Merge pull request #1282 from ucb-bar/full-setup

Rework build-setup | Add single-node CI

.github/actions/cleanup-conda/action.yml

@@ -15,14 +15,19 @@ runs:
             conda env remove -n $env
           done
         fi
+        IS_NUMBER_REGEX='[0-9]+$'
         conda env list | awk '{print $1}' | tail -n +4 | while read envname; do
           ENV_DATE=$(echo $envname | sed "s/cy-[[:digit:]]\+-\(.*\)-\(riscv\|esp\)-tools/\1/")
-          NUM_DIFF=$(( ( $(date +%s) - $(date --date="$ENV_DATE" +%s) )/(60*60*24) ))
-          if (( $NUM_DIFF > 7 )); then
-            echo "Removing $envname since it is $NUM_DIFF days old."
-            conda env remove -n $envname
+          if ! [[ $ENV_DATE =~ $IS_NUMBER_REGEX ]]; then
+            echo "Skipping removal of $envname since it cannot be parsed into a date"
           else
-            echo "Skipping removal of $envname since it is $NUM_DIFF days old."
+            NUM_DIFF=$(( ( $(date +%s) - $(date --date="$ENV_DATE" +%s) )/(60*60*24) ))
+            if (( $NUM_DIFF > 7 )); then
+              echo "Removing $envname since it is $NUM_DIFF days old."
+              conda env remove -n $envname
+            else
+              echo "Skipping removal of $envname since it is $NUM_DIFF days old."
+            fi
           fi
         done
       shell: bash -leo pipefail {0}

.github/actions/create-conda-env/action.yml

@@ -23,10 +23,10 @@ runs:
           if [[ "${{ inputs.install-collateral }}" == 'true' ]]; then
             echo "Add extra toolchain collateral to RISC-V install area"
             conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)-riscv-tools
-            ./scripts/build-toolchain-extra.sh riscv-tools
+            ./scripts/build-toolchain-extra.sh riscv-tools -p $CONDA_PREFIX/riscv-tools
             conda deactivate
             conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)-esp-tools
-            ./scripts/build-toolchain-extra.sh esp-tools
+            ./scripts/build-toolchain-extra.sh esp-tools -p $CONDA_PREFIX/esp-tools
             conda deactivate
           fi
         fi

.github/workflows/chipyard-full-flow.yml

@@ -0,0 +1,104 @@
+name: chipyard-ci-full-flow
+
+on:
+  # run ci on pull requests targeting following branches (runs on the merge commit)
+  pull_request:
+    branches:
+      - main
+      - '1.[0-9]*.x'
+
+defaults:
+  run:
+    shell: bash -leo pipefail {0}
+
+env:
+  REMOTE_WORK_DIR: ${{ secrets.BUILDDIR }}/cy-ci-shared/cy-${{ github.sha }}
+
+jobs:
+  cancel-prior-workflows:
+    name: cancel-prior-workflows
+    runs-on: ubuntu-latest
+    steps:
+      - name: Cancel Previous Runs
+        uses: styfle/[email protected]
+        with:
+          access_token: ${{ github.token }}
+
+  # Set up a set of boolean conditions to control which branches of the CI
+  # workflow will execute This is based off the conditional job execution
+  # example here: https://github.com/dorny/paths-filter#examples
+  change-filters:
+    name: filter-jobs-on-changes
+    runs-on: ubuntu-latest
+    # Queried by downstream jobs to determine if they should run.
+    outputs:
+      needs-rtl: ${{ steps.filter.outputs.all_count != steps.filter.outputs.skip-rtl_count }}
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Git workaround
+        uses: ./.github/actions/git-workaround
+      - uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          filters: |
+            all:
+              - '**'
+
+            # If any of the files changed match, do a doc build
+            docs: &docs-filter
+              - 'docs/**'
+              - '.readthedocs.yml'
+
+            # If all files match to this filter, skip the main ci pipeline
+            skip-rtl:
+              - *docs-filter
+              - '**/*.md'
+              - '**/.gitignore'
+              - '.github/ISSUE_TEMPLATE/**'
+
+  setup-repo:
+    name: setup-repo
+    needs: [change-filters, cancel-prior-workflows]
+    if: needs.change-filters.outputs.needs-rtl == 'true'
+    runs-on: ferry
+    steps:
+      - name: Delete old checkout
+        run: |
+            ls -alh .
+            rm -rf ${{ github.workspace }}/* || true
+            rm -rf ${{ github.workspace }}/.* || true
+            ls -alh .
+      - uses: actions/checkout@v3
+      - name: Setup repo copy
+        run: |
+          git clone $GITHUB_WORKSPACE ${{ env.REMOTE_WORK_DIR }}
+      - name: Setup repo
+        run: |
+          cd ${{ env.REMOTE_WORK_DIR }}
+          eval "$(conda shell.bash hook)"
+          export MAKEFLAGS="-j32"
+          ./build-setup.sh -f
+
+  run-tutorial:
+    name: run-tutorial
+    needs: [setup-repo]
+    runs-on: ferry
+    steps:
+      - name: Run smoke test
+        run: |
+          cd ${{ env.REMOTE_WORK_DIR }}
+          eval "$(conda shell.bash hook)"
+          source env.sh
+          cd sims/verilator
+          make verilog
+
+  cleanup:
+    name: cleanup
+    needs: [run-tutorial]
+    runs-on: ferry
+    if: ${{ always() && contains(join(needs.*.result, ','), 'success') }}
+    steps:
+      - name: Delete repo copy and conda env
+        run: |
+           rm -rf ${{ env.REMOTE_WORK_DIR }}

.github/workflows/chipyard-run-tests.yml

@@ -25,7 +25,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Cancel Previous Runs
-        uses: styfle/cancel-workflow-action@0.9.1
+        uses: styfle/cancel-workflow-action@0.11.0
         with:
           access_token: ${{ github.token }}
 

docs/Chipyard-Basics/Initial-Repo-Setup.rst

@@ -62,27 +62,25 @@ Start by checking out the proper Chipyard version. Run:
     # note: this may not be the latest release if the documentation version != "stable"
     git checkout |version|
 
-Next run the following script to create Chipyard's Conda environment including a pre-built RISC-V toolchain.
+Next run the following script to fully setup Chipyard with a specific toolchain.
 There are two toolchains, one for normal RISC-V programs called ``riscv-tools`` which is the one needed for most Chipyard use-cases, and another for Hwacha/Gemmini called ``esp-tools``.
 Run the following script based off which compiler you would like to use.
 
+.. Warning:: The following script will complete a "full" installation of Chipyard which may take a long time depending on the system.
+    Ensure that this script completes fully (no interruptions) before continuing on.
+
 .. code-block:: shell
 
     ./build-setup.sh riscv-tools # or esp-tools
 
-This script wraps around the conda environment initialization process and also runs the ``init-submodules-no-riscv-tools.sh`` and ``build-toolchain-extra.sh`` scripts.
-
-The ``init-submodules-no-riscv-tools.sh`` script will initialize and checkout all of the necessary git submodules.
-This will also validate that you are on a tagged branch, otherwise it will prompt for confirmation.
-When updating Chipyard to a new version, you will also want to rerun this script to update the submodules.
-Using ``git`` directly will try to initialize all submodules; this is not recommended unless you expressly desire this behavior.
+This script wraps around the conda environment initialization process, initializes all submodules (with the ``init-submodules-no-riscv-tools.sh`` script), installs a toolchain, and runs other setups.
+See ``./build-setup.sh --help`` for more details on what this does and how to disable parts of the setup.
 
-The ``build-toolchain-extra.sh`` script will install extra toolchain utilities/tests used by Chipyard.
-This command builds utilities like Spike, RISC-V Proxy Kernel, libgloss, and RISC-V tests from source for a specific toolchain type.
+.. Warning:: Using ``git`` directly will try to initialize all submodules; this is not recommended unless you expressly desire this behavior.
 
-.. Note:: By default, the ``build-toolchain-extra.sh`` script installs to ``$CONDA_PREFIX/<toolchain-type>``. Thus, if you uninstall the compiler using ``conda remove`` these utilities/tests will also have to be re-installed/built.
+.. Note:: By default, the ``build-setup.sh`` script installs extra toolchain utilities (RISC-V tests, PK, Spike, etc) to ``$CONDA_PREFIX/<toolchain-type>``. Thus, if you uninstall the compiler using ``conda remove`` these utilities/tests will also have to be re-installed/built.
 
-.. Note:: If you already have a working conda environment setup, separate Chipyard clones can use that pre-used environment in combination with running the aforementioned scripts yourself (``init-submodules...`` and ``build-toolchain...``).
+.. Note:: If you already have a working conda environment setup, separate Chipyard clones can use that pre-used environment in combination with running the aforementioned scripts yourself (``init-submodules...``, ``build-toolchain...``, etc).
 
 .. Note:: If you are a power user and would like to build your own compiler/toolchain, you can refer to the https://github.com/ucb-bar/riscv-tools-feedstock and https://github.com/ucb-bar/esp-tools-feedstock repositories (submoduled in the ``toolchains/*`` directories) on how to build the compiler yourself.
 

docs/Simulation/FPGA-Accelerated-Simulation.rst

@@ -14,14 +14,7 @@ FireSim supports running on Amazon EC2 F1 FPGA-enabled cloud instances and on lo
 The rest of this documentation assumes you are running on an Amazon EC2 F1 FPGA-enabled virtual instance.
 In order to simuate your Chipyard design using FireSim, make sure to follow the repository setup as described by
 :ref:`Chipyard-Basics/Initial-Repo-Setup:Initial Repository Setup`, if you have not already.
-
-Next, initalize FireSim as a library in Chipyard by running:
-
-.. code-block:: shell
-
-    # At the root of your chipyard repo
-    ./scripts/firesim-setup.sh
-
+This setup should have setup the Chipyard repository including FireSim by running the ``./scripts/firesim-setup.sh`` script.
 ``firesim-setup.sh`` initializes additional submodules and then invokes
 FireSim's ``build-setup.sh`` script adding ``--library`` to properly
 initialize FireSim as a library submodule in Chipyard. You may run