Merge branch 'main' into jane/source-coverage

.github/PULL_REQUEST_TEMPLATE/rfc.md

@@ -15,10 +15,22 @@ The RFC process is documented in CONTRIBUTING.md.
 Please see the checklist there: https://zebra.zfnd.org/CONTRIBUTING.html
 -->
 
+
+### Summary
+
+Please copy the RFC summary over here. 
+
+### More information
+
 Feature Name: `my_feature`
+
 Start Date: YYYY-MM-DD
+
 Design PR: [ZcashFoundation/zebra#0000](https://github.com/ZcashFoundation/zebra/pull/0000)
-Zebra Issue: [ZcashFoundation/zebra#0000](https://github.com/ZcashFoundation/zebra/pull/0000)
+
+Zebra Issue: [ZcashFoundation/zebra#0000](https://github.com/ZcashFoundation/zebra/issues/0000)
+
+### Document
 
 <!--
 Fill this in, replacing:

.github/workflows/cd.yml

@@ -1,13 +1,16 @@
 name: CD
 
 on:
+  workflow_dispatch:
   push:
     branches:
       - main
-      - cloudbuild
 
 env:
   PROJECT_ID: zealous-zebra
+  REGION: us-east1
+  ZONE: us-east1-b
+
 
 jobs:
 
@@ -43,68 +46,6 @@ jobs:
           --config cloudbuild.yaml \
           --substitutions SHORT_SHA="$SHORT_SHA",BRANCH_NAME="$BRANCH_NAME"
 
-  test:
-    name: Run integration tests
-    needs: build
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Set project and image names
-        run: |
-          BRANCH_NAME=$(expr $GITHUB_REF : '.*/\(.*\)') && \
-          BRANCH_NAME=${BRANCH_NAME,,} && \
-          REPOSITORY=${GITHUB_REPOSITORY,,} && \
-          echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV && \
-          echo "SHORT_SHA=$(git rev-parse --short=7 $GITHUB_SHA)" >> $GITHUB_ENV && \
-          echo "REPOSITORY=$REPOSITORY" >> $GITHUB_ENV
-
-      # Setup gcloud CLI
-      - name: Set up gcloud SDK environment
-        uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
-        with:
-          version: '295.0.0'
-          project_id: ${{ env.PROJECT_ID }}
-          service_account_key: ${{ secrets.GCLOUD_AUTH }}
-
-      # Run once: create firewall rule to allow incoming traffic to the nodes
-      # - name: Create Zcash incoming traffic firewall rule
-      #   run: |
-      #     gcloud compute firewall-rules create "allow-zcash" \
-      #     --target-tags zebrad \
-      #     --allow tcp:8233,tcp:18233 \
-      #     --source-ranges 0.0.0.0/0 \
-      #     --description="Allow incoming Zcash traffic from anywhere" \
-
-      # Creates Compute Engine virtual machine instance w/ test container and disks
-      - name: Create instance with test container image
-        run: |
-          gcloud compute instances create-with-container "zebra-tests-$BRANCH_NAME-$SHORT_SHA" \
-          --container-command='/zebra/target/debug/deps/acceptance' \
-          --container-arg='-Z' --container-arg='unstable-options' --container-arg='--include-ignored' \
-          --container-image "gcr.io/$PROJECT_ID/$REPOSITORY/tests/$BRANCH_NAME:$SHORT_SHA" \
-          --container-mount-disk mount-path='/zebrad-cache',name=zebrad-cache-$SHORT_SHA \
-          --container-restart-policy='never' \
-          --create-disk name=zebrad-cache-$SHORT_SHA,image=zebrad-cache-mainnet-height-1018849 \
-          --service-account [email protected] \
-          --scopes cloud-platform \
-          --tags zebrad \
-          --zone us-central1-a
-
-      # Wait for acceptance test container is up and stream its results
-      - name: Run acceptance tests
-        run: |
-          gcloud compute ssh "zebra-tests-$BRANCH_NAME-$SHORT_SHA" --zone us-central1-a \
-          --command "echo 'Pulling container image...'; until [[ -n \$(docker ps --all -q --filter name=zebra-test) ]]; do sleep 0.1; done; docker attach \$(docker ps -q -a --filter name=zebra-test)"
-
-      # Clean up
-      - name: Delete test instance
-        # Always run even if the earlier step fails
-        if: ${{ always() }}
-        run: |
-          gcloud compute instances delete "zebra-tests-$BRANCH_NAME-$SHORT_SHA" --delete-disks all --zone us-central1-a
-
   deploy:
     name: Deploy mainnet nodes
     needs: build
@@ -144,9 +85,9 @@ jobs:
         run: |
           gcloud compute instance-templates create-with-container "zebrad-$BRANCH_NAME-$SHORT_SHA" \
           --container-image "gcr.io/$PROJECT_ID/$REPOSITORY/$BRANCH_NAME:$SHORT_SHA" \
-          --machine-type n2-standard-4 \
-          --create-disk name=zebrad-cache-$SHORT_SHA,size=100GB,type=pd-balanced,auto-delete=no \
+          --create-disk name=zebrad-cache-$SHORT_SHA,size=100GB,type=pd-balanced \
           --container-mount-disk mount-path="/zebrad-cache",name=zebrad-cache-$SHORT_SHA \
+          --machine-type n2d-standard-4 \
           --service-account [email protected] \
           --scopes cloud-platform \
           --tags zebrad \
@@ -165,7 +106,7 @@ jobs:
         id: does-group-exist
         continue-on-error: true
         run: |
-          gcloud compute instance-groups list | grep "zebrad-$BRANCH_NAME"
+          gcloud compute instance-groups list | grep "zebrad-$BRANCH_NAME"\ \ "$REGION"
 
       # Deploy new managed instance group using the new instance template
       - name: Create managed instance group
@@ -176,7 +117,7 @@ jobs:
           --template "zebrad-$BRANCH_NAME-$SHORT_SHA" \
           --health-check zebrad-tracing-filter \
           --initial-delay 30 \
-          --region us-central1 \
+          --region "$REGION" \
           --size 2
 
       # Rolls out update to existing group using the new instance template
@@ -186,4 +127,4 @@ jobs:
           gcloud compute instance-groups managed rolling-action start-update \
           "zebrad-$BRANCH_NAME" \
           --version template="zebrad-$BRANCH_NAME-$SHORT_SHA" \
-          --region us-central1 \
+          --region "$REGION"

.github/workflows/ci.yml

@@ -6,7 +6,8 @@ jobs:
 
   test:
     name: Test on ${{ matrix.os }}
-    timeout-minutes: 30
+    # The large timeout is to accommodate Windows builds
+    timeout-minutes: 45
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -37,14 +38,21 @@ jobs:
         with:
           command: test
           args: --verbose --all
-      # Explicitly run any tests that are usually #[ignored]
+      # Explicitly run any tests that are usually #[ignored], modulo ZEBRA_SKIP_NETWORK_TESTS
       - name: Run zebrad large sync tests
         env:
           RUST_BACKTRACE: full
         uses: actions-rs/cargo@v1
         with:
           command: test
-          args: --verbose --manifest-path zebrad/Cargo.toml -- --ignored
+          args: --verbose --manifest-path zebrad/Cargo.toml sync_large_checkpoints_ -- --ignored
+      - name: Run zebrad tracing test
+        env:
+          RUST_BACKTRACE: full
+        uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --verbose --manifest-path zebrad/Cargo.toml tracing_endpoint -- --ignored
 
   build-chain-no-features:
     name: Build zebra-chain w/o features on ubuntu-latest
@@ -111,26 +119,6 @@ jobs:
           token: ${{ secrets.GITHUB_TOKEN }}
           args: --all-features --all-targets -- -D warnings
 
-  clippy-nightly:
-    name: Clippy (nightly)
-    timeout-minutes: 30
-    runs-on: ubuntu-latest
-    continue-on-error: true
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
-        with:
-          toolchain: nightly
-          components: clippy
-          override: true
-      - name: Run Clippy (nightly)
-        uses: actions-rs/clippy-check@v1
-        continue-on-error: true
-        with:
-          name: Clippy (nightly)
-          token: ${{ secrets.GITHUB_TOKEN }}
-          args: --all-features --all-targets -- -W clippy::all
-
   fmt:
     name: Rustfmt
     timeout-minutes: 30

.github/workflows/firebase.yml → .github/workflows/docs.yml

@@ -26,7 +26,7 @@ jobs:
         cd book
         curl -L https://github.com/rust-lang/mdBook/releases/download/v0.4.1/mdbook-v0.4.1-x86_64-unknown-linux-gnu.tar.gz | tar xz
         # Add the book directory to the $PATH
-        echo "::add-path::$GITHUB_WORKSPACE/book"
+        echo "$GITHUB_WORKSPACE/book" >> $GITHUB_PATH
 
     - name: Build Zebra book
       run: |

.github/workflows/manual-deploy.yml

@@ -0,0 +1,65 @@
+name: Manual Deploy
+
+on:
+  workflow_dispatch:
+    inputs:
+      network:
+        default: 'Mainnet'
+      checkpoint_sync:
+        default: true
+
+env:
+  PROJECT_ID: zealous-zebra
+
+jobs:
+
+  deploy:
+    name: Deploy one zebrad node
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set project and image names
+      run: |
+        BRANCH_NAME=$(expr $GITHUB_REF : '.*/\(.*\)') && \
+        BRANCH_NAME=${BRANCH_NAME,,} && \
+        REPOSITORY=${GITHUB_REPOSITORY,,} && \
+        echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV && \
+        echo "SHORT_SHA=$(git rev-parse --short=7 $GITHUB_SHA)" >> $GITHUB_ENV && \
+        echo "REPOSITORY=$REPOSITORY" >> $GITHUB_ENV
+
+    - name: Set up gcloud
+      uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
+      with:
+        version: '295.0.0'
+        project_id: ${{ env.PROJECT_ID }}
+        service_account_key: ${{ secrets.GCLOUD_AUTH }}
+
+    # Build and push image to Google Container Registry
+    - name: Build
+      # Tagging w/ the commit SHA blocks the :latest tag on GCR
+      run: |
+        gcloud builds submit \
+          --config cloudbuild.yaml \
+          --substitutions SHORT_SHA="$SHORT_SHA",BRANCH_NAME="$BRANCH_NAME",_CHECKPOINT_SYNC="${{ github.event.inputs.checkpoint_sync }}",_NETWORK="${{ github.event.inputs.network }}"
+
+    # Run once: create firewall rule to allow incoming traffic to the node
+    # - name: Create Zcash incoming traffic firewall rule
+    #   run: |
+    #     gcloud compute firewall-rules create "allow-zcash" \
+    #     --target-tags zebrad \
+    #     --allow tcp:8233,tcp:18233 \
+    #     --source-ranges 0.0.0.0/0 \
+    #     --description="Allow incoming Zcash traffic from anywhere" \
+
+    # Creates Compute Engine virtual machine instance w/ zebrad container and disks
+    - name: Create instance running zebrad container image
+      run: |
+        gcloud compute instances create-with-container "zebrad-$BRANCH_NAME-$SHORT_SHA" \
+        --container-image "gcr.io/$PROJECT_ID/$REPOSITORY/$BRANCH_NAME:$SHORT_SHA" \
+        --container-mount-disk mount-path='/zebrad-cache',name=zebrad-cache-$SHORT_SHA \
+        --create-disk name=zebrad-cache-$SHORT_SHA,size=100GB,type=pd-balanced \
+        --machine-type n2-standard-4 \
+        --service-account [email protected] \
+        --tags zebrad \
+        --zone us-central1-a

.github/workflows/test.yml

@@ -0,0 +1,80 @@
+name: Test
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+
+env:
+  PROJECT_ID: zealous-zebra
+  ZONE: europe-west1-b
+
+jobs:
+
+  test:
+    name: Run all tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set project and image names
+        run: |
+          BRANCH_NAME=$(expr $GITHUB_REF : '.*/\(.*\)') && \
+          BRANCH_NAME=${BRANCH_NAME,,} && \
+          REPOSITORY=${GITHUB_REPOSITORY,,} && \
+          echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV && \
+          echo "SHORT_SHA=$(git rev-parse --short=7 $GITHUB_SHA)" >> $GITHUB_ENV && \
+          echo "REPOSITORY=$REPOSITORY" >> $GITHUB_ENV
+
+      - name: Set up gcloud
+        uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
+        with:
+          version: '295.0.0'
+          project_id: ${{ env.PROJECT_ID }}
+          service_account_key: ${{ secrets.GCLOUD_AUTH }}
+
+      # Run once: create firewall rule to allow incoming traffic to the nodes
+      # - name: Create Zcash incoming traffic firewall rule
+      #   run: |
+      #     gcloud compute firewall-rules create "allow-zcash" \
+      #     --target-tags zebrad \
+      #     --allow tcp:8233,tcp:18233 \
+      #     --source-ranges 0.0.0.0/0 \
+      #     --description="Allow incoming Zcash traffic from anywhere" \
+
+      # Creates Compute Engine virtual machine instance w/ disks
+      - name: Create instance
+        run: |
+          gcloud compute instances create-with-container "zebrad-tests-$BRANCH_NAME-$SHORT_SHA" \
+          --boot-disk-size 100GB \
+          --boot-disk-type pd-ssd \
+          --container-image rust:buster \
+          --container-mount-disk mount-path='/mainnet',name="zebrad-cache-$SHORT_SHA-mainnet-419200" \
+          --container-mount-disk mount-path='/testnet',name="zebrad-cache-$SHORT_SHA-testnet-280000" \
+          --container-restart-policy never \
+          --create-disk name="zebrad-cache-$SHORT_SHA-mainnet-419200",image=zebrad-cache-062a5ae-mainnet-419200 \
+          --create-disk name="zebrad-cache-$SHORT_SHA-testnet-280000",image=zebrad-cache-2935b4e-testnet-280000 \
+          --machine-type n2-standard-4 \
+          --service-account [email protected] \
+          --scopes cloud-platform \
+          --tags zebrad \
+          --zone "$ZONE"
+
+      # Build and run test container
+      - name: Run all tests
+        run: |
+          gcloud compute ssh "zebrad-tests-$BRANCH_NAME-$SHORT_SHA" --zone "$ZONE" --command \
+          "git clone -b $BRANCH_NAME https://github.com/ZcashFoundation/zebra.git;
+          cd zebra/;
+          docker build -f docker/Dockerfile.test -t zebrad-test .;
+          docker run -i zebrad-test cargo test --workspace --no-fail-fast -- -Zunstable-options --include-ignored;
+          docker run -i --mount type=bind,source=/mnt/disks/gce-containers-mounts/gce-persistent-disks/zebrad-cache-$SHORT_SHA-mainnet-419200,target=/zebrad-cache zebrad-test:latest cargo test --verbose --features test_sync_past_sapling_mainnet --manifest-path zebrad/Cargo.toml sync_past_sapling_mainnet;
+          docker run -i --mount type=bind,source=/mnt/disks/gce-containers-mounts/gce-persistent-disks/zebrad-cache-$SHORT_SHA-testnet-280000,target=/zebrad-cache zebrad-test:latest cargo test --verbose --features test_sync_past_sapling_testnet --manifest-path zebrad/Cargo.toml sync_past_sapling_testnet;
+          "
+      # Clean up
+      - name: Delete test instance
+        # Always run even if the earlier step fails
+        if: ${{ always() }}
+        run: |
+          gcloud compute instances delete "zebrad-tests-$BRANCH_NAME-$SHORT_SHA" --delete-disks all --zone "$ZONE"