From 7403897fdac22910c5ca726d29fd4835b3a275fe Mon Sep 17 00:00:00 2001
From: Jane Lusby <jlusby42@gmail.com>
Date: Fri, 13 Nov 2020 10:19:47 -0800
Subject: [PATCH 001/148] Add transcript tests as described in the state
 service tracking issue (#1281)

* Add transcript test for requests while state is empty

* Add happy path test for each query once the state is populated

* let populate logic handle out of order blocks
---
 zebra-state/src/service.rs       |   2 +
 zebra-state/src/service/tests.rs | 182 +++++++++++++++++++++++++++++++
 zebra-test/src/transcript.rs     |   9 +-
 3 files changed, 190 insertions(+), 3 deletions(-)
 create mode 100644 zebra-state/src/service/tests.rs

diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs
index 2a6430e30b4..1deefb1bc75 100644
--- a/zebra-state/src/service.rs
+++ b/zebra-state/src/service.rs
@@ -27,6 +27,8 @@ use crate::{
 
 mod check;
 mod memory_state;
+#[cfg(test)]
+mod tests;
 mod utxo;
 
 // todo: put this somewhere
diff --git a/zebra-state/src/service/tests.rs b/zebra-state/src/service/tests.rs
new file mode 100644
index 00000000000..1acf159075e
--- /dev/null
+++ b/zebra-state/src/service/tests.rs
@@ -0,0 +1,182 @@
+use std::sync::Arc;
+
+use futures::stream::FuturesUnordered;
+use tower::{util::BoxService, Service, ServiceExt};
+use zebra_chain::{
+    block::Block, parameters::Network, serialization::ZcashDeserializeInto, transaction,
+    transparent,
+};
+use zebra_test::{prelude::*, transcript::Transcript};
+
+use crate::{init, BoxError, Config, Request, Response};
+
+const LAST_BLOCK_HEIGHT: u32 = 10;
+
+async fn populated_state(
+    blocks: impl IntoIterator<Item = Arc<Block>>,
+) -> BoxService<Request, Response, BoxError> {
+    let requests = blocks
+        .into_iter()
+        .map(|block| Request::CommitFinalizedBlock { block });
+
+    let config = Config::ephemeral();
+    let network = Network::Mainnet;
+    let mut state = init(config, network);
+
+    let mut responses = FuturesUnordered::new();
+
+    for request in requests {
+        let rsp = state.ready_and().await.unwrap().call(request);
+        responses.push(rsp);
+    }
+
+    use futures::StreamExt;
+    while let Some(rsp) = responses.next().await {
+        rsp.expect("blocks should commit just fine");
+    }
+
+    state
+}
+
+async fn test_populated_state_responds_correctly(
+    mut state: BoxService<Request, Response, BoxError>,
+) -> Result<()> {
+    let blocks = zebra_test::vectors::MAINNET_BLOCKS
+        .range(0..=LAST_BLOCK_HEIGHT)
+        .map(|(_, block_bytes)| block_bytes.zcash_deserialize_into::<Arc<Block>>().unwrap());
+
+    for (ind, block) in blocks.into_iter().enumerate() {
+        let mut transcript = vec![];
+        let height = block.coinbase_height().unwrap();
+        let hash = block.hash();
+
+        transcript.push((
+            Request::Block(hash.into()),
+            Ok(Response::Block(Some(block.clone()))),
+        ));
+
+        transcript.push((
+            Request::Block(height.into()),
+            Ok(Response::Block(Some(block.clone()))),
+        ));
+
+        transcript.push((
+            Request::Depth(block.hash()),
+            Ok(Response::Depth(Some(LAST_BLOCK_HEIGHT - height.0))),
+        ));
+
+        if ind == LAST_BLOCK_HEIGHT as usize {
+            transcript.push((Request::Tip, Ok(Response::Tip(Some((height, hash))))));
+        }
+
+        // Consensus-critical bug in zcashd: transactions in the genesis block
+        // are ignored.
+        if height.0 != 0 {
+            for transaction in &block.transactions {
+                let transaction_hash = transaction.hash();
+
+                transcript.push((
+                    Request::Transaction(transaction_hash),
+                    Ok(Response::Transaction(Some(transaction.clone()))),
+                ));
+
+                for (index, output) in transaction.outputs().iter().enumerate() {
+                    let outpoint = transparent::OutPoint {
+                        hash: transaction_hash,
+                        index: index as _,
+                    };
+
+                    transcript.push((
+                        Request::AwaitUtxo(outpoint),
+                        Ok(Response::Utxo(output.clone())),
+                    ));
+                }
+            }
+        }
+
+        let transcript = Transcript::from(transcript);
+        transcript.check(&mut state).await?;
+    }
+
+    Ok(())
+}
+
+#[tokio::main]
+async fn populate_and_check(blocks: Vec<Arc<Block>>) -> Result<()> {
+    let state = populated_state(blocks).await;
+    test_populated_state_responds_correctly(state).await?;
+    Ok(())
+}
+
+fn out_of_order_committing_strategy() -> BoxedStrategy<Vec<Arc<Block>>> {
+    let blocks = zebra_test::vectors::MAINNET_BLOCKS
+        .range(0..=LAST_BLOCK_HEIGHT)
+        .map(|(_, block_bytes)| block_bytes.zcash_deserialize_into::<Arc<Block>>().unwrap())
+        .collect::<Vec<_>>();
+
+    Just(blocks).prop_shuffle().boxed()
+}
+
+#[tokio::test]
+async fn empty_state_still_responds_to_requests() -> Result<()> {
+    zebra_test::init();
+
+    let block =
+        zebra_test::vectors::BLOCK_MAINNET_419200_BYTES.zcash_deserialize_into::<Arc<Block>>()?;
+
+    let iter = vec![
+        // No checks for CommitBlock or CommitFinalizedBlock because empty state
+        // precondition doesn't matter to them
+        (Request::Depth(block.hash()), Ok(Response::Depth(None))),
+        (Request::Tip, Ok(Response::Tip(None))),
+        (Request::BlockLocator, Ok(Response::BlockLocator(vec![]))),
+        (
+            Request::Transaction(transaction::Hash([0; 32])),
+            Ok(Response::Transaction(None)),
+        ),
+        (
+            Request::Block(block.hash().into()),
+            Ok(Response::Block(None)),
+        ),
+        (
+            Request::Block(block.coinbase_height().unwrap().into()),
+            Ok(Response::Block(None)),
+        ),
+        // No check for AwaitUTXO because it will wait if the UTXO isn't present
+    ]
+    .into_iter();
+    let transcript = Transcript::from(iter);
+
+    let config = Config::ephemeral();
+    let network = Network::Mainnet;
+    let state = init(config, network);
+
+    transcript.check(state).await?;
+
+    Ok(())
+}
+
+#[test]
+fn state_behaves_when_blocks_are_committed_in_order() -> Result<()> {
+    zebra_test::init();
+
+    let blocks = zebra_test::vectors::MAINNET_BLOCKS
+        .range(0..=LAST_BLOCK_HEIGHT)
+        .map(|(_, block_bytes)| block_bytes.zcash_deserialize_into::<Arc<Block>>().unwrap())
+        .collect();
+
+    populate_and_check(blocks)?;
+
+    Ok(())
+}
+
+#[test]
+fn state_behaves_when_blocks_are_committed_out_of_order() -> Result<()> {
+    zebra_test::init();
+
+    proptest!(|(blocks in out_of_order_committing_strategy())| {
+        populate_and_check(blocks).unwrap();
+    });
+
+    Ok(())
+}
diff --git a/zebra-test/src/transcript.rs b/zebra-test/src/transcript.rs
index 8c9a2d44751..feea1b0c861 100644
--- a/zebra-test/src/transcript.rs
+++ b/zebra-test/src/transcript.rs
@@ -51,6 +51,7 @@ impl TransError {
 #[error("ErrorChecker Error: {0}")]
 struct ErrorCheckerError(Error);
 
+#[must_use]
 pub struct Transcript<R, S, I>
 where
     I: Iterator<Item = (R, Result<S, TransError>)>,
@@ -58,12 +59,14 @@ where
     messages: I,
 }
 
-impl<R, S, I> From<I> for Transcript<R, S, I>
+impl<R, S, I> From<I> for Transcript<R, S, I::IntoIter>
 where
-    I: Iterator<Item = (R, Result<S, TransError>)>,
+    I: IntoIterator<Item = (R, Result<S, TransError>)>,
 {
     fn from(messages: I) -> Self {
-        Self { messages }
+        Self {
+            messages: messages.into_iter(),
+        }
     }
 }
 

From 8c5f6d01778004e266e3ba02fc6d2788df084425 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 13 Nov 2020 10:01:37 +0000
Subject: [PATCH 002/148] build(deps): bump once_cell from 1.5.1 to 1.5.2

Bumps [once_cell](https://github.com/matklad/once_cell) from 1.5.1 to 1.5.2.
- [Release notes](https://github.com/matklad/once_cell/releases)
- [Changelog](https://github.com/matklad/once_cell/blob/master/CHANGELOG.md)
- [Commits](https://github.com/matklad/once_cell/compare/v1.5.1...v1.5.2)

Signed-off-by: dependabot[bot] <support@github.com>
---
 Cargo.lock                 | 4 ++--
 zebra-consensus/Cargo.toml | 2 +-
 zebra-state/Cargo.toml     | 2 +-
 zebrad/Cargo.toml          | 2 +-
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index e5c9d970617..2c2362ca36f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1820,9 +1820,9 @@ checksum = "8d3b63360ec3cb337817c2dbd47ab4a0f170d285d8e5a2064600f3def1402397"
 
 [[package]]
 name = "once_cell"
-version = "1.5.1"
+version = "1.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f53cef67919d7d247eb9a2f128ca9e522789967ef1eb4ccd8c71a95a8aedf596"
+checksum = "13bd41f508810a131401606d54ac32a467c97172d74ba7662562ebba5ad07fa0"
 
 [[package]]
 name = "opaque-debug"
diff --git a/zebra-consensus/Cargo.toml b/zebra-consensus/Cargo.toml
index eb84e8b45fb..eeae83d0c9c 100644
--- a/zebra-consensus/Cargo.toml
+++ b/zebra-consensus/Cargo.toml
@@ -10,7 +10,7 @@ chrono = "0.4.19"
 color-eyre = "0.5"
 displaydoc = "0.1.7"
 jubjub = "0.5.1"
-once_cell = "1.4"
+once_cell = "1.5"
 rand = "0.7"
 redjubjub = "0.2"
 serde = { version = "1", features = ["serde_derive"] }
diff --git a/zebra-state/Cargo.toml b/zebra-state/Cargo.toml
index 5bd87e93ed8..06a21f298bf 100644
--- a/zebra-state/Cargo.toml
+++ b/zebra-state/Cargo.toml
@@ -30,7 +30,7 @@ displaydoc = "0.1.7"
 zebra-chain = { path = "../zebra-chain", features = ["proptest-impl"] }
 zebra-test = { path = "../zebra-test/" }
 
-once_cell = "1.4"
+once_cell = "1.5"
 spandoc = "0.2"
 tempdir = "0.3.7"
 tokio = { version = "0.2.22", features = ["full"] }
diff --git a/zebrad/Cargo.toml b/zebrad/Cargo.toml
index cb5431780f4..d88d7d29afa 100644
--- a/zebrad/Cargo.toml
+++ b/zebrad/Cargo.toml
@@ -41,6 +41,6 @@ inferno = { version = "0.10.1", default-features = false }
 
 [dev-dependencies]
 abscissa_core = { version = "0.5", features = ["testing"] }
-once_cell = "1.4"
+once_cell = "1.5"
 tempdir = "0.3.7"
 zebra-test = { path = "../zebra-test" }

From 18eb4d1c317c98fc0c2a2c820e99de27cbc2bc8e Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 13 Nov 2020 10:01:34 +0000
Subject: [PATCH 003/148] build(deps): bump primitive-types from 0.7.2 to 0.7.3

Bumps [primitive-types](https://github.com/paritytech/parity-common) from 0.7.2 to 0.7.3.
- [Release notes](https://github.com/paritytech/parity-common/releases)
- [Commits](https://github.com/paritytech/parity-common/compare/primitive-types-v0.7.2...primitive-types-v0.7.3)

Signed-off-by: dependabot[bot] <support@github.com>
---
 Cargo.lock             | 4 ++--
 zebra-chain/Cargo.toml | 2 +-
 zebra-state/Cargo.toml | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 2c2362ca36f..268023d6476 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2006,9 +2006,9 @@ dependencies = [
 
 [[package]]
 name = "primitive-types"
-version = "0.7.2"
+version = "0.7.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c55c21c64d0eaa4d7ed885d959ef2d62d9e488c27c0e02d9aa5ce6c877b7d5f8"
+checksum = "7dd39dcacf71411ba488570da7bbc89b717225e46478b30ba99b92db6b149809"
 dependencies = [
  "fixed-hash",
  "impl-codec",
diff --git a/zebra-chain/Cargo.toml b/zebra-chain/Cargo.toml
index 93efc7bc91e..54953858336 100644
--- a/zebra-chain/Cargo.toml
+++ b/zebra-chain/Cargo.toml
@@ -24,7 +24,7 @@ futures = "0.3"
 hex = "0.4"
 jubjub = "0.5.1"
 lazy_static = "1.4.0"
-primitive-types = "0.7.2"
+primitive-types = "0.7.3"
 rand_core = "0.5.1"
 ripemd160 = "0.8.0"
 secp256k1 = { version = "0.19.0", features = ["serde"] }
diff --git a/zebra-state/Cargo.toml b/zebra-state/Cargo.toml
index 06a21f298bf..ab5bf229521 100644
--- a/zebra-state/Cargo.toml
+++ b/zebra-state/Cargo.toml
@@ -35,4 +35,4 @@ spandoc = "0.2"
 tempdir = "0.3.7"
 tokio = { version = "0.2.22", features = ["full"] }
 proptest = "0.10.1"
-primitive-types = "0.7.2"
+primitive-types = "0.7.3"

From 472163f8176c429ed889678d088955bd92f775ed Mon Sep 17 00:00:00 2001
From: Deirdre Connolly <deirdre@zfnd.org>
Date: Fri, 13 Nov 2020 16:58:25 -0500
Subject: [PATCH 004/148] Add a manual zebrad node deploy workflow

---
 .github/workflows/manual-deploy.yml | 64 +++++++++++++++++++++++++++++
 1 file changed, 64 insertions(+)
 create mode 100644 .github/workflows/manual-deploy.yml

diff --git a/.github/workflows/manual-deploy.yml b/.github/workflows/manual-deploy.yml
new file mode 100644
index 00000000000..414dabb3dd3
--- /dev/null
+++ b/.github/workflows/manual-deploy.yml
@@ -0,0 +1,64 @@
+name: Deploy a zebrad node
+
+on:
+  workflow_dispatch:
+    inputs:
+      network:
+        default: 'mainnet'
+      checkpoint_sync:
+        default: true
+
+env:
+  PROJECT_ID: zealous-zebra
+
+jobs:
+
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set project and image names
+      run: |
+        BRANCH_NAME=$(expr $GITHUB_REF : '.*/\(.*\)') && \
+        BRANCH_NAME=${BRANCH_NAME,,} && \
+        REPOSITORY=${GITHUB_REPOSITORY,,} && \
+        echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV && \
+        echo "SHORT_SHA=$(git rev-parse --short=7 $GITHUB_SHA)" >> $GITHUB_ENV && \
+        echo "REPOSITORY=$REPOSITORY" >> $GITHUB_ENV
+
+    - name: Set up gcloud
+      uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
+      with:
+        version: '295.0.0'
+        project_id: ${{ env.PROJECT_ID }}
+        service_account_key: ${{ secrets.GCLOUD_AUTH }}
+
+    # Build and push image to Google Container Registry
+    - name: Build
+      # Tagging w/ the commit SHA blocks the :latest tag on GCR
+      run: |
+        gcloud builds submit \
+          --config cloudbuild.yaml \
+          --substitutions SHORT_SHA="$SHORT_SHA",BRANCH_NAME="$BRANCH_NAME"
+
+    # Run once: create firewall rule to allow incoming traffic to the node
+    # - name: Create Zcash incoming traffic firewall rule
+    #   run: |
+    #     gcloud compute firewall-rules create "allow-zcash" \
+    #     --target-tags zebrad \
+    #     --allow tcp:8233,tcp:18233 \
+    #     --source-ranges 0.0.0.0/0 \
+    #     --description="Allow incoming Zcash traffic from anywhere" \
+
+    # Creates Compute Engine virtual machine instance w/ zebrad container and disks
+    - name: Create instance running zebrad container image
+      run: |
+        gcloud compute instances create-with-container "zebra-$BRANCH_NAME-$SHORT_SHA" \
+        --container-image "gcr.io/$PROJECT_ID/$REPOSITORY/$BRANCH_NAME:$SHORT_SHA" \
+        --container-mount-disk mount-path='/zebrad-cache',name=zebrad-cache-$SHORT_SHA \
+        --create-disk name=zebrad-cache-$SHORT_SHA,size=100GB,type=pd-balanced,auto-delete=no \
+        --machine-type n2-standard-4 \
+        --service-account cos-vm@zealous-zebra.iam.gserviceaccount.com \
+        --tags zebrad \
+        --zone us-central1-a

From 1a1240a94351b97781d9723f9132ad63cfa048e8 Mon Sep 17 00:00:00 2001
From: Deirdre Connolly <deirdre@zfnd.org>
Date: Fri, 13 Nov 2020 16:58:41 -0500
Subject: [PATCH 005/148] Rearrange some deploy flags

---
 .github/workflows/cd.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index 4b9d5d598a8..e37bd81401d 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -144,9 +144,9 @@ jobs:
         run: |
           gcloud compute instance-templates create-with-container "zebrad-$BRANCH_NAME-$SHORT_SHA" \
           --container-image "gcr.io/$PROJECT_ID/$REPOSITORY/$BRANCH_NAME:$SHORT_SHA" \
-          --machine-type n2-standard-4 \
           --create-disk name=zebrad-cache-$SHORT_SHA,size=100GB,type=pd-balanced,auto-delete=no \
           --container-mount-disk mount-path="/zebrad-cache",name=zebrad-cache-$SHORT_SHA \
+          --machine-type n2-standard-4 \
           --service-account cos-vm@zealous-zebra.iam.gserviceaccount.com \
           --scopes cloud-platform \
           --tags zebrad \

From 7023465d91023ba388742e670000fd1f88a5a93c Mon Sep 17 00:00:00 2001
From: Deirdre Connolly <deirdre@zfnd.org>
Date: Fri, 13 Nov 2020 22:06:48 -0500
Subject: [PATCH 006/148] Pass in workflow inputs for network and
 checkpoint_sync (with defaults) all the way down

---
 .github/workflows/manual-deploy.yml |  5 ++++-
 Dockerfile                          |  7 ++++++-
 cloudbuild.yaml                     | 19 +++++++++++++++++--
 3 files changed, 27 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/manual-deploy.yml b/.github/workflows/manual-deploy.yml
index 414dabb3dd3..e0db3962d49 100644
--- a/.github/workflows/manual-deploy.yml
+++ b/.github/workflows/manual-deploy.yml
@@ -40,7 +40,10 @@ jobs:
       run: |
         gcloud builds submit \
           --config cloudbuild.yaml \
-          --substitutions SHORT_SHA="$SHORT_SHA",BRANCH_NAME="$BRANCH_NAME"
+          --substitutions SHORT_SHA="$SHORT_SHA",\
+          BRANCH_NAME="$BRANCH_NAME",\
+          _CHECKPOINT_SYNC="${{ github.event.inputs.checkpoint_sync }}",\
+          _NETWORK="${{ github.event.inputs.network }}"
 
     # Run once: create firewall rule to allow incoming traffic to the node
     # - name: Create Zcash incoming traffic firewall rule
diff --git a/Dockerfile b/Dockerfile
index 6bc8f4ad968..d10dc0c7a18 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -35,8 +35,13 @@ FROM debian:buster-slim AS zebrad-release
 
 COPY --from=builder /zebra/target/release/zebrad /
 
+ARG checkpoint_sync=true
+ARG network=Mainnet
+
 RUN printf "[consensus]\n" >> /zebrad.toml
-RUN printf "checkpoint_sync = true\n" >> /zebrad.toml
+RUN printf "checkpoint_sync = $checkpoint_sync\n" >> /zebrad.toml
+RUN printf "[network]\n" >> /zebrad.toml
+RUN printf "network = $network\n" >> /zebrad.toml
 RUN printf "[state]\n" >> /zebrad.toml
 RUN printf "cache_dir = '/zebrad-cache'\n" >> /zebrad.toml
 RUN printf "memory_cache_bytes = 52428800\n" >> /zebrad.toml
diff --git a/cloudbuild.yaml b/cloudbuild.yaml
index 7cd64677f36..b96b0c95ed0 100644
--- a/cloudbuild.yaml
+++ b/cloudbuild.yaml
@@ -1,8 +1,22 @@
 steps:
 - name: 'gcr.io/cloud-builders/docker'
-  args: ['build', '--target', 'zebra-tests', '-t', 'gcr.io/$PROJECT_ID/zcashfoundation/zebra/tests/$BRANCH_NAME:$SHORT_SHA', '.']
+  args: ['build',
+         '--target',
+         'zebra-tests',
+         '-t',
+         'gcr.io/$PROJECT_ID/zcashfoundation/zebra/tests/$BRANCH_NAME:$SHORT_SHA',
+         '.']
 - name: 'gcr.io/cloud-builders/docker'
-  args: ['build', '--target', 'zebrad-release', '-t', 'gcr.io/$PROJECT_ID/zcashfoundation/zebra/$BRANCH_NAME:$SHORT_SHA', '.']
+  args: ['build',
+         '--build-arg',
+         'checkpoint_sync=${_CHECKPOINT_SYNC}',
+         '--build-arg',
+         'network=${_NETWORK}',
+         '--target',
+         'zebrad-release',
+         '-t',
+         'gcr.io/$PROJECT_ID/zcashfoundation/zebra/$BRANCH_NAME:$SHORT_SHA',
+         '.']
 
 images:
 - 'gcr.io/$PROJECT_ID/zcashfoundation/zebra/tests/$BRANCH_NAME:$SHORT_SHA'
@@ -10,5 +24,6 @@ images:
 
 options:
     machineType: 'N1_HIGHCPU_32'
+    substitution_option: 'ALLOW_LOOSE'
 
 timeout: 1800s # 30 mins

From 477eac7f19721d1bc18eac099708e32ad125b259 Mon Sep 17 00:00:00 2001
From: Deirdre Connolly <deirdre@zfnd.org>
Date: Fri, 13 Nov 2020 23:53:26 -0500
Subject: [PATCH 007/148] Properly use Dockerfile ARG values

---
 .github/workflows/manual-deploy.yml | 2 +-
 Dockerfile                          | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/manual-deploy.yml b/.github/workflows/manual-deploy.yml
index e0db3962d49..914ae0d1b4d 100644
--- a/.github/workflows/manual-deploy.yml
+++ b/.github/workflows/manual-deploy.yml
@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
     inputs:
       network:
-        default: 'mainnet'
+        default: 'Mainnet'
       checkpoint_sync:
         default: true
 
diff --git a/Dockerfile b/Dockerfile
index d10dc0c7a18..8a7c279e577 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -39,9 +39,9 @@ ARG checkpoint_sync=true
 ARG network=Mainnet
 
 RUN printf "[consensus]\n" >> /zebrad.toml
-RUN printf "checkpoint_sync = $checkpoint_sync\n" >> /zebrad.toml
+RUN printf "checkpoint_sync = ${checkpoint_sync}\n" >> /zebrad.toml
 RUN printf "[network]\n" >> /zebrad.toml
-RUN printf "network = $network\n" >> /zebrad.toml
+RUN printf "network = '${network}'\n" >> /zebrad.toml
 RUN printf "[state]\n" >> /zebrad.toml
 RUN printf "cache_dir = '/zebrad-cache'\n" >> /zebrad.toml
 RUN printf "memory_cache_bytes = 52428800\n" >> /zebrad.toml

From 87d749ee4fc69f7db839876f63d15c6b1f7ce12d Mon Sep 17 00:00:00 2001
From: Deirdre Connolly <deirdre@zfnd.org>
Date: Sat, 14 Nov 2020 21:36:41 -0500
Subject: [PATCH 008/148] I love it when capitalization matters, contrary to
 the docs

---
 Dockerfile | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 8a7c279e577..12946c086cd 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -35,13 +35,13 @@ FROM debian:buster-slim AS zebrad-release
 
 COPY --from=builder /zebra/target/release/zebrad /
 
-ARG checkpoint_sync=true
-ARG network=Mainnet
+ARG CHECKPOINT_SYNC=true
+ARG NETWORK=Mainnet
 
 RUN printf "[consensus]\n" >> /zebrad.toml
-RUN printf "checkpoint_sync = ${checkpoint_sync}\n" >> /zebrad.toml
+RUN printf "checkpoint_sync = ${CHECKPOINT_SYNC}\n" >> /zebrad.toml
 RUN printf "[network]\n" >> /zebrad.toml
-RUN printf "network = '${network}'\n" >> /zebrad.toml
+RUN printf "network = '${NETWORK}'\n" >> /zebrad.toml
 RUN printf "[state]\n" >> /zebrad.toml
 RUN printf "cache_dir = '/zebrad-cache'\n" >> /zebrad.toml
 RUN printf "memory_cache_bytes = 52428800\n" >> /zebrad.toml

From bb99a5aa2ad2b74a33a985cccdc0a62c4d361b52 Mon Sep 17 00:00:00 2001
From: Deirdre Connolly <deirdre@zfnd.org>
Date: Sat, 14 Nov 2020 22:49:05 -0500
Subject: [PATCH 009/148] Linewrap

---
 .github/workflows/manual-deploy.yml | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/.github/workflows/manual-deploy.yml b/.github/workflows/manual-deploy.yml
index 914ae0d1b4d..5382bf6337e 100644
--- a/.github/workflows/manual-deploy.yml
+++ b/.github/workflows/manual-deploy.yml
@@ -40,10 +40,7 @@ jobs:
       run: |
         gcloud builds submit \
           --config cloudbuild.yaml \
-          --substitutions SHORT_SHA="$SHORT_SHA",\
-          BRANCH_NAME="$BRANCH_NAME",\
-          _CHECKPOINT_SYNC="${{ github.event.inputs.checkpoint_sync }}",\
-          _NETWORK="${{ github.event.inputs.network }}"
+          --substitutions SHORT_SHA="$SHORT_SHA",BRANCH_NAME="$BRANCH_NAME",_CHECKPOINT_SYNC="${{ github.event.inputs.checkpoint_sync }}",_NETWORK="${{ github.event.inputs.network }}"
 
     # Run once: create firewall rule to allow incoming traffic to the node
     # - name: Create Zcash incoming traffic firewall rule

From 2b8d69622162dcbf9bc21b636b19a68834499826 Mon Sep 17 00:00:00 2001
From: Deirdre Connolly <deirdre@zfnd.org>
Date: Sun, 15 Nov 2020 01:17:46 -0500
Subject: [PATCH 010/148] Better naming workflows (#1301)

* Better workflow names

* We run zebrad
---
 .github/workflows/manual-deploy.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/manual-deploy.yml b/.github/workflows/manual-deploy.yml
index 5382bf6337e..fa33d64de7b 100644
--- a/.github/workflows/manual-deploy.yml
+++ b/.github/workflows/manual-deploy.yml
@@ -1,4 +1,4 @@
-name: Deploy a zebrad node
+name: Manual Deploy
 
 on:
   workflow_dispatch:
@@ -14,6 +14,7 @@ env:
 jobs:
 
   deploy:
+    name: Deploy one zebrad node
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
@@ -54,7 +55,7 @@ jobs:
     # Creates Compute Engine virtual machine instance w/ zebrad container and disks
     - name: Create instance running zebrad container image
       run: |
-        gcloud compute instances create-with-container "zebra-$BRANCH_NAME-$SHORT_SHA" \
+        gcloud compute instances create-with-container "zebrad-$BRANCH_NAME-$SHORT_SHA" \
         --container-image "gcr.io/$PROJECT_ID/$REPOSITORY/$BRANCH_NAME:$SHORT_SHA" \
         --container-mount-disk mount-path='/zebrad-cache',name=zebrad-cache-$SHORT_SHA \
         --create-disk name=zebrad-cache-$SHORT_SHA,size=100GB,type=pd-balanced,auto-delete=no \

From 57637560b9c93fab0bfc1632bef4d8a637e36813 Mon Sep 17 00:00:00 2001
From: Jane Lusby <jlusby42@gmail.com>
Date: Sun, 15 Nov 2020 18:22:53 -0800
Subject: [PATCH 011/148] Add internal iterator API for accessing relevant
 chain blocks (#1271)

* Add internal iterator API for accessing relevant chain blocks
* get blocks from all chains in non_finalized state
* Impl FusedIterator for service::Iter
* impl ExactSizedIterator for service::Iter
* let size_hint find heights in side chains

Co-authored-by: teor <teor@riseup.net>
---
 zebra-state/src/service.rs                    | 112 ++++++++++++++++++
 .../memory_state/non_finalized_state.rs       |  26 ++++
 2 files changed, 138 insertions(+)

diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs
index 1deefb1bc75..6b446d990fe 100644
--- a/zebra-state/src/service.rs
+++ b/zebra-state/src/service.rs
@@ -252,10 +252,122 @@ impl StateService {
         self.mem.hash(height).or_else(|| self.sled.hash(height))
     }
 
+    /// Return the height for the block at `hash` in any chain.
+    pub fn height_by_hash(&self, hash: block::Hash) -> Option<block::Height> {
+        self.mem
+            .height_by_hash(hash)
+            .or_else(|| self.sled.height(hash))
+    }
+
     /// Return the utxo pointed to by `outpoint` if it exists in any chain.
     pub fn utxo(&self, outpoint: &transparent::OutPoint) -> Option<transparent::Output> {
         self.mem.utxo(outpoint).or_else(|| self.sled.utxo(outpoint))
     }
+
+    /// Return an iterator over the relevant chain of the block identified by
+    /// `hash`.
+    ///
+    /// The block identified by `hash` is included in the chain of blocks yielded
+    /// by the iterator.
+    #[allow(dead_code)]
+    pub fn chain(&self, hash: block::Hash) -> Iter<'_> {
+        Iter {
+            service: self,
+            state: IterState::NonFinalized(hash),
+        }
+    }
+}
+
+struct Iter<'a> {
+    service: &'a StateService,
+    state: IterState,
+}
+
+enum IterState {
+    NonFinalized(block::Hash),
+    Finalized(block::Height),
+    Finished,
+}
+
+impl Iter<'_> {
+    fn next_non_finalized_block(&mut self) -> Option<Arc<Block>> {
+        let Iter { service, state } = self;
+
+        let hash = match state {
+            IterState::NonFinalized(hash) => *hash,
+            IterState::Finalized(_) | IterState::Finished => unreachable!(),
+        };
+
+        if let Some(block) = service.mem.block_by_hash(hash) {
+            let hash = block.header.previous_block_hash;
+            self.state = IterState::NonFinalized(hash);
+            Some(block)
+        } else {
+            None
+        }
+    }
+
+    fn next_finalized_block(&mut self) -> Option<Arc<Block>> {
+        let Iter { service, state } = self;
+
+        let hash_or_height: HashOrHeight = match *state {
+            IterState::Finalized(height) => height.into(),
+            IterState::NonFinalized(hash) => hash.into(),
+            IterState::Finished => unreachable!(),
+        };
+
+        if let Some(block) = service.sled.block(hash_or_height) {
+            let height = block
+                .coinbase_height()
+                .expect("valid blocks have a coinbase height");
+
+            if let Some(next_height) = height - 1 {
+                self.state = IterState::Finalized(next_height);
+            } else {
+                self.state = IterState::Finished;
+            }
+
+            Some(block)
+        } else {
+            self.state = IterState::Finished;
+            None
+        }
+    }
+}
+
+impl Iterator for Iter<'_> {
+    type Item = Arc<Block>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        match self.state {
+            IterState::NonFinalized(_) => self
+                .next_non_finalized_block()
+                .or_else(|| self.next_finalized_block()),
+            IterState::Finalized(_) => self.next_finalized_block(),
+            IterState::Finished => None,
+        }
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        let len = self.len();
+        (len, Some(len))
+    }
+}
+
+impl std::iter::FusedIterator for Iter<'_> {}
+
+impl ExactSizeIterator for Iter<'_> {
+    fn len(&self) -> usize {
+        match self.state {
+            IterState::NonFinalized(hash) => self
+                .service
+                .height_by_hash(hash)
+                .map(|height| (height.0 + 1) as _)
+                .unwrap_or(0),
+            IterState::Finalized(height) => (height.0 + 1) as _,
+            IterState::Finished => 0,
+        }
+    }
 }
 
 impl Service<Request> for StateService {
diff --git a/zebra-state/src/service/memory_state/non_finalized_state.rs b/zebra-state/src/service/memory_state/non_finalized_state.rs
index 2384bf07073..f4bcbe1e54d 100644
--- a/zebra-state/src/service/memory_state/non_finalized_state.rs
+++ b/zebra-state/src/service/memory_state/non_finalized_state.rs
@@ -143,6 +143,21 @@ impl NonFinalizedState {
         None
     }
 
+    /// Returns the `block` with the given hash in the any chain.
+    pub fn block_by_hash(&self, hash: block::Hash) -> Option<Arc<Block>> {
+        for chain in self.chain_set.iter().rev() {
+            if let Some(block) = chain
+                .height_by_hash
+                .get(&hash)
+                .and_then(|height| chain.blocks.get(height))
+            {
+                return Some(block.clone());
+            }
+        }
+
+        None
+    }
+
     /// Returns the `block` at a given height or hash in the best chain.
     pub fn block(&self, hash_or_height: HashOrHeight) -> Option<Arc<Block>> {
         let best_chain = self.best_chain()?;
@@ -173,6 +188,17 @@ impl NonFinalizedState {
         Some(height)
     }
 
+    /// Returns the height of `hash` in any chain.
+    pub fn height_by_hash(&self, hash: block::Hash) -> Option<block::Height> {
+        for chain in self.chain_set.iter().rev() {
+            if let Some(height) = chain.height_by_hash.get(&hash) {
+                return Some(*height);
+            }
+        }
+
+        None
+    }
+
     /// Returns the given transaction if it exists in the best chain.
     pub fn transaction(&self, hash: transaction::Hash) -> Option<Arc<Transaction>> {
         let best_chain = self.best_chain()?;

From ca4e792f472b1e94f4fcf3d21dbf14861bc9614b Mon Sep 17 00:00:00 2001
From: teor <teor@riseup.net>
Date: Tue, 17 Nov 2020 05:57:57 +1000
Subject: [PATCH 012/148] Put messages in request/response order

And fix a comment typo
---
 .../src/protocol/external/message.rs          | 54 +++++++++----------
 1 file changed, 27 insertions(+), 27 deletions(-)

diff --git a/zebra-network/src/protocol/external/message.rs b/zebra-network/src/protocol/external/message.rs
index 1bf051a4b45..e0e035460b2 100644
--- a/zebra-network/src/protocol/external/message.rs
+++ b/zebra-network/src/protocol/external/message.rs
@@ -121,20 +121,15 @@ pub enum Message {
         data: Option<[u8; 32]>,
     },
 
-    /// An `addr` message.
-    ///
-    /// [Bitcoin reference](https://en.bitcoin.it/wiki/Protocol_documentation#addr)
-    Addr(Vec<MetaAddr>),
-
     /// A `getaddr` message.
     ///
     /// [Bitcoin reference](https://en.bitcoin.it/wiki/Protocol_documentation#getaddr)
     GetAddr,
 
-    /// A `block` message.
+    /// An `addr` message.
     ///
-    /// [Bitcoin reference](https://en.bitcoin.it/wiki/Protocol_documentation#block)
-    Block(Arc<Block>),
+    /// [Bitcoin reference](https://en.bitcoin.it/wiki/Protocol_documentation#addr)
+    Addr(Vec<MetaAddr>),
 
     /// A `getblocks` message.
     ///
@@ -155,16 +150,14 @@ pub enum Message {
         stop: Option<block::Hash>,
     },
 
-    /// A `headers` message.
+    /// An `inv` message.
     ///
-    /// Returns block headers in response to a getheaders packet.
+    /// Allows a node to advertise its knowledge of one or more
+    /// objects. It can be received unsolicited, or in reply to
+    /// `getblocks`.
     ///
-    /// [Bitcoin reference](https://en.bitcoin.it/wiki/Protocol_documentation#headers)
-    // Note that the block headers in this packet include a
-    // transaction count (a var_int, so there can be more than 81
-    // bytes per header) as opposed to the block headers that are
-    // hashed by miners.
-    Headers(Vec<block::Header>),
+    /// [Bitcoin reference](https://en.bitcoin.it/wiki/Protocol_documentation#inv)
+    Inv(Vec<InventoryHash>),
 
     /// A `getheaders` message.
     ///
@@ -173,7 +166,7 @@ pub enum Message {
     /// of its best chain and determine the blocks following the intersection
     /// point.
     ///
-    /// The peer responds with an `headers` packet with the hashes of subsequent blocks.
+    /// The peer responds with an `headers` packet with the headers of subsequent blocks.
     /// If supplied, the `stop` parameter specifies the last header to request.
     /// Otherwise, the maximum number of block headers (160) are sent.
     ///
@@ -185,14 +178,16 @@ pub enum Message {
         stop: Option<block::Hash>,
     },
 
-    /// An `inv` message.
+    /// A `headers` message.
     ///
-    /// Allows a node to advertise its knowledge of one or more
-    /// objects. It can be received unsolicited, or in reply to
-    /// `getblocks`.
+    /// Returns block headers in response to a getheaders packet.
     ///
-    /// [Bitcoin reference](https://en.bitcoin.it/wiki/Protocol_documentation#inv)
-    Inv(Vec<InventoryHash>),
+    /// [Bitcoin reference](https://en.bitcoin.it/wiki/Protocol_documentation#headers)
+    // Note that the block headers in this packet include a
+    // transaction count (a var_int, so there can be more than 81
+    // bytes per header) as opposed to the block headers that are
+    // hashed by miners.
+    Headers(Vec<block::Header>),
 
     /// A `getdata` message.
     ///
@@ -203,17 +198,22 @@ pub enum Message {
     /// [Bitcoin reference](https://en.bitcoin.it/wiki/Protocol_documentation#getdata)
     GetData(Vec<InventoryHash>),
 
-    /// A `notfound` message.
+    /// A `block` message.
     ///
-    /// [Bitcoin reference](https://en.bitcoin.it/wiki/Protocol_documentation#notfound)
-    // See note above on `Inventory`.
-    NotFound(Vec<InventoryHash>),
+    /// [Bitcoin reference](https://en.bitcoin.it/wiki/Protocol_documentation#block)
+    Block(Arc<Block>),
 
     /// A `tx` message.
     ///
     /// [Bitcoin reference](https://en.bitcoin.it/wiki/Protocol_documentation#tx)
     Tx(Arc<Transaction>),
 
+    /// A `notfound` message.
+    ///
+    /// [Bitcoin reference](https://en.bitcoin.it/wiki/Protocol_documentation#notfound)
+    // See note above on `Inventory`.
+    NotFound(Vec<InventoryHash>),
+
     /// A `mempool` message.
     ///
     /// This was defined in [BIP35], which is included in Zcash.

From 2253ab3c00ec8845f1dbea0914c20643d23477e6 Mon Sep 17 00:00:00 2001
From: teor <teor@riseup.net>
Date: Tue, 17 Nov 2020 06:50:57 +1000
Subject: [PATCH 013/148] Improve state request docs

Document best and any chain requests
Explain that the block locator is sparse
---
 zebra-state/src/request.rs | 30 +++++++++++++++++-------------
 1 file changed, 17 insertions(+), 13 deletions(-)

diff --git a/zebra-state/src/request.rs b/zebra-state/src/request.rs
index 974210617bd..6a0da75c818 100644
--- a/zebra-state/src/request.rs
+++ b/zebra-state/src/request.rs
@@ -83,45 +83,49 @@ pub enum Request {
         // sapling_anchor: sapling::tree::Root,
     },
 
-    /// Computes the depth in the best chain of the block identified by the given hash.
+    /// Computes the depth in the current best chain of the block identified by the given hash.
     ///
     /// Returns
     ///
-    /// * [`Response::Depth(Some(depth))`](Response::Depth) if the block is in the main chain;
+    /// * [`Response::Depth(Some(depth))`](Response::Depth) if the block is in the best chain;
     /// * [`Response::Depth(None)`](Response::Depth) otherwise.
     Depth(block::Hash),
 
     /// Returns [`Response::Tip`] with the current best chain tip.
     Tip,
 
-    /// Computes a block locator object based on the current chain state.
+    /// Computes a block locator object based on the current best chain.
     ///
     /// Returns [`Response::BlockLocator`] with hashes starting
-    /// from the current chain tip and reaching backwards towards the genesis
-    /// block. The first hash is the current chain tip. The last hash is the tip
-    /// of the finalized portion of the state. If the state is empty, the block
-    /// locator is also empty.
+    /// from the best chain tip, and following the chain of previous
+    /// hashes. The first hash is the best chain tip. The last hash is
+    /// the tip of the finalized portion of the state. Block locators
+    /// are not continuous - some intermediate hashes might be skipped.
+    ///
+    /// If the state is empty, the block locator is also empty.
     BlockLocator,
 
-    /// Looks up a transaction by hash.
+    /// Looks up a transaction by hash in the current best chain.
     ///
     /// Returns
     ///
-    /// * [`Response::Transaction(Some(Arc<Transaction>))`](Response::Transaction) if the transaction is known;
+    /// * [`Response::Transaction(Some(Arc<Transaction>))`](Response::Transaction) if the transaction is in the best chain;
     /// * [`Response::Transaction(None)`](Response::Transaction) otherwise.
     Transaction(transaction::Hash),
 
-    /// Looks up a block by hash or height.
+    /// Looks up a block by hash or height in the current best chain.
     ///
     /// Returns
     ///
-    /// * [`Response::Block(Some(Arc<Block>))`](Response::Block) if the block is known;
-    /// * [`Response::Block(None)`](Response::Transaction) otherwise.
+    /// * [`Response::Block(Some(Arc<Block>))`](Response::Block) if the block is in the best chain;
+    /// * [`Response::Block(None)`](Response::Block) otherwise.
     ///
     /// Note: the [`HashOrHeight`] can be constructed from a [`block::Hash`] or
     /// [`block::Height`] using `.into()`.
     Block(HashOrHeight),
 
-    /// Request a UTXO identified by the given Outpoint
+    /// Request a UTXO identified by the given Outpoint in any chain.
+    ///
+    /// Returns UTXOs fron any chain, including side-chains.
     AwaitUtxo(transparent::OutPoint),
 }

From 4c2b44be9372938a4010c8c7073bd5dcf131410c Mon Sep 17 00:00:00 2001
From: Jane Lusby <jlusby42@gmail.com>
Date: Mon, 16 Nov 2020 15:31:22 -0800
Subject: [PATCH 014/148] Add tests for QueuedBlocks (#1268)

* Add unit test for QueuedBlocks
* Add test for pruned blocks
---
 .../src/service/memory_state/queued_blocks.rs | 156 +++++++++++++++++-
 1 file changed, 153 insertions(+), 3 deletions(-)

diff --git a/zebra-state/src/service/memory_state/queued_blocks.rs b/zebra-state/src/service/memory_state/queued_blocks.rs
index dc7721857fc..ec16d807c50 100644
--- a/zebra-state/src/service/memory_state/queued_blocks.rs
+++ b/zebra-state/src/service/memory_state/queued_blocks.rs
@@ -96,10 +96,27 @@ impl QueuedBlocks {
         for hash in by_height.into_iter().flat_map(|(_, hashes)| hashes) {
             let expired = self.blocks.remove(&hash).expect("block is present");
             let parent_hash = &expired.block.header.previous_block_hash;
-            self.by_parent
+
+            let parent_list = self
+                .by_parent
                 .get_mut(parent_hash)
-                .expect("parent is present")
-                .remove(&hash);
+                .expect("parent is present");
+
+            if parent_list.len() == 1 {
+                let removed = self
+                    .by_parent
+                    .remove(parent_hash)
+                    .expect("parent is present");
+                assert!(
+                    removed.contains(&hash),
+                    "hash must be present in parent hash list"
+                );
+            } else {
+                assert!(
+                    parent_list.remove(&hash),
+                    "hash must be present in parent hash list"
+                );
+            }
         }
 
         tracing::trace!(num_blocks = %self.blocks.len(), "Finished pruning blocks at or beneath the finalized tip height",);
@@ -122,3 +139,136 @@ impl QueuedBlocks {
         metrics::gauge!("state.memory.queued.block.count", self.blocks.len() as _);
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use tokio::sync::oneshot;
+    use zebra_chain::{block::Block, serialization::ZcashDeserializeInto};
+    use zebra_test::prelude::*;
+
+    use crate::tests::FakeChainHelper;
+
+    use self::assert_eq;
+    use super::*;
+
+    // Quick helper trait for making queued blocks with throw away channels
+    trait IntoQueued {
+        fn into_queued(self) -> QueuedBlock;
+    }
+
+    impl IntoQueued for Arc<Block> {
+        fn into_queued(self) -> QueuedBlock {
+            let (rsp_tx, _) = oneshot::channel();
+
+            QueuedBlock {
+                block: self,
+                rsp_tx,
+            }
+        }
+    }
+
+    #[test]
+    fn dequeue_gives_right_children() -> Result<()> {
+        zebra_test::init();
+
+        let block1: Arc<Block> =
+            zebra_test::vectors::BLOCK_MAINNET_419200_BYTES.zcash_deserialize_into()?;
+        let child1: Arc<Block> =
+            zebra_test::vectors::BLOCK_MAINNET_419201_BYTES.zcash_deserialize_into()?;
+        let child2 = block1.make_fake_child();
+
+        let parent = block1.header.previous_block_hash;
+
+        let mut queue = QueuedBlocks::default();
+        // Empty to start
+        assert_eq!(0, queue.blocks.len());
+        assert_eq!(0, queue.by_parent.len());
+        assert_eq!(0, queue.by_height.len());
+
+        // Inserting the first block gives us 1 in each table
+        queue.queue(block1.clone().into_queued());
+        assert_eq!(1, queue.blocks.len());
+        assert_eq!(1, queue.by_parent.len());
+        assert_eq!(1, queue.by_height.len());
+
+        // The second gives us one in each table because its a child of the first
+        queue.queue(child1.clone().into_queued());
+        assert_eq!(2, queue.blocks.len());
+        assert_eq!(2, queue.by_parent.len());
+        assert_eq!(2, queue.by_height.len());
+
+        // The 3rd only increments blocks, because it is also a child of the
+        // first block, so for the second and third tables it gets added to the
+        // existing HashSet value
+        queue.queue(child2.clone().into_queued());
+        assert_eq!(3, queue.blocks.len());
+        assert_eq!(2, queue.by_parent.len());
+        assert_eq!(2, queue.by_height.len());
+
+        // Dequeueing the first block removes 1 block from each list
+        let children = queue.dequeue_children(parent);
+        assert_eq!(1, children.len());
+        assert_eq!(block1, children[0].block);
+        assert_eq!(2, queue.blocks.len());
+        assert_eq!(1, queue.by_parent.len());
+        assert_eq!(1, queue.by_height.len());
+
+        // Dequeueing the children of the first block removes both of the other
+        // blocks, and empties all lists
+        let parent = children[0].block.hash();
+        let children = queue.dequeue_children(parent);
+        assert_eq!(2, children.len());
+        assert!(children
+            .iter()
+            .any(|QueuedBlock { block, .. }| block == &child1));
+        assert!(children
+            .iter()
+            .any(|QueuedBlock { block, .. }| block == &child2));
+        assert_eq!(0, queue.blocks.len());
+        assert_eq!(0, queue.by_parent.len());
+        assert_eq!(0, queue.by_height.len());
+
+        Ok(())
+    }
+
+    #[test]
+    fn prune_removes_right_children() -> Result<()> {
+        zebra_test::init();
+
+        let block1: Arc<Block> =
+            zebra_test::vectors::BLOCK_MAINNET_419200_BYTES.zcash_deserialize_into()?;
+        let child1: Arc<Block> =
+            zebra_test::vectors::BLOCK_MAINNET_419201_BYTES.zcash_deserialize_into()?;
+        let child2 = block1.make_fake_child();
+
+        let mut queue = QueuedBlocks::default();
+        queue.queue(block1.clone().into_queued());
+        queue.queue(child1.clone().into_queued());
+        queue.queue(child2.clone().into_queued());
+        assert_eq!(3, queue.blocks.len());
+        assert_eq!(2, queue.by_parent.len());
+        assert_eq!(2, queue.by_height.len());
+
+        // Pruning the first height removes only block1
+        queue.prune_by_height(block1.coinbase_height().unwrap());
+        assert_eq!(2, queue.blocks.len());
+        assert_eq!(1, queue.by_parent.len());
+        assert_eq!(1, queue.by_height.len());
+        assert!(queue.get_mut(&block1.hash()).is_none());
+        assert!(queue.get_mut(&child1.hash()).is_some());
+        assert!(queue.get_mut(&child2.hash()).is_some());
+
+        // Dequeueing the children of the first block removes both of the other
+        // blocks, and empties all lists
+        queue.prune_by_height(child1.coinbase_height().unwrap());
+        assert_eq!(0, queue.blocks.len());
+        assert_eq!(0, queue.by_parent.len());
+        assert_eq!(0, queue.by_height.len());
+        assert!(queue.get_mut(&child1.hash()).is_none());
+        assert!(queue.get_mut(&child2.hash()).is_none());
+
+        Ok(())
+    }
+}

From a6bd77e98a3625e8953ec0bda5bed59bea0ef84e Mon Sep 17 00:00:00 2001
From: Jane Lusby <jlusby42@gmail.com>
Date: Mon, 16 Nov 2020 15:53:33 -0800
Subject: [PATCH 015/148] Add check to ensure heights in state service are
 sequential (#1290)

* Add check to ensure heights in state service are sequential

Co-authored-by: teor <teor@riseup.net>
---
 zebra-state/src/service.rs    |  3 +++
 zebra-state/src/sled_state.rs | 38 ++++++++++++++++++++++++++++++++---
 2 files changed, 38 insertions(+), 3 deletions(-)

diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs
index 6b446d990fe..90769ed9d2a 100644
--- a/zebra-state/src/service.rs
+++ b/zebra-state/src/service.rs
@@ -196,7 +196,10 @@ impl StateService {
         let parent_height = parent_block
             .coinbase_height()
             .expect("valid blocks have a coinbase height");
+        let parent_hash = parent_block.hash();
         check::height_one_more_than_parent_height(parent_height, block)?;
+        // should be impossible by design, so no handleable error is thrown
+        assert_eq!(parent_hash, block.header.previous_block_hash);
 
         // TODO: contextual validation design and implementation
         Ok(())
diff --git a/zebra-state/src/sled_state.rs b/zebra-state/src/sled_state.rs
index 002f2b80e91..62a3a2b0aff 100644
--- a/zebra-state/src/sled_state.rs
+++ b/zebra-state/src/sled_state.rs
@@ -148,11 +148,45 @@ impl FinalizedState {
         let height = queued_block.block.coinbase_height().unwrap();
         self.queued_by_prev_hash.insert(prev_hash, queued_block);
 
-        while let Some(queued_block) = self.queued_by_prev_hash.remove(&self.finalized_tip_hash()) {
+        loop {
+            let finalized_tip_hash = self.finalized_tip_hash();
+            let queued_block =
+                if let Some(queued_block) = self.queued_by_prev_hash.remove(&finalized_tip_hash) {
+                    queued_block
+                } else {
+                    break;
+                };
+
             let height = queued_block
                 .block
                 .coinbase_height()
                 .expect("valid blocks must have a height");
+
+            if self.block_by_height.is_empty() {
+                assert_eq!(
+                    block::Hash([0; 32]),
+                    prev_hash,
+                    "the first block added to an empty state must be a genesis block"
+                );
+                assert_eq!(
+                    block::Height(0),
+                    height,
+                    "cannot commit genesis: invalid height"
+                );
+            } else {
+                assert_eq!(
+                    self.finalized_tip_height()
+                        .expect("state must have a genesis block committed")
+                        + 1,
+                    Some(height)
+                );
+
+                assert_eq!(
+                    finalized_tip_hash,
+                    queued_block.block.header.previous_block_hash
+                );
+            }
+
             self.commit_finalized(queued_block);
             metrics::counter!("state.finalized.committed.block.count", 1);
             metrics::gauge!("state.finalized.committed.block.height", height.0 as _);
@@ -216,8 +250,6 @@ impl FinalizedState {
                     sprout_nullifiers,
                     sapling_nullifiers,
                 )| {
-                    // TODO: check highest entry of hash_by_height as in RFC
-
                     // Index the block
                     hash_by_height.zs_insert(height, hash)?;
                     height_by_hash.zs_insert(hash, height)?;

From 54cb9277ef94630197a1ca8be75ebe1f58512d52 Mon Sep 17 00:00:00 2001
From: teor <teor@riseup.net>
Date: Mon, 16 Nov 2020 13:00:08 +1000
Subject: [PATCH 016/148] Allow some new clippy nightly lints

---
 zebra-chain/src/lib.rs     | 2 ++
 zebra-state/src/lib.rs     | 2 ++
 zebra-test/src/lib.rs      | 2 ++
 zebrad/tests/acceptance.rs | 2 ++
 4 files changed, 8 insertions(+)

diff --git a/zebra-chain/src/lib.rs b/zebra-chain/src/lib.rs
index 3c504a26fde..c35a8afde9b 100644
--- a/zebra-chain/src/lib.rs
+++ b/zebra-chain/src/lib.rs
@@ -8,6 +8,8 @@
 #![doc(html_root_url = "https://doc.zebra.zfnd.org/zebra_chain")]
 // #![deny(missing_docs)]
 #![allow(clippy::try_err)]
+#![allow(clippy::unknown_clippy_lints)]
+#![allow(clippy::from_iter_instead_of_collect)]
 
 #[macro_use]
 extern crate serde;
diff --git a/zebra-state/src/lib.rs b/zebra-state/src/lib.rs
index 5e716772efb..42fd4cea2aa 100644
--- a/zebra-state/src/lib.rs
+++ b/zebra-state/src/lib.rs
@@ -5,6 +5,8 @@
 #![doc(html_root_url = "https://doc.zebra.zfnd.org/zebra_state")]
 #![warn(missing_docs)]
 #![allow(clippy::try_err)]
+#![allow(clippy::unknown_clippy_lints)]
+#![allow(clippy::field_reassign_with_default)]
 
 mod config;
 mod constants;
diff --git a/zebra-test/src/lib.rs b/zebra-test/src/lib.rs
index 46919611acb..e45eb4a81e2 100644
--- a/zebra-test/src/lib.rs
+++ b/zebra-test/src/lib.rs
@@ -1,5 +1,7 @@
 //! Miscellaneous test code for Zebra.
 
+#![allow(clippy::unknown_clippy_lints)]
+#![allow(clippy::from_iter_instead_of_collect)]
 // Each lazy_static variable uses additional recursion
 #![recursion_limit = "256"]
 
diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs
index 227d9435431..742fb1ebd0f 100644
--- a/zebrad/tests/acceptance.rs
+++ b/zebrad/tests/acceptance.rs
@@ -14,6 +14,8 @@
 #![warn(warnings, missing_docs, trivial_casts, unused_qualifications)]
 #![forbid(unsafe_code)]
 #![allow(clippy::try_err)]
+#![allow(clippy::unknown_clippy_lints)]
+#![allow(clippy::field_reassign_with_default)]
 
 use color_eyre::eyre::Result;
 use eyre::WrapErr;

From d80a0c74026f6285aada7828220edab9b3f39104 Mon Sep 17 00:00:00 2001
From: teor <teor@riseup.net>
Date: Mon, 16 Nov 2020 13:06:40 +1000
Subject: [PATCH 017/148] Stop panicking during contextual validation

`check_contextual_validity` mistakenly used the new block's hash to try
to get the parent block from the state. This caused a panic, because the
new block isn't in the state yet.

Use `StateService::chain` to get the parent block, because we'll be
using `chain` for difficulty adjustment contextual verification anyway.
---
 zebra-state/src/service.rs | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs
index 90769ed9d2a..d502b3fae31 100644
--- a/zebra-state/src/service.rs
+++ b/zebra-state/src/service.rs
@@ -190,9 +190,10 @@ impl StateService {
         );
         check::block_is_not_orphaned(finalized_tip_height, block)?;
 
-        let parent_block = self
-            .block(block.hash().into())
-            .expect("the parent's presence has already been checked");
+        let mut relevant_chain = self.chain(block.header.previous_block_hash);
+        let parent_block = relevant_chain
+            .next()
+            .expect("state must contain parent block to do contextual validation");
         let parent_height = parent_block
             .coinbase_height()
             .expect("valid blocks have a coinbase height");
@@ -201,7 +202,8 @@ impl StateService {
         // should be impossible by design, so no handleable error is thrown
         assert_eq!(parent_hash, block.header.previous_block_hash);
 
-        // TODO: contextual validation design and implementation
+        // TODO: validate difficulty adjustment
+        // TODO: other contextual validation design and implelentation
         Ok(())
     }
 
@@ -272,7 +274,6 @@ impl StateService {
     ///
     /// The block identified by `hash` is included in the chain of blocks yielded
     /// by the iterator.
-    #[allow(dead_code)]
     pub fn chain(&self, hash: block::Hash) -> Iter<'_> {
         Iter {
             service: self,

From cfe779db69f468408b221cc7a9cafd14948f97fb Mon Sep 17 00:00:00 2001
From: teor <teor@riseup.net>
Date: Mon, 16 Nov 2020 15:46:16 +1000
Subject: [PATCH 018/148] Add an info-level span to check_contextual_validity

---
 zebra-state/src/service.rs | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs
index d502b3fae31..fd0faa2d855 100644
--- a/zebra-state/src/service.rs
+++ b/zebra-state/src/service.rs
@@ -50,6 +50,8 @@ struct StateService {
     queued_blocks: QueuedBlocks,
     /// The set of outpoints with pending requests for their associated transparent::Output
     pending_utxos: utxo::PendingUtxos,
+    /// The configured Zcash network
+    network: Network,
     /// Instant tracking the last time `pending_utxos` was pruned
     last_prune: Instant,
 }
@@ -68,6 +70,7 @@ impl StateService {
             mem,
             queued_blocks,
             pending_utxos,
+            network,
             last_prune: Instant::now(),
         }
     }
@@ -185,6 +188,15 @@ impl StateService {
     /// Check that `block` is contextually valid based on the committed finalized
     /// and non-finalized state.
     fn check_contextual_validity(&mut self, block: &Block) -> Result<(), ValidateContextError> {
+        let height = block
+            .coinbase_height()
+            .expect("semantically valid blocks have a coinbase height");
+        let hash = block.hash();
+
+        let span = tracing::info_span!("StateService::check_contextual_validity",
+                                             ?height, network = ?self.network, ?hash);
+        let _entered = span.enter();
+
         let finalized_tip_height = self.sled.finalized_tip_height().expect(
             "finalized state must contain at least one block to use the non-finalized state",
         );

From c8e6f5843f58bc1125deaa0614007495de3c15d7 Mon Sep 17 00:00:00 2001
From: Alfredo Garcia <oxarbitrage@gmail.com>
Date: Mon, 16 Nov 2020 22:10:21 -0300
Subject: [PATCH 019/148] Update RFC template (#1278)

* update rfc template
* change pull to issues
---
 .github/PULL_REQUEST_TEMPLATE/rfc.md | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/.github/PULL_REQUEST_TEMPLATE/rfc.md b/.github/PULL_REQUEST_TEMPLATE/rfc.md
index bdb39cf8501..e8a363a1eb1 100644
--- a/.github/PULL_REQUEST_TEMPLATE/rfc.md
+++ b/.github/PULL_REQUEST_TEMPLATE/rfc.md
@@ -15,10 +15,22 @@ The RFC process is documented in CONTRIBUTING.md.
 Please see the checklist there: https://zebra.zfnd.org/CONTRIBUTING.html
 -->
 
+
+### Summary
+
+Please copy the RFC summary over here. 
+
+### More information
+
 Feature Name: `my_feature`
+
 Start Date: YYYY-MM-DD
+
 Design PR: [ZcashFoundation/zebra#0000](https://github.com/ZcashFoundation/zebra/pull/0000)
-Zebra Issue: [ZcashFoundation/zebra#0000](https://github.com/ZcashFoundation/zebra/pull/0000)
+
+Zebra Issue: [ZcashFoundation/zebra#0000](https://github.com/ZcashFoundation/zebra/issues/0000)
+
+### Document
 
 <!--
 Fill this in, replacing:

From d7d15984ebcdfe27e1b6caeb8309824ad81e4125 Mon Sep 17 00:00:00 2001
From: teor <teor@riseup.net>
Date: Tue, 17 Nov 2020 10:55:24 +1000
Subject: [PATCH 020/148] Move all contextual validation code into its own
 function

This change has two benefits:
* reduces conflicts with the sled refactor and any replacement
* allows the function to be called independently for testing
---
 zebra-state/src/service.rs       | 37 +++++----------------
 zebra-state/src/service/check.rs | 57 +++++++++++++++++++++++++++++++-
 2 files changed, 64 insertions(+), 30 deletions(-)

diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs
index fd0faa2d855..6ca069e9585 100644
--- a/zebra-state/src/service.rs
+++ b/zebra-state/src/service.rs
@@ -185,37 +185,16 @@ impl StateService {
         }
     }
 
-    /// Check that `block` is contextually valid based on the committed finalized
-    /// and non-finalized state.
+    /// Check that `block` is contextually valid for the configured network,
+    /// based on the committed finalized and non-finalized state.
     fn check_contextual_validity(&mut self, block: &Block) -> Result<(), ValidateContextError> {
-        let height = block
-            .coinbase_height()
-            .expect("semantically valid blocks have a coinbase height");
-        let hash = block.hash();
+        check::block_is_contextually_valid(
+            block,
+            self.network,
+            self.sled.finalized_tip_height(),
+            self.chain(block.header.previous_block_hash),
+        )?;
 
-        let span = tracing::info_span!("StateService::check_contextual_validity",
-                                             ?height, network = ?self.network, ?hash);
-        let _entered = span.enter();
-
-        let finalized_tip_height = self.sled.finalized_tip_height().expect(
-            "finalized state must contain at least one block to use the non-finalized state",
-        );
-        check::block_is_not_orphaned(finalized_tip_height, block)?;
-
-        let mut relevant_chain = self.chain(block.header.previous_block_hash);
-        let parent_block = relevant_chain
-            .next()
-            .expect("state must contain parent block to do contextual validation");
-        let parent_height = parent_block
-            .coinbase_height()
-            .expect("valid blocks have a coinbase height");
-        let parent_hash = parent_block.hash();
-        check::height_one_more_than_parent_height(parent_height, block)?;
-        // should be impossible by design, so no handleable error is thrown
-        assert_eq!(parent_hash, block.header.previous_block_hash);
-
-        // TODO: validate difficulty adjustment
-        // TODO: other contextual validation design and implelentation
         Ok(())
     }
 
diff --git a/zebra-state/src/service/check.rs b/zebra-state/src/service/check.rs
index b62cb61dad2..f5a72700f8c 100644
--- a/zebra-state/src/service/check.rs
+++ b/zebra-state/src/service/check.rs
@@ -1,9 +1,64 @@
 //! Consensus critical contextual checks
 
-use zebra_chain::block::{self, Block};
+use zebra_chain::{
+    block::{self, Block},
+    parameters::Network,
+};
 
 use crate::ValidateContextError;
 
+use super::check;
+
+/// Check that `block` is contextually valid for `network`, based on the
+/// `finalized_tip_height` and `relevant_chain`.
+///
+/// The relevant chain is an iterator over the ancestors of `block`, starting
+/// with its parent block.
+pub(crate) fn block_is_contextually_valid<C>(
+    block: &Block,
+    network: Network,
+    finalized_tip_height: Option<block::Height>,
+    relevant_chain: C,
+) -> Result<(), ValidateContextError>
+where
+    C: IntoIterator,
+    C::Item: AsRef<Block>,
+{
+    let height = block
+        .coinbase_height()
+        .expect("semantically valid blocks have a coinbase height");
+    let hash = block.hash();
+
+    let span = tracing::info_span!(
+        "StateService::check_contextual_validity",
+        ?height,
+        ?network,
+        ?hash
+    );
+    let _entered = span.enter();
+
+    let finalized_tip_height = finalized_tip_height
+        .expect("finalized state must contain at least one block to use the non-finalized state");
+    check::block_is_not_orphaned(finalized_tip_height, block)?;
+
+    let mut relevant_chain = relevant_chain.into_iter();
+    let parent_block = relevant_chain
+        .next()
+        .expect("state must contain parent block to do contextual validation");
+    let parent_block = parent_block.as_ref();
+    let parent_height = parent_block
+        .coinbase_height()
+        .expect("valid blocks have a coinbase height");
+    let parent_hash = parent_block.hash();
+    check::height_one_more_than_parent_height(parent_height, block)?;
+    // should be impossible by design, so no handleable error is thrown
+    assert_eq!(parent_hash, block.header.previous_block_hash);
+
+    // TODO: validate difficulty adjustment
+    // TODO: other contextual validation design and implelentation
+    Ok(())
+}
+
 /// Returns `ValidateContextError::OrphanedBlock` if the height of the given
 /// block is less than or equal to the finalized tip height.
 pub(super) fn block_is_not_orphaned(

From 40b012acefef75d4f1d887172c45356e6995870d Mon Sep 17 00:00:00 2001
From: Deirdre Connolly <deirdre@zfnd.org>
Date: Mon, 16 Nov 2020 20:13:09 -0500
Subject: [PATCH 021/148] Add mdbook stuff to path using environment
 files/variables instead of workflow commands

Fixes #1309
---
 .github/workflows/{firebase.yml => docs.yml} | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 rename .github/workflows/{firebase.yml => docs.yml} (97%)

diff --git a/.github/workflows/firebase.yml b/.github/workflows/docs.yml
similarity index 97%
rename from .github/workflows/firebase.yml
rename to .github/workflows/docs.yml
index ec92d3d371e..01612a21767 100644
--- a/.github/workflows/firebase.yml
+++ b/.github/workflows/docs.yml
@@ -26,7 +26,7 @@ jobs:
         cd book
         curl -L https://github.com/rust-lang/mdBook/releases/download/v0.4.1/mdbook-v0.4.1-x86_64-unknown-linux-gnu.tar.gz | tar xz
         # Add the book directory to the $PATH
-        echo "::add-path::$GITHUB_WORKSPACE/book"
+        echo "$GITHUB_WORKSPACE/book" >> $GITHUB_PATH
 
     - name: Build Zebra book
       run: |

From 2f53ff44f7cac079b45bdfd758efae20d7a2fbe7 Mon Sep 17 00:00:00 2001
From: teor <teor@riseup.net>
Date: Tue, 17 Nov 2020 12:25:35 +1000
Subject: [PATCH 022/148] Move chain order assertions to
 commit_finalized_direct

And remove a duplicate assert in the contextual verification function.
---
 zebra-state/src/service/check.rs |  3 --
 zebra-state/src/sled_state.rs    | 70 ++++++++++++++------------------
 2 files changed, 30 insertions(+), 43 deletions(-)

diff --git a/zebra-state/src/service/check.rs b/zebra-state/src/service/check.rs
index f5a72700f8c..b7a6a18c893 100644
--- a/zebra-state/src/service/check.rs
+++ b/zebra-state/src/service/check.rs
@@ -49,10 +49,7 @@ where
     let parent_height = parent_block
         .coinbase_height()
         .expect("valid blocks have a coinbase height");
-    let parent_hash = parent_block.hash();
     check::height_one_more_than_parent_height(parent_height, block)?;
-    // should be impossible by design, so no handleable error is thrown
-    assert_eq!(parent_hash, block.header.previous_block_hash);
 
     // TODO: validate difficulty adjustment
     // TODO: other contextual validation design and implelentation
diff --git a/zebra-state/src/sled_state.rs b/zebra-state/src/sled_state.rs
index 62a3a2b0aff..31f99da82f6 100644
--- a/zebra-state/src/sled_state.rs
+++ b/zebra-state/src/sled_state.rs
@@ -148,45 +148,7 @@ impl FinalizedState {
         let height = queued_block.block.coinbase_height().unwrap();
         self.queued_by_prev_hash.insert(prev_hash, queued_block);
 
-        loop {
-            let finalized_tip_hash = self.finalized_tip_hash();
-            let queued_block =
-                if let Some(queued_block) = self.queued_by_prev_hash.remove(&finalized_tip_hash) {
-                    queued_block
-                } else {
-                    break;
-                };
-
-            let height = queued_block
-                .block
-                .coinbase_height()
-                .expect("valid blocks must have a height");
-
-            if self.block_by_height.is_empty() {
-                assert_eq!(
-                    block::Hash([0; 32]),
-                    prev_hash,
-                    "the first block added to an empty state must be a genesis block"
-                );
-                assert_eq!(
-                    block::Height(0),
-                    height,
-                    "cannot commit genesis: invalid height"
-                );
-            } else {
-                assert_eq!(
-                    self.finalized_tip_height()
-                        .expect("state must have a genesis block committed")
-                        + 1,
-                    Some(height)
-                );
-
-                assert_eq!(
-                    finalized_tip_hash,
-                    queued_block.block.header.previous_block_hash
-                );
-            }
-
+        while let Some(queued_block) = self.queued_by_prev_hash.remove(&self.finalized_tip_hash()) {
             self.commit_finalized(queued_block);
             metrics::counter!("state.finalized.committed.block.count", 1);
             metrics::gauge!("state.finalized.committed.block.height", height.0 as _);
@@ -229,7 +191,35 @@ impl FinalizedState {
             .expect("finalized blocks are valid and have a coinbase height");
         let hash = block.hash();
 
-        trace!(?height, "Finalized block");
+        trace!(?height, ?hash, "Finalized block");
+
+        // Assert that callers (including unit tests) get the chain order correct
+        if self.block_by_height.is_empty() {
+            assert_eq!(
+                block::Hash([0; 32]),
+                block.header.previous_block_hash,
+                "the first block added to an empty state must be a genesis block"
+            );
+            assert_eq!(
+                block::Height(0),
+                height,
+                "cannot commit genesis: invalid height"
+            );
+        } else {
+            assert_eq!(
+                self.finalized_tip_height()
+                    .expect("state must have a genesis block committed")
+                    + 1,
+                Some(height),
+                "committed block height must be 1 more than the finalized tip height"
+            );
+
+            assert_eq!(
+                self.finalized_tip_hash(),
+                block.header.previous_block_hash,
+                "committed block must be a child of the finalized tip"
+            );
+        }
 
         let result = (
             &self.hash_by_height,

From d5d17a9a71885d1454e05b3a98a53afbad0fbca1 Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Thu, 12 Nov 2020 11:08:20 -0800
Subject: [PATCH 023/148] consensus: remove incorrect comment

The ZcashDeserialize implementation for Block doesn't check that blocks
have a coinbase height.
---
 zebra-consensus/src/block.rs | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/zebra-consensus/src/block.rs b/zebra-consensus/src/block.rs
index 2e0f99af4bc..6c2926ea71f 100644
--- a/zebra-consensus/src/block.rs
+++ b/zebra-consensus/src/block.rs
@@ -138,9 +138,6 @@ where
                 _ => unreachable!("wrong response to Request::Depth"),
             }
 
-            // We repeat the height checks here, to ensure that generated blocks
-            // are valid. (We check the block heights for parsed blocks when we
-            // deserialize them.)
             let height = block
                 .coinbase_height()
                 .ok_or(BlockError::MissingHeight(hash))?;

From a3ab589d89b8013c646a36d7a915767b08b29015 Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Thu, 12 Nov 2020 11:43:17 -0800
Subject: [PATCH 024/148] consensus,state: document cancellation contracts for
 services

This change explicitly documents cancellation contracts for our Tower services,
and tries to correct a bug in the implementation of the CheckpointVerifier,
which duplicates information from the state service but did not ensure that it
would be kept in sync.
---
 zebra-consensus/src/chain.rs                  | 12 +++-
 zebra-consensus/src/checkpoint.rs             | 55 +++++++++++--------
 .../src/protocol/internal/request.rs          | 12 ++++
 zebra-state/src/request.rs                    | 42 ++++++++++----
 zebra-state/src/service.rs                    | 12 ++--
 5 files changed, 88 insertions(+), 45 deletions(-)

diff --git a/zebra-consensus/src/chain.rs b/zebra-consensus/src/chain.rs
index 3e3722ae65a..06dea6e69bd 100644
--- a/zebra-consensus/src/chain.rs
+++ b/zebra-consensus/src/chain.rs
@@ -87,10 +87,18 @@ where
     }
 }
 
-/// Return a block verification service, using `config`, `network` and
-/// `state_service`.
+/// Initialize a block verification service.
+///
+/// The consensus configuration is specified by `config`, and the Zcash network
+/// to verify blocks for is specified by `network`.
+///
+/// The block verification service asynchronously performs semantic verification
+/// checks. Blocks that pass semantic verification are submitted to the supplied
+/// `state_service` for contextual verification before being committed to the chain.
 ///
 /// This function should only be called once for a particular state service.
+///
+/// Dropped requests are cancelled on a best-effort basis, but may continue to be processed.
 #[instrument(skip(state_service))]
 pub async fn init<S>(
     config: Config,
diff --git a/zebra-consensus/src/checkpoint.rs b/zebra-consensus/src/checkpoint.rs
index c9a407eff7a..be8b1285669 100644
--- a/zebra-consensus/src/checkpoint.rs
+++ b/zebra-consensus/src/checkpoint.rs
@@ -98,8 +98,6 @@ where
     S: Service<zs::Request, Response = zs::Response, Error = BoxError> + Send + Clone + 'static,
     S::Future: Send + 'static,
 {
-    // Inputs
-    //
     /// The checkpoint list for this verifier.
     checkpoint_list: CheckpointList,
 
@@ -109,8 +107,6 @@ where
     /// The underlying state service, possibly wrapped in other services.
     state_service: S,
 
-    // Queued Blocks
-    //
     /// A queue of unverified blocks.
     ///
     /// Contains a list of unverified blocks at each block height. In most cases,
@@ -127,9 +123,6 @@ where
     verifier_progress: Progress<block::Height>,
 }
 
-/// The CheckpointVerifier implementation.
-///
-/// Contains non-service utility functions for CheckpointVerifiers.
 impl<S> CheckpointVerifier<S>
 where
     S: Service<zs::Request, Response = zs::Response, Error = BoxError> + Send + Clone + 'static,
@@ -386,8 +379,7 @@ where
                     height,
                     verified_height: previous_height,
                 });
-                // TODO: reduce to trace level once the AlreadyVerified bug is fixed
-                tracing::info!(?e);
+                tracing::trace!(?e);
                 e?;
             }
             InitialTip(_) | PreviousCheckpoint(_) => {}
@@ -580,7 +572,7 @@ where
         valid_qblock
     }
 
-    /// Check all the blocks in the current checkpoint range.
+    /// Try to verify from the previous checkpoint to a target checkpoint.
     ///
     /// Send `Ok` for the blocks that are in the chain, and `Err` for side-chain
     /// blocks.
@@ -830,34 +822,43 @@ where
             return async { Err(VerifyCheckpointError::Finished) }.boxed();
         }
 
-        // Queue the block for verification, until we receive all the blocks for
-        // the current checkpoint range.
         let rx = self.queue_block(block.clone());
+        self.process_checkpoint_range();
 
-        // Try to verify from the previous checkpoint to a target checkpoint.
+        metrics::gauge!("checkpoint.queued_slots", self.queued.len() as i64);
+
+        // Because the checkpoint verifier duplicates state from the state
+        // service (it tracks which checkpoints have been verified), we must
+        // commit blocks transactionally on a per-checkpoint basis. Otherwise,
+        // the checkpoint verifier's state could desync from the underlying
+        // state service. Among other problems, this could cause the checkpoint
+        // verifier to reject blocks not already in the state as
+        // already-verified.
         //
-        // If there are multiple checkpoints in the target range, and one of
-        // the ranges is invalid, we'll try again with a smaller target range
-        // on the next call(). Failures always reject a block, so we know
-        // there will be at least one more call().
+        // To commit blocks transactionally on a per-checkpoint basis, we must
+        // commit all verified blocks in a checkpoint range, regardless of
+        // whether or not the response futures for each block were dropped.
         //
-        // We don't retry with a smaller range on failure, because failures
-        // should be rare.
-        self.process_checkpoint_range();
-
+        // We accomplish this by spawning a new task containing the
+        // commit-if-verified logic. This task will always execute, except if
+        // the program is interrupted, in which case there is no longer a
+        // checkpoint verifier to keep in sync with the state.
         let mut state_service = self.state_service.clone();
-        async move {
+        let commit_finalized_block = tokio::spawn(async move {
             let hash = rx
                 .await
                 .expect("CheckpointVerifier does not leave dangling receivers")?;
 
+            // Once we get a verified hash, we must commit it to the chain state
+            // as a finalized block, or exit the program, so .expect rather than
+            // propagate errors from the state service.
             match state_service
                 .ready_and()
                 .await
-                .map_err(VerifyCheckpointError::CommitFinalized)?
+                .expect("Verified checkpoints must be committed transactionally")
                 .call(zs::Request::CommitFinalizedBlock { block })
                 .await
-                .map_err(VerifyCheckpointError::CommitFinalized)?
+                .expect("Verified checkpoints must be committed transactionally")
             {
                 zs::Response::Committed(committed_hash) => {
                     assert_eq!(committed_hash, hash, "state must commit correct hash");
@@ -865,6 +866,12 @@ where
                 }
                 _ => unreachable!("wrong response for CommitFinalizedBlock"),
             }
+        });
+
+        async move {
+            commit_finalized_block
+                .await
+                .expect("commit_finalized_block should not panic")
         }
         .boxed()
     }
diff --git a/zebra-network/src/protocol/internal/request.rs b/zebra-network/src/protocol/internal/request.rs
index f0f4cc87377..39fc436ffa4 100644
--- a/zebra-network/src/protocol/internal/request.rs
+++ b/zebra-network/src/protocol/internal/request.rs
@@ -14,6 +14,18 @@ use super::super::types::Nonce;
 /// possible [`Response`s](super::Response) it can generate; it is fine (and
 /// recommended!) to match on the expected responses and treat the others as
 /// `unreachable!()`, since their return indicates a bug in the network code.
+///
+/// # Cancellations
+///
+/// The peer set handles cancelled requests (i.e., requests where the future
+/// returned by `Service::call` is dropped before it resolves) on a best-effort
+/// basis. Requests are routed to a particular peer connection, and then
+/// translated into Zcash protocol messages and sent over the network. If a
+/// request is cancelled after it is submitted but before it is processed by a
+/// peer connection, no messages will be sent. Otherwise, if it is cancelled
+/// while waiting for a response, the peer connection resets its state and makes
+/// a best-effort attempt to ignore any messages responsive to the cancelled
+/// request, subject to limitations in the underlying Zcash protocol.
 #[derive(Clone, Debug)]
 pub enum Request {
     /// Requests additional peers from the server.
diff --git a/zebra-state/src/request.rs b/zebra-state/src/request.rs
index 6a0da75c818..761677351f8 100644
--- a/zebra-state/src/request.rs
+++ b/zebra-state/src/request.rs
@@ -53,11 +53,18 @@ pub enum Request {
     /// Performs contextual validation of the given block, committing it to the
     /// state if successful.
     ///
-    /// Returns [`Response::Committed`] with the hash of the newly
-    /// committed block, or an error.
+    /// It is the caller's responsibility to perform semantic validation. This
+    /// request can be made out-of-order; the state service will queue it until
+    /// its parent is ready.
     ///
-    /// This request can be made out-of-order; the state service will buffer it
-    /// until its parent is ready.
+    /// Returns [`Response::Committed`] with the hash of the block when it is
+    /// committed to the state, or an error if the block fails contextual
+    /// validation or has already been committed to the state.
+    ///
+    /// This request cannot be cancelled once submitted; dropping the response
+    /// future will have no effect on whether it is eventually processed. A
+    /// request to commit a block which has been queued internally but not yet
+    /// committed will fail the older request and replace it with the newer request.
     CommitBlock {
         /// The block to commit to the state.
         block: Arc<Block>,
@@ -66,15 +73,20 @@ pub enum Request {
         // sapling_anchor: sapling::tree::Root,
     },
 
-    /// Commit a finalized block to the state, skipping contextual validation.
+    /// Commit a finalized block to the state, skipping all validation.
+    ///
     /// This is exposed for use in checkpointing, which produces finalized
-    /// blocks.
+    /// blocks. It is the caller's responsibility to ensure that the block is
+    /// valid and final. This request can be made out-of-order; the state service
+    /// will queue it until its parent is ready.
     ///
-    /// Returns [`Response::Committed`] with the hash of the newly
-    /// committed block, or an error.
+    /// Returns [`Response::Committed`] with the hash of the newly committed
+    /// block, or an error.
     ///
-    /// This request can be made out-of-order; the state service will buffer it
-    /// until its parent is ready.
+    /// This request cannot be cancelled once submitted; dropping the response
+    /// future will have no effect on whether it is eventually processed.
+    /// Duplicate requests should not be made, because it is the caller's
+    /// responsibility to ensure that each block is valid and final.
     CommitFinalizedBlock {
         /// The block to commit to the state.
         block: Arc<Block>,
@@ -124,8 +136,14 @@ pub enum Request {
     /// [`block::Height`] using `.into()`.
     Block(HashOrHeight),
 
-    /// Request a UTXO identified by the given Outpoint in any chain.
+    /// Request a UTXO identified by the given Outpoint, waiting until it becomes
+    /// available if it is unknown.
+    ///
+    /// This request is purely informational, and there are no guarantees about
+    /// whether the UTXO remains unspent or is on the best chain. Its purpose is
+    /// to allow asynchronous script verification.
     ///
-    /// Returns UTXOs fron any chain, including side-chains.
+    /// Code making this request should apply a timeout layer to the service to
+    /// handle missing UTXOs.
     AwaitUtxo(transparent::OutPoint),
 }
diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs
index 6ca069e9585..a100eaf9333 100644
--- a/zebra-state/src/service.rs
+++ b/zebra-state/src/service.rs
@@ -92,19 +92,17 @@ impl StateService {
 
         if self.contains_committed_block(&block) {
             let (rsp_tx, rsp_rx) = oneshot::channel();
-            let _ = rsp_tx.send(Err("duplicate block".into()));
+            let _ = rsp_tx.send(Err("block is already committed to the state".into()));
             return rsp_rx;
         }
 
-        // The queue of blocks maintained by this service acts as a pipeline for
-        // blocks waiting for contextual verification. We lazily flush the
-        // pipeline here by handling duplicate requests to verify an existing
-        // queued block. We handle those duplicate requests by replacing the old
-        // channel with the new one and sending an error over the old channel.
+        // Request::CommitBlock contract: a request to commit a block which has
+        // been queued but not yet committed to the state fails the older
+        // request and replaces it with the newer request.
         let rsp_rx = if let Some(queued_block) = self.queued_blocks.get_mut(&hash) {
             let (mut rsp_tx, rsp_rx) = oneshot::channel();
             std::mem::swap(&mut queued_block.rsp_tx, &mut rsp_tx);
-            let _ = rsp_tx.send(Err("duplicate block".into()));
+            let _ = rsp_tx.send(Err("replaced by newer request".into()));
             rsp_rx
         } else {
             let (rsp_tx, rsp_rx) = oneshot::channel();

From aa8d95bd23e58a3fffdf4092f149c8bcababee7b Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Thu, 12 Nov 2020 14:38:47 -0800
Subject: [PATCH 025/148] consensus: improve checkpoint request replacement
 diagnostics

---
 zebra-consensus/src/checkpoint.rs | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/zebra-consensus/src/checkpoint.rs b/zebra-consensus/src/checkpoint.rs
index be8b1285669..0e1422b83b9 100644
--- a/zebra-consensus/src/checkpoint.rs
+++ b/zebra-consensus/src/checkpoint.rs
@@ -454,12 +454,7 @@ where
         let height = match self.check_block(&block) {
             Ok(height) => height,
             Err(error) => {
-                // Block errors happen frequently on mainnet, due to bad peers.
-                tracing::trace!(?error);
-
-                // Sending might fail, depending on what the caller does with rx,
-                // but there's nothing we can do about it.
-                let _ = tx.send(Err(error));
+                tx.send(Err(error)).expect("rx has not been dropped yet");
                 return rx;
             }
         };
@@ -474,11 +469,12 @@ where
 
         let hash = block.hash();
 
+        // Replace older requests by newer ones by swapping the oneshot.
         for qb in qblocks.iter_mut() {
             if qb.hash == hash {
-                let old_tx = std::mem::replace(&mut qb.tx, tx);
                 let e = VerifyCheckpointError::NewerRequest { height, hash };
-                tracing::trace!(?e);
+                tracing::trace!(?e, "failing older of duplicate requests");
+                let old_tx = std::mem::replace(&mut qb.tx, tx);
                 let _ = old_tx.send(Err(e));
                 return rx;
             }

From e0b2af7123f4bf5b70dc80100e9663dc772e7bcf Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Thu, 12 Nov 2020 20:34:49 -0800
Subject: [PATCH 026/148] state: add sled tree precommit metrics on tracked
 objects

---
 zebra-state/src/sled_state.rs | 62 +++++++++++++++++++++++++++++++++--
 1 file changed, 60 insertions(+), 2 deletions(-)

diff --git a/zebra-state/src/sled_state.rs b/zebra-state/src/sled_state.rs
index 31f99da82f6..1d211e86a8e 100644
--- a/zebra-state/src/sled_state.rs
+++ b/zebra-state/src/sled_state.rs
@@ -2,7 +2,6 @@
 
 use std::{collections::HashMap, convert::TryInto, sync::Arc};
 
-use tracing::trace;
 use zebra_chain::transparent;
 use zebra_chain::{
     block::{self, Block},
@@ -191,7 +190,7 @@ impl FinalizedState {
             .expect("finalized blocks are valid and have a coinbase height");
         let hash = block.hash();
 
-        trace!(?height, ?hash, "Finalized block");
+        block_precommit_metrics(&hash, height, &block);
 
         // Assert that callers (including unit tests) get the chain order correct
         if self.block_by_height.is_empty() {
@@ -381,3 +380,62 @@ impl FinalizedState {
             })
     }
 }
+
+fn block_precommit_metrics(hash: &block::Hash, height: block::Height, block: &Block) {
+    let transaction_count = block.transactions.len();
+    let transparent_prevout_count = block
+        .transactions
+        .iter()
+        .flat_map(|t| t.inputs().iter())
+        .count()
+        // Each block has a single coinbase input which is not a previous output.
+        - 1;
+    let transparent_newout_count = block
+        .transactions
+        .iter()
+        .flat_map(|t| t.outputs().iter())
+        .count();
+
+    let sprout_nullifier_count = block
+        .transactions
+        .iter()
+        .flat_map(|t| t.sprout_nullifiers())
+        .count();
+
+    let sapling_nullifier_count = block
+        .transactions
+        .iter()
+        .flat_map(|t| t.sapling_nullifiers())
+        .count();
+
+    tracing::debug!(
+        ?hash,
+        ?height,
+        transaction_count,
+        transparent_prevout_count,
+        transparent_newout_count,
+        sprout_nullifier_count,
+        sapling_nullifier_count,
+        "preparing to commit finalized block"
+    );
+    metrics::counter!(
+        "state.finalized.cumulative.transactions",
+        transaction_count as u64
+    );
+    metrics::counter!(
+        "state.finalized.cumulative.transparent_prevouts",
+        transparent_prevout_count as u64
+    );
+    metrics::counter!(
+        "state.finalized.cumulative.transparent_newouts",
+        transparent_newout_count as u64
+    );
+    metrics::counter!(
+        "state.finalized.cumulative.sprout_nullifiers",
+        sprout_nullifier_count as u64
+    );
+    metrics::counter!(
+        "state.finalized.cumulative.sapling_nullifiers",
+        sapling_nullifier_count as u64
+    );
+}

From ec411574ee76637767db60434d48d094fb7ecfef Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Thu, 12 Nov 2020 14:38:57 -0800
Subject: [PATCH 027/148] zebrad: improve sync diagnostics

---
 zebrad/src/components/sync.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/zebrad/src/components/sync.rs b/zebrad/src/components/sync.rs
index 07b3a459dfd..ecf687b2bae 100644
--- a/zebrad/src/components/sync.rs
+++ b/zebrad/src/components/sync.rs
@@ -191,7 +191,7 @@ where
 
             tracing::info!("starting sync, obtaining new tips");
             if let Err(e) = self.obtain_tips().await {
-                tracing::warn!(?e);
+                tracing::warn!(?e, "error obtaining tips");
                 continue 'sync;
             }
             self.update_metrics();
@@ -204,7 +204,7 @@ where
                             tracing::trace!(?hash, "verified and committed block to state");
                         }
                         Err(e) => {
-                            tracing::warn!(?e);
+                            tracing::warn!(?e, "error downloading and verifying block");
                             continue 'sync;
                         }
                     }
@@ -236,7 +236,7 @@ where
                             tracing::trace!(?hash, "verified and committed block to state");
                         }
                         Err(e) => {
-                            tracing::warn!(?e);
+                            tracing::warn!(?e, "error downloading and verifying block");
                             continue 'sync;
                         }
                     }
@@ -252,7 +252,7 @@ where
                 );
 
                 if let Err(e) = self.extend_tips().await {
-                    tracing::warn!(?e);
+                    tracing::warn!(?e, "error extending tips");
                     continue 'sync;
                 }
                 self.update_metrics();

From b632a244364ff4935aa239f4396055602648574d Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Thu, 12 Nov 2020 14:39:03 -0800
Subject: [PATCH 028/148] zebrad: add diagnostics on cancelled download tasks

---
 zebrad/src/components/sync/downloads.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/zebrad/src/components/sync/downloads.rs b/zebrad/src/components/sync/downloads.rs
index 9bda7744ed1..177a27c0a18 100644
--- a/zebrad/src/components/sync/downloads.rs
+++ b/zebrad/src/components/sync/downloads.rs
@@ -140,6 +140,7 @@ where
             async move {
                 let rsp = tokio::select! {
                     _ = &mut cancel_rx => {
+                        tracing::trace!("task cancelled prior to download completion");
                         metrics::counter!("sync.cancelled.download.count", 1);
                         return Err("canceled block_fetch_verify".into())
                     }
@@ -159,6 +160,7 @@ where
                 let rsp = verifier.ready_and().await?.call(block);
                 let verification = tokio::select! {
                     _ = &mut cancel_rx => {
+                        tracing::trace!("task cancelled prior to verification");
                         metrics::counter!("sync.cancelled.verify.count", 1);
                         return Err("canceled block_fetch_verify".into())
                     }

From e9c847bbd774f698f028ab0ce424619fb56cf37f Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Thu, 12 Nov 2020 20:01:16 -0800
Subject: [PATCH 029/148] zebrad: avoid a borrow in the ChainSync future

---
 zebrad/src/commands/start.rs  | 2 +-
 zebrad/src/components/sync.rs | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/zebrad/src/commands/start.rs b/zebrad/src/commands/start.rs
index 619d47bde96..f829f355267 100644
--- a/zebrad/src/commands/start.rs
+++ b/zebrad/src/commands/start.rs
@@ -75,7 +75,7 @@ impl StartCmd {
             .map_err(|_| eyre!("could not send setup data to inbound service"))?;
 
         info!("initializing syncer");
-        let mut syncer = ChainSync::new(&config, peer_set, state, verifier);
+        let syncer = ChainSync::new(&config, peer_set, state, verifier);
 
         syncer.sync().await
     }
diff --git a/zebrad/src/components/sync.rs b/zebrad/src/components/sync.rs
index ecf687b2bae..ba749a4c97e 100644
--- a/zebrad/src/components/sync.rs
+++ b/zebrad/src/components/sync.rs
@@ -169,7 +169,7 @@ where
     }
 
     #[instrument(skip(self))]
-    pub async fn sync(&mut self) -> Result<(), Report> {
+    pub async fn sync(mut self) -> Result<(), Report> {
         // We can't download the genesis block using our normal algorithm,
         // due to protocol limitations
         self.request_genesis().await?;

From 6de824bd99f52039e12178a30979cb27c8170095 Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Thu, 12 Nov 2020 20:08:42 -0800
Subject: [PATCH 030/148] zebrad: remove block verification timeout

Because we set the lookahead limit to be at least twice the size of a checkpoint, we don't have a risk of timeouts.
---
 zebrad/src/components/sync.rs | 36 ++---------------------------------
 1 file changed, 2 insertions(+), 34 deletions(-)

diff --git a/zebrad/src/components/sync.rs b/zebrad/src/components/sync.rs
index ba749a4c97e..6646af7b500 100644
--- a/zebrad/src/components/sync.rs
+++ b/zebrad/src/components/sync.rs
@@ -46,21 +46,6 @@ const TIPS_RESPONSE_TIMEOUT: Duration = Duration::from_secs(6);
 /// Controls how long we wait for a block download request to complete.
 const BLOCK_DOWNLOAD_TIMEOUT: Duration = Duration::from_secs(20);
 
-/// The maximum amount of time that Zebra should take to download a checkpoint
-/// full of blocks. Based on the current `MAX_CHECKPOINT_BYTE_SIZE`.
-///
-/// We assume that Zebra nodes have at least 10 Mbps bandwidth, and allow some
-/// extra time for request latency.
-const MAX_CHECKPOINT_DOWNLOAD_SECONDS: u64 = 300;
-
-/// Controls how long we wait for a block verify task to complete.
-///
-/// This timeout makes sure that the syncer and verifiers do not deadlock.
-/// When the lookahead limit is reached, the syncer waits for blocks to verify
-/// (or fail). If the verifiers are also waiting for more blocks from the syncer,
-/// then without a timeout, Zebra would deadlock.
-const BLOCK_VERIFY_TIMEOUT: Duration = Duration::from_secs(MAX_CHECKPOINT_DOWNLOAD_SECONDS);
-
 /// Controls how long we wait to restart syncing after finishing a sync run.
 ///
 /// This timeout should be long enough to:
@@ -102,10 +87,7 @@ where
     lookahead_limit: usize,
     downloads: Pin<
         Box<
-            Downloads<
-                Hedge<ConcurrencyLimit<Retry<zn::RetryLimit, Timeout<ZN>>>, AlwaysHedge>,
-                Timeout<ZV>,
-            >,
+            Downloads<Hedge<ConcurrencyLimit<Retry<zn::RetryLimit, Timeout<ZN>>>, AlwaysHedge>, ZV>,
         >,
     >,
 }
@@ -158,10 +140,7 @@ where
         Self {
             tip_network,
             state,
-            downloads: Box::pin(Downloads::new(
-                block_network,
-                Timeout::new(verifier, BLOCK_VERIFY_TIMEOUT),
-            )),
+            downloads: Box::pin(Downloads::new(block_network, verifier)),
             prospective_tips: HashSet::new(),
             genesis_hash: genesis_hash(config.network.network),
             lookahead_limit: std::cmp::max(config.sync.lookahead_limit, MIN_LOOKAHEAD_LIMIT),
@@ -587,20 +566,9 @@ mod test {
     fn ensure_timeouts_consistent() {
         zebra_test::init();
 
-        let max_download_retry_time =
-            BLOCK_DOWNLOAD_TIMEOUT.as_secs() * (BLOCK_DOWNLOAD_RETRY_LIMIT as u64);
-        assert!(
-            max_download_retry_time < BLOCK_VERIFY_TIMEOUT.as_secs(),
-            "Verify timeout should allow for previous block download retries"
-        );
         assert!(
             BLOCK_DOWNLOAD_TIMEOUT.as_secs() * 2 < SYNC_RESTART_TIMEOUT.as_secs(),
             "Sync restart should allow for pending and buffered requests to complete"
         );
-
-        assert!(
-            SYNC_RESTART_TIMEOUT < BLOCK_VERIFY_TIMEOUT,
-            "Verify timeout should allow for a sync restart"
-        );
     }
 }

From e55392b61ec5d99e43d5a14f24432aabeb0255a9 Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Thu, 12 Nov 2020 20:32:23 -0800
Subject: [PATCH 031/148] zebrad: explicitly select the threaded scheduler.

---
 zebrad/src/components/tokio.rs | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/zebrad/src/components/tokio.rs b/zebrad/src/components/tokio.rs
index 386c0bf22af..c0dc22d09cc 100644
--- a/zebrad/src/components/tokio.rs
+++ b/zebrad/src/components/tokio.rs
@@ -21,7 +21,13 @@ pub struct TokioComponent {
 impl TokioComponent {
     pub fn new() -> Result<Self, FrameworkError> {
         Ok(Self {
-            rt: Some(Runtime::new().unwrap()),
+            rt: Some(
+                tokio::runtime::Builder::new()
+                    .enable_all()
+                    .threaded_scheduler()
+                    .build()
+                    .unwrap(),
+            ),
         })
     }
 }

From aa7538ab15bbb08c36dd8ffdbf509f705947ca0d Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Thu, 12 Nov 2020 20:35:27 -0800
Subject: [PATCH 032/148] zebrad: hack to skip alreadyverified errors

---
 zebrad/src/components/sync.rs | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/zebrad/src/components/sync.rs b/zebrad/src/components/sync.rs
index 6646af7b500..74e54bbf15f 100644
--- a/zebrad/src/components/sync.rs
+++ b/zebrad/src/components/sync.rs
@@ -214,9 +214,14 @@ where
                         Ok(hash) => {
                             tracing::trace!(?hash, "verified and committed block to state");
                         }
+
                         Err(e) => {
-                            tracing::warn!(?e, "error downloading and verifying block");
-                            continue 'sync;
+                            if format!("{:?}", e).contains("AlreadyVerified {") {
+                                tracing::info!(error = ?e, "error seems like it might be already verified, this is probably fine i guess?")
+                            } else {
+                                tracing::warn!(?e, "error downloading and verifying block");
+                                continue 'sync;
+                            }
                         }
                     }
                     self.update_metrics();

From d2fc01755bd51faefddc4ecb72526d83ffb59020 Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Fri, 13 Nov 2020 10:52:39 -0800
Subject: [PATCH 033/148] zebrad: more reasonable concurrent block limit

This helps prevent overloading the network with too many concurrent
block requests.  On a fast network, we're likely to still have enough
room to saturate our bandwidth.  In the worst case, with 2MB blocks,
downloading 50 blocks concurrently is 100MB of queued downloads.  If we
need to download this in 20 seconds to avoid peer connection timeouts,
the implied worst-case minimum speed is 5MB/s.  In practice, this
minimum speed will likely be much lower.
---
 zebrad/src/config.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/zebrad/src/config.rs b/zebrad/src/config.rs
index 0112edf6ea1..dde94a13aea 100644
--- a/zebrad/src/config.rs
+++ b/zebrad/src/config.rs
@@ -157,7 +157,7 @@ pub struct SyncSection {
 impl Default for SyncSection {
     fn default() -> Self {
         Self {
-            max_concurrent_block_requests: 1_000,
+            max_concurrent_block_requests: 50,
             lookahead_limit: 2_000,
         }
     }

From 608b3953aff6f6f7c327fd4acb5e65a3f0f75631 Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Fri, 13 Nov 2020 17:23:58 -0800
Subject: [PATCH 034/148] deps: cargo update

---
 Cargo.lock | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 268023d6476..f2dc6d52dfd 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2600,9 +2600,9 @@ checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8"
 
 [[package]]
 name = "sled"
-version = "0.34.5"
+version = "0.34.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "34a9981af2a13b88f562777db36ed6e8695eef9be8bbbfb1c8b1c83762b8c15c"
+checksum = "1d0132f3e393bcb7390c60bb45769498cf4550bcb7a21d7f95c02b69f6362cdc"
 dependencies = [
  "crc32fast",
  "crossbeam-epoch 0.9.0",
@@ -2856,9 +2856,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
 
 [[package]]
 name = "tokio"
-version = "0.2.22"
+version = "0.2.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d34ca54d84bf2b5b4d7d31e901a8464f7b60ac145a284fba25ceb801f2ddccd"
+checksum = "a6d7ad61edd59bfcc7e80dababf0f4aed2e6d5e0ba1659356ae889752dfc12ff"
 dependencies = [
  "bytes",
  "fnv",
@@ -3163,9 +3163,9 @@ dependencies = [
 
 [[package]]
 name = "unicode-segmentation"
-version = "1.6.0"
+version = "1.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e83e153d1053cbb5a118eeff7fd5be06ed99153f00dbcd8ae310c5fb2b22edc0"
+checksum = "db8716a166f290ff49dabc18b44aa407cb7c6dbe1aa0971b44b8a24b0ca35aae"
 
 [[package]]
 name = "unicode-width"

From 3edc1f7db48fb58112e164a3f9e9714e5d471fd3 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 17 Nov 2020 10:00:42 +0000
Subject: [PATCH 035/148] build(deps): bump codecov/codecov-action from v1.0.14
 to v1.0.15

Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from v1.0.14 to v1.0.15.
- [Release notes](https://github.com/codecov/codecov-action/releases)
- [Commits](https://github.com/codecov/codecov-action/compare/v1.0.14...239febf655bba88b16ff5dea1d3135ea8663a1f9)

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/coverage.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml
index 021831bd449..10875a7e46a 100644
--- a/.github/workflows/coverage.yml
+++ b/.github/workflows/coverage.yml
@@ -29,4 +29,4 @@ jobs:
           version: '0.16.0'
           timeout: 600
       - name: Upload coverage report to Codecov
-        uses: codecov/codecov-action@v1.0.14
+        uses: codecov/codecov-action@v1.0.15

From 4953f216706f2ee1c49341e9192691e916eb0e97 Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Tue, 17 Nov 2020 16:23:26 -0800
Subject: [PATCH 036/148] fixup! zebrad: hack to skip alreadyverified errors

---
 zebrad/src/components/sync.rs | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/zebrad/src/components/sync.rs b/zebrad/src/components/sync.rs
index 74e54bbf15f..d758f1c2393 100644
--- a/zebrad/src/components/sync.rs
+++ b/zebrad/src/components/sync.rs
@@ -183,8 +183,12 @@ where
                             tracing::trace!(?hash, "verified and committed block to state");
                         }
                         Err(e) => {
-                            tracing::warn!(?e, "error downloading and verifying block");
-                            continue 'sync;
+                            if format!("{:?}", e).contains("AlreadyVerified {") {
+                                tracing::debug!(error = ?e, "block was already verified, possibly from a previous sync run, continuing");
+                            } else {
+                                tracing::warn!(?e, "error downloading and verifying block");
+                                continue 'sync;
+                            }
                         }
                     }
                 }
@@ -217,7 +221,7 @@ where
 
                         Err(e) => {
                             if format!("{:?}", e).contains("AlreadyVerified {") {
-                                tracing::info!(error = ?e, "error seems like it might be already verified, this is probably fine i guess?")
+                                tracing::debug!(error = ?e, "block was already verified, possibly from a previous sync run, continuing");
                             } else {
                                 tracing::warn!(?e, "error downloading and verifying block");
                                 continue 'sync;

From a122a547be55d10c2c82e6bf5e49beb39cbb62d7 Mon Sep 17 00:00:00 2001
From: Jane Lusby <jane@zfnd.org>
Date: Mon, 16 Nov 2020 16:05:35 -0800
Subject: [PATCH 037/148] reorganize modules for consistency

---
 zebra-state/src/lib.rs                                |  4 ----
 zebra-state/src/service.rs                            |  9 ++++++---
 .../src/{sled_state.rs => service/finalized_state.rs} | 10 +++++-----
 .../finalized_state}/sled_format.rs                   |  0
 zebra-state/src/service/memory_state.rs               | 11 -----------
 .../service/{memory_state => }/non_finalized_state.rs | 11 ++++++++++-
 .../{memory_state => non_finalized_state}/chain.rs    |  0
 .../queued_blocks.rs                                  |  0
 8 files changed, 21 insertions(+), 24 deletions(-)
 rename zebra-state/src/{sled_state.rs => service/finalized_state.rs} (98%)
 rename zebra-state/src/{sled_state => service/finalized_state}/sled_format.rs (100%)
 delete mode 100644 zebra-state/src/service/memory_state.rs
 rename zebra-state/src/service/{memory_state => }/non_finalized_state.rs (98%)
 rename zebra-state/src/service/{memory_state => non_finalized_state}/chain.rs (100%)
 rename zebra-state/src/service/{memory_state => non_finalized_state}/queued_blocks.rs (100%)

diff --git a/zebra-state/src/lib.rs b/zebra-state/src/lib.rs
index 42fd4cea2aa..cdf6c339280 100644
--- a/zebra-state/src/lib.rs
+++ b/zebra-state/src/lib.rs
@@ -14,16 +14,12 @@ mod error;
 mod request;
 mod response;
 mod service;
-mod sled_state;
 mod util;
 
 // TODO: move these to integration tests.
 #[cfg(test)]
 mod tests;
 
-use service::QueuedBlock;
-use sled_state::FinalizedState;
-
 pub use config::Config;
 pub use constants::MAX_BLOCK_REORG_HEIGHT;
 pub use error::{BoxError, CloneError, CommitBlockError, ValidateContextError};
diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs
index a100eaf9333..fe7d05c902b 100644
--- a/zebra-state/src/service.rs
+++ b/zebra-state/src/service.rs
@@ -8,7 +8,7 @@ use std::{
 };
 
 use futures::future::FutureExt;
-use memory_state::{NonFinalizedState, QueuedBlocks};
+use non_finalized_state::{NonFinalizedState, QueuedBlocks};
 use tokio::sync::oneshot;
 use tower::{util::BoxService, Service};
 use tracing::instrument;
@@ -21,12 +21,15 @@ use zebra_chain::{
 };
 
 use crate::{
-    request::HashOrHeight, BoxError, CommitBlockError, Config, FinalizedState, Request, Response,
+    request::HashOrHeight, BoxError, CommitBlockError, Config, Request, Response,
     ValidateContextError,
 };
 
+use self::finalized_state::FinalizedState;
+
 mod check;
-mod memory_state;
+mod finalized_state;
+mod non_finalized_state;
 #[cfg(test)]
 mod tests;
 mod utxo;
diff --git a/zebra-state/src/sled_state.rs b/zebra-state/src/service/finalized_state.rs
similarity index 98%
rename from zebra-state/src/sled_state.rs
rename to zebra-state/src/service/finalized_state.rs
index 1d211e86a8e..472022f9890 100644
--- a/zebra-state/src/sled_state.rs
+++ b/zebra-state/src/service/finalized_state.rs
@@ -1,5 +1,7 @@
 //! The primary implementation of the `zebra_state::Service` built upon sled
 
+mod sled_format;
+
 use std::{collections::HashMap, convert::TryInto, sync::Arc};
 
 use zebra_chain::transparent;
@@ -9,13 +11,11 @@ use zebra_chain::{
     transaction::{self, Transaction},
 };
 
-use crate::{BoxError, Config, HashOrHeight, QueuedBlock};
-
-mod sled_format;
+use crate::{BoxError, Config, HashOrHeight};
 
-use sled_format::{FromSled, IntoSled, SledDeserialize, SledSerialize};
+use self::sled_format::{FromSled, IntoSled, SledDeserialize, SledSerialize, TransactionLocation};
 
-use self::sled_format::TransactionLocation;
+use super::QueuedBlock;
 
 /// The finalized part of the chain state, stored in sled.
 ///
diff --git a/zebra-state/src/sled_state/sled_format.rs b/zebra-state/src/service/finalized_state/sled_format.rs
similarity index 100%
rename from zebra-state/src/sled_state/sled_format.rs
rename to zebra-state/src/service/finalized_state/sled_format.rs
diff --git a/zebra-state/src/service/memory_state.rs b/zebra-state/src/service/memory_state.rs
deleted file mode 100644
index 88926220481..00000000000
--- a/zebra-state/src/service/memory_state.rs
+++ /dev/null
@@ -1,11 +0,0 @@
-//! Non-finalized chain state management as defined by [RFC0005]
-//!
-//! [RFC0005]: https://zebra.zfnd.org/dev/rfcs/0005-state-updates.html
-
-mod chain;
-mod non_finalized_state;
-mod queued_blocks;
-
-use chain::Chain;
-pub use non_finalized_state::NonFinalizedState;
-pub use queued_blocks::QueuedBlocks;
diff --git a/zebra-state/src/service/memory_state/non_finalized_state.rs b/zebra-state/src/service/non_finalized_state.rs
similarity index 98%
rename from zebra-state/src/service/memory_state/non_finalized_state.rs
rename to zebra-state/src/service/non_finalized_state.rs
index f4bcbe1e54d..2444e7e105f 100644
--- a/zebra-state/src/service/memory_state/non_finalized_state.rs
+++ b/zebra-state/src/service/non_finalized_state.rs
@@ -1,3 +1,12 @@
+//! Non-finalized chain state management as defined by [RFC0005]
+//!
+//! [RFC0005]: https://zebra.zfnd.org/dev/rfcs/0005-state-updates.html
+
+mod chain;
+mod queued_blocks;
+
+pub use queued_blocks::QueuedBlocks;
+
 use std::{collections::BTreeSet, mem, ops::Deref, sync::Arc};
 
 use zebra_chain::{
@@ -8,7 +17,7 @@ use zebra_chain::{
 
 use crate::request::HashOrHeight;
 
-use super::Chain;
+use self::chain::Chain;
 
 /// The state of the chains in memory, incuding queued blocks.
 #[derive(Default)]
diff --git a/zebra-state/src/service/memory_state/chain.rs b/zebra-state/src/service/non_finalized_state/chain.rs
similarity index 100%
rename from zebra-state/src/service/memory_state/chain.rs
rename to zebra-state/src/service/non_finalized_state/chain.rs
diff --git a/zebra-state/src/service/memory_state/queued_blocks.rs b/zebra-state/src/service/non_finalized_state/queued_blocks.rs
similarity index 100%
rename from zebra-state/src/service/memory_state/queued_blocks.rs
rename to zebra-state/src/service/non_finalized_state/queued_blocks.rs

From 5a6a9fd51e6439a00f68f1749570f7a75dd520e7 Mon Sep 17 00:00:00 2001
From: Jane Lusby <jane@zfnd.org>
Date: Mon, 16 Nov 2020 16:05:59 -0800
Subject: [PATCH 038/148] remove some references to sled in serialization
 definition module

---
 zebra-state/src/service/finalized_state.rs    |   4 +-
 .../{sled_format.rs => disk_format.rs}        | 100 +++++++++---------
 2 files changed, 52 insertions(+), 52 deletions(-)
 rename zebra-state/src/service/finalized_state/{sled_format.rs => disk_format.rs} (86%)

diff --git a/zebra-state/src/service/finalized_state.rs b/zebra-state/src/service/finalized_state.rs
index 472022f9890..a4158d3dcc3 100644
--- a/zebra-state/src/service/finalized_state.rs
+++ b/zebra-state/src/service/finalized_state.rs
@@ -1,6 +1,6 @@
 //! The primary implementation of the `zebra_state::Service` built upon sled
 
-mod sled_format;
+mod disk_format;
 
 use std::{collections::HashMap, convert::TryInto, sync::Arc};
 
@@ -13,7 +13,7 @@ use zebra_chain::{
 
 use crate::{BoxError, Config, HashOrHeight};
 
-use self::sled_format::{FromSled, IntoSled, SledDeserialize, SledSerialize, TransactionLocation};
+use self::disk_format::{DiskDeserialize, DiskSerialize, FromDisk, IntoDisk, TransactionLocation};
 
 use super::QueuedBlock;
 
diff --git a/zebra-state/src/service/finalized_state/sled_format.rs b/zebra-state/src/service/finalized_state/disk_format.rs
similarity index 86%
rename from zebra-state/src/service/finalized_state/sled_format.rs
rename to zebra-state/src/service/finalized_state/disk_format.rs
index ca5d9d95d57..77e9a4f33b5 100644
--- a/zebra-state/src/service/finalized_state/sled_format.rs
+++ b/zebra-state/src/service/finalized_state/disk_format.rs
@@ -15,24 +15,24 @@ pub struct TransactionLocation {
     pub index: u32,
 }
 
-// Helper trait for defining the exact format used to interact with sled per
+// Helper trait for defining the exact format used to interact with disk per
 // type.
-pub trait IntoSled {
+pub trait IntoDisk {
     // The type used to compare a value as a key to other keys stored in a
-    // sled::Tree
+    // database
     type Bytes: AsRef<[u8]>;
 
-    // function to convert the current type to its sled format in `zs_get()`
+    // function to convert the current type to its disk format in `zs_get()`
     // without necessarily allocating a new IVec
     fn as_bytes(&self) -> Self::Bytes;
 
-    // function to convert the current type into its sled format
+    // function to convert the current type into its disk format
     fn into_ivec(&self) -> sled::IVec;
 }
 
-impl<'a, T> IntoSled for &'a T
+impl<'a, T> IntoDisk for &'a T
 where
-    T: IntoSled,
+    T: IntoDisk,
 {
     type Bytes = T::Bytes;
 
@@ -45,9 +45,9 @@ where
     }
 }
 
-impl<T> IntoSled for Arc<T>
+impl<T> IntoDisk for Arc<T>
 where
-    T: IntoSled,
+    T: IntoDisk,
 {
     type Bytes = T::Bytes;
 
@@ -60,11 +60,11 @@ where
     }
 }
 
-/// Helper type for retrieving types from sled with the correct format.
+/// Helper type for retrieving types from the disk with the correct format.
 ///
-/// The ivec should be correctly encoded by IntoSled.
-pub trait FromSled: Sized {
-    /// Function to convert the sled bytes back into the deserialized type.
+/// The ivec should be correctly encoded by IntoDisk.
+pub trait FromDisk: Sized {
+    /// Function to convert the disk bytes back into the deserialized type.
     ///
     /// # Panics
     ///
@@ -72,16 +72,16 @@ pub trait FromSled: Sized {
     fn from_ivec(bytes: sled::IVec) -> Self;
 }
 
-impl<T> FromSled for Arc<T>
+impl<T> FromDisk for Arc<T>
 where
-    T: FromSled,
+    T: FromDisk,
 {
     fn from_ivec(bytes: sled::IVec) -> Self {
         Arc::new(T::from_ivec(bytes))
     }
 }
 
-impl IntoSled for Block {
+impl IntoDisk for Block {
     type Bytes = Vec<u8>;
 
     fn as_bytes(&self) -> Self::Bytes {
@@ -94,14 +94,14 @@ impl IntoSled for Block {
     }
 }
 
-impl FromSled for Block {
+impl FromDisk for Block {
     fn from_ivec(bytes: sled::IVec) -> Self {
         Block::zcash_deserialize(bytes.as_ref())
             .expect("deserialization format should match the serialization format used by IntoSled")
     }
 }
 
-impl IntoSled for TransactionLocation {
+impl IntoDisk for TransactionLocation {
     type Bytes = [u8; 8];
 
     fn as_bytes(&self) -> Self::Bytes {
@@ -121,7 +121,7 @@ impl IntoSled for TransactionLocation {
     }
 }
 
-impl FromSled for TransactionLocation {
+impl FromDisk for TransactionLocation {
     fn from_ivec(sled_bytes: sled::IVec) -> Self {
         let height = {
             let mut bytes = [0; 4];
@@ -140,7 +140,7 @@ impl FromSled for TransactionLocation {
     }
 }
 
-impl IntoSled for transaction::Hash {
+impl IntoDisk for transaction::Hash {
     type Bytes = [u8; 32];
 
     fn as_bytes(&self) -> Self::Bytes {
@@ -152,7 +152,7 @@ impl IntoSled for transaction::Hash {
     }
 }
 
-impl IntoSled for block::Hash {
+impl IntoDisk for block::Hash {
     type Bytes = [u8; 32];
 
     fn as_bytes(&self) -> Self::Bytes {
@@ -163,14 +163,14 @@ impl IntoSled for block::Hash {
     }
 }
 
-impl FromSled for block::Hash {
+impl FromDisk for block::Hash {
     fn from_ivec(bytes: sled::IVec) -> Self {
         let array = bytes.as_ref().try_into().unwrap();
         Self(array)
     }
 }
 
-impl IntoSled for sprout::Nullifier {
+impl IntoDisk for sprout::Nullifier {
     type Bytes = [u8; 32];
 
     fn as_bytes(&self) -> Self::Bytes {
@@ -182,7 +182,7 @@ impl IntoSled for sprout::Nullifier {
     }
 }
 
-impl IntoSled for sapling::Nullifier {
+impl IntoDisk for sapling::Nullifier {
     type Bytes = [u8; 32];
 
     fn as_bytes(&self) -> Self::Bytes {
@@ -194,7 +194,7 @@ impl IntoSled for sapling::Nullifier {
     }
 }
 
-impl IntoSled for () {
+impl IntoDisk for () {
     type Bytes = [u8; 0];
 
     fn as_bytes(&self) -> Self::Bytes {
@@ -206,7 +206,7 @@ impl IntoSled for () {
     }
 }
 
-impl IntoSled for block::Height {
+impl IntoDisk for block::Height {
     type Bytes = [u8; 4];
 
     fn as_bytes(&self) -> Self::Bytes {
@@ -217,14 +217,14 @@ impl IntoSled for block::Height {
     }
 }
 
-impl FromSled for block::Height {
+impl FromDisk for block::Height {
     fn from_ivec(bytes: sled::IVec) -> Self {
         let array = bytes.as_ref().try_into().unwrap();
         block::Height(u32::from_be_bytes(array))
     }
 }
 
-impl IntoSled for transparent::Output {
+impl IntoDisk for transparent::Output {
     type Bytes = Vec<u8>;
 
     fn as_bytes(&self) -> Self::Bytes {
@@ -237,14 +237,14 @@ impl IntoSled for transparent::Output {
     }
 }
 
-impl FromSled for transparent::Output {
+impl FromDisk for transparent::Output {
     fn from_ivec(bytes: sled::IVec) -> Self {
         Self::zcash_deserialize(&*bytes)
             .expect("deserialization format should match the serialization format used by IntoSled")
     }
 }
 
-impl IntoSled for transparent::OutPoint {
+impl IntoDisk for transparent::OutPoint {
     type Bytes = Vec<u8>;
 
     fn as_bytes(&self) -> Self::Bytes {
@@ -259,7 +259,7 @@ impl IntoSled for transparent::OutPoint {
 
 /// Helper trait for inserting (Key, Value) pairs into sled with a consistently
 /// defined format
-pub trait SledSerialize {
+pub trait DiskSerialize {
     /// Serialize and insert the given key and value into a sled tree.
     fn zs_insert<K, V>(
         &self,
@@ -267,19 +267,19 @@ pub trait SledSerialize {
         value: V,
     ) -> Result<(), sled::transaction::UnabortableTransactionError>
     where
-        K: IntoSled + Debug,
-        V: IntoSled;
+        K: IntoDisk + Debug,
+        V: IntoDisk;
 }
 
-impl SledSerialize for sled::transaction::TransactionalTree {
+impl DiskSerialize for sled::transaction::TransactionalTree {
     fn zs_insert<K, V>(
         &self,
         key: K,
         value: V,
     ) -> Result<(), sled::transaction::UnabortableTransactionError>
     where
-        K: IntoSled + Debug,
-        V: IntoSled,
+        K: IntoDisk + Debug,
+        V: IntoDisk,
     {
         use std::any::type_name;
 
@@ -301,20 +301,20 @@ impl SledSerialize for sled::transaction::TransactionalTree {
 
 /// Helper trait for retrieving values from sled trees with a consistently
 /// defined format
-pub trait SledDeserialize {
+pub trait DiskDeserialize {
     /// Serialize the given key and use that to get and deserialize the
     /// corresponding value from a sled tree, if it is present.
     fn zs_get<K, V>(&self, key: &K) -> Option<V>
     where
-        K: IntoSled,
-        V: FromSled;
+        K: IntoDisk,
+        V: FromDisk;
 }
 
-impl SledDeserialize for sled::Tree {
+impl DiskDeserialize for sled::Tree {
     fn zs_get<K, V>(&self, key: &K) -> Option<V>
     where
-        K: IntoSled,
-        V: FromSled,
+        K: IntoDisk,
+        V: FromDisk,
     {
         let key_bytes = key.as_bytes();
 
@@ -345,7 +345,7 @@ mod tests {
 
     fn round_trip<T>(input: T) -> T
     where
-        T: IntoSled + FromSled,
+        T: IntoDisk + FromDisk,
     {
         let bytes = input.into_ivec();
         T::from_ivec(bytes)
@@ -353,7 +353,7 @@ mod tests {
 
     fn assert_round_trip<T>(input: T)
     where
-        T: IntoSled + FromSled + Clone + PartialEq + std::fmt::Debug,
+        T: IntoDisk + FromDisk + Clone + PartialEq + std::fmt::Debug,
     {
         let before = input.clone();
         let after = round_trip(input);
@@ -362,7 +362,7 @@ mod tests {
 
     fn round_trip_ref<T>(input: &T) -> T
     where
-        T: IntoSled + FromSled,
+        T: IntoDisk + FromDisk,
     {
         let bytes = input.into_ivec();
         T::from_ivec(bytes)
@@ -370,7 +370,7 @@ mod tests {
 
     fn assert_round_trip_ref<T>(input: &T)
     where
-        T: IntoSled + FromSled + Clone + PartialEq + std::fmt::Debug,
+        T: IntoDisk + FromDisk + Clone + PartialEq + std::fmt::Debug,
     {
         let before = input;
         let after = round_trip_ref(input);
@@ -379,7 +379,7 @@ mod tests {
 
     fn round_trip_arc<T>(input: Arc<T>) -> T
     where
-        T: IntoSled + FromSled,
+        T: IntoDisk + FromDisk,
     {
         let bytes = input.into_ivec();
         T::from_ivec(bytes)
@@ -387,7 +387,7 @@ mod tests {
 
     fn assert_round_trip_arc<T>(input: Arc<T>)
     where
-        T: IntoSled + FromSled + Clone + PartialEq + std::fmt::Debug,
+        T: IntoDisk + FromDisk + Clone + PartialEq + std::fmt::Debug,
     {
         let before = input.clone();
         let after = round_trip_arc(input);
@@ -399,7 +399,7 @@ mod tests {
     /// ones that implement both `IntoSled` and `FromSled`.
     fn assert_value_properties<T>(input: T)
     where
-        T: IntoSled + FromSled + Clone + PartialEq + std::fmt::Debug,
+        T: IntoDisk + FromDisk + Clone + PartialEq + std::fmt::Debug,
     {
         assert_round_trip_ref(&input);
         assert_round_trip_arc(Arc::new(input.clone()));
@@ -414,7 +414,7 @@ mod tests {
     /// keys in the sled database, such as `contains`.
     fn assert_as_bytes_matches_ivec<T>(input: T)
     where
-        T: IntoSled + Clone,
+        T: IntoDisk + Clone,
     {
         let before = input.clone();
         let ivec = input.into_ivec();

From 65a605520f392811d5cd56b14d592764e3c86d9b Mon Sep 17 00:00:00 2001
From: Jane Lusby <jane@zfnd.org>
Date: Tue, 17 Nov 2020 15:26:21 -0800
Subject: [PATCH 039/148] remove references to sled from service.rs

---
 Cargo.lock                 | 18 ++++++++---------
 zebra-script/Cargo.toml    |  2 +-
 zebra-state/src/service.rs | 40 +++++++++++++++++++-------------------
 3 files changed, 30 insertions(+), 30 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index f2dc6d52dfd..94d2590117a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -193,9 +193,9 @@ dependencies = [
 
 [[package]]
 name = "bindgen"
-version = "0.55.1"
+version = "0.54.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75b13ce559e6433d360c26305643803cb52cfbabbc2b9c47ce04a58493dfb443"
+checksum = "66c0bb6167449588ff70803f4127f0684f9063097eca5016f37eb52b92c2cf36"
 dependencies = [
  "bitflags",
  "cexpr",
@@ -430,9 +430,9 @@ dependencies = [
 
 [[package]]
 name = "clang-sys"
-version = "1.0.1"
+version = "0.29.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa785e9017cb8e8c8045e3f096b7d1ebc4d7337cceccdca8d678a27f788ac133"
+checksum = "fe6837df1d5cba2397b835c8530f51723267e16abbf83892e9e5af4f0e5dd10a"
 dependencies = [
  "glob",
  "libc",
@@ -1468,11 +1468,11 @@ checksum = "4d58d1b70b004888f764dfbf6a26a3b0342a1632d33968e4a179d8011c760614"
 
 [[package]]
 name = "libloading"
-version = "0.6.5"
+version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1090080fe06ec2648d0da3881d9453d97e71a45f00eb179af7fdd7e3f686fdb0"
+checksum = "f2b111a074963af1d37a139918ac6d49ad1d0d5e47f72fd55388619691a7d753"
 dependencies = [
- "cfg-if 1.0.0",
+ "cc",
  "winapi 0.3.9",
 ]
 
@@ -3337,9 +3337,9 @@ dependencies = [
 
 [[package]]
 name = "zcash_script"
-version = "0.1.3"
+version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f50ddde89b1a9ae1677de9eaff3578a005da42879622ddef1a057b037e0ff834"
+checksum = "8bf1995ddb0c827f160357922a96c0cd34c6dd759757cb2b799128d6f420c91e"
 dependencies = [
  "bindgen",
  "blake2b_simd",
diff --git a/zebra-script/Cargo.toml b/zebra-script/Cargo.toml
index 78a99b84f05..7fa9812d08f 100644
--- a/zebra-script/Cargo.toml
+++ b/zebra-script/Cargo.toml
@@ -8,7 +8,7 @@ edition = "2018"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-zcash_script = "0.1.3"
+zcash_script = "0.1.4"
 zebra-chain = { path = "../zebra-chain" }
 thiserror = "1.0.22"
 displaydoc = "0.1.7"
diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs
index fe7d05c902b..0181e8c67af 100644
--- a/zebra-state/src/service.rs
+++ b/zebra-state/src/service.rs
@@ -46,7 +46,7 @@ pub struct QueuedBlock {
 
 struct StateService {
     /// Holds data relating to finalized chain state.
-    sled: FinalizedState,
+    disk: FinalizedState,
     /// Holds data relating to non-finalized chain state.
     mem: NonFinalizedState,
     /// Blocks awaiting their parent blocks for contextual verification.
@@ -63,13 +63,13 @@ impl StateService {
     const PRUNE_INTERVAL: Duration = Duration::from_secs(30);
 
     pub fn new(config: Config, network: Network) -> Self {
-        let sled = FinalizedState::new(&config, network);
+        let disk = FinalizedState::new(&config, network);
         let mem = NonFinalizedState::default();
         let queued_blocks = QueuedBlocks::default();
         let pending_utxos = utxo::PendingUtxos::default();
 
         Self {
-            sled,
+            disk,
             mem,
             queued_blocks,
             pending_utxos,
@@ -121,13 +121,13 @@ impl StateService {
 
         while self.mem.best_chain_len() > crate::constants::MAX_BLOCK_REORG_HEIGHT {
             let finalized = self.mem.finalize();
-            self.sled
+            self.disk
                 .commit_finalized_direct(finalized)
-                .expect("expected that sled errors would not occur");
+                .expect("expected that disk errors would not occur");
         }
 
         self.queued_blocks
-            .prune_by_height(self.sled.finalized_tip_height().expect(
+            .prune_by_height(self.disk.finalized_tip_height().expect(
             "Finalized state must have at least one block before committing non-finalized state",
         ));
 
@@ -140,7 +140,7 @@ impl StateService {
         self.check_contextual_validity(&block)?;
         let parent_hash = block.header.previous_block_hash;
 
-        if self.sled.finalized_tip_hash() == parent_hash {
+        if self.disk.finalized_tip_hash() == parent_hash {
             self.mem.commit_new_chain(block);
         } else {
             self.mem.commit_block(block);
@@ -151,7 +151,7 @@ impl StateService {
 
     /// Returns `true` if `hash` is a valid previous block hash for new non-finalized blocks.
     fn can_fork_chain_at(&self, hash: &block::Hash) -> bool {
-        self.mem.any_chain_contains(hash) || &self.sled.finalized_tip_hash() == hash
+        self.mem.any_chain_contains(hash) || &self.disk.finalized_tip_hash() == hash
     }
 
     /// Returns true if the given hash has been committed to either the finalized
@@ -162,7 +162,7 @@ impl StateService {
             .coinbase_height()
             .expect("coinbase heights should be valid");
 
-        self.mem.any_chain_contains(&hash) || self.sled.hash(height) == Some(hash)
+        self.mem.any_chain_contains(&hash) || self.disk.hash(height) == Some(hash)
     }
 
     /// Attempt to validate and commit all queued blocks whose parents have
@@ -192,7 +192,7 @@ impl StateService {
         check::block_is_contextually_valid(
             block,
             self.network,
-            self.sled.finalized_tip_height(),
+            self.disk.finalized_tip_height(),
             self.chain(block.header.previous_block_hash),
         )?;
 
@@ -217,13 +217,13 @@ impl StateService {
 
     /// Return the tip of the current best chain.
     pub fn tip(&self) -> Option<(block::Height, block::Hash)> {
-        self.mem.tip().or_else(|| self.sled.tip())
+        self.mem.tip().or_else(|| self.disk.tip())
     }
 
     /// Return the depth of block `hash` in the current best chain.
     pub fn depth(&self, hash: block::Hash) -> Option<u32> {
         let tip = self.tip()?.0;
-        let height = self.mem.height(hash).or_else(|| self.sled.height(hash))?;
+        let height = self.mem.height(hash).or_else(|| self.disk.height(hash))?;
 
         Some(tip.0 - height.0)
     }
@@ -233,7 +233,7 @@ impl StateService {
     pub fn block(&self, hash_or_height: HashOrHeight) -> Option<Arc<Block>> {
         self.mem
             .block(hash_or_height)
-            .or_else(|| self.sled.block(hash_or_height))
+            .or_else(|| self.disk.block(hash_or_height))
     }
 
     /// Return the transaction identified by `hash` if it exists in the current
@@ -241,24 +241,24 @@ impl StateService {
     pub fn transaction(&self, hash: transaction::Hash) -> Option<Arc<Transaction>> {
         self.mem
             .transaction(hash)
-            .or_else(|| self.sled.transaction(hash))
+            .or_else(|| self.disk.transaction(hash))
     }
 
     /// Return the hash for the block at `height` in the current best chain.
     pub fn hash(&self, height: block::Height) -> Option<block::Hash> {
-        self.mem.hash(height).or_else(|| self.sled.hash(height))
+        self.mem.hash(height).or_else(|| self.disk.hash(height))
     }
 
     /// Return the height for the block at `hash` in any chain.
     pub fn height_by_hash(&self, hash: block::Hash) -> Option<block::Height> {
         self.mem
             .height_by_hash(hash)
-            .or_else(|| self.sled.height(hash))
+            .or_else(|| self.disk.height(hash))
     }
 
     /// Return the utxo pointed to by `outpoint` if it exists in any chain.
     pub fn utxo(&self, outpoint: &transparent::OutPoint) -> Option<transparent::Output> {
-        self.mem.utxo(outpoint).or_else(|| self.sled.utxo(outpoint))
+        self.mem.utxo(outpoint).or_else(|| self.disk.utxo(outpoint))
     }
 
     /// Return an iterator over the relevant chain of the block identified by
@@ -312,7 +312,7 @@ impl Iter<'_> {
             IterState::Finished => unreachable!(),
         };
 
-        if let Some(block) = service.sled.block(hash_or_height) {
+        if let Some(block) = service.disk.block(hash_or_height) {
             let height = block
                 .coinbase_height()
                 .expect("valid blocks have a coinbase height");
@@ -402,7 +402,7 @@ impl Service<Request> for StateService {
                 let (rsp_tx, rsp_rx) = oneshot::channel();
 
                 self.pending_utxos.check_block(&block);
-                self.sled
+                self.disk
                     .queue_and_commit_finalized_blocks(QueuedBlock { block, rsp_tx });
 
                 async move {
@@ -449,7 +449,7 @@ impl Service<Request> for StateService {
 
 /// Initialize a state service from the provided [`Config`].
 ///
-/// Each `network` has its own separate sled database.
+/// Each `network` has its own separate on-disk database.
 ///
 /// To share access to the state, wrap the returned service in a `Buffer`. It's
 /// possible to construct multiple state services in the same application (as

From 4c9bb87df2e0f9b549bd5f302e9dad8e103f0a34 Mon Sep 17 00:00:00 2001
From: Jane Lusby <jlusby42@gmail.com>
Date: Wed, 18 Nov 2020 18:05:06 -0800
Subject: [PATCH 040/148] zebra-state: replace sled with rocksdb (#1325)

## Motivation

Prior to this PR we've been using `sled` as our database for storing persistent chain data on the disk between boots. We picked sled over rocksdb to minimize our c++ dependencies despite it being a less mature codebase. The theory was if it worked well enough we'd prefer to have a pure rust codebase, but if we ever ran into problems we knew we could easily swap it out with rocksdb.

Well, we ran into problems. Sled's memory usage was particularly high, and it seemed to be leaking memory. On top of all that, the performance for writes was pretty poor, causing us to become bottle-necked on sled instead of the network.

## Solution

This PR replaces `sled` with `rocksdb`. We've seen a 10x improvement in memory usage out of the box, no more leaking, and much better write performance. With this change writing chain data to disk is no longer a limiting factor in how quickly we can sync the chain.

The code in this pull request has:
  - [x] Documentation Comments
  - [x] Unit Tests and Property Tests

## Review

@hdevalence
---
 Cargo.lock                                    | 120 ++------
 Dockerfile                                    |   2 +-
 .../0004-asynchronous-script-verification.md  |  29 +-
 book/src/dev/rfcs/0005-state-updates.md       |  57 ++--
 zebra-state/Cargo.toml                        |   2 +-
 zebra-state/src/config.rs                     |  79 +++--
 zebra-state/src/constants.rs                  |   2 +-
 zebra-state/src/service/finalized_state.rs    | 271 ++++++++----------
 .../service/finalized_state/disk_format.rs    | 214 +++-----------
 zebrad/tests/acceptance.rs                    |   6 +-
 10 files changed, 280 insertions(+), 502 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 94d2590117a..d7b3f4d53fe 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -463,15 +463,6 @@ dependencies = [
  "bitflags",
 ]
 
-[[package]]
-name = "cloudabi"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4344512281c643ae7638bbabc3af17a11307803ec8f0fcad9fae512a8bf36467"
-dependencies = [
- "bitflags",
-]
-
 [[package]]
 name = "color-backtrace"
 version = "0.3.0"
@@ -939,16 +930,6 @@ dependencies = [
  "percent-encoding",
 ]
 
-[[package]]
-name = "fs2"
-version = "0.4.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"
-dependencies = [
- "libc",
- "winapi 0.3.9",
-]
-
 [[package]]
 name = "fuchsia-cprng"
 version = "0.1.1"
@@ -1072,15 +1053,6 @@ dependencies = [
  "slab",
 ]
 
-[[package]]
-name = "fxhash"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
-dependencies = [
- "byteorder",
-]
-
 [[package]]
 name = "generational-arena"
 version = "0.2.8"
@@ -1390,15 +1362,6 @@ dependencies = [
  "str_stack",
 ]
 
-[[package]]
-name = "instant"
-version = "0.1.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cb1fc4429a33e1f80d41dc9fea4d108a88bec1de8053878898ae448a0b52f613"
-dependencies = [
- "cfg-if 1.0.0",
-]
-
 [[package]]
 name = "iovec"
 version = "0.1.4"
@@ -1476,6 +1439,18 @@ dependencies = [
  "winapi 0.3.9",
 ]
 
+[[package]]
+name = "librocksdb-sys"
+version = "6.11.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eb5b56f651c204634b936be2f92dbb42c36867e00ff7fe2405591f3b9fa66f09"
+dependencies = [
+ "bindgen",
+ "cc",
+ "glob",
+ "libc",
+]
+
 [[package]]
 name = "linked-hash-map"
 version = "0.5.3"
@@ -1491,15 +1466,6 @@ dependencies = [
  "scopeguard",
 ]
 
-[[package]]
-name = "lock_api"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "28247cc5a5be2f05fbcd76dd0cf2c7d3b5400cb978a28042abcd4fa0b3f8261c"
-dependencies = [
- "scopeguard",
-]
-
 [[package]]
 name = "log"
 version = "0.4.11"
@@ -1648,7 +1614,7 @@ dependencies = [
  "metrics-observer-prometheus",
  "metrics-observer-yaml",
  "metrics-util",
- "parking_lot 0.10.2",
+ "parking_lot",
  "quanta",
 ]
 
@@ -1878,19 +1844,8 @@ version = "0.10.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d3a704eb390aafdc107b0e392f56a82b668e3a71366993b5340f5833fd62505e"
 dependencies = [
- "lock_api 0.3.4",
- "parking_lot_core 0.7.2",
-]
-
-[[package]]
-name = "parking_lot"
-version = "0.11.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4893845fa2ca272e647da5d0e46660a314ead9c2fdd9a883aabc32e481a8733"
-dependencies = [
- "instant",
- "lock_api 0.4.1",
- "parking_lot_core 0.8.0",
+ "lock_api",
+ "parking_lot_core",
 ]
 
 [[package]]
@@ -1900,22 +1855,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d58c7c768d4ba344e3e8d72518ac13e259d7c7ade24167003b8488e10b6740a3"
 dependencies = [
  "cfg-if 0.1.10",
- "cloudabi 0.0.3",
- "libc",
- "redox_syscall",
- "smallvec 1.4.2",
- "winapi 0.3.9",
-]
-
-[[package]]
-name = "parking_lot_core"
-version = "0.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c361aa727dd08437f2f1447be8b59a33b0edd15e0fcee698f935613d9efbca9b"
-dependencies = [
- "cfg-if 0.1.10",
- "cloudabi 0.1.0",
- "instant",
+ "cloudabi",
  "libc",
  "redox_syscall",
  "smallvec 1.4.2",
@@ -2367,6 +2307,16 @@ dependencies = [
  "opaque-debug 0.2.3",
 ]
 
+[[package]]
+name = "rocksdb"
+version = "0.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "23d83c02c429044d58474eaf5ae31e062d0de894e21125b47437ec0edc1397e6"
+dependencies = [
+ "libc",
+ "librocksdb-sys",
+]
+
 [[package]]
 name = "rust-argon2"
 version = "0.8.2"
@@ -2598,22 +2548,6 @@ version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8"
 
-[[package]]
-name = "sled"
-version = "0.34.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d0132f3e393bcb7390c60bb45769498cf4550bcb7a21d7f95c02b69f6362cdc"
-dependencies = [
- "crc32fast",
- "crossbeam-epoch 0.9.0",
- "crossbeam-utils 0.8.0",
- "fs2",
- "fxhash",
- "libc",
- "log",
- "parking_lot 0.11.0",
-]
-
 [[package]]
 name = "smallvec"
 version = "0.6.13"
@@ -3480,8 +3414,8 @@ dependencies = [
  "once_cell",
  "primitive-types",
  "proptest",
+ "rocksdb",
  "serde",
- "sled",
  "spandoc",
  "tempdir",
  "thiserror",
diff --git a/Dockerfile b/Dockerfile
index 12946c086cd..caf1698052d 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -3,7 +3,7 @@ FROM rust:buster as builder
 
 RUN apt-get update && \
 	apt-get install -y --no-install-recommends \
-	make cmake g++ gcc llvm libclang-dev
+	make cmake g++ gcc llvm libclang-dev clang
 
 RUN mkdir /zebra
 WORKDIR /zebra
diff --git a/book/src/dev/rfcs/0004-asynchronous-script-verification.md b/book/src/dev/rfcs/0004-asynchronous-script-verification.md
index ff8074761ea..9e42917ca81 100644
--- a/book/src/dev/rfcs/0004-asynchronous-script-verification.md
+++ b/book/src/dev/rfcs/0004-asynchronous-script-verification.md
@@ -97,11 +97,11 @@ with a timeout error. Currently, we outsource script verification to
 Implementing the state request correctly requires considering two sets of behaviors:
 
 1. behaviors related to the state's external API (a `Buffer`ed `tower::Service`);
-2. behaviors related to the state's internal implementation (using `sled`).
+2. behaviors related to the state's internal implementation (using `rocksdb`).
 
 Making this distinction helps us to ensure we don't accidentally leak
 "internal" behaviors into "external" behaviors, which would violate
-encapsulation and make it more difficult to replace `sled`.
+encapsulation and make it more difficult to replace `rocksdb`.
 
 In the first category, our state is presented to the rest of the application
 as a `Buffer`ed `tower::Service`. The `Buffer` wrapper allows shared access
@@ -116,19 +116,12 @@ This means that our external API ensures that the state service sees a
 linearized sequence of state requests, although the exact ordering is
 unpredictable when there are multiple senders making requests.
 
-In the second category, the Sled API presents itself synchronously, but
-database and tree handles are clonable and can be moved between threads. All
-that's required to process some request asynchronously is to clone the
-appropriate handle, move it into an async block, and make the call as part of
-the future. (We might want to use Tokio's blocking API for this, but that's a
-side detail).
-
-Because the state service has exclusive access to the sled database, and the
+Because the state service has exclusive access to the rocksdb database, and the
 state service sees a linearized sequence of state requests, we have an easy
-way to opt in to asynchronous database access. We can perform sled operations
+way to opt in to asynchronous database access. We can perform rocksdb operations
 synchronously in the `Service::call`, waiting for them to complete, and be
-sure that all future requests will see the resulting sled state. Or, we can
-perform sled operations asynchronously in the future returned by
+sure that all future requests will see the resulting rocksdb state. Or, we can
+perform rocksdb operations asynchronously in the future returned by
 `Service::call`.
 
 If we perform all *writes* synchronously and allow reads to be either
@@ -139,7 +132,7 @@ time the request was processed, or a later state.
 Now, returning to the UTXO lookup problem, we can map out the possible states
 with this restriction in mind. This description assumes that UTXO storage is
 split into disjoint sets, one in-memory (e.g., blocks after the reorg limit)
-and the other in sled (e.g., blocks after the reorg limit). The details of
+and the other in rocksdb (e.g., blocks after the reorg limit). The details of
 this storage are not important for this design, only that the two sets are
 disjoint.
 
@@ -147,14 +140,14 @@ When the state service processes a `Request::AwaitUtxo(OutPoint)` referencing
 some UTXO `u`, there are three disjoint possibilities:
 
 1. `u` is already contained in an in-memory block storage;
-2. `u` is already contained in the sled UTXO set;
+2. `u` is already contained in the rocksdb UTXO set;
 3. `u` is not yet known to the state service.
 
 In case 3, we need to queue `u` and scan all *future* blocks to see whether
 they contain `u`. However, if we have a mechanism to queue `u`, we can
 perform check 2 asynchronously, because restricting to synchronous writes
 means that any async read will return the current or later state. If `u` was
-in the sled UTXO set when the request was processed, the only way that an
+in the rocksdb UTXO set when the request was processed, the only way that an
 async read would not return `u` is if the UTXO were spent, in which case the
 service is not required to return a response.
 
@@ -184,12 +177,12 @@ The state service should maintain an `Arc<Mutex<PendingUtxos>>`, used as follows
 
 1. In `Service::call(Request::AwaitUtxo(u))`, the service should:
   - call `PendingUtxos::queue(u)` to get a future `f` to return to the caller;
-  spawn a task that does a sled lookup for `u`, calling `PendingUtxos::respond(u, output)` if present;
+  spawn a task that does a rocksdb lookup for `u`, calling `PendingUtxos::respond(u, output)` if present;
   - check the in-memory storage for `u`, calling `PendingUtxos::respond(u, output)` if present;
   - return `f` to the caller (it may already be ready).
   The common case is that `u` references an old UTXO, so spawning the lookup
   task first means that we don't wait to check in-memory storage for `u`
-  before starting the sled lookup.
+  before starting the rocksdb lookup.
 
 2. In `Service::call(Request::CommitBlock(block, ..))`, the service should:
   - call `PendingUtxos::check_block(block.as_ref())`;
diff --git a/book/src/dev/rfcs/0005-state-updates.md b/book/src/dev/rfcs/0005-state-updates.md
index 3d634ab33a4..b685af0eae0 100644
--- a/book/src/dev/rfcs/0005-state-updates.md
+++ b/book/src/dev/rfcs/0005-state-updates.md
@@ -156,7 +156,7 @@ state data at the finality boundary provided by the reorg limit.
 
 State data from blocks *above* the reorg limit (*non-finalized state*) is
 stored in-memory and handles multiple chains. State data from blocks *below*
-the reorg limit (*finalized state*) is stored persistently using `sled` and
+the reorg limit (*finalized state*) is stored persistently using `rocksdb` and
 only tracks a single chain. This allows a simplification of our state
 handling, because only finalized data is persistent and the logic for
 finalized data handles less invariants.
@@ -169,7 +169,7 @@ Another downside of this design is that we do not achieve exactly the same
 behavior as `zcashd` in the event of a 51% attack: `zcashd` limits *each* chain
 reorganization to 100 blocks, but permits multiple reorgs, while Zebra limits
 *all* chain reorgs to 100 blocks. In the event of a successful 51% attack on
-Zcash, this could be resolved by wiping the Sled state and re-syncing the new
+Zcash, this could be resolved by wiping the rocksdb state and re-syncing the new
 chain, but in this scenario there are worse problems.
 
 ## Service Interface
@@ -180,11 +180,11 @@ Determining what guarantees the state service can and should provide to the
 rest of the application requires considering two sets of behaviors:
 
 1. behaviors related to the state's external API (a `Buffer`ed `tower::Service`);
-2. behaviors related to the state's internal implementation (using `sled`).
+2. behaviors related to the state's internal implementation (using `rocksdb`).
 
 Making this distinction helps us to ensure we don't accidentally leak
 "internal" behaviors into "external" behaviors, which would violate
-encapsulation and make it more difficult to replace `sled`.
+encapsulation and make it more difficult to replace `rocksdb`.
 
 In the first category, our state is presented to the rest of the application
 as a `Buffer`ed `tower::Service`. The `Buffer` wrapper allows shared access
@@ -199,19 +199,12 @@ This means that our external API ensures that the state service sees a
 linearized sequence of state requests, although the exact ordering is
 unpredictable when there are multiple senders making requests.
 
-In the second category, the Sled API presents itself synchronously, but
-database and tree handles are cloneable and can be moved between threads. All
-that's required to process some request asynchronously is to clone the
-appropriate handle, move it into an async block, and make the call as part of
-the future. (We might want to use Tokio's blocking API for this, but this is
-an implementation detail).
-
-Because the state service has exclusive access to the sled database, and the
+Because the state service has exclusive access to the rocksdb database, and the
 state service sees a linearized sequence of state requests, we have an easy
-way to opt in to asynchronous database access. We can perform sled operations
+way to opt in to asynchronous database access. We can perform rocksdb operations
 synchronously in the `Service::call`, waiting for them to complete, and be
-sure that all future requests will see the resulting sled state. Or, we can
-perform sled operations asynchronously in the future returned by
+sure that all future requests will see the resulting rocksdb state. Or, we can
+perform rocksdb operations asynchronously in the future returned by
 `Service::call`.
 
 If we perform all *writes* synchronously and allow reads to be either
@@ -221,10 +214,10 @@ time the request was processed, or a later state.
 
 ### Summary
 
-- **Sled reads** may be done synchronously (in `call`) or asynchronously (in
+- **rocksdb reads** may be done synchronously (in `call`) or asynchronously (in
   the `Future`), depending on the context;
 
-- **Sled writes** must be done synchronously (in `call`)
+- **rocksdb writes** must be done synchronously (in `call`)
 
 ## In-memory data structures
 [in-memory]: #in-memory
@@ -580,22 +573,22 @@ New `non-finalized` blocks are commited as follows:
     - Remove the lowest height block from the non-finalized state with
       `self.mem.finalize();`
     - Commit that block to the finalized state with
-      `self.sled.commit_finalized_direct(finalized);`
+      `self.disk.commit_finalized_direct(finalized);`
 
 8. Prune orphaned blocks from `self.queued_blocks` with
    `self.queued_blocks.prune_by_height(finalized_height);`
 
 9. Return the receiver for the block's channel
 
-## Sled data structures
-[sled]: #sled
+## rocksdb data structures
+[rocksdb]: #rocksdb
 
-Sled provides a persistent, thread-safe `BTreeMap<&[u8], &[u8]>`. Each map is
+rocksdb provides a persistent, thread-safe `BTreeMap<&[u8], &[u8]>`. Each map is
 a distinct "tree". Keys are sorted using lex order on byte strings, so
 integer values should be stored using big-endian encoding (so that the lex
 order on byte strings is the numeric ordering).
 
-We use the following Sled trees:
+We use the following rocksdb column families:
 
 | Tree                 |                  Keys |                              Values |
 |----------------------|-----------------------|-------------------------------------|
@@ -613,16 +606,16 @@ Zcash structures are encoded using `ZcashSerialize`/`ZcashDeserialize`.
 
 **Note:** We do not store the cumulative work for the finalized chain, because the finalized work is equal for all non-finalized chains. So the additional non-finalized work can be used to calculate the relative chain order, and choose the best chain.
 
-### Notes on Sled trees
+### Notes on rocksdb column families
 
-- The `hash_by_height` and `height_by_hash` trees provide a bijection between
-  block heights and block hashes.  (Since the Sled state only stores finalized
+- The `hash_by_height` and `height_by_hash` column families provide a bijection between
+  block heights and block hashes.  (Since the rocksdb state only stores finalized
   state, they are actually a bijection).
 
-- The `block_by_height` tree provides a bijection between block heights and block
-  data. There is no corresponding `height_by_block` tree: instead, hash the block,
-  and use `height_by_hash`. (Since the Sled state only stores finalized state,
-  they are actually a bijection).
+- The `block_by_height` column family provides a bijection between block
+  heights and block data. There is no corresponding `height_by_block` column
+  family: instead, hash the block, and use `height_by_hash`. (Since the
+  rocksdb state only stores finalized state, they are actually a bijection).
 
 - Blocks are stored by height, not by hash.  This has the downside that looking
   up a block by hash requires an extra level of indirection.  The upside is
@@ -630,7 +623,7 @@ Zcash structures are encoded using `ZcashSerialize`/`ZcashDeserialize`.
   common access patterns, such as helping a client sync the chain or doing
   analysis, access blocks in (potentially sparse) height order.  In addition,
   the fact that we commit blocks in order means we're writing only to the end
-  of the Sled tree, which may help save space.
+  of the rocksdb column family, which may help save space.
 
 - Transaction references are stored as a `(height, index)` pair referencing the
   height of the transaction's parent block and the transaction's index in that
@@ -645,7 +638,7 @@ commit any queued children.  (Although the checkpointer generates verified
 blocks in order when it completes a checkpoint, the blocks are committed in the
 response futures, so they may arrive out of order).
 
-Committing a block to the sled state should be implemented as a wrapper around
+Committing a block to the rocksdb state should be implemented as a wrapper around
 a function also called by [`Request::CommitBlock`](#request-commit-block),
 which should:
 
@@ -754,7 +747,7 @@ CommitFinalizedBlock {
 }
 ```
 
-Commits a finalized block to the sled state, skipping contextual validation.
+Commits a finalized block to the rocksdb state, skipping contextual validation.
 This is exposed for use in checkpointing, which produces in-order finalized
 blocks. Returns `Response::Added(block::Hash)` with the hash of the
 committed block if successful.
diff --git a/zebra-state/Cargo.toml b/zebra-state/Cargo.toml
index ab5bf229521..4fb34e566ea 100644
--- a/zebra-state/Cargo.toml
+++ b/zebra-state/Cargo.toml
@@ -15,7 +15,6 @@ dirs = "3.0.1"
 hex = "0.4.2"
 lazy_static = "1.4.0"
 serde = { version = "1", features = ["serde_derive"] }
-sled = "0.34.5"
 
 futures = "0.3.7"
 metrics = "0.12"
@@ -25,6 +24,7 @@ tracing-error = "0.1.2"
 thiserror = "1.0.22"
 tokio = { version = "0.2.22", features = ["sync"] }
 displaydoc = "0.1.7"
+rocksdb = "0.15.0"
 
 [dev-dependencies]
 zebra-chain = { path = "../zebra-chain", features = ["proptest-impl"] }
diff --git a/zebra-state/src/config.rs b/zebra-state/src/config.rs
index f0aed0a14fe..720ffaa8e19 100644
--- a/zebra-state/src/config.rs
+++ b/zebra-state/src/config.rs
@@ -27,18 +27,6 @@ pub struct Config {
     /// | Other   | `std::env::current_dir()/cache`                 |                                    |
     pub cache_dir: PathBuf,
 
-    /// Controls the size of the database cache, in bytes.
-    ///
-    /// This corresponds to `sled`'s [`cache_capacity`][cc] parameter.
-    /// Note that the behavior of this parameter is [somewhat
-    /// unintuitive][gh], measuring the on-disk size of the cached data,
-    /// not the in-memory size, which may be much larger, especially for
-    /// smaller keys and values.
-    ///
-    /// [cc]: https://docs.rs/sled/0.34.4/sled/struct.Config.html#method.cache_capacity
-    /// [gh]: https://github.com/spacejam/sled/issues/986#issuecomment-592950100
-    pub memory_cache_bytes: u64,
-
     /// Whether to use an ephemeral database.
     ///
     /// Ephemeral databases are stored in memory on Linux, and in a temporary directory on other OSes.
@@ -54,30 +42,68 @@ pub struct Config {
     pub debug_stop_at_height: Option<u32>,
 }
 
+fn gen_temp_path() -> PathBuf {
+    use std::sync::atomic::{AtomicUsize, Ordering};
+    use std::time::SystemTime;
+
+    static SALT_COUNTER: AtomicUsize = AtomicUsize::new(0);
+
+    let seed = SALT_COUNTER.fetch_add(1, Ordering::SeqCst) as u128;
+
+    let now = SystemTime::now()
+        .duration_since(SystemTime::UNIX_EPOCH)
+        .unwrap()
+        .as_nanos()
+        << 48;
+
+    #[cfg(not(miri))]
+    let pid = u128::from(std::process::id());
+
+    #[cfg(miri)]
+    let pid = 0;
+
+    let salt = (pid << 16) + now + seed;
+
+    if cfg!(target_os = "linux") {
+        // use shared memory for temporary linux files
+        format!("/dev/shm/pagecache.tmp.{}", salt).into()
+    } else {
+        std::env::temp_dir().join(format!("pagecache.tmp.{}", salt))
+    }
+}
+
 impl Config {
-    /// Generate the appropriate `sled::Config` for `network`, based on the
-    /// provided `zebra_state::Config`.
-    pub(crate) fn sled_config(&self, network: Network) -> sled::Config {
+    pub(crate) fn open_db(&self, network: Network) -> rocksdb::DB {
         let net_dir = match network {
             Network::Mainnet => "mainnet",
             Network::Testnet => "testnet",
         };
 
-        let config = sled::Config::default()
-            .cache_capacity(self.memory_cache_bytes)
-            .mode(sled::Mode::LowSpace);
+        let mut opts = rocksdb::Options::default();
 
-        if self.ephemeral {
-            config.temporary(self.ephemeral)
+        let cfs = vec![
+            rocksdb::ColumnFamilyDescriptor::new("hash_by_height", opts.clone()),
+            rocksdb::ColumnFamilyDescriptor::new("height_by_hash", opts.clone()),
+            rocksdb::ColumnFamilyDescriptor::new("block_by_height", opts.clone()),
+            rocksdb::ColumnFamilyDescriptor::new("tx_by_hash", opts.clone()),
+            rocksdb::ColumnFamilyDescriptor::new("utxo_by_outpoint", opts.clone()),
+            rocksdb::ColumnFamilyDescriptor::new("sprout_nullifiers", opts.clone()),
+            rocksdb::ColumnFamilyDescriptor::new("sapling_nullifiers", opts.clone()),
+        ];
+
+        opts.create_if_missing(true);
+        opts.create_missing_column_families(true);
+
+        let path = if self.ephemeral {
+            gen_temp_path()
         } else {
-            let path = self
-                .cache_dir
+            self.cache_dir
                 .join("state")
-                .join(format!("v{}", crate::constants::SLED_FORMAT_VERSION))
-                .join(net_dir);
+                .join(format!("v{}", crate::constants::DATABASE_FORMAT_VERSION))
+                .join(net_dir)
+        };
 
-            config.path(path)
-        }
+        rocksdb::DB::open_cf_descriptors(&opts, path, cfs).unwrap()
     }
 
     /// Construct a config for an ephemeral in memory database
@@ -96,7 +122,6 @@ impl Default for Config {
 
         Self {
             cache_dir,
-            memory_cache_bytes: 50_000_000,
             ephemeral: false,
             debug_stop_at_height: None,
         }
diff --git a/zebra-state/src/constants.rs b/zebra-state/src/constants.rs
index 960db6737cf..29e0d02491e 100644
--- a/zebra-state/src/constants.rs
+++ b/zebra-state/src/constants.rs
@@ -11,4 +11,4 @@ pub const MIN_TRASPARENT_COINBASE_MATURITY: u32 = 100;
 /// coinbase transactions.
 pub const MAX_BLOCK_REORG_HEIGHT: u32 = MIN_TRASPARENT_COINBASE_MATURITY - 1;
 
-pub const SLED_FORMAT_VERSION: u32 = 1;
+pub const DATABASE_FORMAT_VERSION: u32 = 2;
diff --git a/zebra-state/src/service/finalized_state.rs b/zebra-state/src/service/finalized_state.rs
index a4158d3dcc3..2cd3c8e749c 100644
--- a/zebra-state/src/service/finalized_state.rs
+++ b/zebra-state/src/service/finalized_state.rs
@@ -1,4 +1,4 @@
-//! The primary implementation of the `zebra_state::Service` built upon sled
+//! The primary implementation of the `zebra_state::Service` built upon rocksdb
 
 mod disk_format;
 
@@ -17,55 +17,27 @@ use self::disk_format::{DiskDeserialize, DiskSerialize, FromDisk, IntoDisk, Tran
 
 use super::QueuedBlock;
 
-/// The finalized part of the chain state, stored in sled.
-///
-/// This structure has two categories of methods:
-///
-/// - *synchronous* methods that perform writes to the sled state;
-/// - *asynchronous* methods that perform reads.
-///
-/// For more on this distinction, see RFC5. The synchronous methods are
-/// implemented as ordinary methods on the [`FinalizedState`]. The asynchronous
-/// methods are not implemented using `async fn`, but using normal methods that
-/// return `impl Future<Output = ...>`. This allows them to move data (e.g.,
-/// clones of handles for [`sled::Tree`]s) into the futures they return.
-///
-/// This means that the returned futures have a `'static` lifetime and don't
-/// borrow any resources from the [`FinalizedState`], and the actual database work is
-/// performed asynchronously when the returned future is polled, not while it is
-/// created.  This is analogous to the way [`tower::Service::call`] works.
+/// The finalized part of the chain state, stored in the db.
 pub struct FinalizedState {
     /// Queued blocks that arrived out of order, indexed by their parent block hash.
     queued_by_prev_hash: HashMap<block::Hash, QueuedBlock>,
     max_queued_height: i64,
 
-    hash_by_height: sled::Tree,
-    height_by_hash: sled::Tree,
-    block_by_height: sled::Tree,
-    tx_by_hash: sled::Tree,
-    utxo_by_outpoint: sled::Tree,
-    sprout_nullifiers: sled::Tree,
-    sapling_nullifiers: sled::Tree,
-    // sprout_anchors: sled::Tree,
-    // sapling_anchors: sled::Tree,
+    db: rocksdb::DB,
+    ephemeral: bool,
     /// Commit blocks to the finalized state up to this height, then exit Zebra.
     debug_stop_at_height: Option<block::Height>,
 }
 
 impl FinalizedState {
     pub fn new(config: &Config, network: Network) -> Self {
-        let db = config.sled_config(network).open().unwrap();
+        let db = config.open_db(network);
 
         let new_state = Self {
             queued_by_prev_hash: HashMap::new(),
             max_queued_height: -1,
-            hash_by_height: db.open_tree(b"hash_by_height").unwrap(),
-            height_by_hash: db.open_tree(b"height_by_hash").unwrap(),
-            block_by_height: db.open_tree(b"block_by_height").unwrap(),
-            tx_by_hash: db.open_tree(b"tx_by_hash").unwrap(),
-            utxo_by_outpoint: db.open_tree(b"utxo_by_outpoint").unwrap(),
-            sprout_nullifiers: db.open_tree(b"sprout_nullifiers").unwrap(),
-            sapling_nullifiers: db.open_tree(b"sapling_nullifiers").unwrap(),
+            db,
+            ephemeral: config.ephemeral,
             debug_stop_at_height: config.debug_stop_at_height.map(block::Height),
         };
 
@@ -101,30 +73,8 @@ impl FinalizedState {
         new_state
     }
 
-    /// Synchronously flushes all dirty IO buffers and calls fsync.
-    ///
-    /// Returns the number of bytes flushed during this call.
-    /// See sled's `Tree.flush` for more details.
-    pub fn flush(&self) -> sled::Result<usize> {
-        let mut total_flushed = 0;
-
-        total_flushed += self.hash_by_height.flush()?;
-        total_flushed += self.height_by_hash.flush()?;
-        total_flushed += self.block_by_height.flush()?;
-        total_flushed += self.tx_by_hash.flush()?;
-        total_flushed += self.utxo_by_outpoint.flush()?;
-        total_flushed += self.sprout_nullifiers.flush()?;
-        total_flushed += self.sapling_nullifiers.flush()?;
-
-        Ok(total_flushed)
-    }
-
-    /// If `block_height` is greater than or equal to the configured stop height,
-    /// stop the process.
-    ///
-    /// Flushes sled trees before exiting.
-    ///
-    /// `called_from` and `block_hash` are used for assertions and logging.
+    /// Stop the process if `block_height` is greater than or equal to the
+    /// configured stop height.
     fn is_at_stop_height(&self, block_height: block::Height) -> bool {
         let debug_stop_at_height = match self.debug_stop_at_height {
             Some(debug_stop_at_height) => debug_stop_at_height,
@@ -181,10 +131,16 @@ impl FinalizedState {
         self.tip().map(|(height, _)| height)
     }
 
+    fn is_empty(&self, cf: &rocksdb::ColumnFamily) -> bool {
+        // use iterator to check if it's empty
+        !self
+            .db
+            .iterator_cf(cf, rocksdb::IteratorMode::Start)
+            .valid()
+    }
+
     /// Immediately commit `block` to the finalized state.
     pub fn commit_finalized_direct(&mut self, block: Arc<Block>) -> Result<block::Hash, BoxError> {
-        use sled::Transactional;
-
         let height = block
             .coinbase_height()
             .expect("finalized blocks are valid and have a coinbase height");
@@ -192,8 +148,16 @@ impl FinalizedState {
 
         block_precommit_metrics(&hash, height, &block);
 
+        let hash_by_height = self.db.cf_handle("hash_by_height").unwrap();
+        let height_by_hash = self.db.cf_handle("height_by_hash").unwrap();
+        let block_by_height = self.db.cf_handle("block_by_height").unwrap();
+        let tx_by_hash = self.db.cf_handle("tx_by_hash").unwrap();
+        let utxo_by_outpoint = self.db.cf_handle("utxo_by_outpoint").unwrap();
+        let sprout_nullifiers = self.db.cf_handle("sprout_nullifiers").unwrap();
+        let sapling_nullifiers = self.db.cf_handle("sapling_nullifiers").unwrap();
+
         // Assert that callers (including unit tests) get the chain order correct
-        if self.block_by_height.is_empty() {
+        if self.is_empty(hash_by_height) {
             assert_eq!(
                 block::Hash([0; 32]),
                 block.header.previous_block_hash,
@@ -220,94 +184,73 @@ impl FinalizedState {
             );
         }
 
-        let result = (
-            &self.hash_by_height,
-            &self.height_by_hash,
-            &self.block_by_height,
-            &self.utxo_by_outpoint,
-            &self.tx_by_hash,
-            &self.sprout_nullifiers,
-            &self.sapling_nullifiers,
-        )
-            .transaction(
-                move |(
-                    hash_by_height,
-                    height_by_hash,
-                    block_by_height,
-                    utxo_by_outpoint,
-                    tx_by_hash,
-                    sprout_nullifiers,
-                    sapling_nullifiers,
-                )| {
-                    // Index the block
-                    hash_by_height.zs_insert(height, hash)?;
-                    height_by_hash.zs_insert(hash, height)?;
-                    block_by_height.zs_insert(height, &block)?;
-
-                    // TODO: sprout and sapling anchors (per block)
-
-                    // Consensus-critical bug in zcashd: transactions in the
-                    // genesis block are ignored.
-                    if block.header.previous_block_hash == block::Hash([0; 32]) {
-                        return Ok(hash);
-                    }
+        // We use a closure so we can use an early return for control flow in
+        // the genesis case
+        let prepare_commit = || -> rocksdb::WriteBatch {
+            let mut batch = rocksdb::WriteBatch::default();
 
-                    // Index each transaction
-                    for (transaction_index, transaction) in block.transactions.iter().enumerate() {
-                        let transaction_hash = transaction.hash();
-                        let transaction_location = TransactionLocation {
-                            height,
-                            index: transaction_index
-                                .try_into()
-                                .expect("no more than 4 billion transactions per block"),
-                        };
-                        tx_by_hash.zs_insert(transaction_hash, transaction_location)?;
-
-                        // Mark all transparent inputs as spent
-                        for input in transaction.inputs() {
-                            match input {
-                                transparent::Input::PrevOut { outpoint, .. } => {
-                                    utxo_by_outpoint.remove(outpoint.as_bytes())?;
-                                }
-                                // Coinbase inputs represent new coins,
-                                // so there are no UTXOs to mark as spent.
-                                transparent::Input::Coinbase { .. } => {}
-                            }
-                        }
+            // Index the block
+            batch.zs_insert(hash_by_height, height, hash);
+            batch.zs_insert(height_by_hash, hash, height);
+            batch.zs_insert(block_by_height, height, &block);
 
-                        // Index all new transparent outputs
-                        for (index, output) in transaction.outputs().iter().enumerate() {
-                            let outpoint = transparent::OutPoint {
-                                hash: transaction_hash,
-                                index: index as _,
-                            };
-                            utxo_by_outpoint.zs_insert(outpoint, output)?;
-                        }
+            // TODO: sprout and sapling anchors (per block)
 
-                        // Mark sprout and sapling nullifiers as spent
-                        for sprout_nullifier in transaction.sprout_nullifiers() {
-                            sprout_nullifiers.zs_insert(sprout_nullifier, ())?;
-                        }
-                        for sapling_nullifier in transaction.sapling_nullifiers() {
-                            sapling_nullifiers.zs_insert(sapling_nullifier, ())?;
+            // Consensus-critical bug in zcashd: transactions in the
+            // genesis block are ignored.
+            if block.header.previous_block_hash == block::Hash([0; 32]) {
+                return batch;
+            }
+
+            // Index each transaction
+            for (transaction_index, transaction) in block.transactions.iter().enumerate() {
+                let transaction_hash = transaction.hash();
+                let transaction_location = TransactionLocation {
+                    height,
+                    index: transaction_index
+                        .try_into()
+                        .expect("no more than 4 billion transactions per block"),
+                };
+                batch.zs_insert(tx_by_hash, transaction_hash, transaction_location);
+
+                // Mark all transparent inputs as spent
+                for input in transaction.inputs() {
+                    match input {
+                        transparent::Input::PrevOut { outpoint, .. } => {
+                            batch.delete_cf(utxo_by_outpoint, outpoint.as_bytes());
                         }
+                        // Coinbase inputs represent new coins,
+                        // so there are no UTXOs to mark as spent.
+                        transparent::Input::Coinbase { .. } => {}
                     }
+                }
 
-                    // for some reason type inference fails here
-                    Ok::<_, sled::transaction::ConflictableTransactionError>(hash)
-                },
-            );
+                // Index all new transparent outputs
+                for (index, output) in transaction.outputs().iter().enumerate() {
+                    let outpoint = transparent::OutPoint {
+                        hash: transaction_hash,
+                        index: index as _,
+                    };
+                    batch.zs_insert(utxo_by_outpoint, outpoint, output);
+                }
 
-        if result.is_ok() && self.is_at_stop_height(height) {
-            if let Err(e) = self.flush() {
-                tracing::error!(
-                    ?e,
-                    ?height,
-                    ?hash,
-                    "error flushing sled state before stopping"
-                );
+                // Mark sprout and sapling nullifiers as spent
+                for sprout_nullifier in transaction.sprout_nullifiers() {
+                    batch.zs_insert(sprout_nullifiers, sprout_nullifier, ());
+                }
+                for sapling_nullifier in transaction.sapling_nullifiers() {
+                    batch.zs_insert(sapling_nullifiers, sapling_nullifier, ());
+                }
             }
 
+            batch
+        };
+
+        let batch = prepare_commit();
+
+        let result = self.db.write(batch).map(|()| hash);
+
+        if result.is_ok() && self.is_at_stop_height(height) {
             tracing::info!(?height, ?hash, "stopping at configured height");
 
             std::process::exit(0);
@@ -330,15 +273,13 @@ impl FinalizedState {
 
     /// Returns the tip height and hash if there is one.
     pub fn tip(&self) -> Option<(block::Height, block::Hash)> {
-        self.hash_by_height
-            .iter()
-            .rev()
+        let hash_by_height = self.db.cf_handle("hash_by_height").unwrap();
+        self.db
+            .iterator_cf(hash_by_height, rocksdb::IteratorMode::End)
             .next()
-            .transpose()
-            .expect("expected that sled errors would not occur")
             .map(|(height_bytes, hash_bytes)| {
-                let height = block::Height::from_ivec(height_bytes);
-                let hash = block::Hash::from_ivec(hash_bytes);
+                let height = block::Height::from_bytes(height_bytes);
+                let hash = block::Hash::from_bytes(hash_bytes);
 
                 (height, hash)
             })
@@ -346,31 +287,37 @@ impl FinalizedState {
 
     /// Returns the height of the given block if it exists.
     pub fn height(&self, hash: block::Hash) -> Option<block::Height> {
-        self.height_by_hash.zs_get(&hash)
+        let height_by_hash = self.db.cf_handle("height_by_hash").unwrap();
+        self.db.zs_get(&height_by_hash, &hash)
     }
 
     /// Returns the given block if it exists.
     pub fn block(&self, hash_or_height: HashOrHeight) -> Option<Arc<Block>> {
-        let height = hash_or_height.height_or_else(|hash| self.height_by_hash.zs_get(&hash))?;
+        let height_by_hash = self.db.cf_handle("height_by_hash").unwrap();
+        let block_by_height = self.db.cf_handle("block_by_height").unwrap();
+        let height = hash_or_height.height_or_else(|hash| self.db.zs_get(height_by_hash, &hash))?;
 
-        self.block_by_height.zs_get(&height)
+        self.db.zs_get(block_by_height, &height)
     }
 
     /// Returns the `transparent::Output` pointed to by the given
     /// `transparent::OutPoint` if it is present.
     pub fn utxo(&self, outpoint: &transparent::OutPoint) -> Option<transparent::Output> {
-        self.utxo_by_outpoint.zs_get(outpoint)
+        let utxo_by_outpoint = self.db.cf_handle("utxo_by_outpoint").unwrap();
+        self.db.zs_get(utxo_by_outpoint, outpoint)
     }
 
     /// Returns the finalized hash for a given `block::Height` if it is present.
     pub fn hash(&self, height: block::Height) -> Option<block::Hash> {
-        self.hash_by_height.zs_get(&height)
+        let hash_by_height = self.db.cf_handle("hash_by_height").unwrap();
+        self.db.zs_get(hash_by_height, &height)
     }
 
     /// Returns the given transaction if it exists.
     pub fn transaction(&self, hash: transaction::Hash) -> Option<Arc<Transaction>> {
-        self.tx_by_hash
-            .zs_get(&hash)
+        let tx_by_hash = self.db.cf_handle("tx_by_hash").unwrap();
+        self.db
+            .zs_get(tx_by_hash, &hash)
             .map(|TransactionLocation { index, height }| {
                 let block = self
                     .block(height.into())
@@ -381,6 +328,20 @@ impl FinalizedState {
     }
 }
 
+// Drop isn't guaranteed to run, such as when we panic, or if someone stored
+// their FinalizedState in a static, but it should be fine if we don't clean
+// this up since the files are placed in the os temp dir and should be cleaned
+// up automatically eventually.
+impl Drop for FinalizedState {
+    fn drop(&mut self) {
+        if self.ephemeral {
+            let path = self.db.path();
+            tracing::debug!("removing temporary database files {:?}", path);
+            let _res = std::fs::remove_dir_all(path);
+        }
+    }
+}
+
 fn block_precommit_metrics(hash: &block::Hash, height: block::Height, block: &Block) {
     let transaction_count = block.transactions.len();
     let transparent_prevout_count = block
diff --git a/zebra-state/src/service/finalized_state/disk_format.rs b/zebra-state/src/service/finalized_state/disk_format.rs
index 77e9a4f33b5..f3d150f8135 100644
--- a/zebra-state/src/service/finalized_state/disk_format.rs
+++ b/zebra-state/src/service/finalized_state/disk_format.rs
@@ -1,4 +1,4 @@
-//! Module defining exactly how to move types in and out of sled
+//! Module defining exactly how to move types in and out of rocksdb
 use std::{convert::TryInto, fmt::Debug, sync::Arc};
 
 use zebra_chain::{
@@ -25,9 +25,6 @@ pub trait IntoDisk {
     // function to convert the current type to its disk format in `zs_get()`
     // without necessarily allocating a new IVec
     fn as_bytes(&self) -> Self::Bytes;
-
-    // function to convert the current type into its disk format
-    fn into_ivec(&self) -> sled::IVec;
 }
 
 impl<'a, T> IntoDisk for &'a T
@@ -39,10 +36,6 @@ where
     fn as_bytes(&self) -> Self::Bytes {
         T::as_bytes(*self)
     }
-
-    fn into_ivec(&self) -> sled::IVec {
-        T::into_ivec(*self)
-    }
 }
 
 impl<T> IntoDisk for Arc<T>
@@ -54,10 +47,6 @@ where
     fn as_bytes(&self) -> Self::Bytes {
         T::as_bytes(&*self)
     }
-
-    fn into_ivec(&self) -> sled::IVec {
-        T::into_ivec(&*self)
-    }
 }
 
 /// Helper type for retrieving types from the disk with the correct format.
@@ -69,15 +58,15 @@ pub trait FromDisk: Sized {
     /// # Panics
     ///
     /// - if the input data doesn't deserialize correctly
-    fn from_ivec(bytes: sled::IVec) -> Self;
+    fn from_bytes(bytes: impl AsRef<[u8]>) -> Self;
 }
 
 impl<T> FromDisk for Arc<T>
 where
     T: FromDisk,
 {
-    fn from_ivec(bytes: sled::IVec) -> Self {
-        Arc::new(T::from_ivec(bytes))
+    fn from_bytes(bytes: impl AsRef<[u8]>) -> Self {
+        Arc::new(T::from_bytes(bytes))
     }
 }
 
@@ -88,16 +77,12 @@ impl IntoDisk for Block {
         self.zcash_serialize_to_vec()
             .expect("serialization to vec doesn't fail")
     }
-
-    fn into_ivec(&self) -> sled::IVec {
-        self.as_bytes().into()
-    }
 }
 
 impl FromDisk for Block {
-    fn from_ivec(bytes: sled::IVec) -> Self {
+    fn from_bytes(bytes: impl AsRef<[u8]>) -> Self {
         Block::zcash_deserialize(bytes.as_ref())
-            .expect("deserialization format should match the serialization format used by IntoSled")
+            .expect("deserialization format should match the serialization format used by IntoDisk")
     }
 }
 
@@ -115,24 +100,21 @@ impl IntoDisk for TransactionLocation {
 
         bytes
     }
-
-    fn into_ivec(&self) -> sled::IVec {
-        self.as_bytes().as_ref().into()
-    }
 }
 
 impl FromDisk for TransactionLocation {
-    fn from_ivec(sled_bytes: sled::IVec) -> Self {
+    fn from_bytes(disk_bytes: impl AsRef<[u8]>) -> Self {
+        let disk_bytes = disk_bytes.as_ref();
         let height = {
             let mut bytes = [0; 4];
-            bytes.copy_from_slice(&sled_bytes[0..4]);
+            bytes.copy_from_slice(&disk_bytes[0..4]);
             let height = u32::from_be_bytes(bytes);
             block::Height(height)
         };
 
         let index = {
             let mut bytes = [0; 4];
-            bytes.copy_from_slice(&sled_bytes[4..8]);
+            bytes.copy_from_slice(&disk_bytes[4..8]);
             u32::from_be_bytes(bytes)
         };
 
@@ -146,10 +128,6 @@ impl IntoDisk for transaction::Hash {
     fn as_bytes(&self) -> Self::Bytes {
         self.0
     }
-
-    fn into_ivec(&self) -> sled::IVec {
-        self.as_bytes().as_ref().into()
-    }
 }
 
 impl IntoDisk for block::Hash {
@@ -158,13 +136,10 @@ impl IntoDisk for block::Hash {
     fn as_bytes(&self) -> Self::Bytes {
         self.0
     }
-    fn into_ivec(&self) -> sled::IVec {
-        self.as_bytes().as_ref().into()
-    }
 }
 
 impl FromDisk for block::Hash {
-    fn from_ivec(bytes: sled::IVec) -> Self {
+    fn from_bytes(bytes: impl AsRef<[u8]>) -> Self {
         let array = bytes.as_ref().try_into().unwrap();
         Self(array)
     }
@@ -176,10 +151,6 @@ impl IntoDisk for sprout::Nullifier {
     fn as_bytes(&self) -> Self::Bytes {
         self.0
     }
-
-    fn into_ivec(&self) -> sled::IVec {
-        self.as_bytes().as_ref().into()
-    }
 }
 
 impl IntoDisk for sapling::Nullifier {
@@ -188,10 +159,6 @@ impl IntoDisk for sapling::Nullifier {
     fn as_bytes(&self) -> Self::Bytes {
         self.0
     }
-
-    fn into_ivec(&self) -> sled::IVec {
-        self.as_bytes().as_ref().into()
-    }
 }
 
 impl IntoDisk for () {
@@ -200,10 +167,6 @@ impl IntoDisk for () {
     fn as_bytes(&self) -> Self::Bytes {
         []
     }
-
-    fn into_ivec(&self) -> sled::IVec {
-        sled::IVec::default()
-    }
 }
 
 impl IntoDisk for block::Height {
@@ -212,13 +175,10 @@ impl IntoDisk for block::Height {
     fn as_bytes(&self) -> Self::Bytes {
         self.0.to_be_bytes()
     }
-    fn into_ivec(&self) -> sled::IVec {
-        self.as_bytes().as_ref().into()
-    }
 }
 
 impl FromDisk for block::Height {
-    fn from_ivec(bytes: sled::IVec) -> Self {
+    fn from_bytes(bytes: impl AsRef<[u8]>) -> Self {
         let array = bytes.as_ref().try_into().unwrap();
         block::Height(u32::from_be_bytes(array))
     }
@@ -231,16 +191,12 @@ impl IntoDisk for transparent::Output {
         self.zcash_serialize_to_vec()
             .expect("serialization to vec doesn't fail")
     }
-
-    fn into_ivec(&self) -> sled::IVec {
-        self.as_bytes().into()
-    }
 }
 
 impl FromDisk for transparent::Output {
-    fn from_ivec(bytes: sled::IVec) -> Self {
-        Self::zcash_deserialize(&*bytes)
-            .expect("deserialization format should match the serialization format used by IntoSled")
+    fn from_bytes(bytes: impl AsRef<[u8]>) -> Self {
+        Self::zcash_deserialize(bytes.as_ref())
+            .expect("deserialization format should match the serialization format used by IntoDisk")
     }
 }
 
@@ -251,78 +207,57 @@ impl IntoDisk for transparent::OutPoint {
         self.zcash_serialize_to_vec()
             .expect("serialization to vec doesn't fail")
     }
-
-    fn into_ivec(&self) -> sled::IVec {
-        self.as_bytes().into()
-    }
 }
 
-/// Helper trait for inserting (Key, Value) pairs into sled with a consistently
+/// Helper trait for inserting (Key, Value) pairs into rocksdb with a consistently
 /// defined format
 pub trait DiskSerialize {
-    /// Serialize and insert the given key and value into a sled tree.
-    fn zs_insert<K, V>(
-        &self,
-        key: K,
-        value: V,
-    ) -> Result<(), sled::transaction::UnabortableTransactionError>
+    /// Serialize and insert the given key and value into a rocksdb column family.
+    fn zs_insert<K, V>(&mut self, cf: &rocksdb::ColumnFamily, key: K, value: V)
     where
         K: IntoDisk + Debug,
         V: IntoDisk;
 }
 
-impl DiskSerialize for sled::transaction::TransactionalTree {
-    fn zs_insert<K, V>(
-        &self,
-        key: K,
-        value: V,
-    ) -> Result<(), sled::transaction::UnabortableTransactionError>
+impl DiskSerialize for rocksdb::WriteBatch {
+    fn zs_insert<K, V>(&mut self, cf: &rocksdb::ColumnFamily, key: K, value: V)
     where
         K: IntoDisk + Debug,
         V: IntoDisk,
     {
-        use std::any::type_name;
-
-        let key_bytes = key.into_ivec();
-        let value_bytes = value.into_ivec();
-        let previous = self.insert(key_bytes, value_bytes)?;
-
-        assert!(
-            previous.is_none(),
-            "duplicate key: previous value for key {:?} was not none when inserting into ({}, {}) sled Tree",
-            key,
-            type_name::<K>(),
-            type_name::<V>()
-        );
-
-        Ok(())
+        let key_bytes = key.as_bytes();
+        let value_bytes = value.as_bytes();
+        self.put_cf(cf, key_bytes, value_bytes);
     }
 }
 
-/// Helper trait for retrieving values from sled trees with a consistently
+/// Helper trait for retrieving values from rocksdb column familys with a consistently
 /// defined format
 pub trait DiskDeserialize {
     /// Serialize the given key and use that to get and deserialize the
-    /// corresponding value from a sled tree, if it is present.
-    fn zs_get<K, V>(&self, key: &K) -> Option<V>
+    /// corresponding value from a rocksdb column family, if it is present.
+    fn zs_get<K, V>(&self, cf: &rocksdb::ColumnFamily, key: &K) -> Option<V>
     where
         K: IntoDisk,
         V: FromDisk;
 }
 
-impl DiskDeserialize for sled::Tree {
-    fn zs_get<K, V>(&self, key: &K) -> Option<V>
+impl DiskDeserialize for rocksdb::DB {
+    fn zs_get<K, V>(&self, cf: &rocksdb::ColumnFamily, key: &K) -> Option<V>
     where
         K: IntoDisk,
         V: FromDisk,
     {
         let key_bytes = key.as_bytes();
 
+        // We use `get_pinned_cf` to avoid taking ownership of the serialized
+        // format because we're going to deserialize it anyways, which avoids an
+        // extra copy
         let value_bytes = self
-            .get(key_bytes)
-            .expect("expected that sled errors would not occur");
+            .get_pinned_cf(cf, key_bytes)
+            .expect("expected that disk errors would not occur");
 
-        value_bytes.map(V::from_ivec)
+        value_bytes.map(V::from_bytes)
     }
 }
 
@@ -347,8 +282,8 @@ mod tests {
     where
         T: IntoDisk + FromDisk,
     {
-        let bytes = input.into_ivec();
-        T::from_ivec(bytes)
+        let bytes = input.as_bytes();
+        T::from_bytes(bytes)
     }
 
     fn assert_round_trip<T>(input: T)
@@ -364,8 +299,8 @@ mod tests {
     where
         T: IntoDisk + FromDisk,
     {
-        let bytes = input.into_ivec();
-        T::from_ivec(bytes)
+        let bytes = input.as_bytes();
+        T::from_bytes(bytes)
     }
 
     fn assert_round_trip_ref<T>(input: &T)
@@ -381,8 +316,8 @@ mod tests {
     where
         T: IntoDisk + FromDisk,
     {
-        let bytes = input.into_ivec();
-        T::from_ivec(bytes)
+        let bytes = input.as_bytes();
+        T::from_bytes(bytes)
     }
 
     fn assert_round_trip_arc<T>(input: Arc<T>)
@@ -394,9 +329,9 @@ mod tests {
         assert_eq!(*before, after);
     }
 
-    /// The round trip test covers types that are used as value field in a sled
-    /// Tree. Only these types are ever deserialized, and so they're the only
-    /// ones that implement both `IntoSled` and `FromSled`.
+    /// The round trip test covers types that are used as value field in a rocksdb
+    /// column family. Only these types are ever deserialized, and so they're the only
+    /// ones that implement both `IntoDisk` and `FromDisk`.
     fn assert_value_properties<T>(input: T)
     where
         T: IntoDisk + FromDisk + Clone + PartialEq + std::fmt::Debug,
@@ -406,21 +341,6 @@ mod tests {
         assert_round_trip(input);
     }
 
-    /// This test asserts that types that are used as sled keys behave correctly.
-    /// Any type that implements `IntoIVec` can be used as a sled key. The value
-    /// is serialized via `IntoSled::into_ivec` when the `key`, `value` pair is
-    /// inserted into the sled tree. The `as_bytes` impl on the other hand is
-    /// called for most other operations when comparing a key against existing
-    /// keys in the sled database, such as `contains`.
-    fn assert_as_bytes_matches_ivec<T>(input: T)
-    where
-        T: IntoDisk + Clone,
-    {
-        let before = input.clone();
-        let ivec = input.into_ivec();
-        assert_eq!(before.as_bytes().as_ref(), ivec.as_ref());
-    }
-
     #[test]
     fn roundtrip_transaction_location() {
         zebra_test::init();
@@ -452,52 +372,4 @@ mod tests {
 
         proptest!(|(val in any::<transparent::Output>())| assert_value_properties(val));
     }
-
-    #[test]
-    fn key_matches_ivec_transaction_location() {
-        zebra_test::init();
-        proptest!(|(val in any::<TransactionLocation>())| assert_as_bytes_matches_ivec(val));
-    }
-
-    #[test]
-    fn key_matches_ivec_trans_hash() {
-        zebra_test::init();
-        proptest!(|(val in any::<transaction::Hash>())| assert_as_bytes_matches_ivec(val));
-    }
-
-    #[test]
-    fn key_matches_ivec_block_hash() {
-        zebra_test::init();
-        proptest!(|(val in any::<block::Hash>())| assert_as_bytes_matches_ivec(val));
-    }
-
-    #[test]
-    fn key_matches_ivec_sprout_nullifier() {
-        zebra_test::init();
-        proptest!(|(val in any::<sprout::Nullifier>())| assert_as_bytes_matches_ivec(val));
-    }
-
-    #[test]
-    fn key_matches_ivec_sapling_nullifier() {
-        zebra_test::init();
-        proptest!(|(val in any::<sapling::Nullifier>())| assert_as_bytes_matches_ivec(val));
-    }
-
-    #[test]
-    fn key_matches_ivec_block_height() {
-        zebra_test::init();
-        proptest!(|(val in any::<block::Height>())| assert_as_bytes_matches_ivec(val));
-    }
-
-    #[test]
-    fn key_matches_ivec_transparent_output() {
-        zebra_test::init();
-        proptest!(|(val in any::<transparent::Output>())| assert_as_bytes_matches_ivec(val));
-    }
-
-    #[test]
-    fn key_matches_ivec_transparent_outpoint() {
-        zebra_test::init();
-        proptest!(|(val in any::<transparent::OutPoint>())| assert_as_bytes_matches_ivec(val));
-    }
 }
diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs
index 742fb1ebd0f..bd382115ce0 100644
--- a/zebrad/tests/acceptance.rs
+++ b/zebrad/tests/acceptance.rs
@@ -313,7 +313,7 @@ fn persistent_mode() -> Result<()> {
     // Make sure the command was killed
     output.assert_was_killed()?;
 
-    // Check that we have persistent sled database
+    // Check that we have persistent rocksdb database
     let cache_dir = testdir.path().join("state");
     assert_with_context!(cache_dir.read_dir()?.count() > 0, &output);
 
@@ -536,8 +536,8 @@ fn restart_stop_at_height() -> Result<()> {
         SMALL_CHECKPOINT_TIMEOUT,
         None,
     )?;
-    // if stopping corrupts the sled database, zebrad might hang here
-    // if stopping does not sync the sled database, the logs will contain OnCommit
+    // if stopping corrupts the rocksdb database, zebrad might hang here
+    // if stopping does not sync the rocksdb database, the logs will contain OnCommit
     sync_until(
         Height(0),
         Mainnet,

From a23de13af90053257778dd0121da627526617852 Mon Sep 17 00:00:00 2001
From: Deirdre Connolly <deirdre@zfnd.org>
Date: Wed, 18 Nov 2020 14:41:47 -0500
Subject: [PATCH 041/148] Break up Dockerfile into (additional) test and build
 images

---
 docker/Dockerfile.build | 39 +++++++++++++++++++++++++++++++++++++++
 docker/Dockerfile.test  | 22 ++++++++++++++++++++++
 2 files changed, 61 insertions(+)
 create mode 100644 docker/Dockerfile.build
 create mode 100644 docker/Dockerfile.test

diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
new file mode 100644
index 00000000000..9b99a23d864
--- /dev/null
+++ b/docker/Dockerfile.build
@@ -0,0 +1,39 @@
+# Builder image
+FROM rust:buster as builder
+
+RUN apt-get update && \
+	apt-get install -y --no-install-recommends \
+	make cmake g++ gcc llvm libclang-dev
+
+RUN mkdir /zebra
+WORKDIR /zebra
+
+ENV RUST_BACKTRACE 1
+ENV CARGO_HOME /zebra/.cargo/
+RUN rustc -V; cargo -V; rustup -V
+
+COPY . .
+RUN cargo build --release
+
+
+# Runner image
+FROM debian:buster-slim AS zebrad-release
+
+COPY --from=builder /zebra/target/release/zebrad /
+
+ARG CHECKPOINT_SYNC=true
+ARG NETWORK=Mainnet
+
+RUN printf "[consensus]\n" >> /zebrad.toml
+RUN printf "checkpoint_sync = ${CHECKPOINT_SYNC}\n" >> /zebrad.toml
+RUN printf "[network]\n" >> /zebrad.toml
+RUN printf "network = '${NETWORK}'\n" >> /zebrad.toml
+RUN printf "[state]\n" >> /zebrad.toml
+RUN printf "cache_dir = '/zebrad-cache'\n" >> /zebrad.toml
+RUN printf "memory_cache_bytes = 52428800\n" >> /zebrad.toml
+RUN printf "[tracing]\n" >> /zebrad.toml
+RUN printf "endpoint_addr = '0.0.0.0:3000'\n" >> /zebrad.toml
+
+EXPOSE 3000 8233 18233
+
+CMD [ "/zebrad", "-c", "/zebrad.toml", "start" ]
diff --git a/docker/Dockerfile.test b/docker/Dockerfile.test
new file mode 100644
index 00000000000..b41c8c8b870
--- /dev/null
+++ b/docker/Dockerfile.test
@@ -0,0 +1,22 @@
+FROM rustlang/rust:nightly
+
+RUN apt-get update && \
+	apt-get install -y --no-install-recommends \
+	make cmake g++ gcc llvm libclang-dev
+
+RUN mkdir /zebra
+WORKDIR /zebra
+
+ENV RUST_BACKTRACE 1
+ENV CARGO_HOME /zebra/.cargo/
+
+RUN rustc -V; cargo -V; rustup -V
+
+EXPOSE 8233 18233
+
+COPY . .
+RUN cargo test --all --no-run
+RUN find /zebra/target/debug/deps -type f -perm 755 ! -name '*.dylib' ! -name '*.so' | sed -e 'p;s/-.*//' | xargs -n2 mv
+
+# Filtering to only run integration tests requires nightly for now
+CMD ["cargo", "+nightly", "test", "--test", "'*'", "--no-fail-fast", "--", "-Zunstable-options", "--include-ignored"]

From 1c49e57eba34dd79eb632f700ab759f590d7e699 Mon Sep 17 00:00:00 2001
From: Deirdre Connolly <deirdre@zfnd.org>
Date: Wed, 18 Nov 2020 15:53:09 -0500
Subject: [PATCH 042/148] Escape single quotes passed as CMD args to cargo

---
 docker/Dockerfile.test | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/Dockerfile.test b/docker/Dockerfile.test
index b41c8c8b870..95f3566445d 100644
--- a/docker/Dockerfile.test
+++ b/docker/Dockerfile.test
@@ -19,4 +19,4 @@ RUN cargo test --all --no-run
 RUN find /zebra/target/debug/deps -type f -perm 755 ! -name '*.dylib' ! -name '*.so' | sed -e 'p;s/-.*//' | xargs -n2 mv
 
 # Filtering to only run integration tests requires nightly for now
-CMD ["cargo", "+nightly", "test", "--test", "'*'", "--no-fail-fast", "--", "-Zunstable-options", "--include-ignored"]
+CMD ["cargo", "+nightly", "test", "--test", "\'*\'", "--no-fail-fast", "--", "-Zunstable-options", "--include-ignored"]

From 2445d23dd8f9ea1820468a804bed488dbc3b6e06 Mon Sep 17 00:00:00 2001
From: Deirdre Connolly <deirdre@zfnd.org>
Date: Wed, 18 Nov 2020 18:45:25 -0500
Subject: [PATCH 043/148] Shell form CMD

---
 docker/Dockerfile.test | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/Dockerfile.test b/docker/Dockerfile.test
index 95f3566445d..e8909a62788 100644
--- a/docker/Dockerfile.test
+++ b/docker/Dockerfile.test
@@ -19,4 +19,4 @@ RUN cargo test --all --no-run
 RUN find /zebra/target/debug/deps -type f -perm 755 ! -name '*.dylib' ! -name '*.so' | sed -e 'p;s/-.*//' | xargs -n2 mv
 
 # Filtering to only run integration tests requires nightly for now
-CMD ["cargo", "+nightly", "test", "--test", "\'*\'", "--no-fail-fast", "--", "-Zunstable-options", "--include-ignored"]
+CMD cargo +nightly test --test '*' --no-fail-fast -- -Zunstable-options --include-ignored

From 44970af929cbf6df35c2dac87b409628fef312b2 Mon Sep 17 00:00:00 2001
From: Deirdre Connolly <deirdre@zfnd.org>
Date: Thu, 19 Nov 2020 15:21:03 -0500
Subject: [PATCH 044/148] Split up big test job into its own workflow

---
 .github/workflows/cd.yml   | 72 ++++---------------------------------
 .github/workflows/test.yml | 73 ++++++++++++++++++++++++++++++++++++++
 Dockerfile                 | 53 ---------------------------
 cloudbuild.yaml            | 10 ++----
 docker/Dockerfile.test     |  4 +--
 5 files changed, 82 insertions(+), 130 deletions(-)
 create mode 100644 .github/workflows/test.yml
 delete mode 100644 Dockerfile

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index e37bd81401d..60e8248798f 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -1,13 +1,15 @@
 name: CD
 
 on:
+  workflow_dispatch:
   push:
     branches:
       - main
-      - cloudbuild
 
 env:
   PROJECT_ID: zealous-zebra
+  ZONE: us-east1-b
+
 
 jobs:
 
@@ -43,68 +45,6 @@ jobs:
           --config cloudbuild.yaml \
           --substitutions SHORT_SHA="$SHORT_SHA",BRANCH_NAME="$BRANCH_NAME"
 
-  test:
-    name: Run integration tests
-    needs: build
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Set project and image names
-        run: |
-          BRANCH_NAME=$(expr $GITHUB_REF : '.*/\(.*\)') && \
-          BRANCH_NAME=${BRANCH_NAME,,} && \
-          REPOSITORY=${GITHUB_REPOSITORY,,} && \
-          echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV && \
-          echo "SHORT_SHA=$(git rev-parse --short=7 $GITHUB_SHA)" >> $GITHUB_ENV && \
-          echo "REPOSITORY=$REPOSITORY" >> $GITHUB_ENV
-
-      # Setup gcloud CLI
-      - name: Set up gcloud SDK environment
-        uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
-        with:
-          version: '295.0.0'
-          project_id: ${{ env.PROJECT_ID }}
-          service_account_key: ${{ secrets.GCLOUD_AUTH }}
-
-      # Run once: create firewall rule to allow incoming traffic to the nodes
-      # - name: Create Zcash incoming traffic firewall rule
-      #   run: |
-      #     gcloud compute firewall-rules create "allow-zcash" \
-      #     --target-tags zebrad \
-      #     --allow tcp:8233,tcp:18233 \
-      #     --source-ranges 0.0.0.0/0 \
-      #     --description="Allow incoming Zcash traffic from anywhere" \
-
-      # Creates Compute Engine virtual machine instance w/ test container and disks
-      - name: Create instance with test container image
-        run: |
-          gcloud compute instances create-with-container "zebra-tests-$BRANCH_NAME-$SHORT_SHA" \
-          --container-command='/zebra/target/debug/deps/acceptance' \
-          --container-arg='-Z' --container-arg='unstable-options' --container-arg='--include-ignored' \
-          --container-image "gcr.io/$PROJECT_ID/$REPOSITORY/tests/$BRANCH_NAME:$SHORT_SHA" \
-          --container-mount-disk mount-path='/zebrad-cache',name=zebrad-cache-$SHORT_SHA \
-          --container-restart-policy='never' \
-          --create-disk name=zebrad-cache-$SHORT_SHA,image=zebrad-cache-mainnet-height-1018849 \
-          --service-account cos-vm@zealous-zebra.iam.gserviceaccount.com \
-          --scopes cloud-platform \
-          --tags zebrad \
-          --zone us-central1-a
-
-      # Wait for acceptance test container is up and stream its results
-      - name: Run acceptance tests
-        run: |
-          gcloud compute ssh "zebra-tests-$BRANCH_NAME-$SHORT_SHA" --zone us-central1-a \
-          --command "echo 'Pulling container image...'; until [[ -n \$(docker ps --all -q --filter name=zebra-test) ]]; do sleep 0.1; done; docker attach \$(docker ps -q -a --filter name=zebra-test)"
-
-      # Clean up
-      - name: Delete test instance
-        # Always run even if the earlier step fails
-        if: ${{ always() }}
-        run: |
-          gcloud compute instances delete "zebra-tests-$BRANCH_NAME-$SHORT_SHA" --delete-disks all --zone us-central1-a
-
   deploy:
     name: Deploy mainnet nodes
     needs: build
@@ -146,7 +86,7 @@ jobs:
           --container-image "gcr.io/$PROJECT_ID/$REPOSITORY/$BRANCH_NAME:$SHORT_SHA" \
           --create-disk name=zebrad-cache-$SHORT_SHA,size=100GB,type=pd-balanced,auto-delete=no \
           --container-mount-disk mount-path="/zebrad-cache",name=zebrad-cache-$SHORT_SHA \
-          --machine-type n2-standard-4 \
+          --machine-type n2d-standard-4 \
           --service-account cos-vm@zealous-zebra.iam.gserviceaccount.com \
           --scopes cloud-platform \
           --tags zebrad \
@@ -176,7 +116,7 @@ jobs:
           --template "zebrad-$BRANCH_NAME-$SHORT_SHA" \
           --health-check zebrad-tracing-filter \
           --initial-delay 30 \
-          --region us-central1 \
+          --region "$ZONE" \
           --size 2
 
       # Rolls out update to existing group using the new instance template
@@ -186,4 +126,4 @@ jobs:
           gcloud compute instance-groups managed rolling-action start-update \
           "zebrad-$BRANCH_NAME" \
           --version template="zebrad-$BRANCH_NAME-$SHORT_SHA" \
-          --region us-central1 \
+          --region "$ZONE"
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
new file mode 100644
index 00000000000..586c5e46d7d
--- /dev/null
+++ b/.github/workflows/test.yml
@@ -0,0 +1,73 @@
+name: Test
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+
+env:
+  PROJECT_ID: zealous-zebra
+  ZONE: europe-west1-b
+
+jobs:
+
+  test:
+    name: Run all tests
+    needs: build
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set project and image names
+        run: |
+          BRANCH_NAME=$(expr $GITHUB_REF : '.*/\(.*\)') && \
+          BRANCH_NAME=${BRANCH_NAME,,} && \
+          REPOSITORY=${GITHUB_REPOSITORY,,} && \
+          echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV && \
+          echo "SHORT_SHA=$(git rev-parse --short=7 $GITHUB_SHA)" >> $GITHUB_ENV && \
+          echo "REPOSITORY=$REPOSITORY" >> $GITHUB_ENV
+
+      # Setup gcloud CLI
+      - name: Set up gcloud
+        uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
+        with:
+          version: '295.0.0'
+          project_id: ${{ env.PROJECT_ID }}
+          service_account_key: ${{ secrets.GCLOUD_AUTH }}
+
+      # Run once: create firewall rule to allow incoming traffic to the nodes
+      # - name: Create Zcash incoming traffic firewall rule
+      #   run: |
+      #     gcloud compute firewall-rules create "allow-zcash" \
+      #     --target-tags zebrad \
+      #     --allow tcp:8233,tcp:18233 \
+      #     --source-ranges 0.0.0.0/0 \
+      #     --description="Allow incoming Zcash traffic from anywhere" \
+
+      # Creates Compute Engine virtual machine instance w/ disks
+      - name: Create instance
+        run: |
+          gcloud compute instances create "zebrad-tests-$BRANCH_NAME-$SHORT_SHA" \
+          --boot-disk-size 100GB \
+          --boot-disk-type pd-ssd \
+          --image-family cos-stable \
+          --machine-type n2d-standard-16 \
+          --service-account cos-vm@zealous-zebra.iam.gserviceaccount.com \
+          --scopes cloud-platform \
+          --tags zebrad \
+          --zone "$ZONE"
+
+      # Build and run test container
+      - name: Run all tests
+        run: |
+          gcloud compute ssh "zebrad-tests-$BRANCH_NAME-$SHORT_SHA" --zone "$ZONE" \
+          --command "git clone -b test https://github.com/ZcashFoundation/zebra.git; cd zebra/; docker build -f docker/Dockerfile.test -t zebrad-test .; docker run -i zebrad-test"
+
+      # Clean up
+      - name: Delete test instance
+        # Always run even if the earlier step fails
+        if: ${{ always() }}
+        run: |
+          gcloud compute instances delete "zebrad-tests-$BRANCH_NAME-$SHORT_SHA" --delete-disks all --zone "$ZONE"
diff --git a/Dockerfile b/Dockerfile
deleted file mode 100644
index caf1698052d..00000000000
--- a/Dockerfile
+++ /dev/null
@@ -1,53 +0,0 @@
-# Builder image
-FROM rust:buster as builder
-
-RUN apt-get update && \
-	apt-get install -y --no-install-recommends \
-	make cmake g++ gcc llvm libclang-dev clang
-
-RUN mkdir /zebra
-WORKDIR /zebra
-
-ENV RUST_BACKTRACE 1
-ENV CARGO_HOME /zebra/.cargo/
-
-# Copy local code to the container image.
-# Assumes that we are in the git repo.
-COPY . .
-RUN cargo fetch --verbose
-COPY . .
-RUN rustc -V; cargo -V; rustup -V; cargo test --all && cargo build --release
-RUN find /zebra/target/debug/deps -type f -perm 755 ! -name '*.dylib' ! -name '*.so' | sed -e 'p;s/-.*//' | xargs -n2 mv
-
-
-# Test binaries image
-FROM debian:buster AS zebra-tests
-
-RUN mkdir /zebra
-WORKDIR /zebra
-COPY --from=builder /zebra/target/debug/zebrad /zebra/target/debug/zebrad
-COPY --from=builder /zebra/target/debug/deps/[a-z0-9_]* /zebra/target/debug/deps/
-EXPOSE 8233 18233
-
-
-# Runner image
-FROM debian:buster-slim AS zebrad-release
-
-COPY --from=builder /zebra/target/release/zebrad /
-
-ARG CHECKPOINT_SYNC=true
-ARG NETWORK=Mainnet
-
-RUN printf "[consensus]\n" >> /zebrad.toml
-RUN printf "checkpoint_sync = ${CHECKPOINT_SYNC}\n" >> /zebrad.toml
-RUN printf "[network]\n" >> /zebrad.toml
-RUN printf "network = '${NETWORK}'\n" >> /zebrad.toml
-RUN printf "[state]\n" >> /zebrad.toml
-RUN printf "cache_dir = '/zebrad-cache'\n" >> /zebrad.toml
-RUN printf "memory_cache_bytes = 52428800\n" >> /zebrad.toml
-RUN printf "[tracing]\n" >> /zebrad.toml
-RUN printf "endpoint_addr = '0.0.0.0:3000'\n" >> /zebrad.toml
-
-EXPOSE 3000 8233 18233
-
-CMD [ "/zebrad", "-c", "/zebrad.toml", "start" ]
diff --git a/cloudbuild.yaml b/cloudbuild.yaml
index b96b0c95ed0..811a3518656 100644
--- a/cloudbuild.yaml
+++ b/cloudbuild.yaml
@@ -1,11 +1,4 @@
 steps:
-- name: 'gcr.io/cloud-builders/docker'
-  args: ['build',
-         '--target',
-         'zebra-tests',
-         '-t',
-         'gcr.io/$PROJECT_ID/zcashfoundation/zebra/tests/$BRANCH_NAME:$SHORT_SHA',
-         '.']
 - name: 'gcr.io/cloud-builders/docker'
   args: ['build',
          '--build-arg',
@@ -16,10 +9,11 @@ steps:
          'zebrad-release',
          '-t',
          'gcr.io/$PROJECT_ID/zcashfoundation/zebra/$BRANCH_NAME:$SHORT_SHA',
+         '-f',
+         'docker/Dockerfile.build',
          '.']
 
 images:
-- 'gcr.io/$PROJECT_ID/zcashfoundation/zebra/tests/$BRANCH_NAME:$SHORT_SHA'
 - 'gcr.io/$PROJECT_ID/zcashfoundation/zebra/$BRANCH_NAME:$SHORT_SHA'
 
 options:
diff --git a/docker/Dockerfile.test b/docker/Dockerfile.test
index e8909a62788..d501fe01aed 100644
--- a/docker/Dockerfile.test
+++ b/docker/Dockerfile.test
@@ -15,8 +15,6 @@ RUN rustc -V; cargo -V; rustup -V
 EXPOSE 8233 18233
 
 COPY . .
-RUN cargo test --all --no-run
-RUN find /zebra/target/debug/deps -type f -perm 755 ! -name '*.dylib' ! -name '*.so' | sed -e 'p;s/-.*//' | xargs -n2 mv
 
 # Filtering to only run integration tests requires nightly for now
-CMD cargo +nightly test --test '*' --no-fail-fast -- -Zunstable-options --include-ignored
+CMD cargo +nightly test --all --test '*' --no-fail-fast -- -Zunstable-options --include-ignored

From 938b6d6fddc6a4a9a0b056b04af67440cb685866 Mon Sep 17 00:00:00 2001
From: Deirdre Connolly <deirdre@zfnd.org>
Date: Thu, 19 Nov 2020 15:35:08 -0500
Subject: [PATCH 045/148] Make the full test suite command explicit

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 586c5e46d7d..7a3e2ba2721 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -63,7 +63,7 @@ jobs:
       - name: Run all tests
         run: |
           gcloud compute ssh "zebrad-tests-$BRANCH_NAME-$SHORT_SHA" --zone "$ZONE" \
-          --command "git clone -b test https://github.com/ZcashFoundation/zebra.git; cd zebra/; docker build -f docker/Dockerfile.test -t zebrad-test .; docker run -i zebrad-test"
+          --command "git clone -b test https://github.com/ZcashFoundation/zebra.git; cd zebra/; docker build -f docker/Dockerfile.test -t zebrad-test .; docker run -i zebrad-test cargo +nightly test --all --test '*' --no-fail-fast -- -Zunstable-options --include-ignored"
 
       # Clean up
       - name: Delete test instance

From 53d63d0514e5f1d7f1542b6f5281cac26e523f69 Mon Sep 17 00:00:00 2001
From: Deirdre Connolly <deirdre@zfnd.org>
Date: Thu, 19 Nov 2020 15:50:57 -0500
Subject: [PATCH 046/148] Build this branch

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 7a3e2ba2721..47ea661d1af 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -63,7 +63,7 @@ jobs:
       - name: Run all tests
         run: |
           gcloud compute ssh "zebrad-tests-$BRANCH_NAME-$SHORT_SHA" --zone "$ZONE" \
-          --command "git clone -b test https://github.com/ZcashFoundation/zebra.git; cd zebra/; docker build -f docker/Dockerfile.test -t zebrad-test .; docker run -i zebrad-test cargo +nightly test --all --test '*' --no-fail-fast -- -Zunstable-options --include-ignored"
+          --command "git clone -b $BRANCH_NAME https://github.com/ZcashFoundation/zebra.git; cd zebra/; docker build -f docker/Dockerfile.test -t zebrad-test .; docker run -i zebrad-test cargo +nightly test --all --test '*' --no-fail-fast -- -Zunstable-options --include-ignored"
 
       # Clean up
       - name: Delete test instance

From a317cc11c64a1272c1261ce9a4ad8a3806fcbfe3 Mon Sep 17 00:00:00 2001
From: Deirdre Connolly <deirdre@zfnd.org>
Date: Thu, 19 Nov 2020 16:03:15 -0500
Subject: [PATCH 047/148] Install clang to build rocksdb dep

---
 docker/Dockerfile.build | 2 +-
 docker/Dockerfile.test  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index 9b99a23d864..336bd8afc43 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -3,7 +3,7 @@ FROM rust:buster as builder
 
 RUN apt-get update && \
 	apt-get install -y --no-install-recommends \
-	make cmake g++ gcc llvm libclang-dev
+	make cmake g++ gcc llvm libclang-dev clang
 
 RUN mkdir /zebra
 WORKDIR /zebra
diff --git a/docker/Dockerfile.test b/docker/Dockerfile.test
index d501fe01aed..78d830481e5 100644
--- a/docker/Dockerfile.test
+++ b/docker/Dockerfile.test
@@ -2,7 +2,7 @@ FROM rustlang/rust:nightly
 
 RUN apt-get update && \
 	apt-get install -y --no-install-recommends \
-	make cmake g++ gcc llvm libclang-dev
+	make cmake g++ gcc llvm libclang-dev clang
 
 RUN mkdir /zebra
 WORKDIR /zebra

From e325775bf3df9ca867cf0dbcb94c110653342c71 Mon Sep 17 00:00:00 2001
From: Deirdre Connolly <deirdre@zfnd.org>
Date: Thu, 19 Nov 2020 16:44:12 -0500
Subject: [PATCH 048/148] Specify region not just zone

---
 .github/workflows/cd.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index 60e8248798f..d904f841b57 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -8,6 +8,7 @@ on:
 
 env:
   PROJECT_ID: zealous-zebra
+  REGION: us-east1
   ZONE: us-east1-b
 
 
@@ -116,7 +117,7 @@ jobs:
           --template "zebrad-$BRANCH_NAME-$SHORT_SHA" \
           --health-check zebrad-tracing-filter \
           --initial-delay 30 \
-          --region "$ZONE" \
+          --region "$REGION" \
           --size 2
 
       # Rolls out update to existing group using the new instance template
@@ -126,4 +127,4 @@ jobs:
           gcloud compute instance-groups managed rolling-action start-update \
           "zebrad-$BRANCH_NAME" \
           --version template="zebrad-$BRANCH_NAME-$SHORT_SHA" \
-          --region "$ZONE"
+          --region "$REGION"

From 623949bbaaa3b38fac0c4ca0f8e42117f11b09d5 Mon Sep 17 00:00:00 2001
From: Deirdre Connolly <deirdre@zfnd.org>
Date: Thu, 19 Nov 2020 16:47:20 -0500
Subject: [PATCH 049/148] Remove vestigial 'needs'

---
 .github/workflows/test.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 47ea661d1af..eb1117a2b57 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -14,7 +14,6 @@ jobs:
 
   test:
     name: Run all tests
-    needs: build
     runs-on: ubuntu-latest
     timeout-minutes: 30
     steps:

From fb66c7ecdf5335dd0d7ea030c62190f41dbdd98c Mon Sep 17 00:00:00 2001
From: Deirdre Connolly <deirdre@zfnd.org>
Date: Thu, 19 Nov 2020 17:13:00 -0500
Subject: [PATCH 050/148] Supply --image-project, return to N2 not N2D

---
 .github/workflows/test.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index eb1117a2b57..d6a28f6c814 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -52,7 +52,8 @@ jobs:
           --boot-disk-size 100GB \
           --boot-disk-type pd-ssd \
           --image-family cos-stable \
-          --machine-type n2d-standard-16 \
+          --image-project cos-cloud \
+          --machine-type n2-standard-16 \
           --service-account cos-vm@zealous-zebra.iam.gserviceaccount.com \
           --scopes cloud-platform \
           --tags zebrad \

From 06dd39df541ac466455c17e851b98a17c2f374da Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Thu, 19 Nov 2020 15:50:05 -0800
Subject: [PATCH 051/148] network: bump network version for Canopy (#1333)

Per https://zips.z.cash/zip-0251, nodes compatible with Canopy
activation on mainnet MUST advertise protocol version 170013 or later.

Once Canopy activates on testnet or mainnet, Canopy nodes SHOULD reject
new connections from pre-Canopy nodes, so this also increases the
minimum version.
---
 zebra-network/src/constants.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/zebra-network/src/constants.rs b/zebra-network/src/constants.rs
index f4b8d59deba..32f9f6c6849 100644
--- a/zebra-network/src/constants.rs
+++ b/zebra-network/src/constants.rs
@@ -67,7 +67,7 @@ pub const USER_AGENT: &str = "/🦓Zebra🦓:3.0.0-alpha.0/";
 ///
 /// The current protocol version typically changes before Mainnet and Testnet
 /// network upgrades.
-pub const CURRENT_VERSION: Version = Version(170_012);
+pub const CURRENT_VERSION: Version = Version(170_013);
 
 /// The most recent bilateral consensus upgrade implemented by this crate.
 ///
@@ -76,7 +76,7 @@ pub const CURRENT_VERSION: Version = Version(170_012);
 //
 // TODO: replace with NetworkUpgrade::current(network, height).
 //       See the detailed comment in handshake.rs, where this constant is used.
-pub const MIN_NETWORK_UPGRADE: NetworkUpgrade = NetworkUpgrade::Heartwood;
+pub const MIN_NETWORK_UPGRADE: NetworkUpgrade = NetworkUpgrade::Canopy;
 
 /// The default RTT estimate for peer responses.
 ///

From f6dc92a25604d2931e47466434d682efefdd52a4 Mon Sep 17 00:00:00 2001
From: Deirdre Connolly <deirdre@zfnd.org>
Date: Thu, 19 Nov 2020 18:43:36 -0500
Subject: [PATCH 052/148] Correctly grep for instance group & region

---
 .github/workflows/cd.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index d904f841b57..a503284a2d0 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -106,7 +106,7 @@ jobs:
         id: does-group-exist
         continue-on-error: true
         run: |
-          gcloud compute instance-groups list | grep "zebrad-$BRANCH_NAME"
+          gcloud compute instance-groups list | grep "zebrad-$BRANCH_NAME"\ \ "$REGION"
 
       # Deploy new managed instance group using the new instance template
       - name: Create managed instance group

From 2b9819a190b8c269183aa44110f9c7c51c4443d3 Mon Sep 17 00:00:00 2001
From: Deirdre Connolly <deirdre@zfnd.org>
Date: Thu, 19 Nov 2020 19:07:12 -0500
Subject: [PATCH 053/148] Remove defunct memory_cache_bytes

It left with sled
---
 docker/Dockerfile.build | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index 336bd8afc43..4c3386fdc93 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -30,7 +30,6 @@ RUN printf "[network]\n" >> /zebrad.toml
 RUN printf "network = '${NETWORK}'\n" >> /zebrad.toml
 RUN printf "[state]\n" >> /zebrad.toml
 RUN printf "cache_dir = '/zebrad-cache'\n" >> /zebrad.toml
-RUN printf "memory_cache_bytes = 52428800\n" >> /zebrad.toml
 RUN printf "[tracing]\n" >> /zebrad.toml
 RUN printf "endpoint_addr = '0.0.0.0:3000'\n" >> /zebrad.toml
 

From af5f3c1395acfe0817cce710c26dfe6eb90173ff Mon Sep 17 00:00:00 2001
From: Deirdre Connolly <deirdre@zfnd.org>
Date: Thu, 19 Nov 2020 19:15:25 -0500
Subject: [PATCH 054/148] Bump down cores, running into default quotas

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index d6a28f6c814..6b17625a0e3 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -53,7 +53,7 @@ jobs:
           --boot-disk-type pd-ssd \
           --image-family cos-stable \
           --image-project cos-cloud \
-          --machine-type n2-standard-16 \
+          --machine-type n2-standard-4 \
           --service-account cos-vm@zealous-zebra.iam.gserviceaccount.com \
           --scopes cloud-platform \
           --tags zebrad \

From ec00ee4cf01669ec32e7ba047613e239a45af5b2 Mon Sep 17 00:00:00 2001
From: teor <teor@riseup.net>
Date: Fri, 20 Nov 2020 13:01:19 +1000
Subject: [PATCH 055/148] Stop using /dev/shm on Linux (#1338)

Some systems have a very small /dev/shm, for example, see:
https://github.com/docker-library/postgres/issues/416

So we should just use the temporary directory on all operating systems.

Also:
* use TempDir to generate the temporary path
* delete the code that we copied from sled
* prefix the temporary path with the state version and network
---
 zebra-state/Cargo.toml    |  1 +
 zebra-state/src/config.rs | 43 ++++++++++++---------------------------
 2 files changed, 14 insertions(+), 30 deletions(-)

diff --git a/zebra-state/Cargo.toml b/zebra-state/Cargo.toml
index 4fb34e566ea..fa7d8ab0ade 100644
--- a/zebra-state/Cargo.toml
+++ b/zebra-state/Cargo.toml
@@ -25,6 +25,7 @@ thiserror = "1.0.22"
 tokio = { version = "0.2.22", features = ["sync"] }
 displaydoc = "0.1.7"
 rocksdb = "0.15.0"
+tempdir = "0.3.7"
 
 [dev-dependencies]
 zebra-chain = { path = "../zebra-chain", features = ["proptest-impl"] }
diff --git a/zebra-state/src/config.rs b/zebra-state/src/config.rs
index 720ffaa8e19..baa14ddf6f0 100644
--- a/zebra-state/src/config.rs
+++ b/zebra-state/src/config.rs
@@ -1,5 +1,6 @@
 use serde::{Deserialize, Serialize};
 use std::path::PathBuf;
+use tempdir::TempDir;
 use zebra_chain::parameters::Network;
 
 /// Configuration for the state service.
@@ -29,7 +30,9 @@ pub struct Config {
 
     /// Whether to use an ephemeral database.
     ///
-    /// Ephemeral databases are stored in memory on Linux, and in a temporary directory on other OSes.
+    /// Ephemeral databases are stored in a temporary directory.
+    /// They are deleted when Zebra exits successfully.
+    /// (If Zebra panics or crashes, the ephemeral database won't be deleted.)
     ///
     /// Set to `false` by default. If this is set to `true`, [`cache_dir`] is ignored.
     ///
@@ -42,34 +45,10 @@ pub struct Config {
     pub debug_stop_at_height: Option<u32>,
 }
 
-fn gen_temp_path() -> PathBuf {
-    use std::sync::atomic::{AtomicUsize, Ordering};
-    use std::time::SystemTime;
-
-    static SALT_COUNTER: AtomicUsize = AtomicUsize::new(0);
-
-    let seed = SALT_COUNTER.fetch_add(1, Ordering::SeqCst) as u128;
-
-    let now = SystemTime::now()
-        .duration_since(SystemTime::UNIX_EPOCH)
-        .unwrap()
-        .as_nanos()
-        << 48;
-
-    #[cfg(not(miri))]
-    let pid = u128::from(std::process::id());
-
-    #[cfg(miri)]
-    let pid = 0;
-
-    let salt = (pid << 16) + now + seed;
-
-    if cfg!(target_os = "linux") {
-        // use shared memory for temporary linux files
-        format!("/dev/shm/pagecache.tmp.{}", salt).into()
-    } else {
-        std::env::temp_dir().join(format!("pagecache.tmp.{}", salt))
-    }
+fn gen_temp_path(prefix: &str) -> PathBuf {
+    TempDir::new(prefix)
+        .expect("temporary directory is created successfully")
+        .into_path()
 }
 
 impl Config {
@@ -95,7 +74,11 @@ impl Config {
         opts.create_missing_column_families(true);
 
         let path = if self.ephemeral {
-            gen_temp_path()
+            gen_temp_path(&format!(
+                "zebra-state-v{}-{}",
+                crate::constants::DATABASE_FORMAT_VERSION,
+                net_dir
+            ))
         } else {
             self.cache_dir
                 .join("state")

From add94c1c4537b012751e05f93fd2fb5e2bc3c762 Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Thu, 19 Nov 2020 11:54:20 -0800
Subject: [PATCH 056/148] deps: move to tokio 0.3, tower 0.4

This change is mostly mechanical, with the exception of the changes to the
`tower-batch` middleware.  This middleware was adapted from `tower::buffer`,
and the `tower::buffer` code was changed to implement its own bounded queue,
because Tokio 0.3 removed the `mpsc::Sender::poll_send` method.  See

https://github.com/tower-rs/tower/commit/ddc64e8d4d4d499adef569e284f317288edd6513

for more context on the Tower changes.  To match Tower as closely as possible
in order to be able to upstream `tower-batch`, those changes are copied from
`tower::Buffer` to `tower-batch`.
---
 Cargo.lock                                    | 235 ++++++++++++------
 Cargo.toml                                    |   2 +-
 tower-batch/Cargo.toml                        |   6 +-
 tower-batch/src/lib.rs                        |   1 +
 tower-batch/src/message.rs                    |   1 +
 tower-batch/src/semaphore.rs                  |  78 ++++++
 tower-batch/src/service.rs                    |  79 +++---
 tower-batch/src/worker.rs                     |  79 +++---
 tower-batch/tests/ed25519.rs                  |   2 +-
 tower-fallback/Cargo.toml                     |   4 +-
 zebra-consensus/Cargo.toml                    |   6 +-
 zebra-consensus/src/primitives/redjubjub.rs   |   2 +-
 zebra-network/Cargo.toml                      |   8 +-
 zebra-network/src/isolated.rs                 |   2 +-
 zebra-network/src/peer/connection.rs          |   6 +-
 zebra-network/src/peer_set/initialize.rs      |   2 +-
 .../src/peer_set/inventory_registry.rs        |  31 ++-
 zebra-network/src/protocol/external/codec.rs  |  13 +-
 zebra-state/Cargo.toml                        |   6 +-
 zebra-test/Cargo.toml                         |   4 +-
 zebrad/Cargo.toml                             |   4 +-
 zebrad/src/components/sync.rs                 |   6 +-
 zebrad/src/components/tokio.rs                |   3 +-
 23 files changed, 390 insertions(+), 190 deletions(-)
 create mode 100644 tower-batch/src/semaphore.rs

diff --git a/Cargo.lock b/Cargo.lock
index d7b3f4d53fe..489294e610e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -378,6 +378,12 @@ version = "0.5.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38"
 
+[[package]]
+name = "bytes"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e0dcbc35f504eb6fc275a6d20e4ebcda18cf50d40ba6fabff8c711fa16cb3b16"
+
 [[package]]
 name = "canonical-path"
 version = "2.0.2"
@@ -463,6 +469,15 @@ dependencies = [
  "bitflags",
 ]
 
+[[package]]
+name = "cloudabi"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4344512281c643ae7638bbabc3af17a11307803ec8f0fcad9fae512a8bf36467"
+dependencies = [
+ "bitflags",
+]
+
 [[package]]
 name = "color-backtrace"
 version = "0.3.0"
@@ -1046,7 +1061,7 @@ dependencies = [
  "futures-sink",
  "futures-task",
  "memchr",
- "pin-project 1.0.1",
+ "pin-project 1.0.2",
  "pin-utils",
  "proc-macro-hack",
  "proc-macro-nested",
@@ -1166,7 +1181,7 @@ version = "0.2.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5e4728fd124914ad25e99e3d15a9361a879f6620f63cb56bbb08f95abb97a535"
 dependencies = [
- "bytes",
+ "bytes 0.5.6",
  "fnv",
  "futures-core",
  "futures-sink",
@@ -1174,7 +1189,7 @@ dependencies = [
  "http",
  "indexmap",
  "slab",
- "tokio",
+ "tokio 0.2.23",
  "tokio-util 0.3.1",
  "tracing",
  "tracing-futures",
@@ -1230,7 +1245,7 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "28d569972648b2c512421b5f2a405ad6ac9666547189d0c5477a3f200f3e02f9"
 dependencies = [
- "bytes",
+ "bytes 0.5.6",
  "fnv",
  "itoa",
 ]
@@ -1241,7 +1256,7 @@ version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "13d5ff830006f7646652e057693569bfe0d51760c0085a071769d142a205111b"
 dependencies = [
- "bytes",
+ "bytes 0.5.6",
  "http",
 ]
 
@@ -1272,7 +1287,7 @@ version = "0.13.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f6ad767baac13b44d4529fcf58ba2cd0995e36e7b435bc5b039de6f47e880dbf"
 dependencies = [
- "bytes",
+ "bytes 0.5.6",
  "futures-channel",
  "futures-core",
  "futures-util",
@@ -1282,9 +1297,9 @@ dependencies = [
  "httparse",
  "httpdate",
  "itoa",
- "pin-project 1.0.1",
+ "pin-project 1.0.2",
  "socket2",
- "tokio",
+ "tokio 0.2.23",
  "tower-service",
  "tracing",
  "want",
@@ -1362,6 +1377,15 @@ dependencies = [
  "str_stack",
 ]
 
+[[package]]
+name = "instant"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec"
+dependencies = [
+ "cfg-if 1.0.0",
+]
+
 [[package]]
 name = "iovec"
 version = "0.1.4"
@@ -1466,6 +1490,15 @@ dependencies = [
  "scopeguard",
 ]
 
+[[package]]
+name = "lock_api"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dd96ffd135b2fd7b973ac026d28085defbe8983df057ced3eb4f2130b0831312"
+dependencies = [
+ "scopeguard",
+]
+
 [[package]]
 name = "log"
 version = "0.4.11"
@@ -1558,7 +1591,7 @@ checksum = "f3fc63816bd5f8bde5eb31ce471f9633adc69ba1c55b44191b4d5fc7e263e8ab"
 dependencies = [
  "log",
  "metrics-core",
- "tokio",
+ "tokio 0.2.23",
 ]
 
 [[package]]
@@ -1614,15 +1647,15 @@ dependencies = [
  "metrics-observer-prometheus",
  "metrics-observer-yaml",
  "metrics-util",
- "parking_lot",
+ "parking_lot 0.10.2",
  "quanta",
 ]
 
 [[package]]
 name = "metrics-util"
-version = "0.3.1"
+version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d11f8090a8886339f9468a04eeea0711e4cf27538b134014664308041307a1c5"
+checksum = "277619f040719a5a23d75724586d5601286e8fa53451cfaaca3b8c627c2c2378"
 dependencies = [
  "crossbeam-epoch 0.8.2",
  "serde",
@@ -1658,28 +1691,18 @@ dependencies = [
 ]
 
 [[package]]
-name = "mio-named-pipes"
-version = "0.1.7"
+name = "mio"
+version = "0.7.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0840c1c50fd55e521b247f949c241c9997709f23bd7f023b9762cd561e935656"
+checksum = "f33bc887064ef1fd66020c9adfc45bb9f33d75a42096c81e7c56c65b75dd1a8b"
 dependencies = [
+ "libc",
  "log",
- "mio",
- "miow 0.3.5",
+ "miow 0.3.6",
+ "ntapi",
  "winapi 0.3.9",
 ]
 
-[[package]]
-name = "mio-uds"
-version = "0.6.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "afcb699eb26d4332647cc848492bbc15eafb26f08d0304550d5aa1f612e066f0"
-dependencies = [
- "iovec",
- "libc",
- "mio",
-]
-
 [[package]]
 name = "miow"
 version = "0.2.1"
@@ -1694,9 +1717,9 @@ dependencies = [
 
 [[package]]
 name = "miow"
-version = "0.3.5"
+version = "0.3.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07b88fb9795d4d36d62a012dfbf49a8f5cf12751f36d31a9dbe66d528e58979e"
+checksum = "5a33c1b55807fbed163481b5ba66db4b2fa6cde694a5027be10fb724206c5897"
 dependencies = [
  "socket2",
  "winapi 0.3.9",
@@ -1739,6 +1762,15 @@ dependencies = [
  "version_check 0.9.2",
 ]
 
+[[package]]
+name = "ntapi"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44"
+dependencies = [
+ "winapi 0.3.9",
+]
+
 [[package]]
 name = "num-format"
 version = "0.4.0"
@@ -1844,8 +1876,19 @@ version = "0.10.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d3a704eb390aafdc107b0e392f56a82b668e3a71366993b5340f5833fd62505e"
 dependencies = [
- "lock_api",
- "parking_lot_core",
+ "lock_api 0.3.4",
+ "parking_lot_core 0.7.2",
+]
+
+[[package]]
+name = "parking_lot"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb"
+dependencies = [
+ "instant",
+ "lock_api 0.4.2",
+ "parking_lot_core 0.8.0",
 ]
 
 [[package]]
@@ -1855,10 +1898,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d58c7c768d4ba344e3e8d72518ac13e259d7c7ade24167003b8488e10b6740a3"
 dependencies = [
  "cfg-if 0.1.10",
- "cloudabi",
+ "cloudabi 0.0.3",
  "libc",
  "redox_syscall",
- "smallvec 1.4.2",
+ "smallvec 1.5.0",
+ "winapi 0.3.9",
+]
+
+[[package]]
+name = "parking_lot_core"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c361aa727dd08437f2f1447be8b59a33b0edd15e0fcee698f935613d9efbca9b"
+dependencies = [
+ "cfg-if 0.1.10",
+ "cloudabi 0.1.0",
+ "instant",
+ "libc",
+ "redox_syscall",
+ "smallvec 1.5.0",
  "winapi 0.3.9",
 ]
 
@@ -1885,11 +1943,11 @@ dependencies = [
 
 [[package]]
 name = "pin-project"
-version = "1.0.1"
+version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ee41d838744f60d959d7074e3afb6b35c7456d0f61cad38a24e35e6553f73841"
+checksum = "9ccc2237c2c489783abd8c4c80e5450fc0e98644555b1364da68cc29aa151ca7"
 dependencies = [
- "pin-project-internal 1.0.1",
+ "pin-project-internal 1.0.2",
 ]
 
 [[package]]
@@ -1905,9 +1963,9 @@ dependencies = [
 
 [[package]]
 name = "pin-project-internal"
-version = "1.0.1"
+version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "81a4ffa594b66bff340084d4081df649a7dc049ac8d7fc458d8e628bfbbb2f86"
+checksum = "f8e8d2bf0b23038a4424865103a4df472855692821aab4e4f5c3312d461d9e5f"
 dependencies = [
  "proc-macro2 1.0.24",
  "quote 1.0.7",
@@ -1920,6 +1978,12 @@ version = "0.1.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c917123afa01924fc84bb20c4c03f004d9c38e5127e3c039bbf7f4b9c76a2f6b"
 
+[[package]]
+name = "pin-project-lite"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6b063f57ec186e6140e2b8b6921e5f1bd89c7356dda5b33acc5401203ca6131c"
+
 [[package]]
 name = "pin-utils"
 version = "0.1.0"
@@ -2559,9 +2623,9 @@ dependencies = [
 
 [[package]]
 name = "smallvec"
-version = "1.4.2"
+version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fbee7696b84bbf3d89a1c2eccff0850e3047ed46bfcd2e92c29a2d074d57e252"
+checksum = "7acad6f34eb9e8a259d3283d1e8c1d34d7415943d4895f65cc73813c7396fc85"
 
 [[package]]
 name = "socket2"
@@ -2717,9 +2781,9 @@ dependencies = [
 
 [[package]]
 name = "termcolor"
-version = "1.1.0"
+version = "1.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bb6bfa289a4d7c5766392812c0a1f4c1ba45afa1ad47803c11e1f407d846d75f"
+checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4"
 dependencies = [
  "winapi-util",
 ]
@@ -2794,18 +2858,33 @@ version = "0.2.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a6d7ad61edd59bfcc7e80dababf0f4aed2e6d5e0ba1659356ae889752dfc12ff"
 dependencies = [
- "bytes",
+ "bytes 0.5.6",
  "fnv",
  "futures-core",
  "iovec",
  "lazy_static",
+ "memchr",
+ "mio 0.6.22",
+ "pin-project-lite 0.1.11",
+ "slab",
+]
+
+[[package]]
+name = "tokio"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9dfe2523e6fa84ddf5e688151d4e5fddc51678de9752c6512a24714c23818d61"
+dependencies = [
+ "autocfg",
+ "bytes 0.6.0",
+ "futures-core",
+ "lazy_static",
  "libc",
  "memchr",
- "mio",
- "mio-named-pipes",
- "mio-uds",
+ "mio 0.7.6",
  "num_cpus",
- "pin-project-lite",
+ "parking_lot 0.11.1",
+ "pin-project-lite 0.2.0",
  "signal-hook-registry",
  "slab",
  "tokio-macros",
@@ -2815,9 +2894,9 @@ dependencies = [
 
 [[package]]
 name = "tokio-macros"
-version = "0.2.6"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e44da00bfc73a25f814cd8d7e57a68a5c31b74b3152a0a1d1f590c97ed06265a"
+checksum = "21d30fdbb5dc2d8f91049691aa1a9d4d4ae422a21c334ce8936e5886d30c5c45"
 dependencies = [
  "proc-macro2 1.0.24",
  "quote 1.0.7",
@@ -2826,30 +2905,30 @@ dependencies = [
 
 [[package]]
 name = "tokio-util"
-version = "0.2.0"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "571da51182ec208780505a32528fc5512a8fe1443ab960b3f2f3ef093cd16930"
+checksum = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499"
 dependencies = [
- "bytes",
+ "bytes 0.5.6",
  "futures-core",
  "futures-sink",
  "log",
- "pin-project-lite",
- "tokio",
+ "pin-project-lite 0.1.11",
+ "tokio 0.2.23",
 ]
 
 [[package]]
 name = "tokio-util"
-version = "0.3.1"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499"
+checksum = "73af76301319bcacf00d26d3c75534ef248dcad7ceaf36d93ec902453c3b1706"
 dependencies = [
- "bytes",
+ "bytes 0.6.0",
  "futures-core",
  "futures-sink",
  "log",
- "pin-project-lite",
- "tokio",
+ "pin-project-lite 0.1.11",
+ "tokio 0.3.4",
 ]
 
 [[package]]
@@ -2863,14 +2942,14 @@ dependencies = [
 
 [[package]]
 name = "tower"
-version = "0.3.1"
-source = "git+https://github.com/tower-rs/tower?rev=1a84543#1a845433177b31ebc6daff765ee81468c42d6676"
+version = "0.4.0"
+source = "git+https://github.com/tower-rs/tower?rev=5e1e07744820028877654c336a3b9fe057bf46f1#5e1e07744820028877654c336a3b9fe057bf46f1"
 dependencies = [
  "futures-core",
  "futures-util",
  "hdrhistogram",
- "pin-project 0.4.27",
- "tokio",
+ "pin-project 1.0.2",
+ "tokio 0.3.4",
  "tower-layer",
  "tower-service",
  "tracing",
@@ -2886,7 +2965,7 @@ dependencies = [
  "futures-core",
  "pin-project 0.4.27",
  "rand 0.7.3",
- "tokio",
+ "tokio 0.3.4",
  "tower",
  "tower-fallback",
  "tracing",
@@ -2900,7 +2979,7 @@ version = "0.1.0"
 dependencies = [
  "futures-core",
  "pin-project 0.4.27",
- "tokio",
+ "tokio 0.3.4",
  "tower",
  "tracing",
  "zebra-test",
@@ -2909,7 +2988,7 @@ dependencies = [
 [[package]]
 name = "tower-layer"
 version = "0.3.0"
-source = "git+https://github.com/tower-rs/tower?rev=1a84543#1a845433177b31ebc6daff765ee81468c42d6676"
+source = "git+https://github.com/tower-rs/tower?rev=5e1e07744820028877654c336a3b9fe057bf46f1#5e1e07744820028877654c336a3b9fe057bf46f1"
 
 [[package]]
 name = "tower-service"
@@ -2937,7 +3016,7 @@ checksum = "b0987850db3733619253fe60e17cb59b82d37c7e6c0236bb81e4d6b87c879f27"
 dependencies = [
  "cfg-if 0.1.10",
  "log",
- "pin-project-lite",
+ "pin-project-lite 0.1.11",
  "tracing-attributes",
  "tracing-core",
 ]
@@ -3045,7 +3124,7 @@ dependencies = [
  "serde",
  "serde_json",
  "sharded-slab",
- "smallvec 1.4.2",
+ "smallvec 1.5.0",
  "thread_local",
  "tracing",
  "tracing-core",
@@ -3088,9 +3167,9 @@ dependencies = [
 
 [[package]]
 name = "unicode-normalization"
-version = "0.1.14"
+version = "0.1.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b7f98e67a4d84f730d343392f9bfff7d21e3fca562b9cb7a43b768350beeddc6"
+checksum = "a13e63ab62dbe32aeee58d1c5408d35c36c392bba5d9d3142287219721afe606"
 dependencies = [
  "tinyvec",
 ]
@@ -3340,7 +3419,7 @@ dependencies = [
  "serde",
  "spandoc",
  "thiserror",
- "tokio",
+ "tokio 0.3.4",
  "tower",
  "tower-batch",
  "tower-fallback",
@@ -3361,7 +3440,7 @@ version = "3.0.0-alpha.0"
 dependencies = [
  "bitflags",
  "byteorder",
- "bytes",
+ "bytes 0.6.0",
  "chrono",
  "futures",
  "hex",
@@ -3373,8 +3452,8 @@ dependencies = [
  "rand 0.7.3",
  "serde",
  "thiserror",
- "tokio",
- "tokio-util 0.2.0",
+ "tokio 0.3.4",
+ "tokio-util 0.5.0",
  "tower",
  "tracing",
  "tracing-error",
@@ -3419,7 +3498,7 @@ dependencies = [
  "spandoc",
  "tempdir",
  "thiserror",
- "tokio",
+ "tokio 0.3.4",
  "tower",
  "tracing",
  "tracing-error",
@@ -3442,7 +3521,7 @@ dependencies = [
  "spandoc",
  "tempdir",
  "thiserror",
- "tokio",
+ "tokio 0.3.4",
  "tower",
  "tracing",
  "tracing-error",
@@ -3484,7 +3563,7 @@ dependencies = [
  "serde",
  "tempdir",
  "thiserror",
- "tokio",
+ "tokio 0.3.4",
  "toml",
  "tower",
  "tracing",
diff --git a/Cargo.toml b/Cargo.toml
index 13fcf98f8ec..1eb6e23ebbf 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -21,4 +21,4 @@ panic = "abort"
 panic = "abort"
 
 [patch.crates-io]
-tower = { git = "https://github.com/tower-rs/tower", rev = "1a84543" }
+tower = { git = "https://github.com/tower-rs/tower", rev = "5e1e07744820028877654c336a3b9fe057bf46f1" }
diff --git a/tower-batch/Cargo.toml b/tower-batch/Cargo.toml
index 8a65493c3a0..c5063db488a 100644
--- a/tower-batch/Cargo.toml
+++ b/tower-batch/Cargo.toml
@@ -6,8 +6,8 @@ license = "MIT"
 edition = "2018"
 
 [dependencies]
-tokio = { version = "0.2.22", features = ["time", "sync", "stream", "tracing"] }
-tower = { version = "0.3", features = ["util", "buffer"] }
+tokio = { version = "0.3", features = ["time", "sync", "stream", "tracing"] }
+tower = { version = "0.4", features = ["util", "buffer"] }
 futures-core = "0.3.6"
 pin-project = "0.4.27"
 tracing = "0.1.21"
@@ -17,7 +17,7 @@ futures = "0.3.7"
 [dev-dependencies]
 ed25519-zebra = "2.1.0"
 rand = "0.7"
-tokio = { version = "0.2", features = ["full"]}
+tokio = { version = "0.3", features = ["full"]}
 tracing = "0.1.21"
 zebra-test = { path = "../zebra-test/" }
 tower-fallback = { path = "../tower-fallback/" }
diff --git a/tower-batch/src/lib.rs b/tower-batch/src/lib.rs
index 9ed70a9eea3..6eea4fb52c1 100644
--- a/tower-batch/src/lib.rs
+++ b/tower-batch/src/lib.rs
@@ -89,6 +89,7 @@ pub mod error;
 pub mod future;
 mod layer;
 mod message;
+mod semaphore;
 mod service;
 mod worker;
 
diff --git a/tower-batch/src/message.rs b/tower-batch/src/message.rs
index dc73a6adb80..287076f26ce 100644
--- a/tower-batch/src/message.rs
+++ b/tower-batch/src/message.rs
@@ -7,6 +7,7 @@ pub(crate) struct Message<Request, Fut> {
     pub(crate) request: Request,
     pub(crate) tx: Tx<Fut>,
     pub(crate) span: tracing::Span,
+    pub(super) _permit: crate::semaphore::Permit,
 }
 
 /// Response sender
diff --git a/tower-batch/src/semaphore.rs b/tower-batch/src/semaphore.rs
new file mode 100644
index 00000000000..cdab56d0784
--- /dev/null
+++ b/tower-batch/src/semaphore.rs
@@ -0,0 +1,78 @@
+// Copied from tower/src/semaphore.rs
+// When/if tower-batch is upstreamed, delete this file
+// and use the common tower semaphore implementation
+
+pub(crate) use self::sync::OwnedSemaphorePermit as Permit;
+use futures_core::ready;
+use std::{
+    fmt,
+    future::Future,
+    mem,
+    pin::Pin,
+    sync::Arc,
+    task::{Context, Poll},
+};
+use tokio::sync;
+
+#[derive(Debug)]
+pub(crate) struct Semaphore {
+    semaphore: Arc<sync::Semaphore>,
+    state: State,
+}
+
+enum State {
+    Waiting(Pin<Box<dyn Future<Output = Permit> + Send + Sync + 'static>>),
+    Ready(Permit),
+    Empty,
+}
+
+impl Semaphore {
+    pub(crate) fn new(permits: usize) -> Self {
+        Self {
+            semaphore: Arc::new(sync::Semaphore::new(permits)),
+            state: State::Empty,
+        }
+    }
+
+    pub(crate) fn poll_acquire(&mut self, cx: &mut Context<'_>) -> Poll<()> {
+        loop {
+            self.state = match self.state {
+                State::Ready(_) => return Poll::Ready(()),
+                State::Waiting(ref mut fut) => {
+                    let permit = ready!(Pin::new(fut).poll(cx));
+                    State::Ready(permit)
+                }
+                State::Empty => State::Waiting(Box::pin(self.semaphore.clone().acquire_owned())),
+            };
+        }
+    }
+
+    pub(crate) fn take_permit(&mut self) -> Option<Permit> {
+        if let State::Ready(permit) = mem::replace(&mut self.state, State::Empty) {
+            return Some(permit);
+        }
+        None
+    }
+}
+
+impl Clone for Semaphore {
+    fn clone(&self) -> Self {
+        Self {
+            semaphore: self.semaphore.clone(),
+            state: State::Empty,
+        }
+    }
+}
+
+impl fmt::Debug for State {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            State::Waiting(_) => f
+                .debug_tuple("State::Waiting")
+                .field(&format_args!("..."))
+                .finish(),
+            State::Ready(ref r) => f.debug_tuple("State::Ready").field(&r).finish(),
+            State::Empty => f.debug_tuple("State::Empty").finish(),
+        }
+    }
+}
diff --git a/tower-batch/src/service.rs b/tower-batch/src/service.rs
index 28c4e7132ed..6ddeb2157b1 100644
--- a/tower-batch/src/service.rs
+++ b/tower-batch/src/service.rs
@@ -5,6 +5,7 @@ use super::{
     BatchControl,
 };
 
+use crate::semaphore::Semaphore;
 use futures_core::ready;
 use std::task::{Context, Poll};
 use tokio::sync::{mpsc, oneshot};
@@ -18,7 +19,19 @@ pub struct Batch<T, Request>
 where
     T: Service<BatchControl<Request>>,
 {
-    tx: mpsc::Sender<Message<Request, T::Future>>,
+    // Note: this actually _is_ bounded, but rather than using Tokio's unbounded
+    // channel, we use tokio's semaphore separately to implement the bound.
+    tx: mpsc::UnboundedSender<Message<Request, T::Future>>,
+    // When the buffer's channel is full, we want to exert backpressure in
+    // `poll_ready`, so that callers such as load balancers could choose to call
+    // another service rather than waiting for buffer capacity.
+    //
+    // Unfortunately, this can't be done easily using Tokio's bounded MPSC
+    // channel, because it doesn't expose a polling-based interface, only an
+    // `async fn ready`, which borrows the sender. Therefore, we implement our
+    // own bounded MPSC on top of the unbounded channel, using a semaphore to
+    // limit how many items are in the channel.
+    semaphore: Semaphore,
     handle: Handle,
 }
 
@@ -45,10 +58,16 @@ where
         Request: Send + 'static,
     {
         // XXX(hdevalence): is this bound good
-        let (tx, rx) = mpsc::channel(1);
+        let bound = 1;
+        let (tx, rx) = mpsc::unbounded_channel();
         let (handle, worker) = Worker::new(service, rx, max_items, max_latency);
         tokio::spawn(worker.run());
-        Batch { tx, handle }
+        let semaphore = Semaphore::new(bound);
+        Batch {
+            tx,
+            handle,
+            semaphore,
+        }
     }
 
     fn get_worker_error(&self) -> crate::BoxError {
@@ -66,40 +85,43 @@ where
     type Future = ResponseFuture<T::Future>;
 
     fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
-        // If the inner service has errored, then we error here.
-        if ready!(self.tx.poll_ready(cx)).is_err() {
-            Poll::Ready(Err(self.get_worker_error()))
-        } else {
-            Poll::Ready(Ok(()))
+        // First, check if the worker is still alive.
+        if self.tx.is_closed() {
+            // If the inner service has errored, then we error here.
+            return Poll::Ready(Err(self.get_worker_error()));
         }
+
+        // Then, poll to acquire a semaphore permit. If we acquire a permit,
+        // then there's enough buffer capacity to send a new request. Otherwise,
+        // we need to wait for capacity.
+        ready!(self.semaphore.poll_acquire(cx));
+
+        Poll::Ready(Ok(()))
     }
 
     fn call(&mut self, request: Request) -> Self::Future {
-        // TODO:
-        // ideally we'd poll_ready again here so we don't allocate the oneshot
-        // if the try_send is about to fail, but sadly we can't call poll_ready
-        // outside of task context.
-        let (tx, rx) = oneshot::channel();
+        tracing::trace!("sending request to buffer worker");
+        let _permit = self
+            .semaphore
+            .take_permit()
+            .expect("buffer full; poll_ready must be called first");
 
         // get the current Span so that we can explicitly propagate it to the worker
         // if we didn't do this, events on the worker related to this span wouldn't be counted
         // towards that span since the worker would have no way of entering it.
         let span = tracing::Span::current();
-        tracing::trace!(parent: &span, "sending request to batch worker");
-        match self.tx.try_send(Message { request, span, tx }) {
-            Err(mpsc::error::TrySendError::Closed(_)) => {
-                ResponseFuture::failed(self.get_worker_error())
-            }
-            Err(mpsc::error::TrySendError::Full(_)) => {
-                // When `mpsc::Sender::poll_ready` returns `Ready`, a slot
-                // in the channel is reserved for the handle. Other `Sender`
-                // handles may not send a message using that slot. This
-                // guarantees capacity for `request`.
-                //
-                // Given this, the only way to hit this code path is if
-                // `poll_ready` has not been called & `Ready` returned.
-                panic!("buffer full; poll_ready must be called first");
-            }
+
+        // If we've made it here, then a semaphore permit has already been
+        // acquired, so we can freely allocate a oneshot.
+        let (tx, rx) = oneshot::channel();
+
+        match self.tx.send(Message {
+            request,
+            span,
+            tx,
+            _permit,
+        }) {
+            Err(_) => ResponseFuture::failed(self.get_worker_error()),
             Ok(_) => ResponseFuture::new(rx),
         }
     }
@@ -113,6 +135,7 @@ where
         Self {
             tx: self.tx.clone(),
             handle: self.handle.clone(),
+            semaphore: self.semaphore.clone(),
         }
     }
 }
diff --git a/tower-batch/src/worker.rs b/tower-batch/src/worker.rs
index 8d4ab367e2f..ff79395112a 100644
--- a/tower-batch/src/worker.rs
+++ b/tower-batch/src/worker.rs
@@ -1,19 +1,21 @@
-use super::{
-    error::{Closed, ServiceError},
-    message::{self, Message},
-    BatchControl,
-};
+use std::sync::{Arc, Mutex};
+
 use futures::future::TryFutureExt;
 use pin_project::pin_project;
-use std::sync::{Arc, Mutex};
 use tokio::{
     stream::StreamExt,
     sync::mpsc,
-    time::{delay_for, Delay},
+    time::{sleep, Sleep},
 };
 use tower::{Service, ServiceExt};
 use tracing_futures::Instrument;
 
+use super::{
+    error::{Closed, ServiceError},
+    message::{self, Message},
+    BatchControl,
+};
+
 /// Task that handles processing the buffer. This type should not be used
 /// directly, instead `Buffer` requires an `Executor` that can accept this task.
 ///
@@ -28,7 +30,7 @@ where
     T: Service<BatchControl<Request>>,
     T::Error: Into<crate::BoxError>,
 {
-    rx: mpsc::Receiver<Message<Request, T::Future>>,
+    rx: mpsc::UnboundedReceiver<Message<Request, T::Future>>,
     service: T,
     failed: Option<ServiceError>,
     handle: Handle,
@@ -49,7 +51,7 @@ where
 {
     pub(crate) fn new(
         service: T,
-        rx: mpsc::Receiver<Message<Request, T::Future>>,
+        rx: mpsc::UnboundedReceiver<Message<Request, T::Future>>,
         max_items: usize,
         max_latency: std::time::Duration,
     ) -> (Handle, Worker<T, Request>) {
@@ -103,12 +105,11 @@ where
     }
 
     pub async fn run(mut self) {
-        use futures::future::Either::{Left, Right};
         // The timer is started when the first entry of a new batch is
         // submitted, so that the batch latency of all entries is at most
         // self.max_latency. However, we don't keep the timer running unless
         // there is a pending request to prevent wakeups on idle services.
-        let mut timer: Option<Delay> = None;
+        let mut timer: Option<Sleep> = None;
         let mut pending_items = 0usize;
         loop {
             match timer {
@@ -120,40 +121,42 @@ where
                             // Apply the provided span to request processing
                             .instrument(span)
                             .await;
-                        timer = Some(delay_for(self.max_latency));
+                        timer = Some(sleep(self.max_latency));
                         pending_items = 1;
                     }
                     // No more messages, ever.
                     None => return,
                 },
-                Some(delay) => {
+                Some(mut sleep) => {
                     // Wait on either a new message or the batch timer.
-                    match futures::future::select(self.rx.next(), delay).await {
-                        Left((Some(msg), delay)) => {
-                            let span = msg.span;
-                            self.process_req(msg.request, msg.tx)
-                                // Apply the provided span to request processing.
-                                .instrument(span)
-                                .await;
-                            pending_items += 1;
-                            // Check whether we have too many pending items.
-                            if pending_items >= self.max_items {
-                                // XXX(hdevalence): what span should instrument this?
-                                self.flush_service().await;
-                                // Now we have an empty batch.
-                                timer = None;
-                                pending_items = 0;
-                            } else {
-                                // The timer is still running, set it back!
-                                timer = Some(delay);
+                    tokio::select! {
+                        maybe_msg = self.rx.recv() => match maybe_msg {
+                            Some(msg) => {
+                                let span = msg.span;
+                                self.process_req(msg.request, msg.tx)
+                                    // Apply the provided span to request processing.
+                                    .instrument(span)
+                                    .await;
+                                pending_items += 1;
+                                // Check whether we have too many pending items.
+                                if pending_items >= self.max_items {
+                                    // XXX(hdevalence): what span should instrument this?
+                                    self.flush_service().await;
+                                    // Now we have an empty batch.
+                                    timer = None;
+                                    pending_items = 0;
+                                } else {
+                                    // The timer is still running, set it back!
+                                    timer = Some(sleep);
+                                }
                             }
-                        }
-                        // No more messages, ever.
-                        Left((None, _delay)) => {
-                            return;
-                        }
-                        // The batch timer elapsed.
-                        Right(((), _next)) => {
+                            None => {
+                                // No more messages, ever.
+                                return;
+                            }
+                        },
+                        () = &mut sleep => {
+                            // The batch timer elapsed.
                             // XXX(hdevalence): what span should instrument this?
                             self.flush_service().await;
                             timer = None;
diff --git a/tower-batch/tests/ed25519.rs b/tower-batch/tests/ed25519.rs
index a5fb52888fc..947c073680e 100644
--- a/tower-batch/tests/ed25519.rs
+++ b/tower-batch/tests/ed25519.rs
@@ -10,7 +10,7 @@ use color_eyre::{eyre::eyre, Report};
 use ed25519_zebra::*;
 use futures::stream::{FuturesUnordered, StreamExt};
 use rand::thread_rng;
-use tokio::sync::broadcast::{channel, RecvError, Sender};
+use tokio::sync::broadcast::{channel, error::RecvError, Sender};
 use tower::{Service, ServiceExt};
 use tower_batch::{Batch, BatchControl};
 use tower_fallback::Fallback;
diff --git a/tower-fallback/Cargo.toml b/tower-fallback/Cargo.toml
index eb1cb3a949f..67f452589b1 100644
--- a/tower-fallback/Cargo.toml
+++ b/tower-fallback/Cargo.toml
@@ -6,11 +6,11 @@ license = "MIT"
 edition = "2018"
 
 [dependencies]
-tower = "0.3"
+tower = "0.4"
 futures-core = "0.3.6"
 pin-project = "0.4.27"
 tracing = "0.1"
 
 [dev-dependencies]
 zebra-test = { path = "../zebra-test/" }
-tokio = { version = "0.2", features = ["full"]}
+tokio = { version = "0.3", features = ["full"]}
diff --git a/zebra-consensus/Cargo.toml b/zebra-consensus/Cargo.toml
index eeae83d0c9c..db7facb801d 100644
--- a/zebra-consensus/Cargo.toml
+++ b/zebra-consensus/Cargo.toml
@@ -19,8 +19,8 @@ futures = "0.3.7"
 futures-util = "0.3.6"
 metrics = "0.12"
 thiserror = "1.0.22"
-tokio = { version = "0.2.22", features = ["time", "sync", "stream", "tracing"] }
-tower = { version = "0.3", features = ["timeout", "util", "buffer"] }
+tokio = { version = "0.3", features = ["time", "sync", "stream", "tracing"] }
+tower = { version = "0.4", features = ["timeout", "util", "buffer"] }
 tower-util = "0.3"
 tracing = "0.1.21"
 tracing-futures = "0.2.4"
@@ -34,7 +34,7 @@ zebra-script = { path = "../zebra-script" }
 [dev-dependencies]
 rand = "0.7"
 spandoc = "0.2"
-tokio = { version = "0.2", features = ["full"] }
+tokio = { version = "0.3", features = ["full"] }
 tracing-error = "0.1.2"
 tracing-subscriber = "0.2.15"
 
diff --git a/zebra-consensus/src/primitives/redjubjub.rs b/zebra-consensus/src/primitives/redjubjub.rs
index ac9021f20f0..472312ae50d 100644
--- a/zebra-consensus/src/primitives/redjubjub.rs
+++ b/zebra-consensus/src/primitives/redjubjub.rs
@@ -14,7 +14,7 @@ use futures::future::{ready, Ready};
 use once_cell::sync::Lazy;
 use rand::thread_rng;
 
-use tokio::sync::broadcast::{channel, RecvError, Sender};
+use tokio::sync::broadcast::{channel, error::RecvError, Sender};
 use tower::{util::ServiceFn, Service};
 use tower_batch::{Batch, BatchControl};
 use tower_fallback::Fallback;
diff --git a/zebra-network/Cargo.toml b/zebra-network/Cargo.toml
index 19bca8f56c1..3a21a702a15 100644
--- a/zebra-network/Cargo.toml
+++ b/zebra-network/Cargo.toml
@@ -10,7 +10,7 @@ edition = "2018"
 [dependencies]
 bitflags = "1.2"
 byteorder = "1.3"
-bytes = "0.5"
+bytes = "0.6"
 chrono = "0.4"
 hex = "0.4"
 # indexmap has rayon support for parallel iteration,
@@ -22,9 +22,9 @@ serde = { version = "1", features = ["serde_derive"] }
 thiserror = "1"
 
 futures = "0.3"
-tokio = { version = "0.2.22", features = ["net", "time", "stream", "tracing", "macros"] }
-tokio-util = { version = "0.2", features = ["codec"] }
-tower = { version = "0.3", features = ["retry", "discover", "load", "load-shed", "timeout", "util", "buffer"] }
+tokio = { version = "0.3", features = ["net", "time", "stream", "tracing", "macros"] }
+tokio-util = { version = "0.5", features = ["codec"] }
+tower = { version = "0.4", features = ["retry", "discover", "load", "load-shed", "timeout", "util", "buffer"] }
 
 metrics = "0.12"
 tracing = "0.1"
diff --git a/zebra-network/src/isolated.rs b/zebra-network/src/isolated.rs
index 65320b5f99b..1d6132f20e2 100644
--- a/zebra-network/src/isolated.rs
+++ b/zebra-network/src/isolated.rs
@@ -93,7 +93,7 @@ mod tests {
         use futures::stream::StreamExt;
         use tokio_util::codec::Framed;
 
-        let mut listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
         let listen_addr = listener.local_addr().unwrap();
 
         let conn = tokio::net::TcpStream::connect(listen_addr).await.unwrap();
diff --git a/zebra-network/src/peer/connection.rs b/zebra-network/src/peer/connection.rs
index 4893248954f..014050e4da3 100644
--- a/zebra-network/src/peer/connection.rs
+++ b/zebra-network/src/peer/connection.rs
@@ -23,7 +23,7 @@ use futures::{
     prelude::*,
     stream::Stream,
 };
-use tokio::time::{delay_for, Delay};
+use tokio::time::{sleep, Sleep};
 use tower::Service;
 use tracing_futures::Instrument;
 
@@ -221,7 +221,7 @@ pub struct Connection<S, Tx> {
     /// A timeout for a client request. This is stored separately from
     /// State so that we can move the future out of it independently of
     /// other state handling.
-    pub(super) request_timer: Option<Delay>,
+    pub(super) request_timer: Option<Sleep>,
     pub(super) svc: S,
     pub(super) client_rx: mpsc::Receiver<ClientRequest>,
     /// A slot for an error shared between the Connection and the Client that uses it.
@@ -553,7 +553,7 @@ where
         } {
             Ok(new_state) => {
                 self.state = new_state;
-                self.request_timer = Some(delay_for(constants::REQUEST_TIMEOUT));
+                self.request_timer = Some(sleep(constants::REQUEST_TIMEOUT));
             }
             Err(e) => self.fail_with(e),
         }
diff --git a/zebra-network/src/peer_set/initialize.rs b/zebra-network/src/peer_set/initialize.rs
index f679e7b4315..ae273e1fa5b 100644
--- a/zebra-network/src/peer_set/initialize.rs
+++ b/zebra-network/src/peer_set/initialize.rs
@@ -211,7 +211,7 @@ where
     S: Service<(TcpStream, SocketAddr), Response = peer::Client, Error = BoxError> + Clone,
     S::Future: Send + 'static,
 {
-    let mut listener = TcpListener::bind(addr).await?;
+    let listener = TcpListener::bind(addr).await?;
     let local_addr = listener.local_addr()?;
     info!("Opened Zcash protocol endpoint at {}", local_addr);
     loop {
diff --git a/zebra-network/src/peer_set/inventory_registry.rs b/zebra-network/src/peer_set/inventory_registry.rs
index da119b9f278..5d211e479ed 100644
--- a/zebra-network/src/peer_set/inventory_registry.rs
+++ b/zebra-network/src/peer_set/inventory_registry.rs
@@ -1,8 +1,7 @@
 //! Inventory Registry Implementation
 //!
 //! [RFC]: https://zebra.zfnd.org/dev/rfcs/0003-inventory-tracking.html
-use crate::{protocol::external::InventoryHash, BoxError};
-use futures::Stream;
+
 use std::{
     collections::{HashMap, HashSet},
     net::SocketAddr,
@@ -10,17 +9,20 @@ use std::{
     task::{Context, Poll},
     time::Duration,
 };
+
+use futures::{Stream, StreamExt};
 use tokio::{
     sync::broadcast,
     time::{self, Interval},
 };
 
+use crate::{protocol::external::InventoryHash, BoxError};
+
 /// An Inventory Registry for tracking recent inventory advertisements by peer.
 ///
 /// For more details please refer to the [RFC].
 ///
 /// [RFC]: https://zebra.zfnd.org/dev/rfcs/0003-inventory-tracking.html
-#[derive(Debug)]
 pub struct InventoryRegistry {
     /// Map tracking the inventory advertisements from the current interval
     /// period
@@ -28,18 +30,33 @@ pub struct InventoryRegistry {
     /// Map tracking inventory advertisements from the previous interval period
     prev: HashMap<InventoryHash, HashSet<SocketAddr>>,
     /// Stream of incoming inventory hashes to register
-    inv_stream: broadcast::Receiver<(InventoryHash, SocketAddr)>,
+    inv_stream: Pin<
+        Box<
+            dyn Stream<Item = Result<(InventoryHash, SocketAddr), broadcast::error::RecvError>>
+                + Send
+                + 'static,
+        >,
+    >,
     /// Interval tracking how frequently we should rotate our maps
     interval: Interval,
 }
 
+impl std::fmt::Debug for InventoryRegistry {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("InventoryRegistry")
+            .field("current", &self.current)
+            .field("prev", &self.prev)
+            .finish()
+    }
+}
+
 impl InventoryRegistry {
     /// Returns an Inventory Registry
     pub fn new(inv_stream: broadcast::Receiver<(InventoryHash, SocketAddr)>) -> Self {
         Self {
             current: Default::default(),
             prev: Default::default(),
-            inv_stream,
+            inv_stream: inv_stream.into_stream().boxed(),
             interval: time::interval(Duration::from_secs(75)),
         }
     }
@@ -60,7 +77,7 @@ impl InventoryRegistry {
     /// - rotates HashMaps based on interval events
     /// - drains the inv_stream channel and registers all advertised inventory
     pub fn poll_inventory(&mut self, cx: &mut Context<'_>) -> Result<(), BoxError> {
-        while let Poll::Ready(_) = self.interval.poll_tick(cx) {
+        while let Poll::Ready(_) = Pin::new(&mut self.interval).poll_next(cx) {
             self.rotate();
         }
 
@@ -80,7 +97,7 @@ impl InventoryRegistry {
         // rather than propagating it through the peer set's Service::poll_ready
         // implementation, where reporting a failure means reporting a permanent
         // failure of the peer set.
-        use broadcast::RecvError;
+        use broadcast::error::RecvError;
         while let Poll::Ready(Some(channel_result)) = Pin::new(&mut self.inv_stream).poll_next(cx) {
             match channel_result {
                 Ok((hash, addr)) => self.register(hash, addr),
diff --git a/zebra-network/src/protocol/external/codec.rs b/zebra-network/src/protocol/external/codec.rs
index ac3c56adcb3..e8925352a9d 100644
--- a/zebra-network/src/protocol/external/codec.rs
+++ b/zebra-network/src/protocol/external/codec.rs
@@ -104,11 +104,10 @@ impl Builder {
 
 // ======== Encoding =========
 
-impl Encoder for Codec {
-    type Item = Message;
+impl Encoder<Message> for Codec {
     type Error = Error;
 
-    fn encode(&mut self, item: Self::Item, dst: &mut BytesMut) -> Result<(), Self::Error> {
+    fn encode(&mut self, item: Message, dst: &mut BytesMut) -> Result<(), Self::Error> {
         use Error::Parse;
         // XXX(HACK): this is inefficient and does an extra allocation.
         // instead, we should have a size estimator for the message, reserve
@@ -587,7 +586,7 @@ mod tests {
         let services = PeerServices::NODE_NETWORK;
         let timestamp = Utc.timestamp(1_568_000_000, 0);
 
-        let mut rt = Runtime::new().unwrap();
+        let rt = Runtime::new().unwrap();
 
         let v = Message::Version {
             version: crate::constants::CURRENT_VERSION,
@@ -634,7 +633,7 @@ mod tests {
     fn filterload_message_round_trip() {
         zebra_test::init();
 
-        let mut rt = Runtime::new().unwrap();
+        let rt = Runtime::new().unwrap();
 
         let v = Message::FilterLoad {
             filter: Filter(vec![0; 35999]),
@@ -670,7 +669,7 @@ mod tests {
     fn filterload_message_too_large_round_trip() {
         zebra_test::init();
 
-        let mut rt = Runtime::new().unwrap();
+        let rt = Runtime::new().unwrap();
 
         let v = Message::FilterLoad {
             filter: Filter(vec![0; 40000]),
@@ -706,7 +705,7 @@ mod tests {
         use zebra_chain::serialization::ZcashDeserializeInto;
         zebra_test::init();
 
-        let mut rt = Runtime::new().unwrap();
+        let rt = Runtime::new().unwrap();
 
         // make tests with a Tx message
         let tx = zebra_test::vectors::DUMMY_TX1
diff --git a/zebra-state/Cargo.toml b/zebra-state/Cargo.toml
index fa7d8ab0ade..fb6afc6806f 100644
--- a/zebra-state/Cargo.toml
+++ b/zebra-state/Cargo.toml
@@ -18,11 +18,11 @@ serde = { version = "1", features = ["serde_derive"] }
 
 futures = "0.3.7"
 metrics = "0.12"
-tower = { version = "0.3.1", features = ["buffer", "util"] }
+tower = { version = "0.4", features = ["buffer", "util"] }
 tracing = "0.1"
 tracing-error = "0.1.2"
 thiserror = "1.0.22"
-tokio = { version = "0.2.22", features = ["sync"] }
+tokio = { version = "0.3", features = ["sync"] }
 displaydoc = "0.1.7"
 rocksdb = "0.15.0"
 tempdir = "0.3.7"
@@ -34,6 +34,6 @@ zebra-test = { path = "../zebra-test/" }
 once_cell = "1.5"
 spandoc = "0.2"
 tempdir = "0.3.7"
-tokio = { version = "0.2.22", features = ["full"] }
+tokio = { version = "0.3", features = ["full"] }
 proptest = "0.10.1"
 primitive-types = "0.7.3"
diff --git a/zebra-test/Cargo.toml b/zebra-test/Cargo.toml
index 3394e313992..adc40b0540f 100644
--- a/zebra-test/Cargo.toml
+++ b/zebra-test/Cargo.toml
@@ -10,7 +10,7 @@ edition = "2018"
 [dependencies]
 hex = "0.4.2"
 lazy_static = "1.4.0"
-tower = { version = "0.3.1", features = ["util"] }
+tower = { version = "0.4", features = ["util"] }
 futures = "0.3.7"
 color-eyre = "0.5.7"
 tracing = "0.1.21"
@@ -25,4 +25,4 @@ proptest = "0.10.1"
 tempdir = "0.3.7"
 
 [dev-dependencies]
-tokio = { version = "0.2", features = ["full"] }
+tokio = { version = "0.3", features = ["full"] }
diff --git a/zebrad/Cargo.toml b/zebrad/Cargo.toml
index d88d7d29afa..49b8452af42 100644
--- a/zebrad/Cargo.toml
+++ b/zebrad/Cargo.toml
@@ -21,8 +21,8 @@ rand = "0.7"
 
 hyper = "0.13.9"
 futures = "0.3"
-tokio = { version = "0.2.22", features = ["time", "rt-threaded", "stream", "macros", "tracing", "signal"] }
-tower = { version = "0.3", features = ["hedge", "limit"] }
+tokio = { version = "0.3", features = ["time", "rt-multi-thread", "stream", "macros", "tracing", "signal"] }
+tower = { version = "0.4", features = ["hedge", "limit"] }
 pin-project = "0.4.23"
 
 color-eyre = { version = "0.5.7", features = ["issue-url"] }
diff --git a/zebrad/src/components/sync.rs b/zebrad/src/components/sync.rs
index d758f1c2393..1cbf944cf80 100644
--- a/zebrad/src/components/sync.rs
+++ b/zebrad/src/components/sync.rs
@@ -5,7 +5,7 @@ use futures::{
     future::FutureExt,
     stream::{FuturesUnordered, StreamExt},
 };
-use tokio::time::delay_for;
+use tokio::time::sleep;
 use tower::{
     builder::ServiceBuilder, hedge::Hedge, limit::ConcurrencyLimit, retry::Retry, timeout::Timeout,
     Service, ServiceExt,
@@ -153,7 +153,7 @@ where
         // due to protocol limitations
         self.request_genesis().await?;
 
-        // Distinguishes a restart from a start, so we don't delay when starting
+        // Distinguishes a restart from a start, so we don't sleep when starting
         // the sync process, but we can keep restart logic in one place.
         let mut started_once = false;
 
@@ -163,7 +163,7 @@ where
                 self.prospective_tips = HashSet::new();
                 self.downloads.cancel_all();
                 self.update_metrics();
-                delay_for(SYNC_RESTART_TIMEOUT).await;
+                sleep(SYNC_RESTART_TIMEOUT).await;
             } else {
                 started_once = true;
             }
diff --git a/zebrad/src/components/tokio.rs b/zebrad/src/components/tokio.rs
index c0dc22d09cc..66131fe6f13 100644
--- a/zebrad/src/components/tokio.rs
+++ b/zebrad/src/components/tokio.rs
@@ -22,9 +22,8 @@ impl TokioComponent {
     pub fn new() -> Result<Self, FrameworkError> {
         Ok(Self {
             rt: Some(
-                tokio::runtime::Builder::new()
+                tokio::runtime::Builder::new_multi_thread()
                     .enable_all()
-                    .threaded_scheduler()
                     .build()
                     .unwrap(),
             ),

From ba3c19142c53f452ead8f2b1032be43be0dfd043 Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Thu, 19 Nov 2020 12:54:31 -0800
Subject: [PATCH 057/148] deps: update hyper, metrics to tokio 0.3

The metrics code becomes much simpler because the current version of the
metrics crate builds its own single-threaded runtime on a dedicated worker
thread, so no dependency on the main Zebra Tokio runtime is required.
---
 Cargo.lock                       | 516 +++++++++----------------------
 Cargo.toml                       |   4 +
 zebrad/Cargo.toml                |   6 +-
 zebrad/src/components/metrics.rs |  51 +--
 zebrad/src/components/sync.rs    |   4 +-
 5 files changed, 162 insertions(+), 419 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 489294e610e..346da0458fb 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -57,6 +57,15 @@ version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e"
 
+[[package]]
+name = "ahash"
+version = "0.3.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e8fd72866655d1904d6b0997d0b07ba561047d070fbe29de039031c641b61217"
+dependencies = [
+ "const-random",
+]
+
 [[package]]
 name = "ahash"
 version = "0.4.6"
@@ -81,7 +90,7 @@ version = "0.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"
 dependencies = [
- "winapi 0.3.9",
+ "winapi",
 ]
 
 [[package]]
@@ -90,14 +99,14 @@ version = "0.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2"
 dependencies = [
- "winapi 0.3.9",
+ "winapi",
 ]
 
 [[package]]
 name = "arc-swap"
-version = "0.4.7"
+version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4d25d88fd6b8041580a654f9d0c581a047baee2b3efee13275f2fc392fc75034"
+checksum = "0688b520bcc7498f6ca8fa006e8031d353e3fd4f51bd4a50fb03cc4230b28bd2"
 
 [[package]]
 name = "arrayref"
@@ -137,7 +146,7 @@ checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
 dependencies = [
  "hermit-abi",
  "libc",
- "winapi 0.3.9",
+ "winapi",
 ]
 
 [[package]]
@@ -181,6 +190,12 @@ version = "0.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cdcf67bb7ba7797a081cd19009948ab533af7c355d5caf1d08c777582d351e9c"
 
+[[package]]
+name = "beef"
+version = "0.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "474a626a67200bd107d44179bb3d4fc61891172d11696609264589be6a0e6a43"
+
 [[package]]
 name = "bincode"
 version = "1.3.1"
@@ -236,15 +251,6 @@ version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
 
-[[package]]
-name = "bitmaps"
-version = "2.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "031043d04099746d8db04daf1fa424b2bc8bd69d92b25962dcde24da39ab64a2"
-dependencies = [
- "typenum",
-]
-
 [[package]]
 name = "bitvec"
 version = "0.17.4"
@@ -431,7 +437,7 @@ dependencies = [
  "num-traits",
  "serde",
  "time",
- "winapi 0.3.9",
+ "winapi",
 ]
 
 [[package]]
@@ -460,15 +466,6 @@ dependencies = [
  "vec_map",
 ]
 
-[[package]]
-name = "cloudabi"
-version = "0.0.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f"
-dependencies = [
- "bitflags",
-]
-
 [[package]]
 name = "cloudabi"
 version = "0.1.0"
@@ -766,6 +763,17 @@ dependencies = [
  "syn 1.0.48",
 ]
 
+[[package]]
+name = "dashmap"
+version = "3.11.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0f260e2fc850179ef410018660006951c1b55b79e8087e87111a2c388994b9b5"
+dependencies = [
+ "ahash 0.3.8",
+ "cfg-if 0.1.10",
+ "num_cpus",
+]
+
 [[package]]
 name = "difference"
 version = "2.0.0"
@@ -807,7 +815,7 @@ checksum = "8e93d7f5705de3e49895a2b5e0b8855a1c27f080192ae9c32a6432d50741a57a"
 dependencies = [
  "libc",
  "redox_users",
- "winapi 0.3.9",
+ "winapi",
 ]
 
 [[package]]
@@ -821,12 +829,6 @@ dependencies = [
  "syn 1.0.48",
 ]
 
-[[package]]
-name = "dtoa"
-version = "0.4.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "134951f4028bdadb9b84baf4232681efbf277da25144b9b0ad65df75946c422b"
-
 [[package]]
 name = "ed25519-zebra"
 version = "1.0.1"
@@ -951,22 +953,6 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
 
-[[package]]
-name = "fuchsia-zircon"
-version = "0.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82"
-dependencies = [
- "bitflags",
- "fuchsia-zircon-sys",
-]
-
-[[package]]
-name = "fuchsia-zircon-sys"
-version = "0.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7"
-
 [[package]]
 name = "funty"
 version = "1.0.1"
@@ -1087,7 +1073,7 @@ dependencies = [
  "libc",
  "log",
  "rustc_version",
- "winapi 0.3.9",
+ "winapi",
 ]
 
 [[package]]
@@ -1177,9 +1163,8 @@ dependencies = [
 
 [[package]]
 name = "h2"
-version = "0.2.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5e4728fd124914ad25e99e3d15a9361a879f6620f63cb56bbb08f95abb97a535"
+version = "0.3.0"
+source = "git+https://github.com/hyperium/h2#cbbdd305b1afc1eaf19f2e3b26f9419048041e7d"
 dependencies = [
  "bytes 0.5.6",
  "fnv",
@@ -1189,8 +1174,8 @@ dependencies = [
  "http",
  "indexmap",
  "slab",
- "tokio 0.2.23",
- "tokio-util 0.3.1",
+ "tokio",
+ "tokio-util 0.4.0",
  "tracing",
  "tracing-futures",
 ]
@@ -1215,6 +1200,20 @@ dependencies = [
  "num-traits",
 ]
 
+[[package]]
+name = "hdrhistogram"
+version = "7.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b3c22708574c44e924720c5b3a116326c688e6d532f438c77c007ec8768644f9"
+dependencies = [
+ "base64 0.12.3",
+ "byteorder",
+ "crossbeam-channel 0.4.4",
+ "flate2",
+ "nom 5.1.2",
+ "num-traits",
+]
+
 [[package]]
 name = "heck"
 version = "0.3.1"
@@ -1283,9 +1282,8 @@ dependencies = [
 
 [[package]]
 name = "hyper"
-version = "0.13.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f6ad767baac13b44d4529fcf58ba2cd0995e36e7b435bc5b039de6f47e880dbf"
+version = "0.14.0-dev"
+source = "git+https://github.com/hyperium/hyper/?rev=ed2b22a7f66899d338691552fbcb6c0f2f4e06b9#ed2b22a7f66899d338691552fbcb6c0f2f4e06b9"
 dependencies = [
  "bytes 0.5.6",
  "futures-channel",
@@ -1299,7 +1297,7 @@ dependencies = [
  "itoa",
  "pin-project 1.0.2",
  "socket2",
- "tokio 0.2.23",
+ "tokio",
  "tower-service",
  "tracing",
  "want",
@@ -1322,20 +1320,6 @@ dependencies = [
  "unicode-normalization",
 ]
 
-[[package]]
-name = "im"
-version = "15.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "111c1983f3c5bb72732df25cddacee9b546d08325fb584b5ebd38148be7b0246"
-dependencies = [
- "bitmaps",
- "rand_core 0.5.1",
- "rand_xoshiro",
- "sized-chunks",
- "typenum",
- "version_check 0.9.2",
-]
-
 [[package]]
 name = "impl-codec"
 version = "0.4.2"
@@ -1367,7 +1351,7 @@ version = "0.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "48b0b7698c16bccca4617de8f9ede9f58ab44a68ad6b2a4920fd74e491de580d"
 dependencies = [
- "ahash",
+ "ahash 0.4.6",
  "itoa",
  "lazy_static",
  "log",
@@ -1386,15 +1370,6 @@ dependencies = [
  "cfg-if 1.0.0",
 ]
 
-[[package]]
-name = "iovec"
-version = "0.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e"
-dependencies = [
- "libc",
-]
-
 [[package]]
 name = "itoa"
 version = "0.4.6"
@@ -1425,16 +1400,6 @@ dependencies = [
  "subtle",
 ]
 
-[[package]]
-name = "kernel32-sys"
-version = "0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d"
-dependencies = [
- "winapi 0.2.8",
- "winapi-build",
-]
-
 [[package]]
 name = "lazy_static"
 version = "1.4.0"
@@ -1447,6 +1412,19 @@ version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
 
+[[package]]
+name = "lexical-core"
+version = "0.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "db65c6da02e61f55dae90a0ae427b2a5f6b3e8db09f58d10efab23af92592616"
+dependencies = [
+ "arrayvec 0.5.2",
+ "bitflags",
+ "cfg-if 0.1.10",
+ "ryu",
+ "static_assertions",
+]
+
 [[package]]
 name = "libc"
 version = "0.2.80"
@@ -1460,7 +1438,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f2b111a074963af1d37a139918ac6d49ad1d0d5e47f72fd55388619691a7d753"
 dependencies = [
  "cc",
- "winapi 0.3.9",
+ "winapi",
 ]
 
 [[package]]
@@ -1475,21 +1453,6 @@ dependencies = [
  "libc",
 ]
 
-[[package]]
-name = "linked-hash-map"
-version = "0.5.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8dd5a6d5999d9907cda8ed67bbd137d3af8085216c2ac62de5be860bd41f304a"
-
-[[package]]
-name = "lock_api"
-version = "0.3.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75"
-dependencies = [
- "scopeguard",
-]
-
 [[package]]
 name = "lock_api"
 version = "0.4.2"
@@ -1567,98 +1530,61 @@ dependencies = [
 ]
 
 [[package]]
-name = "metrics-core"
-version = "0.5.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7c064b3a1ff41f4bf6c91185c8a0caeccf8a8a27e9d0f92cc54cf3dbec812f48"
-
-[[package]]
-name = "metrics-exporter-http"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e14017d204ae062dc5c68a321e3dbdcd9b30181305cb6b067932f7f03f754e27"
-dependencies = [
- "hyper",
- "log",
- "metrics-core",
-]
-
-[[package]]
-name = "metrics-exporter-log"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f3fc63816bd5f8bde5eb31ce471f9633adc69ba1c55b44191b4d5fc7e263e8ab"
+name = "metrics"
+version = "0.13.0-alpha.8"
+source = "git+https://github.com/ZcashFoundation/metrics?rev=971133128e5aebe3ad177acffc6154449736cfa2#971133128e5aebe3ad177acffc6154449736cfa2"
 dependencies = [
- "log",
- "metrics-core",
- "tokio 0.2.23",
+ "beef",
+ "metrics-macros",
+ "proc-macro-hack",
+ "sharded-slab",
 ]
 
 [[package]]
-name = "metrics-observer-json"
-version = "0.1.1"
+name = "metrics-core"
+version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bbe930460a6c336b8f873dcfb28da3f805fd0dbadbea7beaf3042c7fb1d9fcd3"
-dependencies = [
- "hdrhistogram",
- "metrics-core",
- "metrics-util",
- "serde_json",
-]
+checksum = "7c064b3a1ff41f4bf6c91185c8a0caeccf8a8a27e9d0f92cc54cf3dbec812f48"
 
 [[package]]
-name = "metrics-observer-prometheus"
-version = "0.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4bfe24ad8285ef8b239232135a65f89cc5fa4690bbfaf8907f4bef38f8b08eba"
+name = "metrics-exporter-prometheus"
+version = "0.1.0-alpha.7"
+source = "git+https://github.com/ZcashFoundation/metrics?rev=971133128e5aebe3ad177acffc6154449736cfa2#971133128e5aebe3ad177acffc6154449736cfa2"
 dependencies = [
- "hdrhistogram",
- "metrics-core",
+ "hdrhistogram 7.1.0",
+ "hyper",
+ "metrics 0.13.0-alpha.8",
  "metrics-util",
+ "parking_lot",
+ "thiserror",
+ "tokio",
 ]
 
 [[package]]
-name = "metrics-observer-yaml"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "83f66811013592560efc75d75a92d6e2f415a11b52f085e51d9fb4d1edec6335"
+name = "metrics-macros"
+version = "0.1.0-alpha.5"
+source = "git+https://github.com/ZcashFoundation/metrics?rev=971133128e5aebe3ad177acffc6154449736cfa2#971133128e5aebe3ad177acffc6154449736cfa2"
 dependencies = [
- "hdrhistogram",
- "metrics-core",
- "metrics-util",
- "serde_yaml",
+ "lazy_static",
+ "proc-macro-hack",
+ "proc-macro2 1.0.24",
+ "quote 1.0.7",
+ "regex",
+ "syn 1.0.48",
 ]
 
 [[package]]
-name = "metrics-runtime"
-version = "0.13.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ce0e4f69639ccc0c6b2f0612164f9817349eb25545ed1ffb5ef3e1e1c1d220b4"
+name = "metrics-util"
+version = "0.4.0-alpha.6"
+source = "git+https://github.com/ZcashFoundation/metrics?rev=971133128e5aebe3ad177acffc6154449736cfa2#971133128e5aebe3ad177acffc6154449736cfa2"
 dependencies = [
  "arc-swap",
  "atomic-shim",
- "crossbeam-utils 0.7.2",
- "im",
- "metrics",
- "metrics-core",
- "metrics-exporter-http",
- "metrics-exporter-log",
- "metrics-observer-json",
- "metrics-observer-prometheus",
- "metrics-observer-yaml",
- "metrics-util",
- "parking_lot 0.10.2",
- "quanta",
-]
-
-[[package]]
-name = "metrics-util"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "277619f040719a5a23d75724586d5601286e8fa53451cfaaca3b8c627c2c2378"
-dependencies = [
- "crossbeam-epoch 0.8.2",
- "serde",
+ "crossbeam-epoch 0.9.0",
+ "crossbeam-utils 0.8.0",
+ "dashmap",
+ "indexmap",
+ "metrics 0.13.0-alpha.8",
 ]
 
 [[package]]
@@ -1671,25 +1597,6 @@ dependencies = [
  "autocfg",
 ]
 
-[[package]]
-name = "mio"
-version = "0.6.22"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fce347092656428bc8eaf6201042cb551b8d67855af7374542a92a0fbfcac430"
-dependencies = [
- "cfg-if 0.1.10",
- "fuchsia-zircon",
- "fuchsia-zircon-sys",
- "iovec",
- "kernel32-sys",
- "libc",
- "log",
- "miow 0.2.1",
- "net2",
- "slab",
- "winapi 0.2.8",
-]
-
 [[package]]
 name = "mio"
 version = "0.7.6"
@@ -1698,21 +1605,9 @@ checksum = "f33bc887064ef1fd66020c9adfc45bb9f33d75a42096c81e7c56c65b75dd1a8b"
 dependencies = [
  "libc",
  "log",
- "miow 0.3.6",
+ "miow",
  "ntapi",
- "winapi 0.3.9",
-]
-
-[[package]]
-name = "miow"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919"
-dependencies = [
- "kernel32-sys",
- "net2",
- "winapi 0.2.8",
- "ws2_32-sys",
+ "winapi",
 ]
 
 [[package]]
@@ -1722,18 +1617,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5a33c1b55807fbed163481b5ba66db4b2fa6cde694a5027be10fb724206c5897"
 dependencies = [
  "socket2",
- "winapi 0.3.9",
-]
-
-[[package]]
-name = "net2"
-version = "0.2.35"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3ebc3ec692ed7c9a255596c67808dee269f64655d8baf7b4f0638e51ba1d6853"
-dependencies = [
- "cfg-if 0.1.10",
- "libc",
- "winapi 0.3.9",
+ "winapi",
 ]
 
 [[package]]
@@ -1758,6 +1642,7 @@ version = "5.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ffb4262d26ed83a1c0a33a38fe2bb15797329c85770da05e6b828ddb782627af"
 dependencies = [
+ "lexical-core",
  "memchr",
  "version_check 0.9.2",
 ]
@@ -1768,7 +1653,7 @@ version = "0.3.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44"
 dependencies = [
- "winapi 0.3.9",
+ "winapi",
 ]
 
 [[package]]
@@ -1840,7 +1725,7 @@ version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "53cdc5b785b7a58c5aad8216b3dfa114df64b0b06ae6e1501cef91df2fbdf8f9"
 dependencies = [
- "winapi 0.3.9",
+ "winapi",
 ]
 
 [[package]]
@@ -1870,16 +1755,6 @@ dependencies = [
  "serde",
 ]
 
-[[package]]
-name = "parking_lot"
-version = "0.10.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d3a704eb390aafdc107b0e392f56a82b668e3a71366993b5340f5833fd62505e"
-dependencies = [
- "lock_api 0.3.4",
- "parking_lot_core 0.7.2",
-]
-
 [[package]]
 name = "parking_lot"
 version = "0.11.1"
@@ -1887,22 +1762,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb"
 dependencies = [
  "instant",
- "lock_api 0.4.2",
- "parking_lot_core 0.8.0",
-]
-
-[[package]]
-name = "parking_lot_core"
-version = "0.7.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d58c7c768d4ba344e3e8d72518ac13e259d7c7ade24167003b8488e10b6740a3"
-dependencies = [
- "cfg-if 0.1.10",
- "cloudabi 0.0.3",
- "libc",
- "redox_syscall",
- "smallvec 1.5.0",
- "winapi 0.3.9",
+ "lock_api",
+ "parking_lot_core",
 ]
 
 [[package]]
@@ -1912,12 +1773,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c361aa727dd08437f2f1447be8b59a33b0edd15e0fcee698f935613d9efbca9b"
 dependencies = [
  "cfg-if 0.1.10",
- "cloudabi 0.1.0",
+ "cloudabi",
  "instant",
  "libc",
  "redox_syscall",
  "smallvec 1.5.0",
- "winapi 0.3.9",
+ "winapi",
 ]
 
 [[package]]
@@ -2104,18 +1965,6 @@ dependencies = [
  "syn 0.15.44",
 ]
 
-[[package]]
-name = "quanta"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "21484fda3d8ad7affee37755c77a5d0da527543f0af0c7f731c14e2215645d39"
-dependencies = [
- "atomic-shim",
- "ctor",
- "libc",
- "winapi 0.3.9",
-]
-
 [[package]]
 name = "quick-error"
 version = "1.2.3"
@@ -2171,7 +2020,7 @@ dependencies = [
  "libc",
  "rand_core 0.3.1",
  "rdrand",
- "winapi 0.3.9",
+ "winapi",
 ]
 
 [[package]]
@@ -2239,15 +2088,6 @@ dependencies = [
  "rand_core 0.5.1",
 ]
 
-[[package]]
-name = "rand_xoshiro"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a9fcdd2e881d02f1d9390ae47ad8e5696a9e4be7b547a1da2afbc61973217004"
-dependencies = [
- "rand_core 0.5.1",
-]
-
 [[package]]
 name = "rayon"
 version = "1.5.0"
@@ -2348,7 +2188,7 @@ version = "0.5.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7"
 dependencies = [
- "winapi 0.3.9",
+ "winapi",
 ]
 
 [[package]]
@@ -2536,18 +2376,6 @@ dependencies = [
  "serde",
 ]
 
-[[package]]
-name = "serde_yaml"
-version = "0.8.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f7baae0a99f1a324984bcdc5f0718384c1f69775f1c7eec8b859b71b443e3fd7"
-dependencies = [
- "dtoa",
- "linked-hash-map",
- "serde",
- "yaml-rust",
-]
-
 [[package]]
 name = "sha2"
 version = "0.9.2"
@@ -2596,16 +2424,6 @@ dependencies = [
  "libc",
 ]
 
-[[package]]
-name = "sized-chunks"
-version = "0.6.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ec31ceca5644fa6d444cc77548b88b67f46db6f7c71683b0f9336e671830d2f"
-dependencies = [
- "bitmaps",
- "typenum",
-]
-
 [[package]]
 name = "slab"
 version = "0.4.2"
@@ -2636,7 +2454,7 @@ dependencies = [
  "cfg-if 0.1.10",
  "libc",
  "redox_syscall",
- "winapi 0.3.9",
+ "winapi",
 ]
 
 [[package]]
@@ -2776,7 +2594,7 @@ dependencies = [
  "rand 0.7.3",
  "redox_syscall",
  "remove_dir_all",
- "winapi 0.3.9",
+ "winapi",
 ]
 
 [[package]]
@@ -2834,7 +2652,7 @@ checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255"
 dependencies = [
  "libc",
  "wasi 0.10.0+wasi-snapshot-preview1",
- "winapi 0.3.9",
+ "winapi",
 ]
 
 [[package]]
@@ -2852,23 +2670,6 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
 
-[[package]]
-name = "tokio"
-version = "0.2.23"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a6d7ad61edd59bfcc7e80dababf0f4aed2e6d5e0ba1659356ae889752dfc12ff"
-dependencies = [
- "bytes 0.5.6",
- "fnv",
- "futures-core",
- "iovec",
- "lazy_static",
- "memchr",
- "mio 0.6.22",
- "pin-project-lite 0.1.11",
- "slab",
-]
-
 [[package]]
 name = "tokio"
 version = "0.3.4"
@@ -2881,15 +2682,15 @@ dependencies = [
  "lazy_static",
  "libc",
  "memchr",
- "mio 0.7.6",
+ "mio",
  "num_cpus",
- "parking_lot 0.11.1",
+ "parking_lot",
  "pin-project-lite 0.2.0",
  "signal-hook-registry",
  "slab",
  "tokio-macros",
  "tracing",
- "winapi 0.3.9",
+ "winapi",
 ]
 
 [[package]]
@@ -2905,16 +2706,16 @@ dependencies = [
 
 [[package]]
 name = "tokio-util"
-version = "0.3.1"
+version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499"
+checksum = "24793699f4665ba0416ed287dc794fe6b11a4aa5e4e95b58624f45f6c46b97d4"
 dependencies = [
  "bytes 0.5.6",
  "futures-core",
  "futures-sink",
  "log",
  "pin-project-lite 0.1.11",
- "tokio 0.2.23",
+ "tokio",
 ]
 
 [[package]]
@@ -2928,7 +2729,7 @@ dependencies = [
  "futures-sink",
  "log",
  "pin-project-lite 0.1.11",
- "tokio 0.3.4",
+ "tokio",
 ]
 
 [[package]]
@@ -2947,9 +2748,9 @@ source = "git+https://github.com/tower-rs/tower?rev=5e1e07744820028877654c336a3b
 dependencies = [
  "futures-core",
  "futures-util",
- "hdrhistogram",
+ "hdrhistogram 6.3.4",
  "pin-project 1.0.2",
- "tokio 0.3.4",
+ "tokio",
  "tower-layer",
  "tower-service",
  "tracing",
@@ -2965,7 +2766,7 @@ dependencies = [
  "futures-core",
  "pin-project 0.4.27",
  "rand 0.7.3",
- "tokio 0.3.4",
+ "tokio",
  "tower",
  "tower-fallback",
  "tracing",
@@ -2979,7 +2780,7 @@ version = "0.1.0"
 dependencies = [
  "futures-core",
  "pin-project 0.4.27",
- "tokio 0.3.4",
+ "tokio",
  "tower",
  "tracing",
  "zebra-test",
@@ -3268,12 +3069,6 @@ dependencies = [
  "libc",
 ]
 
-[[package]]
-name = "winapi"
-version = "0.2.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a"
-
 [[package]]
 name = "winapi"
 version = "0.3.9"
@@ -3284,12 +3079,6 @@ dependencies = [
  "winapi-x86_64-pc-windows-gnu",
 ]
 
-[[package]]
-name = "winapi-build"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc"
-
 [[package]]
 name = "winapi-i686-pc-windows-gnu"
 version = "0.4.0"
@@ -3302,7 +3091,7 @@ version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
 dependencies = [
- "winapi 0.3.9",
+ "winapi",
 ]
 
 [[package]]
@@ -3311,16 +3100,6 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
-[[package]]
-name = "ws2_32-sys"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e"
-dependencies = [
- "winapi 0.2.8",
- "winapi-build",
-]
-
 [[package]]
 name = "wyz"
 version = "0.2.0"
@@ -3339,15 +3118,6 @@ dependencies = [
  "zeroize",
 ]
 
-[[package]]
-name = "yaml-rust"
-version = "0.4.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "39f0c922f1a334134dc2f7a8b67dc5d25f0735263feec974345ff706bcf20b0d"
-dependencies = [
- "linked-hash-map",
-]
-
 [[package]]
 name = "zcash_script"
 version = "0.1.4"
@@ -3412,14 +3182,14 @@ dependencies = [
  "futures",
  "futures-util",
  "jubjub 0.5.1",
- "metrics",
+ "metrics 0.12.1",
  "once_cell",
  "rand 0.7.3",
  "redjubjub",
  "serde",
  "spandoc",
  "thiserror",
- "tokio 0.3.4",
+ "tokio",
  "tower",
  "tower-batch",
  "tower-fallback",
@@ -3445,14 +3215,14 @@ dependencies = [
  "futures",
  "hex",
  "indexmap",
- "metrics",
+ "metrics 0.12.1",
  "pin-project 0.4.27",
  "proptest",
  "proptest-derive",
  "rand 0.7.3",
  "serde",
  "thiserror",
- "tokio 0.3.4",
+ "tokio",
  "tokio-util 0.5.0",
  "tower",
  "tracing",
@@ -3489,7 +3259,7 @@ dependencies = [
  "futures",
  "hex",
  "lazy_static",
- "metrics",
+ "metrics 0.12.1",
  "once_cell",
  "primitive-types",
  "proptest",
@@ -3498,7 +3268,7 @@ dependencies = [
  "spandoc",
  "tempdir",
  "thiserror",
- "tokio 0.3.4",
+ "tokio",
  "tower",
  "tracing",
  "tracing-error",
@@ -3521,7 +3291,7 @@ dependencies = [
  "spandoc",
  "tempdir",
  "thiserror",
- "tokio 0.3.4",
+ "tokio",
  "tower",
  "tracing",
  "tracing-error",
@@ -3555,15 +3325,15 @@ dependencies = [
  "gumdrop",
  "hyper",
  "inferno",
- "metrics",
- "metrics-runtime",
+ "metrics 0.13.0-alpha.8",
+ "metrics-exporter-prometheus",
  "once_cell",
  "pin-project 0.4.27",
  "rand 0.7.3",
  "serde",
  "tempdir",
  "thiserror",
- "tokio 0.3.4",
+ "tokio",
  "toml",
  "tower",
  "tracing",
diff --git a/Cargo.toml b/Cargo.toml
index 1eb6e23ebbf..5c9e4ac6565 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -22,3 +22,7 @@ panic = "abort"
 
 [patch.crates-io]
 tower = { git = "https://github.com/tower-rs/tower", rev = "5e1e07744820028877654c336a3b9fe057bf46f1" }
+hyper = { git = "https://github.com/hyperium/hyper/", rev = "ed2b22a7f66899d338691552fbcb6c0f2f4e06b9" }
+metrics = { git = "https://github.com/ZcashFoundation/metrics", rev = "971133128e5aebe3ad177acffc6154449736cfa2" }
+metrics-exporter-prometheus = { git = "https://github.com/ZcashFoundation/metrics", rev = "971133128e5aebe3ad177acffc6154449736cfa2" }
+
diff --git a/zebrad/Cargo.toml b/zebrad/Cargo.toml
index 49b8452af42..96d2ea7b972 100644
--- a/zebrad/Cargo.toml
+++ b/zebrad/Cargo.toml
@@ -19,7 +19,7 @@ toml = "0.5"
 chrono = "0.4"
 rand = "0.7"
 
-hyper = "0.13.9"
+hyper = { version = "0.14.0-dev", features = ["full"] }
 futures = "0.3"
 tokio = { version = "0.3", features = ["time", "rt-multi-thread", "stream", "macros", "tracing", "signal"] }
 tower = { version = "0.4", features = ["hedge", "limit"] }
@@ -33,8 +33,8 @@ tracing-futures = "0.2"
 tracing-flame = "0.1.0"
 tracing-subscriber = { version = "0.2.15", features = ["tracing-log"] }
 tracing-error = "0.1.2"
-metrics-runtime = "0.13"
-metrics = "0.12"
+metrics = "0.13.0-alpha.8"
+metrics-exporter-prometheus = "0.1.0-alpha.7"
 
 dirs = "3.0.1"
 inferno = { version = "0.10.1", default-features = false }
diff --git a/zebrad/src/components/metrics.rs b/zebrad/src/components/metrics.rs
index af003839cd4..83963f3c731 100644
--- a/zebrad/src/components/metrics.rs
+++ b/zebrad/src/components/metrics.rs
@@ -1,54 +1,23 @@
 //! An HTTP endpoint for metrics collection.
 
-use std::net::SocketAddr;
-
 use abscissa_core::{Component, FrameworkError};
-use metrics_runtime::{exporters::HttpExporter, observers::PrometheusBuilder, Receiver};
 
-use crate::{components::tokio::TokioComponent, config::ZebradConfig};
+use crate::config::ZebradConfig;
 
 /// Abscissa component which runs a metrics endpoint.
 #[derive(Debug, Component)]
-#[component(inject = "init_tokio(zebrad::components::tokio::TokioComponent)")]
-pub struct MetricsEndpoint {
-    addr: Option<SocketAddr>,
-}
+pub struct MetricsEndpoint {}
 
 impl MetricsEndpoint {
     /// Create the component.
     pub fn new(config: &ZebradConfig) -> Result<Self, FrameworkError> {
-        Ok(Self {
-            addr: config.metrics.endpoint_addr,
-        })
-    }
-
-    /// Tokio endpoint dependency stub.
-    pub fn init_tokio(&mut self, tokio_component: &TokioComponent) -> Result<(), FrameworkError> {
-        let addr = if let Some(addr) = self.addr {
-            addr
-        } else {
-            return Ok(());
-        };
-
-        info!("Initializing metrics endpoint at {}", addr);
-
-        // XXX do we need to hold on to the receiver?
-        let receiver = Receiver::builder()
-            .build()
-            .expect("Receiver config should be valid");
-        // XXX ???? connect this ???
-        let _sink = receiver.sink();
-
-        let endpoint = HttpExporter::new(receiver.controller(), PrometheusBuilder::new(), addr);
-
-        tokio_component
-            .rt
-            .as_ref()
-            .expect("runtime should not be taken")
-            .spawn(endpoint.async_run());
-
-        metrics::set_boxed_recorder(Box::new(receiver)).expect("XXX FIXME ERROR CONVERSION");
-
-        Ok(())
+        if let Some(addr) = config.metrics.endpoint_addr {
+            info!("Initializing metrics endpoint at {}", addr);
+            metrics_exporter_prometheus::PrometheusBuilder::new()
+                .listen_address(addr)
+                .install()
+                .expect("FIXME ERROR CONVERSION");
+        }
+        Ok(Self {})
     }
 }
diff --git a/zebrad/src/components/sync.rs b/zebrad/src/components/sync.rs
index 1cbf944cf80..d49738db229 100644
--- a/zebrad/src/components/sync.rs
+++ b/zebrad/src/components/sync.rs
@@ -557,11 +557,11 @@ where
     fn update_metrics(&self) {
         metrics::gauge!(
             "sync.prospective_tips.len",
-            self.prospective_tips.len() as i64
+            self.prospective_tips.len() as f64
         );
         metrics::gauge!(
             "sync.downloads.in_flight",
-            self.downloads.in_flight() as i64
+            self.downloads.in_flight() as f64
         );
     }
 }

From faa9cbcadeb10fe1037079a9cc5657f67fe0e668 Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Thu, 19 Nov 2020 14:30:28 -0800
Subject: [PATCH 058/148] deps: bump tower to pick up auto-resize in Hedge

Picks up https://github.com/tower-rs/tower/pull/484
---
 Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Cargo.toml b/Cargo.toml
index 5c9e4ac6565..d74173e4f73 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -21,7 +21,7 @@ panic = "abort"
 panic = "abort"
 
 [patch.crates-io]
-tower = { git = "https://github.com/tower-rs/tower", rev = "5e1e07744820028877654c336a3b9fe057bf46f1" }
+tower = { git = "https://github.com/tower-rs/tower", rev = "d4d1c67c6a0e4213a52abcc2b9df6cc58276ee39" }
 hyper = { git = "https://github.com/hyperium/hyper/", rev = "ed2b22a7f66899d338691552fbcb6c0f2f4e06b9" }
 metrics = { git = "https://github.com/ZcashFoundation/metrics", rev = "971133128e5aebe3ad177acffc6154449736cfa2" }
 metrics-exporter-prometheus = { git = "https://github.com/ZcashFoundation/metrics", rev = "971133128e5aebe3ad177acffc6154449736cfa2" }

From d4da9609eee2230d42b97b8ef4c3f0b23ecbede6 Mon Sep 17 00:00:00 2001
From: teor <teor@riseup.net>
Date: Fri, 20 Nov 2020 10:49:14 +1000
Subject: [PATCH 059/148] Update the max_concurrent_block_requests docs

In #1298, we decreased `max_concurrent_block_requests`,
but forgot to update the docs.
---
 zebrad/src/config.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/zebrad/src/config.rs b/zebrad/src/config.rs
index dde94a13aea..51ac53567ba 100644
--- a/zebrad/src/config.rs
+++ b/zebrad/src/config.rs
@@ -132,10 +132,10 @@ impl Default for MetricsSection {
 pub struct SyncSection {
     /// The maximum number of concurrent block requests during sync.
     ///
-    /// This is set to a high value by default, so that the concurrency limit is
-    /// based only on the number of available peers. However, on a slow network,
-    /// making too many concurrent block requests can overwhelm the connection.
-    /// Lowering this value may help on slow or unreliable networks.
+    /// This is set to a low value by default, to avoid task and
+    /// network contention. Increasing this value may improve
+    /// performance on machines with many cores and a fast network
+    /// connection.
     pub max_concurrent_block_requests: usize,
 
     /// Controls how far ahead of the chain tip the syncer tries to

From 04acc9da6c959810c1282d13a826df912323b578 Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Fri, 20 Nov 2020 12:42:29 -0800
Subject: [PATCH 060/148] consensus: instrument script verification

---
 zebra-consensus/src/script.rs | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/zebra-consensus/src/script.rs b/zebra-consensus/src/script.rs
index 4d37c1a095d..ee0876e7047 100644
--- a/zebra-consensus/src/script.rs
+++ b/zebra-consensus/src/script.rs
@@ -1,6 +1,7 @@
-use std::{pin::Pin, sync::Arc};
+use std::{future::Future, pin::Pin, sync::Arc};
+
+use tracing::Instrument;
 
-use std::future::Future;
 use zebra_chain::{parameters::ConsensusBranchId, transaction::Transaction, transparent};
 
 use crate::BoxError;
@@ -65,25 +66,33 @@ where
 
         match input {
             transparent::Input::PrevOut { outpoint, .. } => {
-                let output = self.state.call(zebra_state::Request::AwaitUtxo(*outpoint));
+                let outpoint = *outpoint;
                 let transaction = req.transaction;
                 let branch_id = self.branch;
                 let input_index = req.input_index;
 
+                let span = tracing::trace_span!("script", ?outpoint);
+                let output =
+                    span.in_scope(|| self.state.call(zebra_state::Request::AwaitUtxo(outpoint)));
+
                 async move {
+                    tracing::trace!("awaiting outpoint lookup");
                     let previous_output = match output.await? {
                         zebra_state::Response::Utxo(output) => output,
                         _ => unreachable!("AwaitUtxo always responds with Utxo"),
                     };
+                    tracing::trace!(?previous_output, "got UTXO");
 
                     zebra_script::is_valid(
                         transaction,
                         branch_id,
                         (input_index as u32, previous_output),
                     )?;
+                    tracing::trace!("script verification succeeded");
 
                     Ok(())
                 }
+                .instrument(span)
                 .boxed()
             }
             transparent::Input::Coinbase { .. } => {

From 3bfe63e38fdcbcd3237944eecf42f5628a07df09 Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Fri, 20 Nov 2020 12:43:50 -0800
Subject: [PATCH 061/148] state: add span to state service

Here the span is added to the body of the `Service::call`
implementation, not to the futures it returns, because the state service
does all of the work synchronously in `call` rather than in the futures
it returns.

The service is skipped as a span field.  We could either include or
exclude the request itself.  It would be useful, but the request body
can be very large.  Instead, we make two spans, one at info level and
one at trace level, and filter that way.
---
 zebra-state/src/service.rs | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs
index 0181e8c67af..b0d22465e47 100644
--- a/zebra-state/src/service.rs
+++ b/zebra-state/src/service.rs
@@ -383,6 +383,13 @@ impl Service<Request> for StateService {
         Poll::Ready(Ok(()))
     }
 
+    // Because the request might have a verbose debug output (e.g.,
+    // an entire block), include it only in a second trace-level span.
+    // For some reason putting them in this order causes the info-level
+    // span to be entered first.
+    #[allow(unused_braces)] // fixes a spurious warning from the proc macro
+    #[instrument(name = "state_trace", level = "trace", skip(self))]
+    #[instrument(name = "state", level = "info", skip(self, req))]
     fn call(&mut self, req: Request) -> Self::Future {
         match req {
             Request::CommitBlock { block } => {

From bbd7a62b2043bac64eb24e30b1b08565373b601d Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Fri, 20 Nov 2020 13:27:57 -0800
Subject: [PATCH 062/148] state: add service request count metrics

These are all one metric, with the type as an attribute, so that we can
display total requests, filter by a particular type, etc.
---
 zebra-state/src/service.rs | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs
index b0d22465e47..c6934eb4706 100644
--- a/zebra-state/src/service.rs
+++ b/zebra-state/src/service.rs
@@ -393,6 +393,8 @@ impl Service<Request> for StateService {
     fn call(&mut self, req: Request) -> Self::Future {
         match req {
             Request::CommitBlock { block } => {
+                metrics::counter!("state.requests", 1, "type" => "commit_block");
+
                 self.pending_utxos.check_block(&block);
                 let rsp_rx = self.queue_and_commit_non_finalized_blocks(block);
 
@@ -406,6 +408,8 @@ impl Service<Request> for StateService {
                 .boxed()
             }
             Request::CommitFinalizedBlock { block } => {
+                metrics::counter!("state.requests", 1, "type" => "commit_finalized_block");
+
                 let (rsp_tx, rsp_rx) = oneshot::channel();
 
                 self.pending_utxos.check_block(&block);
@@ -422,26 +426,33 @@ impl Service<Request> for StateService {
                 .boxed()
             }
             Request::Depth(hash) => {
+                metrics::counter!("state.requests", 1, "type" => "depth");
                 let rsp = Ok(self.depth(hash)).map(Response::Depth);
                 async move { rsp }.boxed()
             }
             Request::Tip => {
+                metrics::counter!("state.requests", 1, "type" => "tip");
                 let rsp = Ok(self.tip()).map(Response::Tip);
                 async move { rsp }.boxed()
             }
             Request::BlockLocator => {
+                metrics::counter!("state.requests", 1, "type" => "block_locator");
                 let rsp = Ok(self.block_locator().unwrap_or_default()).map(Response::BlockLocator);
                 async move { rsp }.boxed()
             }
             Request::Transaction(hash) => {
+                metrics::counter!("state.requests", 1, "type" => "transaction");
                 let rsp = Ok(self.transaction(hash)).map(Response::Transaction);
                 async move { rsp }.boxed()
             }
             Request::Block(hash_or_height) => {
+                metrics::counter!("state.requests", 1, "type" => "block");
                 let rsp = Ok(self.block(hash_or_height)).map(Response::Block);
                 async move { rsp }.boxed()
             }
             Request::AwaitUtxo(outpoint) => {
+                metrics::counter!("state.requests", 1, "type" => "await_utxo");
+
                 let fut = self.pending_utxos.queue(outpoint);
 
                 if let Some(utxo) = self.utxo(&outpoint) {

From 7dfea510d5b54fee7ad1059ff10ce4b158f33ef6 Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Fri, 20 Nov 2020 15:12:30 -0800
Subject: [PATCH 063/148] state: remove state_trace span

This turns out not to give much additional information when stacked with
child spans.
---
 zebra-state/src/service.rs | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs
index c6934eb4706..5593cec27b8 100644
--- a/zebra-state/src/service.rs
+++ b/zebra-state/src/service.rs
@@ -383,13 +383,7 @@ impl Service<Request> for StateService {
         Poll::Ready(Ok(()))
     }
 
-    // Because the request might have a verbose debug output (e.g.,
-    // an entire block), include it only in a second trace-level span.
-    // For some reason putting them in this order causes the info-level
-    // span to be entered first.
-    #[allow(unused_braces)] // fixes a spurious warning from the proc macro
-    #[instrument(name = "state_trace", level = "trace", skip(self))]
-    #[instrument(name = "state", level = "info", skip(self, req))]
+    #[instrument(name = "state", skip(self, req))]
     fn call(&mut self, req: Request) -> Self::Future {
         match req {
             Request::CommitBlock { block } => {

From 706c42de3eb4269ec69dad2259783c0df9f83b6c Mon Sep 17 00:00:00 2001
From: Deirdre Connolly <deirdre@zfnd.org>
Date: Fri, 20 Nov 2020 01:43:05 -0500
Subject: [PATCH 064/148] Filter broken command tests while including ignored
 otherwise

---
 .github/workflows/test.yml | 2 +-
 docker/Dockerfile.test     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 6b17625a0e3..0168e7dce78 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -63,7 +63,7 @@ jobs:
       - name: Run all tests
         run: |
           gcloud compute ssh "zebrad-tests-$BRANCH_NAME-$SHORT_SHA" --zone "$ZONE" \
-          --command "git clone -b $BRANCH_NAME https://github.com/ZcashFoundation/zebra.git; cd zebra/; docker build -f docker/Dockerfile.test -t zebrad-test .; docker run -i zebrad-test cargo +nightly test --all --test '*' --no-fail-fast -- -Zunstable-options --include-ignored"
+          --command "git clone -b $BRANCH_NAME https://github.com/ZcashFoundation/zebra.git; cd zebra/; docker build -f docker/Dockerfile.test -t zebrad-test .; docker run -i zebrad-test cargo +nightly test --all --no-fail-fast -- -Zunstable-options --include-ignored --skip kill_on_timeout"
 
       # Clean up
       - name: Delete test instance
diff --git a/docker/Dockerfile.test b/docker/Dockerfile.test
index 78d830481e5..40d4838681b 100644
--- a/docker/Dockerfile.test
+++ b/docker/Dockerfile.test
@@ -17,4 +17,4 @@ EXPOSE 8233 18233
 COPY . .
 
 # Filtering to only run integration tests requires nightly for now
-CMD cargo +nightly test --all --test '*' --no-fail-fast -- -Zunstable-options --include-ignored
+CMD cargo +nightly test --all --no-fail-fast -- -Zunstable-options --include-ignored

From 52296b96c7e4dec2eff5b04f18672a2748469d3d Mon Sep 17 00:00:00 2001
From: Deirdre Connolly <deirdre@zfnd.org>
Date: Fri, 20 Nov 2020 15:03:27 -0500
Subject: [PATCH 065/148] Bump test job timeout to 45 minutes because Windows
 debug builds are taking a while

---
 .github/workflows/ci.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 80af5e0c109..05ff5c02d6f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -6,7 +6,8 @@ jobs:
 
   test:
     name: Test on ${{ matrix.os }}
-    timeout-minutes: 30
+    # The large timeout is to accommodate Windows builds
+    timeout-minutes: 45
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -37,7 +38,7 @@ jobs:
         with:
           command: test
           args: --verbose --all
-      # Explicitly run any tests that are usually #[ignored]
+      # Explicitly run any tests that are usually #[ignored], modulo ZEBRA_SKIP_NETWORK_TESTS
       - name: Run zebrad large sync tests
         env:
           RUST_BACKTRACE: full

From 036abd50acbef420fbec6f9ffea88ac834c0c9a8 Mon Sep 17 00:00:00 2001
From: Deirdre Connolly <deirdre@zfnd.org>
Date: Fri, 20 Nov 2020 16:02:45 -0500
Subject: [PATCH 066/148] Back to stable for test image

---
 .github/workflows/test.yml | 2 +-
 docker/Dockerfile.test     | 5 ++---
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 0168e7dce78..9eb7ae670e8 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -63,7 +63,7 @@ jobs:
       - name: Run all tests
         run: |
           gcloud compute ssh "zebrad-tests-$BRANCH_NAME-$SHORT_SHA" --zone "$ZONE" \
-          --command "git clone -b $BRANCH_NAME https://github.com/ZcashFoundation/zebra.git; cd zebra/; docker build -f docker/Dockerfile.test -t zebrad-test .; docker run -i zebrad-test cargo +nightly test --all --no-fail-fast -- -Zunstable-options --include-ignored --skip kill_on_timeout"
+          --command "git clone -b $BRANCH_NAME https://github.com/ZcashFoundation/zebra.git; cd zebra/; docker build -f docker/Dockerfile.test -t zebrad-test .; docker run -i zebrad-test cargo test --workspace --no-fail-fast -- -Zunstable-options --include-ignored --skip kill_on_timeout"
 
       # Clean up
       - name: Delete test instance
diff --git a/docker/Dockerfile.test b/docker/Dockerfile.test
index 40d4838681b..8a7838af462 100644
--- a/docker/Dockerfile.test
+++ b/docker/Dockerfile.test
@@ -1,4 +1,4 @@
-FROM rustlang/rust:nightly
+FROM rust:buster
 
 RUN apt-get update && \
 	apt-get install -y --no-install-recommends \
@@ -16,5 +16,4 @@ EXPOSE 8233 18233
 
 COPY . .
 
-# Filtering to only run integration tests requires nightly for now
-CMD cargo +nightly test --all --no-fail-fast -- -Zunstable-options --include-ignored
+CMD cargo test --workspace --no-fail-fast -- -Zunstable-options --include-ignored

From 558661a531d2c7dc86fd630942763faa2c2b540e Mon Sep 17 00:00:00 2001
From: Deirdre Connolly <deirdre@zfnd.org>
Date: Sat, 21 Nov 2020 01:37:43 -0500
Subject: [PATCH 067/148] Remove test attributes and allow(dead_code) for test
 code that tests currently unimplemented functionality

---
 .github/workflows/test.yml              | 2 +-
 zebra-consensus/src/block/tests.rs      | 6 ++++--
 zebra-consensus/src/checkpoint/tests.rs | 6 ++++--
 3 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 9eb7ae670e8..e4d0aac57c0 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -63,7 +63,7 @@ jobs:
       - name: Run all tests
         run: |
           gcloud compute ssh "zebrad-tests-$BRANCH_NAME-$SHORT_SHA" --zone "$ZONE" \
-          --command "git clone -b $BRANCH_NAME https://github.com/ZcashFoundation/zebra.git; cd zebra/; docker build -f docker/Dockerfile.test -t zebrad-test .; docker run -i zebrad-test cargo test --workspace --no-fail-fast -- -Zunstable-options --include-ignored --skip kill_on_timeout"
+          --command "git clone -b $BRANCH_NAME https://github.com/ZcashFoundation/zebra.git; cd zebra/; docker build -f docker/Dockerfile.test -t zebrad-test .; docker run -i zebrad-test cargo test --workspace --no-fail-fast -- -Zunstable-options --include-ignored"
 
       # Clean up
       - name: Delete test instance
diff --git a/zebra-consensus/src/block/tests.rs b/zebra-consensus/src/block/tests.rs
index 82c8c0d1432..11c724d6720 100644
--- a/zebra-consensus/src/block/tests.rs
+++ b/zebra-consensus/src/block/tests.rs
@@ -102,13 +102,15 @@ static INVALID_COINBASE_TRANSCRIPT: Lazy<Vec<(Arc<Block>, Result<block::Hash, Tr
         ]
     });
 
-#[tokio::test]
 // TODO: enable this test after implementing contextual verification
-#[ignore]
+// #[tokio::test]
+// #[ignore]
+#[allow(dead_code)]
 async fn check_transcripts_test() -> Result<(), Report> {
     check_transcripts().await
 }
 
+#[allow(dead_code)]
 #[spandoc::spandoc]
 async fn check_transcripts() -> Result<(), Report> {
     zebra_test::init();
diff --git a/zebra-consensus/src/checkpoint/tests.rs b/zebra-consensus/src/checkpoint/tests.rs
index 4b8eb8039f3..535d9461b6d 100644
--- a/zebra-consensus/src/checkpoint/tests.rs
+++ b/zebra-consensus/src/checkpoint/tests.rs
@@ -213,9 +213,9 @@ async fn multi_item_checkpoint_list() -> Result<(), Report> {
     Ok(())
 }
 
-#[tokio::test]
 // Temporarily ignore this test, until the state can handle out-of-order blocks
-#[ignore]
+// #[tokio::test]
+#[allow(dead_code)]
 async fn continuous_blockchain_test() -> Result<(), Report> {
     continuous_blockchain(None).await?;
     for height in 0..=10 {
@@ -225,6 +225,8 @@ async fn continuous_blockchain_test() -> Result<(), Report> {
 }
 
 /// Test a continuous blockchain, restarting verification at `restart_height`.
+// TODO: does this duplicate the test code commented out in src/chain/tests.rs?
+#[allow(dead_code)]
 #[spandoc::spandoc]
 async fn continuous_blockchain(restart_height: Option<block::Height>) -> Result<(), Report> {
     zebra_test::init();

From 902c6f6b29181391f99c40b23091b461898789c5 Mon Sep 17 00:00:00 2001
From: Deirdre Connolly <deirdre@zfnd.org>
Date: Sat, 21 Nov 2020 04:29:24 -0500
Subject: [PATCH 068/148] Remove test attributes and allow(dead_code) for
 command timeout tests that exercise currently broken properties

---
 zebra-test/tests/command.rs | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/zebra-test/tests/command.rs b/zebra-test/tests/command.rs
index 3c7e9fd1a5b..95f95763574 100644
--- a/zebra-test/tests/command.rs
+++ b/zebra-test/tests/command.rs
@@ -95,8 +95,9 @@ fn finish_before_timeout_output_single_line() -> Result<()> {
 /// is killed after the timeout.
 ///
 /// This test fails due to bugs in TestDirExt, see #1140 for details.
-#[test]
-#[ignore]
+//#[test]
+//#[ignore]
+#[allow(dead_code)]
 fn kill_on_timeout_continuous_output_no_newlines() -> Result<()> {
     zebra_test::init();
 
@@ -146,8 +147,9 @@ fn finish_before_timeout_short_output_no_newlines() -> Result<()> {
 /// Test if the timeout works for a process that produces no output.
 ///
 /// This test fails due to bugs in TestDirExt, see #1140 for details.
-#[test]
-#[ignore]
+// #[test]
+// #[ignore]
+#[allow(dead_code)]
 fn kill_on_timeout_no_output() -> Result<()> {
     zebra_test::init();
 

From 0b6a61c9e8db0f51dd54cfea5c77f6db085ab19c Mon Sep 17 00:00:00 2001
From: Deirdre Connolly <deirdre@zfnd.org>
Date: Sat, 21 Nov 2020 05:02:03 -0500
Subject: [PATCH 069/148] gcloud build is still the only required check for PR
 merge, run tests in release profile

---
 docker/Dockerfile.build | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index 4c3386fdc93..e8c0aead2d1 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -13,7 +13,7 @@ ENV CARGO_HOME /zebra/.cargo/
 RUN rustc -V; cargo -V; rustup -V
 
 COPY . .
-RUN cargo build --release
+RUN cargo test --all --release; cargo build --release
 
 
 # Runner image

From 2e4f4d8e87017dd02c2080e75eb7eb86fdf6e5e6 Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Thu, 19 Nov 2020 16:55:36 -0800
Subject: [PATCH 070/148] consensus: fix span handling in BlockVerifier

The BlockVerifier constructed a tracing span and manually entered it
inside of an async block.  Manually entering spans inside async blocks
can cause problems where the span might not be entered and exited
correctly as the resulting future is polled.  Instead, using the
.instrument creates a wrapper future that handles the bookkeeping.

I changed the span name and contents to be consistent with the spans in
the checkpoint verifier.
---
 zebra-consensus/src/block.rs | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/zebra-consensus/src/block.rs b/zebra-consensus/src/block.rs
index 6c2926ea71f..97ccfb7077f 100644
--- a/zebra-consensus/src/block.rs
+++ b/zebra-consensus/src/block.rs
@@ -19,6 +19,7 @@ use futures::stream::FuturesUnordered;
 use futures_util::FutureExt;
 use thiserror::Error;
 use tower::{Service, ServiceExt};
+use tracing::Instrument;
 
 use zebra_chain::{
     block::{self, Block},
@@ -114,14 +115,14 @@ where
         let mut transaction_verifier = self.transaction_verifier.clone();
         let network = self.network;
 
+        // We don't include the block hash, because it's likely already in a parent span
+        let span = tracing::debug_span!("block", height = ?block.coinbase_height());
+
         // TODO(jlusby): Error = Report, handle errors from state_service.
         async move {
-            let hash = block.hash();
-
-            // The height is already included in the ChainVerifier span
-            let span = tracing::debug_span!("BlockVerifier::call", ?hash);
-            let _entered = span.enter();
+            tracing::trace!("beginning block verification");
 
+            let hash = block.hash();
             // Check that this block is actually a new block.
             match state_service
                 .ready_and()
@@ -200,6 +201,7 @@ where
                 _ => unreachable!("wrong response for CommitBlock"),
             }
         }
+        .instrument(span)
         .boxed()
     }
 }

From d1ee7f263a6d2eed1a25c9c80ec3947c2d1351a3 Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Thu, 19 Nov 2020 17:18:50 -0800
Subject: [PATCH 071/148] consensus: add debug span to TransactionVerifier

---
 zebra-consensus/src/transaction.rs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/zebra-consensus/src/transaction.rs b/zebra-consensus/src/transaction.rs
index 91fa6ed5b5f..b3d2be422c9 100644
--- a/zebra-consensus/src/transaction.rs
+++ b/zebra-consensus/src/transaction.rs
@@ -9,8 +9,8 @@ use futures::{
     stream::{FuturesUnordered, StreamExt},
     FutureExt,
 };
-
 use tower::{Service, ServiceExt};
+use tracing::Instrument;
 
 use zebra_chain::{
     parameters::NetworkUpgrade,
@@ -88,7 +88,9 @@ where
 
         let mut redjubjub_verifier = crate::primitives::redjubjub::VERIFIER.clone();
         let mut script_verifier = self.script_verifier.clone();
+        let span = tracing::debug_span!("tx", hash = ?tx.hash());
         async move {
+            tracing::trace!(?tx);
             match &*tx {
                 Transaction::V1 { .. } | Transaction::V2 { .. } | Transaction::V3 { .. } => {
                     Err(TransactionError::WrongVersion)
@@ -210,6 +212,7 @@ where
                 }
             }
         }
+        .instrument(span)
         .boxed()
     }
 }

From b5515123eb1fc89858cefff7bd07bb35b522ca69 Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Thu, 19 Nov 2020 17:32:27 -0800
Subject: [PATCH 072/148] chain: add custom Debug for CoinbaseData

The derived Debug impl just shows u8s as numbers, which isn't what we
want.  There are basically two reasonable options here:

1. Hex-encoded bytes
2. Escaped ASCII

I picked (2) because a lot of coinbase data has ascii text in it.
---
 zebra-chain/src/transparent.rs | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/zebra-chain/src/transparent.rs b/zebra-chain/src/transparent.rs
index 93aacc3e4e7..97a09683028 100644
--- a/zebra-chain/src/transparent.rs
+++ b/zebra-chain/src/transparent.rs
@@ -23,7 +23,7 @@ use crate::{
 };
 
 /// Arbitrary data inserted by miners into a coinbase transaction.
-#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
+#[derive(Clone, Eq, PartialEq, Serialize, Deserialize)]
 pub struct CoinbaseData(
     /// Invariant: this vec, together with the coinbase height, must be less than
     /// 100 bytes. We enforce this by only constructing CoinbaseData fields by
@@ -40,6 +40,20 @@ impl AsRef<[u8]> for CoinbaseData {
     }
 }
 
+impl std::fmt::Debug for CoinbaseData {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let escaped = String::from_utf8(
+            self.0
+                .iter()
+                .cloned()
+                .flat_map(std::ascii::escape_default)
+                .collect(),
+        )
+        .expect("ascii::escape_default produces utf8");
+        f.debug_tuple("CoinbaseData").field(&escaped).finish()
+    }
+}
+
 /// OutPoint
 ///
 /// A particular transaction output reference.

From 25fd52be519def8ba984e7ce38957c10a0459220 Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Thu, 19 Nov 2020 19:02:33 -0800
Subject: [PATCH 073/148] chain: tidy Debug for Amount

This avoids printing a bunch of PhantomData.
---
 zebra-chain/src/amount.rs | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/zebra-chain/src/amount.rs b/zebra-chain/src/amount.rs
index 81a30b6feef..3e507193660 100644
--- a/zebra-chain/src/amount.rs
+++ b/zebra-chain/src/amount.rs
@@ -19,11 +19,21 @@ use byteorder::{ByteOrder, LittleEndian, ReadBytesExt, WriteBytesExt};
 type Result<T, E = Error> = std::result::Result<T, E>;
 
 /// A runtime validated type for representing amounts of zatoshis
-#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
+#[derive(Clone, Copy, Serialize, Deserialize)]
 #[serde(try_from = "i64")]
 #[serde(bound = "C: Constraint")]
 pub struct Amount<C = NegativeAllowed>(i64, PhantomData<C>);
 
+// in a world where specialization existed
+// https://github.com/rust-lang/rust/issues/31844
+// we could do much better here
+// for now, drop the constraint
+impl<C> std::fmt::Debug for Amount<C> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_tuple("Amount").field(&self.0).finish()
+    }
+}
+
 impl<C> Amount<C> {
     /// Convert this amount to a different Amount type if it satisfies the new constraint
     pub fn constrain<C2>(self) -> Result<Amount<C2>>

From b116cfcd76757d061cf715b72e7f7fee36a47692 Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Thu, 19 Nov 2020 19:25:25 -0800
Subject: [PATCH 074/148] consensus: add debug event on wrong version check

Adding this check reveals that the WrongVersion errors aren't coming
from the correct WrongVersion check.
---
 zebra-consensus/src/transaction.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/zebra-consensus/src/transaction.rs b/zebra-consensus/src/transaction.rs
index b3d2be422c9..3f0986b676e 100644
--- a/zebra-consensus/src/transaction.rs
+++ b/zebra-consensus/src/transaction.rs
@@ -93,6 +93,7 @@ where
             tracing::trace!(?tx);
             match &*tx {
                 Transaction::V1 { .. } | Transaction::V2 { .. } | Transaction::V3 { .. } => {
+                    tracing::debug!(?tx, "got transaction with wrong version");
                     Err(TransactionError::WrongVersion)
                 }
                 Transaction::V4 {

From 96ee32e5d22920a02b54cb37949ddc66bd4820fd Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Thu, 19 Nov 2020 19:06:10 -0800
Subject: [PATCH 075/148] consensus: fix bug in coinbase joinsplit/spend check

This function caused spurious "WrongVersion" errors, because the match
pattern in the first arm was non-exhaustive, but the fallthrough match
arm was present and assumed it would only be reached if the version was
incorrect.

This commit cleans up the implemenation, splits out the error variants,
and renames the check to be more precise.

To avoid this kind of bug in the future, two guidelines are useful:

1. Avoid fallthrough cases that circumvent non-exhaustive match checks;
2. Avoid nested conditionals, preferring a "straight-line" sequence of
   match arm => result pairs rather than nested matches or matches with
   conditionals inside.
---
 zebra-consensus/src/error.rs             | 10 ++++--
 zebra-consensus/src/transaction.rs       |  2 +-
 zebra-consensus/src/transaction/check.rs | 42 +++++++++++++++---------
 3 files changed, 36 insertions(+), 18 deletions(-)

diff --git a/zebra-consensus/src/error.rs b/zebra-consensus/src/error.rs
index 70c002c3291..2f5458d585a 100644
--- a/zebra-consensus/src/error.rs
+++ b/zebra-consensus/src/error.rs
@@ -28,8 +28,14 @@ pub enum TransactionError {
     #[error("coinbase input found in non-coinbase transaction")]
     CoinbaseInputFound,
 
-    #[error("coinbase transaction MUST NOT have any JoinSplit descriptions or Spend descriptions")]
-    CoinbaseHasJoinSplitOrSpend,
+    #[error("coinbase transaction MUST NOT have any JoinSplit descriptions")]
+    CoinbaseHasJoinSplit,
+
+    #[error("coinbase transaction MUST NOT have any Spend descriptions")]
+    CoinbaseHasSpend,
+
+    #[error("coinbase transaction MUST NOT have any Output descriptions pre-Heartwood")]
+    CoinbaseHasOutputPreHeartwood,
 
     #[error("coinbase transaction failed subsidy validation")]
     Subsidy(#[from] SubsidyError),
diff --git a/zebra-consensus/src/transaction.rs b/zebra-consensus/src/transaction.rs
index 3f0986b676e..b3fe435bb4d 100644
--- a/zebra-consensus/src/transaction.rs
+++ b/zebra-consensus/src/transaction.rs
@@ -116,7 +116,7 @@ where
                     #[allow(clippy::if_same_then_else)] // delete when filled in
                     if tx.is_coinbase() {
                         // do something special for coinbase transactions
-                        check::coinbase_tx_does_not_spend_shielded(&tx)?;
+                        check::coinbase_tx_no_joinsplit_or_spend(&tx)?;
                     } else {
                         // otherwise, check no coinbase inputs
                         // feed all of the inputs to the script verifier
diff --git a/zebra-consensus/src/transaction/check.rs b/zebra-consensus/src/transaction/check.rs
index aa3b13bc1a8..9761d4454e7 100644
--- a/zebra-consensus/src/transaction/check.rs
+++ b/zebra-consensus/src/transaction/check.rs
@@ -1,3 +1,7 @@
+//! Transaction checks.
+//!
+//! Code in this file can freely assume that no pre-V4 transactions are present.
+
 use std::convert::TryFrom;
 
 use zebra_chain::{
@@ -86,22 +90,30 @@ pub fn shielded_balances_match(
 
 /// Check that a coinbase tx does not have any JoinSplit or Spend descriptions.
 ///
-/// https://zips.z.cash/protocol/canopy.pdf#consensusfrombitcoin
-pub fn coinbase_tx_does_not_spend_shielded(tx: &Transaction) -> Result<(), TransactionError> {
-    match tx {
-        Transaction::V4 {
-            joinsplit_data: Some(joinsplit_data),
-            shielded_data: Some(shielded_data),
-            ..
-        } => {
-            if !tx.is_coinbase()
-                || (joinsplit_data.joinsplits().count() == 0 && shielded_data.spends().count() == 0)
-            {
-                Ok(())
-            } else {
-                Err(TransactionError::CoinbaseHasJoinSplitOrSpend)
+/// https://zips.z.cash/protocol/canopy.pdf#txnencodingandconsensus
+pub fn coinbase_tx_no_joinsplit_or_spend(tx: &Transaction) -> Result<(), TransactionError> {
+    if tx.is_coinbase() {
+        match tx {
+            // Check if there is any JoinSplitData.
+            Transaction::V4 {
+                joinsplit_data: Some(_),
+                ..
+            } => Err(TransactionError::CoinbaseHasJoinSplit),
+
+            // The ShieldedData contains both Spends and Outputs, and Outputs
+            // are allowed post-Heartwood, so we have to count Spends.
+            Transaction::V4 {
+                shielded_data: Some(shielded_data),
+                ..
+            } if shielded_data.spends().count() > 0 => Err(TransactionError::CoinbaseHasSpend),
+
+            Transaction::V4 { .. } => Ok(()),
+
+            Transaction::V1 { .. } | Transaction::V2 { .. } | Transaction::V3 { .. } => {
+                unreachable!("tx version is checked first")
             }
         }
-        _ => Err(TransactionError::WrongVersion),
+    } else {
+        Ok(())
     }
 }

From ace1103462dafa65c4b95dfce81f2796d9903c9a Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Thu, 19 Nov 2020 19:13:52 -0800
Subject: [PATCH 076/148] consensus: fix bug in tx input/output presence check

Making this check's match statement exhaustive revealed a bug similar to
the previous commit.  The logic in the spec is written in terms of
numbers, but our data is internally represented in terms of enums
(ADTs).  This kind of cross-representation rule translation is a bug
surface, which we can avoid by converting to counts and summing up.  (We
should use one style at a time).
---
 zebra-consensus/src/error.rs             |  7 +++-
 zebra-consensus/src/transaction.rs       |  2 +-
 zebra-consensus/src/transaction/check.rs | 48 ++++++++++++++++++------
 3 files changed, 42 insertions(+), 15 deletions(-)

diff --git a/zebra-consensus/src/error.rs b/zebra-consensus/src/error.rs
index 2f5458d585a..caa2f2edb03 100644
--- a/zebra-consensus/src/error.rs
+++ b/zebra-consensus/src/error.rs
@@ -43,8 +43,11 @@ pub enum TransactionError {
     #[error("transaction version number MUST be >= 4")]
     WrongVersion,
 
-    #[error("at least one of tx_in_count, nShieldedSpend, and nJoinSplit MUST be nonzero")]
-    NoTransfer,
+    #[error("must have at least one input: transparent, shielded spend, or joinsplit")]
+    NoInputs,
+
+    #[error("must have at least one output: transparent, shielded output, or joinsplit")]
+    NoOutputs,
 
     #[error("if there are no Spends or Outputs, the value balance MUST be 0.")]
     BadBalance,
diff --git a/zebra-consensus/src/transaction.rs b/zebra-consensus/src/transaction.rs
index b3fe435bb4d..ef2f5377a93 100644
--- a/zebra-consensus/src/transaction.rs
+++ b/zebra-consensus/src/transaction.rs
@@ -130,7 +130,7 @@ where
                         }
                     }
 
-                    check::some_money_is_spent(&tx)?;
+                    check::has_inputs_and_outputs(&tx)?;
                     check::any_coinbase_inputs_no_transparent_outputs(&tx)?;
 
                     let sighash = tx.sighash(
diff --git a/zebra-consensus/src/transaction/check.rs b/zebra-consensus/src/transaction/check.rs
index 9761d4454e7..3060220f5bf 100644
--- a/zebra-consensus/src/transaction/check.rs
+++ b/zebra-consensus/src/transaction/check.rs
@@ -25,28 +25,52 @@ pub fn validate_joinsplit_sig(
         .map_err(TransactionError::Ed25519)
 }
 
-/// Check that at least one of tx_in_count, nShieldedSpend, and nJoinSplit MUST
-/// be non-zero.
+/// Checks that the transaction has inputs and outputs.
+///
+/// More specifically:
+///
+/// * at least one of tx_in_count, nShieldedSpend, and nJoinSplit MUST be non-zero.
+/// * at least one of tx_out_count, nShieldedOutput, and nJoinSplit MUST be non-zero.
 ///
 /// https://zips.z.cash/protocol/canopy.pdf#txnencodingandconsensus
-pub fn some_money_is_spent(tx: &Transaction) -> Result<(), TransactionError> {
+pub fn has_inputs_and_outputs(tx: &Transaction) -> Result<(), TransactionError> {
+    // The consensus rule is written in terms of numbers, but our transactions
+    // hold enum'd data. Mixing pattern matching and numerical checks is risky,
+    // so convert everything to counts and sum up.
     match tx {
         Transaction::V4 {
             inputs,
-            joinsplit_data: Some(joinsplit_data),
-            shielded_data: Some(shielded_data),
+            outputs,
+            joinsplit_data,
+            shielded_data,
             ..
         } => {
-            if !inputs.is_empty()
-                || joinsplit_data.joinsplits().count() > 0
-                || shielded_data.spends().count() > 0
-            {
-                Ok(())
+            let tx_in_count = inputs.len();
+            let tx_out_count = outputs.len();
+            let n_joinsplit = joinsplit_data
+                .as_ref()
+                .map(|d| d.joinsplits().count())
+                .unwrap_or(0);
+            let n_shielded_spend = shielded_data
+                .as_ref()
+                .map(|d| d.spends().count())
+                .unwrap_or(0);
+            let n_shielded_output = shielded_data
+                .as_ref()
+                .map(|d| d.outputs().count())
+                .unwrap_or(0);
+
+            if tx_in_count + n_shielded_spend + n_joinsplit == 0 {
+                Err(TransactionError::NoInputs)
+            } else if tx_out_count + n_shielded_output + n_joinsplit == 0 {
+                Err(TransactionError::NoOutputs)
             } else {
-                Err(TransactionError::NoTransfer)
+                Ok(())
             }
         }
-        _ => Err(TransactionError::WrongVersion),
+        Transaction::V1 { .. } | Transaction::V2 { .. } | Transaction::V3 { .. } => {
+            unreachable!("tx version is checked first")
+        }
     }
 }
 

From 2eceff421f4730110e005b9238eadbbac7a1098d Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Thu, 19 Nov 2020 19:17:27 -0800
Subject: [PATCH 077/148] consensus: remove incorrect check

This consensus rule is supposed to apply to transactions whose
transparent inputs are the *outputs* of previous coinbase
transactions, not to transactions with coinbase inputs.  Because that
logic is different enough from this logic, and requires different data
flow, it's cleaner to just remove this check for now.
---
 zebra-consensus/src/transaction.rs       |  1 -
 zebra-consensus/src/transaction/check.rs | 22 ----------------------
 2 files changed, 23 deletions(-)

diff --git a/zebra-consensus/src/transaction.rs b/zebra-consensus/src/transaction.rs
index ef2f5377a93..75ff014d247 100644
--- a/zebra-consensus/src/transaction.rs
+++ b/zebra-consensus/src/transaction.rs
@@ -131,7 +131,6 @@ where
                     }
 
                     check::has_inputs_and_outputs(&tx)?;
-                    check::any_coinbase_inputs_no_transparent_outputs(&tx)?;
 
                     let sighash = tx.sighash(
                         NetworkUpgrade::Sapling, // TODO: pass this in
diff --git a/zebra-consensus/src/transaction/check.rs b/zebra-consensus/src/transaction/check.rs
index 3060220f5bf..40e199edaad 100644
--- a/zebra-consensus/src/transaction/check.rs
+++ b/zebra-consensus/src/transaction/check.rs
@@ -74,28 +74,6 @@ pub fn has_inputs_and_outputs(tx: &Transaction) -> Result<(), TransactionError>
     }
 }
 
-/// Check that a transaction with one or more transparent inputs from coinbase
-/// transactions has no transparent outputs.
-///
-/// Note that inputs from coinbase transactions include Founders’ Reward
-/// outputs.
-///
-/// https://zips.z.cash/protocol/canopy.pdf#consensusfrombitcoin
-pub fn any_coinbase_inputs_no_transparent_outputs(
-    tx: &Transaction,
-) -> Result<(), TransactionError> {
-    match tx {
-        Transaction::V4 { outputs, .. } => {
-            if !tx.contains_coinbase_input() || !outputs.is_empty() {
-                Ok(())
-            } else {
-                Err(TransactionError::NoTransfer)
-            }
-        }
-        _ => Err(TransactionError::WrongVersion),
-    }
-}
-
 /// Check that if there are no Spends or Outputs, that valueBalance is also 0.
 ///
 /// https://zips.z.cash/protocol/canopy.pdf#consensusfrombitcoin

From a877da2157212880b40858ad2430d20024b9d8fe Mon Sep 17 00:00:00 2001
From: Deirdre Connolly <deirdre@zfnd.org>
Date: Sun, 22 Nov 2020 21:19:08 -0500
Subject: [PATCH 078/148] Enable RUST_BACKTRACE=full for test and build/deploy
 images

---
 docker/Dockerfile.build | 2 ++
 docker/Dockerfile.test  | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index e8c0aead2d1..12f8af25ceb 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -35,4 +35,6 @@ RUN printf "endpoint_addr = '0.0.0.0:3000'\n" >> /zebrad.toml
 
 EXPOSE 3000 8233 18233
 
+ENV RUST_BACKTRACE full
+
 CMD [ "/zebrad", "-c", "/zebrad.toml", "start" ]
diff --git a/docker/Dockerfile.test b/docker/Dockerfile.test
index 8a7838af462..8972eca5994 100644
--- a/docker/Dockerfile.test
+++ b/docker/Dockerfile.test
@@ -7,7 +7,7 @@ RUN apt-get update && \
 RUN mkdir /zebra
 WORKDIR /zebra
 
-ENV RUST_BACKTRACE 1
+ENV RUST_BACKTRACE full
 ENV CARGO_HOME /zebra/.cargo/
 
 RUN rustc -V; cargo -V; rustup -V

From 196dc6369c77e26094a226857a6846357286a47c Mon Sep 17 00:00:00 2001
From: teor <teor@riseup.net>
Date: Mon, 23 Nov 2020 11:43:15 +1000
Subject: [PATCH 079/148] Delete outdated transaction comments

---
 zebra-consensus/src/transaction.rs | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/zebra-consensus/src/transaction.rs b/zebra-consensus/src/transaction.rs
index 75ff014d247..d44f1a74664 100644
--- a/zebra-consensus/src/transaction.rs
+++ b/zebra-consensus/src/transaction.rs
@@ -111,9 +111,6 @@ where
                     let mut async_checks = FuturesUnordered::new();
 
                     // Handle transparent inputs and outputs.
-                    // These are left unimplemented!() pending implementation
-                    // of the async script RFC.
-                    #[allow(clippy::if_same_then_else)] // delete when filled in
                     if tx.is_coinbase() {
                         // do something special for coinbase transactions
                         check::coinbase_tx_no_joinsplit_or_spend(&tx)?;

From aa45bf2b58b6e24f614156b08fa29672d933a7f3 Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Fri, 20 Nov 2020 15:17:36 -0800
Subject: [PATCH 080/148] consensus: add traces to block verifier

---
 zebra-consensus/src/block.rs | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/zebra-consensus/src/block.rs b/zebra-consensus/src/block.rs
index 97ccfb7077f..bcd3a7d6de1 100644
--- a/zebra-consensus/src/block.rs
+++ b/zebra-consensus/src/block.rs
@@ -120,10 +120,9 @@ where
 
         // TODO(jlusby): Error = Report, handle errors from state_service.
         async move {
-            tracing::trace!("beginning block verification");
-
             let hash = block.hash();
             // Check that this block is actually a new block.
+            tracing::trace!("checking that block is not already in state");
             match state_service
                 .ready_and()
                 .await
@@ -139,6 +138,7 @@ where
                 _ => unreachable!("wrong response to Request::Depth"),
             }
 
+            tracing::trace!("performing block checks");
             let height = block
                 .coinbase_height()
                 .ok_or(BlockError::MissingHeight(hash))?;
@@ -174,9 +174,11 @@ where
                     .call(req);
                 async_checks.push(rsp);
             }
+            tracing::trace!(len = async_checks.len(), "built async tx checks");
 
             use futures::StreamExt;
             while let Some(result) = async_checks.next().await {
+                tracing::trace!(?result, remaining = async_checks.len());
                 result.map_err(VerifyBlockError::Transaction)?;
             }
 

From 77b60f3a30d684ee035413c53ca494acef16d78e Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Fri, 20 Nov 2020 15:18:20 -0800
Subject: [PATCH 081/148] state: add traces for utxo scanning

---
 zebra-state/src/service/utxo.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/zebra-state/src/service/utxo.rs b/zebra-state/src/service/utxo.rs
index 8f4736ad59d..4ade4c536b8 100644
--- a/zebra-state/src/service/utxo.rs
+++ b/zebra-state/src/service/utxo.rs
@@ -37,6 +37,9 @@ impl PendingUtxos {
     /// by the given `transparent::OutPoint` that the `Output` has arrived.
     pub fn respond(&mut self, outpoint: transparent::OutPoint, output: transparent::Output) {
         if let Some(sender) = self.0.remove(&outpoint) {
+            // Adding the outpoint as a field lets us crossreference
+            // with the trace of the verification that made the request.
+            tracing::trace!(?outpoint, "found pending UTXO");
             let _ = sender.send(output);
         }
     }
@@ -48,6 +51,7 @@ impl PendingUtxos {
             return;
         }
 
+        tracing::trace!("scanning new block for pending UTXOs");
         for transaction in block.transactions.iter() {
             let transaction_hash = transaction.hash();
             for (index, output) in transaction.outputs().iter().enumerate() {

From f0810b028d9359035e5f45cde8ab90aa7766f4d6 Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Fri, 20 Nov 2020 19:52:44 -0800
Subject: [PATCH 082/148] state,consensus,sync: shorten span lengths

These changes help reduce the size of the resulting spans, making the
output more compact.  Together they save about 30-40 characters.
---
 zebra-consensus/src/transaction.rs         | 2 +-
 zebra-state/src/service.rs                 | 6 +++---
 zebra-state/src/service/finalized_state.rs | 2 +-
 zebrad/src/components/sync/downloads.rs    | 2 +-
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/zebra-consensus/src/transaction.rs b/zebra-consensus/src/transaction.rs
index d44f1a74664..39ca1a6655a 100644
--- a/zebra-consensus/src/transaction.rs
+++ b/zebra-consensus/src/transaction.rs
@@ -88,7 +88,7 @@ where
 
         let mut redjubjub_verifier = crate::primitives::redjubjub::VERIFIER.clone();
         let mut script_verifier = self.script_verifier.clone();
-        let span = tracing::debug_span!("tx", hash = ?tx.hash());
+        let span = tracing::debug_span!("tx", hash = %tx.hash());
         async move {
             tracing::trace!(?tx);
             match &*tx {
diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs
index 5593cec27b8..e1a0339a33b 100644
--- a/zebra-state/src/service.rs
+++ b/zebra-state/src/service.rs
@@ -86,7 +86,7 @@ impl StateService {
     ///
     /// [1]: https://zebra.zfnd.org/dev/rfcs/0005-state-updates.html#committing-non-finalized-blocks
     #[instrument(skip(self, block))]
-    fn queue_and_commit_non_finalized_blocks(
+    fn queue_and_commit_non_finalized(
         &mut self,
         block: Arc<Block>,
     ) -> oneshot::Receiver<Result<block::Hash, BoxError>> {
@@ -390,7 +390,7 @@ impl Service<Request> for StateService {
                 metrics::counter!("state.requests", 1, "type" => "commit_block");
 
                 self.pending_utxos.check_block(&block);
-                let rsp_rx = self.queue_and_commit_non_finalized_blocks(block);
+                let rsp_rx = self.queue_and_commit_non_finalized(block);
 
                 async move {
                     rsp_rx
@@ -408,7 +408,7 @@ impl Service<Request> for StateService {
 
                 self.pending_utxos.check_block(&block);
                 self.disk
-                    .queue_and_commit_finalized_blocks(QueuedBlock { block, rsp_tx });
+                    .queue_and_commit_finalized(QueuedBlock { block, rsp_tx });
 
                 async move {
                     rsp_rx
diff --git a/zebra-state/src/service/finalized_state.rs b/zebra-state/src/service/finalized_state.rs
index 2cd3c8e749c..dc3e2b0cd0f 100644
--- a/zebra-state/src/service/finalized_state.rs
+++ b/zebra-state/src/service/finalized_state.rs
@@ -92,7 +92,7 @@ impl FinalizedState {
     ///
     /// After queueing a finalized block, this method checks whether the newly
     /// queued block (and any of its descendants) can be committed to the state.
-    pub fn queue_and_commit_finalized_blocks(&mut self, queued_block: QueuedBlock) {
+    pub fn queue_and_commit_finalized(&mut self, queued_block: QueuedBlock) {
         let prev_hash = queued_block.block.header.previous_block_hash;
         let height = queued_block.block.coinbase_height().unwrap();
         self.queued_by_prev_hash.insert(prev_hash, queued_block);
diff --git a/zebrad/src/components/sync/downloads.rs b/zebrad/src/components/sync/downloads.rs
index 177a27c0a18..b3ae4169098 100644
--- a/zebrad/src/components/sync/downloads.rs
+++ b/zebrad/src/components/sync/downloads.rs
@@ -110,7 +110,7 @@ where
     /// This method waits for the network to become ready, and returns an error
     /// only if the network service fails. It returns immediately after queuing
     /// the request.
-    #[instrument(skip(self))]
+    #[instrument(skip(self), fields(%hash))]
     pub async fn download_and_verify(&mut self, hash: block::Hash) -> Result<(), Report> {
         if self.cancel_handles.contains_key(&hash) {
             return Err(eyre!("duplicate hash queued for download"));

From 36cd76d59049e479d8fb342101c578e9a3474990 Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Fri, 20 Nov 2020 19:54:57 -0800
Subject: [PATCH 083/148] state: tidy process_queued tracing

Previously, this function was instrumented with a span containing the
parent hash that was the entry to the function.  But it doesn't make
sense to consider the work done by the function as happening in the
context of the supplied parent hash (as distinct from the context of the
hash of the newly arrived block, which is already contained in an outer
span), so this adds noise without conveying extra context.

Instead, use events that occur within the context of the existing spans.
---
 zebra-state/src/service.rs | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs
index e1a0339a33b..b50f6282996 100644
--- a/zebra-state/src/service.rs
+++ b/zebra-state/src/service.rs
@@ -167,21 +167,21 @@ impl StateService {
 
     /// Attempt to validate and commit all queued blocks whose parents have
     /// recently arrived starting from `new_parent`, in breadth-first ordering.
-    #[instrument(skip(self))]
     fn process_queued(&mut self, new_parent: block::Hash) {
         let mut new_parents = vec![new_parent];
 
-        while let Some(parent) = new_parents.pop() {
-            let queued_children = self.queued_blocks.dequeue_children(parent);
+        while let Some(parent_hash) = new_parents.pop() {
+            let queued_children = self.queued_blocks.dequeue_children(parent_hash);
 
             for QueuedBlock { block, rsp_tx } in queued_children {
-                let hash = block.hash();
+                let child_hash = block.hash();
+                tracing::trace!(?child_hash, "validating queued child");
                 let result = self
                     .validate_and_commit(block)
-                    .map(|()| hash)
+                    .map(|()| child_hash)
                     .map_err(BoxError::from);
                 let _ = rsp_tx.send(result);
-                new_parents.push(hash);
+                new_parents.push(child_hash);
             }
         }
     }

From 3192a5008d8ef90931d73ab4e362c0d6feccaccb Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Fri, 20 Nov 2020 19:58:07 -0800
Subject: [PATCH 084/148] state: add additional traces to block commit logic

---
 zebra-state/src/service.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs
index b50f6282996..62ad137c625 100644
--- a/zebra-state/src/service.rs
+++ b/zebra-state/src/service.rs
@@ -103,6 +103,7 @@ impl StateService {
         // been queued but not yet committed to the state fails the older
         // request and replaces it with the newer request.
         let rsp_rx = if let Some(queued_block) = self.queued_blocks.get_mut(&hash) {
+            tracing::debug!("replacing older queued request with new request");
             let (mut rsp_tx, rsp_rx) = oneshot::channel();
             std::mem::swap(&mut queued_block.rsp_tx, &mut rsp_tx);
             let _ = rsp_tx.send(Err("replaced by newer request".into()));
@@ -114,12 +115,14 @@ impl StateService {
         };
 
         if !self.can_fork_chain_at(&parent_hash) {
+            tracing::trace!("unready to verify, returning early");
             return rsp_rx;
         }
 
         self.process_queued(parent_hash);
 
         while self.mem.best_chain_len() > crate::constants::MAX_BLOCK_REORG_HEIGHT {
+            tracing::trace!("finalizing block past the reorg limit");
             let finalized = self.mem.finalize();
             self.disk
                 .commit_finalized_direct(finalized)
@@ -131,6 +134,7 @@ impl StateService {
             "Finalized state must have at least one block before committing non-finalized state",
         ));
 
+        tracing::trace!("finished processing queued block");
         rsp_rx
     }
 

From 719a48ad9e7521a8671930a0bdc1f54155bbfad1 Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Fri, 20 Nov 2020 19:58:39 -0800
Subject: [PATCH 085/148] state: shorten tracing messages

Make tracing messages more concise by omitting information already
contained in a parent span and by shortening messages.  This makes them
easier to read.
---
 .../src/service/non_finalized_state/chain.rs       |  4 +---
 .../service/non_finalized_state/queued_blocks.rs   | 14 +++++++++-----
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/zebra-state/src/service/non_finalized_state/chain.rs b/zebra-state/src/service/non_finalized_state/chain.rs
index 82b85bff658..f3e61a872ca 100644
--- a/zebra-state/src/service/non_finalized_state/chain.rs
+++ b/zebra-state/src/service/non_finalized_state/chain.rs
@@ -35,12 +35,10 @@ impl Chain {
         let block_height = block
             .coinbase_height()
             .expect("valid non-finalized blocks have a coinbase height");
-
-        trace!(?block_height, "Pushing new block into chain state");
-
         // update cumulative data members
         self.update_chain_state_with(&block);
         self.blocks.insert(block_height, block);
+        trace!("pushed block onto chain");
     }
 
     /// Remove the lowest height block of the non-finalized portion of a chain.
diff --git a/zebra-state/src/service/non_finalized_state/queued_blocks.rs b/zebra-state/src/service/non_finalized_state/queued_blocks.rs
index ec16d807c50..783ddeb05ba 100644
--- a/zebra-state/src/service/non_finalized_state/queued_blocks.rs
+++ b/zebra-state/src/service/non_finalized_state/queued_blocks.rs
@@ -48,17 +48,17 @@ impl QueuedBlocks {
             .insert(new_hash);
         assert!(inserted, "hashes must be unique");
 
-        tracing::trace!(num_blocks = %self.blocks.len(), %parent_hash, ?new_height,  "Finished queueing a new block");
+        tracing::trace!(%parent_hash, queued = %self.blocks.len(), "queued block");
         self.update_metrics();
     }
 
     /// Dequeue and return all blocks that were waiting for the arrival of
     /// `parent`.
-    #[instrument(skip(self))]
-    pub fn dequeue_children(&mut self, parent: block::Hash) -> Vec<QueuedBlock> {
+    #[instrument(skip(self), fields(%parent_hash))]
+    pub fn dequeue_children(&mut self, parent_hash: block::Hash) -> Vec<QueuedBlock> {
         let queued_children = self
             .by_parent
-            .remove(&parent)
+            .remove(&parent_hash)
             .unwrap_or_default()
             .into_iter()
             .map(|hash| {
@@ -73,7 +73,11 @@ impl QueuedBlocks {
             self.by_height.remove(&height);
         }
 
-        tracing::trace!(num_blocks = %self.blocks.len(), "Finished dequeuing blocks waiting for parent hash",);
+        tracing::trace!(
+            dequeued = queued_children.len(),
+            remaining = self.blocks.len(),
+            "dequeued blocks"
+        );
         self.update_metrics();
 
         queued_children

From a2a70e1fc199e465dd9c22906ca4b7148c340edf Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@deimos.local>
Date: Fri, 20 Nov 2020 19:47:30 -0800
Subject: [PATCH 086/148] consensus: fix same-block utxo lookups

The UTXO query system assumes that a transaction will only request
information about UTXOs created in prior blocks.  But transactions are
allowed to spend UTXOs created by prior transactions in the same block.

This doesn't fit with the existing query model, so instead of trying to
change it, allow the script verifier to take an additional set of known
UTXOs, and propagate this set from the block.
---
 Cargo.lock                         |  4 ++--
 zebra-consensus/src/block.rs       | 22 ++++++++++++++++++--
 zebra-consensus/src/script.rs      | 32 +++++++++++++++++-------------
 zebra-consensus/src/transaction.rs | 31 ++++++++++++++++++++++-------
 4 files changed, 64 insertions(+), 25 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 346da0458fb..398349c1fd9 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2744,7 +2744,7 @@ dependencies = [
 [[package]]
 name = "tower"
 version = "0.4.0"
-source = "git+https://github.com/tower-rs/tower?rev=5e1e07744820028877654c336a3b9fe057bf46f1#5e1e07744820028877654c336a3b9fe057bf46f1"
+source = "git+https://github.com/tower-rs/tower?rev=d4d1c67c6a0e4213a52abcc2b9df6cc58276ee39#d4d1c67c6a0e4213a52abcc2b9df6cc58276ee39"
 dependencies = [
  "futures-core",
  "futures-util",
@@ -2789,7 +2789,7 @@ dependencies = [
 [[package]]
 name = "tower-layer"
 version = "0.3.0"
-source = "git+https://github.com/tower-rs/tower?rev=5e1e07744820028877654c336a3b9fe057bf46f1#5e1e07744820028877654c336a3b9fe057bf46f1"
+source = "git+https://github.com/tower-rs/tower?rev=d4d1c67c6a0e4213a52abcc2b9df6cc58276ee39#d4d1c67c6a0e4213a52abcc2b9df6cc58276ee39"
 
 [[package]]
 name = "tower-service"
diff --git a/zebra-consensus/src/block.rs b/zebra-consensus/src/block.rs
index bcd3a7d6de1..acdd56538b6 100644
--- a/zebra-consensus/src/block.rs
+++ b/zebra-consensus/src/block.rs
@@ -8,6 +8,7 @@
 //! verification, where it may be accepted or rejected.
 
 use std::{
+    collections::HashMap,
     future::Future,
     pin::Pin,
     sync::Arc,
@@ -25,6 +26,7 @@ use zebra_chain::{
     block::{self, Block},
     parameters::Network,
     parameters::NetworkUpgrade,
+    transparent,
     work::equihash,
 };
 use zebra_state as zs;
@@ -165,13 +167,16 @@ where
 
             let mut async_checks = FuturesUnordered::new();
 
+            let known_utxos = known_utxos(&block);
             for transaction in &block.transactions {
-                let req = transaction::Request::Block(transaction.clone());
                 let rsp = transaction_verifier
                     .ready_and()
                     .await
                     .expect("transaction verifier is always ready")
-                    .call(req);
+                    .call(transaction::Request::Block {
+                        transaction: transaction.clone(),
+                        known_utxos: known_utxos.clone(),
+                    });
                 async_checks.push(rsp);
             }
             tracing::trace!(len = async_checks.len(), "built async tx checks");
@@ -207,3 +212,16 @@ where
         .boxed()
     }
 }
+
+fn known_utxos(block: &Block) -> Arc<HashMap<transparent::OutPoint, transparent::Output>> {
+    let mut map = HashMap::default();
+    for transaction in &block.transactions {
+        let hash = transaction.hash();
+        for (index, output) in transaction.outputs().iter().cloned().enumerate() {
+            let index = index as u32;
+            map.insert(transparent::OutPoint { hash, index }, output);
+        }
+    }
+
+    Arc::new(map)
+}
diff --git a/zebra-consensus/src/script.rs b/zebra-consensus/src/script.rs
index ee0876e7047..36b986a16c9 100644
--- a/zebra-consensus/src/script.rs
+++ b/zebra-consensus/src/script.rs
@@ -1,4 +1,4 @@
-use std::{future::Future, pin::Pin, sync::Arc};
+use std::{collections::HashMap, future::Future, pin::Pin, sync::Arc};
 
 use tracing::Instrument;
 
@@ -40,6 +40,7 @@ impl<ZS> Verifier<ZS> {
 pub struct Request {
     pub transaction: Arc<Transaction>,
     pub input_index: usize,
+    pub known_utxos: Arc<HashMap<transparent::OutPoint, transparent::Output>>,
 }
 
 impl<ZS> tower::Service<Request> for Verifier<ZS>
@@ -62,32 +63,35 @@ where
     fn call(&mut self, req: Request) -> Self::Future {
         use futures_util::FutureExt;
 
-        let input = &req.transaction.inputs()[req.input_index];
+        let Request {
+            transaction,
+            input_index,
+            known_utxos,
+        } = req;
+        let input = &transaction.inputs()[input_index];
 
         match input {
             transparent::Input::PrevOut { outpoint, .. } => {
                 let outpoint = *outpoint;
-                let transaction = req.transaction;
                 let branch_id = self.branch;
-                let input_index = req.input_index;
 
                 let span = tracing::trace_span!("script", ?outpoint);
-                let output =
+                let query =
                     span.in_scope(|| self.state.call(zebra_state::Request::AwaitUtxo(outpoint)));
 
                 async move {
                     tracing::trace!("awaiting outpoint lookup");
-                    let previous_output = match output.await? {
-                        zebra_state::Response::Utxo(output) => output,
-                        _ => unreachable!("AwaitUtxo always responds with Utxo"),
+                    let output = if let Some(output) = known_utxos.get(&outpoint) {
+                        tracing::trace!("UXTO in known_utxos, discarding query");
+                        output.clone()
+                    } else if let zebra_state::Response::Utxo(output) = query.await? {
+                        output
+                    } else {
+                        unreachable!("AwaitUtxo always responds with Utxo")
                     };
-                    tracing::trace!(?previous_output, "got UTXO");
+                    tracing::trace!(?output, "got UTXO");
 
-                    zebra_script::is_valid(
-                        transaction,
-                        branch_id,
-                        (input_index as u32, previous_output),
-                    )?;
+                    zebra_script::is_valid(transaction, branch_id, (input_index as u32, output))?;
                     tracing::trace!("script verification succeeded");
 
                     Ok(())
diff --git a/zebra-consensus/src/transaction.rs b/zebra-consensus/src/transaction.rs
index 39ca1a6655a..dd33d327b51 100644
--- a/zebra-consensus/src/transaction.rs
+++ b/zebra-consensus/src/transaction.rs
@@ -1,4 +1,5 @@
 use std::{
+    collections::HashMap,
     future::Future,
     pin::Pin,
     sync::Arc,
@@ -15,6 +16,7 @@ use tracing::Instrument;
 use zebra_chain::{
     parameters::NetworkUpgrade,
     transaction::{self, HashType, Transaction},
+    transparent,
 };
 
 use zebra_state as zs;
@@ -51,9 +53,17 @@ where
 #[allow(dead_code)]
 pub enum Request {
     /// Verify the supplied transaction as part of a block.
-    Block(Arc<Transaction>),
+    Block {
+        transaction: Arc<Transaction>,
+        /// Additional UTXOs which are known at the time of verification.
+        known_utxos: Arc<HashMap<transparent::OutPoint, transparent::Output>>,
+    },
     /// Verify the supplied transaction as part of the mempool.
-    Mempool(Arc<Transaction>),
+    Mempool {
+        transaction: Arc<Transaction>,
+        /// Additional UTXOs which are known at the time of verification.
+        known_utxos: Arc<HashMap<transparent::OutPoint, transparent::Output>>,
+    },
 }
 
 impl<ZS> Service<Request> for Verifier<ZS>
@@ -73,17 +83,23 @@ where
     // TODO: break up each chunk into its own method
     fn call(&mut self, req: Request) -> Self::Future {
         let is_mempool = match req {
-            Request::Block(_) => false,
-            Request::Mempool(_) => true,
+            Request::Block { .. } => false,
+            Request::Mempool { .. } => true,
         };
         if is_mempool {
             // XXX determine exactly which rules apply to mempool transactions
             unimplemented!();
         }
 
-        let tx = match req {
-            Request::Block(tx) => tx,
-            Request::Mempool(tx) => tx,
+        let (tx, known_utxos) = match req {
+            Request::Block {
+                transaction,
+                known_utxos,
+            } => (transaction, known_utxos),
+            Request::Mempool {
+                transaction,
+                known_utxos,
+            } => (transaction, known_utxos),
         };
 
         let mut redjubjub_verifier = crate::primitives::redjubjub::VERIFIER.clone();
@@ -119,6 +135,7 @@ where
                         // feed all of the inputs to the script verifier
                         for input_index in 0..inputs.len() {
                             let rsp = script_verifier.ready_and().await?.call(script::Request {
+                                known_utxos: known_utxos.clone(),
                                 transaction: tx.clone(),
                                 input_index,
                             });

From e4f38582fdacbac562b6b04ad0a565ba2968b43e Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Fri, 20 Nov 2020 22:48:21 -0800
Subject: [PATCH 087/148] consensus: add remaining checks trace counter to tx
 verifier

---
 zebra-consensus/src/transaction.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/zebra-consensus/src/transaction.rs b/zebra-consensus/src/transaction.rs
index dd33d327b51..313ff3bcf24 100644
--- a/zebra-consensus/src/transaction.rs
+++ b/zebra-consensus/src/transaction.rs
@@ -219,6 +219,7 @@ where
                     // Finally, wait for all asynchronous checks to complete
                     // successfully, or fail verification if they error.
                     while let Some(check) = async_checks.next().await {
+                        tracing::trace!(?check, remaining = async_checks.len());
                         check?;
                     }
 

From 3f78476693310b4c09e3999c472ab04ffa24b562 Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Fri, 20 Nov 2020 22:50:09 -0800
Subject: [PATCH 088/148] state: check queued blocks for known UTXOs

The behavior of a request for a UTXO from a previous block depends on
whether that block has already been submitted to the state, or not:

* if it has, the state should be able to find it and answer immediately.
* if it has not, the state should see it in a later request.

However, the previous code only checked committed blocks, not queued
blocks, so if the block containing the UTXO had already arrived but had
not been committed, it would never be scanned.

This patch fixes the problem but is a bad solution, duplicating
computation between the block verifier and the state.  A better fix
follows in the next commit.
---
 zebra-state/src/service.rs                    |  5 ++-
 .../non_finalized_state/queued_blocks.rs      | 45 +++++++++++++++++--
 2 files changed, 46 insertions(+), 4 deletions(-)

diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs
index 62ad137c625..da559f7ec55 100644
--- a/zebra-state/src/service.rs
+++ b/zebra-state/src/service.rs
@@ -262,7 +262,10 @@ impl StateService {
 
     /// Return the utxo pointed to by `outpoint` if it exists in any chain.
     pub fn utxo(&self, outpoint: &transparent::OutPoint) -> Option<transparent::Output> {
-        self.mem.utxo(outpoint).or_else(|| self.disk.utxo(outpoint))
+        self.mem
+            .utxo(outpoint)
+            .or_else(|| self.disk.utxo(outpoint))
+            .or_else(|| self.queued_blocks.utxo(outpoint))
     }
 
     /// Return an iterator over the relevant chain of the block identified by
diff --git a/zebra-state/src/service/non_finalized_state/queued_blocks.rs b/zebra-state/src/service/non_finalized_state/queued_blocks.rs
index 783ddeb05ba..790c1238c1e 100644
--- a/zebra-state/src/service/non_finalized_state/queued_blocks.rs
+++ b/zebra-state/src/service/non_finalized_state/queued_blocks.rs
@@ -4,7 +4,7 @@ use std::{
 };
 
 use tracing::instrument;
-use zebra_chain::block;
+use zebra_chain::{block, transparent};
 
 use crate::service::QueuedBlock;
 
@@ -17,6 +17,8 @@ pub struct QueuedBlocks {
     by_parent: HashMap<block::Hash, HashSet<block::Hash>>,
     /// Hashes from `queued_blocks`, indexed by block height.
     by_height: BTreeMap<block::Height, HashSet<block::Hash>>,
+    /// Known UTXOs.
+    known_utxos: HashMap<transparent::OutPoint, transparent::Output>,
 }
 
 impl QueuedBlocks {
@@ -33,6 +35,21 @@ impl QueuedBlocks {
             .expect("validated non-finalized blocks have a coinbase height");
         let parent_hash = new.block.header.previous_block_hash;
 
+        // XXX QueuedBlock should include this data
+        let prev_utxo_count = self.known_utxos.len();
+        for transaction in &new.block.transactions {
+            let hash = transaction.hash();
+            for (index, output) in transaction.outputs().iter().cloned().enumerate() {
+                let index = index as u32;
+                self.known_utxos
+                    .insert(transparent::OutPoint { hash, index }, output);
+            }
+        }
+        tracing::trace!(
+            known_utxos = self.known_utxos.len(),
+            new = self.known_utxos.len() - prev_utxo_count
+        );
+
         let replaced = self.blocks.insert(new_hash, new);
         assert!(replaced.is_none(), "hashes must be unique");
         let inserted = self
@@ -62,9 +79,26 @@ impl QueuedBlocks {
             .unwrap_or_default()
             .into_iter()
             .map(|hash| {
-                self.blocks
+                let queued = self
+                    .blocks
                     .remove(&hash)
-                    .expect("block is present if its hash is in by_parent")
+                    .expect("block is present if its hash is in by_parent");
+
+                let prev_utxo_count = self.known_utxos.len();
+                for transaction in &queued.block.transactions {
+                    let hash = transaction.hash();
+                    for (index, _output) in transaction.outputs().iter().cloned().enumerate() {
+                        let index = index as u32;
+                        self.known_utxos
+                            .remove(&transparent::OutPoint { hash, index });
+                    }
+                }
+                tracing::trace!(
+                    known_utxos = self.known_utxos.len(),
+                    removed = prev_utxo_count - self.known_utxos.len()
+                );
+
+                queued
             })
             .collect::<Vec<_>>();
 
@@ -142,6 +176,11 @@ impl QueuedBlocks {
         }
         metrics::gauge!("state.memory.queued.block.count", self.blocks.len() as _);
     }
+
+    /// Try to look up this UTXO in any queued block.
+    pub fn utxo(&self, outpoint: &transparent::OutPoint) -> Option<transparent::Output> {
+        self.known_utxos.get(outpoint).cloned()
+    }
 }
 
 #[cfg(test)]

From e0817d174739da79c2c3b9b70f657ca3fb05a26f Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Sat, 21 Nov 2020 01:16:14 -0800
Subject: [PATCH 089/148] state: introduce PreparedBlock, FinalizedBlock

This change introduces two new types:

- `PreparedBlock`, representing a block which has undergone semantic
  validation and has been prepared for contextual validation;
- `FinalizedBlock`, representing a block which is ready to be finalized
  immediately;

and changes the `Request::CommitBlock`,`Request::CommitFinalizedBlock`
variants to use these types instead of their previous fields.

This change solves the problem of passing data between semantic
validation and contextual validation, and cleans up the state code by
allowing it to pass around a bundle of data.  Previously, the state code
just passed around an `Arc<Block>`, which forced it to needlessly
recompute block hashes and other data, and was incompatible with the
already-known but not-yet-implemented data transfer requirements, namely
passing in the Sprout and Sapling anchors computed during contextual
validation.

This commit propagates the `PreparedBlock` and `FinalizedBlock` types
through the state code but only uses their data opportunistically, e.g.,
changing .hash() computations to use the precomputed hash.  In the
future, these structures can be extended to pass data through the
verification pipeline for reuse as appropriate.  For instance, these
changes allow the sprout and sapling anchors to be propagated through
the state.
---
 zebra-consensus/src/block.rs                  |  10 +-
 zebra-consensus/src/checkpoint.rs             |   2 +-
 zebra-consensus/src/checkpoint/tests.rs       |   6 +-
 zebra-state/src/lib.rs                        |   2 +-
 zebra-state/src/request.rs                    |  83 +++++++++++---
 zebra-state/src/service.rs                    |  99 +++++++---------
 zebra-state/src/service/check.rs              |  68 +++++------
 zebra-state/src/service/finalized_state.rs    |  43 ++++---
 .../src/service/non_finalized_state.rs        | 108 ++++++++++--------
 .../src/service/non_finalized_state/chain.rs  |  73 ++++++------
 .../non_finalized_state/queued_blocks.rs      |  66 +++--------
 zebra-state/src/service/tests.rs              |   2 +-
 zebra-state/src/service/utxo.rs               |  17 ++-
 zebra-state/src/tests.rs                      |  31 ++++-
 zebra-state/tests/basic.rs                    |   4 +-
 15 files changed, 331 insertions(+), 283 deletions(-)

diff --git a/zebra-consensus/src/block.rs b/zebra-consensus/src/block.rs
index acdd56538b6..564ae804f94 100644
--- a/zebra-consensus/src/block.rs
+++ b/zebra-consensus/src/block.rs
@@ -193,11 +193,19 @@ where
             metrics::counter!("block.verified.block.count", 1);
 
             // Finally, submit the block for contextual verification.
+            let new_outputs = Arc::try_unwrap(known_utxos)
+                .expect("all verification tasks using known_utxos are complete");
+            let prepared_block = zs::PreparedBlock {
+                block,
+                hash,
+                height,
+                new_outputs,
+            };
             match state_service
                 .ready_and()
                 .await
                 .map_err(VerifyBlockError::Commit)?
-                .call(zs::Request::CommitBlock { block })
+                .call(zs::Request::CommitBlock(prepared_block))
                 .await
                 .map_err(VerifyBlockError::Commit)?
             {
diff --git a/zebra-consensus/src/checkpoint.rs b/zebra-consensus/src/checkpoint.rs
index 0e1422b83b9..ec98e0ada17 100644
--- a/zebra-consensus/src/checkpoint.rs
+++ b/zebra-consensus/src/checkpoint.rs
@@ -852,7 +852,7 @@ where
                 .ready_and()
                 .await
                 .expect("Verified checkpoints must be committed transactionally")
-                .call(zs::Request::CommitFinalizedBlock { block })
+                .call(zs::Request::CommitFinalizedBlock(block.into()))
                 .await
                 .expect("Verified checkpoints must be committed transactionally")
             {
diff --git a/zebra-consensus/src/checkpoint/tests.rs b/zebra-consensus/src/checkpoint/tests.rs
index 535d9461b6d..d29d50693e5 100644
--- a/zebra-consensus/src/checkpoint/tests.rs
+++ b/zebra-consensus/src/checkpoint/tests.rs
@@ -320,9 +320,9 @@ async fn continuous_blockchain(restart_height: Option<block::Height>) -> Result<
 
                     /// SPANDOC: Add block to the state {?height}
                     ready_state_service
-                        .call(zebra_state::Request::CommitFinalizedBlock {
-                            block: block.clone(),
-                        })
+                        .call(zebra_state::Request::CommitFinalizedBlock(
+                            block.clone().into(),
+                        ))
                         .await
                         .map_err(|e| eyre!(e))?;
                 }
diff --git a/zebra-state/src/lib.rs b/zebra-state/src/lib.rs
index cdf6c339280..f97a82c2a11 100644
--- a/zebra-state/src/lib.rs
+++ b/zebra-state/src/lib.rs
@@ -23,6 +23,6 @@ mod tests;
 pub use config::Config;
 pub use constants::MAX_BLOCK_REORG_HEIGHT;
 pub use error::{BoxError, CloneError, CommitBlockError, ValidateContextError};
-pub use request::{HashOrHeight, Request};
+pub use request::{FinalizedBlock, HashOrHeight, PreparedBlock, Request};
 pub use response::Response;
 pub use service::init;
diff --git a/zebra-state/src/request.rs b/zebra-state/src/request.rs
index 761677351f8..7c78112ad63 100644
--- a/zebra-state/src/request.rs
+++ b/zebra-state/src/request.rs
@@ -1,4 +1,5 @@
-use std::sync::Arc;
+use std::{collections::HashMap, sync::Arc};
+
 use zebra_chain::{
     block::{self, Block},
     transaction, transparent,
@@ -47,6 +48,66 @@ impl From<block::Height> for HashOrHeight {
     }
 }
 
+/// A block which has undergone semantic validation and has been prepared for
+/// contextual validation.
+///
+/// It is the constructor's responsibility to perform semantic validation and to
+/// ensure that all fields are consistent.
+///
+/// This structure contains data from contextual validation, which is computed in
+/// the *service caller*'s task, not inside the service call itself. This allows
+/// moving work out of the single-threaded state service.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct PreparedBlock {
+    /// The block to commit to the state.
+    pub block: Arc<Block>,
+    /// The hash of the block.
+    pub hash: block::Hash,
+    /// The height of the block.
+    pub height: block::Height,
+    /// New transparent outputs created in this block, indexed by
+    /// [`Outpoint`](transparent::Outpoint).
+    ///
+    /// Note: although these transparent outputs are newly created, they may not
+    /// be unspent, since a later transaction in a block can spend outputs of an
+    /// earlier transaction.
+    pub new_outputs: HashMap<transparent::OutPoint, transparent::Output>,
+    // TODO: add these parameters when we can compute anchors.
+    // sprout_anchor: sprout::tree::Root,
+    // sapling_anchor: sapling::tree::Root,
+}
+
+/// A finalized block, ready to be committed directly to the finalized state with
+/// no checks.
+///
+/// This is exposed for use in checkpointing.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct FinalizedBlock {
+    // These are pub(crate) so we can add whatever db-format-dependent
+    // precomputation we want here without leaking internal details.
+    pub(crate) block: Arc<Block>,
+    pub(crate) hash: block::Hash,
+    pub(crate) height: block::Height,
+}
+
+// Doing precomputation in this From impl means that it will be done in
+// the *service caller*'s task, not inside the service call itself.
+// This allows moving work out of the single-threaded state service.
+impl From<Arc<Block>> for FinalizedBlock {
+    fn from(block: Arc<Block>) -> Self {
+        let height = block
+            .coinbase_height()
+            .expect("finalized blocks must have a valid coinbase height");
+        let hash = block.hash();
+
+        Self {
+            block,
+            height,
+            hash,
+        }
+    }
+}
+
 #[derive(Clone, Debug, PartialEq, Eq)]
 /// A query about or modification to the chain state.
 pub enum Request {
@@ -65,13 +126,7 @@ pub enum Request {
     /// future will have no effect on whether it is eventually processed. A
     /// request to commit a block which has been queued internally but not yet
     /// committed will fail the older request and replace it with the newer request.
-    CommitBlock {
-        /// The block to commit to the state.
-        block: Arc<Block>,
-        // TODO: add these parameters when we can compute anchors.
-        // sprout_anchor: sprout::tree::Root,
-        // sapling_anchor: sapling::tree::Root,
-    },
+    CommitBlock(PreparedBlock),
 
     /// Commit a finalized block to the state, skipping all validation.
     ///
@@ -87,13 +142,7 @@ pub enum Request {
     /// future will have no effect on whether it is eventually processed.
     /// Duplicate requests should not be made, because it is the caller's
     /// responsibility to ensure that each block is valid and final.
-    CommitFinalizedBlock {
-        /// The block to commit to the state.
-        block: Arc<Block>,
-        // TODO: add these parameters when we can compute anchors.
-        // sprout_anchor: sprout::tree::Root,
-        // sapling_anchor: sapling::tree::Root,
-    },
+    CommitFinalizedBlock(FinalizedBlock),
 
     /// Computes the depth in the current best chain of the block identified by the given hash.
     ///
@@ -140,8 +189,8 @@ pub enum Request {
     /// available if it is unknown.
     ///
     /// This request is purely informational, and there are no guarantees about
-    /// whether the UTXO remains unspent or is on the best chain. Its purpose is
-    /// to allow asynchronous script verification.
+    /// whether the UTXO remains unspent or is on the best chain, or any chain.
+    /// Its purpose is to allow asynchronous script verification.
     ///
     /// Code making this request should apply a timeout layer to the service to
     /// handle missing UTXOs.
diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs
index da559f7ec55..0b3d1351512 100644
--- a/zebra-state/src/service.rs
+++ b/zebra-state/src/service.rs
@@ -3,8 +3,7 @@ use std::{
     pin::Pin,
     sync::Arc,
     task::{Context, Poll},
-    time::Duration,
-    time::Instant,
+    time::{Duration, Instant},
 };
 
 use futures::future::FutureExt;
@@ -21,8 +20,8 @@ use zebra_chain::{
 };
 
 use crate::{
-    request::HashOrHeight, BoxError, CommitBlockError, Config, Request, Response,
-    ValidateContextError,
+    request::HashOrHeight, BoxError, CommitBlockError, Config, FinalizedBlock, PreparedBlock,
+    Request, Response, ValidateContextError,
 };
 
 use self::finalized_state::FinalizedState;
@@ -34,15 +33,14 @@ mod non_finalized_state;
 mod tests;
 mod utxo;
 
-// todo: put this somewhere
-#[derive(Debug)]
-pub struct QueuedBlock {
-    pub block: Arc<Block>,
-    // TODO: add these parameters when we can compute anchors.
-    // sprout_anchor: sprout::tree::Root,
-    // sapling_anchor: sapling::tree::Root,
-    pub rsp_tx: oneshot::Sender<Result<block::Hash, BoxError>>,
-}
+pub type QueuedBlock = (
+    PreparedBlock,
+    oneshot::Sender<Result<block::Hash, BoxError>>,
+);
+pub type QueuedFinalized = (
+    FinalizedBlock,
+    oneshot::Sender<Result<block::Hash, BoxError>>,
+);
 
 struct StateService {
     /// Holds data relating to finalized chain state.
@@ -85,15 +83,15 @@ impl StateService {
     /// in RFC0005.
     ///
     /// [1]: https://zebra.zfnd.org/dev/rfcs/0005-state-updates.html#committing-non-finalized-blocks
-    #[instrument(skip(self, block))]
+    #[instrument(skip(self, prepared))]
     fn queue_and_commit_non_finalized(
         &mut self,
-        block: Arc<Block>,
+        prepared: PreparedBlock,
     ) -> oneshot::Receiver<Result<block::Hash, BoxError>> {
-        let hash = block.hash();
-        let parent_hash = block.header.previous_block_hash;
+        let parent_hash = prepared.block.header.previous_block_hash;
 
-        if self.contains_committed_block(&block) {
+        if self.mem.any_chain_contains(&prepared.hash) || self.disk.hash(prepared.height).is_some()
+        {
             let (rsp_tx, rsp_rx) = oneshot::channel();
             let _ = rsp_tx.send(Err("block is already committed to the state".into()));
             return rsp_rx;
@@ -102,15 +100,15 @@ impl StateService {
         // Request::CommitBlock contract: a request to commit a block which has
         // been queued but not yet committed to the state fails the older
         // request and replaces it with the newer request.
-        let rsp_rx = if let Some(queued_block) = self.queued_blocks.get_mut(&hash) {
+        let rsp_rx = if let Some((_, old_rsp_tx)) = self.queued_blocks.get_mut(&prepared.hash) {
             tracing::debug!("replacing older queued request with new request");
             let (mut rsp_tx, rsp_rx) = oneshot::channel();
-            std::mem::swap(&mut queued_block.rsp_tx, &mut rsp_tx);
+            std::mem::swap(old_rsp_tx, &mut rsp_tx);
             let _ = rsp_tx.send(Err("replaced by newer request".into()));
             rsp_rx
         } else {
             let (rsp_tx, rsp_rx) = oneshot::channel();
-            self.queued_blocks.queue(QueuedBlock { block, rsp_tx });
+            self.queued_blocks.queue((prepared, rsp_tx));
             rsp_rx
         };
 
@@ -138,16 +136,16 @@ impl StateService {
         rsp_rx
     }
 
-    /// Run contextual validation on `block` and add it to the non-finalized
-    /// state if it is contextually valid.
-    fn validate_and_commit(&mut self, block: Arc<Block>) -> Result<(), CommitBlockError> {
-        self.check_contextual_validity(&block)?;
-        let parent_hash = block.header.previous_block_hash;
+    /// Run contextual validation on the prepared block and add it to the
+    /// non-finalized state if it is contextually valid.
+    fn validate_and_commit(&mut self, prepared: PreparedBlock) -> Result<(), CommitBlockError> {
+        self.check_contextual_validity(&prepared)?;
+        let parent_hash = prepared.block.header.previous_block_hash;
 
         if self.disk.finalized_tip_hash() == parent_hash {
-            self.mem.commit_new_chain(block);
+            self.mem.commit_new_chain(prepared);
         } else {
-            self.mem.commit_block(block);
+            self.mem.commit_block(prepared);
         }
 
         Ok(())
@@ -158,17 +156,6 @@ impl StateService {
         self.mem.any_chain_contains(hash) || &self.disk.finalized_tip_hash() == hash
     }
 
-    /// Returns true if the given hash has been committed to either the finalized
-    /// or non-finalized state.
-    fn contains_committed_block(&self, block: &Block) -> bool {
-        let hash = block.hash();
-        let height = block
-            .coinbase_height()
-            .expect("coinbase heights should be valid");
-
-        self.mem.any_chain_contains(&hash) || self.disk.hash(height) == Some(hash)
-    }
-
     /// Attempt to validate and commit all queued blocks whose parents have
     /// recently arrived starting from `new_parent`, in breadth-first ordering.
     fn process_queued(&mut self, new_parent: block::Hash) {
@@ -177,11 +164,11 @@ impl StateService {
         while let Some(parent_hash) = new_parents.pop() {
             let queued_children = self.queued_blocks.dequeue_children(parent_hash);
 
-            for QueuedBlock { block, rsp_tx } in queued_children {
-                let child_hash = block.hash();
+            for (child, rsp_tx) in queued_children {
+                let child_hash = child.hash.clone();
                 tracing::trace!(?child_hash, "validating queued child");
                 let result = self
-                    .validate_and_commit(block)
+                    .validate_and_commit(child)
                     .map(|()| child_hash)
                     .map_err(BoxError::from);
                 let _ = rsp_tx.send(result);
@@ -190,14 +177,17 @@ impl StateService {
         }
     }
 
-    /// Check that `block` is contextually valid for the configured network,
-    /// based on the committed finalized and non-finalized state.
-    fn check_contextual_validity(&mut self, block: &Block) -> Result<(), ValidateContextError> {
+    /// Check that the prepared block is contextually valid for the configured
+    /// network, based on the committed finalized and non-finalized state.
+    fn check_contextual_validity(
+        &mut self,
+        prepared: &PreparedBlock,
+    ) -> Result<(), ValidateContextError> {
         check::block_is_contextually_valid(
-            block,
+            prepared,
             self.network,
             self.disk.finalized_tip_height(),
-            self.chain(block.header.previous_block_hash),
+            self.chain(prepared.block.header.previous_block_hash),
         )?;
 
         Ok(())
@@ -393,11 +383,11 @@ impl Service<Request> for StateService {
     #[instrument(name = "state", skip(self, req))]
     fn call(&mut self, req: Request) -> Self::Future {
         match req {
-            Request::CommitBlock { block } => {
+            Request::CommitBlock(prepared) => {
                 metrics::counter!("state.requests", 1, "type" => "commit_block");
 
-                self.pending_utxos.check_block(&block);
-                let rsp_rx = self.queue_and_commit_non_finalized(block);
+                self.pending_utxos.check_against(&prepared.new_outputs);
+                let rsp_rx = self.queue_and_commit_non_finalized(prepared);
 
                 async move {
                     rsp_rx
@@ -408,14 +398,13 @@ impl Service<Request> for StateService {
                 }
                 .boxed()
             }
-            Request::CommitFinalizedBlock { block } => {
+            Request::CommitFinalizedBlock(finalized) => {
                 metrics::counter!("state.requests", 1, "type" => "commit_finalized_block");
 
                 let (rsp_tx, rsp_rx) = oneshot::channel();
 
-                self.pending_utxos.check_block(&block);
-                self.disk
-                    .queue_and_commit_finalized(QueuedBlock { block, rsp_tx });
+                self.pending_utxos.scan_block(&finalized.block);
+                self.disk.queue_and_commit_finalized((finalized, rsp_tx));
 
                 async move {
                     rsp_rx
@@ -457,7 +446,7 @@ impl Service<Request> for StateService {
                 let fut = self.pending_utxos.queue(outpoint);
 
                 if let Some(utxo) = self.utxo(&outpoint) {
-                    self.pending_utxos.respond(outpoint, utxo);
+                    self.pending_utxos.respond(&outpoint, utxo);
                 }
 
                 fut.boxed()
diff --git a/zebra-state/src/service/check.rs b/zebra-state/src/service/check.rs
index b7a6a18c893..cdc0439e938 100644
--- a/zebra-state/src/service/check.rs
+++ b/zebra-state/src/service/check.rs
@@ -5,7 +5,7 @@ use zebra_chain::{
     parameters::Network,
 };
 
-use crate::ValidateContextError;
+use crate::{PreparedBlock, ValidateContextError};
 
 use super::check;
 
@@ -14,8 +14,13 @@ use super::check;
 ///
 /// The relevant chain is an iterator over the ancestors of `block`, starting
 /// with its parent block.
+#[tracing::instrument(
+    name = "contextual_validation",
+    fields(?network),
+    skip(prepared, network, finalized_tip_height, relevant_chain)
+)]
 pub(crate) fn block_is_contextually_valid<C>(
-    block: &Block,
+    prepared: &PreparedBlock,
     network: Network,
     finalized_tip_height: Option<block::Height>,
     relevant_chain: C,
@@ -24,22 +29,9 @@ where
     C: IntoIterator,
     C::Item: AsRef<Block>,
 {
-    let height = block
-        .coinbase_height()
-        .expect("semantically valid blocks have a coinbase height");
-    let hash = block.hash();
-
-    let span = tracing::info_span!(
-        "StateService::check_contextual_validity",
-        ?height,
-        ?network,
-        ?hash
-    );
-    let _entered = span.enter();
-
     let finalized_tip_height = finalized_tip_height
         .expect("finalized state must contain at least one block to use the non-finalized state");
-    check::block_is_not_orphaned(finalized_tip_height, block)?;
+    check::block_is_not_orphaned(finalized_tip_height, prepared.height)?;
 
     let mut relevant_chain = relevant_chain.into_iter();
     let parent_block = relevant_chain
@@ -49,7 +41,7 @@ where
     let parent_height = parent_block
         .coinbase_height()
         .expect("valid blocks have a coinbase height");
-    check::height_one_more_than_parent_height(parent_height, block)?;
+    check::height_one_more_than_parent_height(parent_height, prepared.height)?;
 
     // TODO: validate difficulty adjustment
     // TODO: other contextual validation design and implelentation
@@ -60,13 +52,9 @@ where
 /// block is less than or equal to the finalized tip height.
 pub(super) fn block_is_not_orphaned(
     finalized_tip_height: block::Height,
-    block: &Block,
+    height: block::Height,
 ) -> Result<(), ValidateContextError> {
-    if block
-        .coinbase_height()
-        .expect("valid blocks have a coinbase height")
-        <= finalized_tip_height
-    {
+    if height <= finalized_tip_height {
         Err(ValidateContextError::OrphanedBlock)
     } else {
         Ok(())
@@ -77,12 +65,8 @@ pub(super) fn block_is_not_orphaned(
 /// equal to the parent_height+1.
 pub(super) fn height_one_more_than_parent_height(
     parent_height: block::Height,
-    block: &Block,
+    height: block::Height,
 ) -> Result<(), ValidateContextError> {
-    let height = block
-        .coinbase_height()
-        .expect("valid blocks have a coinbase height");
-
     if parent_height + 1 != Some(height) {
         Err(ValidateContextError::NonSequentialBlock)
     } else {
@@ -102,16 +86,18 @@ mod tests {
     fn test_orphan_consensus_check() {
         zebra_test::init();
 
-        let block = zebra_test::vectors::BLOCK_MAINNET_347499_BYTES
+        let height = zebra_test::vectors::BLOCK_MAINNET_347499_BYTES
             .zcash_deserialize_into::<Arc<Block>>()
+            .unwrap()
+            .coinbase_height()
             .unwrap();
 
-        block_is_not_orphaned(block::Height(0), &block).expect("tip is lower so it should be fine");
-        block_is_not_orphaned(block::Height(347498), &block)
+        block_is_not_orphaned(block::Height(0), height).expect("tip is lower so it should be fine");
+        block_is_not_orphaned(block::Height(347498), height)
             .expect("tip is lower so it should be fine");
-        block_is_not_orphaned(block::Height(347499), &block)
+        block_is_not_orphaned(block::Height(347499), height)
             .expect_err("tip is equal so it should error");
-        block_is_not_orphaned(block::Height(500000), &block)
+        block_is_not_orphaned(block::Height(500000), height)
             .expect_err("tip is higher so it should error");
     }
 
@@ -119,21 +105,23 @@ mod tests {
     fn test_sequential_height_check() {
         zebra_test::init();
 
-        let block = zebra_test::vectors::BLOCK_MAINNET_347499_BYTES
+        let height = zebra_test::vectors::BLOCK_MAINNET_347499_BYTES
             .zcash_deserialize_into::<Arc<Block>>()
+            .unwrap()
+            .coinbase_height()
             .unwrap();
 
-        height_one_more_than_parent_height(block::Height(0), &block)
+        height_one_more_than_parent_height(block::Height(0), height)
             .expect_err("block is much lower, should panic");
-        height_one_more_than_parent_height(block::Height(347497), &block)
+        height_one_more_than_parent_height(block::Height(347497), height)
             .expect_err("parent height is 2 less, should panic");
-        height_one_more_than_parent_height(block::Height(347498), &block)
+        height_one_more_than_parent_height(block::Height(347498), height)
             .expect("parent height is 1 less, should be good");
-        height_one_more_than_parent_height(block::Height(347499), &block)
+        height_one_more_than_parent_height(block::Height(347499), height)
             .expect_err("parent height is equal, should panic");
-        height_one_more_than_parent_height(block::Height(347500), &block)
+        height_one_more_than_parent_height(block::Height(347500), height)
             .expect_err("parent height is way more, should panic");
-        height_one_more_than_parent_height(block::Height(500000), &block)
+        height_one_more_than_parent_height(block::Height(500000), height)
             .expect_err("parent height is way more, should panic");
     }
 }
diff --git a/zebra-state/src/service/finalized_state.rs b/zebra-state/src/service/finalized_state.rs
index dc3e2b0cd0f..e801b5a3f54 100644
--- a/zebra-state/src/service/finalized_state.rs
+++ b/zebra-state/src/service/finalized_state.rs
@@ -11,16 +11,16 @@ use zebra_chain::{
     transaction::{self, Transaction},
 };
 
-use crate::{BoxError, Config, HashOrHeight};
+use crate::{BoxError, Config, FinalizedBlock, HashOrHeight};
 
 use self::disk_format::{DiskDeserialize, DiskSerialize, FromDisk, IntoDisk, TransactionLocation};
 
-use super::QueuedBlock;
+use super::QueuedFinalized;
 
 /// The finalized part of the chain state, stored in the db.
 pub struct FinalizedState {
     /// Queued blocks that arrived out of order, indexed by their parent block hash.
-    queued_by_prev_hash: HashMap<block::Hash, QueuedBlock>,
+    queued_by_prev_hash: HashMap<block::Hash, QueuedFinalized>,
     max_queued_height: i64,
 
     db: rocksdb::DB,
@@ -92,10 +92,10 @@ impl FinalizedState {
     ///
     /// After queueing a finalized block, this method checks whether the newly
     /// queued block (and any of its descendants) can be committed to the state.
-    pub fn queue_and_commit_finalized(&mut self, queued_block: QueuedBlock) {
-        let prev_hash = queued_block.block.header.previous_block_hash;
-        let height = queued_block.block.coinbase_height().unwrap();
-        self.queued_by_prev_hash.insert(prev_hash, queued_block);
+    pub fn queue_and_commit_finalized(&mut self, queued: QueuedFinalized) {
+        let prev_hash = queued.0.block.header.previous_block_hash;
+        let height = queued.0.height;
+        self.queued_by_prev_hash.insert(prev_hash, queued);
 
         while let Some(queued_block) = self.queued_by_prev_hash.remove(&self.finalized_tip_hash()) {
             self.commit_finalized(queued_block);
@@ -111,7 +111,6 @@ impl FinalizedState {
         }
 
         metrics::gauge!("state.finalized.queued.max.height", self.max_queued_height);
-
         metrics::gauge!(
             "state.finalized.queued.block.count",
             self.queued_by_prev_hash.len() as _
@@ -139,14 +138,18 @@ impl FinalizedState {
             .valid()
     }
 
-    /// Immediately commit `block` to the finalized state.
-    pub fn commit_finalized_direct(&mut self, block: Arc<Block>) -> Result<block::Hash, BoxError> {
-        let height = block
-            .coinbase_height()
-            .expect("finalized blocks are valid and have a coinbase height");
-        let hash = block.hash();
+    /// Immediately commit `finalized` to the finalized state.
+    pub fn commit_finalized_direct(
+        &mut self,
+        finalized: FinalizedBlock,
+    ) -> Result<block::Hash, BoxError> {
+        block_precommit_metrics(&finalized);
 
-        block_precommit_metrics(&hash, height, &block);
+        let FinalizedBlock {
+            block,
+            hash,
+            height,
+        } = finalized;
 
         let hash_by_height = self.db.cf_handle("hash_by_height").unwrap();
         let height_by_hash = self.db.cf_handle("height_by_hash").unwrap();
@@ -265,9 +268,9 @@ impl FinalizedState {
     /// order. This function is called by [`queue`], which ensures order.
     /// It is intentionally not exposed as part of the public API of the
     /// [`FinalizedState`].
-    fn commit_finalized(&mut self, queued_block: QueuedBlock) {
-        let QueuedBlock { block, rsp_tx } = queued_block;
-        let result = self.commit_finalized_direct(block);
+    fn commit_finalized(&mut self, queued_block: QueuedFinalized) {
+        let (finalized, rsp_tx) = queued_block;
+        let result = self.commit_finalized_direct(finalized);
         let _ = rsp_tx.send(result.map_err(Into::into));
     }
 
@@ -342,7 +345,9 @@ impl Drop for FinalizedState {
     }
 }
 
-fn block_precommit_metrics(hash: &block::Hash, height: block::Height, block: &Block) {
+fn block_precommit_metrics(finalized: &FinalizedBlock) {
+    let (hash, height, block) = (finalized.hash, finalized.height, finalized.block.as_ref());
+
     let transaction_count = block.transactions.len();
     let transparent_prevout_count = block
         .transactions
diff --git a/zebra-state/src/service/non_finalized_state.rs b/zebra-state/src/service/non_finalized_state.rs
index 2444e7e105f..bf849cf8e7e 100644
--- a/zebra-state/src/service/non_finalized_state.rs
+++ b/zebra-state/src/service/non_finalized_state.rs
@@ -15,7 +15,7 @@ use zebra_chain::{
     transparent,
 };
 
-use crate::request::HashOrHeight;
+use crate::{FinalizedBlock, HashOrHeight, PreparedBlock};
 
 use self::chain::Chain;
 
@@ -31,7 +31,7 @@ pub struct NonFinalizedState {
 impl NonFinalizedState {
     /// Finalize the lowest height block in the non-finalized portion of the best
     /// chain and update all side-chains to match.
-    pub fn finalize(&mut self) -> Arc<Block> {
+    pub fn finalize(&mut self) -> FinalizedBlock {
         let chains = mem::take(&mut self.chain_set);
         let mut chains = chains.into_iter();
 
@@ -40,8 +40,8 @@ impl NonFinalizedState {
         // extract the rest into side_chains so they can be mutated
         let side_chains = chains;
 
-        // remove the lowest height block from the best_chain as finalized_block
-        let finalized_block = best_chain.pop_root();
+        // remove the lowest height block from the best_chain to be finalized
+        let finalizing = best_chain.pop_root();
 
         // add best_chain back to `self.chain_set`
         if !best_chain.is_empty() {
@@ -53,7 +53,7 @@ impl NonFinalizedState {
             // remove the first block from `chain`
             let chain_start = chain.pop_root();
             // if block equals finalized_block
-            if !chain.is_empty() && chain_start == finalized_block {
+            if !chain.is_empty() && chain_start.hash == finalizing.hash {
                 // add the chain back to `self.chain_set`
                 self.chain_set.insert(chain);
             } else {
@@ -64,13 +64,17 @@ impl NonFinalizedState {
 
         self.update_metrics_for_chains();
 
-        // return the finalized block
-        finalized_block
+        // Construct a finalized block.
+        FinalizedBlock {
+            block: finalizing.block,
+            hash: finalizing.hash,
+            height: finalizing.height,
+        }
     }
 
     /// Commit block to the non-finalized state.
-    pub fn commit_block(&mut self, block: Arc<Block>) {
-        let parent_hash = block.header.previous_block_hash;
+    pub fn commit_block(&mut self, prepared: PreparedBlock) {
+        let parent_hash = prepared.block.header.previous_block_hash;
 
         let mut parent_chain = self
             .take_chain_if(|chain| chain.non_finalized_tip_hash() == parent_hash)
@@ -82,18 +86,20 @@ impl NonFinalizedState {
             })
             .expect("commit_block is only called with blocks that are ready to be commited");
 
-        parent_chain.push(block.clone());
+        let (height, hash) = (prepared.height, prepared.hash);
+        parent_chain.push(prepared);
         self.chain_set.insert(parent_chain);
-        self.update_metrics_for_committed_block(block);
+        self.update_metrics_for_committed_block(height, hash);
     }
 
     /// Commit block to the non-finalized state as a new chain where its parent
     /// is the finalized tip.
-    pub fn commit_new_chain(&mut self, block: Arc<Block>) {
+    pub fn commit_new_chain(&mut self, prepared: PreparedBlock) {
         let mut chain = Chain::default();
-        chain.push(block.clone());
+        let (height, hash) = (prepared.height, prepared.hash);
+        chain.push(prepared);
         self.chain_set.insert(Box::new(chain));
-        self.update_metrics_for_committed_block(block);
+        self.update_metrics_for_committed_block(height, hash);
     }
 
     /// Returns the length of the non-finalized portion of the current best chain.
@@ -155,12 +161,12 @@ impl NonFinalizedState {
     /// Returns the `block` with the given hash in the any chain.
     pub fn block_by_hash(&self, hash: block::Hash) -> Option<Arc<Block>> {
         for chain in self.chain_set.iter().rev() {
-            if let Some(block) = chain
+            if let Some(prepared) = chain
                 .height_by_hash
                 .get(&hash)
                 .and_then(|height| chain.blocks.get(height))
             {
-                return Some(block.clone());
+                return Some(prepared.block.clone());
             }
         }
 
@@ -173,7 +179,10 @@ impl NonFinalizedState {
         let height =
             hash_or_height.height_or_else(|hash| best_chain.height_by_hash.get(&hash).cloned())?;
 
-        best_chain.blocks.get(&height).cloned()
+        best_chain
+            .blocks
+            .get(&height)
+            .map(|prepared| prepared.block.clone())
     }
 
     /// Returns the hash for a given `block::Height` if it is present in the best chain.
@@ -211,10 +220,10 @@ impl NonFinalizedState {
     /// Returns the given transaction if it exists in the best chain.
     pub fn transaction(&self, hash: transaction::Hash) -> Option<Arc<Transaction>> {
         let best_chain = self.best_chain()?;
-        best_chain.tx_by_hash.get(&hash).map(|(height, index)| {
-            let block = &best_chain.blocks[height];
-            block.transactions[*index].clone()
-        })
+        best_chain
+            .tx_by_hash
+            .get(&hash)
+            .map(|(height, index)| best_chain.blocks[height].block.transactions[*index].clone())
     }
 
     /// Return the non-finalized portion of the current best chain
@@ -226,9 +235,7 @@ impl NonFinalizedState {
     }
 
     /// Update the metrics after `block` is committed
-    fn update_metrics_for_committed_block(&self, block: Arc<Block>) {
-        let height = block.coinbase_height().unwrap();
-
+    fn update_metrics_for_committed_block(&self, height: block::Height, hash: block::Hash) {
         metrics::counter!("state.memory.committed.block.count", 1);
         metrics::gauge!("state.memory.committed.block.height", height.0 as _);
 
@@ -240,7 +247,8 @@ impl NonFinalizedState {
             .next_back()
             .unwrap()
             .1
-            == &block
+            .hash
+            == hash
         {
             metrics::counter!("state.memory.best.committed.block.count", 1);
             metrics::gauge!("state.memory.best.committed.block.height", height.0 as _);
@@ -261,7 +269,7 @@ mod tests {
     use zebra_chain::serialization::ZcashDeserializeInto;
     use zebra_test::prelude::*;
 
-    use crate::tests::FakeChainHelper;
+    use crate::tests::{FakeChainHelper, Prepare};
 
     use self::assert_eq;
     use super::*;
@@ -278,8 +286,8 @@ mod tests {
         let expected_hash = block2.hash();
 
         let mut state = NonFinalizedState::default();
-        state.commit_new_chain(block2);
-        state.commit_new_chain(child);
+        state.commit_new_chain(block2.prepare());
+        state.commit_new_chain(child.prepare());
 
         let best_chain = state.best_chain().unwrap();
         assert!(best_chain.height_by_hash.contains_key(&expected_hash));
@@ -297,15 +305,15 @@ mod tests {
         let child = block1.make_fake_child().set_work(1);
 
         let mut state = NonFinalizedState::default();
-        state.commit_new_chain(block1.clone());
-        state.commit_block(block2.clone());
-        state.commit_block(child);
+        state.commit_new_chain(block1.clone().prepare());
+        state.commit_block(block2.clone().prepare());
+        state.commit_block(child.prepare());
 
         let finalized = state.finalize();
-        assert_eq!(block1, finalized);
+        assert_eq!(block1, finalized.block);
 
         let finalized = state.finalize();
-        assert_eq!(block2, finalized);
+        assert_eq!(block2, finalized.block);
 
         assert!(state.best_chain().is_none());
 
@@ -325,13 +333,13 @@ mod tests {
 
         let mut state = NonFinalizedState::default();
         assert_eq!(0, state.chain_set.len());
-        state.commit_new_chain(block1);
+        state.commit_new_chain(block1.prepare());
         assert_eq!(1, state.chain_set.len());
-        state.commit_block(block2);
+        state.commit_block(block2.prepare());
         assert_eq!(1, state.chain_set.len());
-        state.commit_block(child1);
+        state.commit_block(child1.prepare());
         assert_eq!(2, state.chain_set.len());
-        state.commit_block(child2);
+        state.commit_block(child2.prepare());
         assert_eq!(2, state.chain_set.len());
 
         Ok(())
@@ -350,10 +358,10 @@ mod tests {
         let short_chain_block = block1.make_fake_child().set_work(3);
 
         let mut state = NonFinalizedState::default();
-        state.commit_new_chain(block1);
-        state.commit_block(long_chain_block1);
-        state.commit_block(long_chain_block2);
-        state.commit_block(short_chain_block);
+        state.commit_new_chain(block1.prepare());
+        state.commit_block(long_chain_block1.prepare());
+        state.commit_block(long_chain_block2.prepare());
+        state.commit_block(short_chain_block.prepare());
         assert_eq!(2, state.chain_set.len());
 
         assert_eq!(2, state.best_chain_len());
@@ -376,12 +384,12 @@ mod tests {
         let short_chain_block = block1.make_fake_child().set_work(3);
 
         let mut state = NonFinalizedState::default();
-        state.commit_new_chain(block1);
-        state.commit_block(long_chain_block1);
-        state.commit_block(long_chain_block2);
-        state.commit_block(long_chain_block3);
-        state.commit_block(long_chain_block4);
-        state.commit_block(short_chain_block);
+        state.commit_new_chain(block1.prepare());
+        state.commit_block(long_chain_block1.prepare());
+        state.commit_block(long_chain_block2.prepare());
+        state.commit_block(long_chain_block3.prepare());
+        state.commit_block(long_chain_block4.prepare());
+        state.commit_block(short_chain_block.prepare());
         assert_eq!(2, state.chain_set.len());
 
         assert_eq!(5, state.best_chain_len());
@@ -401,9 +409,9 @@ mod tests {
         let expected_hash = more_work_child.hash();
 
         let mut state = NonFinalizedState::default();
-        state.commit_new_chain(block1);
-        state.commit_block(less_work_child);
-        state.commit_block(more_work_child);
+        state.commit_new_chain(block1.prepare());
+        state.commit_block(less_work_child.prepare());
+        state.commit_block(more_work_child.prepare());
         assert_eq!(2, state.chain_set.len());
 
         let tip_hash = state.tip().unwrap().1;
diff --git a/zebra-state/src/service/non_finalized_state/chain.rs b/zebra-state/src/service/non_finalized_state/chain.rs
index f3e61a872ca..bb4e2bb3ad9 100644
--- a/zebra-state/src/service/non_finalized_state/chain.rs
+++ b/zebra-state/src/service/non_finalized_state/chain.rs
@@ -2,20 +2,19 @@ use std::{
     cmp::Ordering,
     collections::{BTreeMap, HashMap, HashSet},
     ops::Deref,
-    sync::Arc,
 };
 
 use tracing::{debug_span, instrument, trace};
 use zebra_chain::{
-    block::{self, Block},
-    primitives::Groth16Proof,
-    sapling, sprout, transaction, transparent,
+    block, primitives::Groth16Proof, sapling, sprout, transaction, transparent,
     work::difficulty::PartialCumulativeWork,
 };
 
+use crate::PreparedBlock;
+
 #[derive(Default, Clone)]
 pub struct Chain {
-    pub blocks: BTreeMap<block::Height, Arc<Block>>,
+    pub blocks: BTreeMap<block::Height, PreparedBlock>,
     pub height_by_hash: HashMap<block::Hash, block::Height>,
     pub tx_by_hash: HashMap<transaction::Hash, (block::Height, usize)>,
 
@@ -30,20 +29,17 @@ pub struct Chain {
 
 impl Chain {
     /// Push a contextually valid non-finalized block into a chain as the new tip.
-    #[instrument(skip(self), fields(%block))]
-    pub fn push(&mut self, block: Arc<Block>) {
-        let block_height = block
-            .coinbase_height()
-            .expect("valid non-finalized blocks have a coinbase height");
+    #[instrument(skip(self, block), fields(block = %block.block))]
+    pub fn push(&mut self, block: PreparedBlock) {
         // update cumulative data members
         self.update_chain_state_with(&block);
-        self.blocks.insert(block_height, block);
+        self.blocks.insert(block.height, block);
         trace!("pushed block onto chain");
     }
 
     /// Remove the lowest height block of the non-finalized portion of a chain.
     #[instrument(skip(self))]
-    pub fn pop_root(&mut self) -> Arc<Block> {
+    pub fn pop_root(&mut self) -> PreparedBlock {
         let block_height = self.lowest_height();
 
         // remove the lowest height block from self.blocks
@@ -55,7 +51,7 @@ impl Chain {
         // update cumulative data members
         self.revert_chain_state_with(&block);
 
-        // return the block
+        // return the prepared block
         block
     }
 
@@ -88,7 +84,7 @@ impl Chain {
             .values()
             .next_back()
             .expect("only called while blocks is populated")
-            .hash()
+            .hash
     }
 
     /// Remove the highest height block of the non-finalized portion of a chain.
@@ -138,15 +134,12 @@ trait UpdateWith<T> {
     fn revert_chain_state_with(&mut self, _: &T);
 }
 
-impl UpdateWith<Arc<Block>> for Chain {
-    fn update_chain_state_with(&mut self, block: &Arc<Block>) {
-        let block_height = block
-            .coinbase_height()
-            .expect("valid non-finalized blocks have a coinbase height");
-        let block_hash = block.hash();
+impl UpdateWith<PreparedBlock> for Chain {
+    fn update_chain_state_with(&mut self, prepared: &PreparedBlock) {
+        let (block, hash, height) = (prepared.block.as_ref(), prepared.hash, prepared.height);
 
         // add hash to height_by_hash
-        let prior_height = self.height_by_hash.insert(block_hash, block_height);
+        let prior_height = self.height_by_hash.insert(hash, height);
         assert!(
             prior_height.is_none(),
             "block heights must be unique within a single chain"
@@ -179,7 +172,7 @@ impl UpdateWith<Arc<Block>> for Chain {
             let transaction_hash = transaction.hash();
             let prior_pair = self
                 .tx_by_hash
-                .insert(transaction_hash, (block_height, transaction_index));
+                .insert(transaction_hash, (height, transaction_index));
             assert!(
                 prior_pair.is_none(),
                 "transactions must be unique within a single chain"
@@ -196,13 +189,13 @@ impl UpdateWith<Arc<Block>> for Chain {
         }
     }
 
-    #[instrument(skip(self), fields(%block))]
-    fn revert_chain_state_with(&mut self, block: &Arc<Block>) {
-        let block_hash = block.hash();
+    #[instrument(skip(self, prepared), fields(block = %prepared.block))]
+    fn revert_chain_state_with(&mut self, prepared: &PreparedBlock) {
+        let (block, hash) = (prepared.block.as_ref(), prepared.hash);
 
         // remove the blocks hash from `height_by_hash`
         assert!(
-            self.height_by_hash.remove(&block_hash).is_some(),
+            self.height_by_hash.remove(&hash).is_some(),
             "hash must be present if block was"
         );
 
@@ -395,14 +388,14 @@ impl Ord for Chain {
                 .values()
                 .last()
                 .expect("always at least 1 element")
-                .hash();
+                .hash;
 
             let other_hash = other
                 .blocks
                 .values()
                 .last()
                 .expect("always at least 1 element")
-                .hash();
+                .hash;
 
             // This comparison is a tie-breaker within the local node, so it does not need to
             // be consistent with the ordering on `ExpandedDifficulty` and `block::Hash`.
@@ -416,16 +409,17 @@ impl Ord for Chain {
 
 #[cfg(test)]
 mod tests {
-    use std::{env, fmt};
+    use std::{env, fmt, sync::Arc};
 
     use zebra_chain::serialization::ZcashDeserializeInto;
     use zebra_chain::{
+        block::Block,
         parameters::{Network, NetworkUpgrade},
         LedgerState,
     };
     use zebra_test::prelude::*;
 
-    use crate::tests::FakeChainHelper;
+    use crate::tests::{FakeChainHelper, Prepare};
 
     use self::assert_eq;
     use super::*;
@@ -447,10 +441,11 @@ mod tests {
     #[test]
     fn construct_single() -> Result<()> {
         zebra_test::init();
-        let block = zebra_test::vectors::BLOCK_MAINNET_434873_BYTES.zcash_deserialize_into()?;
+        let block: Arc<Block> =
+            zebra_test::vectors::BLOCK_MAINNET_434873_BYTES.zcash_deserialize_into()?;
 
         let mut chain = Chain::default();
-        chain.push(block);
+        chain.push(block.prepare());
 
         assert_eq!(1, chain.blocks.len());
 
@@ -474,7 +469,7 @@ mod tests {
         let mut chain = Chain::default();
 
         for block in blocks {
-            chain.push(block);
+            chain.push(block.prepare());
         }
 
         assert_eq!(100, chain.blocks.len());
@@ -491,10 +486,10 @@ mod tests {
         let more_block = less_block.clone().set_work(10);
 
         let mut lesser_chain = Chain::default();
-        lesser_chain.push(less_block);
+        lesser_chain.push(less_block.prepare());
 
         let mut bigger_chain = Chain::default();
-        bigger_chain.push(more_block);
+        bigger_chain.push(more_block.prepare());
 
         assert!(bigger_chain > lesser_chain);
 
@@ -529,11 +524,11 @@ mod tests {
             let mut partial_chain = Chain::default();
 
             for block in chain.iter().take(count) {
-                partial_chain.push(block.clone());
+                partial_chain.push(block.clone().prepare());
             }
 
             for block in chain {
-                full_chain.push(block);
+                full_chain.push(block.prepare());
             }
 
             let forked = full_chain.fork(fork_tip_hash).expect("hash is present");
@@ -560,11 +555,11 @@ mod tests {
             let mut partial_chain = Chain::default();
 
             for block in chain.iter().skip(finalized_count) {
-                partial_chain.push(block.clone());
+                partial_chain.push(block.clone().prepare());
             }
 
             for block in chain {
-                full_chain.push(block);
+                full_chain.push(block.prepare());
             }
 
             for _ in 0..finalized_count {
diff --git a/zebra-state/src/service/non_finalized_state/queued_blocks.rs b/zebra-state/src/service/non_finalized_state/queued_blocks.rs
index 790c1238c1e..0bea77fda23 100644
--- a/zebra-state/src/service/non_finalized_state/queued_blocks.rs
+++ b/zebra-state/src/service/non_finalized_state/queued_blocks.rs
@@ -28,27 +28,14 @@ impl QueuedBlocks {
     ///
     /// - if a block with the same `block::Hash` has already been queued.
     pub fn queue(&mut self, new: QueuedBlock) {
-        let new_hash = new.block.hash();
-        let new_height = new
-            .block
-            .coinbase_height()
-            .expect("validated non-finalized blocks have a coinbase height");
-        let parent_hash = new.block.header.previous_block_hash;
-
-        // XXX QueuedBlock should include this data
-        let prev_utxo_count = self.known_utxos.len();
-        for transaction in &new.block.transactions {
-            let hash = transaction.hash();
-            for (index, output) in transaction.outputs().iter().cloned().enumerate() {
-                let index = index as u32;
-                self.known_utxos
-                    .insert(transparent::OutPoint { hash, index }, output);
-            }
+        let new_hash = new.0.hash;
+        let new_height = new.0.height;
+        let parent_hash = new.0.block.header.previous_block_hash;
+
+        // Track known UTXOs in queued blocks.
+        for (outpoint, output) in new.0.new_outputs.iter() {
+            self.known_utxos.insert(*outpoint, output.clone());
         }
-        tracing::trace!(
-            known_utxos = self.known_utxos.len(),
-            new = self.known_utxos.len() - prev_utxo_count
-        );
 
         let replaced = self.blocks.insert(new_hash, new);
         assert!(replaced.is_none(), "hashes must be unique");
@@ -83,28 +70,15 @@ impl QueuedBlocks {
                     .blocks
                     .remove(&hash)
                     .expect("block is present if its hash is in by_parent");
-
-                let prev_utxo_count = self.known_utxos.len();
-                for transaction in &queued.block.transactions {
-                    let hash = transaction.hash();
-                    for (index, _output) in transaction.outputs().iter().cloned().enumerate() {
-                        let index = index as u32;
-                        self.known_utxos
-                            .remove(&transparent::OutPoint { hash, index });
-                    }
-                }
-                tracing::trace!(
-                    known_utxos = self.known_utxos.len(),
-                    removed = prev_utxo_count - self.known_utxos.len()
-                );
-
                 queued
             })
             .collect::<Vec<_>>();
 
         for queued in &queued_children {
-            let height = queued.block.coinbase_height().unwrap();
-            self.by_height.remove(&height);
+            self.by_height.remove(&queued.0.height);
+            for outpoint in queued.0.new_outputs.keys() {
+                self.known_utxos.remove(outpoint);
+            }
         }
 
         tracing::trace!(
@@ -133,7 +107,7 @@ impl QueuedBlocks {
 
         for hash in by_height.into_iter().flat_map(|(_, hashes)| hashes) {
             let expired = self.blocks.remove(&hash).expect("block is present");
-            let parent_hash = &expired.block.header.previous_block_hash;
+            let parent_hash = &expired.0.block.header.previous_block_hash;
 
             let parent_list = self
                 .by_parent
@@ -191,7 +165,7 @@ mod tests {
     use zebra_chain::{block::Block, serialization::ZcashDeserializeInto};
     use zebra_test::prelude::*;
 
-    use crate::tests::FakeChainHelper;
+    use crate::tests::{FakeChainHelper, Prepare};
 
     use self::assert_eq;
     use super::*;
@@ -204,11 +178,7 @@ mod tests {
     impl IntoQueued for Arc<Block> {
         fn into_queued(self) -> QueuedBlock {
             let (rsp_tx, _) = oneshot::channel();
-
-            QueuedBlock {
-                block: self,
-                rsp_tx,
-            }
+            (self.prepare(), rsp_tx)
         }
     }
 
@@ -253,22 +223,22 @@ mod tests {
         // Dequeueing the first block removes 1 block from each list
         let children = queue.dequeue_children(parent);
         assert_eq!(1, children.len());
-        assert_eq!(block1, children[0].block);
+        assert_eq!(block1, children[0].0.block);
         assert_eq!(2, queue.blocks.len());
         assert_eq!(1, queue.by_parent.len());
         assert_eq!(1, queue.by_height.len());
 
         // Dequeueing the children of the first block removes both of the other
         // blocks, and empties all lists
-        let parent = children[0].block.hash();
+        let parent = children[0].0.block.hash();
         let children = queue.dequeue_children(parent);
         assert_eq!(2, children.len());
         assert!(children
             .iter()
-            .any(|QueuedBlock { block, .. }| block == &child1));
+            .any(|(block, _)| block.hash == child1.hash()));
         assert!(children
             .iter()
-            .any(|QueuedBlock { block, .. }| block == &child2));
+            .any(|(block, _)| block.hash == child2.hash()));
         assert_eq!(0, queue.blocks.len());
         assert_eq!(0, queue.by_parent.len());
         assert_eq!(0, queue.by_height.len());
diff --git a/zebra-state/src/service/tests.rs b/zebra-state/src/service/tests.rs
index 1acf159075e..3442cad4f84 100644
--- a/zebra-state/src/service/tests.rs
+++ b/zebra-state/src/service/tests.rs
@@ -17,7 +17,7 @@ async fn populated_state(
 ) -> BoxService<Request, Response, BoxError> {
     let requests = blocks
         .into_iter()
-        .map(|block| Request::CommitFinalizedBlock { block });
+        .map(|block| Request::CommitFinalizedBlock(block.into()));
 
     let config = Config::ephemeral();
     let network = Network::Mainnet;
diff --git a/zebra-state/src/service/utxo.rs b/zebra-state/src/service/utxo.rs
index 4ade4c536b8..099dee24279 100644
--- a/zebra-state/src/service/utxo.rs
+++ b/zebra-state/src/service/utxo.rs
@@ -35,7 +35,7 @@ impl PendingUtxos {
 
     /// Notify all utxo requests waiting for the `transparent::Output` pointed to
     /// by the given `transparent::OutPoint` that the `Output` has arrived.
-    pub fn respond(&mut self, outpoint: transparent::OutPoint, output: transparent::Output) {
+    pub fn respond(&mut self, outpoint: &transparent::OutPoint, output: transparent::Output) {
         if let Some(sender) = self.0.remove(&outpoint) {
             // Adding the outpoint as a field lets us crossreference
             // with the trace of the verification that made the request.
@@ -44,9 +44,16 @@ impl PendingUtxos {
         }
     }
 
-    /// For each notifies waiting utxo requests for each `transparent::Output` in
-    /// `block` that the output has arrived.
-    pub fn check_block(&mut self, block: &Block) {
+    /// Check the list of pending UTXO requests against the supplied UTXO index.
+    pub fn check_against(&mut self, utxos: &HashMap<transparent::OutPoint, transparent::Output>) {
+        for (outpoint, output) in utxos.iter() {
+            self.respond(outpoint, output.clone());
+        }
+    }
+
+    /// Scan through unindexed transactions in the given `block`
+    /// to determine whether it contains any requested UTXOs.
+    pub fn scan_block(&mut self, block: &Block) {
         if self.0.is_empty() {
             return;
         }
@@ -60,7 +67,7 @@ impl PendingUtxos {
                     index: index as _,
                 };
 
-                self.respond(outpoint, output.clone());
+                self.respond(&outpoint, output.clone());
             }
         }
     }
diff --git a/zebra-state/src/tests.rs b/zebra-state/src/tests.rs
index 86ea3323741..dee35a69bb6 100644
--- a/zebra-state/src/tests.rs
+++ b/zebra-state/src/tests.rs
@@ -1,4 +1,4 @@
-use std::{convert::TryFrom, mem, sync::Arc};
+use std::{collections::HashMap, convert::TryFrom, mem, sync::Arc};
 
 use primitive_types::U256;
 use zebra_chain::{
@@ -11,6 +11,35 @@ use zebra_chain::{
 
 use super::*;
 
+/// Mocks computation done during semantic validation
+pub trait Prepare {
+    fn prepare(self) -> PreparedBlock;
+}
+
+impl Prepare for Arc<Block> {
+    fn prepare(self) -> PreparedBlock {
+        let block = self;
+        let hash = block.hash();
+        let height = block.coinbase_height().unwrap();
+
+        let mut new_outputs = HashMap::new();
+        for transaction in &block.transactions {
+            let hash = transaction.hash();
+            for (index, output) in transaction.outputs().iter().cloned().enumerate() {
+                let index = index as u32;
+                new_outputs.insert(transparent::OutPoint { hash, index }, output);
+            }
+        }
+
+        PreparedBlock {
+            block,
+            hash,
+            height,
+            new_outputs,
+        }
+    }
+}
+
 /// Helper trait for constructing "valid" looking chains of blocks
 pub trait FakeChainHelper {
     fn make_fake_child(&self) -> Arc<Block>;
diff --git a/zebra-state/tests/basic.rs b/zebra-state/tests/basic.rs
index 27b93cb420f..6a7a6f67f85 100644
--- a/zebra-state/tests/basic.rs
+++ b/zebra-state/tests/basic.rs
@@ -17,7 +17,7 @@ static COMMIT_FINALIZED_BLOCK_MAINNET: Lazy<Vec<(Request, Result<Response, Trans
         let hash = block.hash();
         vec![
             (
-                Request::CommitFinalizedBlock { block },
+                Request::CommitFinalizedBlock(block.into()),
                 Ok(Response::Committed(hash)),
             ),
             (
@@ -37,7 +37,7 @@ static COMMIT_FINALIZED_BLOCK_TESTNET: Lazy<Vec<(Request, Result<Response, Trans
         let hash = block.hash();
         vec![
             (
-                Request::CommitFinalizedBlock { block },
+                Request::CommitFinalizedBlock(block.into()),
                 Ok(Response::Committed(hash)),
             ),
             (

From 2a4a89c00200f3dc83b0a24795f863ad78234b6d Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Sat, 21 Nov 2020 10:41:53 -0800
Subject: [PATCH 090/148] state,zebrad: tidy span levels for good INFO output

This provides useful and not too noisy output at INFO level.  We do an
info-level message on every block commit instead of trying to do one
message every N blocks, because this is useful both for initial block
sync as well as continuous state updates on new blocks.
---
 zebra-state/src/service.rs                           | 3 ++-
 zebra-state/src/service/non_finalized_state/chain.rs | 6 +++---
 zebrad/src/components/sync/downloads.rs              | 2 +-
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs
index 0b3d1351512..e6fe6c69362 100644
--- a/zebra-state/src/service.rs
+++ b/zebra-state/src/service.rs
@@ -83,11 +83,12 @@ impl StateService {
     /// in RFC0005.
     ///
     /// [1]: https://zebra.zfnd.org/dev/rfcs/0005-state-updates.html#committing-non-finalized-blocks
-    #[instrument(skip(self, prepared))]
+    #[instrument(level = "debug", skip(self, prepared))]
     fn queue_and_commit_non_finalized(
         &mut self,
         prepared: PreparedBlock,
     ) -> oneshot::Receiver<Result<block::Hash, BoxError>> {
+        tracing::debug!(block = %prepared.block, "queueing block for contextual verification");
         let parent_hash = prepared.block.header.previous_block_hash;
 
         if self.mem.any_chain_contains(&prepared.hash) || self.disk.hash(prepared.height).is_some()
diff --git a/zebra-state/src/service/non_finalized_state/chain.rs b/zebra-state/src/service/non_finalized_state/chain.rs
index bb4e2bb3ad9..583f4320e82 100644
--- a/zebra-state/src/service/non_finalized_state/chain.rs
+++ b/zebra-state/src/service/non_finalized_state/chain.rs
@@ -29,16 +29,16 @@ pub struct Chain {
 
 impl Chain {
     /// Push a contextually valid non-finalized block into a chain as the new tip.
-    #[instrument(skip(self, block), fields(block = %block.block))]
+    #[instrument(level = "debug", skip(self, block), fields(block = %block.block))]
     pub fn push(&mut self, block: PreparedBlock) {
         // update cumulative data members
         self.update_chain_state_with(&block);
+        tracing::info!(block = %block.block, "adding block to chain");
         self.blocks.insert(block.height, block);
-        trace!("pushed block onto chain");
     }
 
     /// Remove the lowest height block of the non-finalized portion of a chain.
-    #[instrument(skip(self))]
+    #[instrument(level = "debug", skip(self))]
     pub fn pop_root(&mut self) -> PreparedBlock {
         let block_height = self.lowest_height();
 
diff --git a/zebrad/src/components/sync/downloads.rs b/zebrad/src/components/sync/downloads.rs
index b3ae4169098..ca59abd73d6 100644
--- a/zebrad/src/components/sync/downloads.rs
+++ b/zebrad/src/components/sync/downloads.rs
@@ -110,7 +110,7 @@ where
     /// This method waits for the network to become ready, and returns an error
     /// only if the network service fails. It returns immediately after queuing
     /// the request.
-    #[instrument(skip(self), fields(%hash))]
+    #[instrument(level = "debug", skip(self), fields(%hash))]
     pub async fn download_and_verify(&mut self, hash: block::Hash) -> Result<(), Report> {
         if self.cancel_handles.contains_key(&hash) {
             return Err(eyre!("duplicate hash queued for download"));

From acf6096103c62bd251c6c3eca1e8dd6438fa78c5 Mon Sep 17 00:00:00 2001
From: teor <teor@riseup.net>
Date: Mon, 23 Nov 2020 13:38:25 +1000
Subject: [PATCH 091/148] Appease clippy stable

---
 zebra-state/src/service.rs                                   | 2 +-
 zebra-state/src/service/non_finalized_state/queued_blocks.rs | 5 ++---
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs
index e6fe6c69362..35f09eb7120 100644
--- a/zebra-state/src/service.rs
+++ b/zebra-state/src/service.rs
@@ -166,7 +166,7 @@ impl StateService {
             let queued_children = self.queued_blocks.dequeue_children(parent_hash);
 
             for (child, rsp_tx) in queued_children {
-                let child_hash = child.hash.clone();
+                let child_hash = child.hash;
                 tracing::trace!(?child_hash, "validating queued child");
                 let result = self
                     .validate_and_commit(child)
diff --git a/zebra-state/src/service/non_finalized_state/queued_blocks.rs b/zebra-state/src/service/non_finalized_state/queued_blocks.rs
index 0bea77fda23..6593a57c6be 100644
--- a/zebra-state/src/service/non_finalized_state/queued_blocks.rs
+++ b/zebra-state/src/service/non_finalized_state/queued_blocks.rs
@@ -66,11 +66,10 @@ impl QueuedBlocks {
             .unwrap_or_default()
             .into_iter()
             .map(|hash| {
-                let queued = self
+                self
                     .blocks
                     .remove(&hash)
-                    .expect("block is present if its hash is in by_parent");
-                queued
+                    .expect("block is present if its hash is in by_parent")
             })
             .collect::<Vec<_>>();
 

From 00c52d28cdfc19936ba11dc58ab9e363e07e01b3 Mon Sep 17 00:00:00 2001
From: teor <teor@riseup.net>
Date: Mon, 23 Nov 2020 13:42:02 +1000
Subject: [PATCH 092/148] Appease rustfmt

---
 zebra-state/src/service/non_finalized_state/queued_blocks.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/zebra-state/src/service/non_finalized_state/queued_blocks.rs b/zebra-state/src/service/non_finalized_state/queued_blocks.rs
index 6593a57c6be..f8007e57a92 100644
--- a/zebra-state/src/service/non_finalized_state/queued_blocks.rs
+++ b/zebra-state/src/service/non_finalized_state/queued_blocks.rs
@@ -66,8 +66,7 @@ impl QueuedBlocks {
             .unwrap_or_default()
             .into_iter()
             .map(|hash| {
-                self
-                    .blocks
+                self.blocks
                     .remove(&hash)
                     .expect("block is present if its hash is in by_parent")
             })

From d2173cae7e4dff5a3eb89b1517245989fd742417 Mon Sep 17 00:00:00 2001
From: teor <teor@riseup.net>
Date: Tue, 24 Nov 2020 10:49:41 +1000
Subject: [PATCH 093/148] Basic documentation for zebra-checkpoints (#1361)

---
 book/src/dev/zebra-checkpoints.md | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/book/src/dev/zebra-checkpoints.md b/book/src/dev/zebra-checkpoints.md
index 4dbd4504bab..d627cc5ddc0 100644
--- a/book/src/dev/zebra-checkpoints.md
+++ b/book/src/dev/zebra-checkpoints.md
@@ -1 +1,8 @@
 # zebra-checkpoints
+
+`zebra-checkpoints` uses a local `zcashd` instance to generate a list of checkpoints for Zebra's checkpoint verifier.
+
+Developers should run this tool every few months to add new checkpoints for the `checkpoint_sync = true` mode.
+(By default, Zebra syncs up to Sapling using checkpoints. These checkpoints don't need to be updated.)
+
+See the source code for more details.

From 342eb166ff3376af7224b0a68d7002813bf27b2d Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Mon, 23 Nov 2020 12:02:57 -0800
Subject: [PATCH 094/148] state: track UTXO provenance

This commit changes the state system and database format to track the
provenance of UTXOs, in addition to the outputs themselves.
Specifically, it tracks the following additional metadata:

- the height at which the UTXO was created;
- whether or not the UTXO was created from a coinbase transaction or
  not.

This metadata will allow us to:

- check the coinbase maturity consensus rule;
- check the coinbase inputs => no transparent outputs rule;
- implement lookup of transactions by utxo (using the height to find the
  block and then scanning the block) for a future RPC mechanism.

Closes #1342
---
 Cargo.lock                                    |  1 +
 zebra-consensus/src/block.rs                  | 19 +++++--
 zebra-consensus/src/script.rs                 | 17 +++---
 zebra-consensus/src/transaction.rs            |  4 +-
 zebra-state/Cargo.toml                        |  1 +
 zebra-state/src/constants.rs                  |  3 +-
 zebra-state/src/lib.rs                        |  2 +
 zebra-state/src/request.rs                    | 46 +++++++++++++++-
 zebra-state/src/response.rs                   |  5 +-
 zebra-state/src/service.rs                    | 10 ++--
 zebra-state/src/service/finalized_state.rs    | 22 ++++----
 .../service/finalized_state/disk_format.rs    | 32 ++++++++---
 .../src/service/non_finalized_state.rs        | 11 ++--
 .../src/service/non_finalized_state/chain.rs  | 54 ++++++-------------
 .../non_finalized_state/queued_blocks.rs      |  6 +--
 zebra-state/src/service/tests.rs              | 15 +++---
 zebra-state/src/service/utxo.rs               | 48 ++++++-----------
 zebra-state/src/tests.rs                      | 12 +----
 zebra-state/src/utxo.rs                       | 42 +++++++++++++++
 19 files changed, 210 insertions(+), 140 deletions(-)
 create mode 100644 zebra-state/src/utxo.rs

diff --git a/Cargo.lock b/Cargo.lock
index 398349c1fd9..7c240829de8 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3263,6 +3263,7 @@ dependencies = [
  "once_cell",
  "primitive-types",
  "proptest",
+ "proptest-derive",
  "rocksdb",
  "serde",
  "spandoc",
diff --git a/zebra-consensus/src/block.rs b/zebra-consensus/src/block.rs
index 564ae804f94..c86c3983de3 100644
--- a/zebra-consensus/src/block.rs
+++ b/zebra-consensus/src/block.rs
@@ -167,7 +167,7 @@ where
 
             let mut async_checks = FuturesUnordered::new();
 
-            let known_utxos = known_utxos(&block);
+            let known_utxos = new_outputs(&block);
             for transaction in &block.transactions {
                 let rsp = transaction_verifier
                     .ready_and()
@@ -221,15 +221,24 @@ where
     }
 }
 
-fn known_utxos(block: &Block) -> Arc<HashMap<transparent::OutPoint, transparent::Output>> {
-    let mut map = HashMap::default();
+fn new_outputs(block: &Block) -> Arc<HashMap<transparent::OutPoint, zs::Utxo>> {
+    let mut new_outputs = HashMap::default();
+    let height = block.coinbase_height().expect("block has coinbase height");
     for transaction in &block.transactions {
         let hash = transaction.hash();
+        let from_coinbase = transaction.is_coinbase();
         for (index, output) in transaction.outputs().iter().cloned().enumerate() {
             let index = index as u32;
-            map.insert(transparent::OutPoint { hash, index }, output);
+            new_outputs.insert(
+                transparent::OutPoint { hash, index },
+                zs::Utxo {
+                    output,
+                    height,
+                    from_coinbase,
+                },
+            );
         }
     }
 
-    Arc::new(map)
+    Arc::new(new_outputs)
 }
diff --git a/zebra-consensus/src/script.rs b/zebra-consensus/src/script.rs
index 36b986a16c9..9dc4a4d105e 100644
--- a/zebra-consensus/src/script.rs
+++ b/zebra-consensus/src/script.rs
@@ -3,6 +3,7 @@ use std::{collections::HashMap, future::Future, pin::Pin, sync::Arc};
 use tracing::Instrument;
 
 use zebra_chain::{parameters::ConsensusBranchId, transaction::Transaction, transparent};
+use zebra_state::Utxo;
 
 use crate::BoxError;
 
@@ -40,7 +41,7 @@ impl<ZS> Verifier<ZS> {
 pub struct Request {
     pub transaction: Arc<Transaction>,
     pub input_index: usize,
-    pub known_utxos: Arc<HashMap<transparent::OutPoint, transparent::Output>>,
+    pub known_utxos: Arc<HashMap<transparent::OutPoint, Utxo>>,
 }
 
 impl<ZS> tower::Service<Request> for Verifier<ZS>
@@ -81,17 +82,21 @@ where
 
                 async move {
                     tracing::trace!("awaiting outpoint lookup");
-                    let output = if let Some(output) = known_utxos.get(&outpoint) {
+                    let utxo = if let Some(output) = known_utxos.get(&outpoint) {
                         tracing::trace!("UXTO in known_utxos, discarding query");
                         output.clone()
-                    } else if let zebra_state::Response::Utxo(output) = query.await? {
-                        output
+                    } else if let zebra_state::Response::Utxo(utxo) = query.await? {
+                        utxo
                     } else {
                         unreachable!("AwaitUtxo always responds with Utxo")
                     };
-                    tracing::trace!(?output, "got UTXO");
+                    tracing::trace!(?utxo, "got UTXO");
 
-                    zebra_script::is_valid(transaction, branch_id, (input_index as u32, output))?;
+                    zebra_script::is_valid(
+                        transaction,
+                        branch_id,
+                        (input_index as u32, utxo.output),
+                    )?;
                     tracing::trace!("script verification succeeded");
 
                     Ok(())
diff --git a/zebra-consensus/src/transaction.rs b/zebra-consensus/src/transaction.rs
index 313ff3bcf24..894e6450548 100644
--- a/zebra-consensus/src/transaction.rs
+++ b/zebra-consensus/src/transaction.rs
@@ -56,13 +56,13 @@ pub enum Request {
     Block {
         transaction: Arc<Transaction>,
         /// Additional UTXOs which are known at the time of verification.
-        known_utxos: Arc<HashMap<transparent::OutPoint, transparent::Output>>,
+        known_utxos: Arc<HashMap<transparent::OutPoint, zs::Utxo>>,
     },
     /// Verify the supplied transaction as part of the mempool.
     Mempool {
         transaction: Arc<Transaction>,
         /// Additional UTXOs which are known at the time of verification.
-        known_utxos: Arc<HashMap<transparent::OutPoint, transparent::Output>>,
+        known_utxos: Arc<HashMap<transparent::OutPoint, zs::Utxo>>,
     },
 }
 
diff --git a/zebra-state/Cargo.toml b/zebra-state/Cargo.toml
index fb6afc6806f..5f1e5da34c0 100644
--- a/zebra-state/Cargo.toml
+++ b/zebra-state/Cargo.toml
@@ -36,4 +36,5 @@ spandoc = "0.2"
 tempdir = "0.3.7"
 tokio = { version = "0.3", features = ["full"] }
 proptest = "0.10.1"
+proptest-derive = "0.2"
 primitive-types = "0.7.3"
diff --git a/zebra-state/src/constants.rs b/zebra-state/src/constants.rs
index 29e0d02491e..42d9b648c50 100644
--- a/zebra-state/src/constants.rs
+++ b/zebra-state/src/constants.rs
@@ -11,4 +11,5 @@ pub const MIN_TRASPARENT_COINBASE_MATURITY: u32 = 100;
 /// coinbase transactions.
 pub const MAX_BLOCK_REORG_HEIGHT: u32 = MIN_TRASPARENT_COINBASE_MATURITY - 1;
 
-pub const DATABASE_FORMAT_VERSION: u32 = 2;
+/// The database format version, incremented each time the database format changes.
+pub const DATABASE_FORMAT_VERSION: u32 = 3;
diff --git a/zebra-state/src/lib.rs b/zebra-state/src/lib.rs
index f97a82c2a11..2558a62d0e1 100644
--- a/zebra-state/src/lib.rs
+++ b/zebra-state/src/lib.rs
@@ -15,6 +15,7 @@ mod request;
 mod response;
 mod service;
 mod util;
+mod utxo;
 
 // TODO: move these to integration tests.
 #[cfg(test)]
@@ -26,3 +27,4 @@ pub use error::{BoxError, CloneError, CommitBlockError, ValidateContextError};
 pub use request::{FinalizedBlock, HashOrHeight, PreparedBlock, Request};
 pub use response::Response;
 pub use service::init;
+pub use utxo::Utxo;
diff --git a/zebra-state/src/request.rs b/zebra-state/src/request.rs
index 7c78112ad63..8f9fa39740c 100644
--- a/zebra-state/src/request.rs
+++ b/zebra-state/src/request.rs
@@ -5,6 +5,8 @@ use zebra_chain::{
     transaction, transparent,
 };
 
+use crate::Utxo;
+
 // Allow *only* this unused import, so that rustdoc link resolution
 // will work with inline links.
 #[allow(unused_imports)]
@@ -71,7 +73,7 @@ pub struct PreparedBlock {
     /// Note: although these transparent outputs are newly created, they may not
     /// be unspent, since a later transaction in a block can spend outputs of an
     /// earlier transaction.
-    pub new_outputs: HashMap<transparent::OutPoint, transparent::Output>,
+    pub new_outputs: HashMap<transparent::OutPoint, Utxo>,
     // TODO: add these parameters when we can compute anchors.
     // sprout_anchor: sprout::tree::Root,
     // sapling_anchor: sapling::tree::Root,
@@ -88,6 +90,13 @@ pub struct FinalizedBlock {
     pub(crate) block: Arc<Block>,
     pub(crate) hash: block::Hash,
     pub(crate) height: block::Height,
+    /// New transparent outputs created in this block, indexed by
+    /// [`Outpoint`](transparent::Outpoint).
+    ///
+    /// Note: although these transparent outputs are newly created, they may not
+    /// be unspent, since a later transaction in a block can spend outputs of an
+    /// earlier transaction.
+    pub(crate) new_outputs: HashMap<transparent::OutPoint, Utxo>,
 }
 
 // Doing precomputation in this From impl means that it will be done in
@@ -100,10 +109,45 @@ impl From<Arc<Block>> for FinalizedBlock {
             .expect("finalized blocks must have a valid coinbase height");
         let hash = block.hash();
 
+        let mut new_outputs = HashMap::default();
+        for transaction in &block.transactions {
+            let hash = transaction.hash();
+            let from_coinbase = transaction.is_coinbase();
+            for (index, output) in transaction.outputs().iter().cloned().enumerate() {
+                let index = index as u32;
+                new_outputs.insert(
+                    transparent::OutPoint { hash, index },
+                    Utxo {
+                        output,
+                        height,
+                        from_coinbase,
+                    },
+                );
+            }
+        }
+
+        Self {
+            block,
+            height,
+            hash,
+            new_outputs,
+        }
+    }
+}
+
+impl From<PreparedBlock> for FinalizedBlock {
+    fn from(prepared: PreparedBlock) -> Self {
+        let PreparedBlock {
+            block,
+            height,
+            hash,
+            new_outputs,
+        } = prepared;
         Self {
             block,
             height,
             hash,
+            new_outputs,
         }
     }
 }
diff --git a/zebra-state/src/response.rs b/zebra-state/src/response.rs
index b171bfe69a4..fd6df1af127 100644
--- a/zebra-state/src/response.rs
+++ b/zebra-state/src/response.rs
@@ -2,9 +2,10 @@ use std::sync::Arc;
 use zebra_chain::{
     block::{self, Block},
     transaction::Transaction,
-    transparent,
 };
 
+use crate::Utxo;
+
 // Allow *only* this unused import, so that rustdoc link resolution
 // will work with inline links.
 #[allow(unused_imports)]
@@ -33,5 +34,5 @@ pub enum Response {
     Block(Option<Arc<Block>>),
 
     /// The response to a `AwaitUtxo` request
-    Utxo(transparent::Output),
+    Utxo(Utxo),
 }
diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs
index 35f09eb7120..79caea6bd31 100644
--- a/zebra-state/src/service.rs
+++ b/zebra-state/src/service.rs
@@ -21,7 +21,7 @@ use zebra_chain::{
 
 use crate::{
     request::HashOrHeight, BoxError, CommitBlockError, Config, FinalizedBlock, PreparedBlock,
-    Request, Response, ValidateContextError,
+    Request, Response, Utxo, ValidateContextError,
 };
 
 use self::finalized_state::FinalizedState;
@@ -251,12 +251,12 @@ impl StateService {
             .or_else(|| self.disk.height(hash))
     }
 
-    /// Return the utxo pointed to by `outpoint` if it exists in any chain.
-    pub fn utxo(&self, outpoint: &transparent::OutPoint) -> Option<transparent::Output> {
+    /// Return the [`Utxo`] pointed to by `outpoint` if it exists in any chain.
+    pub fn utxo(&self, outpoint: &transparent::OutPoint) -> Option<Utxo> {
         self.mem
             .utxo(outpoint)
-            .or_else(|| self.disk.utxo(outpoint))
             .or_else(|| self.queued_blocks.utxo(outpoint))
+            .or_else(|| self.disk.utxo(outpoint))
     }
 
     /// Return an iterator over the relevant chain of the block identified by
@@ -404,7 +404,7 @@ impl Service<Request> for StateService {
 
                 let (rsp_tx, rsp_rx) = oneshot::channel();
 
-                self.pending_utxos.scan_block(&finalized.block);
+                self.pending_utxos.check_against(&finalized.new_outputs);
                 self.disk.queue_and_commit_finalized((finalized, rsp_tx));
 
                 async move {
diff --git a/zebra-state/src/service/finalized_state.rs b/zebra-state/src/service/finalized_state.rs
index e801b5a3f54..732982442b6 100644
--- a/zebra-state/src/service/finalized_state.rs
+++ b/zebra-state/src/service/finalized_state.rs
@@ -11,7 +11,7 @@ use zebra_chain::{
     transaction::{self, Transaction},
 };
 
-use crate::{BoxError, Config, FinalizedBlock, HashOrHeight};
+use crate::{BoxError, Config, FinalizedBlock, HashOrHeight, Utxo};
 
 use self::disk_format::{DiskDeserialize, DiskSerialize, FromDisk, IntoDisk, TransactionLocation};
 
@@ -149,6 +149,7 @@ impl FinalizedState {
             block,
             hash,
             height,
+            new_outputs,
         } = finalized;
 
         let hash_by_height = self.db.cf_handle("hash_by_height").unwrap();
@@ -205,7 +206,13 @@ impl FinalizedState {
                 return batch;
             }
 
-            // Index each transaction
+            // Index all new transparent outputs
+            for (outpoint, utxo) in new_outputs.into_iter() {
+                batch.zs_insert(utxo_by_outpoint, outpoint, utxo);
+            }
+
+            // Index each transaction, spent inputs, nullifiers
+            // TODO: move computation into FinalizedBlock as with transparent outputs
             for (transaction_index, transaction) in block.transactions.iter().enumerate() {
                 let transaction_hash = transaction.hash();
                 let transaction_location = TransactionLocation {
@@ -228,15 +235,6 @@ impl FinalizedState {
                     }
                 }
 
-                // Index all new transparent outputs
-                for (index, output) in transaction.outputs().iter().enumerate() {
-                    let outpoint = transparent::OutPoint {
-                        hash: transaction_hash,
-                        index: index as _,
-                    };
-                    batch.zs_insert(utxo_by_outpoint, outpoint, output);
-                }
-
                 // Mark sprout and sapling nullifiers as spent
                 for sprout_nullifier in transaction.sprout_nullifiers() {
                     batch.zs_insert(sprout_nullifiers, sprout_nullifier, ());
@@ -305,7 +303,7 @@ impl FinalizedState {
 
     /// Returns the `transparent::Output` pointed to by the given
     /// `transparent::OutPoint` if it is present.
-    pub fn utxo(&self, outpoint: &transparent::OutPoint) -> Option<transparent::Output> {
+    pub fn utxo(&self, outpoint: &transparent::OutPoint) -> Option<Utxo> {
         let utxo_by_outpoint = self.db.cf_handle("utxo_by_outpoint").unwrap();
         self.db.zs_get(utxo_by_outpoint, outpoint)
     }
diff --git a/zebra-state/src/service/finalized_state/disk_format.rs b/zebra-state/src/service/finalized_state/disk_format.rs
index f3d150f8135..92ec14abf3c 100644
--- a/zebra-state/src/service/finalized_state/disk_format.rs
+++ b/zebra-state/src/service/finalized_state/disk_format.rs
@@ -5,10 +5,12 @@ use zebra_chain::{
     block,
     block::Block,
     sapling,
-    serialization::{ZcashDeserialize, ZcashSerialize},
+    serialization::{ZcashDeserialize, ZcashDeserializeInto, ZcashSerialize},
     sprout, transaction, transparent,
 };
 
+use crate::Utxo;
+
 #[derive(Debug, Clone, Copy, PartialEq)]
 pub struct TransactionLocation {
     pub height: block::Height,
@@ -184,19 +186,33 @@ impl FromDisk for block::Height {
     }
 }
 
-impl IntoDisk for transparent::Output {
+impl IntoDisk for Utxo {
     type Bytes = Vec<u8>;
 
     fn as_bytes(&self) -> Self::Bytes {
-        self.zcash_serialize_to_vec()
-            .expect("serialization to vec doesn't fail")
+        let mut bytes = vec![0; 5];
+        bytes[0..4].copy_from_slice(&self.height.0.to_be_bytes());
+        bytes[4] = self.from_coinbase as u8;
+        self.output
+            .zcash_serialize(&mut bytes)
+            .expect("serialization to vec doesn't fail");
+        bytes
     }
 }
 
-impl FromDisk for transparent::Output {
+impl FromDisk for Utxo {
     fn from_bytes(bytes: impl AsRef<[u8]>) -> Self {
-        Self::zcash_deserialize(bytes.as_ref())
-            .expect("deserialization format should match the serialization format used by IntoDisk")
+        let (meta_bytes, output_bytes) = bytes.as_ref().split_at(5);
+        let height = block::Height(u32::from_be_bytes(meta_bytes[0..4].try_into().unwrap()));
+        let from_coinbase = meta_bytes[4] == 1u8;
+        let output = output_bytes
+            .zcash_deserialize_into()
+            .expect("db has serialized data");
+        Self {
+            output,
+            height,
+            from_coinbase,
+        }
     }
 }
 
@@ -370,6 +386,6 @@ mod tests {
     fn roundtrip_transparent_output() {
         zebra_test::init();
 
-        proptest!(|(val in any::<transparent::Output>())| assert_value_properties(val));
+        proptest!(|(val in any::<Utxo>())| assert_value_properties(val));
     }
 }
diff --git a/zebra-state/src/service/non_finalized_state.rs b/zebra-state/src/service/non_finalized_state.rs
index bf849cf8e7e..5b1fdc75080 100644
--- a/zebra-state/src/service/non_finalized_state.rs
+++ b/zebra-state/src/service/non_finalized_state.rs
@@ -15,7 +15,7 @@ use zebra_chain::{
     transparent,
 };
 
-use crate::{FinalizedBlock, HashOrHeight, PreparedBlock};
+use crate::{FinalizedBlock, HashOrHeight, PreparedBlock, Utxo};
 
 use self::chain::Chain;
 
@@ -64,12 +64,7 @@ impl NonFinalizedState {
 
         self.update_metrics_for_chains();
 
-        // Construct a finalized block.
-        FinalizedBlock {
-            block: finalizing.block,
-            hash: finalizing.hash,
-            height: finalizing.height,
-        }
+        finalizing.into()
     }
 
     /// Commit block to the non-finalized state.
@@ -148,7 +143,7 @@ impl NonFinalizedState {
 
     /// Returns the `transparent::Output` pointed to by the given
     /// `transparent::OutPoint` if it is present.
-    pub fn utxo(&self, outpoint: &transparent::OutPoint) -> Option<transparent::Output> {
+    pub fn utxo(&self, outpoint: &transparent::OutPoint) -> Option<Utxo> {
         for chain in self.chain_set.iter().rev() {
             if let Some(output) = chain.created_utxos.get(outpoint) {
                 return Some(output.clone());
diff --git a/zebra-state/src/service/non_finalized_state/chain.rs b/zebra-state/src/service/non_finalized_state/chain.rs
index 583f4320e82..3dc3ec0c11e 100644
--- a/zebra-state/src/service/non_finalized_state/chain.rs
+++ b/zebra-state/src/service/non_finalized_state/chain.rs
@@ -10,7 +10,7 @@ use zebra_chain::{
     work::difficulty::PartialCumulativeWork,
 };
 
-use crate::PreparedBlock;
+use crate::{PreparedBlock, Utxo};
 
 #[derive(Default, Clone)]
 pub struct Chain {
@@ -18,7 +18,7 @@ pub struct Chain {
     pub height_by_hash: HashMap<block::Hash, block::Height>,
     pub tx_by_hash: HashMap<transaction::Hash, (block::Height, usize)>,
 
-    pub created_utxos: HashMap<transparent::OutPoint, transparent::Output>,
+    pub created_utxos: HashMap<transparent::OutPoint, Utxo>,
     spent_utxos: HashSet<transparent::OutPoint>,
     sprout_anchors: HashSet<sprout::tree::Root>,
     sapling_anchors: HashSet<sapling::tree::Root>,
@@ -155,14 +155,13 @@ impl UpdateWith<PreparedBlock> for Chain {
 
         // for each transaction in block
         for (transaction_index, transaction) in block.transactions.iter().enumerate() {
-            let (inputs, outputs, shielded_data, joinsplit_data) = match transaction.deref() {
+            let (inputs, shielded_data, joinsplit_data) = match transaction.deref() {
                 transaction::Transaction::V4 {
                     inputs,
-                    outputs,
                     shielded_data,
                     joinsplit_data,
                     ..
-                } => (inputs, outputs, shielded_data, joinsplit_data),
+                } => (inputs, shielded_data, joinsplit_data),
                 _ => unreachable!(
                     "older transaction versions only exist in finalized blocks pre sapling",
                 ),
@@ -179,7 +178,7 @@ impl UpdateWith<PreparedBlock> for Chain {
             );
 
             // add the utxos this produced
-            self.update_chain_state_with(&(transaction_hash, outputs));
+            self.update_chain_state_with(&prepared.new_outputs);
             // add the utxos this consumed
             self.update_chain_state_with(inputs);
             // add sprout anchor and nullifiers
@@ -209,14 +208,13 @@ impl UpdateWith<PreparedBlock> for Chain {
 
         // for each transaction in block
         for transaction in &block.transactions {
-            let (inputs, outputs, shielded_data, joinsplit_data) = match transaction.deref() {
+            let (inputs, shielded_data, joinsplit_data) = match transaction.deref() {
                 transaction::Transaction::V4 {
                     inputs,
-                    outputs,
                     shielded_data,
                     joinsplit_data,
                     ..
-                } => (inputs, outputs, shielded_data, joinsplit_data),
+                } => (inputs, shielded_data, joinsplit_data),
                 _ => unreachable!(
                     "older transaction versions only exist in finalized blocks pre sapling",
                 ),
@@ -230,7 +228,7 @@ impl UpdateWith<PreparedBlock> for Chain {
             );
 
             // remove the utxos this produced
-            self.revert_chain_state_with(&(transaction_hash, outputs));
+            self.revert_chain_state_with(&prepared.new_outputs);
             // remove the utxos this consumed
             self.revert_chain_state_with(inputs);
             // remove sprout anchor and nullifiers
@@ -241,37 +239,15 @@ impl UpdateWith<PreparedBlock> for Chain {
     }
 }
 
-impl UpdateWith<(transaction::Hash, &Vec<transparent::Output>)> for Chain {
-    fn update_chain_state_with(
-        &mut self,
-        (transaction_hash, outputs): &(transaction::Hash, &Vec<transparent::Output>),
-    ) {
-        for (utxo_index, output) in outputs.iter().enumerate() {
-            self.created_utxos.insert(
-                transparent::OutPoint {
-                    hash: *transaction_hash,
-                    index: utxo_index as u32,
-                },
-                output.clone(),
-            );
-        }
+impl UpdateWith<HashMap<transparent::OutPoint, Utxo>> for Chain {
+    fn update_chain_state_with(&mut self, utxos: &HashMap<transparent::OutPoint, Utxo>) {
+        self.created_utxos
+            .extend(utxos.iter().map(|(k, v)| (*k, v.clone())));
     }
 
-    fn revert_chain_state_with(
-        &mut self,
-        (transaction_hash, outputs): &(transaction::Hash, &Vec<transparent::Output>),
-    ) {
-        for (utxo_index, _) in outputs.iter().enumerate() {
-            assert!(
-                self.created_utxos
-                    .remove(&transparent::OutPoint {
-                        hash: *transaction_hash,
-                        index: utxo_index as u32,
-                    })
-                    .is_some(),
-                "created_utxos must be present if block was"
-            );
-        }
+    fn revert_chain_state_with(&mut self, utxos: &HashMap<transparent::OutPoint, Utxo>) {
+        self.created_utxos
+            .retain(|outpoint, _| !utxos.contains_key(outpoint));
     }
 }
 
diff --git a/zebra-state/src/service/non_finalized_state/queued_blocks.rs b/zebra-state/src/service/non_finalized_state/queued_blocks.rs
index f8007e57a92..8e0f4090ce1 100644
--- a/zebra-state/src/service/non_finalized_state/queued_blocks.rs
+++ b/zebra-state/src/service/non_finalized_state/queued_blocks.rs
@@ -6,7 +6,7 @@ use std::{
 use tracing::instrument;
 use zebra_chain::{block, transparent};
 
-use crate::service::QueuedBlock;
+use crate::{service::QueuedBlock, Utxo};
 
 /// A queue of blocks, awaiting the arrival of parent blocks.
 #[derive(Default)]
@@ -18,7 +18,7 @@ pub struct QueuedBlocks {
     /// Hashes from `queued_blocks`, indexed by block height.
     by_height: BTreeMap<block::Height, HashSet<block::Hash>>,
     /// Known UTXOs.
-    known_utxos: HashMap<transparent::OutPoint, transparent::Output>,
+    known_utxos: HashMap<transparent::OutPoint, Utxo>,
 }
 
 impl QueuedBlocks {
@@ -150,7 +150,7 @@ impl QueuedBlocks {
     }
 
     /// Try to look up this UTXO in any queued block.
-    pub fn utxo(&self, outpoint: &transparent::OutPoint) -> Option<transparent::Output> {
+    pub fn utxo(&self, outpoint: &transparent::OutPoint) -> Option<Utxo> {
         self.known_utxos.get(outpoint).cloned()
     }
 }
diff --git a/zebra-state/src/service/tests.rs b/zebra-state/src/service/tests.rs
index 3442cad4f84..504a1b3ca20 100644
--- a/zebra-state/src/service/tests.rs
+++ b/zebra-state/src/service/tests.rs
@@ -8,7 +8,7 @@ use zebra_chain::{
 };
 use zebra_test::{prelude::*, transcript::Transcript};
 
-use crate::{init, BoxError, Config, Request, Response};
+use crate::{init, BoxError, Config, Request, Response, Utxo};
 
 const LAST_BLOCK_HEIGHT: u32 = 10;
 
@@ -80,16 +80,19 @@ async fn test_populated_state_responds_correctly(
                     Ok(Response::Transaction(Some(transaction.clone()))),
                 ));
 
-                for (index, output) in transaction.outputs().iter().enumerate() {
+                let from_coinbase = transaction.is_coinbase();
+                for (index, output) in transaction.outputs().iter().cloned().enumerate() {
                     let outpoint = transparent::OutPoint {
                         hash: transaction_hash,
                         index: index as _,
                     };
+                    let utxo = Utxo {
+                        output,
+                        height,
+                        from_coinbase,
+                    };
 
-                    transcript.push((
-                        Request::AwaitUtxo(outpoint),
-                        Ok(Response::Utxo(output.clone())),
-                    ));
+                    transcript.push((Request::AwaitUtxo(outpoint), Ok(Response::Utxo(utxo))));
                 }
             }
         }
diff --git a/zebra-state/src/service/utxo.rs b/zebra-state/src/service/utxo.rs
index 099dee24279..c0740096f46 100644
--- a/zebra-state/src/service/utxo.rs
+++ b/zebra-state/src/service/utxo.rs
@@ -1,12 +1,14 @@
-#![allow(dead_code)]
-use crate::{BoxError, Response};
 use std::collections::HashMap;
 use std::future::Future;
+
 use tokio::sync::broadcast;
-use zebra_chain::{block::Block, transparent};
+
+use zebra_chain::transparent;
+
+use crate::{BoxError, Response, Utxo};
 
 #[derive(Debug, Default)]
-pub struct PendingUtxos(HashMap<transparent::OutPoint, broadcast::Sender<transparent::Output>>);
+pub struct PendingUtxos(HashMap<transparent::OutPoint, broadcast::Sender<Utxo>>);
 
 impl PendingUtxos {
     /// Returns a future that will resolve to the `transparent::Output` pointed
@@ -33,41 +35,23 @@ impl PendingUtxos {
         }
     }
 
-    /// Notify all utxo requests waiting for the `transparent::Output` pointed to
-    /// by the given `transparent::OutPoint` that the `Output` has arrived.
-    pub fn respond(&mut self, outpoint: &transparent::OutPoint, output: transparent::Output) {
-        if let Some(sender) = self.0.remove(&outpoint) {
+    /// Notify all requests waiting for the [`Utxo`] pointed to by the given
+    /// [`transparent::OutPoint`] that the [`Utxo`] has arrived.
+    pub fn respond(&mut self, outpoint: &transparent::OutPoint, utxo: Utxo) {
+        if let Some(sender) = self.0.remove(outpoint) {
             // Adding the outpoint as a field lets us crossreference
             // with the trace of the verification that made the request.
             tracing::trace!(?outpoint, "found pending UTXO");
-            let _ = sender.send(output);
+            let _ = sender.send(utxo);
         }
     }
 
     /// Check the list of pending UTXO requests against the supplied UTXO index.
-    pub fn check_against(&mut self, utxos: &HashMap<transparent::OutPoint, transparent::Output>) {
-        for (outpoint, output) in utxos.iter() {
-            self.respond(outpoint, output.clone());
-        }
-    }
-
-    /// Scan through unindexed transactions in the given `block`
-    /// to determine whether it contains any requested UTXOs.
-    pub fn scan_block(&mut self, block: &Block) {
-        if self.0.is_empty() {
-            return;
-        }
-
-        tracing::trace!("scanning new block for pending UTXOs");
-        for transaction in block.transactions.iter() {
-            let transaction_hash = transaction.hash();
-            for (index, output) in transaction.outputs().iter().enumerate() {
-                let outpoint = transparent::OutPoint {
-                    hash: transaction_hash,
-                    index: index as _,
-                };
-
-                self.respond(&outpoint, output.clone());
+    pub fn check_against(&mut self, utxos: &HashMap<transparent::OutPoint, Utxo>) {
+        for (outpoint, utxo) in utxos.iter() {
+            if let Some(sender) = self.0.remove(outpoint) {
+                tracing::trace!(?outpoint, "found pending UTXO");
+                let _ = sender.send(utxo.clone());
             }
         }
     }
diff --git a/zebra-state/src/tests.rs b/zebra-state/src/tests.rs
index dee35a69bb6..3c788967330 100644
--- a/zebra-state/src/tests.rs
+++ b/zebra-state/src/tests.rs
@@ -1,4 +1,4 @@
-use std::{collections::HashMap, convert::TryFrom, mem, sync::Arc};
+use std::{convert::TryFrom, mem, sync::Arc};
 
 use primitive_types::U256;
 use zebra_chain::{
@@ -21,15 +21,7 @@ impl Prepare for Arc<Block> {
         let block = self;
         let hash = block.hash();
         let height = block.coinbase_height().unwrap();
-
-        let mut new_outputs = HashMap::new();
-        for transaction in &block.transactions {
-            let hash = transaction.hash();
-            for (index, output) in transaction.outputs().iter().cloned().enumerate() {
-                let index = index as u32;
-                new_outputs.insert(transparent::OutPoint { hash, index }, output);
-            }
-        }
+        let new_outputs = crate::utxo::new_outputs(&block);
 
         PreparedBlock {
             block,
diff --git a/zebra-state/src/utxo.rs b/zebra-state/src/utxo.rs
new file mode 100644
index 00000000000..5ccfa3a19cf
--- /dev/null
+++ b/zebra-state/src/utxo.rs
@@ -0,0 +1,42 @@
+// needed to make clippy happy with derive(Arbitrary)
+#![allow(clippy::unit_arg)]
+
+use zebra_chain::{block, transparent};
+
+/// An unspent `transparent::Output`, with accompanying metadata.
+#[derive(Clone, Debug, PartialEq, Eq)]
+#[cfg_attr(test, derive(proptest_derive::Arbitrary))]
+pub struct Utxo {
+    /// The output itself.
+    pub output: transparent::Output,
+    /// The height at which the output was created.
+    pub height: block::Height,
+    /// Whether the output originated in a coinbase transaction.
+    pub from_coinbase: bool,
+}
+
+#[cfg(test)]
+pub fn new_outputs(block: &block::Block) -> std::collections::HashMap<transparent::OutPoint, Utxo> {
+    use std::collections::HashMap;
+
+    let height = block.coinbase_height().expect("block has coinbase height");
+
+    let mut new_outputs = HashMap::default();
+    for transaction in &block.transactions {
+        let hash = transaction.hash();
+        let from_coinbase = transaction.is_coinbase();
+        for (index, output) in transaction.outputs().iter().cloned().enumerate() {
+            let index = index as u32;
+            new_outputs.insert(
+                transparent::OutPoint { hash, index },
+                Utxo {
+                    output,
+                    height,
+                    from_coinbase,
+                },
+            );
+        }
+    }
+
+    new_outputs
+}

From 040e50b183b56b8edc55c683f66b5d80e40fdc2d Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Mon, 23 Nov 2020 12:54:27 -0800
Subject: [PATCH 095/148] state: service::utxo -> service::pending_utxos

---
 zebra-state/src/service.rs                            | 10 +++++-----
 zebra-state/src/service/{utxo.rs => pending_utxos.rs} |  0
 2 files changed, 5 insertions(+), 5 deletions(-)
 rename zebra-state/src/service/{utxo.rs => pending_utxos.rs} (100%)

diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs
index 79caea6bd31..07daf6c2395 100644
--- a/zebra-state/src/service.rs
+++ b/zebra-state/src/service.rs
@@ -24,14 +24,14 @@ use crate::{
     Request, Response, Utxo, ValidateContextError,
 };
 
-use self::finalized_state::FinalizedState;
-
 mod check;
 mod finalized_state;
 mod non_finalized_state;
+mod pending_utxos;
 #[cfg(test)]
 mod tests;
-mod utxo;
+
+use self::{finalized_state::FinalizedState, pending_utxos::PendingUtxos};
 
 pub type QueuedBlock = (
     PreparedBlock,
@@ -50,7 +50,7 @@ struct StateService {
     /// Blocks awaiting their parent blocks for contextual verification.
     queued_blocks: QueuedBlocks,
     /// The set of outpoints with pending requests for their associated transparent::Output
-    pending_utxos: utxo::PendingUtxos,
+    pending_utxos: PendingUtxos,
     /// The configured Zcash network
     network: Network,
     /// Instant tracking the last time `pending_utxos` was pruned
@@ -64,7 +64,7 @@ impl StateService {
         let disk = FinalizedState::new(&config, network);
         let mem = NonFinalizedState::default();
         let queued_blocks = QueuedBlocks::default();
-        let pending_utxos = utxo::PendingUtxos::default();
+        let pending_utxos = PendingUtxos::default();
 
         Self {
             disk,
diff --git a/zebra-state/src/service/utxo.rs b/zebra-state/src/service/pending_utxos.rs
similarity index 100%
rename from zebra-state/src/service/utxo.rs
rename to zebra-state/src/service/pending_utxos.rs

From 6202cb45b1ade4200b6197a2c855f1403ce46469 Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Mon, 23 Nov 2020 14:29:57 -0800
Subject: [PATCH 096/148] ci: remove nightly clippy

Closes #1212
---
 .github/workflows/ci.yml | 20 --------------------
 1 file changed, 20 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 05ff5c02d6f..61e2473d913 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -112,26 +112,6 @@ jobs:
           token: ${{ secrets.GITHUB_TOKEN }}
           args: --all-features --all-targets -- -D warnings
 
-  clippy-nightly:
-    name: Clippy (nightly)
-    timeout-minutes: 30
-    runs-on: ubuntu-latest
-    continue-on-error: true
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
-        with:
-          toolchain: nightly
-          components: clippy
-          override: true
-      - name: Run Clippy (nightly)
-        uses: actions-rs/clippy-check@v1
-        continue-on-error: true
-        with:
-          name: Clippy (nightly)
-          token: ${{ secrets.GITHUB_TOKEN }}
-          args: --all-features --all-targets -- -W clippy::all
-
   fmt:
     name: Rustfmt
     timeout-minutes: 30

From 2d60c00fb068178d7cb765133d24241575baff9b Mon Sep 17 00:00:00 2001
From: teor <teor@riseup.net>
Date: Tue, 24 Nov 2020 16:46:02 +1000
Subject: [PATCH 097/148] Avoid a panic when downcasting to redjubjub::Error
 fails (#1363)

Instead, format the original error as a string, to provide better
diagnostics.

Temporary fix for #1357, the permanent fix ticket is #1186.
---
 zebra-consensus/src/error.rs | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/zebra-consensus/src/error.rs b/zebra-consensus/src/error.rs
index caa2f2edb03..419dc2e8d9e 100644
--- a/zebra-consensus/src/error.rs
+++ b/zebra-consensus/src/error.rs
@@ -67,13 +67,20 @@ pub enum TransactionError {
 
     #[error("bindingSig MUST represent a valid signature under the transaction binding validating key bvk of SigHash")]
     RedJubjub(redjubjub::Error),
+
+    // temporary error type until #1186 is fixed
+    #[error("Downcast from BoxError to redjubjub::Error failed")]
+    InternalDowncastError(String),
 }
 
 impl From<BoxError> for TransactionError {
     fn from(err: BoxError) -> Self {
         match err.downcast::<redjubjub::Error>() {
             Ok(e) => TransactionError::RedJubjub(*e),
-            Err(e) => panic!(e),
+            Err(e) => TransactionError::InternalDowncastError(format!(
+                "downcast to redjubjub::Error failed, original error: {:?}",
+                e
+            )),
         }
     }
 }

From 6387dfe1d0048cce89f5e59de506609516f1114b Mon Sep 17 00:00:00 2001
From: teor <teor@riseup.net>
Date: Tue, 24 Nov 2020 16:52:24 +1000
Subject: [PATCH 098/148] Fix individual crate compilation failures

Some Zebra crates don't compile individually due to missing features in
their dependencies. Add those features to each crate's dependency list.
---
 tower-batch/Cargo.toml   | 2 +-
 zebra-network/Cargo.toml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tower-batch/Cargo.toml b/tower-batch/Cargo.toml
index c5063db488a..b4caa44fa1e 100644
--- a/tower-batch/Cargo.toml
+++ b/tower-batch/Cargo.toml
@@ -6,7 +6,7 @@ license = "MIT"
 edition = "2018"
 
 [dependencies]
-tokio = { version = "0.3", features = ["time", "sync", "stream", "tracing"] }
+tokio = { version = "0.3", features = ["time", "sync", "stream", "tracing", "macros"] }
 tower = { version = "0.4", features = ["util", "buffer"] }
 futures-core = "0.3.6"
 pin-project = "0.4.27"
diff --git a/zebra-network/Cargo.toml b/zebra-network/Cargo.toml
index 3a21a702a15..23fd4380e01 100644
--- a/zebra-network/Cargo.toml
+++ b/zebra-network/Cargo.toml
@@ -22,7 +22,7 @@ serde = { version = "1", features = ["serde_derive"] }
 thiserror = "1"
 
 futures = "0.3"
-tokio = { version = "0.3", features = ["net", "time", "stream", "tracing", "macros"] }
+tokio = { version = "0.3", features = ["net", "time", "stream", "tracing", "macros", "rt-multi-thread"] }
 tokio-util = { version = "0.5", features = ["codec"] }
 tower = { version = "0.4", features = ["retry", "discover", "load", "load-shed", "timeout", "util", "buffer"] }
 

From f1155297a3c0278da8a23085d2d843ca128c0528 Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Tue, 24 Nov 2020 01:31:03 -0800
Subject: [PATCH 099/148] consensus: temporarily skip scripts on big
 transactions (#1360)

Temporary workaround for #1350 until that issue is properly fixed.
---
 zebra-consensus/src/script.rs | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/zebra-consensus/src/script.rs b/zebra-consensus/src/script.rs
index 9dc4a4d105e..cad3c6b093e 100644
--- a/zebra-consensus/src/script.rs
+++ b/zebra-consensus/src/script.rs
@@ -92,12 +92,19 @@ where
                     };
                     tracing::trace!(?utxo, "got UTXO");
 
-                    zebra_script::is_valid(
-                        transaction,
-                        branch_id,
-                        (input_index as u32, utxo.output),
-                    )?;
-                    tracing::trace!("script verification succeeded");
+                    if transaction.inputs().len() < 20 {
+                        zebra_script::is_valid(
+                            transaction,
+                            branch_id,
+                            (input_index as u32, utxo.output),
+                        )?;
+                        tracing::trace!("script verification succeeded");
+                    } else {
+                        tracing::debug!(
+                            inputs.len = transaction.inputs().len(),
+                            "skipping verification of script with many inputs to avoid quadratic work until we fix zebra_script/zcash_script interface"
+                        );
+                    }
 
                     Ok(())
                 }

From f36a4800b2736d15117e129f5df9bf22240dca7f Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Mon, 23 Nov 2020 23:02:48 -0800
Subject: [PATCH 100/148] network: fix invariant violation in peer set

Closes #1183.

The peer set maintains a preselected ready service that it can use to
perform power-of-two-choices (p2c) routing of requests.  Ready services
are stored by key (socket address) in an `IndexMap`, and the preselected
service is represented by an `Option<usize>` indexing that map.  This
means that whenever the set of ready services changes (e.g., a service
is removed from the peer set, or a service is taken to be used to
process a request), the preselected index is invalidated.  The original
P2C-only implementation maintained this invariant but did not document
it.

The change to inventory-based routing introduced a bug by failing to
maintain this invariant and appropriately invalidate the preselected
index.  However, this was only noticeable approximately 1/N of the time
on the next request after an inventory-directed request, so the bug
occurred infrequently.  Luckily, the use of `.expect` caused the bug to
be an immediate panic, making it possible to identify by inspecting all
uses of the ready service map.
---
 zebra-network/src/peer_set/set.rs | 27 ++++++++++++++++-----------
 1 file changed, 16 insertions(+), 11 deletions(-)

diff --git a/zebra-network/src/peer_set/set.rs b/zebra-network/src/peer_set/set.rs
index a8df59e5be2..fe9d50d7139 100644
--- a/zebra-network/src/peer_set/set.rs
+++ b/zebra-network/src/peer_set/set.rs
@@ -82,10 +82,13 @@ where
     D: Discover<Key = SocketAddr>,
 {
     discover: D,
+    /// A preselected index for a ready service.
+    /// INVARIANT: If `next_idx` is `Some(i)`, `i` must be a valid index for `ready_services`.
+    /// This means that every change to `ready_services` must invalidate or correct `next_idx`.
+    next_idx: Option<usize>,
     ready_services: IndexMap<D::Key, D::Service>,
     cancel_handles: HashMap<D::Key, oneshot::Sender<()>>,
     unready_services: FuturesUnordered<UnreadyService<D::Key, D::Service, Request>>,
-    next_idx: Option<usize>,
     demand_signal: mpsc::Sender<()>,
     /// Channel for passing ownership of tokio JoinHandles from PeerSet's background tasks
     ///
@@ -117,10 +120,10 @@ where
     ) -> Self {
         Self {
             discover,
+            next_idx: None,
             ready_services: IndexMap::new(),
             cancel_handles: HashMap::new(),
             unready_services: FuturesUnordered::new(),
-            next_idx: None,
             demand_signal,
             guards: futures::stream::FuturesUnordered::new(),
             handle_rx,
@@ -149,20 +152,20 @@ where
     }
 
     fn remove(&mut self, key: &D::Key) {
-        // Remove key from either the set of ready services,
-        // or else from the set of unready services.
         if let Some((i, _, _)) = self.ready_services.swap_remove_full(key) {
             // swap_remove perturbs the position of the last element of
             // ready_services, so we may have invalidated self.next_idx, in
             // which case we need to fix it. Specifically, swap_remove swaps the
             // position of the removee and the last element, then drops the
             // removee from the end, so we compare the active and removed indices:
-            let len = self.ready_services.len();
+            //
+            // We just removed one element, so this was the index of the last element.
+            let last_idx = self.ready_services.len();
             self.next_idx = match self.next_idx {
-                None => None,                   // No active index
-                Some(j) if j == i => None,      // We removed j
-                Some(j) if j == len => Some(i), // We swapped i and j
-                Some(j) => Some(j),             // We swapped an unrelated service.
+                None => None,                        // No active index
+                Some(j) if j == i => None,           // We removed j
+                Some(j) if j == last_idx => Some(i), // We swapped i and j
+                Some(j) => Some(j),                  // We swapped an unrelated service.
             };
             // No Heisenservices: they must be ready or unready.
             assert!(!self.cancel_handles.contains_key(key));
@@ -240,7 +243,6 @@ where
             0 => None,
             1 => Some(0),
             len => {
-                // XXX avoid relying on rand complexity
                 let (a, b) = {
                     let idxs = rand::seq::index::sample(&mut rand::thread_rng(), len, 2);
                     (idxs.index(0), idxs.index(1))
@@ -299,8 +301,10 @@ where
                     .ready_services
                     .swap_remove_index(index)
                     .expect("found index must be valid");
-                tracing::debug!(?hash, ?key, "routing based on inventory");
+                // We changed ready_services, so next_idx is invalid
+                self.next_idx = None;
 
+                tracing::debug!(?hash, ?key, "routing based on inventory");
                 let fut = svc.call(req);
                 self.push_unready(key, svc);
                 fut.map_err(Into::into).boxed()
@@ -320,6 +324,7 @@ where
         // This is not needless: otherwise, we'd hold a &mut reference to self.ready_services,
         // blocking us from passing &mut self to push_unready.
         let ready_services = std::mem::take(&mut self.ready_services);
+        self.next_idx = None; // We changed ready_services, so next_idx is invalid
 
         let futs = FuturesUnordered::new();
         for (key, mut svc) in ready_services {

From d90e709ce15734014a9e3271dd1054b5282eba2d Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Mon, 23 Nov 2020 23:52:40 -0800
Subject: [PATCH 101/148] network: tidy peer set implementation

- rename functions more descriptively
- create a common `take_ready_service` function
- organize poll_ functions separately
---
 zebra-network/src/peer_set/set.rs | 189 +++++++++++++++---------------
 1 file changed, 95 insertions(+), 94 deletions(-)

diff --git a/zebra-network/src/peer_set/set.rs b/zebra-network/src/peer_set/set.rs
index fe9d50d7139..ad95dc8443e 100644
--- a/zebra-network/src/peer_set/set.rs
+++ b/zebra-network/src/peer_set/set.rs
@@ -83,9 +83,9 @@ where
 {
     discover: D,
     /// A preselected index for a ready service.
-    /// INVARIANT: If `next_idx` is `Some(i)`, `i` must be a valid index for `ready_services`.
-    /// This means that every change to `ready_services` must invalidate or correct `next_idx`.
-    next_idx: Option<usize>,
+    /// INVARIANT: If this is `Some(i)`, `i` must be a valid index for `ready_services`.
+    /// This means that every change to `ready_services` must invalidate or correct it.
+    preselected_p2c_index: Option<usize>,
     ready_services: IndexMap<D::Key, D::Service>,
     cancel_handles: HashMap<D::Key, oneshot::Sender<()>>,
     unready_services: FuturesUnordered<UnreadyService<D::Key, D::Service, Request>>,
@@ -120,7 +120,7 @@ where
     ) -> Self {
         Self {
             discover,
-            next_idx: None,
+            preselected_p2c_index: None,
             ready_services: IndexMap::new(),
             cancel_handles: HashMap::new(),
             unready_services: FuturesUnordered::new(),
@@ -131,61 +131,7 @@ where
         }
     }
 
-    fn poll_discover(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), BoxError>> {
-        use futures::ready;
-        loop {
-            match ready!(Pin::new(&mut self.discover).poll_discover(cx))
-                .ok_or("discovery stream closed")?
-                .map_err(Into::into)?
-            {
-                Change::Remove(key) => {
-                    trace!(?key, "got Change::Remove from Discover");
-                    self.remove(&key);
-                }
-                Change::Insert(key, svc) => {
-                    trace!(?key, "got Change::Insert from Discover");
-                    self.remove(&key);
-                    self.push_unready(key, svc);
-                }
-            }
-        }
-    }
-
-    fn remove(&mut self, key: &D::Key) {
-        if let Some((i, _, _)) = self.ready_services.swap_remove_full(key) {
-            // swap_remove perturbs the position of the last element of
-            // ready_services, so we may have invalidated self.next_idx, in
-            // which case we need to fix it. Specifically, swap_remove swaps the
-            // position of the removee and the last element, then drops the
-            // removee from the end, so we compare the active and removed indices:
-            //
-            // We just removed one element, so this was the index of the last element.
-            let last_idx = self.ready_services.len();
-            self.next_idx = match self.next_idx {
-                None => None,                        // No active index
-                Some(j) if j == i => None,           // We removed j
-                Some(j) if j == last_idx => Some(i), // We swapped i and j
-                Some(j) => Some(j),                  // We swapped an unrelated service.
-            };
-            // No Heisenservices: they must be ready or unready.
-            assert!(!self.cancel_handles.contains_key(key));
-        } else if let Some(handle) = self.cancel_handles.remove(key) {
-            let _ = handle.send(());
-        }
-    }
-
-    fn push_unready(&mut self, key: D::Key, svc: D::Service) {
-        let (tx, rx) = oneshot::channel();
-        self.cancel_handles.insert(key, tx);
-        self.unready_services.push(UnreadyService {
-            key: Some(key),
-            service: Some(svc),
-            cancel: rx,
-            _req: PhantomData,
-        });
-    }
-
-    fn check_for_background_errors(&mut self, cx: &mut Context) -> Result<(), BoxError> {
+    fn poll_background_errors(&mut self, cx: &mut Context) -> Result<(), BoxError> {
         if self.guards.is_empty() {
             match self.handle_rx.try_recv() {
                 Ok(handles) => {
@@ -237,8 +183,71 @@ where
         }
     }
 
+    fn poll_discover(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), BoxError>> {
+        use futures::ready;
+        loop {
+            match ready!(Pin::new(&mut self.discover).poll_discover(cx))
+                .ok_or("discovery stream closed")?
+                .map_err(Into::into)?
+            {
+                Change::Remove(key) => {
+                    trace!(?key, "got Change::Remove from Discover");
+                    self.remove(&key);
+                }
+                Change::Insert(key, svc) => {
+                    trace!(?key, "got Change::Insert from Discover");
+                    self.remove(&key);
+                    self.push_unready(key, svc);
+                }
+            }
+        }
+    }
+
+    /// Takes a ready service by key, preserving `preselected_p2c_index` if possible.
+    fn take_ready_service(&mut self, key: &D::Key) -> Option<(D::Key, D::Service)> {
+        if let Some((i, key, svc)) = self.ready_services.swap_remove_full(key) {
+            // swap_remove perturbs the position of the last element of
+            // ready_services, so we may have invalidated self.next_idx, in
+            // which case we need to fix it. Specifically, swap_remove swaps the
+            // position of the removee and the last element, then drops the
+            // removee from the end, so we compare the active and removed indices:
+            //
+            // We just removed one element, so this was the index of the last element.
+            let last_idx = self.ready_services.len();
+            self.preselected_p2c_index = match self.preselected_p2c_index {
+                None => None,                        // No active index
+                Some(j) if j == i => None,           // We removed j
+                Some(j) if j == last_idx => Some(i), // We swapped i and j
+                Some(j) => Some(j),                  // We swapped an unrelated service.
+            };
+            // No Heisenservices: they must be ready or unready.
+            assert!(!self.cancel_handles.contains_key(&key));
+            Some((key, svc))
+        } else {
+            None
+        }
+    }
+
+    fn remove(&mut self, key: &D::Key) {
+        if let Some(_) = self.take_ready_service(key) {
+        } else if let Some(handle) = self.cancel_handles.remove(key) {
+            let _ = handle.send(());
+        }
+    }
+
+    fn push_unready(&mut self, key: D::Key, svc: D::Service) {
+        let (tx, rx) = oneshot::channel();
+        self.cancel_handles.insert(key, tx);
+        self.unready_services.push(UnreadyService {
+            key: Some(key),
+            service: Some(svc),
+            cancel: rx,
+            _req: PhantomData,
+        });
+    }
+
     /// Performs P2C on inner services to select a ready service.
-    fn select_next_ready_index(&mut self) -> Option<usize> {
+    fn preselect_p2c_index(&mut self) -> Option<usize> {
         match self.ready_services.len() {
             0 => None,
             1 => Some(0),
@@ -248,8 +257,8 @@ where
                     (idxs.index(0), idxs.index(1))
                 };
 
-                let a_load = self.ready_index_load(a);
-                let b_load = self.ready_index_load(b);
+                let a_load = self.query_load(a);
+                let b_load = self.query_load(b);
 
                 let selected = if a_load <= b_load { a } else { b };
 
@@ -261,7 +270,7 @@ where
     }
 
     /// Accesses a ready endpoint by index and returns its current load.
-    fn ready_index_load(&self, index: usize) -> <D::Service as Load>::Metric {
+    fn query_load(&self, index: usize) -> <D::Service as Load>::Metric {
         let (_, svc) = self.ready_services.get_index(index).expect("invalid index");
         svc.load()
     }
@@ -269,7 +278,7 @@ where
     /// Routes a request using P2C load-balancing.
     fn route_p2c(&mut self, req: Request) -> <Self as tower::Service<Request>>::Future {
         let index = self
-            .next_idx
+            .preselected_p2c_index
             .take()
             .expect("ready service must have valid preselected index");
 
@@ -290,30 +299,21 @@ where
         req: Request,
         hash: InventoryHash,
     ) -> <Self as tower::Service<Request>>::Future {
-        let candidate_index = self
+        let peer = self
             .inventory_registry
             .peers(&hash)
-            .find_map(|addr| self.ready_services.get_index_of(addr));
-
-        match candidate_index {
-            Some(index) => {
-                let (key, mut svc) = self
-                    .ready_services
-                    .swap_remove_index(index)
-                    .expect("found index must be valid");
-                // We changed ready_services, so next_idx is invalid
-                self.next_idx = None;
+            .find(|&key| self.ready_services.contains_key(key))
+            .cloned();
 
+        match peer.and_then(|key| self.take_ready_service(&key)) {
+            Some((key, mut svc)) => {
                 tracing::debug!(?hash, ?key, "routing based on inventory");
                 let fut = svc.call(req);
                 self.push_unready(key, svc);
                 fut.map_err(Into::into).boxed()
             }
             None => {
-                tracing::debug!(
-                    ?hash,
-                    "could not find ready peer for inventory hash, falling back to p2c"
-                );
+                tracing::debug!(?hash, "no ready peer for inventory, falling back to p2c");
                 self.route_p2c(req)
             }
         }
@@ -324,7 +324,7 @@ where
         // This is not needless: otherwise, we'd hold a &mut reference to self.ready_services,
         // blocking us from passing &mut self to push_unready.
         let ready_services = std::mem::take(&mut self.ready_services);
-        self.next_idx = None; // We changed ready_services, so next_idx is invalid
+        self.preselected_p2c_index = None; // All services are now unready.
 
         let futs = FuturesUnordered::new();
         for (key, mut svc) in ready_services {
@@ -342,6 +342,15 @@ where
         }
         .boxed()
     }
+
+    fn update_metrics(&self) {
+        let num_ready = self.ready_services.len();
+        let num_unready = self.unready_services.len();
+        let num_peers = num_ready + num_unready;
+        metrics::gauge!("pool.num_ready", num_ready.try_into().unwrap());
+        metrics::gauge!("pool.num_unready", num_unready.try_into().unwrap());
+        metrics::gauge!("pool.num_peers", num_peers.try_into().unwrap());
+    }
 }
 
 impl<D> Service<Request> for PeerSet<D>
@@ -359,27 +368,19 @@ where
         Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send + 'static>>;
 
     fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
-        self.check_for_background_errors(cx)?;
+        self.poll_background_errors(cx)?;
         // Process peer discovery updates.
         let _ = self.poll_discover(cx)?;
         self.inventory_registry.poll_inventory(cx)?;
-
-        // Poll unready services to drive them to readiness.
         self.poll_unready(cx);
-        let num_ready = self.ready_services.len();
-        let num_unready = self.unready_services.len();
-        metrics::gauge!("pool.num_ready", num_ready.try_into().unwrap(),);
-        metrics::gauge!("pool.num_unready", num_unready.try_into().unwrap(),);
-        metrics::gauge!(
-            "pool.num_peers",
-            (num_ready + num_unready).try_into().unwrap(),
-        );
+
+        self.update_metrics();
 
         loop {
             // Re-check that the pre-selected service is ready, in case
             // something has happened since (e.g., it failed, peer closed
             // connection, ...)
-            if let Some(index) = self.next_idx {
+            if let Some(index) = self.preselected_p2c_index {
                 let (key, service) = self
                     .ready_services
                     .get_index_mut(index)
@@ -406,9 +407,9 @@ where
             }
 
             trace!("preselected service was not ready, reselecting");
-            self.next_idx = self.select_next_ready_index();
+            self.preselected_p2c_index = self.preselect_p2c_index();
 
-            if self.next_idx.is_none() {
+            if self.preselected_p2c_index.is_none() {
                 trace!("no ready services, sending demand signal");
                 let _ = self.demand_signal.try_send(());
                 return Poll::Pending;

From 8d6ac8eecede083e4fbadbfa08ab57dc70c99776 Mon Sep 17 00:00:00 2001
From: teor <teor@riseup.net>
Date: Tue, 24 Nov 2020 19:19:50 +1000
Subject: [PATCH 102/148] Placate clippy

---
 zebra-network/src/peer_set/set.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/zebra-network/src/peer_set/set.rs b/zebra-network/src/peer_set/set.rs
index ad95dc8443e..d59032bc10d 100644
--- a/zebra-network/src/peer_set/set.rs
+++ b/zebra-network/src/peer_set/set.rs
@@ -229,7 +229,7 @@ where
     }
 
     fn remove(&mut self, key: &D::Key) {
-        if let Some(_) = self.take_ready_service(key) {
+        if self.take_ready_service(key).is_some() {
         } else if let Some(handle) = self.cancel_handles.remove(key) {
             let _ = handle.send(());
         }

From a4af90c2b065d5f16cd32a2bd9b436fb6f2fb4ba Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 24 Nov 2020 10:01:16 +0000
Subject: [PATCH 103/148] build(deps): bump color-eyre from 0.5.7 to 0.5.8

Bumps [color-eyre](https://github.com/yaahc/color-eyre) from 0.5.7 to 0.5.8.
- [Release notes](https://github.com/yaahc/color-eyre/releases)
- [Changelog](https://github.com/yaahc/color-eyre/blob/master/CHANGELOG.md)
- [Commits](https://github.com/yaahc/color-eyre/compare/v0.5.7...v0.5.8)

Signed-off-by: dependabot[bot] <support@github.com>
---
 Cargo.lock             | 4 ++--
 zebra-test/Cargo.toml  | 2 +-
 zebra-utils/Cargo.toml | 2 +-
 zebrad/Cargo.toml      | 2 +-
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 7c240829de8..5ea5cdd2bf1 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -488,9 +488,9 @@ dependencies = [
 
 [[package]]
 name = "color-eyre"
-version = "0.5.7"
+version = "0.5.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "86bc0bb03923141924d5b713a4acd7607c790f3fbc769abe63fe3f38bb268112"
+checksum = "8fb57305b07ffcc1a4d08808f1f2200647c8e3d91a4c83d2810ae20c997274e0"
 dependencies = [
  "backtrace",
  "color-spantrace",
diff --git a/zebra-test/Cargo.toml b/zebra-test/Cargo.toml
index adc40b0540f..055dc933ce3 100644
--- a/zebra-test/Cargo.toml
+++ b/zebra-test/Cargo.toml
@@ -12,7 +12,7 @@ hex = "0.4.2"
 lazy_static = "1.4.0"
 tower = { version = "0.4", features = ["util"] }
 futures = "0.3.7"
-color-eyre = "0.5.7"
+color-eyre = "0.5.8"
 tracing = "0.1.21"
 tracing-subscriber = "0.2.15"
 tracing-error = "0.1.2"
diff --git a/zebra-utils/Cargo.toml b/zebra-utils/Cargo.toml
index 0b04dafb8d4..ea261316e7f 100644
--- a/zebra-utils/Cargo.toml
+++ b/zebra-utils/Cargo.toml
@@ -9,7 +9,7 @@ publish = false
 
 [dependencies]
 structopt = "0.3.20"
-color-eyre = "0.5.7"
+color-eyre = "0.5.8"
 hex = "0.4"
 serde_json = "1.0"
 tracing-error = { version = "0.1.2", features = ["traced-error"] }
diff --git a/zebrad/Cargo.toml b/zebrad/Cargo.toml
index 96d2ea7b972..e86c437cf14 100644
--- a/zebrad/Cargo.toml
+++ b/zebrad/Cargo.toml
@@ -25,7 +25,7 @@ tokio = { version = "0.3", features = ["time", "rt-multi-thread", "stream", "mac
 tower = { version = "0.4", features = ["hedge", "limit"] }
 pin-project = "0.4.23"
 
-color-eyre = { version = "0.5.7", features = ["issue-url"] }
+color-eyre = { version = "0.5.8", features = ["issue-url"] }
 thiserror = "1"
 
 tracing = "0.1"

From 133ebd078ae707132f4d9fb3f3e2480302eeef8a Mon Sep 17 00:00:00 2001
From: teor <teor@riseup.net>
Date: Tue, 24 Nov 2020 20:51:30 +1000
Subject: [PATCH 104/148] Remove a post-Canopy panic in funding stream block
 subsidy validation

Funding stream block subsidy validation will be implemented as part of
block subsidy validation in #801.
---
 zebra-consensus/src/block/check.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/zebra-consensus/src/block/check.rs b/zebra-consensus/src/block/check.rs
index b522d0f3659..c06d999227e 100644
--- a/zebra-consensus/src/block/check.rs
+++ b/zebra-consensus/src/block/check.rs
@@ -132,7 +132,9 @@ pub fn subsidy_is_valid(block: &Block, network: Network) -> Result<(), BlockErro
         // Funding streams are paid from Canopy activation to the second halving
         // Note: Canopy activation is at the first halving on mainnet, but not on testnet
         // ZIP-1014 only applies to mainnet, ZIP-214 contains the specific rules for testnet
-        unimplemented!("funding stream block subsidy validation is not implemented")
+        tracing::trace!("funding stream block subsidy validation is not implemented");
+        // Return ok for now
+        Ok(())
     } else {
         // Future halving, with no founders reward or funding streams
         Ok(())

From e832f70c2c472a9730aa25df425c428117ae86eb Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 24 Nov 2020 15:00:25 +0000
Subject: [PATCH 105/148] build(deps): bump tracing from 0.1.21 to 0.1.22

Bumps [tracing](https://github.com/tokio-rs/tracing) from 0.1.21 to 0.1.22.
- [Release notes](https://github.com/tokio-rs/tracing/releases)
- [Commits](https://github.com/tokio-rs/tracing/compare/tracing-0.1.21...tracing-0.1.22)

Signed-off-by: dependabot[bot] <support@github.com>
---
 Cargo.lock                 | 8 ++++----
 tower-batch/Cargo.toml     | 4 ++--
 zebra-chain/Cargo.toml     | 2 +-
 zebra-consensus/Cargo.toml | 2 +-
 zebra-test/Cargo.toml      | 2 +-
 5 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 5ea5cdd2bf1..bf9b8cc33f4 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2811,13 +2811,13 @@ dependencies = [
 
 [[package]]
 name = "tracing"
-version = "0.1.21"
+version = "0.1.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b0987850db3733619253fe60e17cb59b82d37c7e6c0236bb81e4d6b87c879f27"
+checksum = "9f47026cdc4080c07e49b37087de021820269d996f581aac150ef9e5583eefe3"
 dependencies = [
- "cfg-if 0.1.10",
+ "cfg-if 1.0.0",
  "log",
- "pin-project-lite 0.1.11",
+ "pin-project-lite 0.2.0",
  "tracing-attributes",
  "tracing-core",
 ]
diff --git a/tower-batch/Cargo.toml b/tower-batch/Cargo.toml
index b4caa44fa1e..c53fc5d5086 100644
--- a/tower-batch/Cargo.toml
+++ b/tower-batch/Cargo.toml
@@ -10,7 +10,7 @@ tokio = { version = "0.3", features = ["time", "sync", "stream", "tracing", "mac
 tower = { version = "0.4", features = ["util", "buffer"] }
 futures-core = "0.3.6"
 pin-project = "0.4.27"
-tracing = "0.1.21"
+tracing = "0.1.22"
 tracing-futures = "0.2.4"
 futures = "0.3.7"
 
@@ -18,7 +18,7 @@ futures = "0.3.7"
 ed25519-zebra = "2.1.0"
 rand = "0.7"
 tokio = { version = "0.3", features = ["full"]}
-tracing = "0.1.21"
+tracing = "0.1.22"
 zebra-test = { path = "../zebra-test/" }
 tower-fallback = { path = "../tower-fallback/" }
 color-eyre = "0.5"
diff --git a/zebra-chain/Cargo.toml b/zebra-chain/Cargo.toml
index 54953858336..bc2fa0899d0 100644
--- a/zebra-chain/Cargo.toml
+++ b/zebra-chain/Cargo.toml
@@ -48,7 +48,7 @@ bitflags = "1.2.1"
 bincode = "1"
 color-eyre = "0.5"
 spandoc = "0.2"
-tracing = "0.1.21"
+tracing = "0.1.22"
 proptest = "0.10"
 proptest-derive = "0.2"
 
diff --git a/zebra-consensus/Cargo.toml b/zebra-consensus/Cargo.toml
index db7facb801d..57677d90332 100644
--- a/zebra-consensus/Cargo.toml
+++ b/zebra-consensus/Cargo.toml
@@ -22,7 +22,7 @@ thiserror = "1.0.22"
 tokio = { version = "0.3", features = ["time", "sync", "stream", "tracing"] }
 tower = { version = "0.4", features = ["timeout", "util", "buffer"] }
 tower-util = "0.3"
-tracing = "0.1.21"
+tracing = "0.1.22"
 tracing-futures = "0.2.4"
 
 tower-fallback = { path = "../tower-fallback/" }
diff --git a/zebra-test/Cargo.toml b/zebra-test/Cargo.toml
index 055dc933ce3..79b06879737 100644
--- a/zebra-test/Cargo.toml
+++ b/zebra-test/Cargo.toml
@@ -13,7 +13,7 @@ lazy_static = "1.4.0"
 tower = { version = "0.4", features = ["util"] }
 futures = "0.3.7"
 color-eyre = "0.5.8"
-tracing = "0.1.21"
+tracing = "0.1.22"
 tracing-subscriber = "0.2.15"
 tracing-error = "0.1.2"
 spandoc = "0.2.0"

From 15d26e3c47b3d06b11e08fa7f78d230bf7115b3f Mon Sep 17 00:00:00 2001
From: teor <teor@riseup.net>
Date: Tue, 24 Nov 2020 21:10:38 +1000
Subject: [PATCH 106/148] Downgrade a per-block log to debug level

---
 zebra-state/src/service/non_finalized_state/chain.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/zebra-state/src/service/non_finalized_state/chain.rs b/zebra-state/src/service/non_finalized_state/chain.rs
index 3dc3ec0c11e..5ea43426ab3 100644
--- a/zebra-state/src/service/non_finalized_state/chain.rs
+++ b/zebra-state/src/service/non_finalized_state/chain.rs
@@ -33,7 +33,7 @@ impl Chain {
     pub fn push(&mut self, block: PreparedBlock) {
         // update cumulative data members
         self.update_chain_state_with(&block);
-        tracing::info!(block = %block.block, "adding block to chain");
+        tracing::debug!(block = %block.block, "adding block to chain");
         self.blocks.insert(block.height, block);
     }
 

From d093b4e528ee60f9f50aa285e406920d0a7ff1b3 Mon Sep 17 00:00:00 2001
From: Jane Lusby <jane@zfnd.org>
Date: Wed, 28 Oct 2020 12:34:43 -0700
Subject: [PATCH 107/148] Add network integration test for quick post sapling
 sync testing

---
 zebrad/tests/acceptance.rs | 64 +++++++++++++++++++++++++++++++++++---
 1 file changed, 60 insertions(+), 4 deletions(-)

diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs
index bd382115ce0..2c6d3051e09 100644
--- a/zebrad/tests/acceptance.rs
+++ b/zebrad/tests/acceptance.rs
@@ -18,14 +18,17 @@
 #![allow(clippy::field_reassign_with_default)]
 
 use color_eyre::eyre::Result;
-use eyre::WrapErr;
+use eyre::{eyre, WrapErr};
 use tempdir::TempDir;
 
-use std::{borrow::Borrow, env, fs, io::Write, time::Duration};
+use std::{borrow::Borrow, env, fs, io::Write, path::Path, time::Duration};
 
 use zebra_chain::{
     block::Height,
-    parameters::Network::{self, *},
+    parameters::{
+        Network::{self, *},
+        NetworkUpgrade,
+    },
 };
 use zebra_test::{command::TestDirExt, prelude::*};
 use zebrad::config::ZebradConfig;
@@ -611,9 +614,10 @@ fn sync_until(
     network: Network,
     stop_regex: &str,
     timeout: Duration,
-    reuse_tempdir: Option<TempDir>,
+    reuse_tempdir: impl Into<Option<TempDir>>,
 ) -> Result<TempDir> {
     zebra_test::init();
+    let reuse_tempdir = reuse_tempdir.into();
 
     if env::var_os("ZEBRA_SKIP_NETWORK_TESTS").is_some() {
         // This message is captured by the test runner, use
@@ -644,6 +648,58 @@ fn sync_until(
     Ok(child.dir)
 }
 
+#[test]
+#[ignore]
+fn sync_to_sapling() -> Result<()> {
+    let mut testdir = testdir()?;
+    let backup_dir = dirs::cache_dir()
+        .ok_or_else(|| eyre!("no cache dir found"))?
+        .join("sapling_backup");
+
+    if !backup_dir.exists() {
+        testdir = create_sapling_backup(testdir, &backup_dir)?;
+    }
+
+    for entry in std::fs::read_dir(backup_dir)? {
+        let entry = entry?;
+        std::process::Command::new("cp")
+            .arg("-r")
+            .arg(entry.path())
+            .arg(testdir.path())
+            .status2()?
+            .assert_success()?;
+    }
+
+    let height = LARGE_CHECKPOINT_TEST_HEIGHT;
+    let network = Mainnet;
+    let timeout = Duration::from_secs(60);
+
+    println!("Running real sync");
+    sync_until(height, network, STOP_AT_HEIGHT_REGEX, timeout, testdir)?;
+
+    Ok(())
+}
+
+fn create_sapling_backup(testdir: TempDir, dest: &Path) -> Result<TempDir> {
+    println!("Creating backup database");
+    // 1 hour
+    let timeout = Duration::from_secs(60 * 60);
+    let network = Mainnet;
+    let height = NetworkUpgrade::Sapling.activation_height(network).unwrap();
+
+    let tempdir = sync_until(height, network, STOP_AT_HEIGHT_REGEX, timeout, testdir)?;
+    let path = tempdir.path();
+
+    std::process::Command::new("cp")
+        .arg("-r")
+        .arg(path)
+        .arg(dest)
+        .status2()?
+        .assert_success()?;
+
+    Ok(tempdir)
+}
+
 #[tokio::test]
 async fn metrics_endpoint() -> Result<()> {
     use hyper::{Client, Uri};

From 4bfe747f347d2eb2eed21ed1e62fa4e9ebb3a26e Mon Sep 17 00:00:00 2001
From: Jane Lusby <jane@zfnd.org>
Date: Thu, 29 Oct 2020 18:00:02 -0700
Subject: [PATCH 108/148] update acceptance tests

---
 zebra-test/src/command.rs  |  19 +++---
 zebrad/tests/acceptance.rs | 116 ++++++++++++++++++++++---------------
 2 files changed, 78 insertions(+), 57 deletions(-)

diff --git a/zebra-test/src/command.rs b/zebra-test/src/command.rs
index 6b613f7e063..8280efe4e36 100644
--- a/zebra-test/src/command.rs
+++ b/zebra-test/src/command.rs
@@ -2,16 +2,16 @@ use color_eyre::{
     eyre::{eyre, Context, Report, Result},
     Help, SectionExt,
 };
-use tempdir::TempDir;
 use tracing::instrument;
 
-use std::convert::Infallible as NoDir;
-use std::fmt::Write as _;
+use std::{convert::Infallible as NoDir, io::BufRead};
+use std::{
+    fmt::Write as _,
+    io::{BufReader, Lines},
+};
 #[cfg(unix)]
-use std::os::unix::process::ExitStatusExt;
+use std::{io::Read, os::unix::process::ExitStatusExt};
 use std::{
-    borrow::Borrow,
-    io::{BufRead, BufReader, Lines, Read},
     path::Path,
     process::{Child, ChildStdout, Command, ExitStatus, Output, Stdio},
     time::{Duration, Instant},
@@ -98,7 +98,7 @@ impl CommandExt for Command {
 /// `zebra_test::command` without running `zebrad`.
 pub trait TestDirExt
 where
-    Self: Borrow<TempDir> + Sized,
+    Self: AsRef<Path> + Sized,
 {
     /// Spawn `cmd` with `args` as a child process in this test directory,
     /// potentially taking ownership of the tempdir for the duration of the
@@ -108,11 +108,10 @@ where
 
 impl<T> TestDirExt for T
 where
-    Self: Borrow<TempDir> + Sized,
+    Self: AsRef<Path> + Sized,
 {
     fn spawn_child_with_command(self, cmd: &str, args: &[&str]) -> Result<TestChild<Self>> {
-        let tempdir = self.borrow();
-        let mut cmd = test_cmd(cmd, tempdir.path())?;
+        let mut cmd = test_cmd(cmd, self.as_ref())?;
 
         Ok(cmd
             .args(args)
diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs
index 2c6d3051e09..849b3b2de84 100644
--- a/zebrad/tests/acceptance.rs
+++ b/zebrad/tests/acceptance.rs
@@ -18,10 +18,10 @@
 #![allow(clippy::field_reassign_with_default)]
 
 use color_eyre::eyre::Result;
-use eyre::{eyre, WrapErr};
+use eyre::WrapErr;
 use tempdir::TempDir;
 
-use std::{borrow::Borrow, env, fs, io::Write, path::Path, time::Duration};
+use std::{env, fs, io::Write, path::Path, path::PathBuf, time::Duration};
 
 use zebra_chain::{
     block::Height,
@@ -55,7 +55,7 @@ fn testdir() -> Result<TempDir> {
 /// directory for `zebrad`.
 trait ZebradTestDirExt
 where
-    Self: Borrow<TempDir> + Sized,
+    Self: AsRef<Path> + Sized,
 {
     /// Spawn `zebrad` with `args` as a child process in this test directory,
     /// potentially taking ownership of the tempdir for the duration of the
@@ -73,11 +73,11 @@ where
 
 impl<T> ZebradTestDirExt for T
 where
-    Self: TestDirExt + Borrow<TempDir> + Sized,
+    Self: TestDirExt + AsRef<Path> + Sized,
 {
     fn spawn_child(self, args: &[&str]) -> Result<TestChild<Self>> {
-        let tempdir = self.borrow();
-        let default_config_path = tempdir.path().join("zebrad.toml");
+        let path = self.as_ref();
+        let default_config_path = path.join("zebrad.toml");
 
         if default_config_path.exists() {
             let mut extra_args: Vec<_> = Vec::new();
@@ -96,7 +96,7 @@ where
     }
 
     fn with_config(self, mut config: ZebradConfig) -> Result<Self> {
-        let dir = self.borrow().path();
+        let dir = self.as_ref();
 
         if !config.state.ephemeral {
             let cache_dir = dir.join("state");
@@ -111,7 +111,7 @@ where
     }
 
     fn replace_config(self, mut config: ZebradConfig) -> Result<Self> {
-        let dir = self.borrow().path();
+        let dir = self.as_ref();
 
         if !config.state.ephemeral {
             let cache_dir = dir.join("state");
@@ -648,56 +648,78 @@ fn sync_until(
     Ok(child.dir)
 }
 
-#[test]
-#[ignore]
-fn sync_to_sapling() -> Result<()> {
-    let mut testdir = testdir()?;
-    let backup_dir = dirs::cache_dir()
-        .ok_or_else(|| eyre!("no cache dir found"))?
-        .join("sapling_backup");
-
-    if !backup_dir.exists() {
-        testdir = create_sapling_backup(testdir, &backup_dir)?;
-    }
+fn cached_sapling_test_config() -> Result<ZebradConfig> {
+    let mut config = persistent_test_config()?;
+    config.consensus.checkpoint_sync = true;
+    config.state.cache_dir = "/zebrad-cache".into();
+    config.state.memory_cache_bytes = 52428800;
+    config.tracing.endpoint_addr = Some("0.0.0.0:3000".parse().unwrap());
+    Ok(config)
+}
 
-    for entry in std::fs::read_dir(backup_dir)? {
-        let entry = entry?;
-        std::process::Command::new("cp")
-            .arg("-r")
-            .arg(entry.path())
-            .arg(testdir.path())
-            .status2()?
-            .assert_success()?;
-    }
+fn create_cached_database_height(network: Network, height: Height) -> Result<()> {
+    println!("Creating cached database");
+    // 8 hours
+    let timeout = Duration::from_secs(60 * 60 * 8);
 
-    let height = LARGE_CHECKPOINT_TEST_HEIGHT;
-    let network = Mainnet;
-    let timeout = Duration::from_secs(60);
+    // Use a persistent state, so we can handle large syncs
+    let mut config = cached_sapling_test_config()?;
+    // TODO: add convenience methods?
+    config.network.network = network;
+    config.state.debug_stop_at_height = Some(height.0);
+    let dir = PathBuf::from("/");
+    fs::File::create(dir.join("zebrad.toml"))?.write_all(toml::to_string(&config)?.as_bytes())?;
 
-    println!("Running real sync");
-    sync_until(height, network, STOP_AT_HEIGHT_REGEX, timeout, testdir)?;
+    let mut child = dir.spawn_child(&["start"])?.with_timeout(timeout);
+
+    // TODO: is there a way to check for testnet or mainnet here?
+    // For example: "network=Mainnet" or "network=Testnet"
+    child.expect_stdout("network: Mainnet,")?;
+    child.expect_stdout(STOP_AT_HEIGHT_REGEX)?;
+    child.kill()?;
 
     Ok(())
 }
 
-fn create_sapling_backup(testdir: TempDir, dest: &Path) -> Result<TempDir> {
-    println!("Creating backup database");
-    // 1 hour
-    let timeout = Duration::from_secs(60 * 60);
-    let network = Mainnet;
+fn create_cached_database(network: Network) -> Result<()> {
     let height = NetworkUpgrade::Sapling.activation_height(network).unwrap();
+    create_cached_database_height(network, height)
+}
 
-    let tempdir = sync_until(height, network, STOP_AT_HEIGHT_REGEX, timeout, testdir)?;
-    let path = tempdir.path();
+fn sync_past_sapling(network: Network) -> Result<()> {
+    let height = NetworkUpgrade::Sapling.activation_height(network).unwrap() + 1000;
+    create_cached_database_height(network, height.unwrap())
+}
 
-    std::process::Command::new("cp")
-        .arg("-r")
-        .arg(path)
-        .arg(dest)
-        .status2()?
-        .assert_success()?;
+#[test]
+#[ignore]
+fn create_mainnet_cache() {
+    zebra_test::init();
+    let network = Mainnet;
+    create_cached_database(network).unwrap();
+}
+#[test]
+#[ignore]
+fn create_testnet_cache() {
+    zebra_test::init();
+    let network = Testnet;
+    create_cached_database(network).unwrap();
+}
 
-    Ok(tempdir)
+#[test]
+#[ignore]
+fn sync_past_sapling_mainnet() {
+    zebra_test::init();
+    let network = Mainnet;
+    sync_past_sapling(network).unwrap();
+}
+
+#[test]
+#[ignore]
+fn sync_past_sapling_testnet() {
+    zebra_test::init();
+    let network = Testnet;
+    sync_past_sapling(network).unwrap();
 }
 
 #[tokio::test]

From a21ddfeefc5fc953c8069cf6b53458c58300d4a9 Mon Sep 17 00:00:00 2001
From: Jane Lusby <jane@zfnd.org>
Date: Thu, 29 Oct 2020 18:03:00 -0700
Subject: [PATCH 109/148] fix use cfg

---
 zebra-test/src/command.rs | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/zebra-test/src/command.rs b/zebra-test/src/command.rs
index 8280efe4e36..5eac79146b0 100644
--- a/zebra-test/src/command.rs
+++ b/zebra-test/src/command.rs
@@ -4,14 +4,12 @@ use color_eyre::{
 };
 use tracing::instrument;
 
+#[cfg(unix)]
+use std::os::unix::process::ExitStatusExt;
 use std::{convert::Infallible as NoDir, io::BufRead};
 use std::{
     fmt::Write as _,
-    io::{BufReader, Lines},
-};
-#[cfg(unix)]
-use std::{io::Read, os::unix::process::ExitStatusExt};
-use std::{
+    io::{BufReader, Lines, Read},
     path::Path,
     process::{Child, ChildStdout, Command, ExitStatus, Output, Stdio},
     time::{Duration, Instant},

From 9fd38f29b4da7f29b9040ff480ee7f32c25f3b1e Mon Sep 17 00:00:00 2001
From: Jane Lusby <jane@zfnd.org>
Date: Thu, 29 Oct 2020 18:03:58 -0700
Subject: [PATCH 110/148] further make use statement consistent

---
 zebra-test/src/command.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/zebra-test/src/command.rs b/zebra-test/src/command.rs
index 5eac79146b0..763b4d401ef 100644
--- a/zebra-test/src/command.rs
+++ b/zebra-test/src/command.rs
@@ -6,9 +6,10 @@ use tracing::instrument;
 
 #[cfg(unix)]
 use std::os::unix::process::ExitStatusExt;
-use std::{convert::Infallible as NoDir, io::BufRead};
 use std::{
+    convert::Infallible as NoDir,
     fmt::Write as _,
+    io::BufRead,
     io::{BufReader, Lines, Read},
     path::Path,
     process::{Child, ChildStdout, Command, ExitStatus, Output, Stdio},

From 0f518913592996210406fb75eefd04e4696de704 Mon Sep 17 00:00:00 2001
From: Jane Lusby <jane@zfnd.org>
Date: Thu, 29 Oct 2020 18:05:39 -0700
Subject: [PATCH 111/148] revert unnecessary change in sync_until

---
 zebrad/tests/acceptance.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs
index 849b3b2de84..d20164bc7c3 100644
--- a/zebrad/tests/acceptance.rs
+++ b/zebrad/tests/acceptance.rs
@@ -614,10 +614,9 @@ fn sync_until(
     network: Network,
     stop_regex: &str,
     timeout: Duration,
-    reuse_tempdir: impl Into<Option<TempDir>>,
+    reuse_tempdir: Option<TempDir>,
 ) -> Result<TempDir> {
     zebra_test::init();
-    let reuse_tempdir = reuse_tempdir.into();
 
     if env::var_os("ZEBRA_SKIP_NETWORK_TESTS").is_some() {
         // This message is captured by the test runner, use
@@ -668,6 +667,7 @@ fn create_cached_database_height(network: Network, height: Height) -> Result<()>
     config.network.network = network;
     config.state.debug_stop_at_height = Some(height.0);
     let dir = PathBuf::from("/");
+
     fs::File::create(dir.join("zebrad.toml"))?.write_all(toml::to_string(&config)?.as_bytes())?;
 
     let mut child = dir.spawn_child(&["start"])?.with_timeout(timeout);

From 17fdbe941b9c36ea3df2c4d779bb5e3190710912 Mon Sep 17 00:00:00 2001
From: Jane Lusby <jane@zfnd.org>
Date: Fri, 30 Oct 2020 12:36:20 -0700
Subject: [PATCH 112/148] fix stdout issue with test framework for cached data
 tests

---
 zebra-chain/src/parameters/network.rs | 11 +++++++++++
 zebra-test/src/command.rs             | 17 ++++++++++++++++-
 zebrad/tests/acceptance.rs            | 13 +++++++------
 3 files changed, 34 insertions(+), 7 deletions(-)

diff --git a/zebra-chain/src/parameters/network.rs b/zebra-chain/src/parameters/network.rs
index bbf78e83bc3..6c6091c5770 100644
--- a/zebra-chain/src/parameters/network.rs
+++ b/zebra-chain/src/parameters/network.rs
@@ -1,3 +1,5 @@
+use std::fmt;
+
 #[cfg(any(test, feature = "proptest-impl"))]
 use proptest_derive::Arbitrary;
 
@@ -11,6 +13,15 @@ pub enum Network {
     Testnet,
 }
 
+impl fmt::Display for Network {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Network::Mainnet => f.write_str("Mainnet"),
+            Network::Testnet => f.write_str("Testnet"),
+        }
+    }
+}
+
 impl Network {
     /// Get the default port associated to this network.
     pub fn default_port(&self) -> u16 {
diff --git a/zebra-test/src/command.rs b/zebra-test/src/command.rs
index 763b4d401ef..f70b2fcb83f 100644
--- a/zebra-test/src/command.rs
+++ b/zebra-test/src/command.rs
@@ -86,6 +86,7 @@ impl CommandExt for Command {
             dir,
             deadline: None,
             stdout: None,
+            bypass_test_stdout: false,
         })
     }
 }
@@ -152,6 +153,7 @@ pub struct TestChild<T> {
     pub child: Child,
     pub stdout: Option<Lines<BufReader<ChildStdout>>>,
     pub deadline: Option<Instant>,
+    bypass_test_stdout: bool,
 }
 
 impl<T> TestChild<T> {
@@ -190,6 +192,13 @@ impl<T> TestChild<T> {
         self
     }
 
+    /// Configures testrunner to forward stdout to the true stdout rather than
+    /// fakestdout used by cargo tests.
+    pub fn bypass_test_stdout(mut self, cond: bool) -> Self {
+        self.bypass_test_stdout = cond;
+        self
+    }
+
     /// Checks each line of the child's stdout against `regex`, and returns matching lines.
     ///
     /// Kills the child after the configured timeout has elapsed.
@@ -221,7 +230,13 @@ impl<T> TestChild<T> {
             // since we're about to discard this line write it to stdout so our
             // test runner can capture it and display if the test fails, may
             // cause weird reordering for stdout / stderr
-            println!("{}", line);
+            if !self.bypass_test_stdout {
+                println!("{}", line);
+            } else {
+                use std::io::Write;
+                #[allow(clippy::explicit_write)]
+                writeln!(std::io::stdout(), "{}", line).unwrap();
+            }
 
             if re.is_match(&line) {
                 self.stdout = Some(lines);
diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs
index d20164bc7c3..96808bf0777 100644
--- a/zebrad/tests/acceptance.rs
+++ b/zebrad/tests/acceptance.rs
@@ -652,7 +652,6 @@ fn cached_sapling_test_config() -> Result<ZebradConfig> {
     config.consensus.checkpoint_sync = true;
     config.state.cache_dir = "/zebrad-cache".into();
     config.state.memory_cache_bytes = 52428800;
-    config.tracing.endpoint_addr = Some("0.0.0.0:3000".parse().unwrap());
     Ok(config)
 }
 
@@ -666,15 +665,17 @@ fn create_cached_database_height(network: Network, height: Height) -> Result<()>
     // TODO: add convenience methods?
     config.network.network = network;
     config.state.debug_stop_at_height = Some(height.0);
-    let dir = PathBuf::from("/");
+    let dir = PathBuf::from("/zebrad-cache");
 
     fs::File::create(dir.join("zebrad.toml"))?.write_all(toml::to_string(&config)?.as_bytes())?;
 
-    let mut child = dir.spawn_child(&["start"])?.with_timeout(timeout);
+    let mut child = dir
+        .spawn_child(&["start"])?
+        .with_timeout(timeout)
+        .bypass_test_stdout(true);
 
-    // TODO: is there a way to check for testnet or mainnet here?
-    // For example: "network=Mainnet" or "network=Testnet"
-    child.expect_stdout("network: Mainnet,")?;
+    let network = format!("network: {},", network);
+    child.expect_stdout(&network)?;
     child.expect_stdout(STOP_AT_HEIGHT_REGEX)?;
     child.kill()?;
 

From 583ec8738ba369881686eca1747f8afe7e613f83 Mon Sep 17 00:00:00 2001
From: Jane Lusby <jane@zfnd.org>
Date: Fri, 30 Oct 2020 12:41:37 -0700
Subject: [PATCH 113/148] change ci to not blanket enable all ignored tests

---
 .github/workflows/ci.yml | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 61e2473d913..4abf11f1d07 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -45,7 +45,14 @@ jobs:
         uses: actions-rs/cargo@v1
         with:
           command: test
-          args: --verbose --manifest-path zebrad/Cargo.toml -- --ignored
+          args: --verbose --manifest-path zebrad/Cargo.toml sync_large_checkpoints_mainnet sync_large_checkpoints_testnet -- --ignored
+      - name: Run zebrad tracing test
+        env:
+          RUST_BACKTRACE: full
+        uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --verbose --manifest-path zebrad/Cargo.toml tracing_endpoint -- --ignored
 
   build-chain-no-features:
     name: Build zebra-chain w/o features on ubuntu-latest

From 602d8c4898f3c00faccdf81b56b2882d9be5b13f Mon Sep 17 00:00:00 2001
From: Jane Lusby <jane@zfnd.org>
Date: Fri, 30 Oct 2020 12:45:26 -0700
Subject: [PATCH 114/148] document tests

---
 zebrad/tests/acceptance.rs | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs
index 96808bf0777..5c7772c5509 100644
--- a/zebrad/tests/acceptance.rs
+++ b/zebrad/tests/acceptance.rs
@@ -688,10 +688,19 @@ fn create_cached_database(network: Network) -> Result<()> {
 }
 
 fn sync_past_sapling(network: Network) -> Result<()> {
-    let height = NetworkUpgrade::Sapling.activation_height(network).unwrap() + 1000;
+    let height = NetworkUpgrade::Sapling.activation_height(network).unwrap() + 1200;
     create_cached_database_height(network, height.unwrap())
 }
 
+// These tests are ignored because they're too long running to run during our
+// traditional CI, and they depend on persistent state that cannot be made
+// available in github actions or google cloud build. Instead we run these tests
+// directly in a vm we spin up on google compute engine, where we can mount
+// drives populated by the first two tests and then use those to more quickly
+// run the second two tests.
+
+// Create a cached copy of the mainnet database up to the sapling activation
+// height.
 #[test]
 #[ignore]
 fn create_mainnet_cache() {
@@ -699,6 +708,8 @@ fn create_mainnet_cache() {
     let network = Mainnet;
     create_cached_database(network).unwrap();
 }
+// Create a cached copy of the testnet database up to the sapling activation
+// height.
 #[test]
 #[ignore]
 fn create_testnet_cache() {
@@ -707,6 +718,7 @@ fn create_testnet_cache() {
     create_cached_database(network).unwrap();
 }
 
+/// Test syncing 1200 blocks (3 checkpoints) past the last checkpoint on mainnet.
 #[test]
 #[ignore]
 fn sync_past_sapling_mainnet() {
@@ -715,6 +727,7 @@ fn sync_past_sapling_mainnet() {
     sync_past_sapling(network).unwrap();
 }
 
+/// Test syncing 1200 blocks (3 checkpoints) past the last checkpoint on testnet.
 #[test]
 #[ignore]
 fn sync_past_sapling_testnet() {

From 99c5acc94f282f9fbeedf57c8533843aa0f82edc Mon Sep 17 00:00:00 2001
From: Jane Lusby <jane@zfnd.org>
Date: Fri, 30 Oct 2020 12:46:59 -0700
Subject: [PATCH 115/148] rename test fn

---
 zebra-test/src/command.rs  | 10 +++++-----
 zebrad/tests/acceptance.rs |  2 +-
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/zebra-test/src/command.rs b/zebra-test/src/command.rs
index f70b2fcb83f..e48bc993470 100644
--- a/zebra-test/src/command.rs
+++ b/zebra-test/src/command.rs
@@ -86,7 +86,7 @@ impl CommandExt for Command {
             dir,
             deadline: None,
             stdout: None,
-            bypass_test_stdout: false,
+            bypass_test_capture: false,
         })
     }
 }
@@ -153,7 +153,7 @@ pub struct TestChild<T> {
     pub child: Child,
     pub stdout: Option<Lines<BufReader<ChildStdout>>>,
     pub deadline: Option<Instant>,
-    bypass_test_stdout: bool,
+    bypass_test_capture: bool,
 }
 
 impl<T> TestChild<T> {
@@ -194,8 +194,8 @@ impl<T> TestChild<T> {
 
     /// Configures testrunner to forward stdout to the true stdout rather than
     /// fakestdout used by cargo tests.
-    pub fn bypass_test_stdout(mut self, cond: bool) -> Self {
-        self.bypass_test_stdout = cond;
+    pub fn bypass_test_capture(mut self, cond: bool) -> Self {
+        self.bypass_test_capture = cond;
         self
     }
 
@@ -230,7 +230,7 @@ impl<T> TestChild<T> {
             // since we're about to discard this line write it to stdout so our
             // test runner can capture it and display if the test fails, may
             // cause weird reordering for stdout / stderr
-            if !self.bypass_test_stdout {
+            if !self.bypass_test_capture {
                 println!("{}", line);
             } else {
                 use std::io::Write;
diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs
index 5c7772c5509..aec8e9f02b8 100644
--- a/zebrad/tests/acceptance.rs
+++ b/zebrad/tests/acceptance.rs
@@ -672,7 +672,7 @@ fn create_cached_database_height(network: Network, height: Height) -> Result<()>
     let mut child = dir
         .spawn_child(&["start"])?
         .with_timeout(timeout)
-        .bypass_test_stdout(true);
+        .bypass_test_capture(true);
 
     let network = format!("network: {},", network);
     child.expect_stdout(&network)?;

From c2a57d7e4950cd46e97c510d111bae715cf5b540 Mon Sep 17 00:00:00 2001
From: Jane Lusby <jane@zfnd.org>
Date: Fri, 30 Oct 2020 12:47:46 -0700
Subject: [PATCH 116/148] slight comment tweek

---
 zebrad/tests/acceptance.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs
index aec8e9f02b8..9bfb13bfa48 100644
--- a/zebrad/tests/acceptance.rs
+++ b/zebrad/tests/acceptance.rs
@@ -696,8 +696,8 @@ fn sync_past_sapling(network: Network) -> Result<()> {
 // traditional CI, and they depend on persistent state that cannot be made
 // available in github actions or google cloud build. Instead we run these tests
 // directly in a vm we spin up on google compute engine, where we can mount
-// drives populated by the first two tests and then use those to more quickly
-// run the second two tests.
+// drives populated by the first two tests, snapshot those drives, and then use
+// those to more quickly run the second two tests.
 
 // Create a cached copy of the mainnet database up to the sapling activation
 // height.

From b96ca38d1c36492bb40e99bd433ebfe033af5e39 Mon Sep 17 00:00:00 2001
From: Jane Lusby <jane@zfnd.org>
Date: Fri, 30 Oct 2020 13:06:48 -0700
Subject: [PATCH 117/148] fix args issue

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 4abf11f1d07..d110a39765d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -45,7 +45,7 @@ jobs:
         uses: actions-rs/cargo@v1
         with:
           command: test
-          args: --verbose --manifest-path zebrad/Cargo.toml sync_large_checkpoints_mainnet sync_large_checkpoints_testnet -- --ignored
+          args: --verbose --manifest-path zebrad/Cargo.toml sync_large_checkpoints_ -- --ignored
       - name: Run zebrad tracing test
         env:
           RUST_BACKTRACE: full

From d813603bac063bb5c11b7a2753806baf5b9106b6 Mon Sep 17 00:00:00 2001
From: Deirdre Connolly <deirdre@zfnd.org>
Date: Sat, 21 Nov 2020 06:25:20 -0500
Subject: [PATCH 118/148] Remove defunct memory_cache_bytes from test config

---
 zebrad/tests/acceptance.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs
index 9bfb13bfa48..789cc8c1163 100644
--- a/zebrad/tests/acceptance.rs
+++ b/zebrad/tests/acceptance.rs
@@ -651,7 +651,6 @@ fn cached_sapling_test_config() -> Result<ZebradConfig> {
     let mut config = persistent_test_config()?;
     config.consensus.checkpoint_sync = true;
     config.state.cache_dir = "/zebrad-cache".into();
-    config.state.memory_cache_bytes = 52428800;
     Ok(config)
 }
 

From b3908b4425294870509c59cfb86a2a905c3ff8e6 Mon Sep 17 00:00:00 2001
From: Deirdre Connolly <deirdre@zfnd.org>
Date: Sat, 21 Nov 2020 17:52:56 -0500
Subject: [PATCH 119/148] Compile tests without running as container layer

---
 docker/Dockerfile.test | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docker/Dockerfile.test b/docker/Dockerfile.test
index 8972eca5994..f6a8e826684 100644
--- a/docker/Dockerfile.test
+++ b/docker/Dockerfile.test
@@ -16,4 +16,6 @@ EXPOSE 8233 18233
 
 COPY . .
 
+RUN cargo test --all --no-run
+
 CMD cargo test --workspace --no-fail-fast -- -Zunstable-options --include-ignored

From 4a67e0e7bb20797cb9c1dd5226e89b7db0cd7664 Mon Sep 17 00:00:00 2001
From: Deirdre Connolly <deirdre@zfnd.org>
Date: Mon, 23 Nov 2020 00:12:12 -0500
Subject: [PATCH 120/148] Enable stateful/long sync tests by features, mount
 rocksdb-based state at Sapling activation for sync_past_sapling_mainnet test

---
 .github/workflows/test.yml | 17 +++++++++++------
 zebrad/Cargo.toml          |  6 ++++++
 zebrad/tests/acceptance.rs | 33 ++++++++++++++++++---------------
 3 files changed, 35 insertions(+), 21 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index e4d0aac57c0..cfcfa150ef4 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -48,11 +48,12 @@ jobs:
       # Creates Compute Engine virtual machine instance w/ disks
       - name: Create instance
         run: |
-          gcloud compute instances create "zebrad-tests-$BRANCH_NAME-$SHORT_SHA" \
+          gcloud compute instances create-with-container "zebrad-tests-$BRANCH_NAME-$SHORT_SHA" \
           --boot-disk-size 100GB \
           --boot-disk-type pd-ssd \
-          --image-family cos-stable \
-          --image-project cos-cloud \
+          --container-image rust:buster \
+          --container-mount-disk mount-path='/zebrad-cache',name=zebrad-cache-mainnet-sapling-activation \
+          --create-disk name=zebrad-cache-mainnet-sapling-activation,image=zebrad-cache-062a5ae-mainnet-height-419200 \
           --machine-type n2-standard-4 \
           --service-account cos-vm@zealous-zebra.iam.gserviceaccount.com \
           --scopes cloud-platform \
@@ -62,9 +63,13 @@ jobs:
       # Build and run test container
       - name: Run all tests
         run: |
-          gcloud compute ssh "zebrad-tests-$BRANCH_NAME-$SHORT_SHA" --zone "$ZONE" \
-          --command "git clone -b $BRANCH_NAME https://github.com/ZcashFoundation/zebra.git; cd zebra/; docker build -f docker/Dockerfile.test -t zebrad-test .; docker run -i zebrad-test cargo test --workspace --no-fail-fast -- -Zunstable-options --include-ignored"
-
+          gcloud compute ssh "zebrad-tests-$BRANCH_NAME-$SHORT_SHA" --zone "$ZONE" --command \
+          "git clone -b $BRANCH_NAME https://github.com/ZcashFoundation/zebra.git;
+          cd zebra/;
+          docker build -f docker/Dockerfile.test -t zebrad-test .;
+          docker run -i zebrad-test cargo test --workspace --no-fail-fast -- -Zunstable-options --include-ignored;
+          docker run -it --mount type=bind,source=/mnt/disks/gce-containers-mounts/gce-persistent-disks/zebrad-cache-mainnet-sapling-activation,target=/zebrad-cache zebrad-test:latest cargo test --verbose --features test_sync_to_sapling_mainnet --manifest-path zebrad/Cargo.toml sync_to_sapling_mainnet;
+          "
       # Clean up
       - name: Delete test instance
         # Always run even if the earlier step fails
diff --git a/zebrad/Cargo.toml b/zebrad/Cargo.toml
index e86c437cf14..0d18f4df9b9 100644
--- a/zebrad/Cargo.toml
+++ b/zebrad/Cargo.toml
@@ -44,3 +44,9 @@ abscissa_core = { version = "0.5", features = ["testing"] }
 once_cell = "1.5"
 tempdir = "0.3.7"
 zebra-test = { path = "../zebra-test" }
+
+[features]
+test_sync_to_sapling_mainnet = []
+test_sync_to_sapling_testnet = []
+test_sync_past_sapling_mainnet = []
+test_sync_past_sapling_testnet = []
diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs
index 789cc8c1163..581f1b8a1d7 100644
--- a/zebrad/tests/acceptance.rs
+++ b/zebrad/tests/acceptance.rs
@@ -13,9 +13,10 @@
 
 #![warn(warnings, missing_docs, trivial_casts, unused_qualifications)]
 #![forbid(unsafe_code)]
+#![allow(clippy::dead_code)]
+#![allow(clippy::field_reassign_with_default)]
 #![allow(clippy::try_err)]
 #![allow(clippy::unknown_clippy_lints)]
-#![allow(clippy::field_reassign_with_default)]
 
 use color_eyre::eyre::Result;
 use eyre::WrapErr;
@@ -698,28 +699,27 @@ fn sync_past_sapling(network: Network) -> Result<()> {
 // drives populated by the first two tests, snapshot those drives, and then use
 // those to more quickly run the second two tests.
 
-// Create a cached copy of the mainnet database up to the sapling activation
-// height.
-#[test]
-#[ignore]
-fn create_mainnet_cache() {
+// Sync up to the sapling activation height on mainnet and stop.
+#[cfg_attr(feature = "test_sync_to_sapling_mainnet", test)]
+fn sync_to_sapling_mainnet() {
     zebra_test::init();
     let network = Mainnet;
     create_cached_database(network).unwrap();
 }
-// Create a cached copy of the testnet database up to the sapling activation
-// height.
-#[test]
-#[ignore]
-fn create_testnet_cache() {
+// Sync to the sapling activation height testnet and stop.
+#[cfg_attr(feature = "test_sync_to_sapling_testnet", test)]
+fn sync_to_sapling_testnet() {
     zebra_test::init();
     let network = Testnet;
     create_cached_database(network).unwrap();
 }
 
 /// Test syncing 1200 blocks (3 checkpoints) past the last checkpoint on mainnet.
-#[test]
-#[ignore]
+///
+/// This assumes that the config'd state is already synced at or near Sapling
+/// activation on mainnet. If the state has already synced past Sapling
+/// activation by 1200 blocks, it will fail.
+#[cfg_attr(feature = "test_sync_past_sapling_mainnet", test)]
 fn sync_past_sapling_mainnet() {
     zebra_test::init();
     let network = Mainnet;
@@ -727,8 +727,11 @@ fn sync_past_sapling_mainnet() {
 }
 
 /// Test syncing 1200 blocks (3 checkpoints) past the last checkpoint on testnet.
-#[test]
-#[ignore]
+///
+/// This assumes that the config'd state is already synced at or near Sapling
+/// activation on testnet. If the state has already synced past Sapling
+/// activation by 1200 blocks, it will fail.
+#[cfg_attr(feature = "test_sync_past_sapling_testnet", test)]
 fn sync_past_sapling_testnet() {
     zebra_test::init();
     let network = Testnet;

From 6dab76bc36e2dbdd02e776a7554da136d64d78d2 Mon Sep 17 00:00:00 2001
From: Deirdre Connolly <deirdre@zfnd.org>
Date: Mon, 23 Nov 2020 00:26:20 -0500
Subject: [PATCH 121/148] Avoid disk naming collisions

---
 .github/workflows/test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index cfcfa150ef4..5f58f0f82c8 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -52,8 +52,8 @@ jobs:
           --boot-disk-size 100GB \
           --boot-disk-type pd-ssd \
           --container-image rust:buster \
-          --container-mount-disk mount-path='/zebrad-cache',name=zebrad-cache-mainnet-sapling-activation \
-          --create-disk name=zebrad-cache-mainnet-sapling-activation,image=zebrad-cache-062a5ae-mainnet-height-419200 \
+          --container-mount-disk mount-path='/zebrad-cache',name="zebrad-cache-$SHORT_SHA-mainnet-419200" \
+          --create-disk name="zebrad-cache-$SHORT_SHA-mainnet-419200",image=zebrad-cache-062a5ae-mainnet-height-419200 \
           --machine-type n2-standard-4 \
           --service-account cos-vm@zealous-zebra.iam.gserviceaccount.com \
           --scopes cloud-platform \

From 466b9ea303c4f771cd277586d0da6d63848df533 Mon Sep 17 00:00:00 2001
From: Deirdre Connolly <deirdre@zfnd.org>
Date: Mon, 23 Nov 2020 01:40:19 -0500
Subject: [PATCH 122/148] Not a tty

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 5f58f0f82c8..d4d46d64de2 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -68,7 +68,7 @@ jobs:
           cd zebra/;
           docker build -f docker/Dockerfile.test -t zebrad-test .;
           docker run -i zebrad-test cargo test --workspace --no-fail-fast -- -Zunstable-options --include-ignored;
-          docker run -it --mount type=bind,source=/mnt/disks/gce-containers-mounts/gce-persistent-disks/zebrad-cache-mainnet-sapling-activation,target=/zebrad-cache zebrad-test:latest cargo test --verbose --features test_sync_to_sapling_mainnet --manifest-path zebrad/Cargo.toml sync_to_sapling_mainnet;
+          docker run -i --mount type=bind,source=/mnt/disks/gce-containers-mounts/gce-persistent-disks/zebrad-cache-mainnet-sapling-activation,target=/zebrad-cache zebrad-test:latest cargo test --verbose --features test_sync_to_sapling_mainnet --manifest-path zebrad/Cargo.toml sync_to_sapling_mainnet;
           "
       # Clean up
       - name: Delete test instance

From cf82560538de8707ed99c32f061e3ea149ee5f8f Mon Sep 17 00:00:00 2001
From: Deirdre Connolly <deirdre@zfnd.org>
Date: Mon, 23 Nov 2020 02:18:17 -0500
Subject: [PATCH 123/148] Reference correct disk name when mounting in
 container

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index d4d46d64de2..278448c461c 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -68,7 +68,7 @@ jobs:
           cd zebra/;
           docker build -f docker/Dockerfile.test -t zebrad-test .;
           docker run -i zebrad-test cargo test --workspace --no-fail-fast -- -Zunstable-options --include-ignored;
-          docker run -i --mount type=bind,source=/mnt/disks/gce-containers-mounts/gce-persistent-disks/zebrad-cache-mainnet-sapling-activation,target=/zebrad-cache zebrad-test:latest cargo test --verbose --features test_sync_to_sapling_mainnet --manifest-path zebrad/Cargo.toml sync_to_sapling_mainnet;
+          docker run -i --mount type=bind,source=/mnt/disks/gce-containers-mounts/gce-persistent-disks/zebrad-cache-$SHORT_SHA-mainnet-419200,target=/zebrad-cache zebrad-test:latest cargo test --verbose --features test_sync_to_sapling_mainnet --manifest-path zebrad/Cargo.toml sync_to_sapling_mainnet;
           "
       # Clean up
       - name: Delete test instance

From 3c9b7927a56e158d142e39620ac66b76cc101b4b Mon Sep 17 00:00:00 2001
From: Deirdre Connolly <deirdre@zfnd.org>
Date: Mon, 23 Nov 2020 03:13:15 -0500
Subject: [PATCH 124/148] Run sync_past_sapling_mainnet, not
 sync_to_sapling_mainnet

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 278448c461c..209032c42a2 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -68,7 +68,7 @@ jobs:
           cd zebra/;
           docker build -f docker/Dockerfile.test -t zebrad-test .;
           docker run -i zebrad-test cargo test --workspace --no-fail-fast -- -Zunstable-options --include-ignored;
-          docker run -i --mount type=bind,source=/mnt/disks/gce-containers-mounts/gce-persistent-disks/zebrad-cache-$SHORT_SHA-mainnet-419200,target=/zebrad-cache zebrad-test:latest cargo test --verbose --features test_sync_to_sapling_mainnet --manifest-path zebrad/Cargo.toml sync_to_sapling_mainnet;
+          docker run -i --mount type=bind,source=/mnt/disks/gce-containers-mounts/gce-persistent-disks/zebrad-cache-$SHORT_SHA-mainnet-419200,target=/zebrad-cache zebrad-test:latest cargo test --verbose --features test_sync_past_sapling_mainnet --manifest-path zebrad/Cargo.toml sync_past_sapling_mainnet;
           "
       # Clean up
       - name: Delete test instance

From 6a0a6f6d3741bda8ec2ad6dd4e91b82f41076f3c Mon Sep 17 00:00:00 2001
From: Deirdre Connolly <deirdre@zfnd.org>
Date: Mon, 23 Nov 2020 16:48:44 -0500
Subject: [PATCH 125/148] allow(dead_code) not allow(clippy::dead_code)

---
 zebrad/tests/acceptance.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs
index 581f1b8a1d7..db34bac9334 100644
--- a/zebrad/tests/acceptance.rs
+++ b/zebrad/tests/acceptance.rs
@@ -13,7 +13,7 @@
 
 #![warn(warnings, missing_docs, trivial_casts, unused_qualifications)]
 #![forbid(unsafe_code)]
-#![allow(clippy::dead_code)]
+#![allow(dead_code)]
 #![allow(clippy::field_reassign_with_default)]
 #![allow(clippy::try_err)]
 #![allow(clippy::unknown_clippy_lints)]

From e6210caffb2bb32ef5f02f1a1981b21df3194794 Mon Sep 17 00:00:00 2001
From: Deirdre Connolly <deirdre@zfnd.org>
Date: Mon, 23 Nov 2020 22:42:53 -0500
Subject: [PATCH 126/148] Create and mount another state cache for Sapling
 activation on testnet and use for running sync_past_sapling_testnet

---
 .github/workflows/test.yml | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 209032c42a2..4f05e39383e 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -28,7 +28,6 @@ jobs:
           echo "SHORT_SHA=$(git rev-parse --short=7 $GITHUB_SHA)" >> $GITHUB_ENV && \
           echo "REPOSITORY=$REPOSITORY" >> $GITHUB_ENV
 
-      # Setup gcloud CLI
       - name: Set up gcloud
         uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
         with:
@@ -52,8 +51,11 @@ jobs:
           --boot-disk-size 100GB \
           --boot-disk-type pd-ssd \
           --container-image rust:buster \
-          --container-mount-disk mount-path='/zebrad-cache',name="zebrad-cache-$SHORT_SHA-mainnet-419200" \
-          --create-disk name="zebrad-cache-$SHORT_SHA-mainnet-419200",image=zebrad-cache-062a5ae-mainnet-height-419200 \
+          --container-mount-disk mount-path='/mainnet',name="zebrad-cache-$SHORT_SHA-mainnet-419200" \
+          --container-mount-disk mount-path='/testnet',name="zebrad-cache-$SHORT_SHA-testnet-280000" \
+          --container-restart-policy never \
+          --create-disk name="zebrad-cache-$SHORT_SHA-mainnet-419200",image=zebrad-cache-062a5ae-mainnet-419200 \
+          --create-disk name="zebrad-cache-$SHORT_SHA-testnet-280000",image=zebrad-cache-2935b4e-testnet-280000 \
           --machine-type n2-standard-4 \
           --service-account cos-vm@zealous-zebra.iam.gserviceaccount.com \
           --scopes cloud-platform \
@@ -69,6 +71,7 @@ jobs:
           docker build -f docker/Dockerfile.test -t zebrad-test .;
           docker run -i zebrad-test cargo test --workspace --no-fail-fast -- -Zunstable-options --include-ignored;
           docker run -i --mount type=bind,source=/mnt/disks/gce-containers-mounts/gce-persistent-disks/zebrad-cache-$SHORT_SHA-mainnet-419200,target=/zebrad-cache zebrad-test:latest cargo test --verbose --features test_sync_past_sapling_mainnet --manifest-path zebrad/Cargo.toml sync_past_sapling_mainnet;
+          docker run -i --mount type=bind,source=/mnt/disks/gce-containers-mounts/gce-persistent-disks/zebrad-cache-$SHORT_SHA-testnet-280000,target=/zebrad-cache zebrad-test:latest cargo test --verbose --features test_sync_past_sapling_testnet --manifest-path zebrad/Cargo.toml sync_past_sapling_testnet;
           "
       # Clean up
       - name: Delete test instance

From 487ee6d31765407e5df88948ed2a960b2f68d874 Mon Sep 17 00:00:00 2001
From: Deirdre Connolly <deirdre@zfnd.org>
Date: Tue, 24 Nov 2020 00:13:19 -0500
Subject: [PATCH 127/148] Use cloud disk default of auto-delete=true

---
 .github/workflows/cd.yml            | 2 +-
 .github/workflows/manual-deploy.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index a503284a2d0..559390f45f0 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -85,7 +85,7 @@ jobs:
         run: |
           gcloud compute instance-templates create-with-container "zebrad-$BRANCH_NAME-$SHORT_SHA" \
           --container-image "gcr.io/$PROJECT_ID/$REPOSITORY/$BRANCH_NAME:$SHORT_SHA" \
-          --create-disk name=zebrad-cache-$SHORT_SHA,size=100GB,type=pd-balanced,auto-delete=no \
+          --create-disk name=zebrad-cache-$SHORT_SHA,size=100GB,type=pd-balanced \
           --container-mount-disk mount-path="/zebrad-cache",name=zebrad-cache-$SHORT_SHA \
           --machine-type n2d-standard-4 \
           --service-account cos-vm@zealous-zebra.iam.gserviceaccount.com \
diff --git a/.github/workflows/manual-deploy.yml b/.github/workflows/manual-deploy.yml
index fa33d64de7b..d8f7a9d7f1d 100644
--- a/.github/workflows/manual-deploy.yml
+++ b/.github/workflows/manual-deploy.yml
@@ -58,7 +58,7 @@ jobs:
         gcloud compute instances create-with-container "zebrad-$BRANCH_NAME-$SHORT_SHA" \
         --container-image "gcr.io/$PROJECT_ID/$REPOSITORY/$BRANCH_NAME:$SHORT_SHA" \
         --container-mount-disk mount-path='/zebrad-cache',name=zebrad-cache-$SHORT_SHA \
-        --create-disk name=zebrad-cache-$SHORT_SHA,size=100GB,type=pd-balanced,auto-delete=no \
+        --create-disk name=zebrad-cache-$SHORT_SHA,size=100GB,type=pd-balanced \
         --machine-type n2-standard-4 \
         --service-account cos-vm@zealous-zebra.iam.gserviceaccount.com \
         --tags zebrad \

From 2e0ed94b22d8430ec6697393e7239894d8970bbd Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Tue, 24 Nov 2020 10:08:14 -0800
Subject: [PATCH 128/148] Revert "Downgrade a per-block log to debug level"

This reverts commit 15d26e3c47b3d06b11e08fa7f78d230bf7115b3f.
---
 zebra-state/src/service/non_finalized_state/chain.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/zebra-state/src/service/non_finalized_state/chain.rs b/zebra-state/src/service/non_finalized_state/chain.rs
index 5ea43426ab3..3dc3ec0c11e 100644
--- a/zebra-state/src/service/non_finalized_state/chain.rs
+++ b/zebra-state/src/service/non_finalized_state/chain.rs
@@ -33,7 +33,7 @@ impl Chain {
     pub fn push(&mut self, block: PreparedBlock) {
         // update cumulative data members
         self.update_chain_state_with(&block);
-        tracing::debug!(block = %block.block, "adding block to chain");
+        tracing::info!(block = %block.block, "adding block to chain");
         self.blocks.insert(block.height, block);
     }
 

From 61b3286085708f0947a95842011ab49292edcfc2 Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Tue, 24 Nov 2020 12:30:58 -0800
Subject: [PATCH 129/148] consensus: add NU plumbing to block,tx,script
 verifiers

Closes #1367 by propagating the network upgrade through the service
requests.
---
 zebra-consensus/src/block.rs       |  7 +++---
 zebra-consensus/src/script.rs      | 32 +++++++++++++++++---------
 zebra-consensus/src/transaction.rs | 37 ++++++++++++++++++++++--------
 3 files changed, 52 insertions(+), 24 deletions(-)

diff --git a/zebra-consensus/src/block.rs b/zebra-consensus/src/block.rs
index c86c3983de3..f1623b7897e 100644
--- a/zebra-consensus/src/block.rs
+++ b/zebra-consensus/src/block.rs
@@ -25,7 +25,6 @@ use tracing::Instrument;
 use zebra_chain::{
     block::{self, Block},
     parameters::Network,
-    parameters::NetworkUpgrade,
     transparent,
     work::equihash,
 };
@@ -83,9 +82,8 @@ where
     S::Future: Send + 'static,
 {
     pub fn new(network: Network, state_service: S) -> Self {
-        let branch = NetworkUpgrade::Sapling.branch_id().unwrap();
-        let script_verifier = script::Verifier::new(state_service.clone(), branch);
-        let transaction_verifier = transaction::Verifier::new(script_verifier);
+        let transaction_verifier =
+            transaction::Verifier::new(network, script::Verifier::new(state_service.clone()));
 
         Self {
             network,
@@ -176,6 +174,7 @@ where
                     .call(transaction::Request::Block {
                         transaction: transaction.clone(),
                         known_utxos: known_utxos.clone(),
+                        height,
                     });
                 async_checks.push(rsp);
             }
diff --git a/zebra-consensus/src/script.rs b/zebra-consensus/src/script.rs
index cad3c6b093e..a4bc2e5c592 100644
--- a/zebra-consensus/src/script.rs
+++ b/zebra-consensus/src/script.rs
@@ -2,7 +2,7 @@ use std::{collections::HashMap, future::Future, pin::Pin, sync::Arc};
 
 use tracing::Instrument;
 
-use zebra_chain::{parameters::ConsensusBranchId, transaction::Transaction, transparent};
+use zebra_chain::{parameters::NetworkUpgrade, transaction::Transaction, transparent};
 use zebra_state::Utxo;
 
 use crate::BoxError;
@@ -21,27 +21,34 @@ use crate::BoxError;
 #[derive(Debug, Clone)]
 pub struct Verifier<ZS> {
     state: ZS,
-    branch: ConsensusBranchId,
 }
 
 impl<ZS> Verifier<ZS> {
-    pub fn new(state: ZS, branch: ConsensusBranchId) -> Self {
-        Self { state, branch }
+    pub fn new(state: ZS) -> Self {
+        Self { state }
     }
 }
 
 /// A script verification request.
-///
-/// Ideally, this would supply only an `Outpoint` and the unlock script,
-/// rather than the entire `Transaction`, but we call a C++
-/// implementation, and its FFI requires the entire transaction.
-/// At some future point, we could investigate reducing the size of the
-/// request.
 #[derive(Debug)]
 pub struct Request {
+    /// Ideally, this would supply only an `Outpoint` and the unlock script,
+    /// rather than the entire `Transaction`, but we call a C++
+    /// implementation, and its FFI requires the entire transaction.
+    ///
+    /// This causes quadratic script verification behavior, so
+    /// at some future point, we need to reform this data.
     pub transaction: Arc<Transaction>,
     pub input_index: usize,
+    /// A set of additional UTXOs known in the context of this verification request.
+    ///
+    /// This allows specifying additional UTXOs that are not already known to the chain state.
     pub known_utxos: Arc<HashMap<transparent::OutPoint, Utxo>>,
+    /// The network upgrade active in the context of this verification request.
+    ///
+    /// Because the consensus branch ID changes with each network upgrade,
+    /// it has to be specified on a per-request basis.
+    pub upgrade: NetworkUpgrade,
 }
 
 impl<ZS> tower::Service<Request> for Verifier<ZS>
@@ -68,13 +75,16 @@ where
             transaction,
             input_index,
             known_utxos,
+            upgrade,
         } = req;
         let input = &transaction.inputs()[input_index];
+        let branch_id = upgrade
+            .branch_id()
+            .expect("post-Sapling NUs have a consensus branch ID");
 
         match input {
             transparent::Input::PrevOut { outpoint, .. } => {
                 let outpoint = *outpoint;
-                let branch_id = self.branch;
 
                 let span = tracing::trace_span!("script", ?outpoint);
                 let query =
diff --git a/zebra-consensus/src/transaction.rs b/zebra-consensus/src/transaction.rs
index 894e6450548..27c6417a52c 100644
--- a/zebra-consensus/src/transaction.rs
+++ b/zebra-consensus/src/transaction.rs
@@ -14,7 +14,8 @@ use tower::{Service, ServiceExt};
 use tracing::Instrument;
 
 use zebra_chain::{
-    parameters::NetworkUpgrade,
+    block,
+    parameters::{Network, NetworkUpgrade},
     transaction::{self, HashType, Transaction},
     transparent,
 };
@@ -28,6 +29,7 @@ mod check;
 /// Asynchronous transaction verification.
 #[derive(Debug, Clone)]
 pub struct Verifier<ZS> {
+    network: Network,
     script_verifier: script::Verifier<ZS>,
     // spend_verifier: groth16::Verifier,
     // output_verifier: groth16::Verifier,
@@ -40,8 +42,11 @@ where
     ZS::Future: Send + 'static,
 {
     // XXX: how should this struct be constructed?
-    pub fn new(script_verifier: script::Verifier<ZS>) -> Self {
-        Self { script_verifier }
+    pub fn new(network: Network, script_verifier: script::Verifier<ZS>) -> Self {
+        Self {
+            network,
+            script_verifier,
+        }
     }
 }
 
@@ -54,15 +59,22 @@ where
 pub enum Request {
     /// Verify the supplied transaction as part of a block.
     Block {
+        /// The transaction itself.
         transaction: Arc<Transaction>,
         /// Additional UTXOs which are known at the time of verification.
         known_utxos: Arc<HashMap<transparent::OutPoint, zs::Utxo>>,
+        /// The height of the block containing this transaction, used to
+        /// determine the applicable network upgrade.
+        height: block::Height,
     },
     /// Verify the supplied transaction as part of the mempool.
     Mempool {
+        /// The transaction itself.
         transaction: Arc<Transaction>,
         /// Additional UTXOs which are known at the time of verification.
         known_utxos: Arc<HashMap<transparent::OutPoint, zs::Utxo>>,
+        /// The active NU in the context of this verification.
+        upgrade: NetworkUpgrade,
     },
 }
 
@@ -91,15 +103,20 @@ where
             unimplemented!();
         }
 
-        let (tx, known_utxos) = match req {
+        let (tx, known_utxos, upgrade) = match req {
             Request::Block {
                 transaction,
                 known_utxos,
-            } => (transaction, known_utxos),
+                height,
+            } => {
+                let upgrade = NetworkUpgrade::current(self.network, height);
+                (transaction, known_utxos, upgrade)
+            }
             Request::Mempool {
                 transaction,
                 known_utxos,
-            } => (transaction, known_utxos),
+                upgrade,
+            } => (transaction, known_utxos, upgrade),
         };
 
         let mut redjubjub_verifier = crate::primitives::redjubjub::VERIFIER.clone();
@@ -135,6 +152,7 @@ where
                         // feed all of the inputs to the script verifier
                         for input_index in 0..inputs.len() {
                             let rsp = script_verifier.ready_and().await?.call(script::Request {
+                                upgrade,
                                 known_utxos: known_utxos.clone(),
                                 transaction: tx.clone(),
                                 input_index,
@@ -146,10 +164,11 @@ where
 
                     check::has_inputs_and_outputs(&tx)?;
 
+                    // TODO: rework this code
                     let sighash = tx.sighash(
-                        NetworkUpgrade::Sapling, // TODO: pass this in
-                        HashType::ALL,           // TODO: check these
-                        None,                    // TODO: check these
+                        upgrade,
+                        HashType::ALL, // TODO: check these
+                        None,          // TODO: check these
                     );
 
                     if let Some(joinsplit_data) = joinsplit_data {

From f798aef843f6f8a4468ee37e5f0c541a8cf7607f Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Tue, 24 Nov 2020 13:07:50 -0800
Subject: [PATCH 130/148] consensus: increase buffer for batch redjubjub
 results

---
 zebra-consensus/src/primitives/redjubjub.rs | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/zebra-consensus/src/primitives/redjubjub.rs b/zebra-consensus/src/primitives/redjubjub.rs
index 472312ae50d..465d59d9a5c 100644
--- a/zebra-consensus/src/primitives/redjubjub.rs
+++ b/zebra-consensus/src/primitives/redjubjub.rs
@@ -62,8 +62,9 @@ pub struct Verifier {
 impl Default for Verifier {
     fn default() -> Self {
         let batch = batch::Verifier::default();
-        // XXX(hdevalence) what's a reasonable choice here?
-        let (tx, _) = channel(10);
+        // This bound should be big enough to avoid RecvErrors
+        // when tasks wait on results for a long time.
+        let (tx, _) = channel(512);
         Self { tx, batch }
     }
 }
@@ -90,8 +91,8 @@ impl Service<BatchControl<Item>> for Verifier {
                     match rx.recv().await {
                         Ok(result) => result,
                         Err(RecvError::Lagged(_)) => {
-                            tracing::warn!(
-                                "missed channel updates for the correct signature batch!"
+                            tracing::error!(
+                                "batch verification reciever lagged and lost verification results"
                             );
                             Err(Error::InvalidSignature)
                         }

From b9347e01e43f02745253c33340a13d50cb8162ec Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Tue, 24 Nov 2020 16:13:55 -0800
Subject: [PATCH 131/148] consensus: ignore sighash-derived checks until #1377

---
 zebra-consensus/src/transaction.rs | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/zebra-consensus/src/transaction.rs b/zebra-consensus/src/transaction.rs
index 27c6417a52c..5fdd0869b10 100644
--- a/zebra-consensus/src/transaction.rs
+++ b/zebra-consensus/src/transaction.rs
@@ -164,7 +164,7 @@ where
 
                     check::has_inputs_and_outputs(&tx)?;
 
-                    // TODO: rework this code
+                    // TODO: rework this code #1377
                     let sighash = tx.sighash(
                         upgrade,
                         HashType::ALL, // TODO: check these
@@ -179,7 +179,8 @@ where
 
                         // Then, pass those items to self.joinsplit to verify them.
 
-                        check::validate_joinsplit_sig(joinsplit_data, sighash.as_bytes())?;
+                        // Ignore pending sighash check #1377
+                        let _ = check::validate_joinsplit_sig(joinsplit_data, sighash.as_bytes());
                     }
 
                     if let Some(shielded_data) = shielded_data {
@@ -194,12 +195,13 @@ where
                             // description while adding the resulting future to
                             // our collection of async checks that (at a
                             // minimum) must pass for the transaction to verify.
-                            let rsp = redjubjub_verifier
+                            let _rsp = redjubjub_verifier
                                 .ready_and()
                                 .await?
                                 .call((spend.rk, spend.spend_auth_sig, &sighash).into());
 
-                            async_checks.push(rsp.boxed());
+                            // Disable pending sighash check #1377
+                            //async_checks.push(rsp.boxed());
 
                             // TODO: prepare public inputs for spends, then create
                             // a groth16::Item and pass to self.spend
@@ -227,12 +229,14 @@ where
                         });
 
                         let bvk = shielded_data.binding_verification_key(*value_balance);
-                        let rsp = redjubjub_verifier
+                        let _rsp = redjubjub_verifier
                             .ready_and()
                             .await?
                             .call((bvk, shielded_data.binding_sig, &sighash).into())
                             .boxed();
-                        async_checks.push(rsp);
+
+                        // Disable pending sighash check #1377
+                        //async_checks.push(rsp);
                     }
 
                     // Finally, wait for all asynchronous checks to complete

From 5c465abed288a96dd03a4b85c677520faace2a8f Mon Sep 17 00:00:00 2001
From: Deirdre Connolly <deirdre@zfnd.org>
Date: Tue, 24 Nov 2020 22:07:25 -0500
Subject: [PATCH 132/148] Remove timeout on test job

---
 .github/workflows/test.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 4f05e39383e..e6fb2cd8309 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -15,7 +15,6 @@ jobs:
   test:
     name: Run all tests
     runs-on: ubuntu-latest
-    timeout-minutes: 30
     steps:
       - uses: actions/checkout@v2
 

From 2a21c86b9103535e1078e988744a8001e220e260 Mon Sep 17 00:00:00 2001
From: Deirdre Connolly <deirdre@zfnd.org>
Date: Tue, 24 Nov 2020 22:14:22 -0500
Subject: [PATCH 133/148] I before E except after C (or uh, not-english)

---
 book/src/dev/rfcs/0005-state-updates.md     | 2 +-
 zebra-consensus/src/primitives/redjubjub.rs | 2 +-
 zebra-state/src/service/pending_utxos.rs    | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/book/src/dev/rfcs/0005-state-updates.md b/book/src/dev/rfcs/0005-state-updates.md
index b685af0eae0..f72ce35611d 100644
--- a/book/src/dev/rfcs/0005-state-updates.md
+++ b/book/src/dev/rfcs/0005-state-updates.md
@@ -527,7 +527,7 @@ New `non-finalized` blocks are commited as follows:
    it has already been successfully verified:
      - create a new oneshot channel
      - immediately send `Err(DuplicateBlockHash)` drop the sender
-     - return the reciever
+     - return the receiver
 
 2. If a duplicate block hash exists in the queue:
      - Find the `QueuedBlock` for that existing duplicate block
diff --git a/zebra-consensus/src/primitives/redjubjub.rs b/zebra-consensus/src/primitives/redjubjub.rs
index 465d59d9a5c..08d9239a9a5 100644
--- a/zebra-consensus/src/primitives/redjubjub.rs
+++ b/zebra-consensus/src/primitives/redjubjub.rs
@@ -92,7 +92,7 @@ impl Service<BatchControl<Item>> for Verifier {
                         Ok(result) => result,
                         Err(RecvError::Lagged(_)) => {
                             tracing::error!(
-                                "batch verification reciever lagged and lost verification results"
+                                "batch verification receiver lagged and lost verification results"
                             );
                             Err(Error::InvalidSignature)
                         }
diff --git a/zebra-state/src/service/pending_utxos.rs b/zebra-state/src/service/pending_utxos.rs
index c0740096f46..4ad38c2b061 100644
--- a/zebra-state/src/service/pending_utxos.rs
+++ b/zebra-state/src/service/pending_utxos.rs
@@ -56,7 +56,7 @@ impl PendingUtxos {
         }
     }
 
-    /// Scan the set of waiting utxo requests for channels where all recievers
+    /// Scan the set of waiting utxo requests for channels where all receivers
     /// have been dropped and remove the corresponding sender.
     pub fn prune(&mut self) {
         self.0.retain(|_, chan| chan.receiver_count() > 0);

From b1bbb13978c6d4a337322e4a88b716a0b9b26064 Mon Sep 17 00:00:00 2001
From: teor <teor@riseup.net>
Date: Wed, 25 Nov 2020 15:04:18 +1000
Subject: [PATCH 134/148] Make debug_stop_at_height and ephemeral work together
 (#1339)

* Make debug_stop_at_height and ephemeral work together

* if `debug_stop_at_height` and `ephemeral` are set, delete the database
  files after reaching the stop height
* drop or flush the database before `debug_stop_at_height` exits Zebra
---
 zebra-state/src/service/finalized_state.rs | 37 ++++++++++++++++++----
 1 file changed, 30 insertions(+), 7 deletions(-)

diff --git a/zebra-state/src/service/finalized_state.rs b/zebra-state/src/service/finalized_state.rs
index 732982442b6..ba93f9c0b19 100644
--- a/zebra-state/src/service/finalized_state.rs
+++ b/zebra-state/src/service/finalized_state.rs
@@ -65,7 +65,9 @@ impl FinalizedState {
                     "state is already at the configured height"
                 );
 
-                // There's no need to sync before exit, because the trees have just been opened
+                // RocksDB can do a cleanup when column families are opened.
+                // So we want to drop it before we exit.
+                std::mem::drop(new_state);
                 std::process::exit(0);
             }
         }
@@ -253,7 +255,11 @@ impl FinalizedState {
 
         if result.is_ok() && self.is_at_stop_height(height) {
             tracing::info!(?height, ?hash, "stopping at configured height");
-
+            // We'd like to drop the database here, because that closes the
+            // column families and the database. But Rust's ownership rules
+            // make that difficult, so we just flush instead.
+            self.db.flush().expect("flush is successful");
+            self.delete_ephemeral();
             std::process::exit(0);
         }
 
@@ -327,6 +333,27 @@ impl FinalizedState {
                 block.transactions[index as usize].clone()
             })
     }
+
+    /// If the database is `ephemeral`, delete it.
+    fn delete_ephemeral(&self) {
+        if self.ephemeral {
+            let path = self.db.path();
+            tracing::debug!("removing temporary database files {:?}", path);
+            // We'd like to use `rocksdb::Env::mem_env` for ephemeral databases,
+            // but the Zcash blockchain might not fit in memory. So we just
+            // delete the database files instead.
+            //
+            // We'd like to call `DB::destroy` here, but calling destroy on a
+            // live DB is undefined behaviour:
+            // https://github.com/facebook/rocksdb/wiki/RocksDB-FAQ#basic-readwrite
+            //
+            // So we assume that all the database files are under `path`, and
+            // delete them using standard filesystem APIs. Deleting open files
+            // might cause errors on non-Unix platforms, so we ignore the result.
+            // (The OS will delete them eventually anyway.)
+            let _res = std::fs::remove_dir_all(path);
+        }
+    }
 }
 
 // Drop isn't guaranteed to run, such as when we panic, or if someone stored
@@ -335,11 +362,7 @@ impl FinalizedState {
 // up automatically eventually.
 impl Drop for FinalizedState {
     fn drop(&mut self) {
-        if self.ephemeral {
-            let path = self.db.path();
-            tracing::debug!("removing temporary database files {:?}", path);
-            let _res = std::fs::remove_dir_all(path);
-        }
+        self.delete_ephemeral()
     }
 }
 

From 31eb0a5126da22bf6a0ad40fb2513b07fcdbc296 Mon Sep 17 00:00:00 2001
From: teor <teor@riseup.net>
Date: Wed, 25 Nov 2020 13:30:29 +1000
Subject: [PATCH 135/148] Avoid verbose default logs

Temporary fix so that Zebra's default logs support a typical workflow:
1. Developer or user runs Zebra with the default config
2. They send the logs to a terminal
3. When they see a bug, they copy-paste the last few log lines into a
   bug report

This is the same change that was merged in #1373 and reverted in #1375.
We'll create a consistent logging design for Zebra in ticket #1381.
---
 zebra-state/src/service/non_finalized_state/chain.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/zebra-state/src/service/non_finalized_state/chain.rs b/zebra-state/src/service/non_finalized_state/chain.rs
index 3dc3ec0c11e..5ea43426ab3 100644
--- a/zebra-state/src/service/non_finalized_state/chain.rs
+++ b/zebra-state/src/service/non_finalized_state/chain.rs
@@ -33,7 +33,7 @@ impl Chain {
     pub fn push(&mut self, block: PreparedBlock) {
         // update cumulative data members
         self.update_chain_state_with(&block);
-        tracing::info!(block = %block.block, "adding block to chain");
+        tracing::debug!(block = %block.block, "adding block to chain");
         self.blocks.insert(block.height, block);
     }
 

From 486e55104a7880c64683fffa93801666b4c7dbd6 Mon Sep 17 00:00:00 2001
From: Alfredo Garcia <oxarbitrage@gmail.com>
Date: Thu, 19 Nov 2020 14:55:06 -0300
Subject: [PATCH 136/148] create Downloads for Inbound

---
 zebrad/src/commands/start.rs               |   2 +-
 zebrad/src/components/inbound.rs           |  50 ++++-
 zebrad/src/components/inbound/downloads.rs | 206 +++++++++++++++++++++
 3 files changed, 253 insertions(+), 5 deletions(-)
 create mode 100644 zebrad/src/components/inbound/downloads.rs

diff --git a/zebrad/src/commands/start.rs b/zebrad/src/commands/start.rs
index f829f355267..ee8910737d1 100644
--- a/zebrad/src/commands/start.rs
+++ b/zebrad/src/commands/start.rs
@@ -67,7 +67,7 @@ impl StartCmd {
         let inbound = ServiceBuilder::new()
             .load_shed()
             .buffer(20)
-            .service(Inbound::new(setup_rx, state.clone()));
+            .service(Inbound::new(setup_rx, state.clone(), verifier.clone()));
 
         let (peer_set, address_book) = zebra_network::init(config.network.clone(), inbound).await;
         setup_tx
diff --git a/zebrad/src/components/inbound.rs b/zebrad/src/components/inbound.rs
index ef94a301bbf..67fb41d8758 100644
--- a/zebrad/src/components/inbound.rs
+++ b/zebrad/src/components/inbound.rs
@@ -12,12 +12,21 @@ use futures::{
 use tokio::sync::oneshot;
 use tower::{buffer::Buffer, util::BoxService, Service, ServiceExt};
 
+use zebra_chain as zc;
+use zebra_consensus as zcon;
 use zebra_network as zn;
 use zebra_network::AddressBook;
 use zebra_state as zs;
 
+mod downloads;
+use downloads::Downloads;
+
 type Outbound = Buffer<BoxService<zn::Request, zn::Response, zn::BoxError>, zn::Request>;
 type State = Buffer<BoxService<zs::Request, zs::Response, zs::BoxError>, zs::Request>;
+type Verifier = Buffer<
+    BoxService<Arc<zc::block::Block>, zc::block::Hash, zcon::chain::VerifyChainError>,
+    Arc<zc::block::Block>,
+>;
 
 pub type SetupData = (Outbound, Arc<Mutex<AddressBook>>);
 
@@ -52,15 +61,23 @@ pub struct Inbound {
     outbound: Option<Outbound>,
     address_book: Option<Arc<Mutex<zn::AddressBook>>>,
     state: State,
+    verifier: Verifier,
+    downloads: Option<Downloads<Outbound, Verifier, State>>,
 }
 
 impl Inbound {
-    pub fn new(network_setup: oneshot::Receiver<SetupData>, state: State) -> Self {
+    pub fn new(
+        network_setup: oneshot::Receiver<SetupData>,
+        state: State,
+        verifier: Verifier,
+    ) -> Self {
         Self {
             network_setup: Some(network_setup),
             outbound: None,
             address_book: None,
             state,
+            verifier,
+            downloads: None,
         }
     }
 }
@@ -85,6 +102,11 @@ impl Service<zn::Request> for Inbound {
                     self.outbound = Some(outbound);
                     self.address_book = Some(address_book);
                     self.network_setup = None;
+                    self.downloads = Some(Downloads::new(
+                        self.outbound.clone().unwrap(),
+                        self.verifier.clone(),
+                        self.state.clone(),
+                    ));
                 }
                 Err(TryRecvError::Empty) => {
                     self.network_setup = Some(rx);
@@ -98,6 +120,10 @@ impl Service<zn::Request> for Inbound {
                 }
             };
         }
+
+        // Clean up completed download tasks
+        while let Poll::Ready(Some(_)) = self.downloads.unwrap().poll_next(cx) {}
+
         // Now report readiness based on readiness of the inner services, if they're available.
         // XXX do we want to propagate backpressure from the network here?
         match (
@@ -171,9 +197,25 @@ impl Service<zn::Request> for Inbound {
                 debug!("ignoring unimplemented request");
                 async { Ok(zn::Response::Nil) }.boxed()
             }
-            zn::Request::AdvertiseBlock(_block) => {
-                debug!("ignoring unimplemented request");
-                async { Ok(zn::Response::Nil) }.boxed()
+            zn::Request::AdvertiseBlock(hash) => {
+                // this sucks
+                let mut downloads = self.downloads.take().unwrap();
+                self.downloads = Some(Downloads::new(
+                    self.outbound.as_ref().unwrap().clone(),
+                    self.verifier.clone(),
+                    self.state.clone(),
+                ));
+
+                async move {
+                    if downloads.download_and_verify(hash).await? {
+                        tracing::info!(?hash, "queued download and verification of gossiped block");
+                    } else {
+                        tracing::debug!(?hash, "gossiped block already queued or verified");
+                    }
+
+                    Ok(zn::Response::Nil)
+                }
+                .boxed()
             }
             zn::Request::MempoolTransactions => {
                 debug!("ignoring unimplemented request");
diff --git a/zebrad/src/components/inbound/downloads.rs b/zebrad/src/components/inbound/downloads.rs
new file mode 100644
index 00000000000..05ba567defe
--- /dev/null
+++ b/zebrad/src/components/inbound/downloads.rs
@@ -0,0 +1,206 @@
+use std::{
+    collections::HashMap,
+    pin::Pin,
+    sync::Arc,
+    task::{Context, Poll},
+};
+
+use color_eyre::eyre::{eyre, Report};
+use futures::{
+    future::TryFutureExt,
+    ready,
+    stream::{FuturesUnordered, Stream},
+};
+use pin_project::pin_project;
+use tokio::{sync::oneshot, task::JoinHandle};
+use tower::{Service, ServiceExt};
+use tracing_futures::Instrument;
+
+use zebra_chain::block::{self, Block};
+use zebra_network as zn;
+use zebra_state as zs;
+
+type BoxError = Box<dyn std::error::Error + Send + Sync + 'static>;
+
+/// Manages download and verification of blocks gossiped to this peer.
+#[pin_project]
+#[derive(Debug)]
+pub struct Downloads<ZN, ZV, ZS>
+where
+    ZN: Service<zn::Request, Response = zn::Response, Error = BoxError> + Send + 'static,
+    ZN::Future: Send,
+    ZV: Service<Arc<Block>, Response = block::Hash, Error = BoxError> + Send + Clone + 'static,
+    ZV::Future: Send,
+    ZS: Service<zs::Request, Response = zs::Response, Error = BoxError> + Send + 'static,
+    ZS::Future: Send,
+{
+    network: ZN,
+    verifier: ZV,
+    state: ZS,
+    #[pin]
+    pending: FuturesUnordered<JoinHandle<Result<block::Hash, (BoxError, block::Hash)>>>,
+    cancel_handles: HashMap<block::Hash, oneshot::Sender<()>>,
+}
+
+impl<ZN, ZV, ZS> Stream for Downloads<ZN, ZV, ZS>
+where
+    ZN: Service<zn::Request, Response = zn::Response, Error = BoxError> + Send + 'static,
+    ZN::Future: Send,
+    ZV: Service<Arc<Block>, Response = block::Hash, Error = BoxError> + Send + Clone + 'static,
+    ZV::Future: Send,
+    ZS: Service<zs::Request, Response = zs::Response, Error = BoxError> + Send + 'static,
+    ZS::Future: Send,
+{
+    type Item = Result<block::Hash, BoxError>;
+
+    fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
+        let this = self.project();
+        // This would be cleaner with poll_map #63514, but that's nightly only.
+        if let Some(join_result) = ready!(this.pending.poll_next(cx)) {
+            match join_result.expect("block download and verify tasks must not panic") {
+                Ok(hash) => {
+                    this.cancel_handles.remove(&hash);
+                    Poll::Ready(Some(Ok(hash)))
+                }
+                Err((e, hash)) => {
+                    this.cancel_handles.remove(&hash);
+                    Poll::Ready(Some(Err(e)))
+                }
+            }
+        } else {
+            Poll::Ready(None)
+        }
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        self.pending.size_hint()
+    }
+}
+
+impl<ZN, ZV, ZS> Downloads<ZN, ZV, ZS>
+where
+    ZN: Service<zn::Request, Response = zn::Response, Error = BoxError> + Send + 'static,
+    ZN::Future: Send,
+    ZV: Service<Arc<Block>, Response = block::Hash, Error = BoxError> + Send + Clone + 'static,
+    ZV::Future: Send,
+    ZS: Service<zs::Request, Response = zs::Response, Error = BoxError> + Send + 'static,
+    ZS::Future: Send,
+{
+    /// Initialize a new download stream with the provided `network` and
+    /// `verifier` services.
+    ///
+    /// The [`Downloads`] stream is agnostic to the network policy, so retry and
+    /// timeout limits should be applied to the `network` service passed into
+    /// this constructor.
+    pub fn new(network: ZN, verifier: ZV, state: ZS) -> Self {
+        Self {
+            network,
+            verifier,
+            state,
+            pending: FuturesUnordered::new(),
+            cancel_handles: HashMap::new(),
+        }
+    }
+
+    /// Queue a block for download and verification.
+    ///
+    /// This method waits for the network to become ready, and returns an error
+    /// only if the network service fails. It returns immediately after queuing
+    /// the request.
+    #[instrument(skip(self))]
+    pub async fn download_and_verify(&mut self, hash: block::Hash) -> Result<bool, Report> {
+        if self.cancel_handles.contains_key(&hash) {
+            return Ok(false);
+        }
+
+        // Check if the block is already in the state.
+        let block_state_req = self
+            .state
+            .ready_and()
+            .await
+            .map_err(|e| eyre!(e))?
+            .call(zs::Request::Block(zs::HashOrHeight::from(hash)))
+            .await;
+
+        let block_in_chain = match block_state_req {
+            Ok(zs::Response::Block(block)) => block,
+            _ => None,
+        };
+
+        // Block already in state, get out.
+        if block_in_chain.is_some() {
+            return Ok(false);
+        }
+
+        // We construct the block requests sequentially, waiting for the peer
+        // set to be ready to process each request. This ensures that we start
+        // block downloads in the order we want them (though they may resolve
+        // out of order), and it means that we respect backpressure. Otherwise,
+        // if we waited for readiness and did the service call in the spawned
+        // tasks, all of the spawned tasks would race each other waiting for the
+        // network to become ready.
+        tracing::debug!("waiting to request block");
+        let block_req = self
+            .network
+            .ready_and()
+            .await
+            .map_err(|e| eyre!(e))?
+            .call(zn::Request::BlocksByHash(std::iter::once(hash).collect()));
+        tracing::debug!("requested block");
+
+        // This oneshot is used to signal cancellation to the download task.
+        let (cancel_tx, mut cancel_rx) = oneshot::channel::<()>();
+
+        let mut verifier = self.verifier.clone();
+        let task = tokio::spawn(
+            async move {
+                let rsp = tokio::select! {
+                    _ = &mut cancel_rx => {
+                        tracing::trace!("task cancelled prior to download completion");
+                        metrics::counter!("gossip.cancelled.download.count", 1);
+                        return Err("canceled block_fetch_verify".into())
+                    }
+                    rsp = block_req => rsp?,
+                };
+
+                let block = if let zn::Response::Blocks(blocks) = rsp {
+                    blocks
+                        .into_iter()
+                        .next()
+                        .expect("successful response has the block in it")
+                } else {
+                    unreachable!("wrong response to block request");
+                };
+                metrics::counter!("gossip.downloaded.block.count", 1);
+
+                let rsp = verifier.ready_and().await?.call(block);
+                let verification = tokio::select! {
+                    _ = &mut cancel_rx => {
+                        tracing::trace!("task cancelled prior to verification");
+                        metrics::counter!("gossip.cancelled.verify.count", 1);
+                        return Err("canceled block_fetch_verify".into())
+                    }
+                    verification = rsp => verification,
+                };
+                if verification.is_ok() {
+                    metrics::counter!("sync.verified.block.count", 1);
+                }
+
+                verification
+            }
+            .in_current_span()
+            // Tack the hash onto the error so we can remove the cancel handle
+            // on failure as well as on success.
+            .map_err(move |e| (e, hash)),
+        );
+
+        self.pending.push(task);
+        // XXX replace with expect_none when stable
+        assert!(
+            self.cancel_handles.insert(hash, cancel_tx).is_none(),
+            "blocks are only queued once"
+        );
+
+        Ok(true)
+    }
+}

From 7d42c637901f6b6314bb7a2a7a55a9eeab6709d2 Mon Sep 17 00:00:00 2001
From: Alfredo Garcia <oxarbitrage@gmail.com>
Date: Thu, 19 Nov 2020 15:17:03 -0300
Subject: [PATCH 137/148] fix comment

---
 zebra-network/src/protocol/internal/request.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/zebra-network/src/protocol/internal/request.rs b/zebra-network/src/protocol/internal/request.rs
index 39fc436ffa4..b5b18e9cd98 100644
--- a/zebra-network/src/protocol/internal/request.rs
+++ b/zebra-network/src/protocol/internal/request.rs
@@ -152,7 +152,7 @@ pub enum Request {
     ///
     /// This is implemented by sending an `inv` message containing the
     /// block hash, allowing the remote peer to choose whether to download
-    /// it. Remote peers who choose to download the transaction will generate a
+    /// it. Remote peers who choose to download the block will generate a
     /// [`Request::BlocksByHash`] against the "inbound" service passed to
     /// [`zebra_network::init`].
     ///

From 65699775493a5b827ef07ea2c04ad7575dd0ece5 Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Tue, 24 Nov 2020 13:21:11 -0800
Subject: [PATCH 138/148] test compile change

---
 zebrad/src/components/inbound.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/zebrad/src/components/inbound.rs b/zebrad/src/components/inbound.rs
index 67fb41d8758..7f1f1e565d3 100644
--- a/zebrad/src/components/inbound.rs
+++ b/zebrad/src/components/inbound.rs
@@ -122,7 +122,9 @@ impl Service<zn::Request> for Inbound {
         }
 
         // Clean up completed download tasks
-        while let Poll::Ready(Some(_)) = self.downloads.unwrap().poll_next(cx) {}
+        if let Some(downloads) = self.downloads.as_mut() {
+            while let Poll::Ready(Some(_)) = Pin::new(downloads).poll_next(cx) {}
+        }
 
         // Now report readiness based on readiness of the inner services, if they're available.
         // XXX do we want to propagate backpressure from the network here?

From e645e3bf0c71786e4a96a0747c6a4a1a49db7c85 Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Tue, 24 Nov 2020 14:06:30 -0800
Subject: [PATCH 139/148] remove async

---
 zebrad/src/components/inbound.rs           |   2 +-
 zebrad/src/components/inbound/downloads.rs | 130 ++++++++-------------
 2 files changed, 49 insertions(+), 83 deletions(-)

diff --git a/zebrad/src/components/inbound.rs b/zebrad/src/components/inbound.rs
index 7f1f1e565d3..786f10b1725 100644
--- a/zebrad/src/components/inbound.rs
+++ b/zebrad/src/components/inbound.rs
@@ -7,7 +7,7 @@ use std::{
 
 use futures::{
     future::{FutureExt, TryFutureExt},
-    stream::TryStreamExt,
+    stream::{Stream, TryStreamExt},
 };
 use tokio::sync::oneshot;
 use tower::{buffer::Buffer, util::BoxService, Service, ServiceExt};
diff --git a/zebrad/src/components/inbound/downloads.rs b/zebrad/src/components/inbound/downloads.rs
index 05ba567defe..be312865f94 100644
--- a/zebrad/src/components/inbound/downloads.rs
+++ b/zebrad/src/components/inbound/downloads.rs
@@ -79,11 +79,11 @@ where
 
 impl<ZN, ZV, ZS> Downloads<ZN, ZV, ZS>
 where
-    ZN: Service<zn::Request, Response = zn::Response, Error = BoxError> + Send + 'static,
+    ZN: Service<zn::Request, Response = zn::Response, Error = BoxError> + Send + Clone + 'static,
     ZN::Future: Send,
     ZV: Service<Arc<Block>, Response = block::Hash, Error = BoxError> + Send + Clone + 'static,
     ZV::Future: Send,
-    ZS: Service<zs::Request, Response = zs::Response, Error = BoxError> + Send + 'static,
+    ZS: Service<zs::Request, Response = zs::Response, Error = BoxError> + Send + Clone + 'static,
     ZS::Future: Send,
 {
     /// Initialize a new download stream with the provided `network` and
@@ -104,95 +104,61 @@ where
 
     /// Queue a block for download and verification.
     ///
-    /// This method waits for the network to become ready, and returns an error
-    /// only if the network service fails. It returns immediately after queuing
-    /// the request.
+    /// Returns true if the block was newly queued, and false if it was already queued.
     #[instrument(skip(self))]
-    pub async fn download_and_verify(&mut self, hash: block::Hash) -> Result<bool, Report> {
+    pub fn download_and_verify(&mut self, hash: block::Hash) -> bool {
         if self.cancel_handles.contains_key(&hash) {
-            return Ok(false);
+            return false;
         }
 
-        // Check if the block is already in the state.
-        let block_state_req = self
-            .state
-            .ready_and()
-            .await
-            .map_err(|e| eyre!(e))?
-            .call(zs::Request::Block(zs::HashOrHeight::from(hash)))
-            .await;
-
-        let block_in_chain = match block_state_req {
-            Ok(zs::Response::Block(block)) => block,
-            _ => None,
-        };
-
-        // Block already in state, get out.
-        if block_in_chain.is_some() {
-            return Ok(false);
-        }
-
-        // We construct the block requests sequentially, waiting for the peer
-        // set to be ready to process each request. This ensures that we start
-        // block downloads in the order we want them (though they may resolve
-        // out of order), and it means that we respect backpressure. Otherwise,
-        // if we waited for readiness and did the service call in the spawned
-        // tasks, all of the spawned tasks would race each other waiting for the
-        // network to become ready.
-        tracing::debug!("waiting to request block");
-        let block_req = self
-            .network
-            .ready_and()
-            .await
-            .map_err(|e| eyre!(e))?
-            .call(zn::Request::BlocksByHash(std::iter::once(hash).collect()));
-        tracing::debug!("requested block");
-
         // This oneshot is used to signal cancellation to the download task.
         let (cancel_tx, mut cancel_rx) = oneshot::channel::<()>();
 
+        let mut state = self.state.clone();
+        let mut network = self.network.clone();
         let mut verifier = self.verifier.clone();
-        let task = tokio::spawn(
-            async move {
-                let rsp = tokio::select! {
-                    _ = &mut cancel_rx => {
-                        tracing::trace!("task cancelled prior to download completion");
-                        metrics::counter!("gossip.cancelled.download.count", 1);
-                        return Err("canceled block_fetch_verify".into())
-                    }
-                    rsp = block_req => rsp?,
-                };
-
-                let block = if let zn::Response::Blocks(blocks) = rsp {
-                    blocks
-                        .into_iter()
-                        .next()
-                        .expect("successful response has the block in it")
-                } else {
-                    unreachable!("wrong response to block request");
-                };
-                metrics::counter!("gossip.downloaded.block.count", 1);
-
-                let rsp = verifier.ready_and().await?.call(block);
-                let verification = tokio::select! {
-                    _ = &mut cancel_rx => {
-                        tracing::trace!("task cancelled prior to verification");
-                        metrics::counter!("gossip.cancelled.verify.count", 1);
-                        return Err("canceled block_fetch_verify".into())
-                    }
-                    verification = rsp => verification,
-                };
-                if verification.is_ok() {
-                    metrics::counter!("sync.verified.block.count", 1);
-                }
 
-                verification
+        let fut = async move {
+            // Check if the block is already in the state.
+            match state.oneshot(zs::Request::Depth(hash.into())).await {
+                Ok(zs::Response::Depth(None)) => Ok(()),
+                Ok(zs::Response::Depth(Some(_))) => Err("already present".into()),
+                Err(e) => Err(e),
+            }?;
+
+            let block = if let zn::Response::Blocks(blocks) = network
+                .oneshot(zn::Request::BlocksByHash(std::iter::once(hash).collect()))
+                .await?
+            {
+                blocks
+                    .into_iter()
+                    .next()
+                    .expect("successful response has the block in it")
+            } else {
+                unreachable!("wrong response to block request");
+            };
+            metrics::counter!("gossip.downloaded.block.count", 1);
+
+            verifier.oneshot(block).await
+        }
+        .map_ok(|hash| {
+            tracing::info!(?hash, "verified advertised block");
+            metrics::counter!("gossip.verified.block.count", 1);
+        })
+        // Tack the hash onto the error so we can remove the cancel handle
+        // on failure as well as on success.
+        .map_err(move |e| (e, hash))
+        .in_current_span();
+
+        let task = tokio::spawn(async move {
+            tokio::select! {
+                _ = &mut cancel_rx => {
+                    tracing::trace!("task cancelled prior to completion");
+                    metrics::counter!("gossip.cancelled.count", 1);
+                }
+                verification = fut => verification,
             }
-            .in_current_span()
-            // Tack the hash onto the error so we can remove the cancel handle
-            // on failure as well as on success.
-            .map_err(move |e| (e, hash)),
-        );
+        });
 
         self.pending.push(task);
         // XXX replace with expect_none when stable
@@ -201,6 +167,6 @@ where
             "blocks are only queued once"
         );
 
-        Ok(true)
+        true
     }
 }

From 527f4d39edcf38e42114065863a6ff5e12781174 Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Tue, 24 Nov 2020 14:20:34 -0800
Subject: [PATCH 140/148] fix errors

---
 zebrad/src/components/inbound.rs           | 23 +++++-----------------
 zebrad/src/components/inbound/downloads.rs | 12 +++++++----
 2 files changed, 13 insertions(+), 22 deletions(-)

diff --git a/zebrad/src/components/inbound.rs b/zebrad/src/components/inbound.rs
index 786f10b1725..d4233df83c3 100644
--- a/zebrad/src/components/inbound.rs
+++ b/zebrad/src/components/inbound.rs
@@ -123,7 +123,8 @@ impl Service<zn::Request> for Inbound {
 
         // Clean up completed download tasks
         if let Some(downloads) = self.downloads.as_mut() {
-            while let Poll::Ready(Some(_)) = Pin::new(downloads).poll_next(cx) {}
+            let downloads = Pin::new(downloads);
+            while let Poll::Ready(Some(_)) = downloads.poll_next(cx) {}
         }
 
         // Now report readiness based on readiness of the inner services, if they're available.
@@ -200,24 +201,10 @@ impl Service<zn::Request> for Inbound {
                 async { Ok(zn::Response::Nil) }.boxed()
             }
             zn::Request::AdvertiseBlock(hash) => {
-                // this sucks
-                let mut downloads = self.downloads.take().unwrap();
-                self.downloads = Some(Downloads::new(
-                    self.outbound.as_ref().unwrap().clone(),
-                    self.verifier.clone(),
-                    self.state.clone(),
-                ));
-
-                async move {
-                    if downloads.download_and_verify(hash).await? {
-                        tracing::info!(?hash, "queued download and verification of gossiped block");
-                    } else {
-                        tracing::debug!(?hash, "gossiped block already queued or verified");
-                    }
-
-                    Ok(zn::Response::Nil)
+                if let Some(downloads) = self.downloads.as_mut() {
+                    downloads.download_and_verify(hash);
                 }
-                .boxed()
+                async { Ok(zn::Response::Nil) }.boxed()
             }
             zn::Request::MempoolTransactions => {
                 debug!("ignoring unimplemented request");
diff --git a/zebrad/src/components/inbound/downloads.rs b/zebrad/src/components/inbound/downloads.rs
index be312865f94..136c2959edc 100644
--- a/zebrad/src/components/inbound/downloads.rs
+++ b/zebrad/src/components/inbound/downloads.rs
@@ -108,21 +108,23 @@ where
     #[instrument(skip(self))]
     pub fn download_and_verify(&mut self, hash: block::Hash) -> bool {
         if self.cancel_handles.contains_key(&hash) {
+            tracing::debug!("hash already queued for download");
             return false;
         }
 
         // This oneshot is used to signal cancellation to the download task.
         let (cancel_tx, mut cancel_rx) = oneshot::channel::<()>();
 
-        let mut state = self.state.clone();
-        let mut network = self.network.clone();
-        let mut verifier = self.verifier.clone();
+        let state = self.state.clone();
+        let network = self.network.clone();
+        let verifier = self.verifier.clone();
 
         let fut = async move {
             // Check if the block is already in the state.
             match state.oneshot(zs::Request::Depth(hash.into())).await {
                 Ok(zs::Response::Depth(None)) => Ok(()),
                 Ok(zs::Response::Depth(Some(_))) => Err("already present".into()),
+                Ok(_) => unreachable!("wrong response"),
                 Err(e) => Err(e),
             }?;
 
@@ -142,8 +144,8 @@ where
             verifier.oneshot(block).await
         }
         .map_ok(|hash| {
-            tracing::info!(?hash, "verified advertised block");
             metrics::counter!("gossip.verified.block.count", 1);
+            hash
         })
         // Tack the hash onto the error so we can remove the cancel handle
         // on failure as well as on success.
@@ -155,6 +157,7 @@ where
                 _ = &mut cancel_rx => {
                     tracing::trace!("task cancelled prior to completion");
                     metrics::counter!("gossip.cancelled.count", 1);
+                    Err(("canceled".into(), hash))
                 }
                 verification = fut => verification,
             }
@@ -167,6 +170,7 @@ where
             "blocks are only queued once"
         );
 
+        tracing::debug!("queued hash for download");
         true
     }
 }

From 77bf327b070d1dc36ab74aad19490b489163d02e Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Tue, 24 Nov 2020 14:50:02 -0800
Subject: [PATCH 141/148] fix errors (2)

---
 zebrad/src/components/inbound.rs           | 9 ++++-----
 zebrad/src/components/inbound/downloads.rs | 9 ++++-----
 2 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/zebrad/src/components/inbound.rs b/zebrad/src/components/inbound.rs
index d4233df83c3..cb81e0a4faf 100644
--- a/zebrad/src/components/inbound.rs
+++ b/zebrad/src/components/inbound.rs
@@ -62,7 +62,7 @@ pub struct Inbound {
     address_book: Option<Arc<Mutex<zn::AddressBook>>>,
     state: State,
     verifier: Verifier,
-    downloads: Option<Downloads<Outbound, Verifier, State>>,
+    downloads: Option<Pin<Box<Downloads<Outbound, Verifier, State>>>>,
 }
 
 impl Inbound {
@@ -102,11 +102,11 @@ impl Service<zn::Request> for Inbound {
                     self.outbound = Some(outbound);
                     self.address_book = Some(address_book);
                     self.network_setup = None;
-                    self.downloads = Some(Downloads::new(
+                    self.downloads = Some(Box::pin(Downloads::new(
                         self.outbound.clone().unwrap(),
                         self.verifier.clone(),
                         self.state.clone(),
-                    ));
+                    )));
                 }
                 Err(TryRecvError::Empty) => {
                     self.network_setup = Some(rx);
@@ -123,8 +123,7 @@ impl Service<zn::Request> for Inbound {
 
         // Clean up completed download tasks
         if let Some(downloads) = self.downloads.as_mut() {
-            let downloads = Pin::new(downloads);
-            while let Poll::Ready(Some(_)) = downloads.poll_next(cx) {}
+            while let Poll::Ready(Some(_)) = downloads.as_mut().poll_next(cx) {}
         }
 
         // Now report readiness based on readiness of the inner services, if they're available.
diff --git a/zebrad/src/components/inbound/downloads.rs b/zebrad/src/components/inbound/downloads.rs
index 136c2959edc..3f82f51c393 100644
--- a/zebrad/src/components/inbound/downloads.rs
+++ b/zebrad/src/components/inbound/downloads.rs
@@ -5,7 +5,6 @@ use std::{
     task::{Context, Poll},
 };
 
-use color_eyre::eyre::{eyre, Report};
 use futures::{
     future::TryFutureExt,
     ready,
@@ -27,11 +26,11 @@ type BoxError = Box<dyn std::error::Error + Send + Sync + 'static>;
 #[derive(Debug)]
 pub struct Downloads<ZN, ZV, ZS>
 where
-    ZN: Service<zn::Request, Response = zn::Response, Error = BoxError> + Send + 'static,
+    ZN: Service<zn::Request, Response = zn::Response, Error = BoxError> + Send + Clone + 'static,
     ZN::Future: Send,
     ZV: Service<Arc<Block>, Response = block::Hash, Error = BoxError> + Send + Clone + 'static,
     ZV::Future: Send,
-    ZS: Service<zs::Request, Response = zs::Response, Error = BoxError> + Send + 'static,
+    ZS: Service<zs::Request, Response = zs::Response, Error = BoxError> + Send + Clone + 'static,
     ZS::Future: Send,
 {
     network: ZN,
@@ -44,11 +43,11 @@ where
 
 impl<ZN, ZV, ZS> Stream for Downloads<ZN, ZV, ZS>
 where
-    ZN: Service<zn::Request, Response = zn::Response, Error = BoxError> + Send + 'static,
+    ZN: Service<zn::Request, Response = zn::Response, Error = BoxError> + Send + Clone + 'static,
     ZN::Future: Send,
     ZV: Service<Arc<Block>, Response = block::Hash, Error = BoxError> + Send + Clone + 'static,
     ZV::Future: Send,
-    ZS: Service<zs::Request, Response = zs::Response, Error = BoxError> + Send + 'static,
+    ZS: Service<zs::Request, Response = zs::Response, Error = BoxError> + Send + Clone + 'static,
     ZS::Future: Send,
 {
     type Item = Result<block::Hash, BoxError>;

From 05837797b1b9ea58fd92d3eff36e99bed28d1193 Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Tue, 24 Nov 2020 14:53:57 -0800
Subject: [PATCH 142/148] tidy imports

---
 zebrad/src/components/inbound.rs | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/zebrad/src/components/inbound.rs b/zebrad/src/components/inbound.rs
index cb81e0a4faf..6fcd0f8dd18 100644
--- a/zebrad/src/components/inbound.rs
+++ b/zebrad/src/components/inbound.rs
@@ -12,21 +12,19 @@ use futures::{
 use tokio::sync::oneshot;
 use tower::{buffer::Buffer, util::BoxService, Service, ServiceExt};
 
-use zebra_chain as zc;
-use zebra_consensus as zcon;
 use zebra_network as zn;
-use zebra_network::AddressBook;
 use zebra_state as zs;
 
+use zebra_chain::block::{self, Block};
+use zebra_consensus::chain::VerifyChainError;
+use zebra_network::AddressBook;
+
 mod downloads;
 use downloads::Downloads;
 
 type Outbound = Buffer<BoxService<zn::Request, zn::Response, zn::BoxError>, zn::Request>;
 type State = Buffer<BoxService<zs::Request, zs::Response, zs::BoxError>, zs::Request>;
-type Verifier = Buffer<
-    BoxService<Arc<zc::block::Block>, zc::block::Hash, zcon::chain::VerifyChainError>,
-    Arc<zc::block::Block>,
->;
+type Verifier = Buffer<BoxService<Arc<Block>, block::Hash, VerifyChainError>, Arc<Block>>;
 
 pub type SetupData = (Outbound, Arc<Mutex<AddressBook>>);
 

From de8415dcb1ab36f449b156b89aad1462af0fadea Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Tue, 24 Nov 2020 16:05:11 -0800
Subject: [PATCH 143/148] tidy spans

---
 zebrad/src/components/inbound.rs           | 3 +--
 zebrad/src/components/inbound/downloads.rs | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/zebrad/src/components/inbound.rs b/zebrad/src/components/inbound.rs
index 6fcd0f8dd18..e190cbbd287 100644
--- a/zebrad/src/components/inbound.rs
+++ b/zebrad/src/components/inbound.rs
@@ -86,7 +86,6 @@ impl Service<zn::Request> for Inbound {
     type Future =
         Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send + 'static>>;
 
-    #[instrument(skip(self, cx))]
     fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
         // Check whether the network setup is finished, but don't wait for it to
         // become ready before reporting readiness. We expect to get it "soon",
@@ -139,7 +138,7 @@ impl Service<zn::Request> for Inbound {
         }
     }
 
-    #[instrument(skip(self))]
+    #[instrument(name = "inbound", skip(self, req))]
     fn call(&mut self, req: zn::Request) -> Self::Future {
         match req {
             zn::Request::Peers => match self.address_book.as_ref() {
diff --git a/zebrad/src/components/inbound/downloads.rs b/zebrad/src/components/inbound/downloads.rs
index 3f82f51c393..65d9b25ef8a 100644
--- a/zebrad/src/components/inbound/downloads.rs
+++ b/zebrad/src/components/inbound/downloads.rs
@@ -104,7 +104,7 @@ where
     /// Queue a block for download and verification.
     ///
     /// Returns true if the block was newly queued, and false if it was already queued.
-    #[instrument(skip(self))]
+    #[instrument(skip(self, hash), fields(hash = %hash))]
     pub fn download_and_verify(&mut self, hash: block::Hash) -> bool {
         if self.cancel_handles.contains_key(&hash) {
             tracing::debug!("hash already queued for download");

From fa02b266cac021d6b69c6fee1a480d47f5ae4f59 Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Tue, 24 Nov 2020 16:25:35 -0800
Subject: [PATCH 144/148] clippy

---
 zebrad/src/components/inbound/downloads.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/zebrad/src/components/inbound/downloads.rs b/zebrad/src/components/inbound/downloads.rs
index 65d9b25ef8a..d1ce7a570bf 100644
--- a/zebrad/src/components/inbound/downloads.rs
+++ b/zebrad/src/components/inbound/downloads.rs
@@ -120,7 +120,7 @@ where
 
         let fut = async move {
             // Check if the block is already in the state.
-            match state.oneshot(zs::Request::Depth(hash.into())).await {
+            match state.oneshot(zs::Request::Depth(hash)).await {
                 Ok(zs::Response::Depth(None)) => Ok(()),
                 Ok(zs::Response::Depth(Some(_))) => Err("already present".into()),
                 Ok(_) => unreachable!("wrong response"),

From e11e8e1373092e77bd2a91cd7f76ce9f39b8179e Mon Sep 17 00:00:00 2001
From: Deirdre Connolly <deirdre@zfnd.org>
Date: Wed, 25 Nov 2020 17:12:01 -0500
Subject: [PATCH 145/148] s/TRASPARENT/TRANSPARENT/g

---
 zebra-state/src/constants.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/zebra-state/src/constants.rs b/zebra-state/src/constants.rs
index 42d9b648c50..888ca8b4058 100644
--- a/zebra-state/src/constants.rs
+++ b/zebra-state/src/constants.rs
@@ -3,13 +3,13 @@
 /// A transaction MUST NOT spend a transparent output of a coinbase transaction
 /// from a block less than 100 blocks prior to the spend. Note that transparent
 /// outputs of coinbase transactions include Founders' Reward outputs.
-pub const MIN_TRASPARENT_COINBASE_MATURITY: u32 = 100;
+pub const MIN_TRANSPARENT_COINBASE_MATURITY: u32 = 100;
 
 /// The maximum chain reorganisation height.
 ///
 /// Allowing reorganisations past this height could allow double-spends of
 /// coinbase transactions.
-pub const MAX_BLOCK_REORG_HEIGHT: u32 = MIN_TRASPARENT_COINBASE_MATURITY - 1;
+pub const MAX_BLOCK_REORG_HEIGHT: u32 = MIN_TRANSPARENT_COINBASE_MATURITY - 1;
 
 /// The database format version, incremented each time the database format changes.
 pub const DATABASE_FORMAT_VERSION: u32 = 3;

From fc7d37c984f06c97ab161b1572cdd5108fc96898 Mon Sep 17 00:00:00 2001
From: teor <teor@riseup.net>
Date: Thu, 26 Nov 2020 11:45:01 +1000
Subject: [PATCH 146/148] RFC: Contextual Difficulty (#1246)

* Difficulty Contextual RFC: Introduction
Add a header, summary, and motivation

* Difficulty RFC: Add draft definitions
And update the state RFC definitions to match.

* Difficulty RFC: Add relevant chain
* Difficulty RFC: draft guide-level explanation
Outline the core calculations and checks.

* Difficulty RFC: Revised based on spec fixes
Update the design based on the spec bugs in #1276, #1277, and
zcash/zips#416.

These changes make the difficulty filter into a context-free check,
so we remove it from this contextual validation RFC.

* Difficulty RFC: Explain how Zebra's calculations can match the spec
* Difficulty RFC: write most of the reference section
Includes most of the implementation, modules for each function, and
draft notes for some of the remaining parts of the RFC.

* Difficulty RFC: Add an AdjustedDifficulty struct
* Difficulty RFC: Summarise module structure in the one place
* Difficulty RFC: Create implementation notes subsections
* Difficulty RFC: add consensus critical order of operations
* Difficulty RFC: Use the ValidateContextError type
* Difficulty RFC: make the median_time arg mut owned

We have to clone the data to pass a fixed-length array to a function,
so we might as well sort that array to find the median, and avoid a
copy.
---
 book/src/dev/rfcs/0005-state-updates.md       |  72 +-
 .../dev/rfcs/0006-contextual-difficulty.md    | 784 ++++++++++++++++++
 2 files changed, 851 insertions(+), 5 deletions(-)
 create mode 100644 book/src/dev/rfcs/0006-contextual-difficulty.md

diff --git a/book/src/dev/rfcs/0005-state-updates.md b/book/src/dev/rfcs/0005-state-updates.md
index f72ce35611d..4e95e0dcabd 100644
--- a/book/src/dev/rfcs/0005-state-updates.md
+++ b/book/src/dev/rfcs/0005-state-updates.md
@@ -34,15 +34,20 @@ state service.
 
 * **block chain**: A sequence of valid blocks linked by inclusion of the
   previous block hash in the subsequent block. Chains are rooted at the
-  *genesis* block and extend to a *tip*.
+  genesis block and extend to a **tip**.
 
 * **chain state**: The state of the ledger after application of a particular
   sequence of blocks (state transitions).
 
-* **cumulative difficulty**: The cumulative proof-of-work from genesis to the
-  chain tip.
+* **block work**: The approximate amount of work required for a miner to generate
+  a block hash that passes the difficulty filter. The number of block header
+  attempts and the mining time are proportional to the work value. Numerically
+  higher work values represent longer processing times.
 
-* **best chain**: The chain with the greatest cumulative difficulty. This chain
+* **cumulative work**: The sum of the **block work** of all blocks in a chain, from
+  genesis to the chain tip.
+
+* **best chain**: The chain with the greatest **cumulative work**. This chain
   represents the consensus state of the Zcash network and transactions.
 
 * **side chain**: A chain which is not contained in the best chain.
@@ -70,6 +75,11 @@ state service.
   be lower during the initial sync, and after a chain reorganization, if the new
   best chain is at a lower height.
 
+* **relevant chain**: The relevant chain for a block starts at the previous
+  block, and extends back to genesis.
+
+* **relevant tip**: The tip of the relevant chain.
+
 # Guide-level explanation
 [guide-level-explanation]: #guide-level-explanation
 
@@ -230,7 +240,6 @@ chains, so that the map ordering is the ordering of worst to best chains.
 ### The `Chain` type
 [chain-type]: #chain-type
 
-
 The `Chain` type represents a chain of blocks. Each block represents an
 incremental state update, and the `Chain` type caches the cumulative state
 update from its root to its tip.
@@ -706,6 +715,59 @@ These updates can be performed in a batch or without necessarily iterating
 over all transactions, if the data is available by other means; they're
 specified this way for clarity.
 
+## Accessing previous blocks for contextual validation
+[previous-block-context]: #previous-block-context
+
+The state service performs contextual validation of blocks received via the
+`CommitBlock` request. Since `CommitBlock` is synchronous, contextual validation
+must also be performed synchronously.
+
+The relevant chain for a block starts at its previous block, and follows the
+chain of previous blocks back to the genesis block.
+
+### Relevant chain iterator
+[relevant-chain-iterator]: #relevant-chain-iterator
+
+The relevant chain can be retrieved from the state service as follows:
+* if the previous block is the finalized tip:
+  * get recent blocks from the finalized state
+* if the previous block is in the non-finalized state:
+  * get recent blocks from the relevant chain, then
+  * get recent blocks from the finalized state, if required
+
+The relevant chain can start at any non-finalized block, or at the finalized tip.
+
+### Relevant chain implementation
+[relevant-chain-implementation]: #relevant-chain-implementation
+
+The relevant chain is implemented as a `StateService` iterator, which returns
+`Arc<Block>`s.
+
+The chain iterator implements `ExactSizeIterator`, so Zebra can efficiently
+assert that the relevant chain contains enough blocks to perform each contextual
+validation check.
+
+```rust
+impl StateService {
+    /// Return an iterator over the relevant chain of the block identified by
+    /// `hash`.
+    ///
+    /// The block identified by `hash` is included in the chain of blocks yielded
+    /// by the iterator.
+    pub fn chain(&self, hash: block::Hash) -> Iter<'_> { ... }
+}
+
+impl Iterator for Iter<'_>  {
+    type Item = Arc<Block>;
+    ...
+}
+impl ExactSizeIterator for Iter<'_> { ... }
+impl FusedIterator for Iter<'_> {}
+```
+
+For further details, see [PR 1271].
+
+[PR 1271]: https://github.com/ZcashFoundation/zebra/pull/1271
 
 ## Request / Response API
 [request-response]: #request-response
diff --git a/book/src/dev/rfcs/0006-contextual-difficulty.md b/book/src/dev/rfcs/0006-contextual-difficulty.md
new file mode 100644
index 00000000000..92808097327
--- /dev/null
+++ b/book/src/dev/rfcs/0006-contextual-difficulty.md
@@ -0,0 +1,784 @@
+- Feature Name: contextual_difficulty_validation
+- Start Date: 2020-11-02
+- Design PR: [ZcashFoundation/zebra#0000](https://github.com/ZcashFoundation/zebra/pull/0000)
+- Zebra Issue: [ZcashFoundation/zebra#1036](https://github.com/ZcashFoundation/zebra/issues/1036)
+
+# Summary
+[summary]: #summary
+
+Zcash nodes use a Proof of Work algorithm to reach consensus on the best chain.
+Valid blocks must reach a difficulty threshold, which is adjusted after every
+block. The difficulty adjustment calculations depend on the difficulties and
+times of recent blocks. So Zebra performs contextual validation [RFC2] of
+difficulty adjustments as part of committing blocks to the state.
+
+[RFC2]: ./0002-parallel-verification.md
+
+# Motivation
+[motivation]: #motivation
+
+The Zcash block difficulty adjustment is one of the core Zcash consensus rules.
+Zebra must implement this consensus rule to make sure that its cached chain
+state is consistent with the consensus of Zcash nodes.
+
+Difficulty adjustment is also a significant part of Zcash's security guarantees.
+It ensures that the network continues to resist takeover attacks, even as the
+number of Zcash miners grows.
+
+Difficulty adjustment also ensures that blocks are regularly spaced, which
+allows users to create and finalise transactions with short, consistent delays.
+These predictable delays contribute to Zcash's usability.
+
+# Definitions
+[definitions]: #definitions
+
+Difficulty:
+* **hash difficulty**: An arbitrary ranking of blocks, based on their hashes.
+  Defined as the hash of the block, interpreted as a big-endian 256-bit number.
+  Numerically smaller difficulties are harder to generate.
+
+* **difficulty threshold**: The easiest valid hash difficulty for a block.
+  Numerically lower thresholds are harder to satisfy.
+
+* **difficulty filter**: A block passes the difficulty filter if the hash
+  difficulty is less than or equal to the difficulty threshold (based on the
+  block's difficulty field).
+
+* **block work**: The approximate amount of work required for a miner to generate
+  a block hash that passes the difficulty filter. The number of block header
+  attempts and the mining time are proportional to the work value. Numerically
+  higher work values represent longer processing times.
+
+* **averaging window**: The 17 most recent blocks in the relevant chain.
+
+* **median block span**: The 11 most recent blocks from a chosen tip, typically
+  the relevant tip.
+
+* **target spacing**: 150 seconds per block before Blossom activation, 75 seconds
+  per block from Blossom activation onwards.
+
+* **adjusted difficulty**: After each block is mined, the difficulty threshold of
+  the next block is adjusted, to keep the block gap close to the target spacing.
+
+* **mean target difficulty**: The arithmetic mean of the difficulty thresholds
+  of the blocks in the averaging window.
+
+* **median timespan**: The average number of seconds taken to generate the blocks
+  in the averaging window. Calculated using the difference of median block spans
+  in and after the averaging window, then damped and bounded.
+
+* **target timespan**: The target spacing for an averaging window's worth of
+  blocks.
+
+Consensus:
+* **consensus rule:** A protocol rule which all nodes must apply consistently,
+                      so they can converge on the same chain fork.
+
+* **structural/semantic/contextual verification**: as defined in [RFC2].
+
+State:
+* **block chain**: A sequence of valid blocks linked by inclusion of the
+  previous block hash in the subsequent block. Chains are rooted at the
+  genesis block and extend to a tip.
+
+* **relevant chain**: The relevant chain for a block starts at the previous
+  block, and extends back to genesis.
+
+* **relevant tip**: The tip of the relevant chain.
+
+* **non-finalized state**: State data corresponding to blocks above the reorg
+  limit. This data can change in the event of a chain reorg.
+
+* **finalized state**: State data corresponding to blocks below the reorg
+  limit. This data cannot change in the event of a chain reorg.
+
+* **non-finalized tips**: The highest blocks in each non-finalized chain. These
+  tips might be at different heights.
+
+* **finalized tip**: The highest block in the finalized state. The tip of the best
+  chain is usually 100 blocks (the reorg limit) above the finalized tip. But it can
+  be lower during the initial sync, and after a chain reorganization, if the new
+  best chain is at a lower height.
+
+# Guide-level explanation
+[guide-level-explanation]: #guide-level-explanation
+
+Zcash's difficulty consensus rules are similar to Bitcoin.
+
+Each block contains a **difficulty threshold** in its header. The hash of the
+block header must be less than this **difficulty threshold**. (When interpreted
+as a 256-bit integer in big-endian byte order.) This context-free semantic
+verification check is performed by the `BlockVerifier`.
+
+After each block, the difficulty threshold is adjusted so that the block gap is
+close to the target spacing. On average, harder blocks take longer to mine, and
+easier blocks take less time.
+
+The **adjusted difficulty** for the next block is calculated using the difficulty
+thresholds and times of recent blocks. Zcash uses the most recent 28 blocks in
+the **relevant chain** in its difficulty adjustment calculations.
+
+The difficulty adjustment calculations adjust the **mean target difficulty**,
+based on the difference between the **median timespan** and the
+**target timespan**. If the median timespan is less than the target timespan, the
+next block is harder to mine.
+
+The `StateService` calculates the adjusted difficulty using the context from the
+**relevant chain**. The difficulty contextual verification check ensures that the
+**difficulty threshold** of the next block is equal to the **adjusted difficulty**
+for its relevant chain.
+
+## State service interface changes
+[state-service-interface]: #state-service-interface
+
+Contextual validation accesses recent blocks. So we modify the internal state
+service interface to provide an abstraction for accessing recent blocks.
+
+### The relevant chain
+[relevant-chain]: #relevant-chain
+
+The relevant chain consists of the ancestors of a block, starting with its
+parent block, and extending back to the genesis block.
+
+In Zebra, recent blocks are part of the non-finalized state, which can contain
+multiple chains. Past the reorganization limit, Zebra commits a single chain to
+the finalized state.
+
+The relevant chain can start at any block in the non-finalized state, or at the
+finalized tip. See [RFC5] for details.
+
+[RFC5]: ./0005-state-updates.md
+
+## Contextual validation design
+[contextual-validation-design]: #contextual-validation-design
+
+Contextual validation is performed synchronously by the state service, as soon
+as the state has:
+* received the semantically valid next block (via `CommitBlock`), and
+* committed the previous block.
+
+The difficulty adjustment check calculates the correct adjusted difficulty
+threshold value for a candidate block, and ensures that the block's
+`difficulty_threshold` field is equal to that value.
+
+This check is implemented as follows:
+
+### Difficulty adjustment
+[difficulty-adjustment]: #difficulty-adjustment
+
+The block difficulty threshold is adjusted by scaling the mean target difficulty
+by the median timespan.
+
+On Testnet, if a long time has elapsed since the previous block, the difficulty
+adjustment is modified to allow minimum-difficulty blocks.
+
+#### Mean target difficulty
+[mean-target-difficulty]: #mean-target-difficulty
+
+The mean target difficulty is the arithmetic mean of the difficulty
+thresholds of the `PoWAveragingWindow` (17) most recent blocks in the relevant
+chain.
+
+Zcash uses block difficulty thresholds in its difficulty adjustment calculations.
+(Block hashes are not used for difficulty adjustment.)
+
+#### Median timespan
+[median-timespan]: #median-timespan
+
+The average number of seconds taken to generate the 17 blocks in the averaging
+window.
+
+The median timespan is calculated by taking the difference of the median times
+for:
+* the relevant tip: the `PoWMedianBlockSpan` (11) most recent blocks, and
+* the 11 blocks after the 17-block `PoWAveragingWindow`: that is, blocks 18-28
+  behind the relevant tip.
+
+The median timespan is damped by the `PoWDampingFactor`, and bounded by
+`PoWMaxAdjustDown` and `PoWMaxAdjustUp`.
+
+#### Test network minimum difficulty blocks
+[test-net-min-difficulty]: #test-net-min-difficulty
+
+If there is a large gap after a Testnet block, the next block becomes a minimum
+difficulty block. Testnet minimum difficulty blocks have their
+`difficulty_threshold` set to the minimum difficulty for Testnet.
+
+#### Block difficulty threshold
+[block-difficulty-threshold]: #block-difficulty-threshold
+
+The block difficulty threshold for the next block is calculated by scaling the
+mean target difficulty by the ratio between the median timespan and the averaging
+window timespan.
+
+The result of this calculation is limited by `ToCompact(PoWLimit(network))`, a
+per-network minimum block difficulty. This minimum difficulty is also used when
+a Testnet block's time gap exceeds the minimum difficulty gap.
+
+# Reference-level explanation
+[reference-level-explanation]: #reference-level-explanation
+
+## Contextual validation
+[contextual-validation]: #contextual-validation
+
+Contextual validation is implemented in
+`StateService::check_contextual_validity`, which calls a separate function for
+each contextual validity check.
+
+In Zebra, contextual validation starts after Sapling activation, so we can assume
+that the relevant chain contains at least 28 blocks on Mainnet and Testnet. (And
+panic if this assumption does not hold at runtime.)
+
+## Fundamental data types
+[fundamental-data-types]: #fundamental-data-types
+
+Zebra is free to implement its difficulty calculations in any way that produces
+equivalent results to `zcashd` and the Zcash specification.
+
+### Difficulty
+
+In Zcash block headers, difficulty thresholds are stored as a "compact" `nBits`
+value, which uses a custom 32-bit floating-point encoding. Zebra calls this type
+`CompactDifficulty`.
+
+In Zcash, difficulty threshold calculations are performed using unsigned 256-bit
+integers. Rust has no standard `u256` type, but there are a number of crates
+available which implement the required operations on 256-bit integers. Zebra
+abstracts over the chosen `u256` implementation using its `ExpandedDifficulty`
+type.
+
+### Time
+
+In Zcash, time values are unsigned 32-bit integers. But the difficulty adjustment
+calculations include time subtractions which could overflow an unsigned type, so
+they are performed using signed 64-bit integers in `zcashd`.
+
+Zebra parses the `header.time` field into a `DateTime<Utc>`. Conveniently, the
+`chrono::DateTime<_>::timestamp()` function returns `i64` values. So Zebra can do
+its signed time calculations using `i64` values internally.
+
+Note: `i32` is an unsuitable type for signed time calculations. It is
+theoretically possible for the time gap between blocks to be larger than
+`i32::MAX`, because those times are provided by miners. Even if the median time
+gap is that large, the bounds and minimum difficulty in Zcash's difficulty
+adjustment algorithm will preserve a reasonable difficulty threshold. So Zebra
+must support this edge case.
+
+### Consensus-Critical Operations
+
+The order of operations and overflow semantics for 256-bit integers can be
+consensus-critical.
+
+For example:
+  - dividing before multiplying discards lower-order bits, but
+  - multiplying before dividing can cause overflow.
+
+Zebra's implementation should try to match zcashd's order of operations and
+overflow handling as closely as possible.
+
+## Difficulty adjustment check
+[difficulty-adjustment-check]: #difficulty-adjustment-check
+
+The difficulty adjustment check calculates the correct difficulty threshold
+value for a candidate block, and ensures that the block's
+`difficulty_threshold` field is equal to that value.
+
+### Context data type
+[context-data-type]: #context-data-type
+
+The difficulty adjustment functions use a context consisting of the difficulties
+and times from the previous 28 blocks in the relevant chain.
+
+These functions also use the candidate block's `height` and `network`.
+
+To make these functions more ergonomic, we create a `AdjustedDifficulty`
+type, and implement the difficulty adjustment calculations as methods on that
+type.
+
+```rust
+/// The averaging window for difficulty threshold arithmetic mean calculations.                               
+///                                                                                                           
+/// `PoWAveragingWindow` in the Zcash specification.                                                          
+pub const POW_AVERAGING_WINDOW: usize = 17;
+
+/// The median block span for time median calculations.                                                       
+///                                                                                                           
+/// `PoWMedianBlockSpan` in the Zcash specification.                                                          
+pub const POW_MEDIAN_BLOCK_SPAN: usize = 11;
+
+/// Contains the context needed to calculate the adjusted difficulty for a block. 
+struct AdjustedDifficulty {
+    candidate_time: DateTime<Utc>,
+    candidate_height: block::Height,
+    network: Network,
+    relevant_difficulty_thresholds: [CompactDifficulty; POW_AVERAGING_WINDOW + POW_MEDIAN_BLOCK_SPAN],
+    relevant_times: [DateTime<Utc>; POW_AVERAGING_WINDOW + POW_MEDIAN_BLOCK_SPAN],
+}
+```
+
+We implement some initialiser methods on `AdjustedDifficulty` for convenience.
+We might want to validate downloaded headers in future, so we include a
+`new_from_header` initialiser.
+
+```rust
+/// Initialise and return a new `AdjustedDifficulty` using a `candidate_block`,
+/// `network`, and a `context`.
+///
+/// The `context` contains the previous
+/// `PoWAveragingWindow + PoWMedianBlockSpan` (28) `difficulty_threshold`s and
+/// `time`s from the relevant chain for `candidate_block`, in reverse height
+/// order, starting with the previous block.
+///
+/// Note that the `time`s might not be in reverse chronological order, because
+/// block times are supplied by miners.
+///
+/// Panics:
+/// If the `context` contains fewer than 28 items.
+pub fn new_from_block<C>(candidate_block: &Block,
+                         network: Network,
+                         context: C)
+                         -> AdjustedDifficulty
+    where
+        C: IntoIterator<Item = (CompactDifficulty, DateTime<Utc>)>,
+    { ... }
+
+/// Initialise and return a new `AdjustedDifficulty` using a
+/// `candidate_header`, `previous_block_height`, `network`, and a `context`.
+///
+/// Designed for use when validating block headers, where the full block has not
+/// been downloaded yet.
+///
+/// See `new_from_block` for detailed information about the `context`.
+///
+/// Panics:
+/// If the context contains fewer than 28 items.
+pub fn new_from_header<C>(candidate_header: &block::Header,
+                          previous_block_height: block::Height,
+                          network: Network,
+                          context: C)
+                          -> AdjustedDifficulty
+    where
+        C: IntoIterator<Item = (CompactDifficulty, DateTime<Utc>)>,
+    { ... }
+```
+
+#### Memory usage note
+
+Copying `CompactDifficulty` values into the `AdjustedDifficulty` struct uses
+less memory than borrowing those values. `CompactDifficulty` values are 32 bits,
+but pointers are 64-bit on most modern machines. (And since they all come from
+different blocks, we need a pointer to each individual value.)
+
+Borrowing `DateTime<Utc>` values might use slightly less memory than copying
+them - but that depends on the exact way that Rust stores associated types
+derived from a generic argument.
+
+In any case, the overall size of each `AdjustedDifficulty` is only a few
+hundred bytes. If it turns up in profiles, we can look at borrowing the block
+header data.
+
+### Difficulty adjustment check implementation
+[difficulty-adjustment-check-implementation]: #difficulty-adjustment-check-implementation
+
+The difficulty adjustment check ensures that the
+`candidate_difficulty_threshold` is equal to the `difficulty_threshold` value
+calculated using `AdjustedDifficulty::adjusted_difficulty_threshold`.
+
+We implement this function:
+```rust
+/// Validate the `difficulty_threshold` from a candidate block's header, based
+/// on an `expected_difficulty` for that block.
+///
+/// Uses `expected_difficulty` to calculate the expected `ToCompact(Threshold())`
+/// value, then compares that value to the `difficulty_threshold`. Returns
+/// `Ok(())` if the values are equal.
+pub fn difficulty_threshold_is_valid(difficulty_threshold: CompactDifficulty,
+                                     expected_difficulty: AdjustedDifficulty)
+                                     -> Result<(), ValidateContextError> { ... }
+```
+
+[Issue 1166]: https://github.com/ZcashFoundation/zebra/issues/1166
+
+### Mean target difficulty calculation
+[mean-target-difficulty-calculation]: #mean-target-difficulty-calculation
+
+The mean target difficulty is the arithmetic mean of the difficulty
+thresholds of the `PoWAveragingWindow` (17) most recent blocks in the relevant
+chain.
+
+We implement this method on `AdjustedDifficulty`:
+```rust
+/// Calculate the arithmetic mean of the averaging window thresholds: the
+/// expanded `difficulty_threshold`s from the previous `PoWAveragingWindow` (17)
+/// blocks in the relevant chain.
+///
+/// Implements `MeanTarget` from the Zcash specification.
+fn mean_target_difficulty(&self) -> ExpandedDifficulty { ... }
+```
+
+#### Implementation notes
+
+Since the `PoWLimit`s are `2^251 − 1` for Testnet, and `2^243 − 1` for Mainnet,
+the sum of these difficulty thresholds will be less than or equal to
+`(2^251 − 1)*17 = 2^255 + 2^251 - 17`. Therefore, this calculation can not
+overflow a `u256` value. So the function is infalliable.
+
+In Zebra, contextual validation starts after Sapling activation, so we can assume
+that the relevant chain contains at least 17 blocks. Therefore, the `PoWLimit`
+case of `MeanTarget()` in the Zcash specification is unreachable.
+
+### Median timespan calculation
+[median-timespan-calculation]: #median-timespan-calculation
+
+The median timespan is the difference of the median times for:
+* the relevant tip: the `PoWMedianBlockSpan` (11) most recent blocks, and
+* the 11 blocks after the 17-block `PoWAveragingWindow`: that is, blocks 18-28
+  behind the relevant tip.
+
+(The median timespan is known as the `ActualTimespan` in the Zcash specification,
+but this terminology is confusing, because it is a difference of medians, rather
+than any "actual" elapsed time.)
+
+Zebra implements the median timespan using the following methods on
+`AdjustedDifficulty`:
+```rust
+/// Calculate the bounded median timespan. The median timespan is the
+/// difference of medians of the timespan times, which are the `time`s from
+/// the previous `PoWAveragingWindow + PoWMedianBlockSpan` (28) blocks in the
+/// relevant chain.
+///
+/// Uses the candidate block's `height' and `network` to calculate the
+/// `AveragingWindowTimespan` for that block.
+///
+/// The median timespan is damped by the `PoWDampingFactor`, and bounded by
+/// `PoWMaxAdjustDown` and `PoWMaxAdjustUp`.
+///
+/// Implements `ActualTimespanBounded` from the Zcash specification.
+///
+/// Note: This calculation only uses `PoWMedianBlockSpan` (11) times at the
+/// start and end of the timespan times. timespan times `[11..=16]` are ignored.
+fn median_timespan_bounded(&self) -> Duration { ... }
+
+/// Calculate the median timespan. The median timespan is the difference of
+/// medians of the timespan times, which are the `time`s from the previous
+/// `PoWAveragingWindow + PoWMedianBlockSpan` (28) blocks in the relevant chain.
+///
+/// Implements `ActualTimespan` from the Zcash specification.
+///
+/// See `median_timespan_bounded` for details.
+fn median_timespan(&self) -> Duration { ... }
+
+/// Calculate the median of the `median_block_span_times`: the `time`s from a
+/// slice of `PoWMedianBlockSpan` (11) blocks in the relevant chain.
+///
+/// Implements `MedianTime` from the Zcash specification.
+fn median_time(mut median_block_span_times: [DateTime<Utc>; POW_MEDIAN_BLOCK_SPAN])
+               -> DateTime<Utc> { ... }
+```
+
+Zebra implements the `AveragingWindowTimespan` using the following methods on
+`NetworkUpgrade`:
+```rust
+impl NetworkUpgrade {
+    /// Returns the `AveragingWindowTimespan` for the network upgrade.
+    pub fn averaging_window_timespan(&self) -> Duration { ... }
+
+    /// Returns the `AveragingWindowTimespan` for `network` and `height`.
+    pub fn averaging_window_timespan_for_height(network: Network,
+                                                height: block::Height)
+                                                -> Duration { ... }
+}
+```
+
+#### Implementation notes
+
+In Zebra, contextual validation starts after Sapling activation, so we can assume
+that the relevant chain contains at least 28 blocks. Therefore:
+* `max(0, height − PoWMedianBlockSpan)` in the `MedianTime()` calculation
+   simplifies to `height − PoWMedianBlockSpan`, and
+* there is always an odd number of blocks in `MedianTime()`, so the median is
+  always the exact middle of the sequence.
+
+Therefore, the function is infalliable.
+
+### Test network minimum difficulty calculation
+[test-net-min-difficulty-calculation]: #test-net-min-difficulty-calculation
+
+A block is a Testnet minimum difficulty block if:
+* the block is a Testnet block,
+* the block's height is 299188 or greater, and
+* the time gap from the previous block is greater than the Testnet minimum
+  difficulty gap, which is 6 times the target spacing for the block's height.
+  (The target spacing was halved from the Blossom network upgrade onwards.)
+
+The difficulty adjustment is modified for Testnet minimum difficulty blocks as
+follows:
+* the difficulty threshold in the block header is set to the Testnet minimum
+  difficulty threshold, `ToCompact(PoWLimit(network))`.
+
+Since the new difficulty changes the block header, Testnet blocks can only
+satisfy one of the alternate difficulty adjustment rules:
+* if the time gap is less than or equal to the Testnet minimum difficulty gap:
+  the difficulty threshold is calculated using the default difficulty adjustment
+  rule,
+* if the time gap is greater than the Testnet minimum difficulty gap:
+  the difficulty threshold is the Testnet minimum difficulty threshold.
+
+See [ZIP-208] for details.
+
+Note: some older versions of ZIPs 205 and 208 incorrectly said that:
+* the time gap threshold uses an "at least" check (it is strictly greater than),
+* the minimum difficulty threshold value was `PoWLimit`
+  (it is `ToCompact(PoWLimit)`),
+* the `difficulty_threshold` (`nBits`) field is not modified in Testnet minimum
+  difficulty blocks (the field is modified), and
+* the Testnet minimum difficulty value is not used to calculate future difficulty
+  adjustments (the modified value is used in future adjustments).
+
+ZIP 205 and 208 were fixed on 14 November 2020, see [ZIP PR 417] and
+[ZIP commit 806076c] for details.
+
+[ZIP-208]: https://zips.z.cash/zip-0208#minimum-difficulty-blocks-on-the-test-network
+[ZIP PR 417]: https://github.com/zcash/zips/pull/417
+[ZIP commit 806076c]: https://github.com/zcash/zips/commit/806076c48c9834fd9941b940a32310d737975a3a
+
+#### Test network minimum difficulty implementation
+[test-net-min-difficulty-implementation]: #test-net-min-difficulty-implementation
+
+The Testnet minimum difficulty calculation uses the existing
+`NetworkUpgrade::minimum_difficulty_spacing_for_height` function to calculate the
+minimum difficulty gap.
+
+We implement this method on `NetworkUpgrade`:
+```rust
+/// Returns true if the gap between `block_time` and `previous_block_time` is                             
+/// greater than the Testnet minimum difficulty time gap. This time gap                                   
+/// depends on the `network` and `block_height`.                                                          
+///                                                                                                       
+/// Returns false on Mainnet, when `block_height` is less than the minimum                                
+/// difficulty start height, and when the time gap is too small.                                          
+///                                                                                                       
+/// `block_time` can be less than, equal to, or greater than                                              
+/// `previous_block_time`, because block times are provided by miners.                                    
+///                                                                                                       
+/// Implements the Testnet minimum difficulty adjustment from ZIPs 205 and 208.                           
+///                                                                                                       
+/// Spec Note: Some parts of ZIPs 205 and 208 previously specified an incorrect                           
+/// check for the time gap. This function implements the correct "greater than"                           
+/// check.                                                                                                
+pub fn is_testnet_min_difficulty_block(
+    network: Network,
+    block_height: block::Height,
+    block_time: DateTime<Utc>,
+    previous_block_time: DateTime<Utc>,
+) -> bool { ... }
+```
+
+#### Implementation notes
+
+In Zcash, the Testnet minimum difficulty rule starts at block 299188, and in
+Zebra, contextual validation starts after Sapling activation. So we can assume
+that there is always a previous block.
+
+Therefore, this function is infalliable.
+
+### Block difficulty threshold calculation
+[block-difficulty-threshold-calculation]: #block-difficulty-threshold-calculation
+
+The block difficulty threshold for the next block is calculated by scaling the
+mean target difficulty by the ratio between the median timespan and the averaging
+window timespan.
+
+The result of the scaled threshold calculation is limited by
+`ToCompact(PoWLimit(network))`, a per-network minimum block difficulty. This
+minimum difficulty is also used when a Testnet block's time gap exceeds the
+minimum difficulty gap. We use the existing
+`ExpandedDifficulty::target_difficulty_limit` function to calculate the value of
+`ToCompact(PoWLimit(network))`.
+
+In Zebra, contextual validation starts after Sapling activation, so the genesis
+case of `Threshold()` in the Zcash specification is unreachable.
+
+#### Block difficulty threshold implementation
+[block-difficulty-threshold-implementation]: #block-difficulty-threshold-implementation
+
+We implement these methods on `AdjustedDifficulty`:
+```rust
+/// Calculate the expected `difficulty_threshold` for a candidate block, based
+/// on the `candidate_time`, `candidate_height`, `network`, and the
+/// `difficulty_threshold`s and `time`s from the previous
+/// `PoWAveragingWindow + PoWMedianBlockSpan` (28) blocks in the relevant chain.
+///
+/// Implements `ThresholdBits` from the Zcash specification, and the Testnet
+/// minimum difficulty adjustment from ZIPs 205 and 208.
+pub fn expected_difficulty_threshold(&self) -> CompactDifficulty { ... }
+
+/// Calculate the `difficulty_threshold` for a candidate block, based on the
+/// `candidate_height`, `network`, and the relevant `difficulty_threshold`s and
+/// `time`s.
+///
+/// See `expected_difficulty_threshold` for details.
+///
+/// Implements `ThresholdBits` from the Zcash specification. (Which excludes the
+/// Testnet minimum difficulty adjustment.)
+fn threshold_bits(&self) -> CompactDifficulty { ... }
+```
+
+#### Implementation notes
+
+Since:
+* the `PoWLimit`s are `2^251 − 1` for Testnet, and `2^243 − 1` for Mainnet,
+* the `ActualTimespanBounded` can be at most `MaxActualTimespan`, which is
+  `floor(PoWAveragingWindow * PoWTargetSpacing * (1 + PoWMaxAdjustDown))` or
+  `floor(17 * 150 * (1 + 32/100)) =  3366`,
+* `AveragingWindowTimespan` is at most `17 * 150 = 2250`, and
+* `MeanTarget` is at most `PoWLimit`, ...
+
+The maximum scaled value inside the `Threshold()` calculation is:
+* `floor(PoWLimit / 2250) * 3366`, which equals
+* `floor((2^251 − 1) / 2250) * 3366`, which equals
+* `(2^251 − 1) * 132/100`,
+* which is less than `2^252`.
+
+Therefore, this calculation can not overflow a `u256` value. (And even if it did
+overflow, it would be constrained to a valid value by the `PoWLimit` minimum.)
+
+Note that the multiplication by `ActualTimespanBounded` must happen after the
+division by `AveragingWindowTimespan`. Performing the multiplication first
+could overflow.
+
+If implemented in this way, the function is infalliable.
+
+`zcashd` truncates the `MeanTarget` after the mean calculation, and
+after dividing by `AveragingWindowTimespan`. But as long as there is no overflow,
+this is [equivalent to the single truncation of the final result] in the Zcash
+specification. However, Zebra should follow the order of operations in `zcashd`,
+and use repeated divisions, because that can't overflow. See the relevant
+[comment in the zcashd souce code].
+
+[equivalent to the single truncation of the final result]: https://math.stackexchange.com/questions/147771/rewriting-repeated-integer-division-with-multiplication
+[comment in the zcashd souce code]: https://github.com/zcash/zcash/pull/4860/files
+
+## Module Structure
+[module-structure]: #module-structure
+
+The structs and functions in this RFC are implemented in a new
+`zebra_state::service::check::difficulty` module.
+
+This module has two entry points:
+* `DifficultyAdjustment::new_from_block`
+* `difficulty_threshold_is_valid`
+
+These entry points are both called from
+`StateService::check_contextual_validity`.
+
+## Test Plan
+[test-plan]: #test-plan
+
+Explain how the feature will be tested, including:
+- [ ] tests for consensus-critical functionality
+- [ ] existing test vectors, if available
+- [ ] Zcash blockchain block test vectors (specify the network upgrade, feature, or block height and network)
+- [ ] property testing or fuzzing
+
+The tests should cover:
+- [ ] positive cases: make sure the feature accepts valid inputs
+    - using block test vectors for each network upgrade provides some coverage of valid inputs
+- [ ] negative cases: make sure the feature rejects invalid inputs
+    - make sure there is a test case for each error condition in the code
+    - if there are lots of potential errors, prioritise:
+        - consensus-critical errors
+        - security-critical errors, and
+        - likely errors
+- [ ] edge cases: make sure that boundary conditions are correctly handled
+
+# Drawbacks
+[drawbacks]: #drawbacks
+
+Why should we *not* do this?
+
+## Alternate consensus parameters
+[alternate-consensus-parameters]: #alternate-consensus-parameters
+
+Any alternate consensus parameters or `regtest` mode would have to respect the constraints set by this design.
+
+In particular:
+  * the `PoWLimit` must be less than or equal to
+    `(2^256 - 1) / PoWAveragingWindow` (approximately `2^251`) to avoid overflow,
+  * the `PoWAveragingWindow` and `PoWMedianBlockSpan` are fixed by function argument types
+    (at least until Rust gets stable const generics), and
+  * the design eliminates a significant number of edge cases by assuming that difficulty adjustments aren't
+    validated for the first `PoWAveragingWindow + PoWMedianBlockSpan` (28) blocks in the chain.
+
+# Rationale and alternatives
+[rationale-and-alternatives]: #rationale-and-alternatives
+
+## Is this design a good basis for later designs or implementations?
+[good-basis]: #good-basis
+
+The design includes specific methods for a future header-only validation design.
+
+## What other designs have been considered and what is the rationale for not choosing them?
+[alternate-designs]: #alternate-designs
+
+A previous version of the RFC did not have the `AdjustedDifficulty` struct and
+methods. That design was easy to misuse, because each function had a complicated
+argument list.
+
+## What is the impact of not doing this?
+[no-action]: #no-action
+
+Zebra could accept invalid, low-difficulty blocks from arbitrary miners. That
+would be a security issue.
+
+# Prior art
+[prior-art]: #prior-art
+
+* `zcashd`
+* the Zcash specification
+* Bitcoin
+
+# Unresolved questions
+[unresolved-questions]: #unresolved-questions
+
+- What parts of the design do you expect to resolve through the implementation of this feature before stabilization?
+  - Guide-level examples
+  - Reference-level examples
+  - Corner case examples
+  - Testing
+
+- What related issues do you consider out of scope for this RFC that could be addressed in the future independently of the solution that comes out of this RFC?
+  - Monitoring and maintainence
+
+# Future possibilities
+[future-possibilities]: #future-possibilities
+
+## Re-using the relevant chain API in other contextual checks
+[relevant-chain-api-reuse]: #relevant-chain-api-reuse
+
+The relevant chain iterator can be re-used to implement other contextual
+validation checks.
+
+For example, responding to peer requests for block locators, which means
+implementing relevant chain hash queries as a `StateService` request
+
+## Header-only difficulty adjustment validation
+[header-only-validation]: #header-only-validation
+
+Implementing header-only difficulty adjustment validation as a `StateService` request.
+
+## Caching difficulty calculations
+[caching-calculations]: #caching-calculations
+
+Difficulty calculations use `u256` could be a bit expensive, particularly if we
+get a flood of low-difficulty blocks. To reduce the impact of this kind of DoS,
+we could cache the value returned by `threshold_bits` for each block in the
+non-finalized state, and for the finalized tip. This value could be used to
+quickly calculate the difficulties for any child blocks of these blocks.
+
+There's no need to persist this cache, or pre-fill it. (Minimum-difficulty
+Testnet blocks don't call `threshold_bits`, and some side-chain blocks will
+never have a next block.)
+
+This caching is only worth implementing if these calculations show up in `zebrad`
+profiles.

From 176923a7713e4a2d0d3a59f7075b81dd21adde72 Mon Sep 17 00:00:00 2001
From: teor <teor@riseup.net>
Date: Thu, 26 Nov 2020 17:26:10 +1000
Subject: [PATCH 147/148] Add an info-level log when UTXO requests are pruned
 (#1396)

And a debug-level log when no requests are pruned.

I'm seeing some hangs during the initial sync, these logs might help
identify the cause.
---
 zebra-state/src/service.rs               | 19 +++++++++++++++++++
 zebra-state/src/service/pending_utxos.rs |  5 +++++
 2 files changed, 24 insertions(+)

diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs
index 07daf6c2395..bb368eaf786 100644
--- a/zebra-state/src/service.rs
+++ b/zebra-state/src/service.rs
@@ -374,8 +374,27 @@ impl Service<Request> for StateService {
         let now = Instant::now();
 
         if self.last_prune + Self::PRUNE_INTERVAL < now {
+            let tip = self.tip();
+            let old_len = self.pending_utxos.len();
+
             self.pending_utxos.prune();
             self.last_prune = now;
+
+            let new_len = self.pending_utxos.len();
+            let prune_count = old_len
+                .checked_sub(new_len)
+                .expect("prune does not add any utxo requests");
+            if prune_count > 0 {
+                tracing::info!(
+                    ?old_len,
+                    ?new_len,
+                    ?prune_count,
+                    ?tip,
+                    "pruned utxo requests"
+                );
+            } else {
+                tracing::debug!(len = ?old_len, ?tip, "no utxo requests needed pruning");
+            }
         }
 
         Poll::Ready(Ok(()))
diff --git a/zebra-state/src/service/pending_utxos.rs b/zebra-state/src/service/pending_utxos.rs
index 4ad38c2b061..277173f7b4a 100644
--- a/zebra-state/src/service/pending_utxos.rs
+++ b/zebra-state/src/service/pending_utxos.rs
@@ -61,4 +61,9 @@ impl PendingUtxos {
     pub fn prune(&mut self) {
         self.0.retain(|_, chan| chan.receiver_count() > 0);
     }
+
+    /// Returns the number of utxos that are being waited on.
+    pub fn len(&self) -> usize {
+        self.0.len()
+    }
 }

From 5e48acf3a4d345b448b8800f4adb30e096722ccb Mon Sep 17 00:00:00 2001
From: Henry de Valence <hdevalence@hdevalence.ca>
Date: Thu, 26 Nov 2020 15:55:37 -0800
Subject: [PATCH 148/148] consensus: add timeout to UTXO queries (#1391)

The state service API says explicitly that AwaitUTXO requests should be coupled
with a timeout layer. I didn't add this when I was testing and fixing the UTXO
lookup code (#1348, #1358) because causing zebrad to hang on a failed
dependency was useful for identifying cases where the code wasn't useful (and
then inspecting execution traces).

As a side effect, this ticket resolves most of the hangs in #1389, because
far-future gossiped blocks will have their UTXO lookups time out, though we
may wish to do other work as part of debugging the combined sync+gossip logic.
---
 zebra-consensus/src/script.rs | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/zebra-consensus/src/script.rs b/zebra-consensus/src/script.rs
index a4bc2e5c592..94c741ac800 100644
--- a/zebra-consensus/src/script.rs
+++ b/zebra-consensus/src/script.rs
@@ -1,5 +1,6 @@
 use std::{collections::HashMap, future::Future, pin::Pin, sync::Arc};
 
+use tower::timeout::Timeout;
 use tracing::Instrument;
 
 use zebra_chain::{parameters::NetworkUpgrade, transaction::Transaction, transparent};
@@ -7,6 +8,15 @@ use zebra_state::Utxo;
 
 use crate::BoxError;
 
+/// A timeout applied to UTXO lookup requests.
+///
+/// The exact value is non-essential, but this should be long enough to allow
+/// out-of-order verification of blocks (UTXOs are not required to be ready
+/// immediately) while being short enough to prune blocks that are too far in the
+/// future to be worth keeping in the queue, and to fail blocks that reference
+/// invalid UTXOs.
+const UTXO_LOOKUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10 * 60);
+
 /// Asynchronous script verification.
 ///
 /// The verifier asynchronously requests the UTXO a transaction attempts
@@ -20,12 +30,14 @@ use crate::BoxError;
 /// [RFC4]: https://zebra.zfnd.org/dev/rfcs/0004-asynchronous-script-verification.html
 #[derive(Debug, Clone)]
 pub struct Verifier<ZS> {
-    state: ZS,
+    state: Timeout<ZS>,
 }
 
 impl<ZS> Verifier<ZS> {
     pub fn new(state: ZS) -> Self {
-        Self { state }
+        Self {
+            state: Timeout::new(state, UTXO_LOOKUP_TIMEOUT),
+        }
     }
 }