zecfoundation
/
zebra
mirror of https://github.com/ZcashFoundation/zebra.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Download Zcash Sapling parameters and load them from cached files (#3057) 

* Replace Zcash parameters crates with pre-downloaded local parameter files

* Download Zcash parameters using the `zcashd` script in CI and Docker

* Add a zcash_proofs dependency to zebra-consensus

* Download Sapling parameters using zcash_proofs, rather than fetch-params.sh

* Add a new `zebrad download` subcommand

This command isn't required for nomrmal usage.
But it's useful when testing, or launching multiple Zebra instances.

* Use `zebrad download` in CI to pre-download parameters

* Log a helpful hint if downloading fails

* Allow some duplicate dependencies currently hidden by orchard

* Spawn a separate task to download Groth16 parameters

* Run the parameter download with code coverage

This avoids re-compining Zebra with and without coverage.

* Update Cargo.lock after rebase

* Try to pass `download` as an argument to `zebrad` in coverage CI

* Fix copy and paste comment typos

* Add path and download examples, like zcash_proofs

* Download params in CI just like zcash_proofs does

* Delete a redundant build step

* Implement graceful shutdown for zebrad start

* Send coverage summary to /dev/null when getting the params path

* Use the correct parameters path and download commands in CI

* Explain pre-downloads

* Avoid calling params_folder twice

* Rename parameter types and methods for consistency

```sh
fastmod SaplingParams SaplingParameters zebra*
fastmod Groth16Params Groth16Parameters zebra*
fastmod PARAMS GROTH16_PARAMETERS zebra*
fastmod params_folder directory zebra*
```

And a manual variable name tweak.

* rustfmt

* Remove a redundant coverage step

Co-authored-by: Janito Vaqueiro Ferreira Filho <janito.vff@gmail.com>
This commit is contained in:
teor 2021-11-20 09:02:56 +10:00 committed by GitHub
parent 26b3a50e01
commit f7202bfbc0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
24 changed files with 514 additions and 280 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
.github/workflows/ci.yml vendored
View File

@ -48,22 +48,13 @@ jobs:
echo "C:\Program Files\LLVM\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append echo "C:\Program Files\LLVM\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
echo "LIBCLANG_PATH=C:\Program Files\LLVM\bin" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append echo "LIBCLANG_PATH=C:\Program Files\LLVM\bin" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
- name: Skip network tests on Ubuntu - name: Skip network tests on Ubuntu and Windows
# Ubuntu runners don't have network or DNS configured during test steps # Ubuntu runners don't have network or DNS configured during test steps.
if: matrix.os == 'ubuntu-latest' # Windows runners have an unreliable network.
shell: bash
if: matrix.os != 'macOS-latest'
run: echo "ZEBRA_SKIP_NETWORK_TESTS=1" >> $GITHUB_ENV run: echo "ZEBRA_SKIP_NETWORK_TESTS=1" >> $GITHUB_ENV
- name: Skip network tests on Windows
# Windows runners have an unreliable network
if: matrix.os == 'windows-latest'
run: echo "ZEBRA_SKIP_NETWORK_TESTS=1" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
- name: Show env vars
run: |
echo "ZEBRA_SKIP_NETWORK_TESTS=${{ env.ZEBRA_SKIP_NETWORK_TESTS }}"
echo "CARGO_INCREMENTAL=${{ env.CARGO_INCREMENTAL }}"
echo "RUST_BACKTRACE=${{ env.RUST_BACKTRACE }}"
- name: Change target output directory on Windows - name: Change target output directory on Windows
# Windows doesn't have enough space on the D: drive, so we redirect the build output to the # Windows doesn't have enough space on the D: drive, so we redirect the build output to the
# larger C: drive. # larger C: drive.
@ -73,6 +64,31 @@ jobs:
mkdir C:\zebra-target mkdir C:\zebra-target
echo "CARGO_TARGET_DIR=C:\zebra-target" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append echo "CARGO_TARGET_DIR=C:\zebra-target" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
# Modified from:
# https://github.com/zcash/librustzcash/blob/c48bb4def2e122289843ddb3cb2984c325c03ca0/.github/workflows/ci.yml#L20-L33
- name: Fetch path to Zcash parameters
working-directory: ./zebra-consensus
shell: bash
run: echo "ZCASH_PARAMS=$(cargo run --example get-params-path)" >> $GITHUB_ENV
- name: Cache Zcash parameters
id: cache-params
uses: actions/cache@v2
with:
path: ${{ env.ZCASH_PARAMS }}
key: ${{ runner.os }}-params
- name: Fetch Zcash parameters
if: steps.cache-params.outputs.cache-hit != 'true'
working-directory: ./zebra-consensus
run: cargo run --example download-params
- name: Show env vars
run: |
echo "ZEBRA_SKIP_NETWORK_TESTS=${{ env.ZEBRA_SKIP_NETWORK_TESTS }}"
echo "ZCASH_PARAMS=${{ env.ZCASH_PARAMS }}"
echo "CARGO_INCREMENTAL=${{ env.CARGO_INCREMENTAL }}"
echo "CARGO_TARGET_DIR=${{ env.CARGO_TARGET_DIR }}"
echo "RUST_BACKTRACE=${{ env.RUST_BACKTRACE }}"
- name: Run tests - name: Run tests
uses: actions-rs/cargo@v1.0.3 uses: actions-rs/cargo@v1.0.3
with: with:
@ -132,7 +148,9 @@ jobs:
- name: Show env vars - name: Show env vars
run: | run: |
echo "ZEBRA_SKIP_NETWORK_TESTS=${{ env.ZEBRA_SKIP_NETWORK_TESTS }}" echo "ZEBRA_SKIP_NETWORK_TESTS=${{ env.ZEBRA_SKIP_NETWORK_TESTS }}"
echo "ZCASH_PARAMS=${{ env.ZCASH_PARAMS }}"
echo "CARGO_INCREMENTAL=${{ env.CARGO_INCREMENTAL }}" echo "CARGO_INCREMENTAL=${{ env.CARGO_INCREMENTAL }}"
echo "CARGO_TARGET_DIR=${{ env.CARGO_TARGET_DIR }}"
echo "RUST_BACKTRACE=${{ env.RUST_BACKTRACE }}" echo "RUST_BACKTRACE=${{ env.RUST_BACKTRACE }}"
- name: Run build without features enabled - name: Run build without features enabled
@ -169,7 +187,9 @@ jobs:
- name: Show env vars - name: Show env vars
run: | run: |
echo "ZEBRA_SKIP_NETWORK_TESTS=${{ env.ZEBRA_SKIP_NETWORK_TESTS }}" echo "ZEBRA_SKIP_NETWORK_TESTS=${{ env.ZEBRA_SKIP_NETWORK_TESTS }}"
echo "ZCASH_PARAMS=${{ env.ZCASH_PARAMS }}"
echo "CARGO_INCREMENTAL=${{ env.CARGO_INCREMENTAL }}" echo "CARGO_INCREMENTAL=${{ env.CARGO_INCREMENTAL }}"
echo "CARGO_TARGET_DIR=${{ env.CARGO_TARGET_DIR }}"
echo "RUST_BACKTRACE=${{ env.RUST_BACKTRACE }}" echo "RUST_BACKTRACE=${{ env.RUST_BACKTRACE }}"
- name: Build - name: Build
@ -202,7 +222,9 @@ jobs:
- name: Show env vars - name: Show env vars
run: | run: |
echo "ZEBRA_SKIP_NETWORK_TESTS=${{ env.ZEBRA_SKIP_NETWORK_TESTS }}" echo "ZEBRA_SKIP_NETWORK_TESTS=${{ env.ZEBRA_SKIP_NETWORK_TESTS }}"
echo "ZCASH_PARAMS=${{ env.ZCASH_PARAMS }}"
echo "CARGO_INCREMENTAL=${{ env.CARGO_INCREMENTAL }}" echo "CARGO_INCREMENTAL=${{ env.CARGO_INCREMENTAL }}"
echo "CARGO_TARGET_DIR=${{ env.CARGO_TARGET_DIR }}"
echo "RUST_BACKTRACE=${{ env.RUST_BACKTRACE }}" echo "RUST_BACKTRACE=${{ env.RUST_BACKTRACE }}"
- name: Run clippy - name: Run clippy
@ -244,7 +266,9 @@ jobs:
- name: Show env vars - name: Show env vars
run: | run: |
echo "ZEBRA_SKIP_NETWORK_TESTS=${{ env.ZEBRA_SKIP_NETWORK_TESTS }}" echo "ZEBRA_SKIP_NETWORK_TESTS=${{ env.ZEBRA_SKIP_NETWORK_TESTS }}"
echo "ZCASH_PARAMS=${{ env.ZCASH_PARAMS }}"
echo "CARGO_INCREMENTAL=${{ env.CARGO_INCREMENTAL }}" echo "CARGO_INCREMENTAL=${{ env.CARGO_INCREMENTAL }}"
echo "CARGO_TARGET_DIR=${{ env.CARGO_TARGET_DIR }}"
echo "RUST_BACKTRACE=${{ env.RUST_BACKTRACE }}" echo "RUST_BACKTRACE=${{ env.RUST_BACKTRACE }}"
- name: Check rustfmt - name: Check rustfmt

47
.github/workflows/coverage.yml vendored
View File

@ -16,6 +16,10 @@ jobs:
# The large timeout is to accommodate nightly builds # The large timeout is to accommodate nightly builds
timeout-minutes: 60 timeout-minutes: 60
runs-on: ubuntu-latest runs-on: ubuntu-latest
env:
CARGO_INCREMENTAL: 0
RUST_BACKTRACE: full
steps: steps:
- uses: actions/checkout@v2.4.0 - uses: actions/checkout@v2.4.0
with: with:
@ -31,11 +35,44 @@ jobs:
- name: Install cargo-llvm-cov cargo command - name: Install cargo-llvm-cov cargo command
run: cargo install cargo-llvm-cov run: cargo install cargo-llvm-cov
- name: Generate code coverage - name: Skip network tests on Ubuntu and Windows
env: # Ubuntu runners don't have network or DNS configured during test steps.
ZEBRA_SKIP_NETWORK_TESTS: 1 # Windows runners have an unreliable network.
CARGO_INCREMENTAL: 0 shell: bash
run: cargo llvm-cov --lcov > lcov.info if: matrix.os != 'macOS-latest'
run: echo "ZEBRA_SKIP_NETWORK_TESTS=1" >> $GITHUB_ENV
# Modified from:
# https://github.com/zcash/librustzcash/blob/c48bb4def2e122289843ddb3cb2984c325c03ca0/.github/workflows/ci.yml#L20-L33
- name: Fetch path to Zcash parameters
working-directory: ./zebra-consensus
shell: bash
# cargo-llvm-cov doesn't have a silent mode, so we have to extract the path from stderr
run: echo "ZCASH_PARAMS=$(cargo llvm-cov --lcov --no-report run --example get-params-path 2>&1 >/dev/null | tail -1)" >> $GITHUB_ENV
- name: Cache Zcash parameters
id: cache-params
uses: actions/cache@v2
with:
path: ${{ env.ZCASH_PARAMS }}
key: ${{ runner.os }}-params
- name: Fetch Zcash parameters
if: steps.cache-params.outputs.cache-hit != 'true'
working-directory: ./zebra-consensus
run: cargo llvm-cov --lcov --no-report run --example download-params
- name: Show env vars
run: |
echo "ZEBRA_SKIP_NETWORK_TESTS=${{ env.ZEBRA_SKIP_NETWORK_TESTS }}"
echo "ZCASH_PARAMS=${{ env.ZCASH_PARAMS }}"
echo "CARGO_INCREMENTAL=${{ env.CARGO_INCREMENTAL }}"
echo "CARGO_TARGET_DIR=${{ env.CARGO_TARGET_DIR }}"
echo "RUST_BACKTRACE=${{ env.RUST_BACKTRACE }}"
- name: Run Zebra tests
run: cargo llvm-cov --lcov --no-report
- name: Generate coverage report
run: cargo llvm-cov --lcov --no-run --output-path lcov.info
- name: Upload coverage report to Codecov - name: Upload coverage report to Codecov
uses: codecov/codecov-action@v2.1.0 uses: codecov/codecov-action@v2.1.0

122
Cargo.lock generated
View File

@ -974,6 +974,15 @@ dependencies = [
"generic-array", "generic-array",
] ]
[[package]]
name = "directories"
version = "4.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f51c5d4ddabd36886dd3e1438cb358cdcb0d7c499cb99cb4ac2e38e18b5cb210"
dependencies = [
"dirs-sys",
]
[[package]] [[package]]
name = "dirs" name = "dirs"
version = "4.0.0" version = "4.0.0"
@ -1074,16 +1083,6 @@ dependencies = [
"termcolor", "termcolor",
] ]
[[package]]
name = "equihash"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4127688f6177e3f57521881cb1cfd90d1228214f9dc43b8efe6f6c6948cd8280"
dependencies = [
"blake2b_simd",
"byteorder",
]
[[package]] [[package]]
name = "equihash" name = "equihash"
version = "0.1.0" version = "0.1.0"
@ -1614,7 +1613,7 @@ dependencies = [
[[package]] [[package]]
name = "incrementalmerkletree" name = "incrementalmerkletree"
version = "0.1.0" version = "0.1.0"
source = "git+https://github.com/zcash/incrementalmerkletree?rev=b7bd6246122a6e9ace8edb51553fbf5228906cbb#b7bd6246122a6e9ace8edb51553fbf5228906cbb" source = "git+https://github.com/zcash/incrementalmerkletree.git?rev=b7bd6246122a6e9ace8edb51553fbf5228906cbb#b7bd6246122a6e9ace8edb51553fbf5228906cbb"
dependencies = [ dependencies = [
"serde", "serde",
] ]
@ -1987,6 +1986,19 @@ dependencies = [
"autocfg", "autocfg",
] ]
[[package]]
name = "minreq"
version = "2.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d5f7db7a675c4b46b8842105b9371d6151e95fbbecd9b0e54dc2ea814397d2cc"
dependencies = [
"lazy_static",
"log",
"rustls",
"webpki",
"webpki-roots 0.18.0",
]
[[package]] [[package]]
name = "mio" name = "mio"
version = "0.7.6" version = "0.7.6"
@ -2817,7 +2829,7 @@ dependencies = [
"wasm-bindgen", "wasm-bindgen",
"wasm-bindgen-futures", "wasm-bindgen-futures",
"web-sys", "web-sys",
"webpki-roots", "webpki-roots 0.21.1",
"winreg", "winreg",
] ]
@ -3961,56 +3973,6 @@ version = "0.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed"
[[package]]
name = "wagyu-zcash-parameters"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61c904628658374e651288f000934c33ef738b2d8b3e65d4100b70b395dbe2bb"
dependencies = [
"wagyu-zcash-parameters-1",
"wagyu-zcash-parameters-2",
"wagyu-zcash-parameters-3",
"wagyu-zcash-parameters-4",
"wagyu-zcash-parameters-5",
"wagyu-zcash-parameters-6",
]
[[package]]
name = "wagyu-zcash-parameters-1"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90bf2e21bb027d3f8428c60d6a720b54a08bf6ce4e6f834ef8e0d38bb5695da8"
[[package]]
name = "wagyu-zcash-parameters-2"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a616ab2e51e74cc48995d476e94de810fb16fc73815f390bf2941b046cc9ba2c"
[[package]]
name = "wagyu-zcash-parameters-3"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "14da1e2e958ff93c0830ee68e91884069253bf3462a67831b02b367be75d6147"
[[package]]
name = "wagyu-zcash-parameters-4"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f058aeef03a2070e8666ffb5d1057d8bb10313b204a254a6e6103eb958e9a6d6"
[[package]]
name = "wagyu-zcash-parameters-5"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ffe916b30e608c032ae1b734f02574a3e12ec19ab5cc5562208d679efe4969d"
[[package]]
name = "wagyu-zcash-parameters-6"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7b6d5a78adc3e8f198e9cd730f219a695431467f7ec29dcfc63ade885feebe1"
[[package]] [[package]]
name = "wait-timeout" name = "wait-timeout"
version = "0.2.0" version = "0.2.0"
@ -4139,6 +4101,15 @@ dependencies = [
"untrusted", "untrusted",
] ]
[[package]]
name = "webpki-roots"
version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "91cd5736df7f12a964a5067a12c62fa38e1bd8080aff1f80bc29be7c80d19ab4"
dependencies = [
"webpki",
]
[[package]] [[package]]
name = "webpki-roots" name = "webpki-roots"
version = "0.21.1" version = "0.21.1"
@ -4265,7 +4236,7 @@ dependencies = [
"bls12_381 0.6.0", "bls12_381 0.6.0",
"byteorder", "byteorder",
"chacha20poly1305", "chacha20poly1305",
"equihash 0.1.0 (git+https://github.com/zcash/librustzcash.git?rev=53d0a51d33a421cb76d3e3124d1e4c1c9036068e)", "equihash",
"ff 0.11.0", "ff 0.11.0",
"fpe", "fpe",
"group 0.11.0", "group 0.11.0",
@ -4286,6 +4257,25 @@ dependencies = [
"zcash_note_encryption", "zcash_note_encryption",
] ]
[[package]]
name = "zcash_proofs"
version = "0.5.0"
source = "git+https://github.com/zcash/librustzcash.git?rev=53d0a51d33a421cb76d3e3124d1e4c1c9036068e#53d0a51d33a421cb76d3e3124d1e4c1c9036068e"
dependencies = [
"bellman",
"blake2b_simd",
"bls12_381 0.6.0",
"byteorder",
"directories",
"ff 0.11.0",
"group 0.11.0",
"jubjub 0.8.0",
"lazy_static",
"minreq",
"rand_core 0.6.3",
"zcash_primitives",
]
[[package]] [[package]]
name = "zcash_script" name = "zcash_script"
version = "0.1.6-alpha.0" version = "0.1.6-alpha.0"
@ -4324,7 +4314,7 @@ dependencies = [
"criterion", "criterion",
"displaydoc", "displaydoc",
"ed25519-zebra", "ed25519-zebra",
"equihash 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "equihash",
"fpe", "fpe",
"futures", "futures",
"group 0.11.0", "group 0.11.0",
@ -4372,6 +4362,7 @@ dependencies = [
"bls12_381 0.6.0", "bls12_381 0.6.0",
"chrono", "chrono",
"color-eyre", "color-eyre",
"dirs",
"displaydoc", "displaydoc",
"futures", "futures",
"futures-util", "futures-util",
@ -4397,7 +4388,7 @@ dependencies = [
"tracing-error", "tracing-error",
"tracing-futures", "tracing-futures",
"tracing-subscriber 0.2.25", "tracing-subscriber 0.2.25",
"wagyu-zcash-parameters", "zcash_proofs",
"zebra-chain", "zebra-chain",
"zebra-script", "zebra-script",
"zebra-state", "zebra-state",
@ -4538,6 +4529,7 @@ dependencies = [
"gumdrop", "gumdrop",
"hyper", "hyper",
"inferno", "inferno",
"lazy_static",
"metrics", "metrics",
"metrics-exporter-prometheus", "metrics-exporter-prometheus",
"once_cell", "once_cell",

20
Cargo.toml
View File

@ -22,11 +22,25 @@ panic = "abort"
[patch.crates-io] [patch.crates-io]
# TODO: remove these after a new librustzcash release.
# These are librustzcash requirements specified in its workspace Cargo.toml that we must replicate here
incrementalmerkletree = { git = "https://github.com/zcash/incrementalmerkletree", rev = "b7bd6246122a6e9ace8edb51553fbf5228906cbb" }
# TODO: replace with upstream orchard when these changes are merged # TODO: replace with upstream orchard when these changes are merged
# https://github.com/ZcashFoundation/zebra/issues/3056 # https://github.com/ZcashFoundation/zebra/issues/3056
orchard = { git = "https://github.com/ZcashFoundation/orchard.git", rev = "568e24cd5f129158375d7ac7d98c89ebff4f982f" } orchard = { git = "https://github.com/ZcashFoundation/orchard.git", rev = "568e24cd5f129158375d7ac7d98c89ebff4f982f" }
# TODO: remove these after a new librustzcash release.
# These are librustzcash git requirements specified in its workspace Cargo.toml,
# that we must replicate here
incrementalmerkletree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "b7bd6246122a6e9ace8edb51553fbf5228906cbb" }
# Replaced by the ZcashFoundation fork above
#orchard = { git = "https://github.com/zcash/orchard.git", rev = "2c8241f25b943aa05203eacf9905db117c69bd29" }
# These are librustzcash file requirements specified in its workspace Cargo.toml,
# that we must replace with git requirements
zcash_note_encryption = { git = "https://github.com/zcash/librustzcash.git", rev = "53d0a51d33a421cb76d3e3124d1e4c1c9036068e" } zcash_note_encryption = { git = "https://github.com/zcash/librustzcash.git", rev = "53d0a51d33a421cb76d3e3124d1e4c1c9036068e" }
# These patches are not strictly required,
# but they help avoid duplicate dependencies
equihash = { git = "https://github.com/zcash/librustzcash.git", rev = "53d0a51d33a421cb76d3e3124d1e4c1c9036068e" }
zcash_history = { git = "https://github.com/zcash/librustzcash.git", rev = "53d0a51d33a421cb76d3e3124d1e4c1c9036068e" }
zcash_primitives = { git = "https://github.com/zcash/librustzcash.git", rev = "53d0a51d33a421cb76d3e3124d1e4c1c9036068e" } zcash_primitives = { git = "https://github.com/zcash/librustzcash.git", rev = "53d0a51d33a421cb76d3e3124d1e4c1c9036068e" }
zcash_proofs = { git = "https://github.com/zcash/librustzcash.git", rev = "53d0a51d33a421cb76d3e3124d1e4c1c9036068e" }

21
deny.toml
View File

@ -34,12 +34,6 @@ skip-tree = [
# ticket #2953: tracing dependencies # ticket #2953: tracing dependencies
{ name = "tracing-subscriber", version = "=0.1.6" }, { name = "tracing-subscriber", version = "=0.1.6" },
# ticket #2982: librustzcash and orchard git versions
# Note that the equihash duplication is probably because `zcash_primitives`
# (which imports it with a path import) is being imported as a git dependency.
{ name = "equihash", version = "=0.1.0" },
{ name = "orchard", version = "=0.0.0" },
# ticket #2983: criterion dependencies # ticket #2983: criterion dependencies
{ name = "criterion", version = "=0.3.4" }, { name = "criterion", version = "=0.3.4" },
@ -49,6 +43,9 @@ skip-tree = [
# ticket #2981: bindgen dependencies # ticket #2981: bindgen dependencies
{ name = "rocksdb", version = "=0.16.0" }, { name = "rocksdb", version = "=0.16.0" },
# ticket #3063: redjubjub dependencies
{ name = "redjubjub", version = "=0.4.0" },
# ticket #2984: owo-colors dependencies # ticket #2984: owo-colors dependencies
{ name = "color-eyre", version = "=0.5.11" }, { name = "color-eyre", version = "=0.5.11" },
@ -61,14 +58,24 @@ skip-tree = [
# ticket #2999: http dependencies # ticket #2999: http dependencies
{ name = "bytes", version = "=0.5.6" }, { name = "bytes", version = "=0.5.6" },
# ticket #3061: reqwest and minreq dependencies
{ name = "webpki-roots", version = "=0.18.0" },
# ticket #2980: inferno and orchard/cryptographic dependencies
{ name = "inferno", version = "=0.10.8" },
{ name = "orchard", version = "=0.0.0" },
# upgrade orchard from deprecated `bigint` to `uint`: https://github.com/zcash/orchard/issues/219 # upgrade orchard from deprecated `bigint` to `uint`: https://github.com/zcash/orchard/issues/219
# alternative: downgrade Zebra to `bigint` # alternative: downgrade Zebra to `bigint`
{ name = "bigint", version = "=4.4.3" }, { name = "bigint", version = "=4.4.3" },
# recent major version bumps # recent major version bumps
# we should re-check these dependencies in February 2022
# wait for lots of crates in the cryptographic ecosystem to upgrade
{ name = "rand", version = "=0.7.3" },
# wait for lots of crates in the tokio ecosystem to upgrade # wait for lots of crates in the tokio ecosystem to upgrade
# we should re-check these dependencies in February 2022
{ name = "redox_syscall", version = "=0.1.57" }, { name = "redox_syscall", version = "=0.1.57" },
{ name = "socket2", version = "=0.3.16" }, { name = "socket2", version = "=0.3.16" },
] ]

4
docker/Dockerfile.build
View File

@ -20,7 +20,6 @@ COPY . .
RUN cd zebrad/; cargo build --release --features enable-sentry RUN cd zebrad/; cargo build --release --features enable-sentry
# Runner image # Runner image
FROM debian:buster-slim AS zebrad-release FROM debian:buster-slim AS zebrad-release
@ -45,6 +44,9 @@ RUN printf "[tracing]\n" >> /zebrad.toml
RUN printf "endpoint_addr = '0.0.0.0:3000'\n" >> /zebrad.toml RUN printf "endpoint_addr = '0.0.0.0:3000'\n" >> /zebrad.toml
RUN cat /zebrad.toml RUN cat /zebrad.toml
# Pre-download Zcash Sprout and Sapling parameters
RUN /zebrad download
EXPOSE 3000 8233 18233 EXPOSE 3000 8233 18233
ENV RUST_LOG debug ENV RUST_LOG debug

3
docker/Dockerfile.test
View File

@ -22,6 +22,9 @@ EXPOSE 8233 18233
COPY . . COPY . .
# Pre-download Zcash Sprout and Sapling parameters
RUN cargo run --verbose --bin zebrad download
RUN cargo test --all --no-run RUN cargo test --all --no-run
CMD cargo test --workspace --no-fail-fast -- -Zunstable-options --include-ignored CMD cargo test --workspace --no-fail-fast -- -Zunstable-options --include-ignored

19
zebra-consensus/Cargo.toml
View File

@ -13,16 +13,19 @@ proptest-impl = ["proptest", "proptest-derive", "zebra-chain/proptest-impl"]
blake2b_simd = "0.5.11" blake2b_simd = "0.5.11"
bellman = "0.11.1" bellman = "0.11.1"
bls12_381 = "0.6.0" bls12_381 = "0.6.0"
chrono = "0.4.19"
displaydoc = "0.2.2"
halo2 = "=0.1.0-beta.1"
jubjub = "0.8.0" jubjub = "0.8.0"
lazy_static = "1.4.0" rand = "0.8"
once_cell = "1.8"
halo2 = "=0.1.0-beta.1"
# TODO: replace with upstream orchard when these changes are merged # TODO: replace with upstream orchard when these changes are merged
# https://github.com/ZcashFoundation/zebra/issues/3056 # https://github.com/ZcashFoundation/zebra/issues/3056
orchard = "0.0.0" orchard = "0.0.0"
rand = "0.8"
chrono = "0.4.19"
dirs = "4.0.0"
displaydoc = "0.2.2"
lazy_static = "1.4.0"
once_cell = "1.8"
serde = { version = "1", features = ["serde_derive"] } serde = { version = "1", features = ["serde_derive"] }
futures = "0.3.17" futures = "0.3.17"
@ -34,12 +37,14 @@ tower = { version = "0.4.9", features = ["timeout", "util", "buffer"] }
tracing = "0.1.29" tracing = "0.1.29"
tracing-futures = "0.2.5" tracing-futures = "0.2.5"
zcash_proofs = { git = "https://github.com/zcash/librustzcash.git", rev = "53d0a51d33a421cb76d3e3124d1e4c1c9036068e", features = ["local-prover", "multicore", "download-params"] }
tower-fallback = { path = "../tower-fallback/" } tower-fallback = { path = "../tower-fallback/" }
tower-batch = { path = "../tower-batch/" } tower-batch = { path = "../tower-batch/" }
zebra-chain = { path = "../zebra-chain" } zebra-chain = { path = "../zebra-chain" }
zebra-state = { path = "../zebra-state" } zebra-state = { path = "../zebra-state" }
zebra-script = { path = "../zebra-script" } zebra-script = { path = "../zebra-script" }
wagyu-zcash-parameters = "0.2.0"
proptest = { version = "0.10", optional = true } proptest = { version = "0.10", optional = true }
proptest-derive = { version = "0.3.0", optional = true } proptest-derive = { version = "0.3.0", optional = true }

11
zebra-consensus/examples/download-params.rs Normal file
View File

@ -0,0 +1,11 @@
//! Download the Sapling and Sprout Groth16 parameters if needed,
//! check they were downloaded correctly, and load them into Zebra.
// Has the same functionality as:
// https://github.com/zcash/librustzcash/blob/c48bb4def2e122289843ddb3cb2984c325c03ca0/zcash_proofs/examples/download-params.rs
fn main() {
// The lazy static initializer does the download, if needed,
// and the file hash checks.
lazy_static::initialize(&zebra_consensus::groth16::GROTH16_PARAMETERS);
}

11
zebra-consensus/examples/get-params-path.rs Normal file
View File

@ -0,0 +1,11 @@
//! Print the Zcash parameter directory path to standard output.
// Modified from:
// https://github.com/zcash/librustzcash/blob/c48bb4def2e122289843ddb3cb2984c325c03ca0/zcash_proofs/examples/get-params-path.rs
fn main() {
let path = zebra_consensus::groth16::Groth16Parameters::directory();
if let Some(path) = path.to_str() {
println!("{}", path);
}
}

26
zebra-consensus/src/chain.rs
View File

@ -22,6 +22,7 @@ use std::{
use displaydoc::Display; use displaydoc::Display;
use futures::{FutureExt, TryFutureExt}; use futures::{FutureExt, TryFutureExt};
use thiserror::Error; use thiserror::Error;
use tokio::task::{spawn_blocking, JoinHandle};
use tower::{buffer::Buffer, util::BoxService, Service, ServiceExt}; use tower::{buffer::Buffer, util::BoxService, Service, ServiceExt};
use tracing::instrument; use tracing::instrument;
@ -148,7 +149,11 @@ where
} }
} }
/// Initialize block and transaction verification services. /// Initialize block and transaction verification services,
/// and pre-download Groth16 parameters if needed.
///
/// Returns a block verifier, transaction verifier,
/// and the Groth16 parameter download task [`JoinHandle`].
/// ///
/// The consensus configuration is specified by `config`, and the Zcash network /// The consensus configuration is specified by `config`, and the Zcash network
/// to verify blocks for is specified by `network`. /// to verify blocks for is specified by `network`.
@ -160,6 +165,12 @@ where
/// The transaction verification service asynchronously performs semantic verification /// The transaction verification service asynchronously performs semantic verification
/// checks. Transactions that pass semantic verification return an `Ok` result to the caller. /// checks. Transactions that pass semantic verification return an `Ok` result to the caller.
/// ///
/// Pre-downloads the Sapling and Sprout Groth16 parameters if needed,
/// checks they were downloaded correctly, and loads them into Zebra.
/// (The transaction verifier automatically downloads the parameters on first use.
/// But the parameter downloads can take around 10 minutes.
/// So we pre-download the parameters, to avoid verification timeouts.)
///
/// This function should only be called once for a particular state service. /// This function should only be called once for a particular state service.
/// ///
/// Dropped requests are cancelled on a best-effort basis, but may continue to be processed. /// Dropped requests are cancelled on a best-effort basis, but may continue to be processed.
@ -180,11 +191,22 @@ pub async fn init<S>(
BoxService<transaction::Request, transaction::Response, TransactionError>, BoxService<transaction::Request, transaction::Response, TransactionError>,
transaction::Request, transaction::Request,
>, >,
JoinHandle<()>,
) )
where where
S: Service<zs::Request, Response = zs::Response, Error = BoxError> + Send + Clone + 'static, S: Service<zs::Request, Response = zs::Response, Error = BoxError> + Send + Clone + 'static,
S::Future: Send + 'static, S::Future: Send + 'static,
{ {
// pre-download Groth16 parameters async
let groth16_download_handle = spawn_blocking(|| {
tracing::info!("checking if Zcash Sapling and Sprout parameters have been downloaded");
// The lazy static initializer does the download, if needed,
// and the file hash checks.
lazy_static::initialize(&crate::groth16::GROTH16_PARAMETERS);
});
// transaction verification // transaction verification
let script = script::Verifier::new(state_service.clone()); let script = script::Verifier::new(state_service.clone());
@ -225,5 +247,5 @@ where
let chain = Buffer::new(BoxService::new(chain), VERIFIER_BUFFER_BOUND); let chain = Buffer::new(BoxService::new(chain), VERIFIER_BUFFER_BOUND);
(chain, transaction) (chain, transaction, groth16_download_handle)
} }

14
zebra-consensus/src/chain/tests.rs
View File

@ -41,8 +41,8 @@ pub fn block_no_transactions() -> Block {
} }
} }
/// Return a new `(chain_verifier, state_service)` using the hard-coded /// Return a new chain verifier and state service,
/// checkpoint list for `network`. /// using the hard-coded checkpoint list for `network`.
async fn verifiers_from_network( async fn verifiers_from_network(
network: Network, network: Network,
) -> ( ) -> (
@ -64,9 +64,13 @@ async fn verifiers_from_network(
+ 'static, + 'static,
) { ) {
let state_service = zs::init_test(network); let state_service = zs::init_test(network);
let (chain_verifier, _transaction_verifier) = let (chain_verifier, _transaction_verifier, _groth16_download_handle) =
crate::chain::init(Config::default(), network, state_service.clone()).await; crate::chain::init(Config::default(), network, state_service.clone()).await;
// We can drop the download task handle here, because:
// - if the download task fails, the tests will panic, and
// - if the download task hangs, the tests will hang.
(chain_verifier, state_service) (chain_verifier, state_service)
} }
@ -153,7 +157,9 @@ async fn verify_checkpoint(config: Config) -> Result<(), Report> {
// Test that the chain::init function works. Most of the other tests use // Test that the chain::init function works. Most of the other tests use
// init_from_verifiers. // init_from_verifiers.
let (chain_verifier, _transaction_verifier) = //
// Download task panics and timeouts are propagated to the tests that use Groth16 verifiers.
let (chain_verifier, _transaction_verifier, _groth16_download_handle) =
super::init(config.clone(), network, zs::init_test(network)).await; super::init(config.clone(), network, zs::init_test(network)).await;
// Add a timeout layer // Add a timeout layer

1
zebra-consensus/src/lib.rs
View File

@ -59,6 +59,7 @@ pub use checkpoint::MAX_CHECKPOINT_BYTE_COUNT;
pub use checkpoint::MAX_CHECKPOINT_HEIGHT_GAP; pub use checkpoint::MAX_CHECKPOINT_HEIGHT_GAP;
pub use config::Config; pub use config::Config;
pub use error::BlockError; pub use error::BlockError;
pub use primitives::groth16;
/// A boxed [`std::error::Error`]. /// A boxed [`std::error::Error`].
pub type BoxError = Box<dyn std::error::Error + Send + Sync + 'static>; pub type BoxError = Box<dyn std::error::Error + Send + Sync + 'static>;

60
zebra-consensus/src/primitives/groth16.rs
View File

@ -18,17 +18,17 @@ use once_cell::sync::Lazy;
use rand::thread_rng; use rand::thread_rng;
use tokio::sync::broadcast::{channel, error::RecvError, Sender}; use tokio::sync::broadcast::{channel, error::RecvError, Sender};
use tower::{util::ServiceFn, Service}; use tower::{util::ServiceFn, Service};
use tower_batch::{Batch, BatchControl}; use tower_batch::{Batch, BatchControl};
use tower_fallback::Fallback; use tower_fallback::Fallback;
use zebra_chain::sapling::{Output, PerSpendAnchor, Spend}; use zebra_chain::sapling::{Output, PerSpendAnchor, Spend};
mod hash_reader;
mod params; mod params;
#[cfg(test)] #[cfg(test)]
mod tests; mod tests;
use self::hash_reader::HashReader; pub use params::{Groth16Parameters, GROTH16_PARAMETERS};
use params::PARAMS;
/// Global batch verification context for Groth16 proofs of Spend statements. /// Global batch verification context for Groth16 proofs of Spend statements.
/// ///
@ -40,28 +40,31 @@ use params::PARAMS;
/// handle. /// handle.
pub static SPEND_VERIFIER: Lazy< pub static SPEND_VERIFIER: Lazy<
Fallback<Batch<Verifier, Item>, ServiceFn<fn(Item) -> Ready<Result<(), VerificationError>>>>, Fallback<Batch<Verifier, Item>, ServiceFn<fn(Item) -> Ready<Result<(), VerificationError>>>>,
> = Lazy::new(|| { > =
Fallback::new( Lazy::new(|| {
Batch::new( Fallback::new(
Verifier::new(&PARAMS.sapling.spend.vk), Batch::new(
super::MAX_BATCH_SIZE, Verifier::new(&GROTH16_PARAMETERS.sapling.spend.vk),
super::MAX_BATCH_LATENCY, super::MAX_BATCH_SIZE,
), super::MAX_BATCH_LATENCY,
// We want to fallback to individual verification if batch verification ),
// fails, so we need a Service to use. The obvious way to do this would // We want to fallback to individual verification if batch verification
// be to write a closure that returns an async block. But because we // fails, so we need a Service to use. The obvious way to do this would
// have to specify the type of a static, we need to be able to write the // be to write a closure that returns an async block. But because we
// type of the closure and its return value, and both closures and async // have to specify the type of a static, we need to be able to write the
// blocks have eldritch types whose names cannot be written. So instead, // type of the closure and its return value, and both closures and async
// we use a Ready to avoid an async block and cast the closure to a // blocks have eldritch types whose names cannot be written. So instead,
// function (which is possible because it doesn't capture any state). // we use a Ready to avoid an async block and cast the closure to a
tower::service_fn( // function (which is possible because it doesn't capture any state).
(|item: Item| { tower::service_fn(
ready(item.verify_single(&prepare_verifying_key(&PARAMS.sapling.spend.vk))) (|item: Item| {
}) as fn(_) -> _, ready(item.verify_single(&prepare_verifying_key(
), &GROTH16_PARAMETERS.sapling.spend.vk,
) )))
}); }) as fn(_) -> _,
),
)
});
/// Global batch verification context for Groth16 proofs of Output statements. /// Global batch verification context for Groth16 proofs of Output statements.
/// ///
@ -76,7 +79,7 @@ pub static OUTPUT_VERIFIER: Lazy<
> = Lazy::new(|| { > = Lazy::new(|| {
Fallback::new( Fallback::new(
Batch::new( Batch::new(
Verifier::new(&PARAMS.sapling.output.vk), Verifier::new(&GROTH16_PARAMETERS.sapling.output.vk),
super::MAX_BATCH_SIZE, super::MAX_BATCH_SIZE,
super::MAX_BATCH_LATENCY, super::MAX_BATCH_LATENCY,
), ),
@ -90,7 +93,9 @@ pub static OUTPUT_VERIFIER: Lazy<
// function (which is possible because it doesn't capture any state). // function (which is possible because it doesn't capture any state).
tower::service_fn( tower::service_fn(
(|item: Item| { (|item: Item| {
ready(item.verify_single(&prepare_verifying_key(&PARAMS.sapling.output.vk))) ready(item.verify_single(&prepare_verifying_key(
&GROTH16_PARAMETERS.sapling.output.vk,
)))
}) as fn(_) -> _, }) as fn(_) -> _,
), ),
) )
@ -99,6 +104,7 @@ pub static OUTPUT_VERIFIER: Lazy<
/// A Groth16 verification item, used as the request type of the service. /// A Groth16 verification item, used as the request type of the service.
pub type Item = batch::Item<Bls12>; pub type Item = batch::Item<Bls12>;
/// A wrapper to workaround the missing `ServiceExt::map_err` method.
pub struct ItemWrapper(Item); pub struct ItemWrapper(Item);
impl From<&Spend<PerSpendAnchor>> for ItemWrapper { impl From<&Spend<PerSpendAnchor>> for ItemWrapper {

43
zebra-consensus/src/primitives/groth16/hash_reader.rs
View File

@ -1,43 +0,0 @@
use std::io::{self, Read};
use blake2b_simd::State;
/// Abstraction over a reader which hashes the data being read.
pub struct HashReader<R: Read> {
reader: R,
hasher: State,
}
impl<R: Read> HashReader<R> {
/// Construct a new `HashReader` given an existing `reader` by value.
pub fn new(reader: R) -> Self {
HashReader {
reader,
hasher: State::new(),
}
}
/// Destroy this reader and return the hash of what was read.
pub fn into_hash(self) -> String {
let hash = self.hasher.finalize();
let mut s = String::new();
for c in hash.as_bytes().iter() {
s += &format!("{:02x}", c);
}
s
}
}
impl<R: Read> Read for HashReader<R> {
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
let bytes = self.reader.read(buf)?;
if bytes > 0 {
self.hasher.update(&buf[0..bytes]);
}
Ok(bytes)
}
}

129
zebra-consensus/src/primitives/groth16/params.rs
View File

@ -1,32 +1,60 @@
use std::io::{self, BufReader}; //! Downloading, checking, and loading Groth16 Sapling and Sprout parameters.
use std::path::PathBuf;
use bellman::groth16; use bellman::groth16;
use bls12_381::Bls12; use bls12_381::Bls12;
use super::HashReader;
const SAPLING_SPEND_HASH: &str = "8270785a1a0d0bc77196f000ee6d221c9c9894f55307bd9357c3f0105d31ca63991ab91324160d8f53e2bbd3c2633a6eb8bdf5205d822e7f3f73edac51b2b70c";
const SAPLING_OUTPUT_HASH: &str = "657e3d38dbb5cb5e7dd2970e8b03d69b4787dd907285b5a7f0790dcc8072f60bf593b32cc2d1c030e00ff5ae64bf84c5c3beb84ddc841d48264b4a171744d028";
lazy_static::lazy_static! { lazy_static::lazy_static! {
pub static ref PARAMS: Groth16Params = Groth16Params::new(); /// Groth16 Zero-Knowledge Proof parameters for the Sapling and Sprout circuits.
///
/// When this static is accessed:
/// - the parameters are downloded if needed, then cached to a shared directory,
/// - the file hashes are checked, for both newly downloaded and previously cached files,
/// - the parameters are loaded into Zebra.
///
/// # Panics
///
/// If the downloaded or pre-existing parameter files are invalid.
pub static ref GROTH16_PARAMETERS: Groth16Parameters = Groth16Parameters::new();
} }
/// Groth16 Zero-Knowledge Proof parameters for the Sapling and Sprout circuits.
#[non_exhaustive] #[non_exhaustive]
pub struct Groth16Params { pub struct Groth16Parameters {
pub sapling: SaplingParams, /// The Sapling circuit Groth16 parameters.
pub sapling: SaplingParameters,
} }
impl Groth16Params { impl Groth16Parameters {
fn new() -> Self { /// Download if needed, cache, check, and load the Sprout and Sapling Groth16 parameters.
Self { ///
sapling: SaplingParams::new(), /// # Panics
///
/// If the downloaded or pre-existing parameter files are invalid.
fn new() -> Groth16Parameters {
Groth16Parameters {
sapling: SaplingParameters::new(),
} }
} }
/// Returns the path to the Groth16 parameters directory.
pub fn directory() -> PathBuf {
zcash_proofs::default_params_folder().expect("unable to find user home directory")
}
/// Returns a hint that helps users recover from parameter download failures.
pub fn failure_hint() -> String {
format!(
"Hint: try deleting {:?}, then running 'zebrad download' to re-download the parameters",
Groth16Parameters::directory(),
)
}
} }
/// Groth16 Zero-Knowledge Proof spend and output parameters for the Sapling circuit.
#[non_exhaustive] #[non_exhaustive]
pub struct SaplingParams { pub struct SaplingParameters {
pub spend: groth16::Parameters<Bls12>, pub spend: groth16::Parameters<Bls12>,
pub spend_prepared_verifying_key: groth16::PreparedVerifyingKey<Bls12>, pub spend_prepared_verifying_key: groth16::PreparedVerifyingKey<Bls12>,
@ -34,50 +62,41 @@ pub struct SaplingParams {
pub output_prepared_verifying_key: groth16::PreparedVerifyingKey<Bls12>, pub output_prepared_verifying_key: groth16::PreparedVerifyingKey<Bls12>,
} }
impl SaplingParams { impl SaplingParameters {
fn new() -> Self { /// Download if needed, cache, check, and load the Sapling Groth16 parameters.
let (spend, output) = wagyu_zcash_parameters::load_sapling_parameters(); ///
let spend_fs = BufReader::with_capacity(1024 * 1024, &spend[..]); /// # Panics
let output_fs = BufReader::with_capacity(1024 * 1024, &output[..]); ///
/// If the downloaded or pre-existing parameter files are invalid.
fn new() -> SaplingParameters {
// TODO: Sprout
Self::read(spend_fs, output_fs) let params_directory = Groth16Parameters::directory();
.expect("reading parameters from wagyu zcash parameter's vec will always succeed") let spend_path = params_directory.join("sapling-spend.params");
} let output_path = params_directory.join("sapling-output.params");
fn read<R: io::Read>(spend_fs: R, output_fs: R) -> Result<Self, io::Error> { // Download parameters if needed.
let mut spend_fs = HashReader::new(spend_fs); //
let mut output_fs = HashReader::new(output_fs); // TODO: use try_exists when it stabilises, to exit early on permissions errors (#83186)
if !spend_path.exists() || !output_path.exists() {
tracing::info!("downloading Zcash Sapling parameters");
zcash_proofs::download_parameters().unwrap_or_else(|_| {
panic!(
"error downloading parameter files. {}",
Groth16Parameters::failure_hint()
)
});
}
// Deserialize params // TODO: if loading fails, log a message including `failure_hint`
let spend = groth16::Parameters::<Bls12>::read(&mut spend_fs, false)?; tracing::info!("checking and loading Zcash Sapling parameters");
let output = groth16::Parameters::<Bls12>::read(&mut output_fs, false)?; let parameters = zcash_proofs::load_parameters(&spend_path, &output_path, None);
// There is extra stuff (the transcript) at the end of the parameter file which is SaplingParameters {
// used to verify the parameter validity, but we're not interested in that. We do spend: parameters.spend_params,
// want to read it, though, so that the BLAKE2b computed afterward is consistent spend_prepared_verifying_key: parameters.spend_vk,
// with `b2sum` on the files. output: parameters.output_params,
let mut sink = io::sink(); output_prepared_verifying_key: parameters.output_vk,
io::copy(&mut spend_fs, &mut sink)?; }
io::copy(&mut output_fs, &mut sink)?;
assert!(
spend_fs.into_hash() == SAPLING_SPEND_HASH,
"Sapling spend parameter is not correct."
);
assert!(
output_fs.into_hash() == SAPLING_OUTPUT_HASH,
"Sapling output parameter is not correct."
);
// Prepare verifying keys
let spend_prepared_verifying_key = groth16::prepare_verifying_key(&spend.vk);
let output_prepared_verifying_key = groth16::prepare_verifying_key(&output.vk);
Ok(Self {
spend,
spend_prepared_verifying_key,
output,
output_prepared_verifying_key,
})
} }
} }

39
zebra-consensus/src/primitives/groth16/tests.rs
View File

@ -62,27 +62,32 @@ where
#[tokio::test] #[tokio::test]
async fn verify_sapling_groth16() { async fn verify_sapling_groth16() {
// Use separate verifiers so shared batch tasks aren't killed when the test ends (#2390) // Use separate verifiers so shared batch tasks aren't killed when the test ends (#2390)
let mut spend_verifier = Fallback::new( let mut spend_verifier =
Batch::new( Fallback::new(
Verifier::new(&PARAMS.sapling.spend.vk), Batch::new(
crate::primitives::MAX_BATCH_SIZE, Verifier::new(&GROTH16_PARAMETERS.sapling.spend.vk),
crate::primitives::MAX_BATCH_LATENCY, crate::primitives::MAX_BATCH_SIZE,
), crate::primitives::MAX_BATCH_LATENCY,
tower::service_fn( ),
(|item: Item| { tower::service_fn(
ready(item.verify_single(&prepare_verifying_key(&PARAMS.sapling.spend.vk))) (|item: Item| {
}) as fn(_) -> _, ready(item.verify_single(&prepare_verifying_key(
), &GROTH16_PARAMETERS.sapling.spend.vk,
); )))
}) as fn(_) -> _,
),
);
let mut output_verifier = Fallback::new( let mut output_verifier = Fallback::new(
Batch::new( Batch::new(
Verifier::new(&PARAMS.sapling.output.vk), Verifier::new(&GROTH16_PARAMETERS.sapling.output.vk),
crate::primitives::MAX_BATCH_SIZE, crate::primitives::MAX_BATCH_SIZE,
crate::primitives::MAX_BATCH_LATENCY, crate::primitives::MAX_BATCH_LATENCY,
), ),
tower::service_fn( tower::service_fn(
(|item: Item| { (|item: Item| {
ready(item.verify_single(&prepare_verifying_key(&PARAMS.sapling.output.vk))) ready(item.verify_single(&prepare_verifying_key(
&GROTH16_PARAMETERS.sapling.output.vk,
)))
}) as fn(_) -> _, }) as fn(_) -> _,
), ),
); );
@ -152,13 +157,15 @@ async fn correctly_err_on_invalid_output_proof() {
// Also, since we expect these to fail, we don't want to slow down the communal verifiers. // Also, since we expect these to fail, we don't want to slow down the communal verifiers.
let mut output_verifier = Fallback::new( let mut output_verifier = Fallback::new(
Batch::new( Batch::new(
Verifier::new(&PARAMS.sapling.output.vk), Verifier::new(&GROTH16_PARAMETERS.sapling.output.vk),
crate::primitives::MAX_BATCH_SIZE, crate::primitives::MAX_BATCH_SIZE,
crate::primitives::MAX_BATCH_LATENCY, crate::primitives::MAX_BATCH_LATENCY,
), ),
tower::service_fn( tower::service_fn(
(|item: Item| { (|item: Item| {
ready(item.verify_single(&prepare_verifying_key(&PARAMS.sapling.output.vk))) ready(item.verify_single(&prepare_verifying_key(
&GROTH16_PARAMETERS.sapling.output.vk,
)))
}) as fn(_) -> _, }) as fn(_) -> _,
), ),
); );

2
zebra-consensus/src/transaction.rs
View File

@ -342,6 +342,8 @@ where
)?, )?,
}; };
// If the Groth16 parameter download hangs,
// Zebra will timeout here, waiting for the async checks.
async_checks.check().await?; async_checks.check().await?;
let mut spent_utxos = HashMap::new(); let mut spent_utxos = HashMap::new();

3
zebrad/Cargo.toml
View File

@ -16,10 +16,11 @@ zebra-network = { path = "../zebra-network" }
zebra-state = { path = "../zebra-state" } zebra-state = { path = "../zebra-state" }
abscissa_core = "0.5" abscissa_core = "0.5"
chrono = "0.4"
gumdrop = "0.7" gumdrop = "0.7"
lazy_static = "1.4.0"
serde = { version = "1", features = ["serde_derive"] } serde = { version = "1", features = ["serde_derive"] }
toml = "0.5" toml = "0.5"
chrono = "0.4"
hyper = { version = "0.14.15", features = ["full"] } hyper = { version = "0.14.15", features = ["full"] }
futures = "0.3" futures = "0.3"

10
zebrad/src/commands.rs
View File

@ -1,11 +1,12 @@
//! Zebrad Subcommands //! Zebrad Subcommands
mod download;
mod generate; mod generate;
mod start; mod start;
mod version; mod version;
use self::ZebradCmd::*; use self::ZebradCmd::*;
use self::{generate::GenerateCmd, start::StartCmd, version::VersionCmd}; use self::{download::DownloadCmd, generate::GenerateCmd, start::StartCmd, version::VersionCmd};
use crate::config::ZebradConfig; use crate::config::ZebradConfig;
@ -20,6 +21,10 @@ pub const CONFIG_FILE: &str = "zebrad.toml";
/// Zebrad Subcommands /// Zebrad Subcommands
#[derive(Command, Debug, Options)] #[derive(Command, Debug, Options)]
pub enum ZebradCmd { pub enum ZebradCmd {
/// The `download` subcommand
#[options(help = "pre-download required parameter files")]
Download(DownloadCmd),
/// The `generate` subcommand /// The `generate` subcommand
#[options(help = "generate a skeleton configuration")] #[options(help = "generate a skeleton configuration")]
Generate(GenerateCmd), Generate(GenerateCmd),
@ -45,7 +50,7 @@ impl ZebradCmd {
match self { match self {
// List all the commands, so new commands have to make a choice here // List all the commands, so new commands have to make a choice here
Start(_) => true, Start(_) => true,
Generate(_) | Help(_) | Version(_) => false, Download(_) | Generate(_) | Help(_) | Version(_) => false,
} }
} }
} }
@ -53,6 +58,7 @@ impl ZebradCmd {
impl Runnable for ZebradCmd { impl Runnable for ZebradCmd {
fn run(&self) { fn run(&self) {
match self { match self {
Download(cmd) => cmd.run(),
Generate(cmd) => cmd.run(), Generate(cmd) => cmd.run(),
ZebradCmd::Help(cmd) => cmd.run(), ZebradCmd::Help(cmd) => cmd.run(),
Start(cmd) => cmd.run(), Start(cmd) => cmd.run(),

35
zebrad/src/commands/download.rs Normal file
View File

@ -0,0 +1,35 @@
//! `download` subcommand - pre-download required parameter files
//!
//! `zebrad download` automatically downloads required paramter files the first time it is run.
//!
//! This command should be used if you're launching lots of `zebrad start` instances for testing,
//! or you want to include the parameter files in a distribution package.
use abscissa_core::{Command, Options, Runnable};
/// `download` subcommand
#[derive(Command, Debug, Default, Options)]
pub struct DownloadCmd {}
impl DownloadCmd {
/// Download the Sapling and Sprout Groth16 parameters if needed,
/// check they were downloaded correctly, and load them into Zebra.
///
/// # Panics
///
/// If the downloaded or pre-existing parameter files are invalid.
fn download_and_check(&self) {
// The lazy static initializer does the download, if needed,
// and the file hash checks.
lazy_static::initialize(&zebra_consensus::groth16::GROTH16_PARAMETERS);
}
}
impl Runnable for DownloadCmd {
/// Run the download command.
fn run(&self) {
info!("checking if Zcash Sapling and Sprout parameters have been downloaded");
self.download_and_check();
}
}

117
zebrad/src/commands/start.rs
View File

@ -50,8 +50,8 @@
use abscissa_core::{config, Command, FrameworkError, Options, Runnable}; use abscissa_core::{config, Command, FrameworkError, Options, Runnable};
use color_eyre::eyre::{eyre, Report}; use color_eyre::eyre::{eyre, Report};
use futures::{select, FutureExt}; use futures::FutureExt;
use tokio::sync::oneshot; use tokio::{pin, select, sync::oneshot};
use tower::{builder::ServiceBuilder, util::BoxService}; use tower::{builder::ServiceBuilder, util::BoxService};
use crate::{ use crate::{
@ -79,19 +79,18 @@ impl StartCmd {
info!(?config); info!(?config);
info!("initializing node state"); info!("initializing node state");
// TODO: use ChainTipChange to get tip changes (#2374, #2710, #2711, #2712, #2713, #2714)
let (state_service, latest_chain_tip, chain_tip_change) = let (state_service, latest_chain_tip, chain_tip_change) =
zebra_state::init(config.state.clone(), config.network.network); zebra_state::init(config.state.clone(), config.network.network);
let state = ServiceBuilder::new().buffer(20).service(state_service); let state = ServiceBuilder::new().buffer(20).service(state_service);
info!("initializing verifiers"); info!("initializing verifiers");
// TODO: use the transaction verifier to verify mempool transactions (#2637, #2606) let (chain_verifier, tx_verifier, mut groth16_download_handle) =
let (chain_verifier, tx_verifier) = zebra_consensus::chain::init( zebra_consensus::chain::init(
config.consensus.clone(), config.consensus.clone(),
config.network.network, config.network.network,
state.clone(), state.clone(),
) )
.await; .await;
info!("initializing network"); info!("initializing network");
// The service that our node uses to respond to requests by peers. The // The service that our node uses to respond to requests by peers. The
@ -133,7 +132,7 @@ impl StartCmd {
let syncer_error_future = syncer.sync(); let syncer_error_future = syncer.sync();
let sync_gossip_task_handle = tokio::spawn(sync::gossip_best_tip_block_hashes( let mut sync_gossip_task_handle = tokio::spawn(sync::gossip_best_tip_block_hashes(
sync_status.clone(), sync_status.clone(),
chain_tip_change.clone(), chain_tip_change.clone(),
peer_set.clone(), peer_set.clone(),
@ -154,25 +153,91 @@ impl StartCmd {
peer_set, peer_set,
)); ));
select! { info!("started initial Zebra tasks");
sync_result = syncer_error_future.fuse() => sync_result,
sync_gossip_result = sync_gossip_task_handle.fuse() => sync_gossip_result // TODO: spawn the syncer task, after making the PeerSet sync and send
.expect("unexpected panic in the chain tip block gossip task") // turn these tasks into a FuturesUnordered?
.map_err(|e| eyre!(e)),
mempool_crawl_result = mempool_crawler_task_handle.fuse() => mempool_crawl_result // ongoing futures & tasks
.expect("unexpected panic in the mempool crawler") pin!(syncer_error_future);
.map_err(|e| eyre!(e)), pin!(mempool_crawler_task_handle);
pin!(mempool_queue_checker_task_handle);
pin!(tx_gossip_task_handle);
mempool_queue_result = mempool_queue_checker_task_handle.fuse() => mempool_queue_result // startup tasks
.expect("unexpected panic in the mempool queue checker") let groth16_download_handle_fused = (&mut groth16_download_handle).fuse();
.map_err(|e| eyre!(e)), pin!(groth16_download_handle_fused);
tx_gossip_result = tx_gossip_task_handle.fuse() => tx_gossip_result // Wait for tasks to finish
.expect("unexpected panic in the transaction gossip task") let exit_status = loop {
.map_err(|e| eyre!(e)), let mut exit_when_task_finishes = true;
}
let result = select! {
// We don't spawn the syncer future into a separate task yet.
// So syncer panics automatically propagate to the main zebrad task.
sync_result = &mut syncer_error_future => sync_result
.map(|_| info!("syncer task exited")),
sync_gossip_result = &mut sync_gossip_task_handle => sync_gossip_result
.expect("unexpected panic in the chain tip block gossip task")
.map(|_| info!("chain tip block gossip task exited"))
.map_err(|e| eyre!(e)),
mempool_crawl_result = &mut mempool_crawler_task_handle => mempool_crawl_result
.expect("unexpected panic in the mempool crawler")
.map(|_| info!("mempool crawler task exited"))
.map_err(|e| eyre!(e)),
mempool_queue_result = &mut mempool_queue_checker_task_handle => mempool_queue_result
.expect("unexpected panic in the mempool queue checker")
.map(|_| info!("mempool queue checker task exited"))
.map_err(|e| eyre!(e)),
tx_gossip_result = &mut tx_gossip_task_handle => tx_gossip_result
.expect("unexpected panic in the transaction gossip task")
.map(|_| info!("transaction gossip task exited"))
.map_err(|e| eyre!(e)),
// Unlike other tasks, we expect the download task to finish while Zebra is running.
groth16_download_result = &mut groth16_download_handle_fused => {
groth16_download_result
.unwrap_or_else(|_| panic!(
"unexpected panic in the Groth16 pre-download and check task. {}",
zebra_consensus::groth16::Groth16Parameters::failure_hint())
);
info!("Groth16 pre-download and check task finished");
exit_when_task_finishes = false;
Ok(())
}
};
// Stop Zebra if a task finished and returned an error,
// or if an ongoing task exited.
if let Err(err) = result {
break Err(err);
}
if exit_when_task_finishes {
break Ok(());
}
};
info!("exiting Zebra because an ongoing task exited: stopping other tasks");
// futures
std::mem::drop(syncer_error_future);
// ongoing tasks
sync_gossip_task_handle.abort();
mempool_crawler_task_handle.abort();
mempool_queue_checker_task_handle.abort();
tx_gossip_task_handle.abort();
// startup tasks
groth16_download_handle.abort();
exit_status
} }
} }

3
zebrad/src/components/inbound/tests.rs
View File

@ -588,7 +588,8 @@ async fn setup(
let mut state_service = ServiceBuilder::new().buffer(1).service(state); let mut state_service = ServiceBuilder::new().buffer(1).service(state);
let (block_verifier, _transaction_verifier) = // Download task panics and timeouts are propagated to the tests that use Groth16 verifiers.
let (block_verifier, _transaction_verifier, _groth16_download_handle) =
zebra_consensus::chain::init(consensus_config.clone(), network, state_service.clone()) zebra_consensus::chain::init(consensus_config.clone(), network, state_service.clone())
.await; .await;