Implement Async Batch verification API for groth16

This PR is the first step in getting a groth16 proving system fully
integrated with the rest of zebra. This PR implements the initial async
API, but none of the actual batching logic necessary for our eventual
verifier design.

Once the batch verification API from bellman has been implemented we
will need to swap out the "Batch" type defined in this crate with the
new `batch::Verifier` defined in bellman.
This commit is contained in:
Jane Lusby 2021-02-03 16:13:34 -08:00 committed by Deirdre Connolly
parent b3a3b8f7c7
commit 0ac259430a
7 changed files with 366 additions and 45 deletions

125
Cargo.lock generated
View File

@ -204,6 +204,26 @@ version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "474a626a67200bd107d44179bb3d4fc61891172d11696609264589be6a0e6a43"
[[package]]
name = "bellman"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7089887635778eabf0038a166f586eee5413fb85c8fa6c9a754914f0f644f49f"
dependencies = [
"bitvec 0.18.4",
"blake2s_simd",
"byteorder",
"crossbeam",
"ff 0.8.0",
"futures 0.1.30",
"futures-cpupool",
"group 0.8.0",
"num_cpus",
"pairing",
"rand_core 0.5.1",
"subtle",
]
[[package]]
name = "bincode"
version = "1.3.1"
@ -278,6 +298,17 @@ dependencies = [
"radium 0.3.0",
]
[[package]]
name = "bitvec"
version = "0.18.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d2838fdd79e8776dbe07a106c784b0f8dda571a21b2750a092cc4cbaa653c8e"
dependencies = [
"funty",
"radium 0.4.1",
"wyz",
]
[[package]]
name = "bitvec"
version = "0.20.1"
@ -351,13 +382,27 @@ dependencies = [
"subtle",
]
[[package]]
name = "bls12_381"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4caf0101205582491f772d60a6fcb6bcec19963e68209cb631851eeadb01421f"
dependencies = [
"bitvec 0.18.4",
"ff 0.8.0",
"group 0.8.0",
"pairing",
"rand_core 0.5.1",
"subtle",
]
[[package]]
name = "bls12_381"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c56609cc42c628848e7b18e0baf42a4ef626b8c50442dc08b8094bd21d8ad32"
dependencies = [
"ff",
"ff 0.9.0",
"rand_core 0.6.1",
"subtle",
]
@ -949,6 +994,17 @@ dependencies = [
"once_cell",
]
[[package]]
name = "ff"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "01646e077d4ebda82b73f1bca002ea1e91561a77df2431a9e79729bcc31950ef"
dependencies = [
"bitvec 0.18.4",
"rand_core 0.5.1",
"subtle",
]
[[package]]
name = "ff"
version = "0.9.0"
@ -1028,6 +1084,12 @@ version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ba62103ce691c2fd80fbae2213dfdda9ce60804973ac6b6e97de818ea7f52c8"
[[package]]
name = "futures"
version = "0.1.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c7e4c2612746b0df8fed4ce0c69156021b704c9aefa360311c04e6e9e002eed"
[[package]]
name = "futures"
version = "0.3.12"
@ -1059,6 +1121,16 @@ version = "0.3.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "79e5145dde8da7d1b3892dad07a9c98fc04bc39892b1ecc9692cf53e2b780a65"
[[package]]
name = "futures-cpupool"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab90cde24b3319636588d0c35fe03b1333857621051837ed769faefb4c2162e4"
dependencies = [
"futures 0.1.30",
"num_cpus",
]
[[package]]
name = "futures-executor"
version = "0.3.12"
@ -1198,6 +1270,18 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
[[package]]
name = "group"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cc11f9f5fbf1943b48ae7c2bf6846e7d827a512d1be4f23af708f5ca5d01dde1"
dependencies = [
"byteorder",
"ff 0.8.0",
"rand_core 0.5.1",
"subtle",
]
[[package]]
name = "group"
version = "0.9.0"
@ -1205,7 +1289,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61b3c1e8b4f1ca07e6605ea1be903a5f6956aec5c8a67fd44d56076631675ed8"
dependencies = [
"byteorder",
"ff",
"ff 0.9.0",
"rand_core 0.6.1",
"subtle",
]
@ -1573,8 +1657,8 @@ checksum = "4d7e7fef85ae7b26dd89f34175b7f3c5ace64067a110c2ac86cf92407a6666ca"
dependencies = [
"bitvec 0.20.1",
"bls12_381 0.4.0",
"ff",
"group",
"ff 0.9.0",
"group 0.9.0",
"rand_core 0.6.1",
"subtle",
]
@ -1981,6 +2065,16 @@ version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2386b4ebe91c2f7f51082d4cefa145d030e33a1842a96b12e4885cc3c01f7a55"
[[package]]
name = "pairing"
version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4f702cdbee9e0a6272452c20dec82465bc821116598b4eeb63e9a71a69dbf7fd"
dependencies = [
"ff 0.8.0",
"group 0.8.0",
]
[[package]]
name = "parity-scale-codec"
version = "2.0.0"
@ -2242,6 +2336,12 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "def50a86306165861203e7f84ecffbbdfdea79f0e51039b33de1e952358c47ac"
[[package]]
name = "radium"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "64de9a0c5361e034f1aefc9f71a86871ec870e766fe31a009734a989b329286a"
[[package]]
name = "radium"
version = "0.6.2"
@ -3295,7 +3395,7 @@ version = "0.1.0"
dependencies = [
"color-eyre",
"ed25519-zebra 2.2.0",
"futures",
"futures 0.3.12",
"futures-core",
"pin-project 0.4.27",
"rand 0.7.3",
@ -3880,7 +3980,7 @@ dependencies = [
"displaydoc",
"ed25519-zebra 1.0.1",
"equihash",
"futures",
"futures 0.3.12",
"hex",
"jubjub 0.6.0",
"lazy_static",
@ -3909,14 +4009,17 @@ version = "1.0.0-alpha.0"
name = "zebra-consensus"
version = "1.0.0-alpha.1"
dependencies = [
"bellman",
"bls12_381 0.3.1",
"chrono",
"color-eyre",
"displaydoc",
"futures",
"futures 0.3.12",
"futures-util",
"jubjub 0.6.0",
"metrics",
"once_cell",
"pairing",
"rand 0.7.3",
"redjubjub",
"serde",
@ -3944,7 +4047,7 @@ dependencies = [
"byteorder",
"bytes 0.6.0",
"chrono",
"futures",
"futures 0.3.12",
"hex",
"indexmap",
"lazy_static",
@ -3991,7 +4094,7 @@ dependencies = [
"color-eyre",
"dirs",
"displaydoc",
"futures",
"futures 0.3.12",
"hex",
"lazy_static",
"metrics",
@ -4018,7 +4121,7 @@ name = "zebra-test"
version = "1.0.0-alpha.1"
dependencies = [
"color-eyre",
"futures",
"futures 0.3.12",
"hex",
"lazy_static",
"owo-colors",
@ -4059,7 +4162,7 @@ dependencies = [
"chrono",
"color-eyre",
"dirs",
"futures",
"futures 0.3.12",
"gumdrop",
"hyper 0.14.0-dev",
"inferno",

View File

@ -7,14 +7,16 @@ use super::{
use crate::semaphore::Semaphore;
use futures_core::ready;
use std::task::{Context, Poll};
use std::{
fmt,
task::{Context, Poll},
};
use tokio::sync::{mpsc, oneshot};
use tower::Service;
/// Allows batch processing of requests.
///
/// See the module documentation for more details.
#[derive(Debug)]
pub struct Batch<T, Request>
where
T: Service<BatchControl<Request>>,
@ -35,6 +37,20 @@ where
handle: Handle,
}
impl<T, Request> fmt::Debug for Batch<T, Request>
where
T: Service<BatchControl<Request>>,
{
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let name = std::any::type_name::<Self>();
f.debug_struct(name)
.field("tx", &self.tx)
.field("semaphore", &self.semaphore)
.field("handle", &self.handle)
.finish()
}
}
impl<T, Request> Batch<T, Request>
where
T: Service<BatchControl<Request>>,

View File

@ -13,6 +13,8 @@ once_cell = "1.5"
rand = "0.7"
redjubjub = "0.2"
serde = { version = "1", features = ["serde_derive"] }
bellman = "0.8"
bls12_381 = "0.3.1"
futures = "0.3.12"
futures-util = "0.3.6"
@ -28,6 +30,7 @@ tower-batch = { path = "../tower-batch/" }
zebra-chain = { path = "../zebra-chain" }
zebra-state = { path = "../zebra-state" }
zebra-script = { path = "../zebra-script" }
pairing = "0.18.0"
[dev-dependencies]
color-eyre = "0.5.10"

View File

@ -5,5 +5,13 @@ pub mod redjubjub;
/// The maximum batch size for any of the batch verifiers.
const MAX_BATCH_SIZE: usize = 64;
/// The maximum latency bound for any of the batch verifiers.
const MAX_BATCH_LATENCY: std::time::Duration = std::time::Duration::from_millis(100);
/// The size of the buffer in the broadcast channels used by batch verifiers.
///
/// This bound limits the number of concurrent batches for each verifier.
/// If tasks delay checking for verifier results, and the bound is too small,
/// new batches will be rejected with `RecvError`s.
const BROADCAST_BUFFER_SIZE: usize = 512;

View File

@ -1,55 +1,244 @@
//! Async Groth16 batch verifier service
use std::{
fmt,
future::Future,
mem,
pin::Pin,
task::{Context, Poll},
};
use bellman::{
groth16::{PreparedVerifyingKey, Proof},
VerificationError,
};
use bls12_381::Bls12;
use pairing::MultiMillerLoop;
use rand::{thread_rng, CryptoRng, RngCore};
use tokio::sync::broadcast::{channel, error::RecvError, Sender};
use tower::Service;
use zebra_chain::primitives::Groth16Proof;
use tower_batch::BatchControl;
use tower_fallback::Fallback;
use crate::BoxError;
/// Provides verification of Groth16 proofs for a specific statement.
///
/// Groth16 proofs require a proof verification key; the [`Verifier`] type is
/// responsible for ownership of the PVK.
pub struct Verifier {
// XXX this needs to hold on to a verification key
// === TEMPORARY BATCH BELLMAN SUBSTITUTE ===
// These types are meant to be API compatible with the work in progress batch
// verification API being implemented in Bellman. Once we've finished that
// implementation and upgraded our dependency, we should be able to remove this
// section of code and replace each of these types with the commented out items
// from the rest of this file.
#[derive(Clone)]
pub struct Item<E: MultiMillerLoop> {
proof: Proof<E>,
public_inputs: Vec<E::Fr>,
}
impl Verifier {
/// Create a new Groth16 verifier, supplying the encoding of the verification key.
pub fn new(_encoded_verification_key: &[u8]) -> Result<Self, BoxError> {
// parse and turn into a bellman type,
// so that users don't have to have the entire bellman api
unimplemented!();
impl<E: MultiMillerLoop> Item<E> {
fn verify_single(self, pvk: &PreparedVerifyingKey<E>) -> Result<(), VerificationError> {
let Item {
proof,
public_inputs,
} = self;
bellman::groth16::verify_proof(pvk, &proof, &public_inputs)
}
}
// XXX this is copied from the WIP batch bellman impl,
// in the future, replace with a re export
pub struct Item {
pub proof: Groth16Proof,
pub public_inputs: Vec<jubjub::Fr>,
impl<E: MultiMillerLoop> From<(&Proof<E>, &[E::Fr])> for Item<E> {
fn from((proof, public_inputs): (&Proof<E>, &[E::Fr])) -> Self {
(proof.clone(), public_inputs.to_owned()).into()
}
}
// XXX in the future, Verifier will implement
// Service<BatchControl<Item>>> and be wrapped in a Batch
// to get a Service<Item>
// but for now, just implement Service<Item> and do unbatched verif.
//impl Service<BatchControl<Item>> for Verifier {
impl Service<Item> for Verifier {
impl<E: MultiMillerLoop> From<(Proof<E>, Vec<E::Fr>)> for Item<E> {
fn from((proof, public_inputs): (Proof<E>, Vec<E::Fr>)) -> Self {
Self {
proof,
public_inputs,
}
}
}
#[derive(Default)]
struct Batch {
queue: Vec<Item<Bls12>>,
}
impl Batch {
fn queue(&mut self, item: Item<Bls12>) {
self.queue.push(item);
}
fn verify<R: RngCore + CryptoRng>(
self,
_rng: R,
pvk: &PreparedVerifyingKey<Bls12>,
) -> Result<(), VerificationError> {
for item in self.queue {
item.verify_single(pvk)?;
}
Ok(())
}
}
// === TEMPORARY BATCH BELLMAN SUBSTITUTE END ===
// /// A Groth16 verification item, used as the request type of the service.
// pub type Item = batch::Item<Bls12>;
/// Groth16 signature verifier service
#[derive(Clone, Debug)]
pub struct Verifier {
inner: Fallback<tower_batch::Batch<VerifierImpl, Item<Bls12>>, FallbackVerifierImpl>,
}
impl Verifier {
/// Constructs a new verifier.
pub fn new(pvk: &'static PreparedVerifyingKey<Bls12>) -> Self {
let verifier_impl = VerifierImpl::new(pvk);
let fallback_impl = FallbackVerifierImpl::new(pvk);
let max_items = super::MAX_BATCH_SIZE;
let max_latency = super::MAX_BATCH_LATENCY;
let inner = tower_batch::Batch::new(verifier_impl, max_items, max_latency);
let inner = Fallback::new(inner, fallback_impl);
Self { inner }
}
}
impl Service<Item<Bls12>> for Verifier {
type Response = ();
type Error = BoxError;
type Future = Pin<Box<dyn Future<Output = Result<(), BoxError>> + Send + 'static>>;
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
self.inner.poll_ready(cx)
}
fn call(&mut self, req: Item<Bls12>) -> Self::Future {
use futures::FutureExt;
self.inner.call(req).boxed()
}
}
/// Groth16 signature verifier implementation
///
/// This is the core implementation for the batch verification logic of the groth
/// verifier. It handles batching incoming requests, driving batches to
/// completion, and reporting results.
struct VerifierImpl {
// batch: batch::Verifier<Bls12>,
batch: Batch,
// Making this 'static makes managing lifetimes much easier.
pvk: &'static PreparedVerifyingKey<Bls12>,
/// Broadcast sender used to send the result of a batch verification to each
/// request source in the batch.
tx: Sender<Result<(), VerificationError>>,
}
impl VerifierImpl {
fn new(pvk: &'static PreparedVerifyingKey<Bls12>) -> Self {
// let batch = batch::Verifier::default();
let batch = Batch::default();
let (tx, _) = channel(super::BROADCAST_BUFFER_SIZE);
Self { batch, tx, pvk }
}
}
impl fmt::Debug for VerifierImpl {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let name = "VerifierImpl";
f.debug_struct(name)
.field("batch", &"..")
.field("pvk", &"..")
.field("tx", &self.tx)
.finish()
}
}
impl Service<BatchControl<Item<Bls12>>> for VerifierImpl {
type Response = ();
type Error = VerificationError;
type Future = Pin<Box<dyn Future<Output = Result<(), VerificationError>> + Send + 'static>>;
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
Poll::Ready(Ok(()))
}
fn call(&mut self, _req: Item) -> Self::Future {
unimplemented!()
fn call(&mut self, req: BatchControl<Item<Bls12>>) -> Self::Future {
match req {
BatchControl::Item(item) => {
tracing::trace!("got item");
self.batch.queue(item);
let mut rx = self.tx.subscribe();
Box::pin(async move {
match rx.recv().await {
Ok(result) => result,
Err(RecvError::Lagged(_)) => {
tracing::error!(
"missed channel updates, BROADCAST_BUFFER_SIZE is too low!!"
);
Err(VerificationError::InvalidProof)
}
Err(RecvError::Closed) => panic!("verifier was dropped without flushing"),
}
})
}
BatchControl::Flush => {
tracing::trace!("got flush command");
let batch = mem::take(&mut self.batch);
let _ = self.tx.send(batch.verify(thread_rng(), self.pvk));
Box::pin(async { Ok(()) })
}
}
}
}
impl Drop for VerifierImpl {
fn drop(&mut self) {
// We need to flush the current batch in case there are still any pending futures.
let batch = mem::take(&mut self.batch);
let _ = self.tx.send(batch.verify(thread_rng(), self.pvk));
}
}
/// Groth16 signature verifier fallback implementation
#[derive(Clone)]
struct FallbackVerifierImpl {
pvk: &'static PreparedVerifyingKey<Bls12>,
}
impl FallbackVerifierImpl {
fn new(pvk: &'static PreparedVerifyingKey<Bls12>) -> Self {
Self { pvk }
}
}
impl fmt::Debug for FallbackVerifierImpl {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let name = "FallbackVerifierImpl";
f.debug_struct(name).field("pvk", &"..").finish()
}
}
impl Service<Item<Bls12>> for FallbackVerifierImpl {
type Response = ();
type Error = VerificationError;
type Future = Pin<Box<dyn Future<Output = Result<(), VerificationError>> + Send + 'static>>;
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
Poll::Ready(Ok(()))
}
fn call(&mut self, item: Item<Bls12>) -> Self::Future {
tracing::trace!("got item");
let pvk = self.pvk;
Box::pin(async move { item.verify_single(pvk) })
}
}

View File

@ -62,10 +62,7 @@ pub struct Verifier {
impl Default for Verifier {
fn default() -> Self {
let batch = batch::Verifier::default();
// This bound limits the number of concurrent batches for this verifier.
// If tasks delay checking for verifier results, and the bound is too small,
// new batches will be rejected with `RecvError`s.
let (tx, _) = channel(512);
let (tx, _) = channel(super::BROADCAST_BUFFER_SIZE);
Self { tx, batch }
}
}

View File

@ -44,9 +44,14 @@ where
{
// XXX: how should this struct be constructed?
pub fn new(network: Network, script_verifier: script::Verifier<ZS>) -> Self {
// let (spend_verifier, output_verifier, joinsplit_verifier) = todo!();
Self {
network,
script_verifier,
// spend_verifier,
// output_verifier,
// joinsplit_verifier,
}
}
}