From 0ca22969215b958057d0e9495da8bbfaa2756e4e Mon Sep 17 00:00:00 2001 From: Nalin Bhardwaj Date: Fri, 16 Sep 2022 11:45:52 +0530 Subject: [PATCH 1/4] Vkey serialisation Signed-off-by: Daira Emma Hopwood --- halo2_proofs/src/plonk.rs | 70 ++++++++++++++++++++++++++- halo2_proofs/src/plonk/keygen.rs | 3 +- halo2_proofs/src/plonk/permutation.rs | 20 ++++++++ 3 files changed, 90 insertions(+), 3 deletions(-) diff --git a/halo2_proofs/src/plonk.rs b/halo2_proofs/src/plonk.rs index e96cf3fe..4f56e8c7 100644 --- a/halo2_proofs/src/plonk.rs +++ b/halo2_proofs/src/plonk.rs @@ -10,8 +10,8 @@ use group::ff::{Field, FromUniformBytes, PrimeField}; use crate::arithmetic::CurveAffine; use crate::poly::{ - Coeff, EvaluationDomain, ExtendedLagrangeCoeff, LagrangeCoeff, PinnedEvaluationDomain, - Polynomial, + commitment::Params, Coeff, EvaluationDomain, ExtendedLagrangeCoeff, LagrangeCoeff, + PinnedEvaluationDomain, Polynomial, }; use crate::transcript::{ChallengeScalar, EncodedChallenge, Transcript}; @@ -33,6 +33,7 @@ pub use keygen::*; pub use prover::*; pub use verifier::*; +use crate::helpers::CurveRead; use std::io; /// This is a verifying key which allows for the verification of proofs for a @@ -47,17 +48,81 @@ pub struct VerifyingKey { cs_degree: usize, /// The representative of this `VerifyingKey` in transcripts. transcript_repr: C::Scalar, + selectors: Vec>, } impl VerifyingKey where C::Scalar: FromUniformBytes<64>, { + /// Writes a verifying key to a buffer. + pub fn write(&self, writer: &mut W) -> io::Result<()> { + for commitment in &self.fixed_commitments { + writer.write_all(commitment.to_bytes().as_ref())?; + } + self.permutation.write(writer)?; + + // write self.selectors + for selector in &self.selectors { + let mut selector_bytes = vec![0u8; selector.len() / 8 + 1]; + for i in 0..selector.len() { + let byte_index = i / 8; + let bit_index = i % 8; + selector_bytes[byte_index] |= (selector[i] as u8) << bit_index; + } + writer.write_all(&selector_bytes)?; + } + + Ok(()) + } + + /// Reads a verification key from a buffer. + pub fn read>( + reader: &mut R, + params: &Params, + ) -> io::Result { + let (domain, cs, _) = keygen::create_domain::(params); + + let fixed_commitments: Vec<_> = (0..cs.num_fixed_columns) + .map(|_| C::read(reader)) + .collect::>()?; + + let permutation = permutation::VerifyingKey::read(reader, &cs.permutation)?; + + // read selectors + let selectors: Vec> = vec![vec![false; params.n as usize]; cs.num_selectors] + .into_iter() + .map(|mut selector| { + let mut selector_bytes = vec![0u8; selector.len() / 8 + 1]; + reader + .read_exact(&mut selector_bytes) + .expect("unable to read selector bytes"); + for i in 0..selector.len() { + let byte_index = i / 8; + let bit_index = i % 8; + selector[i] = (selector_bytes[byte_index] >> bit_index) & 1 == 1; + } + Ok(selector) + }) + .collect::>, &str>>() + .unwrap(); + let (cs, _) = cs.compress_selectors(selectors.clone()); + + Ok(Self::from_parts( + domain, + fixed_commitments, + permutation, + cs, + selectors, + )) + } + fn from_parts( domain: EvaluationDomain, fixed_commitments: Vec, permutation: permutation::VerifyingKey, cs: ConstraintSystem, + selectors: Vec>, ) -> Self { // Compute cached values. let cs_degree = cs.degree(); @@ -70,6 +135,7 @@ where cs_degree, // Temporary, this is not pinned. transcript_repr: C::Scalar::ZERO, + selectors, }; let mut hasher = Blake2bParams::new() diff --git a/halo2_proofs/src/plonk/keygen.rs b/halo2_proofs/src/plonk/keygen.rs index bb26d5a6..1b2be6d0 100644 --- a/halo2_proofs/src/plonk/keygen.rs +++ b/halo2_proofs/src/plonk/keygen.rs @@ -219,7 +219,7 @@ where )?; let mut fixed = batch_invert_assigned(assembly.fixed); - let (cs, selector_polys) = cs.compress_selectors(assembly.selectors); + let (cs, selector_polys) = cs.compress_selectors(assembly.selectors.clone()); fixed.extend( selector_polys .into_iter() @@ -240,6 +240,7 @@ where fixed_commitments, permutation_vk, cs, + assembly.selectors, )) } diff --git a/halo2_proofs/src/plonk/permutation.rs b/halo2_proofs/src/plonk/permutation.rs index c81de276..c7180f89 100644 --- a/halo2_proofs/src/plonk/permutation.rs +++ b/halo2_proofs/src/plonk/permutation.rs @@ -8,6 +8,9 @@ pub(crate) mod keygen; pub(crate) mod prover; pub(crate) mod verifier; +use crate::helpers::CurveRead; +use std::io; + /// A permutation argument. #[derive(Debug, Clone)] pub(crate) struct Argument { @@ -75,6 +78,23 @@ pub(crate) struct VerifyingKey { commitments: Vec, } +impl VerifyingKey { + pub(crate) fn write(&self, writer: &mut W) -> io::Result<()> { + for commitment in &self.commitments { + writer.write_all(commitment.to_bytes().as_ref())?; + } + + Ok(()) + } + + pub(crate) fn read(reader: &mut R, argument: &Argument) -> io::Result { + let commitments = (0..argument.columns.len()) + .map(|_| C::read(reader)) + .collect::, _>>()?; + Ok(VerifyingKey { commitments }) + } +} + /// The proving key for a single permutation argument. #[derive(Clone, Debug)] pub(crate) struct ProvingKey { From 3489dcde9e5bf273b4afeb50660b974108f36ec7 Mon Sep 17 00:00:00 2001 From: Jonathan Wang Date: Sat, 8 Oct 2022 15:55:42 -0700 Subject: [PATCH 2/4] fix: add `num_fixed_commitments` to vkey serialization so correct number of fixed columns, post selector compression, are read Signed-off-by: Daira Emma Hopwood --- halo2_proofs/src/plonk.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/halo2_proofs/src/plonk.rs b/halo2_proofs/src/plonk.rs index 4f56e8c7..04e1657b 100644 --- a/halo2_proofs/src/plonk.rs +++ b/halo2_proofs/src/plonk.rs @@ -57,6 +57,7 @@ where { /// Writes a verifying key to a buffer. pub fn write(&self, writer: &mut W) -> io::Result<()> { + writer.write_all(&(self.fixed_commitments.len() as u32).to_be_bytes())?; for commitment in &self.fixed_commitments { writer.write_all(commitment.to_bytes().as_ref())?; } @@ -65,10 +66,10 @@ where // write self.selectors for selector in &self.selectors { let mut selector_bytes = vec![0u8; selector.len() / 8 + 1]; - for i in 0..selector.len() { + for (i, selector_idx) in selector.iter().enumerate() { let byte_index = i / 8; let bit_index = i % 8; - selector_bytes[byte_index] |= (selector[i] as u8) << bit_index; + selector_bytes[byte_index] |= (*selector_idx as u8) << bit_index; } writer.write_all(&selector_bytes)?; } @@ -82,8 +83,11 @@ where params: &Params, ) -> io::Result { let (domain, cs, _) = keygen::create_domain::(params); + let mut num_fixed_columns_be_bytes = [0u8; 4]; + reader.read_exact(&mut num_fixed_columns_be_bytes)?; + let num_fixed_columns = u32::from_be_bytes(num_fixed_columns_be_bytes); - let fixed_commitments: Vec<_> = (0..cs.num_fixed_columns) + let fixed_commitments: Vec<_> = (0..num_fixed_columns) .map(|_| C::read(reader)) .collect::>()?; @@ -97,10 +101,10 @@ where reader .read_exact(&mut selector_bytes) .expect("unable to read selector bytes"); - for i in 0..selector.len() { + for (i, selector_idx) in selector.iter_mut().enumerate() { let byte_index = i / 8; let bit_index = i % 8; - selector[i] = (selector_bytes[byte_index] >> bit_index) & 1 == 1; + *selector_idx = (selector_bytes[byte_index] >> bit_index) & 1 == 1; } Ok(selector) }) From bae30538b5cecb5685bbe1883851c469ac8303c8 Mon Sep 17 00:00:00 2001 From: Daira Emma Hopwood Date: Mon, 1 May 2023 10:41:40 +0100 Subject: [PATCH 3/4] Apply review suggestions for VerifyingKey serialization: * write an initial 0x01 version byte, and check it on read; * use little-endian byte order for lengths; * add length fields before the selectors and the permutation commitments, and check their consistency with the expected lengths on read; * do not write an extra byte for a selector bit vector if the number of bits is a multiple of 8; * ensure that original I/O errors are preserved. Signed-off-by: Daira Emma Hopwood --- halo2_proofs/src/plonk.rs | 45 ++++++++++++++++++++------- halo2_proofs/src/plonk/permutation.rs | 14 +++++++-- 2 files changed, 45 insertions(+), 14 deletions(-) diff --git a/halo2_proofs/src/plonk.rs b/halo2_proofs/src/plonk.rs index 04e1657b..024cca5f 100644 --- a/halo2_proofs/src/plonk.rs +++ b/halo2_proofs/src/plonk.rs @@ -57,15 +57,19 @@ where { /// Writes a verifying key to a buffer. pub fn write(&self, writer: &mut W) -> io::Result<()> { - writer.write_all(&(self.fixed_commitments.len() as u32).to_be_bytes())?; + // Version byte that will be checked on read. + writer.write_all(&[0x01])?; + + writer.write_all(&(u32::try_from(self.fixed_commitments.len()).unwrap()).to_le_bytes())?; for commitment in &self.fixed_commitments { writer.write_all(commitment.to_bytes().as_ref())?; } self.permutation.write(writer)?; // write self.selectors + writer.write_all(&(u32::try_from(self.selectors.len()).unwrap()).to_le_bytes())?; for selector in &self.selectors { - let mut selector_bytes = vec![0u8; selector.len() / 8 + 1]; + let mut selector_bytes = vec![0u8; (selector.len() + 7) / 8]; for (i, selector_idx) in selector.iter().enumerate() { let byte_index = i / 8; let bit_index = i % 8; @@ -83,24 +87,41 @@ where params: &Params, ) -> io::Result { let (domain, cs, _) = keygen::create_domain::(params); - let mut num_fixed_columns_be_bytes = [0u8; 4]; - reader.read_exact(&mut num_fixed_columns_be_bytes)?; - let num_fixed_columns = u32::from_be_bytes(num_fixed_columns_be_bytes); + + let mut version_byte = [0u8; 1]; + reader.read_exact(&mut version_byte)?; + if 0x01 != version_byte[0] { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "unexpected version byte", + )); + } + + let mut num_fixed_columns_le_bytes = [0u8; 4]; + reader.read_exact(&mut num_fixed_columns_le_bytes)?; + let num_fixed_columns = u32::from_le_bytes(num_fixed_columns_le_bytes); let fixed_commitments: Vec<_> = (0..num_fixed_columns) .map(|_| C::read(reader)) - .collect::>()?; + .collect::>()?; let permutation = permutation::VerifyingKey::read(reader, &cs.permutation)?; // read selectors + let mut num_selectors_le_bytes = [0u8; 4]; + reader.read_exact(&mut num_selectors_le_bytes)?; + let num_selectors = u32::from_le_bytes(num_selectors_le_bytes); + if cs.num_selectors != num_selectors.try_into().unwrap() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "unexpected number of selectors", + )); + } let selectors: Vec> = vec![vec![false; params.n as usize]; cs.num_selectors] .into_iter() .map(|mut selector| { - let mut selector_bytes = vec![0u8; selector.len() / 8 + 1]; - reader - .read_exact(&mut selector_bytes) - .expect("unable to read selector bytes"); + let mut selector_bytes = vec![0u8; (selector.len() + 7) / 8]; + reader.read_exact(&mut selector_bytes)?; for (i, selector_idx) in selector.iter_mut().enumerate() { let byte_index = i / 8; let bit_index = i % 8; @@ -108,8 +129,8 @@ where } Ok(selector) }) - .collect::>, &str>>() - .unwrap(); + .collect::>()?; + let (cs, _) = cs.compress_selectors(selectors.clone()); Ok(Self::from_parts( diff --git a/halo2_proofs/src/plonk/permutation.rs b/halo2_proofs/src/plonk/permutation.rs index c7180f89..d6b00994 100644 --- a/halo2_proofs/src/plonk/permutation.rs +++ b/halo2_proofs/src/plonk/permutation.rs @@ -80,6 +80,7 @@ pub(crate) struct VerifyingKey { impl VerifyingKey { pub(crate) fn write(&self, writer: &mut W) -> io::Result<()> { + writer.write_all(&(u32::try_from(self.commitments.len()).unwrap()).to_le_bytes())?; for commitment in &self.commitments { writer.write_all(commitment.to_bytes().as_ref())?; } @@ -88,9 +89,18 @@ impl VerifyingKey { } pub(crate) fn read(reader: &mut R, argument: &Argument) -> io::Result { - let commitments = (0..argument.columns.len()) + let mut num_commitments_le_bytes = [0u8; 4]; + reader.read_exact(&mut num_commitments_le_bytes)?; + let num_commitments = u32::from_le_bytes(num_commitments_le_bytes); + if argument.columns.len() != num_commitments.try_into().unwrap() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "unexpected number of column commitments", + )); + } + let commitments: Vec<_> = (0..argument.columns.len()) .map(|_| C::read(reader)) - .collect::, _>>()?; + .collect::>()?; Ok(VerifyingKey { commitments }) } } From 58e797da937c5193ce43ad333f22ead664f1ba61 Mon Sep 17 00:00:00 2001 From: Daira Emma Hopwood Date: Mon, 1 May 2023 16:23:49 +0100 Subject: [PATCH 4/4] feat: added `to/from_bytes` for `ProvingKey` and `VerifyingKey` * add `pack`/`unpack` helper functions between `&[bool]` and `u8`. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extracted from https://github.com/privacy-scaling-explorations/halo2/pull/103/commits/2c7f2889325370142963cab68c31ffebe83c50d6 Co-authored-by: Carlos Pérez <37264926+CPerezz@users.noreply.github.com> Signed-off-by: Daira Emma Hopwood --- halo2_proofs/src/helpers.rs | 19 +++++++++++ halo2_proofs/src/plonk.rs | 49 ++++++++++++++++++++------- halo2_proofs/src/plonk/permutation.rs | 4 +++ 3 files changed, 60 insertions(+), 12 deletions(-) diff --git a/halo2_proofs/src/helpers.rs b/halo2_proofs/src/helpers.rs index 2158a369..e56758fb 100644 --- a/halo2_proofs/src/helpers.rs +++ b/halo2_proofs/src/helpers.rs @@ -14,3 +14,22 @@ pub(crate) trait CurveRead: CurveAffine { } impl CurveRead for C {} + +/// Converts a slice of `bool` into a `u8`. +/// +/// Panics if the slice has length greater than 8. +pub fn pack(bits: &[bool]) -> u8 { + let mut value = 0u8; + assert!(bits.len() <= 8); + for (bit_index, bit) in bits.iter().enumerate() { + value |= (*bit as u8) << bit_index; + } + value +} + +/// Writes the first `bits.len()` bits of a `u8` into `bits`. +pub fn unpack(byte: u8, bits: &mut [bool]) { + for (bit_index, bit) in bits.iter_mut().enumerate() { + *bit = (byte >> bit_index) & 1 == 1; + } +} diff --git a/halo2_proofs/src/plonk.rs b/halo2_proofs/src/plonk.rs index 024cca5f..3557517a 100644 --- a/halo2_proofs/src/plonk.rs +++ b/halo2_proofs/src/plonk.rs @@ -9,6 +9,7 @@ use blake2b_simd::Params as Blake2bParams; use group::ff::{Field, FromUniformBytes, PrimeField}; use crate::arithmetic::CurveAffine; +use crate::helpers::{pack, unpack, CurveRead}; use crate::poly::{ commitment::Params, Coeff, EvaluationDomain, ExtendedLagrangeCoeff, LagrangeCoeff, PinnedEvaluationDomain, Polynomial, @@ -33,7 +34,6 @@ pub use keygen::*; pub use prover::*; pub use verifier::*; -use crate::helpers::CurveRead; use std::io; /// This is a verifying key which allows for the verification of proofs for a @@ -69,19 +69,16 @@ where // write self.selectors writer.write_all(&(u32::try_from(self.selectors.len()).unwrap()).to_le_bytes())?; for selector in &self.selectors { - let mut selector_bytes = vec![0u8; (selector.len() + 7) / 8]; - for (i, selector_idx) in selector.iter().enumerate() { - let byte_index = i / 8; - let bit_index = i % 8; - selector_bytes[byte_index] |= (*selector_idx as u8) << bit_index; + // since `selector` is filled with `bool`, we pack them 8 at a time into bytes and then write + for bits in selector.chunks(8) { + writer.write_all(&[pack(bits)])?; } - writer.write_all(&selector_bytes)?; } Ok(()) } - /// Reads a verification key from a buffer. + /// Reads a verifying key from a buffer. pub fn read>( reader: &mut R, params: &Params, @@ -122,10 +119,8 @@ where .map(|mut selector| { let mut selector_bytes = vec![0u8; (selector.len() + 7) / 8]; reader.read_exact(&mut selector_bytes)?; - for (i, selector_idx) in selector.iter_mut().enumerate() { - let byte_index = i / 8; - let bit_index = i % 8; - *selector_idx = (selector_bytes[byte_index] >> bit_index) & 1 == 1; + for (bits, byte) in selector.chunks_mut(8).zip(selector_bytes) { + unpack(byte, bits); } Ok(selector) }) @@ -142,6 +137,36 @@ where )) } + /// Writes a verifying key to a vector of bytes. + pub fn to_bytes(&self) -> Vec { + let mut bytes = Vec::::with_capacity(self.bytes_length()); + self.write(&mut bytes) + .expect("Writing to vector should not fail"); + bytes + } + + /// Reads a verifying key from a slice of bytes. + pub fn from_bytes>( + mut bytes: &[u8], + params: &Params, + ) -> io::Result { + Self::read::<_, ConcreteCircuit>(&mut bytes, params) + } + + /// Gets the total number of bytes in the serialization of `self`. + fn bytes_length(&self) -> usize { + 1 + 4 + + self.fixed_commitments.len() * C::default().to_bytes().as_ref().len() + + self.permutation.bytes_length() + + 4 + + self.selectors.len() + * self + .selectors + .get(0) + .map(|selector| (selector.len() + 7) / 8) + .unwrap_or(0) + } + fn from_parts( domain: EvaluationDomain, fixed_commitments: Vec, diff --git a/halo2_proofs/src/plonk/permutation.rs b/halo2_proofs/src/plonk/permutation.rs index d6b00994..2f001715 100644 --- a/halo2_proofs/src/plonk/permutation.rs +++ b/halo2_proofs/src/plonk/permutation.rs @@ -103,6 +103,10 @@ impl VerifyingKey { .collect::>()?; Ok(VerifyingKey { commitments }) } + + pub(crate) fn bytes_length(&self) -> usize { + 4 + self.commitments.len() * C::default().to_bytes().as_ref().len() + } } /// The proving key for a single permutation argument.