blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

adds shred::layout::get_signed_data (#29438) 

Working towards removing merkle root from shreds payload, the commit
implements api to obtain signed data from shreds binary.
This commit is contained in:
behzad nouri 2022-12-30 18:52:10 +00:00 committed by GitHub
parent 1d456a82a9
commit 50afb80f52
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 185 additions and 138 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

117
ledger/src/shred.rs
View File

@ -309,7 +309,7 @@ use dispatch;
impl Shred {
dispatch!(fn common_header(&self) -> &ShredCommonHeader);
dispatch!(fn set_signature(&mut self, signature: Signature));
dispatch!(fn signed_message(&self) -> &[u8]);
dispatch!(fn signed_data(&self) -> &[u8]);
// Returns the portion of the shred's payload which is erasure coded.
dispatch!(pub(crate) fn erasure_shard(self) -> Result<Vec<u8>, Error>);
@ -460,7 +460,7 @@ impl Shred {
}
pub fn sign(&mut self, keypair: &Keypair) {
let signature = keypair.sign_message(self.signed_message());
let signature = keypair.sign_message(self.signed_data());
self.set_signature(signature);
}
@ -508,8 +508,8 @@ impl Shred {
#[must_use]
pub fn verify(&self, pubkey: &Pubkey) -> bool {
let message = self.signed_message();
self.signature().verify(pubkey.as_ref(), message)
let data = self.signed_data();
self.signature().verify(pubkey.as_ref(), data)
}
// Returns true if the erasure coding of the two shreds mismatch.
@ -538,9 +538,26 @@ impl Shred {
// Helper methods to extract pieces of the shred from the payload
// without deserializing the entire payload.
pub mod layout {
use {super::*, crate::shred::merkle::MerkleRoot, std::ops::Range};
#[cfg(test)]
use crate::shred::merkle::MerkleRoot;
use {super::*, std::ops::Range};
use {
rand::{seq::SliceRandom, Rng},
std::collections::HashMap,
};
pub(crate) enum SignedData<'a> {
Chunk(&'a [u8]),
MerkleRoot(MerkleRoot),
}
impl<'a> AsRef<[u8]> for SignedData<'a> {
fn as_ref(&self) -> &[u8] {
match self {
Self::Chunk(chunk) => chunk,
Self::MerkleRoot(root) => root,
}
}
}
fn get_shred_size(packet: &Packet) -> Option<usize> {
let size = packet.data(..)?.len();
@ -615,18 +632,36 @@ pub mod layout {
))
}
// Returns slice range of the shred payload which is signed.
pub(crate) fn get_signed_message_range(shred: &[u8]) -> Option<Range<usize>> {
let range = match get_shred_variant(shred).ok()? {
ShredVariant::LegacyCode | ShredVariant::LegacyData => legacy::SIGNED_MESSAGE_RANGE,
pub(crate) fn get_signed_data(shred: &[u8]) -> Option<SignedData> {
let data = match get_shred_variant(shred).ok()? {
ShredVariant::LegacyCode | ShredVariant::LegacyData => {
let chunk = shred.get(self::legacy::SIGNED_MESSAGE_OFFSETS)?;
SignedData::Chunk(chunk)
}
ShredVariant::MerkleCode(proof_size) => {
merkle::ShredCode::get_signed_message_range(proof_size)?
let merkle_root = self::merkle::ShredCode::get_merkle_root(shred, proof_size)?;
SignedData::MerkleRoot(merkle_root)
}
ShredVariant::MerkleData(proof_size) => {
merkle::ShredData::get_signed_message_range(proof_size)?
let merkle_root = self::merkle::ShredData::get_merkle_root(shred, proof_size)?;
SignedData::MerkleRoot(merkle_root)
}
};
(range.end <= shred.len()).then_some(range)
Some(data)
}
// Returns offsets within the shred payload which is signed.
pub(crate) fn get_signed_data_offsets(shred: &[u8]) -> Option<Range<usize>> {
let offsets = match get_shred_variant(shred).ok()? {
ShredVariant::LegacyCode | ShredVariant::LegacyData => legacy::SIGNED_MESSAGE_OFFSETS,
ShredVariant::MerkleCode(proof_size) => {
merkle::ShredCode::get_signed_data_offsets(proof_size)?
}
ShredVariant::MerkleData(proof_size) => {
merkle::ShredData::get_signed_data_offsets(proof_size)?
}
};
(offsets.end <= shred.len()).then_some(offsets)
}
pub(crate) fn get_reference_tick(shred: &[u8]) -> Result<u8, Error> {
@ -652,6 +687,62 @@ pub mod layout {
}
}
}
// Minimally corrupts the packet so that the signature no longer verifies.
#[cfg(test)]
pub(crate) fn corrupt_packet<R: Rng>(
rng: &mut R,
packet: &mut Packet,
keypairs: &HashMap<Slot, Keypair>,
) {
fn modify_packet<R: Rng>(rng: &mut R, packet: &mut Packet, offsets: Range<usize>) {
let buffer = packet.buffer_mut();
let byte = buffer[offsets].choose_mut(rng).unwrap();
*byte = rng.gen::<u8>().max(1u8).wrapping_add(*byte);
}
let shred = get_shred(packet).unwrap();
let merkle_proof_size = match get_shred_variant(shred).unwrap() {
ShredVariant::LegacyCode | ShredVariant::LegacyData => None,
ShredVariant::MerkleCode(proof_size) | ShredVariant::MerkleData(proof_size) => {
Some(proof_size)
}
};
let coin_flip: bool = rng.gen();
if coin_flip {
// Corrupt one byte within the signature offsets.
modify_packet(rng, packet, 0..SIGNATURE_BYTES);
} else {
// Corrupt one byte within the signed data offsets.
let size = shred.len();
let offsets = get_signed_data_offsets(shred).unwrap();
modify_packet(rng, packet, offsets);
if let Some(proof_size) = merkle_proof_size {
// Also need to corrupt the merkle proof.
// Proof entries are each 20 bytes at the end of shreds.
let offset = usize::from(proof_size) * 20;
modify_packet(rng, packet, size - offset..size);
}
}
// Assert that the signature no longer verifies.
let shred = get_shred(packet).unwrap();
let slot = get_slot(shred).unwrap();
let signature = get_signature(shred).unwrap();
if coin_flip {
let pubkey = keypairs[&slot].pubkey();
let data = get_signed_data(shred).unwrap();
assert!(!signature.verify(pubkey.as_ref(), data.as_ref()));
let offsets = get_signed_data_offsets(shred).unwrap();
assert!(!signature.verify(pubkey.as_ref(), &shred[offsets]));
} else {
// Slot may have been corrupted and no longer mapping to a keypair.
let pubkey = keypairs.get(&slot).map(Keypair::pubkey).unwrap_or_default();
if let Some(data) = get_signed_data(shred) {
assert!(!signature.verify(pubkey.as_ref(), data.as_ref()));
}
let offsets = get_signed_data_offsets(shred).unwrap_or_default();
assert!(!signature.verify(pubkey.as_ref(), &shred[offsets]));
}
}
}
impl From<ShredCode> for Shred {

7
ledger/src/shred/legacy.rs
View File

@ -15,7 +15,8 @@ use {
// All payload including any zero paddings are signed.
// Code and data shreds have the same payload size.
pub(super) const SIGNED_MESSAGE_RANGE: Range<usize> = SIZE_OF_SIGNATURE..ShredData::SIZE_OF_PAYLOAD;
pub(super) const SIGNED_MESSAGE_OFFSETS: Range<usize> =
SIZE_OF_SIGNATURE..ShredData::SIZE_OF_PAYLOAD;
const_assert_eq!(ShredData::SIZE_OF_PAYLOAD, ShredCode::SIZE_OF_PAYLOAD);
const_assert_eq!(ShredData::SIZE_OF_PAYLOAD, 1228);
const_assert_eq!(ShredData::CAPACITY, 1051);
@ -108,7 +109,7 @@ impl Shred for ShredData {
shred_data::sanitize(self)
}
fn signed_message(&self) -> &[u8] {
fn signed_data(&self) -> &[u8] {
debug_assert_eq!(self.payload.len(), Self::SIZE_OF_PAYLOAD);
&self.payload[SIZE_OF_SIGNATURE..]
}
@ -170,7 +171,7 @@ impl Shred for ShredCode {
shred_code::sanitize(self)
}
fn signed_message(&self) -> &[u8] {
fn signed_data(&self) -> &[u8] {
debug_assert_eq!(self.payload.len(), Self::SIZE_OF_PAYLOAD);
&self.payload[SIZE_OF_SIGNATURE..]
}

30
ledger/src/shred/merkle.rs
View File

@ -91,12 +91,12 @@ impl Shred {
dispatch!(fn sanitize(&self, verify_merkle_proof: bool) -> Result<(), Error>);
dispatch!(fn set_merkle_branch(&mut self, merkle_branch: &MerkleBranch) -> Result<(), Error>);
dispatch!(fn set_signature(&mut self, signature: Signature));
dispatch!(fn signed_message(&self) -> &[u8]);
dispatch!(fn signed_data(&self) -> &[u8]);
#[must_use]
fn verify(&self, pubkey: &Pubkey) -> bool {
let message = self.signed_message();
self.signature().verify(pubkey.as_ref(), message)
let data = self.signed_data();
self.signature().verify(pubkey.as_ref(), data)
}
fn signature(&self) -> Signature {
@ -147,7 +147,7 @@ impl ShredData {
.ok_or(Error::InvalidProofSize(proof_size))
}
pub(super) fn get_signed_message_range(proof_size: u8) -> Option<Range<usize>> {
pub(super) fn get_signed_data_offsets(proof_size: u8) -> Option<Range<usize>> {
let data_buffer_size = Self::capacity(proof_size).ok()?;
let offset = Self::SIZE_OF_HEADERS + data_buffer_size;
Some(offset..offset + SIZE_OF_MERKLE_ROOT)
@ -247,7 +247,6 @@ impl ShredData {
shred_data::sanitize(self)
}
#[cfg(test)]
pub(super) fn get_merkle_root(shred: &[u8], proof_size: u8) -> Option<MerkleRoot> {
debug_assert_eq!(
shred::layout::get_shred_variant(shred).unwrap(),
@ -332,7 +331,7 @@ impl ShredCode {
Ok(verify_merkle_proof(index, node, &self.merkle_branch()?))
}
pub(super) fn get_signed_message_range(proof_size: u8) -> Option<Range<usize>> {
pub(super) fn get_signed_data_offsets(proof_size: u8) -> Option<Range<usize>> {
let offset = Self::SIZE_OF_HEADERS + Self::capacity(proof_size).ok()?;
Some(offset..offset + SIZE_OF_MERKLE_ROOT)
}
@ -403,7 +402,6 @@ impl ShredCode {
shred_code::sanitize(self)
}
#[cfg(test)]
pub(super) fn get_merkle_root(shred: &[u8], proof_size: u8) -> Option<MerkleRoot> {
debug_assert_eq!(
shred::layout::get_shred_variant(shred).unwrap(),
@ -494,7 +492,7 @@ impl ShredTrait for ShredData {
self.sanitize(/*verify_merkle_proof:*/ true)
}
fn signed_message(&self) -> &[u8] {
fn signed_data(&self) -> &[u8] {
self.merkle_root().map(AsRef::as_ref).unwrap_or_default()
}
}
@ -559,7 +557,7 @@ impl ShredTrait for ShredCode {
self.sanitize(/*verify_merkle_proof:*/ true)
}
fn signed_message(&self) -> &[u8] {
fn signed_data(&self) -> &[u8] {
self.merkle_root().map(AsRef::as_ref).unwrap_or_default()
}
}
@ -646,7 +644,6 @@ where
})?
}
#[cfg(test)]
fn get_merkle_root(
shred: &[u8],
proof_size: u8,
@ -1353,7 +1350,7 @@ mod test {
let merkle_branch = make_merkle_branch(index, num_shreds, &tree).unwrap();
assert_eq!(merkle_branch.proof.len(), usize::from(proof_size));
shred.set_merkle_branch(&merkle_branch).unwrap();
let signature = keypair.sign_message(shred.signed_message());
let signature = keypair.sign_message(shred.signed_data());
shred.set_signature(signature);
assert!(shred.verify(&keypair.pubkey()));
assert_matches!(shred.sanitize(/*verify_merkle_proof:*/ true), Ok(()));
@ -1485,8 +1482,9 @@ mod test {
.collect::<Vec<_>>()
);
// Assert that shreds sanitize and verify.
let pubkey = keypair.pubkey();
for shred in &shreds {
assert!(shred.verify(&keypair.pubkey()));
assert!(shred.verify(&pubkey));
assert_matches!(shred.sanitize(/*verify_merkle_proof:*/ true), Ok(()));
let ShredCommonHeader {
signature,
@ -1511,9 +1509,11 @@ mod test {
assert_eq!(shred::layout::get_version(shred), Some(version));
assert_eq!(shred::layout::get_shred_id(shred), Some(key));
assert_eq!(shred::layout::get_merkle_root(shred), merkle_root);
let slice = shred::layout::get_signed_message_range(shred).unwrap();
let message = shred.get(slice).unwrap();
assert!(signature.verify(keypair.pubkey().as_ref(), message));
let offsets = shred::layout::get_signed_data_offsets(shred).unwrap();
let data = shred.get(offsets).unwrap();
assert!(signature.verify(pubkey.as_ref(), data));
let data = shred::layout::get_signed_data(shred).unwrap();
assert!(signature.verify(pubkey.as_ref(), data.as_ref()));
}
// Verify common, data and coding headers.
let mut num_data_shreds = 0;

2
ledger/src/shred/shred_code.rs
View File

@ -39,7 +39,7 @@ impl ShredCode {
dispatch!(pub(super) fn payload(&self) -> &Vec<u8>);
dispatch!(pub(super) fn sanitize(&self) -> Result<(), Error>);
dispatch!(pub(super) fn set_signature(&mut self, signature: Signature));
dispatch!(pub(super) fn signed_message(&self) -> &[u8]);
dispatch!(pub(super) fn signed_data(&self) -> &[u8]);
// Only for tests.
dispatch!(pub(super) fn set_index(&mut self, index: u32));

2
ledger/src/shred/shred_data.rs
View File

@ -29,7 +29,7 @@ impl ShredData {
dispatch!(pub(super) fn payload(&self) -> &Vec<u8>);
dispatch!(pub(super) fn sanitize(&self) -> Result<(), Error>);
dispatch!(pub(super) fn set_signature(&mut self, signature: Signature));
dispatch!(pub(super) fn signed_message(&self) -> &[u8]);
dispatch!(pub(super) fn signed_data(&self) -> &[u8]);
// Only for tests.
dispatch!(pub(super) fn set_index(&mut self, index: u32));

2
ledger/src/shred/traits.rs
View File

@ -27,7 +27,7 @@ pub(super) trait Shred: Sized {
fn erasure_shard_as_slice(&self) -> Result<&[u8], Error>;
// Portion of the payload which is signed.
fn signed_message(&self) -> &[u8];
fn signed_data(&self) -> &[u8];
// Only for tests.
fn set_index(&mut self, index: u32);

163
ledger/src/sigverify_shreds.rs
View File

@ -57,12 +57,11 @@ pub fn verify_shred_cpu(
Some(signature) => signature,
};
trace!("signature {}", signature);
let message =
match shred::layout::get_signed_message_range(shred).and_then(|slice| shred.get(slice)) {
None => return false,
Some(message) => message,
};
signature.verify(pubkey, message)
let data = match shred::layout::get_signed_data(shred) {
None => return false,
Some(data) => data,
};
signature.verify(pubkey, data.as_ref())
}
fn verify_shreds_cpu(
@ -182,7 +181,7 @@ fn shred_gpu_offsets(
let shred = shred::layout::get_shred(packet);
// Signature may verify for an empty message but the packet will be
// discarded during deserialization.
let msg = shred.and_then(shred::layout::get_signed_message_range);
let msg = shred.and_then(shred::layout::get_signed_data_offsets);
let msg = add_offset(msg.unwrap_or_default(), offset);
signature_offsets.push(sig.start as u32);
msg_start_offsets.push(msg.start as u32);
@ -261,13 +260,13 @@ pub fn verify_shreds_gpu(
fn sign_shred_cpu(keypair: &Keypair, packet: &mut Packet) {
let sig = shred::layout::get_signature_range();
let msg = shred::layout::get_shred(packet)
.and_then(shred::layout::get_signed_message_range)
.and_then(shred::layout::get_signed_data)
.unwrap();
assert!(
packet.meta().size >= sig.end,
"packet is not large enough for a signature"
);
let signature = keypair.sign_message(packet.data(msg).unwrap());
let signature = keypair.sign_message(msg.as_ref());
trace!("signature {:?}", signature);
packet.buffer_mut()[sig].copy_from_slice(signature.as_ref());
}
@ -413,7 +412,7 @@ mod tests {
solana_sdk::{
hash,
hash::Hash,
signature::{Keypair, Signer, SIGNATURE_BYTES},
signature::{Keypair, Signer},
system_transaction,
transaction::Transaction,
},
@ -688,98 +687,42 @@ mod tests {
)
}
// Minimally corrupts the packet so that the signature no longer verifies.
fn corrupt_packet<R: Rng>(rng: &mut R, packet: &mut Packet, keypairs: &HashMap<Slot, Keypair>) {
let coin_flip: bool = rng.gen();
if coin_flip {
// Corrupt one byte within the signature offsets.
let k = rng.gen_range(0, SIGNATURE_BYTES);
let buffer = packet.buffer_mut();
buffer[k] = buffer[k].wrapping_add(1u8);
} else {
// Corrupt one byte within the signed message offsets.
let shred = shred::layout::get_shred(packet).unwrap();
let offsets: Range<usize> = shred::layout::get_signed_message_range(shred).unwrap();
let k = rng.gen_range(offsets.start, offsets.end);
let buffer = packet.buffer_mut();
buffer[k] = buffer[k].wrapping_add(1u8);
}
// Assert that the signature no longer verifies.
let shred = shred::layout::get_shred(packet).unwrap();
let slot = shred::layout::get_slot(shred).unwrap();
let signature = shred::layout::get_signature(shred).unwrap();
if coin_flip {
let pubkey = keypairs[&slot].pubkey();
let offsets = shred::layout::get_signed_message_range(shred).unwrap();
assert!(!signature.verify(pubkey.as_ref(), &shred[offsets]));
} else {
// Slot may have been corrupted and no longer mapping to a keypair.
let pubkey = keypairs.get(&slot).map(Keypair::pubkey).unwrap_or_default();
let offsets = shred::layout::get_signed_message_range(shred).unwrap_or_default();
assert!(!signature.verify(pubkey.as_ref(), &shred[offsets]));
};
fn make_entries<R: Rng>(rng: &mut R, num_entries: usize) -> Vec<Entry> {
let prev_hash = hash::hashv(&[&rng.gen::<[u8; 32]>()]);
let entry = make_entry(rng, &prev_hash);
std::iter::successors(Some(entry), |entry| Some(make_entry(rng, &entry.hash)))
.take(num_entries)
.collect()
}
fn make_shreds<R: Rng>(rng: &mut R) -> (Vec<Shred>, HashMap<Slot, Keypair>) {
fn make_shreds<R: Rng>(rng: &mut R, keypairs: &HashMap<Slot, Keypair>) -> Vec<Shred> {
let reed_solomon_cache = ReedSolomonCache::default();
let entries: Vec<_> = {
let prev_hash = hash::hashv(&[&rng.gen::<[u8; 32]>()]);
let entry = make_entry(rng, &prev_hash);
let num_entries = rng.gen_range(64, 128);
std::iter::successors(Some(entry), |entry| Some(make_entry(rng, &entry.hash)))
.take(num_entries)
.collect()
};
let mut keypairs = HashMap::<Slot, Keypair>::new();
// Legacy shreds.
let (mut shreds, coding_shreds) = {
let slot = 169_367_809;
let parent_slot = slot - rng.gen::<u16>().max(1) as Slot;
keypairs.insert(slot, Keypair::new());
Shredder::new(
slot,
parent_slot,
rng.gen_range(0, 0x3F), // reference_tick
rng.gen(), // version
)
.unwrap()
.entries_to_shreds(
&keypairs[&slot],
&entries,
rng.gen(), // is_last_in_slot
rng.gen_range(0, 671), // next_shred_index
rng.gen_range(0, 781), // next_code_index
false, // merkle_variant
&reed_solomon_cache,
&mut ProcessShredsStats::default(),
)
};
shreds.extend(coding_shreds);
// Merkle shreds.
let (data_shreds, coding_shreds) = {
let slot = 169_376_655;
let parent_slot = slot - rng.gen::<u16>().max(1) as Slot;
keypairs.insert(slot, Keypair::new());
Shredder::new(
slot,
parent_slot,
rng.gen_range(0, 0x3F), // reference_tick
rng.gen(), // version
)
.unwrap()
.entries_to_shreds(
&keypairs[&slot],
&entries,
rng.gen(), // is_last_in_slot
rng.gen_range(0, 671), // next_shred_index
rng.gen_range(0, 781), // next_code_index
true, // merkle_variant
&reed_solomon_cache,
&mut ProcessShredsStats::default(),
)
};
shreds.extend(data_shreds);
shreds.extend(coding_shreds);
let mut shreds: Vec<_> = keypairs
.iter()
.flat_map(|(&slot, keypair)| {
let parent_slot = slot - rng.gen::<u16>().max(1) as Slot;
let num_entries = rng.gen_range(64, 128);
let (data_shreds, coding_shreds) = Shredder::new(
slot,
parent_slot,
rng.gen_range(0, 0x40), // reference_tick
rng.gen(), // version
)
.unwrap()
.entries_to_shreds(
keypair,
&make_entries(rng, num_entries),
rng.gen(), // is_last_in_slot
rng.gen_range(0, 2671), // next_shred_index
rng.gen_range(0, 2781), // next_code_index
rng.gen(), // merkle_variant,
&reed_solomon_cache,
&mut ProcessShredsStats::default(),
);
[data_shreds, coding_shreds]
})
.flatten()
.collect();
shreds.shuffle(rng);
// Assert that all shreds verfiy and sanitize.
for shred in &shreds {
@ -792,11 +735,13 @@ mod tests {
let shred = shred.payload();
let slot = shred::layout::get_slot(shred).unwrap();
let signature = shred::layout::get_signature(shred).unwrap();
let offsets = shred::layout::get_signed_message_range(shred).unwrap();
let offsets = shred::layout::get_signed_data_offsets(shred).unwrap();
let pubkey = keypairs[&slot].pubkey();
assert!(signature.verify(pubkey.as_ref(), &shred[offsets]));
let data = shred::layout::get_signed_data(shred).unwrap();
assert!(signature.verify(pubkey.as_ref(), data.as_ref()));
}
(shreds, keypairs)
shreds
}
fn make_packets<R: Rng>(rng: &mut R, shreds: &[Shred]) -> Vec<PacketBatch> {
@ -825,7 +770,11 @@ mod tests {
fn test_verify_shreds_fuzz() {
let mut rng = rand::thread_rng();
let recycler_cache = RecyclerCache::default();
let (shreds, keypairs) = make_shreds(&mut rng);
let keypairs = repeat_with(|| rng.gen_range(169_367_809, 169_906_789))
.map(|slot| (slot, Keypair::new()))
.take(3)
.collect();
let shreds = make_shreds(&mut rng, &keypairs);
let pubkeys: HashMap<Slot, [u8; 32]> = keypairs
.iter()
.map(|(&slot, keypair)| (slot, keypair.pubkey().to_bytes()))
@ -848,7 +797,7 @@ mod tests {
.map(|packet| {
let coin_flip: bool = rng.gen();
if !coin_flip {
corrupt_packet(&mut rng, packet, &keypairs);
shred::layout::corrupt_packet(&mut rng, packet, &keypairs);
}
u8::from(coin_flip)
})
@ -862,7 +811,13 @@ mod tests {
fn test_sign_shreds_gpu() {
let mut rng = rand::thread_rng();
let recycler_cache = RecyclerCache::default();
let (shreds, _) = make_shreds(&mut rng);
let shreds = {
let keypairs = repeat_with(|| rng.gen_range(169_367_809, 169_906_789))
.map(|slot| (slot, Keypair::new()))
.take(3)
.collect();
make_shreds(&mut rng, &keypairs)
};
let keypair = Keypair::new();
let pubkeys: HashMap<Slot, [u8; 32]> = {
let pubkey = keypair.pubkey().to_bytes();