From aeee25e703f23c62a99e02ce7a866bf9c97f0cd2 Mon Sep 17 00:00:00 2001
From: Rob Walker <rob@solana.com>
Date: Mon, 10 Dec 2018 20:03:04 -0800
Subject: [PATCH] add tick_height to Entry to be able to repair by period,
 chain forks of Entries, etc. (#2096)

---
 src/bank.rs             | 16 +++++------
 src/cluster_info.rs     |  4 +--
 src/entry.rs            | 61 ++++++++++++++++++++++++++---------------
 src/leader_scheduler.rs |  6 ++--
 src/ledger.rs           | 17 ++++++++----
 src/mint.rs             |  6 ++--
 src/packet.rs           |  2 +-
 src/poh.rs              |  5 ++++
 src/poh_recorder.rs     | 16 ++++++++---
 src/replay_stage.rs     |  8 +++---
 src/storage_stage.rs    |  2 +-
 src/tvu.rs              | 10 +++----
 12 files changed, 95 insertions(+), 58 deletions(-)

diff --git a/src/bank.rs b/src/bank.rs
index fd7b9b0d08..171c42b946 100644
--- a/src/bank.rs
+++ b/src/bank.rs
@@ -1821,7 +1821,7 @@ mod tests {
             let mut e = ledger::next_entries(&hash, 0, txs);
             entries.append(&mut e);
             hash = entries.last().unwrap().id;
-            let tick = Entry::new(&hash, num_hashes, vec![]);
+            let tick = Entry::new(&hash, 0, num_hashes, vec![]);
             hash = tick.id;
             last_id = hash;
             entries.push(tick);
@@ -1842,11 +1842,11 @@ mod tests {
         for i in 0..length {
             let keypair = Keypair::new();
             let tx = Transaction::system_new(&mint.keypair(), keypair.pubkey(), 1, last_id);
-            let entry = Entry::new(&hash, num_hashes, vec![tx]);
+            let entry = Entry::new(&hash, 0, num_hashes, vec![tx]);
             hash = entry.id;
             entries.push(entry);
             if (i + 1) % ticks == 0 {
-                let tick = Entry::new(&hash, num_hashes, vec![]);
+                let tick = Entry::new(&hash, 0, num_hashes, vec![]);
                 hash = tick.id;
                 last_id = hash;
                 entries.push(tick);
@@ -2154,16 +2154,16 @@ mod tests {
         // ensure bank can process 2 entries that do not have a common account and tick is registered
         let tx = Transaction::system_new(&keypair2, keypair3.pubkey(), 1, bank.last_id());
         let entry_1 = next_entry(&last_id, 1, vec![tx]);
-        let new_tick = next_entry(&entry_1.id, 1, vec![]);
-        let tx = Transaction::system_new(&keypair1, keypair4.pubkey(), 1, new_tick.id);
-        let entry_2 = next_entry(&new_tick.id, 1, vec![tx]);
+        let tick = next_entry(&entry_1.id, 1, vec![]);
+        let tx = Transaction::system_new(&keypair1, keypair4.pubkey(), 1, tick.id);
+        let entry_2 = next_entry(&tick.id, 1, vec![tx]);
         assert_eq!(
-            bank.par_process_entries(&[entry_1.clone(), new_tick.clone(), entry_2]),
+            bank.par_process_entries(&[entry_1.clone(), tick.clone(), entry_2]),
             Ok(())
         );
         assert_eq!(bank.get_balance(&keypair3.pubkey()), 1);
         assert_eq!(bank.get_balance(&keypair4.pubkey()), 1);
-        assert_eq!(bank.last_id(), new_tick.id);
+        assert_eq!(bank.last_id(), tick.id);
         // ensure that errors are returned
         assert_eq!(
             bank.par_process_entries(&[entry_1]),
diff --git a/src/cluster_info.rs b/src/cluster_info.rs
index c208eb27c2..9c269b54cf 100644
--- a/src/cluster_info.rs
+++ b/src/cluster_info.rs
@@ -1292,8 +1292,8 @@ mod tests {
             writer
                 .write_entries(
                     &vec![
-                        Entry::new_tick(&zero, 0, &zero),
-                        Entry::new_tick(&one, 0, &one),
+                        Entry::new_tick(&zero, 0, 0, &zero),
+                        Entry::new_tick(&one, 1, 0, &one),
                     ]
                     .to_vec(),
                 )
diff --git a/src/entry.rs b/src/entry.rs
index 1432ab54bb..acb7585b7c 100644
--- a/src/entry.rs
+++ b/src/entry.rs
@@ -37,6 +37,9 @@ pub struct Entry {
     /// The the previous Entry ID.
     pub prev_id: Hash,
 
+    /// tick height of the ledger, not including any tick implied by this Entry
+    pub tick_height: u64,
+
     /// The number of hashes since the previous Entry ID.
     pub num_hashes: u64,
 
@@ -51,11 +54,17 @@ pub struct Entry {
 
 impl Entry {
     /// Creates the next Entry `num_hashes` after `start_hash`.
-    pub fn new(prev_id: &Hash, num_hashes: u64, transactions: Vec<Transaction>) -> Self {
+    pub fn new(
+        prev_id: &Hash,
+        tick_height: u64,
+        num_hashes: u64,
+        transactions: Vec<Transaction>,
+    ) -> Self {
         let entry = {
             if num_hashes == 0 && transactions.is_empty() {
                 Entry {
                     prev_id: *prev_id,
+                    tick_height,
                     num_hashes: 0,
                     id: *prev_id,
                     transactions,
@@ -66,6 +75,7 @@ impl Entry {
                 let id = next_hash(prev_id, 1, &transactions);
                 Entry {
                     prev_id: *prev_id,
+                    tick_height,
                     num_hashes: 1,
                     id,
                     transactions,
@@ -77,6 +87,7 @@ impl Entry {
                 let id = next_hash(prev_id, num_hashes, &transactions);
                 Entry {
                     prev_id: *prev_id,
+                    tick_height,
                     num_hashes,
                     id,
                     transactions,
@@ -129,9 +140,9 @@ impl Entry {
     pub fn serialized_size(transactions: &[Transaction]) -> u64 {
         let txs_size = serialized_size(transactions).unwrap();
 
-        // num_hashes    +      id + prev_id                + txs
+        // tick_height+num_hashes    +     id+prev_id  +        txs
 
-        (size_of::<u64>() + 2 * size_of::<Hash>()) as u64 + txs_size
+        (2 * size_of::<u64>() + 2 * size_of::<Hash>()) as u64 + txs_size
     }
 
     pub fn num_will_fit(transactions: &[Transaction]) -> usize {
@@ -176,18 +187,22 @@ impl Entry {
         num_hashes: &mut u64,
         transactions: Vec<Transaction>,
     ) -> Self {
-        let entry = Self::new(start_hash, *num_hashes, transactions);
+        let entry = Self::new(start_hash, 0, *num_hashes, transactions);
         *start_hash = entry.id;
         *num_hashes = 0;
         assert!(serialized_size(&entry).unwrap() <= BLOB_DATA_SIZE as u64);
         entry
     }
 
-    /// Creates a Entry from the number of hashes `num_hashes` since the previous transaction
-    /// and that resulting `id`.
-    pub fn new_tick(prev_id: &Hash, num_hashes: u64, id: &Hash) -> Self {
+    /// Creates a Entry from the number of hashes `num_hashes`
+    /// since the previous transaction and that resulting `id`.
+
+    #[cfg(test)]
+    pub fn new_tick(prev_id: &Hash, tick_height: u64, num_hashes: u64, id: &Hash) -> Self {
         Entry {
             prev_id: *prev_id,
+            tick_height,
+
             num_hashes,
             id: *id,
             transactions: vec![],
@@ -239,17 +254,6 @@ fn next_hash(start_hash: &Hash, num_hashes: u64, transactions: &[Transaction]) -
     }
 }
 
-/// Creates the next Tick or Transaction Entry `num_hashes` after `start_hash`.
-pub fn next_entry(prev_id: &Hash, num_hashes: u64, transactions: Vec<Transaction>) -> Entry {
-    assert!(num_hashes > 0 || transactions.is_empty());
-    Entry {
-        prev_id: *prev_id,
-        num_hashes,
-        id: next_hash(prev_id, num_hashes, &transactions),
-        transactions,
-    }
-}
-
 pub fn reconstruct_entries_from_blobs(blobs: Vec<SharedBlob>) -> Result<(Vec<Entry>, u64)> {
     let mut entries: Vec<Entry> = Vec::with_capacity(blobs.len());
     let mut num_ticks = 0;
@@ -269,6 +273,19 @@ pub fn reconstruct_entries_from_blobs(blobs: Vec<SharedBlob>) -> Result<(Vec<Ent
     Ok((entries, num_ticks))
 }
 
+#[cfg(test)]
+/// Creates the next Tick or Transaction Entry `num_hashes` after `start_hash`.
+pub fn next_entry(prev_id: &Hash, num_hashes: u64, transactions: Vec<Transaction>) -> Entry {
+    assert!(num_hashes > 0 || transactions.is_empty());
+    Entry {
+        prev_id: *prev_id,
+        tick_height: 0,
+        num_hashes,
+        id: next_hash(prev_id, num_hashes, &transactions),
+        transactions,
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -284,8 +301,8 @@ mod tests {
     fn test_entry_verify() {
         let zero = Hash::default();
         let one = hash(&zero.as_ref());
-        assert!(Entry::new_tick(&zero, 0, &zero).verify(&zero)); // base case, never used
-        assert!(!Entry::new_tick(&zero, 0, &zero).verify(&one)); // base case, bad
+        assert!(Entry::new_tick(&zero, 0, 0, &zero).verify(&zero)); // base case, never used
+        assert!(!Entry::new_tick(&zero, 1, 0, &zero).verify(&one)); // base case, bad
         assert!(next_entry(&zero, 1, vec![]).verify(&zero)); // inductive step
         assert!(next_entry(&zero, 1, vec![]).verify_self()); // also inductive step
         assert!(!next_entry(&zero, 1, vec![]).verify(&one)); // inductive step, bad
@@ -299,7 +316,7 @@ mod tests {
         let keypair = Keypair::new();
         let tx0 = Transaction::system_new(&keypair, keypair.pubkey(), 0, zero);
         let tx1 = Transaction::system_new(&keypair, keypair.pubkey(), 1, zero);
-        let mut e0 = Entry::new(&zero, 0, vec![tx0.clone(), tx1.clone()]);
+        let mut e0 = Entry::new(&zero, 0, 0, vec![tx0.clone(), tx1.clone()]);
         assert!(e0.verify(&zero));
 
         // Next, swap two transactions and ensure verification fails.
@@ -323,7 +340,7 @@ mod tests {
         );
         let tx1 =
             Transaction::budget_new_signature(&keypair, keypair.pubkey(), keypair.pubkey(), zero);
-        let mut e0 = Entry::new(&zero, 0, vec![tx0.clone(), tx1.clone()]);
+        let mut e0 = Entry::new(&zero, 0, 0, vec![tx0.clone(), tx1.clone()]);
         assert!(e0.verify(&zero));
 
         // Next, swap two witness transactions and ensure verification fails.
diff --git a/src/leader_scheduler.rs b/src/leader_scheduler.rs
index ad01188297..71aa1e32e1 100644
--- a/src/leader_scheduler.rs
+++ b/src/leader_scheduler.rs
@@ -483,20 +483,20 @@ pub fn make_active_set_entries(
     // 1) Create transfer token entry
     let transfer_tx =
         Transaction::system_new(&token_source, active_keypair.pubkey(), 3, *last_tick_id);
-    let transfer_entry = Entry::new(last_entry_id, 1, vec![transfer_tx]);
+    let transfer_entry = Entry::new(last_entry_id, 0, 1, vec![transfer_tx]);
     let mut last_entry_id = transfer_entry.id;
 
     // 2) Create and register the vote account
     let vote_account = Keypair::new();
     let new_vote_account_tx =
         Transaction::vote_account_new(active_keypair, vote_account.pubkey(), *last_tick_id, 1, 1);
-    let new_vote_account_entry = Entry::new(&last_entry_id, 1, vec![new_vote_account_tx]);
+    let new_vote_account_entry = Entry::new(&last_entry_id, 0, 1, vec![new_vote_account_tx]);
     last_entry_id = new_vote_account_entry.id;
 
     // 3) Create vote entry
     let vote = Vote { tick_height: 1 };
     let vote_tx = Transaction::vote_new(&vote_account, vote, *last_tick_id, 0);
-    let vote_entry = Entry::new(&last_entry_id, 1, vec![vote_tx]);
+    let vote_entry = Entry::new(&last_entry_id, 0, 1, vec![vote_tx]);
     last_entry_id = vote_entry.id;
 
     // 4) Create the ending empty ticks
diff --git a/src/ledger.rs b/src/ledger.rs
index be6c151148..558c53d836 100644
--- a/src/ledger.rs
+++ b/src/ledger.rs
@@ -458,7 +458,13 @@ pub trait Block {
 
 impl Block for [Entry] {
     fn verify(&self, start_hash: &Hash) -> bool {
-        let genesis = [Entry::new_tick(start_hash, 0, start_hash)];
+        let genesis = [Entry {
+            prev_id: *start_hash,
+            tick_height: 0,
+            num_hashes: 0,
+            id: *start_hash,
+            transactions: vec![],
+        }];
         let entry_pairs = genesis.par_iter().chain(self).zip(self);
         entry_pairs.all(|(x0, x1)| {
             let r = x1.verify(&x0.id);
@@ -609,8 +615,8 @@ pub fn create_tmp_genesis(
 
 pub fn create_ticks(num_ticks: usize, mut hash: Hash) -> Vec<Entry> {
     let mut ticks = Vec::with_capacity(num_ticks as usize);
-    for _ in 0..num_ticks {
-        let new_tick = Entry::new(&hash, 1, vec![]);
+    for _ in 0..num_ticks as u64 {
+        let new_tick = Entry::new(&hash, 0, 1, vec![]);
         hash = new_tick.id;
         ticks.push(new_tick);
     }
@@ -719,8 +725,8 @@ mod tests {
         let zero = Hash::default();
         let one = hash(&zero.as_ref());
         assert!(vec![][..].verify(&zero)); // base case
-        assert!(vec![Entry::new_tick(&zero, 0, &zero)][..].verify(&zero)); // singleton case 1
-        assert!(!vec![Entry::new_tick(&zero, 0, &zero)][..].verify(&one)); // singleton case 2, bad
+        assert!(vec![Entry::new_tick(&zero, 0, 0, &zero)][..].verify(&zero)); // singleton case 1
+        assert!(!vec![Entry::new_tick(&zero, 0, 0, &zero)][..].verify(&one)); // singleton case 2, bad
         assert!(vec![next_entry(&zero, 0, vec![]); 2][..].verify(&zero)); // inductive step
 
         let mut bad_ticks = vec![next_entry(&zero, 0, vec![]); 2];
@@ -791,6 +797,7 @@ mod tests {
         let tx_large_size = serialized_size(&tx_large).unwrap() as usize;
         let entry_size = serialized_size(&Entry {
             prev_id: Hash::default(),
+            tick_height: 0,
             num_hashes: 0,
             id: Hash::default(),
             transactions: vec![],
diff --git a/src/mint.rs b/src/mint.rs
index c1b89f455a..83c2d60f9f 100644
--- a/src/mint.rs
+++ b/src/mint.rs
@@ -96,9 +96,9 @@ impl Mint {
     }
 
     pub fn create_entries(&self) -> Vec<Entry> {
-        let e0 = Entry::new(&self.seed(), 0, vec![]);
-        let e1 = Entry::new(&e0.id, 1, self.create_transaction());
-        let e2 = Entry::new(&e1.id, 1, vec![]); // include a tick
+        let e0 = Entry::new(&self.seed(), 0, 0, vec![]);
+        let e1 = Entry::new(&e0.id, 0, 1, self.create_transaction());
+        let e2 = Entry::new(&e1.id, 0, 1, vec![]); // include a tick
         vec![e0, e1, e2]
     }
 }
diff --git a/src/packet.rs b/src/packet.rs
index 311236ae89..6a787ac5db 100644
--- a/src/packet.rs
+++ b/src/packet.rs
@@ -462,7 +462,7 @@ pub fn make_consecutive_blobs(
     let num_hashes = 1;
     let mut all_entries = Vec::with_capacity(num_blobs_to_make as usize);
     for _ in 0..num_blobs_to_make {
-        let entry = Entry::new(&last_hash, num_hashes, vec![]);
+        let entry = Entry::new(&last_hash, 0, num_hashes, vec![]);
         last_hash = entry.id;
         all_entries.push(entry);
     }
diff --git a/src/poh.rs b/src/poh.rs
index 3bee643131..58d73b767a 100644
--- a/src/poh.rs
+++ b/src/poh.rs
@@ -12,6 +12,7 @@ pub struct Poh {
 #[derive(Debug)]
 pub struct PohEntry {
     pub prev_id: Hash,
+    pub tick_height: u64,
     pub num_hashes: u64,
     pub id: Hash,
     pub mixin: Option<Hash>,
@@ -43,6 +44,7 @@ impl Poh {
 
         PohEntry {
             prev_id,
+            tick_height: self.tick_height,
             num_hashes,
             id: self.id,
             mixin: Some(mixin),
@@ -60,10 +62,12 @@ impl Poh {
         let prev_id = self.prev_id;
         self.prev_id = self.id;
 
+        let tick_height = self.tick_height;
         self.tick_height += 1;
 
         PohEntry {
             prev_id,
+            tick_height,
             num_hashes,
             id: self.id,
             mixin: None,
@@ -106,6 +110,7 @@ mod tests {
             Hash::default(),
             &[PohEntry {
                 prev_id: Hash::default(),
+                tick_height: 0,
                 num_hashes: 0,
                 id: Hash::default(),
                 mixin: None,
diff --git a/src/poh_recorder.rs b/src/poh_recorder.rs
index 044b61933d..18982ae8e0 100644
--- a/src/poh_recorder.rs
+++ b/src/poh_recorder.rs
@@ -93,6 +93,7 @@ impl PohRecorder {
         assert!(!txs.is_empty(), "Entries without transactions are used to track real-time passing in the ledger and can only be generated with PohRecorder::tick function");
         let entry = Entry {
             prev_id: entry.prev_id,
+            tick_height: entry.tick_height,
             num_hashes: entry.num_hashes,
             id: entry.id,
             transactions: txs,
@@ -105,6 +106,7 @@ impl PohRecorder {
         let tick = poh.tick();
         let tick = Entry {
             prev_id: tick.prev_id,
+            tick_height: tick.tick_height,
             num_hashes: tick.num_hashes,
             id: tick.id,
             transactions: vec![],
@@ -136,11 +138,17 @@ mod tests {
         let h1 = hash(b"hello world!");
         let tx = test_tx();
         assert!(poh_recorder.record(h1, vec![tx]).is_ok());
-        assert!(poh_recorder.tick().is_ok());
-
         //get some events
-        let _ = entry_receiver.recv().unwrap();
-        let _ = entry_receiver.recv().unwrap();
+        let e = entry_receiver.recv().unwrap();
+        assert_eq!(e[0].tick_height, 1);
+
+        assert!(poh_recorder.tick().is_ok());
+        let e = entry_receiver.recv().unwrap();
+        assert_eq!(e[0].tick_height, 1);
+
+        assert!(poh_recorder.tick().is_ok());
+        let e = entry_receiver.recv().unwrap();
+        assert_eq!(e[0].tick_height, 2);
 
         //make sure it handles channel close correctly
         drop(entry_receiver);
diff --git a/src/replay_stage.rs b/src/replay_stage.rs
index 16beba796d..758e76c514 100644
--- a/src/replay_stage.rs
+++ b/src/replay_stage.rs
@@ -348,7 +348,7 @@ mod test {
         let mut entries_to_send = vec![];
 
         while entries_to_send.len() < total_entries_to_send {
-            let entry = Entry::new(&mut last_id, num_hashes, vec![]);
+            let entry = Entry::new(&mut last_id, 0, num_hashes, vec![]);
             last_id = entry.id;
             entries_to_send.push(entry);
         }
@@ -567,7 +567,7 @@ mod test {
         let leader_rotation_index = (bootstrap_height - initial_tick_height - 1) as usize;
         let mut expected_last_id = Hash::default();
         for i in 0..total_entries_to_send {
-            let entry = Entry::new(&mut last_id, num_hashes, vec![]);
+            let entry = Entry::new(&mut last_id, 0, num_hashes, vec![]);
             last_id = entry.id;
             entry_sender
                 .send(vec![entry.clone()])
@@ -626,7 +626,7 @@ mod test {
         let mut last_id = Hash::default();
         let mut entries = Vec::new();
         for _ in 0..5 {
-            let entry = Entry::new(&mut last_id, 1, vec![]); //just ticks
+            let entry = Entry::new(&mut last_id, 0, 1, vec![]); //just ticks
             last_id = entry.id;
             entries.push(entry);
         }
@@ -653,7 +653,7 @@ mod test {
 
         entries.clear();
         for _ in 0..5 {
-            let entry = Entry::new(&mut Hash::default(), 0, vec![]); //just broken entries
+            let entry = Entry::new(&mut Hash::default(), 0, 0, vec![]); //just broken entries
             entries.push(entry);
         }
         entry_sender
diff --git a/src/storage_stage.rs b/src/storage_stage.rs
index b88094e5fe..cd1b2d8109 100644
--- a/src/storage_stage.rs
+++ b/src/storage_stage.rs
@@ -419,7 +419,7 @@ mod tests {
         let keypair = Keypair::new();
         let vote_tx = VoteTransaction::vote_new(&keypair, vote, Hash::default(), 1);
         vote_txs.push(vote_tx);
-        let vote_entries = vec![Entry::new(&Hash::default(), 1, vote_txs)];
+        let vote_entries = vec![Entry::new(&Hash::default(), 0, 1, vote_txs)];
         storage_entry_sender.send(vote_entries).unwrap();
 
         for _ in 0..5 {
diff --git a/src/tvu.rs b/src/tvu.rs
index 0ccd261e8c..6fa003b084 100644
--- a/src/tvu.rs
+++ b/src/tvu.rs
@@ -290,10 +290,10 @@ pub mod tests {
         let transfer_amount = 501;
         let bob_keypair = Keypair::new();
         for i in 0..num_transfers {
-            let entry0 = Entry::new(&cur_hash, i, vec![]);
+            let entry0 = Entry::new(&cur_hash, 0, i, vec![]);
             cur_hash = entry0.id;
             bank.register_tick(&cur_hash);
-            let entry_tick0 = Entry::new(&cur_hash, i + 1, vec![]);
+            let entry_tick0 = Entry::new(&cur_hash, 0, i + 1, vec![]);
             cur_hash = entry_tick0.id;
 
             let tx0 = Transaction::system_new(
@@ -303,11 +303,11 @@ pub mod tests {
                 cur_hash,
             );
             bank.register_tick(&cur_hash);
-            let entry_tick1 = Entry::new(&cur_hash, i + 1, vec![]);
+            let entry_tick1 = Entry::new(&cur_hash, 0, i + 1, vec![]);
             cur_hash = entry_tick1.id;
-            let entry1 = Entry::new(&cur_hash, i + num_transfers, vec![tx0]);
+            let entry1 = Entry::new(&cur_hash, 0, i + num_transfers, vec![tx0]);
             bank.register_tick(&entry1.id);
-            let entry_tick2 = Entry::new(&entry1.id, i + 1, vec![]);
+            let entry_tick2 = Entry::new(&entry1.id, 0, i + 1, vec![]);
             cur_hash = entry_tick2.id;
 
             alice_ref_balance -= transfer_amount;