* add merge_parents(), which means 'eat your parent' (#2851)

* add is_root(), which is false if the bank has a parent
* use is_root() for store_slow and store_accounts to decide whether to purge on zero balance

runtime/src/accounts.rs

@@ -255,13 +255,29 @@ impl AccountsDB {
     pub fn transaction_count(&self) -> u64 {
         self.transaction_count
     }
-    //pub fn account_values_slow(&self) -> Vec<(Pubkey, solana_sdk::account::Account)> {
+
-    //    self.accounts.iter().map(|(x, y)| (*x, y.clone())).collect()
+    /// become the root accountsDB
-    //}
+    fn merge_parents<U>(&mut self, parents: &[U])
-    //fn merge(&mut self, other: Self) {
+    where
-    //    self.transaction_count += other.transaction_count;
+        U: Deref<Target = Self>,
-    //    self.accounts.extend(other.accounts)
+    {
-    //}
+        self.transaction_count += parents
+            .iter()
+            .fold(0, |sum, parent| sum + parent.transaction_count);
+
+        // for every account in all the parents, load latest and update self if
+        //   absent
+        for pubkey in parents.iter().flat_map(|parent| parent.accounts.keys()) {
+            // update self with data from parents unless in self
+            if self.accounts.get(pubkey).is_none() {
+                self.accounts
+                    .insert(pubkey.clone(), Self::load(parents, pubkey).unwrap().clone());
+            }
+        }
+
+        // toss any zero-balance accounts, since self is root now
+        self.accounts.retain(|_, account| account.tokens != 0);
+    }
 }
 
 impl Accounts {
@@ -389,27 +405,22 @@ impl Accounts {
     pub fn transaction_count(&self) -> u64 {
         self.accounts_db.read().unwrap().transaction_count()
     }
-    ///// accounts starts with an empty data structure for every fork
-    ///// self is root, merge the fork into self
-    //pub fn merge_into_root(&self, other: Self) {
-    //    assert!(other.account_locks.lock().unwrap().is_empty());
-    //    let db = other.accounts_db.into_inner().unwrap();
-    //    let mut mydb = self.accounts_db.write().unwrap();
-    //    mydb.merge(db)
-    //}
-    //pub fn copy_for_tpu(&self) -> Self {
-    //    //TODO: deprecate this in favor of forks and merge_into_root
-    //    let copy = Accounts::default();
 
-    //    {
+    /// accounts starts with an empty data structure for every child/fork
-    //        let mut accounts_db = copy.accounts_db.write().unwrap();
+    ///   this merges all the parents/checkpoints
-    //        for (key, val) in self.accounts_db.read().unwrap().accounts.iter() {
+    pub fn merge_parents<U>(&self, parents: &[U])
-    //            accounts_db.accounts.insert(key.clone(), val.clone());
+    where
-    //        }
+        U: Deref<Target = Self>,
-    //        accounts_db.transaction_count = self.transaction_count();
+    {
-    //    }
+        assert!(self.account_locks.lock().unwrap().is_empty());
-    //    copy
+
-    //}
+        let dbs: Vec<_> = parents
+            .iter()
+            .map(|obj| obj.accounts_db.read().unwrap())
+            .collect();
+
+        self.accounts_db.write().unwrap().merge_parents(&dbs);
+    }
 }
 
 #[cfg(test)]
@@ -819,4 +830,24 @@ mod tests {
         loaded_accounts[0].clone().unwrap_err();
         assert_eq!(loaded_accounts[0], Err(BankError::AccountLoadedTwice));
     }
+
+    #[test]
+    fn test_accounts_merge_parents() {
+        let mut db0 = AccountsDB::default();
+        let key = Pubkey::default();
+        let account = Account::new(1, 0, key);
+
+        // store value 1 in the "root", i.e. db zero
+        db0.store(true, &key, &account);
+
+        // store value 0 in the child, but don't purge (see purge test above)
+        let mut db1 = AccountsDB::default();
+        db1.store(false, &key, &Account::new(0, 0, key));
+
+        // merge, which should whack key's account
+        db1.merge_parents(&[&db0]);
+
+        assert_eq!(AccountsDB::load(&[&db1], &key), None);
+    }
+
 }

runtime/src/bank.rs

@@ -135,10 +135,33 @@ impl Bank {
         bank
     }
 
+    /// merge (i.e. pull) the parent's state up into this Bank,
+    ///   this Bank becomes a root
+    pub fn merge_parents(&mut self) {
+        let parents = self.parents();
+        self.parent = None;
+
+        let parent_accounts: Vec<_> = parents.iter().map(|b| &b.accounts).collect();
+        self.accounts.merge_parents(&parent_accounts);
+
+        let parent_caches: Vec<_> = parents
+            .iter()
+            .map(|b| b.status_cache.read().unwrap())
+            .collect();
+        self.status_cache
+            .write()
+            .unwrap()
+            .merge_parents(&parent_caches);
+    }
+
     /// Return the more recent checkpoint of this bank instance.
     pub fn parent(&self) -> Option<Arc<Bank>> {
         self.parent.clone()
     }
+    /// Returns whether this bank is the root
+    pub fn is_root(&self) -> bool {
+        self.parent.is_none()
+    }
 
     fn process_genesis_block(&self, genesis_block: &GenesisBlock) {
         assert!(genesis_block.mint_id != Pubkey::default());
@@ -172,7 +195,7 @@ impl Bank {
             .unwrap();
 
         self.accounts.store_slow(
-            true,
+            self.is_root(),
             &genesis_block.bootstrap_leader_vote_account_id,
             &bootstrap_leader_vote_account,
         );
@@ -185,7 +208,8 @@ impl Bank {
 
     pub fn add_native_program(&self, name: &str, program_id: &Pubkey) {
         let account = native_loader::create_program_account(name);
-        self.accounts.store_slow(true, program_id, &account);
+        self.accounts
+            .store_slow(self.is_root(), program_id, &account);
     }
 
     fn add_builtin_programs(&self) {
@@ -197,8 +221,11 @@ impl Bank {
         self.add_native_program("solana_erc20", &token_program::id());
 
         let storage_system_account = Account::new(1, 16 * 1024, storage_program::system_id());
-        self.accounts
+        self.accounts.store_slow(
-            .store_slow(true, &storage_program::system_id(), &storage_system_account);
+            self.is_root(),
+            &storage_program::system_id(),
+            &storage_system_account,
+        );
     }
 
     /// Return the last entry ID registered.
@@ -466,7 +493,7 @@ impl Bank {
     ) {
         let now = Instant::now();
         self.accounts
-            .store_accounts(true, txs, executed, loaded_accounts);
+            .store_accounts(self.is_root(), txs, executed, loaded_accounts);
 
         // once committed there is no way to unroll
         let write_elapsed = now.elapsed();
@@ -546,7 +573,7 @@ impl Bank {
     pub fn deposit(&self, pubkey: &Pubkey, tokens: u64) {
         let mut account = self.get_account(pubkey).unwrap_or_default();
         account.tokens += tokens;
-        self.accounts.store_slow(true, pubkey, &account);
+        self.accounts.store_slow(self.is_root(), pubkey, &account);
     }
 
     pub fn get_account(&self, pubkey: &Pubkey) -> Option<Account> {
@@ -1170,4 +1197,48 @@ mod tests {
         let bank1 = Bank::new(&GenesisBlock::new(20).0);
         assert_ne!(bank0.hash_internal_state(), bank1.hash_internal_state());
     }
+
+    /// Verifies that last ids and accounts are correctly referenced from parent
+    #[test]
+    fn test_bank_merge_parents() {
+        let (genesis_block, mint_keypair) = GenesisBlock::new(2);
+        let key1 = Keypair::new();
+        let key2 = Keypair::new();
+        let parent = Arc::new(Bank::new(&genesis_block));
+
+        let tx_move_mint_to_1 = SystemTransaction::new_move(
+            &mint_keypair,
+            key1.pubkey(),
+            1,
+            genesis_block.last_id(),
+            0,
+        );
+        assert_eq!(parent.process_transaction(&tx_move_mint_to_1), Ok(()));
+        let mut bank = Bank::new_from_parent(&parent);
+        let tx_move_1_to_2 =
+            SystemTransaction::new_move(&key1, key2.pubkey(), 1, genesis_block.last_id(), 0);
+        assert_eq!(bank.process_transaction(&tx_move_1_to_2), Ok(()));
+        assert_eq!(
+            parent.get_signature_status(&tx_move_1_to_2.signatures[0]),
+            None
+        );
+
+        for _ in 0..3 {
+            // first time these should match what happened above, assert that parents are ok
+            assert_eq!(bank.get_balance(&key1.pubkey()), 0);
+            assert_eq!(bank.get_balance(&key2.pubkey()), 1);
+            assert_eq!(
+                bank.get_signature_status(&tx_move_mint_to_1.signatures[0]),
+                Some(Ok(()))
+            );
+            assert_eq!(
+                bank.get_signature_status(&tx_move_1_to_2.signatures[0]),
+                Some(Ok(()))
+            );
+
+            // works iteration 0, no-ops on iteration 1 and 2
+            bank.merge_parents();
+        }
+    }
+
 }

runtime/src/last_id_queue.rs

@@ -123,23 +123,6 @@ impl LastIdQueue {
         self.tick_height = 0;
         self.last_id = None;
     }
-    /// fork for LastIdQueue is a simple clone
-    #[cfg(test)]
-    pub fn fork(&self) -> Self {
-        Self {
-            entries: self.entries.clone(),
-            tick_height: self.tick_height,
-            last_id: self.last_id,
-        }
-    }
-    /// merge for entryq is a swap
-    #[cfg(test)]
-    pub fn merge_into_root(&mut self, other: Self) {
-        let (entries, tick_height, last_id) = { (other.entries, other.tick_height, other.last_id) };
-        self.entries = entries;
-        self.tick_height = tick_height;
-        self.last_id = last_id;
-    }
 }
 #[cfg(test)]
 mod tests {
@@ -166,23 +149,4 @@ mod tests {
         // Assert we're no longer able to use the oldest entry ID.
         assert!(!entry_queue.check_entry(last_id));
     }
-    #[test]
-    fn test_fork() {
-        let last_id = Hash::default();
-        let mut first = LastIdQueue::default();
-        assert!(!first.check_entry(last_id));
-        first.register_tick(&last_id);
-        let second = first.fork();
-        assert!(second.check_entry(last_id));
-    }
-    #[test]
-    fn test_merge() {
-        let last_id = Hash::default();
-        let mut first = LastIdQueue::default();
-        assert!(!first.check_entry(last_id));
-        let mut second = first.fork();
-        second.register_tick(&last_id);
-        first.merge_into_root(second);
-        assert!(first.check_entry(last_id));
-    }
 }

runtime/src/status_cache.rs

@@ -83,17 +83,37 @@ impl<T: Clone> StatusCache<T> {
         }
         self.get_signature_status_merged(sig)
     }
-    /// like accounts, status cache starts with an new data structure for every checkpoint
+
-    /// so only merge is implemented
+    fn merge_parent_is_full(&mut self, parent: &Self) -> bool {
-    /// but the merges maintains a history
+        // flatten the parent and their merges into self.merges, limit
-    #[cfg(test)]
+
-    pub fn merge_into_root(&mut self, other: Self) {
+        self.merges.push_back(StatusCache {
-        // merges should be empty for every other checkpoint accept the root
+            signatures: parent.signatures.clone(),
-        // which cannot be rolled back
+            failures: parent.failures.clone(),
-        assert!(other.merges.is_empty());
+            merges: VecDeque::new(),
-        self.merges.push_front(other);
+        });
-        if self.merges.len() > MAX_CACHE_ENTRIES {
+        for merge in &parent.merges {
-            self.merges.pop_back();
+            self.merges.push_back(StatusCache {
+                signatures: merge.signatures.clone(),
+                failures: merge.failures.clone(),
+                merges: VecDeque::new(),
+            });
+        }
+        self.merges.truncate(MAX_CACHE_ENTRIES);
+
+        self.merges.len() == MAX_CACHE_ENTRIES
+    }
+
+    /// copy the parents and parents' merges up to this instance, up to
+    ///   MAX_CACHE_ENTRIES deep
+    pub fn merge_parents<U>(&mut self, parents: &[U])
+    where
+        U: Deref<Target = Self>,
+    {
+        for parent in parents {
+            if self.merge_parent_is_full(parent) {
+                break;
+            }
         }
     }
 
@@ -158,10 +178,10 @@ mod tests {
         let last_id = hash(Hash::default().as_ref());
         let mut status_cache = BankStatusCache::new(&last_id);
         assert_eq!(status_cache.has_signature(&sig), false);
-        assert_eq!(status_cache.get_signature_status(&sig), None,);
+        assert_eq!(status_cache.get_signature_status(&sig), None);
         status_cache.add(&sig);
         assert_eq!(status_cache.has_signature(&sig), true);
-        assert_eq!(status_cache.get_signature_status(&sig), Some(Ok(())),);
+        assert_eq!(status_cache.get_signature_status(&sig), Some(Ok(())));
     }
 
     #[test]
@@ -190,7 +210,7 @@ mod tests {
         assert_eq!(first.get_signature_status(&sig), Some(Ok(())));
         let last_id = hash(last_id.as_ref());
         first.new_cache(&last_id);
-        assert_eq!(first.get_signature_status(&sig), Some(Ok(())),);
+        assert_eq!(first.get_signature_status(&sig), Some(Ok(())));
         assert!(first.has_signature(&sig));
         first.clear();
         assert_eq!(first.get_signature_status(&sig), None);
@@ -213,31 +233,58 @@ mod tests {
     }
 
     #[test]
-    fn test_has_signature_merged1() {
+    fn test_status_cache_merge_parents_has_signature() {
         let sig = Signature::default();
         let last_id = hash(Hash::default().as_ref());
         let mut first = BankStatusCache::new(&last_id);
         first.add(&sig);
         assert_eq!(first.get_signature_status(&sig), Some(Ok(())));
+
+        // give first a merge
         let last_id = hash(last_id.as_ref());
-        let second = BankStatusCache::new(&last_id);
+        first.new_cache(&last_id);
-        first.merge_into_root(second);
+
-        assert_eq!(first.get_signature_status(&sig), Some(Ok(())),);
+        let last_id = hash(last_id.as_ref());
-        assert!(first.has_signature(&sig));
+        let mut second = BankStatusCache::new(&last_id);
+
+        second.merge_parents(&[&first]);
+
+        assert_eq!(second.get_signature_status(&sig), Some(Ok(())));
+        assert!(second.has_signature(&sig));
     }
 
     #[test]
-    fn test_has_signature_merged2() {
+    #[ignore] // takes a lot of time or RAM or both..
+    fn test_status_cache_merge_parents_overflow() {
+        let mut last_id = hash(Hash::default().as_ref());
+        let mut cache = BankStatusCache::new(&last_id);
+
+        let parents: Vec<_> = (0..MAX_CACHE_ENTRIES)
+            .map(|_| {
+                last_id = hash(last_id.as_ref());
+
+                BankStatusCache::new(&last_id)
+            })
+            .collect();
+
+        let mut parents_refs: Vec<_> = parents.iter().collect();
+
+        last_id = hash(Hash::default().as_ref());
+        let mut root = BankStatusCache::new(&last_id);
+
         let sig = Signature::default();
-        let last_id = hash(Hash::default().as_ref());
+        root.add(&sig);
-        let mut first = BankStatusCache::new(&last_id);
+
-        first.add(&sig);
+        parents_refs.push(&root);
-        assert_eq!(first.get_signature_status(&sig), Some(Ok(())));
+
-        let last_id = hash(last_id.as_ref());
+        assert_eq!(root.get_signature_status(&sig), Some(Ok(())));
-        let mut second = BankStatusCache::new(&last_id);
+        assert!(root.has_signature(&sig));
-        second.merge_into_root(first);
+
-        assert_eq!(second.get_signature_status(&sig), Some(Ok(())),);
+        // will overflow
-        assert!(second.has_signature(&sig));
+        cache.merge_parents(&parents_refs);
+
+        assert_eq!(cache.get_signature_status(&sig), None);
+        assert!(!cache.has_signature(&sig));
     }
 
     #[test]
@@ -268,7 +315,7 @@ mod tests {
         );
         first.clear();
         assert_eq!(first.has_signature(&sig), false);
-        assert_eq!(first.get_signature_status(&sig), None,);
+        assert_eq!(first.get_signature_status(&sig), None);
     }
     #[test]
     fn test_clear_signatures_all() {