Continue processing the ledger on InvalidTickCount errors

ledger/src/blocktree_processor.rs

@@ -239,6 +239,9 @@ pub enum BlocktreeProcessorError {
 
     #[error("invalid transaction")]
     InvalidTransaction,
+
+    #[error("no valid forks found")]
+    NoValidForksFound,
 }
 
 /// Callback for accessing bank state while processing the blocktree
@@ -316,6 +319,9 @@ pub fn process_blocktree_from_root(
                 opts,
             )?;
             let (banks, bank_forks_info): (Vec<_>, Vec<_>) = fork_info.into_iter().unzip();
+            if banks.is_empty() {
+                return Err(BlocktreeProcessorError::NoValidForksFound);
+            }
             let bank_forks = BankForks::new_from_banks(&banks, rooted_path);
             (bank_forks, bank_forks_info, leader_schedule_cache)
         } else {
@@ -513,7 +519,10 @@ fn process_pending_slots(
             BlocktreeProcessorError::FailedToLoadEntries
         })?;
 
-        verify_and_process_slot_entries(&bank, &entries, last_entry_hash, opts)?;
+        if let Err(err) = verify_and_process_slot_entries(&bank, &entries, last_entry_hash, opts) {
+            warn!("slot {} failed to verify: {}", slot, err);
+            continue;
+        }
 
         bank.freeze(); // all banks handled by this routine are created from complete slots
 
@@ -646,8 +655,8 @@ pub mod tests {
         let parent_slot = 0;
         let slot = 1;
         let entries = create_ticks(ticks_per_slot, hashes_per_tick - 1, blockhash);
-        blocktree
+        assert_matches!(
-            .write_entries(
+            blocktree.write_entries(
                 slot,
                 0,
                 0,
@@ -657,18 +666,22 @@ pub mod tests {
                 &Arc::new(Keypair::new()),
                 entries,
                 0,
-            )
+            ),
-            .expect("Expected to write shredded entries to blocktree");
+            Ok(_)
+        );
 
-        let opts = ProcessOptions {
-            poh_verify: true,
-            ..ProcessOptions::default()
-        };
         assert_eq!(
-            process_blocktree(&genesis_config, &blocktree, Vec::new(), opts).err(),
+            process_blocktree(
-            Some(BlocktreeProcessorError::InvalidBlock(
+                &genesis_config,
-                BlockError::InvalidTickHashCount
+                &blocktree,
-            )),
+                Vec::new(),
+                ProcessOptions {
+                    poh_verify: true,
+                    ..ProcessOptions::default()
+                }
+            )
+            .err(),
+            Some(BlocktreeProcessorError::NoValidForksFound)
         );
     }
 
@@ -687,8 +700,8 @@ pub mod tests {
         let parent_slot = 0;
         let slot = 1;
         let entries = create_ticks(ticks_per_slot - 1, 0, blockhash);
-        blocktree
+        assert_matches!(
-            .write_entries(
+            blocktree.write_entries(
                 slot,
                 0,
                 0,
@@ -698,19 +711,42 @@ pub mod tests {
                 &Arc::new(Keypair::new()),
                 entries,
                 0,
-            )
+            ),
-            .expect("Expected to write shredded entries to blocktree");
+            Ok(_)
-
-        let opts = ProcessOptions {
-            poh_verify: true,
-            ..ProcessOptions::default()
-        };
-        assert_eq!(
-            process_blocktree(&genesis_config, &blocktree, Vec::new(), opts).err(),
-            Some(BlocktreeProcessorError::InvalidBlock(
-                BlockError::InvalidTickCount
-            )),
         );
+
+        // No valid forks in blocktree, expect a failure
+        assert_eq!(
+            process_blocktree(
+                &genesis_config,
+                &blocktree,
+                Vec::new(),
+                ProcessOptions {
+                    poh_verify: true,
+                    ..ProcessOptions::default()
+                }
+            )
+            .err(),
+            Some(BlocktreeProcessorError::NoValidForksFound)
+        );
+
+        // Write slot 2 fully
+        let _last_slot2_entry_hash =
+            fill_blocktree_slot_with_ticks(&blocktree, ticks_per_slot, 2, 0, blockhash);
+
+        let (_bank_forks, bank_forks_info, _) = process_blocktree(
+            &genesis_config,
+            &blocktree,
+            Vec::new(),
+            ProcessOptions {
+                poh_verify: true,
+                ..ProcessOptions::default()
+            },
+        )
+        .unwrap();
+
+        // One valid fork, one bad fork.  process_blocktree() should only return the valid fork
+        assert_eq!(bank_forks_info, vec![BankForksInfo { bank_slot: 2 }]);
     }
 
     #[test]
@@ -739,8 +775,8 @@ pub mod tests {
         // per slot.
         let parent_slot = 0;
         let slot = 1;
-        blocktree
+        assert_matches!(
-            .write_entries(
+            blocktree.write_entries(
                 slot,
                 0,
                 0,
@@ -750,8 +786,9 @@ pub mod tests {
                 &Arc::new(Keypair::new()),
                 entries,
                 0,
-            )
+            ),
-            .expect("Expected to write shredded entries to blocktree");
+            Ok(_)
+        );
 
         let opts = ProcessOptions {
             poh_verify: true,
@@ -759,9 +796,7 @@ pub mod tests {
         };
         assert_eq!(
             process_blocktree(&genesis_config, &blocktree, Vec::new(), opts).err(),
-            Some(BlocktreeProcessorError::InvalidBlock(
+            Some(BlocktreeProcessorError::NoValidForksFound)
-                BlockError::TrailingEntry
-            )),
         );
     }
 
@@ -802,8 +837,8 @@ pub mod tests {
             // throw away last one
             entries.pop();
 
-            blocktree
+            assert_matches!(
-                .write_entries(
+                blocktree.write_entries(
                     slot,
                     0,
                     0,
@@ -813,8 +848,9 @@ pub mod tests {
                     &Arc::new(Keypair::new()),
                     entries,
                     0,
-                )
+                ),
-                .expect("Expected to write shredded entries to blocktree");
+                Ok(_)
+            );
         }
 
         // slot 2, points at slot 1