Add `shardtree` batch insertion.

shardtree/src/lib.rs

@@ -1713,6 +1713,89 @@ impl<
         Ok(())
     }
 
+    /// Put a range of values into the subtree to fill leaves starting from the given position.
+    ///
+    /// This operation will pad the tree until it contains enough subtrees to reach the starting
+    /// position. It will fully consume the provided iterator, constructing successive subtrees
+    /// until no more values are available. It aggressively prunes the tree as it goes, retaining
+    /// only nodes that either have [`MARKED`] retention, are required to construct a witness for
+    /// such marked nodes, or that must be retained in order to make it possible to truncate the
+    /// tree to any position with [`CHECKPOINT`] retention.
+    ///
+    /// This operation returns the final position at which a leaf was inserted, and the vector of
+    /// [`IncompleteAt`] values that identify addresses at which [`Node::Nil`] nodes were
+    /// introduced to the tree, as well as whether or not those newly introduced nodes will need to
+    /// be filled with values in order to produce witnesses for inserted leaves with [`MARKED`]
+    /// retention.
+    #[allow(clippy::type_complexity)]
+    pub fn batch_insert<I: Iterator<Item = (H, Retention<C>)>>(
+        &mut self,
+        mut start: Position,
+        values: I,
+    ) -> Result<Option<(Position, Vec<IncompleteAt>)>, InsertionError<S::Error>> {
+        let mut values = values.peekable();
+        let mut subtree_root_addr = Address::above_position(Self::subtree_level(), start);
+        let mut max_insert_position = None;
+        let mut all_incomplete = vec![];
+        loop {
+            if values.peek().is_some() {
+                let empty = LocatedTree::empty(subtree_root_addr);
+                let mut res = self
+                    .store
+                    .get_shard(subtree_root_addr)
+                    .unwrap_or(&empty)
+                    .batch_insert(start, values)?
+                    .expect(
+                        "Iterator containing leaf values to insert was verified to be nonempty.",
+                    );
+                self.store
+                    .put_shard(res.subtree)
+                    .map_err(InsertionError::Storage)?;
+                for (id, position) in res.checkpoints.into_iter() {
+                    self.checkpoints
+                        .insert(id, Checkpoint::at_position(position));
+                }
+
+                values = res.remainder;
+                subtree_root_addr = subtree_root_addr.next_at_level();
+                max_insert_position = res.max_insert_position;
+                start = max_insert_position.unwrap() + 1;
+                all_incomplete.append(&mut res.incomplete);
+            } else {
+                break;
+            }
+        }
+
+        self.prune_excess_checkpoints()
+            .map_err(InsertionError::Storage)?;
+
+        Ok(max_insert_position.map(|p| (p, all_incomplete)))
+    }
+
+    /// Insert a tree by decomposing it into its [`SHARD_HEIGHT`] or smaller parts (if necessary)
+    /// and inserting those at their appropriate locations.
+    pub fn insert_tree(
+        &mut self,
+        tree: LocatedPrunableTree<H>,
+    ) -> Result<Vec<IncompleteAt>, InsertionError<S::Error>> {
+        let mut all_incomplete = vec![];
+        for subtree in tree.decompose_to_level(Self::subtree_level()).into_iter() {
+            let root_addr = subtree.root_addr;
+            let contains_marked = subtree.root.try_reduce(&contains_marked).is_left();
+            let empty = LocatedTree::empty(root_addr);
+            let (new_subtree, mut incomplete) = self
+                .store
+                .get_shard(root_addr)
+                .unwrap_or(&empty)
+                .insert_subtree(subtree, contains_marked)?;
+            self.store
+                .put_shard(new_subtree)
+                .map_err(InsertionError::Storage)?;
+            all_incomplete.append(&mut incomplete);
+        }
+        Ok(all_incomplete)
+    }
+
     /// Adds a checkpoint at the rightmost leaf state of the tree.
     pub fn checkpoint(&mut self, checkpoint_id: C) -> bool {
         fn go<H: Hashable + Clone + PartialEq>(