Move blocktree_processor to solana_ledger (#6460)

* Drop core::result dependency in bank_forks

* Move blocktree_processor into solana_ledger

Cargo.lock

@@ -3268,12 +3268,10 @@ dependencies = [
  "bincode 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "bs58 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "bzip2 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "chrono 0.4.9 (registry+https://github.com/rust-lang/crates.io-index)",
  "core_affinity 0.5.9 (registry+https://github.com/rust-lang/crates.io-index)",
  "crc 1.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "crossbeam-channel 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
- "dir-diff 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "fs_extra 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "hex-literal 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "indexmap 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -3322,7 +3320,6 @@ dependencies = [
  "solana-vote-signer 0.20.0",
  "symlink 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "sys-info 0.5.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)",
  "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio-codec 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -3534,9 +3531,12 @@ version = "0.20.0"
 dependencies = [
  "bincode 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "bzip2 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "chrono 0.4.9 (registry+https://github.com/rust-lang/crates.io-index)",
+ "dir-diff 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "dlopen 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "dlopen_derive 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "fs_extra 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -3549,6 +3549,7 @@ dependencies = [
  "serde_derive 1.0.101 (registry+https://github.com/rust-lang/crates.io-index)",
  "solana-budget-api 0.20.0",
  "solana-logger 0.20.0",
+ "solana-measure 0.20.0",
  "solana-merkle-tree 0.20.0",
  "solana-metrics 0.20.0",
  "solana-rayon-threadlimit 0.20.0",
@@ -3558,6 +3559,8 @@ dependencies = [
  "solana-stake-api 0.20.0",
  "solana-vote-api 0.20.0",
  "sys-info 0.5.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -3571,7 +3574,6 @@ dependencies = [
  "serde_derive 1.0.101 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_yaml 0.8.11 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-core 0.20.0",
  "solana-ledger 0.20.0",
  "solana-logger 0.20.0",
  "solana-runtime 0.20.0",

banking_bench/src/main.rs

@@ -6,7 +6,6 @@ use crossbeam_channel::unbounded;
 use log::*;
 use rand::{thread_rng, Rng};
 use rayon::prelude::*;
-use solana_core::bank_forks::BankForks;
 use solana_core::banking_stage::{create_test_recorder, BankingStage};
 use solana_core::cluster_info::ClusterInfo;
 use solana_core::cluster_info::Node;
@@ -15,6 +14,7 @@ use solana_core::packet::to_packets_chunked;
 use solana_core::poh_recorder::PohRecorder;
 use solana_core::poh_recorder::WorkingBankEntry;
 use solana_core::service::Service;
+use solana_ledger::bank_forks::BankForks;
 use solana_ledger::blocktree::{get_tmp_ledger_path, Blocktree};
 use solana_measure::measure::Measure;
 use solana_runtime::bank::Bank;

core/Cargo.toml

@@ -21,12 +21,10 @@ move = []
 bincode = "1.2.0"
 bs58 = "0.3.0"
 byteorder = "1.3.2"
-bzip2 = "0.3.3"
 chrono = { version = "0.4.9", features = ["serde"] }
 core_affinity = "0.5.9"
 crc = { version = "1.8.1", optional = true }
 crossbeam-channel = "0.3"
-dir-diff = "0.3.2"
 fs_extra = "1.1.0"
 indexmap = "1.1"
 itertools = "0.8.0"
@@ -68,7 +66,6 @@ solana-vote-api = { path = "../programs/vote_api", version = "0.20.0" }
 solana-vote-signer = { path = "../vote-signer", version = "0.20.0" }
 symlink = "0.1.0"
 sys-info = "0.5.8"
-tar = "0.4.26"
 tempfile = "3.1.0"
 tokio = "0.1"
 tokio-codec = "0.1"

core/benches/banking_stage.rs

@@ -9,7 +9,6 @@ use log::*;
 use rand::{thread_rng, Rng};
 use rayon::prelude::*;
 use solana_core::banking_stage::{create_test_recorder, BankingStage};
-use solana_core::blocktree_processor::process_entries;
 use solana_core::cluster_info::ClusterInfo;
 use solana_core::cluster_info::Node;
 use solana_core::genesis_utils::{create_genesis_block, GenesisBlockInfo};
@@ -18,6 +17,7 @@ use solana_core::poh_recorder::WorkingBankEntry;
 use solana_core::service::Service;
 use solana_core::test_tx::test_tx;
 use solana_ledger::blocktree::{get_tmp_ledger_path, Blocktree};
+use solana_ledger::blocktree_processor::process_entries;
 use solana_ledger::entry::{next_hash, Entry};
 use solana_runtime::bank::Bank;
 use solana_sdk::genesis_block::GenesisBlock;

core/benches/retransmit_stage.rs

@@ -4,13 +4,13 @@ extern crate solana_core;
 extern crate test;
 
 use log::*;
-use solana_core::bank_forks::BankForks;
 use solana_core::cluster_info::{ClusterInfo, Node};
 use solana_core::contact_info::ContactInfo;
 use solana_core::genesis_utils::{create_genesis_block, GenesisBlockInfo};
 use solana_core::packet::to_packets_chunked;
 use solana_core::retransmit_stage::retransmitter;
 use solana_core::test_tx::test_tx;
+use solana_ledger::bank_forks::BankForks;
 use solana_ledger::leader_schedule_cache::LeaderScheduleCache;
 use solana_measure::measure::Measure;
 use solana_runtime::bank::Bank;

core/src/cluster_info.rs

@@ -12,7 +12,6 @@
 //! * layer 2 - Everyone else, if layer 1 is `2^10`, layer 2 should be able to fit `2^20` number of nodes.
 //!
 //! Bank needs to provide an interface for us to query the stake weight
-use crate::bank_forks::BankForks;
 use crate::contact_info::ContactInfo;
 use crate::crds_gossip::CrdsGossip;
 use crate::crds_gossip_error::CrdsGossipError;
@@ -30,6 +29,7 @@ use itertools::Itertools;
 use rand::SeedableRng;
 use rand::{thread_rng, Rng};
 use rand_chacha::ChaChaRng;
+use solana_ledger::bank_forks::BankForks;
 use solana_ledger::blocktree::Blocktree;
 use solana_ledger::staking_utils;
 use solana_metrics::{datapoint_debug, inc_new_counter_debug, inc_new_counter_error};
@@ -1779,7 +1779,6 @@ fn report_time_spent(label: &str, time: &Duration, extra: &str) {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::blocktree_processor::tests::fill_blocktree_slot_with_ticks;
     use crate::crds_value::CrdsValueLabel;
     use crate::repair_service::RepairType;
     use crate::result::Error;
@@ -1788,6 +1787,7 @@ mod tests {
     use solana_ledger::blocktree::get_tmp_ledger_path;
     use solana_ledger::blocktree::make_many_slot_entries;
     use solana_ledger::blocktree::Blocktree;
+    use solana_ledger::blocktree_processor::fill_blocktree_slot_with_ticks;
     use solana_ledger::shred::{
         max_ticks_per_n_shreds, CodingShredHeader, DataShredHeader, Shred, ShredCommonHeader,
     };

core/src/cluster_info_repair_listener.rs

@@ -1,13 +1,13 @@
 use crate::cluster_info::ClusterInfo;
 use crate::crds_value::EpochSlots;
 use crate::result::Result;
-use crate::rooted_slot_iterator::RootedSlotIterator;
 use crate::service::Service;
 use byteorder::{ByteOrder, LittleEndian};
 use rand::seq::SliceRandom;
 use rand::SeedableRng;
 use rand_chacha::ChaChaRng;
 use solana_ledger::blocktree::Blocktree;
+use solana_ledger::rooted_slot_iterator::RootedSlotIterator;
 use solana_metrics::datapoint;
 use solana_sdk::{epoch_schedule::EpochSchedule, pubkey::Pubkey};
 use std::{

core/src/consensus.rs

@@ -1,4 +1,4 @@
-use crate::bank_forks::BankForks;
+use solana_ledger::bank_forks::BankForks;
 use solana_metrics::datapoint_debug;
 use solana_runtime::bank::Bank;
 use solana_sdk::account::Account;

core/src/gossip_service.rs

@@ -1,12 +1,12 @@
 //! The `gossip_service` module implements the network control plane.
 
-use crate::bank_forks::BankForks;
 use crate::cluster_info::{ClusterInfo, VALIDATOR_PORT_RANGE};
 use crate::contact_info::ContactInfo;
 use crate::service::Service;
 use crate::streamer;
 use rand::{thread_rng, Rng};
 use solana_client::thin_client::{create_client, ThinClient};
+use solana_ledger::bank_forks::BankForks;
 use solana_ledger::blocktree::Blocktree;
 use solana_sdk::pubkey::Pubkey;
 use solana_sdk::signature::{Keypair, KeypairUtil};

core/src/lib.rs

@@ -5,7 +5,6 @@
 //! command-line tools to spin up validators and a Rust library
 //!
 
-pub mod bank_forks;
 pub mod banking_stage;
 pub mod broadcast_stage;
 pub mod chacha;
@@ -18,7 +17,6 @@ pub mod shred_fetch_stage;
 pub mod contact_info;
 pub mod blockstream;
 pub mod blockstream_service;
-pub mod blocktree_processor;
 pub mod cluster_info;
 pub mod cluster_info_repair_listener;
 pub mod consensus;
@@ -44,7 +42,6 @@ pub mod replay_stage;
 pub mod replicator;
 pub mod result;
 pub mod retransmit_stage;
-pub mod rooted_slot_iterator;
 pub mod rpc;
 pub mod rpc_pubsub;
 pub mod rpc_pubsub_service;
@@ -54,9 +51,7 @@ pub mod sendmmsg;
 pub mod service;
 pub mod sigverify;
 pub mod sigverify_stage;
-pub mod snapshot_package;
 pub mod snapshot_packager_service;
-pub mod snapshot_utils;
 pub mod storage_stage;
 pub mod streamer;
 pub mod test_tx;
@@ -70,8 +65,6 @@ pub mod window_service;
 #[macro_use]
 extern crate solana_budget_program;
 
-extern crate solana_storage_program;
-
 #[cfg(test)]
 #[macro_use]
 extern crate hex_literal;
@@ -95,11 +88,3 @@ extern crate matches;
 
 #[macro_use]
 extern crate solana_ledger;
-
-extern crate bzip2;
-extern crate crossbeam_channel;
-extern crate dir_diff;
-extern crate fs_extra;
-extern crate symlink;
-extern crate tar;
-extern crate tempfile;

core/src/repair_service.rs

@@ -1,9 +1,10 @@
 //! The `repair_service` module implements the tools necessary to generate a thread which
 //! regularly finds missing blobs in the ledger and sends repair requests for those blobs
 use crate::{
-    bank_forks::BankForks, cluster_info::ClusterInfo,
-    cluster_info_repair_listener::ClusterInfoRepairListener, result::Result, service::Service,
+    cluster_info::ClusterInfo, cluster_info_repair_listener::ClusterInfoRepairListener,
+    result::Result, service::Service,
 };
+use solana_ledger::bank_forks::BankForks;
 use solana_ledger::blocktree::{Blocktree, CompletedSlotsReceiver, SlotMeta};
 use solana_sdk::{epoch_schedule::EpochSchedule, pubkey::Pubkey};
 use std::{

core/src/replay_stage.rs

@@ -1,7 +1,5 @@
 //! The `replay_stage` replays transactions broadcast by the leader.
 
-use crate::bank_forks::BankForks;
-use crate::blocktree_processor;
 use crate::cluster_info::ClusterInfo;
 use crate::confidence::{
     AggregateConfidenceService, ConfidenceAggregationData, ForkConfidenceCache,
@@ -12,10 +10,12 @@ use crate::poh_recorder::PohRecorder;
 use crate::result::{Error, Result};
 use crate::rpc_subscriptions::RpcSubscriptions;
 use crate::service::Service;
-use crate::snapshot_package::SnapshotPackageSender;
+use solana_ledger::bank_forks::BankForks;
 use solana_ledger::blocktree::{Blocktree, BlocktreeError};
+use solana_ledger::blocktree_processor;
 use solana_ledger::entry::{Entry, EntrySlice};
 use solana_ledger::leader_schedule_cache::LeaderScheduleCache;
+use solana_ledger::snapshot_package::SnapshotPackageSender;
 use solana_metrics::{datapoint_warn, inc_new_counter_info};
 use solana_runtime::bank::Bank;
 use solana_sdk::hash::Hash;

core/src/result.rs

@@ -3,12 +3,9 @@
 use crate::cluster_info;
 use crate::packet;
 use crate::poh_recorder;
-use crate::snapshot_utils;
-use bincode;
-use serde_json;
 use solana_ledger::blocktree;
+use solana_ledger::snapshot_utils;
 use solana_sdk::transaction;
-use std;
 use std::any::Any;
 
 #[derive(Debug)]

core/src/retransmit_stage.rs

@@ -1,7 +1,6 @@
 //! The `retransmit_stage` retransmits blobs between validators
 
 use crate::{
-    bank_forks::BankForks,
     cluster_info::{compute_retransmit_peers, ClusterInfo, DATA_PLANE_FANOUT},
     repair_service::RepairStrategy,
     result::{Error, Result},
@@ -12,6 +11,7 @@ use crate::{
 use rand::SeedableRng;
 use rand_chacha::ChaChaRng;
 use solana_ledger::{
+    bank_forks::BankForks,
     blocktree::{Blocktree, CompletedSlotsReceiver},
     leader_schedule_cache::LeaderScheduleCache,
     staking_utils,
@@ -262,11 +262,11 @@ impl Service for RetransmitStage {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::blocktree_processor::{process_blocktree, ProcessOptions};
     use crate::contact_info::ContactInfo;
     use crate::genesis_utils::{create_genesis_block, GenesisBlockInfo};
     use crate::packet::{Meta, Packet, Packets};
     use solana_ledger::blocktree::create_new_tmp_ledger;
+    use solana_ledger::blocktree_processor::{process_blocktree, ProcessOptions};
     use solana_netutil::find_available_port_in_range;
     use solana_sdk::pubkey::Pubkey;
 

core/src/rpc.rs

@@ -1,7 +1,6 @@
 //! The `rpc` module implements the Solana RPC interface.
 
 use crate::{
-    bank_forks::BankForks,
     cluster_info::ClusterInfo,
     confidence::{BankConfidence, ForkConfidenceCache},
     contact_info::ContactInfo,
@@ -15,6 +14,7 @@ use jsonrpc_core::{Error, Metadata, Result};
 use jsonrpc_derive::rpc;
 use solana_client::rpc_request::{RpcEpochInfo, RpcVoteAccountInfo, RpcVoteAccountStatus};
 use solana_drone::drone::request_airdrop_transaction;
+use solana_ledger::bank_forks::BankForks;
 use solana_runtime::bank::Bank;
 use solana_sdk::{
     account::Account,

core/src/rpc_pubsub.rs

@@ -238,13 +238,13 @@ impl RpcSolPubSub for RpcSolPubSubImpl {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::bank_forks::BankForks;
     use crate::genesis_utils::{create_genesis_block, GenesisBlockInfo};
     use jsonrpc_core::futures::sync::mpsc;
     use jsonrpc_core::Response;
     use jsonrpc_pubsub::{PubSubHandler, Session};
     use solana_budget_api;
     use solana_budget_api::budget_instruction;
+    use solana_ledger::bank_forks::BankForks;
     use solana_runtime::bank::Bank;
     use solana_sdk::pubkey::Pubkey;
     use solana_sdk::signature::{Keypair, KeypairUtil};

core/src/rpc_service.rs

@@ -1,14 +1,15 @@
 //! The `rpc_service` module implements the Solana JSON RPC service.
 
 use crate::{
-    bank_forks::BankForks, cluster_info::ClusterInfo, confidence::ForkConfidenceCache, rpc::*,
-    service::Service, storage_stage::StorageState, validator::ValidatorExit,
+    cluster_info::ClusterInfo, confidence::ForkConfidenceCache, rpc::*, service::Service,
+    storage_stage::StorageState, validator::ValidatorExit,
 };
 use jsonrpc_core::MetaIoHandler;
 use jsonrpc_http_server::{
     hyper, AccessControlAllowOrigin, CloseHandle, DomainsValidation, RequestMiddleware,
     RequestMiddlewareAction, ServerBuilder,
 };
+use solana_ledger::bank_forks::BankForks;
 use solana_sdk::hash::Hash;
 use std::{
     net::SocketAddr,

core/src/rpc_subscriptions.rs

@@ -1,11 +1,11 @@
 //! The `pubsub` module implements a threaded subscription service on client RPC request
 
-use crate::bank_forks::BankForks;
 use core::hash::Hash;
 use jsonrpc_core::futures::Future;
 use jsonrpc_pubsub::typed::Sink;
 use jsonrpc_pubsub::SubscriptionId;
 use serde::Serialize;
+use solana_ledger::bank_forks::BankForks;
 use solana_runtime::bank::Bank;
 use solana_sdk::account::Account;
 use solana_sdk::pubkey::Pubkey;

core/src/snapshot_packager_service.rs

@@ -1,8 +1,8 @@
 use crate::result::{Error, Result};
 use crate::service::Service;
-use crate::snapshot_package::{SnapshotPackage, SnapshotPackageReceiver};
-use crate::snapshot_utils::{self, TAR_ACCOUNTS_DIR, TAR_SNAPSHOTS_DIR};
 use bincode::serialize_into;
+use solana_ledger::snapshot_package::{SnapshotPackage, SnapshotPackageReceiver};
+use solana_ledger::snapshot_utils::{self, TAR_ACCOUNTS_DIR, TAR_SNAPSHOTS_DIR};
 use solana_measure::measure::Measure;
 use solana_metrics::datapoint_info;
 use solana_runtime::status_cache::SlotDelta;
@@ -187,7 +187,7 @@ impl Service for SnapshotPackagerService {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::snapshot_utils;
+    use solana_ledger::snapshot_utils;
     use solana_runtime::accounts_db::AccountStorageEntry;
     use std::fs::OpenOptions;
     use std::io::Write;

core/src/storage_stage.rs

@@ -2,7 +2,6 @@
 // for storage mining. Replicators submit storage proofs, validator then bundles them
 // to submit its proof for mining to be rewarded.
 
-use crate::bank_forks::BankForks;
 use crate::chacha_cuda::chacha_cbc_encrypt_file_many_keys;
 use crate::cluster_info::ClusterInfo;
 use crate::contact_info::ContactInfo;
@@ -10,6 +9,7 @@ use crate::result::{Error, Result};
 use crate::service::Service;
 use rand::{Rng, SeedableRng};
 use rand_chacha::ChaChaRng;
+use solana_ledger::bank_forks::BankForks;
 use solana_ledger::blocktree::Blocktree;
 use solana_runtime::bank::Bank;
 use solana_runtime::storage_utils::replicator_accounts;

core/src/tvu.rs

@@ -12,7 +12,6 @@
 //! 4. StorageStage
 //! - Generating the keys used to encrypt the ledger and sample it for storage mining.
 
-use crate::bank_forks::BankForks;
 use crate::blockstream_service::BlockstreamService;
 use crate::cluster_info::ClusterInfo;
 use crate::confidence::ForkConfidenceCache;
@@ -25,6 +24,7 @@ use crate::service::Service;
 use crate::shred_fetch_stage::ShredFetchStage;
 use crate::snapshot_packager_service::SnapshotPackagerService;
 use crate::storage_stage::{StorageStage, StorageState};
+use solana_ledger::bank_forks::BankForks;
 use solana_ledger::blocktree::{Blocktree, CompletedSlotsReceiver};
 use solana_ledger::leader_schedule_cache::LeaderScheduleCache;
 use solana_sdk::pubkey::Pubkey;

core/src/validator.rs

@@ -1,7 +1,5 @@
 //! The `validator` module hosts all the validator microservices.
 
-use crate::bank_forks::{BankForks, SnapshotConfig};
-use crate::blocktree_processor::{self, BankForksInfo};
 use crate::broadcast_stage::BroadcastStageType;
 use crate::cluster_info::{ClusterInfo, Node};
 use crate::confidence::ForkConfidenceCache;
@@ -15,12 +13,14 @@ use crate::rpc_service::JsonRpcService;
 use crate::rpc_subscriptions::RpcSubscriptions;
 use crate::service::Service;
 use crate::sigverify;
-use crate::snapshot_utils;
 use crate::storage_stage::StorageState;
 use crate::tpu::Tpu;
 use crate::tvu::{Sockets, Tvu};
+use solana_ledger::bank_forks::{BankForks, SnapshotConfig};
 use solana_ledger::blocktree::{Blocktree, CompletedSlotsReceiver};
+use solana_ledger::blocktree_processor::{self, BankForksInfo};
 use solana_ledger::leader_schedule_cache::LeaderScheduleCache;
+use solana_ledger::snapshot_utils;
 use solana_metrics::datapoint_info;
 use solana_sdk::clock::{Slot, DEFAULT_SLOTS_PER_TURN};
 use solana_sdk::genesis_block::GenesisBlock;

core/tests/bank_forks.rs

@@ -5,11 +5,11 @@ mod tests {
     use bincode::serialize_into;
     use fs_extra::dir::CopyOptions;
     use itertools::Itertools;
-    use solana_core::bank_forks::{BankForks, SnapshotConfig};
     use solana_core::genesis_utils::{create_genesis_block, GenesisBlockInfo};
     use solana_core::service::Service;
     use solana_core::snapshot_packager_service::SnapshotPackagerService;
-    use solana_core::snapshot_utils;
+    use solana_ledger::bank_forks::{BankForks, SnapshotConfig};
+    use solana_ledger::snapshot_utils;
     use solana_runtime::bank::Bank;
     use solana_runtime::status_cache::SlotDelta;
     use solana_runtime::status_cache::MAX_CACHE_ENTRIES;

core/tests/storage_stage.rs

@@ -6,13 +6,13 @@ extern crate solana_ledger;
 #[cfg(test)]
 mod tests {
     use log::*;
-    use solana_core::bank_forks::BankForks;
-    use solana_core::blocktree_processor;
     use solana_core::genesis_utils::{create_genesis_block, GenesisBlockInfo};
     use solana_core::service::Service;
     use solana_core::storage_stage::{test_cluster_info, SLOTS_PER_TURN_TEST};
     use solana_core::storage_stage::{StorageStage, StorageState};
+    use solana_ledger::bank_forks::BankForks;
     use solana_ledger::blocktree::{create_new_tmp_ledger, Blocktree};
+    use solana_ledger::blocktree_processor;
     use solana_ledger::entry;
     use solana_runtime::bank::Bank;
     use solana_sdk::clock::DEFAULT_TICKS_PER_SLOT;

ledger-tool/Cargo.toml

@@ -15,7 +15,6 @@ serde = "1.0.101"
 serde_derive = "1.0.101"
 serde_json = "1.0.41"
 serde_yaml = "0.8.11"
-solana-core = { path = "../core", version = "0.20.0" }
 solana-ledger = { path = "../ledger", version = "0.20.0" }
 solana-logger = { path = "../logger", version = "0.20.0" }
 solana-runtime = { path = "../runtime", version = "0.20.0" }

ledger-tool/src/main.rs

@@ -1,7 +1,7 @@
 use clap::{crate_description, crate_name, crate_version, value_t_or_exit, App, Arg, SubCommand};
-use solana_core::blocktree_processor::{process_blocktree, ProcessOptions};
-use solana_core::rooted_slot_iterator::RootedSlotIterator;
 use solana_ledger::blocktree::Blocktree;
+use solana_ledger::blocktree_processor::{process_blocktree, ProcessOptions};
+use solana_ledger::rooted_slot_iterator::RootedSlotIterator;
 use solana_sdk::clock::Slot;
 use solana_sdk::genesis_block::GenesisBlock;
 use std::collections::BTreeMap;

ledger-tool/tests/basic.rs

@@ -2,8 +2,8 @@
 extern crate solana_ledger;
 
 use assert_cmd::prelude::*;
-use solana_core::genesis_utils::create_genesis_block;
 use solana_ledger::blocktree::create_new_tmp_ledger;
+use solana_ledger::genesis_utils::create_genesis_block;
 use std::process::Command;
 use std::process::Output;
 

ledger/Cargo.toml

@@ -11,9 +11,12 @@ edition = "2018"
 [dependencies]
 bincode = "1.2.0"
 byteorder = "1.3.2"
+bzip2 = "0.3.3"
 chrono = { version = "0.4.9", features = ["serde"] }
+dir-diff = "0.3.2"
 dlopen = "0.1.8"
 dlopen_derive = "0.1.4"
+fs_extra = "1.1.0"
 itertools = "0.8.0"
 lazy_static = "1.4.0"
 log = { version = "0.4.8" }
@@ -24,6 +27,7 @@ reed-solomon-erasure = { package = "solana-reed-solomon-erasure", version = "4.0
 serde = "1.0.101"
 serde_derive = "1.0.101"
 solana-logger = { path = "../logger", version = "0.20.0" }
+solana-measure = { path = "../measure", version = "0.20.0" }
 solana-merkle-tree = { path = "../merkle-tree", version = "0.20.0" }
 solana-metrics = { path = "../metrics", version = "0.20.0" }
 solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "0.20.0" }
@@ -32,6 +36,8 @@ solana-sdk = { path = "../sdk", version = "0.20.0" }
 solana-stake-api = { path = "../programs/stake_api", version = "0.20.0" }
 solana-vote-api = { path = "../programs/vote_api", version = "0.20.0" }
 sys-info = "0.5.8"
+tar = "0.4.26"
+tempfile = "3.1.0"
 
 [dependencies.rocksdb]
 # Avoid the vendored bzip2 within rocksdb-sys that can cause linker conflicts

ledger/src/bank_forks.rs

@@ -1,8 +1,8 @@
 //! The `bank_forks` module implments BankForks a DAG of checkpointed Banks
 
-use crate::result::Result;
-use crate::snapshot_package::SnapshotPackageSender;
-use crate::snapshot_utils;
+use crate::snapshot_package::{SnapshotPackageSendError, SnapshotPackageSender};
+use crate::snapshot_utils::{self, SnapshotError};
+use log::*;
 use solana_measure::measure::Measure;
 use solana_metrics::inc_new_counter_info;
 use solana_runtime::bank::Bank;
@@ -26,6 +26,25 @@ pub struct SnapshotConfig {
     pub snapshot_path: PathBuf,
 }
 
+#[derive(Debug)]
+pub enum BankForksError {
+    SnapshotError(SnapshotError),
+    SnapshotPackageSendError(SnapshotPackageSendError),
+}
+pub type Result<T> = std::result::Result<T, BankForksError>;
+
+impl std::convert::From<SnapshotError> for BankForksError {
+    fn from(e: SnapshotError) -> BankForksError {
+        BankForksError::SnapshotError(e)
+    }
+}
+
+impl std::convert::From<SnapshotPackageSendError> for BankForksError {
+    fn from(e: SnapshotPackageSendError) -> BankForksError {
+        BankForksError::SnapshotPackageSendError(e)
+    }
+}
+
 pub struct BankForks {
     pub banks: HashMap<u64, Arc<Bank>>,
     working_bank: Arc<Bank>,

ledger/src/blocktree_processor.rs

@@ -1,17 +1,19 @@
 use crate::bank_forks::BankForks;
+use crate::blocktree::{Blocktree, SlotMeta};
+use crate::entry::{create_ticks, Entry, EntrySlice};
+use crate::leader_schedule_cache::LeaderScheduleCache;
+use log::*;
 use rand::seq::SliceRandom;
 use rand::thread_rng;
 use rayon::prelude::*;
 use rayon::ThreadPool;
-use solana_ledger::blocktree::{Blocktree, SlotMeta};
-use solana_ledger::entry::{Entry, EntrySlice};
-use solana_ledger::leader_schedule_cache::LeaderScheduleCache;
 use solana_metrics::{datapoint, datapoint_error, inc_new_counter_debug};
 use solana_runtime::bank::Bank;
 use solana_runtime::transaction_batch::TransactionBatch;
 use solana_sdk::clock::{Slot, MAX_RECENT_BLOCKHASHES};
 use solana_sdk::genesis_block::GenesisBlock;
 use solana_sdk::hash::Hash;
+use solana_sdk::signature::{Keypair, KeypairUtil};
 use solana_sdk::timing::duration_as_ms;
 use solana_sdk::transaction::Result;
 use std::result;
@@ -447,15 +449,42 @@ fn process_pending_slots(
     Ok(fork_info)
 }
 
+pub fn fill_blocktree_slot_with_ticks(
+    blocktree: &Blocktree,
+    ticks_per_slot: u64,
+    slot: u64,
+    parent_slot: u64,
+    last_entry_hash: Hash,
+) -> Hash {
+    let entries = create_ticks(ticks_per_slot, last_entry_hash);
+    let last_entry_hash = entries.last().unwrap().hash;
+
+    blocktree
+        .write_entries(
+            slot,
+            0,
+            0,
+            ticks_per_slot,
+            Some(parent_slot),
+            true,
+            &Arc::new(Keypair::new()),
+            entries,
+        )
+        .unwrap();
+
+    last_entry_hash
+}
+
 #[cfg(test)]
 pub mod tests {
     use super::*;
+    use crate::blocktree::create_new_tmp_ledger;
+    use crate::entry::{create_ticks, next_entry, next_entry_mut, Entry};
     use crate::genesis_utils::{
         create_genesis_block, create_genesis_block_with_leader, GenesisBlockInfo,
     };
+    use matches::assert_matches;
     use rand::{thread_rng, Rng};
-    use solana_ledger::blocktree::create_new_tmp_ledger;
-    use solana_ledger::entry::{create_ticks, next_entry, next_entry_mut, Entry};
     use solana_sdk::{
         epoch_schedule::EpochSchedule,
         hash::Hash,
@@ -467,32 +496,6 @@ pub mod tests {
     };
     use std::sync::RwLock;
 
-    pub fn fill_blocktree_slot_with_ticks(
-        blocktree: &Blocktree,
-        ticks_per_slot: u64,
-        slot: u64,
-        parent_slot: u64,
-        last_entry_hash: Hash,
-    ) -> Hash {
-        let entries = create_ticks(ticks_per_slot, last_entry_hash);
-        let last_entry_hash = entries.last().unwrap().hash;
-
-        blocktree
-            .write_entries(
-                slot,
-                0,
-                0,
-                ticks_per_slot,
-                Some(parent_slot),
-                true,
-                &Arc::new(Keypair::new()),
-                entries,
-            )
-            .unwrap();
-
-        last_entry_hash
-    }
-
     #[test]
     fn test_process_blocktree_with_incomplete_slot() {
         solana_logger::setup();

ledger/src/lib.rs

@@ -1,5 +1,7 @@
+pub mod bank_forks;
 #[macro_use]
 pub mod blocktree;
+pub mod blocktree_processor;
 pub mod entry;
 pub mod erasure;
 pub mod genesis_utils;
@@ -8,5 +10,11 @@ pub mod leader_schedule_cache;
 pub mod leader_schedule_utils;
 pub mod perf_libs;
 pub mod poh;
+pub mod rooted_slot_iterator;
 pub mod shred;
+pub mod snapshot_package;
+pub mod snapshot_utils;
 pub mod staking_utils;
+
+#[macro_use]
+extern crate solana_metrics;

ledger/src/rooted_slot_iterator.rs

@@ -1,4 +1,5 @@
-use solana_ledger::blocktree::*;
+use crate::blocktree::*;
+use log::*;
 
 pub struct RootedSlotIterator<'a> {
     next_slots: Vec<u64>,
@@ -52,7 +53,7 @@ impl<'a> Iterator for RootedSlotIterator<'a> {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::blocktree_processor::tests::fill_blocktree_slot_with_ticks;
+    use crate::blocktree_processor::fill_blocktree_slot_with_ticks;
     use solana_sdk::hash::Hash;
 
     #[test]

ledger/src/snapshot_package.rs

@@ -2,12 +2,13 @@ use solana_runtime::accounts_db::AccountStorageEntry;
 use solana_runtime::status_cache::SlotDelta;
 use solana_sdk::transaction::Result as TransactionResult;
 use std::path::PathBuf;
-use std::sync::mpsc::{Receiver, Sender};
+use std::sync::mpsc::{Receiver, SendError, Sender};
 use std::sync::Arc;
 use tempfile::TempDir;
 
 pub type SnapshotPackageSender = Sender<SnapshotPackage>;
 pub type SnapshotPackageReceiver = Receiver<SnapshotPackage>;
+pub type SnapshotPackageSendError = SendError<SnapshotPackage>;
 
 pub struct SnapshotPackage {
     pub root: u64,

ledger/src/snapshot_utils.rs

@@ -2,6 +2,7 @@ use crate::snapshot_package::SnapshotPackage;
 use bincode::{deserialize_from, serialize_into};
 use bzip2::bufread::BzDecoder;
 use fs_extra::dir::CopyOptions;
+use log::*;
 use solana_measure::measure::Measure;
 use solana_runtime::bank::Bank;
 use solana_runtime::status_cache::SlotDelta;

local_cluster/src/tests/local_cluster.rs

@@ -6,10 +6,10 @@ use crate::{
 use log::*;
 use serial_test_derive::serial;
 use solana_core::{
-    bank_forks::SnapshotConfig, broadcast_stage::BroadcastStageType,
-    gossip_service::discover_cluster, snapshot_utils, validator::ValidatorConfig,
+    broadcast_stage::BroadcastStageType, gossip_service::discover_cluster,
+    validator::ValidatorConfig,
 };
-use solana_ledger::blocktree::Blocktree;
+use solana_ledger::{bank_forks::SnapshotConfig, blocktree::Blocktree, snapshot_utils};
 use solana_runtime::accounts_db::AccountsDB;
 use solana_sdk::{
     client::SyncClient,

validator/src/lib.rs

@@ -4,7 +4,6 @@ use console::{style, Emoji};
 use indicatif::{ProgressBar, ProgressStyle};
 use log::*;
 use solana_client::rpc_client::RpcClient;
-use solana_core::bank_forks::SnapshotConfig;
 use solana_core::cluster_info::{Node, VALIDATOR_PORT_RANGE};
 use solana_core::contact_info::ContactInfo;
 use solana_core::gossip_service::discover;
@@ -12,6 +11,7 @@ use solana_core::ledger_cleanup_service::DEFAULT_MAX_LEDGER_SLOTS;
 use solana_core::service::Service;
 use solana_core::socketaddr;
 use solana_core::validator::{Validator, ValidatorConfig};
+use solana_ledger::bank_forks::SnapshotConfig;
 use solana_sdk::clock::Slot;
 use solana_sdk::hash::Hash;
 use solana_sdk::signature::{read_keypair_file, Keypair, KeypairUtil};
@@ -195,7 +195,7 @@ fn initialize_ledger_path(
     download_tar_bz2(&rpc_addr, "genesis.tar.bz2", ledger_path, true)?;
 
     if !no_snapshot_fetch {
-        let snapshot_package = solana_core::snapshot_utils::get_snapshot_tar_path(ledger_path);
+        let snapshot_package = solana_ledger::snapshot_utils::get_snapshot_tar_path(ledger_path);
         if snapshot_package.exists() {
             fs::remove_file(&snapshot_package)
                 .unwrap_or_else(|err| warn!("error removing {:?}: {}", snapshot_package, err));