From 5468be2ef971b6f4d66af18a70036a3fb975e20e Mon Sep 17 00:00:00 2001 From: Greg Fitzgerald Date: Fri, 18 Oct 2019 10:28:51 -0600 Subject: [PATCH] Add solana-ledger crate (#6415) automerge --- Cargo.lock | 42 ++++++- Cargo.toml | 1 + banking_bench/Cargo.toml | 1 + banking_bench/src/main.rs | 4 +- core/Cargo.toml | 10 +- core/benches/banking_stage.rs | 7 +- core/benches/blocktree.rs | 8 +- core/benches/poh.rs | 2 +- core/benches/poh_verify.rs | 3 +- core/benches/retransmit_stage.rs | 2 +- core/benches/shredder.rs | 7 +- core/src/banking_stage.rs | 13 +- core/src/blockstream.rs | 3 +- core/src/blockstream_service.rs | 6 +- core/src/blocktree_processor.rs | 14 +-- core/src/broadcast_stage.rs | 8 +- .../broadcast_fake_blobs_run.rs | 4 +- core/src/broadcast_stage/broadcast_utils.rs | 2 +- .../fail_entry_verification_broadcast_run.rs | 2 +- .../broadcast_stage/standard_broadcast_run.rs | 10 +- core/src/chacha.rs | 8 +- core/src/chacha_cuda.rs | 8 +- core/src/cluster_info.rs | 13 +- core/src/cluster_info_repair_listener.rs | 9 +- core/src/cuda_runtime.rs | 2 +- core/src/genesis_utils.rs | 15 +-- core/src/gossip_service.rs | 2 +- core/src/ledger_cleanup_service.rs | 6 +- core/src/lib.rs | 18 +-- core/src/packet.rs | 2 +- core/src/poh_recorder.rs | 10 +- core/src/poh_service.rs | 4 +- core/src/repair_service.rs | 19 ++- core/src/replay_stage.rs | 14 +-- core/src/replicator.rs | 6 +- core/src/result.rs | 2 +- core/src/retransmit_stage.rs | 10 +- .../{blocktree => }/rooted_slot_iterator.rs | 9 +- core/src/rpc.rs | 2 +- core/src/sigverify.rs | 2 +- core/src/sigverify_stage.rs | 2 +- core/src/storage_stage.rs | 13 +- core/src/tpu.rs | 2 +- core/src/tvu.rs | 6 +- core/src/validator.rs | 10 +- core/src/window_service.rs | 16 +-- genesis/Cargo.toml | 2 +- genesis/src/main.rs | 4 +- ledger-tool/Cargo.toml | 1 + ledger-tool/src/main.rs | 17 ++- ledger-tool/tests/basic.rs | 4 +- ledger/Cargo.toml | 49 ++++++++ {core => ledger}/src/blocktree.rs | 114 ++++++++---------- {core => ledger}/src/blocktree/db.rs | 1 + {core => ledger}/src/blocktree/meta.rs | 1 + {core => ledger}/src/entry.rs | 5 +- {core => ledger}/src/erasure.rs | 2 + ledger/src/genesis_utils.rs | 14 +++ {core => ledger}/src/leader_schedule.rs | 2 +- {core => ledger}/src/leader_schedule_cache.rs | 3 +- {core => ledger}/src/leader_schedule_utils.rs | 1 - ledger/src/lib.rs | 12 ++ {core => ledger}/src/perf_libs.rs | 2 + {core => ledger}/src/poh.rs | 2 + {core => ledger}/src/shred.rs | 5 +- {core => ledger}/src/staking_utils.rs | 0 local_cluster/Cargo.toml | 1 + local_cluster/src/cluster_tests.rs | 10 +- local_cluster/src/lib.rs | 2 +- local_cluster/src/local_cluster.rs | 2 +- local_cluster/src/tests/local_cluster.rs | 3 +- local_cluster/src/tests/replicator.rs | 2 +- validator/Cargo.toml | 1 + validator/src/lib.rs | 2 +- 74 files changed, 350 insertions(+), 273 deletions(-) rename core/src/{blocktree => }/rooted_slot_iterator.rs (94%) create mode 100644 ledger/Cargo.toml rename {core => ledger}/src/blocktree.rs (98%) rename {core => ledger}/src/blocktree/db.rs (99%) rename {core => ledger}/src/blocktree/meta.rs (99%) rename {core => ledger}/src/entry.rs (99%) rename {core => ledger}/src/erasure.rs (99%) create mode 100644 ledger/src/genesis_utils.rs rename {core => ledger}/src/leader_schedule.rs (98%) rename {core => ledger}/src/leader_schedule_cache.rs (99%) rename {core => ledger}/src/leader_schedule_utils.rs (99%) create mode 100644 ledger/src/lib.rs rename {core => ledger}/src/perf_libs.rs (99%) rename {core => ledger}/src/poh.rs (99%) rename {core => ledger}/src/shred.rs (99%) rename {core => ledger}/src/staking_utils.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index db8441f1e8..b80bb50681 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2993,6 +2993,7 @@ dependencies = [ "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", "rayon 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "solana-core 0.20.0", + "solana-ledger 0.20.0", "solana-logger 0.20.0", "solana-measure 0.20.0", "solana-runtime 0.20.0", @@ -3265,8 +3266,6 @@ dependencies = [ "crc 1.8.1 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam-channel 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "dir-diff 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "dlopen 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", - "dlopen_derive 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "fs_extra 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "hex-literal 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "indexmap 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3287,7 +3286,6 @@ dependencies = [ "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "rayon 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "reqwest 0.9.22 (registry+https://github.com/rust-lang/crates.io-index)", - "rocksdb 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.101 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.101 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3299,6 +3297,7 @@ dependencies = [ "solana-client 0.20.0", "solana-drone 0.20.0", "solana-ed25519-dalek 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "solana-ledger 0.20.0", "solana-logger 0.20.0", "solana-measure 0.20.0", "solana-merkle-tree 0.20.0", @@ -3428,8 +3427,8 @@ dependencies = [ "serde_derive 1.0.101 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)", "serde_yaml 0.8.11 (registry+https://github.com/rust-lang/crates.io-index)", - "solana-core 0.20.0", "solana-genesis-programs 0.20.0", + "solana-ledger 0.20.0", "solana-sdk 0.20.0", "solana-stake-api 0.20.0", "solana-storage-api 0.20.0", @@ -3519,6 +3518,38 @@ dependencies = [ "tiny-bip39 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "solana-ledger" +version = "0.20.0" +dependencies = [ + "bincode 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "chrono 0.4.9 (registry+https://github.com/rust-lang/crates.io-index)", + "dlopen 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "dlopen_derive 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rayon 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rocksdb 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.101 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.101 (registry+https://github.com/rust-lang/crates.io-index)", + "solana-budget-api 0.20.0", + "solana-logger 0.20.0", + "solana-merkle-tree 0.20.0", + "solana-metrics 0.20.0", + "solana-rayon-threadlimit 0.20.0", + "solana-reed-solomon-erasure 4.0.1-3 (registry+https://github.com/rust-lang/crates.io-index)", + "solana-runtime 0.20.0", + "solana-sdk 0.20.0", + "solana-stake-api 0.20.0", + "solana-vote-api 0.20.0", + "sys-info 0.5.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "solana-ledger-tool" version = "0.20.0" @@ -3531,6 +3562,7 @@ dependencies = [ "serde_json 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)", "serde_yaml 0.8.11 (registry+https://github.com/rust-lang/crates.io-index)", "solana-core 0.20.0", + "solana-ledger 0.20.0", "solana-logger 0.20.0", "solana-runtime 0.20.0", "solana-sdk 0.20.0", @@ -3565,6 +3597,7 @@ dependencies = [ "solana-drone 0.20.0", "solana-exchange-api 0.20.0", "solana-exchange-program 0.20.0", + "solana-ledger 0.20.0", "solana-logger 0.20.0", "solana-move-loader-api 0.20.0", "solana-move-loader-program 0.20.0", @@ -3886,6 +3919,7 @@ dependencies = [ "solana-client 0.20.0", "solana-core 0.20.0", "solana-drone 0.20.0", + "solana-ledger 0.20.0", "solana-logger 0.20.0", "solana-metrics 0.20.0", "solana-netutil 0.20.0", diff --git a/Cargo.toml b/Cargo.toml index 574ac3de35..95e54efd96 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ members = [ "gossip", "install", "keygen", + "ledger", "ledger-tool", "local_cluster", "logger", diff --git a/banking_bench/Cargo.toml b/banking_bench/Cargo.toml index eaa65d9cd6..4d80bb40fb 100644 --- a/banking_bench/Cargo.toml +++ b/banking_bench/Cargo.toml @@ -11,6 +11,7 @@ homepage = "https://solana.com/" log = "0.4.6" rayon = "1.2.0" solana-core = { path = "../core", version = "0.20.0" } +solana-ledger = { path = "../ledger", version = "0.20.0" } solana-logger = { path = "../logger", version = "0.20.0" } solana-runtime = { path = "../runtime", version = "0.20.0" } solana-measure = { path = "../measure", version = "0.20.0" } diff --git a/banking_bench/src/main.rs b/banking_bench/src/main.rs index c53d5f2124..aa5a483e2a 100644 --- a/banking_bench/src/main.rs +++ b/banking_bench/src/main.rs @@ -1,5 +1,5 @@ #[macro_use] -extern crate solana_core; +extern crate solana_ledger; extern crate crossbeam_channel; use crossbeam_channel::unbounded; @@ -8,7 +8,6 @@ use rand::{thread_rng, Rng}; use rayon::prelude::*; use solana_core::bank_forks::BankForks; use solana_core::banking_stage::{create_test_recorder, BankingStage}; -use solana_core::blocktree::{get_tmp_ledger_path, Blocktree}; use solana_core::cluster_info::ClusterInfo; use solana_core::cluster_info::Node; use solana_core::genesis_utils::{create_genesis_block, GenesisBlockInfo}; @@ -16,6 +15,7 @@ use solana_core::packet::to_packets_chunked; use solana_core::poh_recorder::PohRecorder; use solana_core::poh_recorder::WorkingBankEntry; use solana_core::service::Service; +use solana_ledger::blocktree::{get_tmp_ledger_path, Blocktree}; use solana_measure::measure::Measure; use solana_runtime::bank::Bank; use solana_sdk::hash::Hash; diff --git a/core/Cargo.toml b/core/Cargo.toml index 1301176a5f..39eee5f4eb 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -27,8 +27,6 @@ core_affinity = "0.5.9" crc = { version = "1.8.1", optional = true } crossbeam-channel = "0.3" dir-diff = "0.3.2" -dlopen = "0.1.8" -dlopen_derive = "0.1.4" fs_extra = "1.1.0" indexmap = "1.1" itertools = "0.8.0" @@ -55,6 +53,7 @@ solana-chacha-sys = { path = "../chacha-sys", version = "0.20.0" } solana-client = { path = "../client", version = "0.20.0" } solana-drone = { path = "../drone", version = "0.20.0" } solana-ed25519-dalek = "0.2.0" +solana-ledger = { path = "../ledger", version = "0.20.0" } solana-logger = { path = "../logger", version = "0.20.0" } solana-merkle-tree = { path = "../merkle-tree", version = "0.20.0" } solana-metrics = { path = "../metrics", version = "0.20.0" } @@ -79,13 +78,6 @@ untrusted = "0.7.0" solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "0.20.0" } reed-solomon-erasure = { package = "solana-reed-solomon-erasure", version = "4.0.1-3", features = ["simd-accel"] } -[dependencies.rocksdb] -# Avoid the vendored bzip2 within rocksdb-sys that can cause linker conflicts -# when also using the bzip2 crate -version = "0.11.0" -default-features = false -features = ["lz4"] - [dev-dependencies] hex-literal = "0.2.1" matches = "0.1.6" diff --git a/core/benches/banking_stage.rs b/core/benches/banking_stage.rs index b57ae0bb0a..15a928f771 100644 --- a/core/benches/banking_stage.rs +++ b/core/benches/banking_stage.rs @@ -2,24 +2,23 @@ extern crate test; #[macro_use] -extern crate solana_core; +extern crate solana_ledger; use crossbeam_channel::unbounded; use log::*; use rand::{thread_rng, Rng}; use rayon::prelude::*; use solana_core::banking_stage::{create_test_recorder, BankingStage}; -use solana_core::blocktree::{get_tmp_ledger_path, Blocktree}; use solana_core::blocktree_processor::process_entries; use solana_core::cluster_info::ClusterInfo; use solana_core::cluster_info::Node; -use solana_core::entry::next_hash; -use solana_core::entry::Entry; use solana_core::genesis_utils::{create_genesis_block, GenesisBlockInfo}; use solana_core::packet::to_packets_chunked; use solana_core::poh_recorder::WorkingBankEntry; use solana_core::service::Service; use solana_core::test_tx::test_tx; +use solana_ledger::blocktree::{get_tmp_ledger_path, Blocktree}; +use solana_ledger::entry::{next_hash, Entry}; use solana_runtime::bank::Bank; use solana_sdk::genesis_block::GenesisBlock; use solana_sdk::hash::Hash; diff --git a/core/benches/blocktree.rs b/core/benches/blocktree.rs index c2615af704..287f85380e 100644 --- a/core/benches/blocktree.rs +++ b/core/benches/blocktree.rs @@ -4,13 +4,11 @@ use rand; extern crate test; #[macro_use] -extern crate solana_core; +extern crate solana_ledger; use rand::Rng; -use solana_core::{ - blocktree::{entries_to_test_shreds, get_tmp_ledger_path, Blocktree}, - entry::{create_ticks, Entry}, -}; +use solana_ledger::blocktree::{entries_to_test_shreds, get_tmp_ledger_path, Blocktree}; +use solana_ledger::entry::{create_ticks, Entry}; use solana_sdk::hash::Hash; use std::path::Path; use test::Bencher; diff --git a/core/benches/poh.rs b/core/benches/poh.rs index ff2ff4dffd..726a28a970 100644 --- a/core/benches/poh.rs +++ b/core/benches/poh.rs @@ -3,8 +3,8 @@ #![feature(test)] extern crate test; -use solana_core::poh::Poh; use solana_core::poh_service::NUM_HASHES_PER_BATCH; +use solana_ledger::poh::Poh; use solana_sdk::hash::Hash; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; diff --git a/core/benches/poh_verify.rs b/core/benches/poh_verify.rs index b13c503fb6..b611999273 100644 --- a/core/benches/poh_verify.rs +++ b/core/benches/poh_verify.rs @@ -1,8 +1,7 @@ #![feature(test)] extern crate test; -use solana_core::entry::EntrySlice; -use solana_core::entry::{next_entry_mut, Entry}; +use solana_ledger::entry::{next_entry_mut, Entry, EntrySlice}; use solana_sdk::hash::{hash, Hash}; use solana_sdk::signature::{Keypair, KeypairUtil}; use solana_sdk::system_transaction; diff --git a/core/benches/retransmit_stage.rs b/core/benches/retransmit_stage.rs index a14e967aec..c4f68bda06 100644 --- a/core/benches/retransmit_stage.rs +++ b/core/benches/retransmit_stage.rs @@ -8,10 +8,10 @@ use solana_core::bank_forks::BankForks; use solana_core::cluster_info::{ClusterInfo, Node}; use solana_core::contact_info::ContactInfo; use solana_core::genesis_utils::{create_genesis_block, GenesisBlockInfo}; -use solana_core::leader_schedule_cache::LeaderScheduleCache; use solana_core::packet::to_packets_chunked; use solana_core::retransmit_stage::retransmitter; use solana_core::test_tx::test_tx; +use solana_ledger::leader_schedule_cache::LeaderScheduleCache; use solana_measure::measure::Measure; use solana_runtime::bank::Bank; use solana_sdk::pubkey::Pubkey; diff --git a/core/benches/shredder.rs b/core/benches/shredder.rs index 7a496397f6..696d991acf 100644 --- a/core/benches/shredder.rs +++ b/core/benches/shredder.rs @@ -2,13 +2,12 @@ extern crate test; -use solana_core::entry::create_ticks; -use solana_core::entry::Entry; -use solana_core::shred::{ +use solana_core::test_tx; +use solana_ledger::entry::{create_ticks, Entry}; +use solana_ledger::shred::{ max_entries_per_n_shred, max_ticks_per_n_shreds, Shred, Shredder, RECOMMENDED_FEC_RATE, SIZE_OF_SHRED_HEADER, }; -use solana_core::test_tx; use solana_sdk::hash::Hash; use solana_sdk::packet::PACKET_DATA_SIZE; use solana_sdk::signature::{Keypair, KeypairUtil}; diff --git a/core/src/banking_stage.rs b/core/src/banking_stage.rs index 85059176fb..721ef63106 100644 --- a/core/src/banking_stage.rs +++ b/core/src/banking_stage.rs @@ -2,13 +2,9 @@ //! to contruct a software pipeline. The stage uses all available CPU cores and //! can do its processing in parallel with signature verification on the GPU. use crate::{ - blocktree::Blocktree, cluster_info::ClusterInfo, - entry::hash_transactions, - leader_schedule_cache::LeaderScheduleCache, packet::PACKETS_PER_BATCH, packet::{Packet, Packets}, - perf_libs, poh_recorder::{PohRecorder, PohRecorderError, WorkingBankEntry}, poh_service::PohService, result::{Error, Result}, @@ -18,6 +14,10 @@ use crate::{ use bincode::deserialize; use crossbeam_channel::{Receiver as CrossbeamReceiver, RecvTimeoutError}; use itertools::Itertools; +use solana_ledger::{ + blocktree::Blocktree, entry::hash_transactions, leader_schedule_cache::LeaderScheduleCache, + perf_libs, +}; use solana_measure::measure::Measure; use solana_metrics::{inc_new_counter_debug, inc_new_counter_info, inc_new_counter_warn}; use solana_runtime::{accounts_db::ErrorCounters, bank::Bank, transaction_batch::TransactionBatch}; @@ -967,15 +967,14 @@ pub fn create_test_recorder( #[cfg(test)] mod tests { use super::*; - use crate::blocktree::get_tmp_ledger_path; use crate::cluster_info::Node; - use crate::entry::{Entry, EntrySlice}; use crate::genesis_utils::{create_genesis_block, GenesisBlockInfo}; use crate::packet::to_packets; use crate::poh_recorder::WorkingBank; - use crate::{get_tmp_ledger_path, tmp_ledger_name}; use crossbeam_channel::unbounded; use itertools::Itertools; + use solana_ledger::blocktree::get_tmp_ledger_path; + use solana_ledger::entry::{Entry, EntrySlice}; use solana_sdk::instruction::InstructionError; use solana_sdk::signature::{Keypair, KeypairUtil}; use solana_sdk::system_transaction; diff --git a/core/src/blockstream.rs b/core/src/blockstream.rs index 4c0156b5cb..1159eaae22 100644 --- a/core/src/blockstream.rs +++ b/core/src/blockstream.rs @@ -2,11 +2,11 @@ //! local unix socket, to provide client services such as a block explorer with //! real-time access to entries. -use crate::entry::Entry; use crate::result::Result; use bincode::serialize; use chrono::{SecondsFormat, Utc}; use serde_json::json; +use solana_ledger::entry::Entry; use solana_sdk::hash::Hash; use solana_sdk::pubkey::Pubkey; use std::cell::RefCell; @@ -178,7 +178,6 @@ fn serialize_transactions(entry: &Entry) -> Vec> { #[cfg(test)] mod test { use super::*; - use crate::entry::Entry; use chrono::{DateTime, FixedOffset}; use serde_json::Value; use solana_sdk::hash::Hash; diff --git a/core/src/blockstream_service.rs b/core/src/blockstream_service.rs index 85c874081d..cbe931ea63 100644 --- a/core/src/blockstream_service.rs +++ b/core/src/blockstream_service.rs @@ -7,9 +7,9 @@ use crate::blockstream::BlockstreamEvents; use crate::blockstream::MockBlockstream as Blockstream; #[cfg(not(test))] use crate::blockstream::SocketBlockstream as Blockstream; -use crate::blocktree::Blocktree; use crate::result::{Error, Result}; use crate::service::Service; +use solana_ledger::blocktree::Blocktree; use solana_sdk::pubkey::Pubkey; use std::path::Path; use std::sync::atomic::{AtomicBool, Ordering}; @@ -101,12 +101,12 @@ impl Service for BlockstreamService { #[cfg(test)] mod test { use super::*; - use crate::blocktree::create_new_tmp_ledger; - use crate::entry::{create_ticks, Entry}; use crate::genesis_utils::{create_genesis_block, GenesisBlockInfo}; use bincode::{deserialize, serialize}; use chrono::{DateTime, FixedOffset}; use serde_json::Value; + use solana_ledger::blocktree::create_new_tmp_ledger; + use solana_ledger::entry::{create_ticks, Entry}; use solana_sdk::hash::Hash; use solana_sdk::signature::{Keypair, KeypairUtil}; use solana_sdk::system_transaction; diff --git a/core/src/blocktree_processor.rs b/core/src/blocktree_processor.rs index 29f9b93cf7..ba976be313 100644 --- a/core/src/blocktree_processor.rs +++ b/core/src/blocktree_processor.rs @@ -1,11 +1,11 @@ use crate::bank_forks::BankForks; -use crate::blocktree::{Blocktree, SlotMeta}; -use crate::entry::{Entry, EntrySlice}; -use crate::leader_schedule_cache::LeaderScheduleCache; use rand::seq::SliceRandom; use rand::thread_rng; use rayon::prelude::*; use rayon::ThreadPool; +use solana_ledger::blocktree::{Blocktree, SlotMeta}; +use solana_ledger::entry::{Entry, EntrySlice}; +use solana_ledger::leader_schedule_cache::LeaderScheduleCache; use solana_metrics::{datapoint, datapoint_error, inc_new_counter_debug}; use solana_runtime::bank::Bank; use solana_runtime::transaction_batch::TransactionBatch; @@ -450,12 +450,12 @@ fn process_pending_slots( #[cfg(test)] pub mod tests { use super::*; - use crate::{ - blocktree::create_new_tmp_ledger, - entry::{create_ticks, next_entry, next_entry_mut, Entry}, - genesis_utils::{create_genesis_block, create_genesis_block_with_leader, GenesisBlockInfo}, + use crate::genesis_utils::{ + create_genesis_block, create_genesis_block_with_leader, GenesisBlockInfo, }; use rand::{thread_rng, Rng}; + use solana_ledger::blocktree::create_new_tmp_ledger; + use solana_ledger::entry::{create_ticks, next_entry, next_entry_mut, Entry}; use solana_sdk::{ epoch_schedule::EpochSchedule, hash::Hash, diff --git a/core/src/broadcast_stage.rs b/core/src/broadcast_stage.rs index 94fc9e8044..6fa1b9fb39 100644 --- a/core/src/broadcast_stage.rs +++ b/core/src/broadcast_stage.rs @@ -2,12 +2,12 @@ use self::broadcast_fake_blobs_run::BroadcastFakeBlobsRun; use self::fail_entry_verification_broadcast_run::FailEntryVerificationBroadcastRun; use self::standard_broadcast_run::StandardBroadcastRun; -use crate::blocktree::Blocktree; use crate::cluster_info::{ClusterInfo, ClusterInfoError}; use crate::poh_recorder::WorkingBankEntry; use crate::result::{Error, Result}; use crate::service::Service; -use crate::staking_utils; +use solana_ledger::blocktree::Blocktree; +use solana_ledger::staking_utils; use solana_metrics::{inc_new_counter_error, inc_new_counter_info}; use std::net::UdpSocket; use std::sync::atomic::{AtomicBool, Ordering}; @@ -191,11 +191,11 @@ impl Service for BroadcastStage { #[cfg(test)] mod test { use super::*; - use crate::blocktree::{get_tmp_ledger_path, Blocktree}; use crate::cluster_info::{ClusterInfo, Node}; - use crate::entry::create_ticks; use crate::genesis_utils::{create_genesis_block, GenesisBlockInfo}; use crate::service::Service; + use solana_ledger::blocktree::{get_tmp_ledger_path, Blocktree}; + use solana_ledger::entry::create_ticks; use solana_runtime::bank::Bank; use solana_sdk::hash::Hash; use solana_sdk::pubkey::Pubkey; diff --git a/core/src/broadcast_stage/broadcast_fake_blobs_run.rs b/core/src/broadcast_stage/broadcast_fake_blobs_run.rs index 452ac20299..5c44351351 100644 --- a/core/src/broadcast_stage/broadcast_fake_blobs_run.rs +++ b/core/src/broadcast_stage/broadcast_fake_blobs_run.rs @@ -1,6 +1,6 @@ use super::*; -use crate::entry::Entry; -use crate::shred::{Shredder, RECOMMENDED_FEC_RATE}; +use solana_ledger::entry::Entry; +use solana_ledger::shred::{Shredder, RECOMMENDED_FEC_RATE}; use solana_sdk::hash::Hash; pub(super) struct BroadcastFakeBlobsRun { diff --git a/core/src/broadcast_stage/broadcast_utils.rs b/core/src/broadcast_stage/broadcast_utils.rs index 9e7ee0ca62..2886014752 100644 --- a/core/src/broadcast_stage/broadcast_utils.rs +++ b/core/src/broadcast_stage/broadcast_utils.rs @@ -1,6 +1,6 @@ -use crate::entry::Entry; use crate::poh_recorder::WorkingBankEntry; use crate::result::Result; +use solana_ledger::entry::Entry; use solana_runtime::bank::Bank; use std::sync::mpsc::Receiver; use std::sync::Arc; diff --git a/core/src/broadcast_stage/fail_entry_verification_broadcast_run.rs b/core/src/broadcast_stage/fail_entry_verification_broadcast_run.rs index 2d48f41a01..990592f86a 100644 --- a/core/src/broadcast_stage/fail_entry_verification_broadcast_run.rs +++ b/core/src/broadcast_stage/fail_entry_verification_broadcast_run.rs @@ -1,5 +1,5 @@ use super::*; -use crate::shred::{Shredder, RECOMMENDED_FEC_RATE}; +use solana_ledger::shred::{Shredder, RECOMMENDED_FEC_RATE}; use solana_sdk::hash::Hash; pub(super) struct FailEntryVerificationBroadcastRun {} diff --git a/core/src/broadcast_stage/standard_broadcast_run.rs b/core/src/broadcast_stage/standard_broadcast_run.rs index 63b7b01b43..6b0718e687 100644 --- a/core/src/broadcast_stage/standard_broadcast_run.rs +++ b/core/src/broadcast_stage/standard_broadcast_run.rs @@ -1,8 +1,8 @@ use super::broadcast_utils::{self, ReceiveResults}; use super::*; use crate::broadcast_stage::broadcast_utils::UnfinishedSlotInfo; -use crate::entry::Entry; -use crate::shred::{Shred, Shredder, RECOMMENDED_FEC_RATE}; +use solana_ledger::entry::Entry; +use solana_ledger::shred::{Shred, Shredder, RECOMMENDED_FEC_RATE}; use solana_sdk::signature::Keypair; use solana_sdk::timing::duration_as_us; use std::time::Duration; @@ -294,11 +294,11 @@ impl BroadcastRun for StandardBroadcastRun { #[cfg(test)] mod test { use super::*; - use crate::blocktree::{get_tmp_ledger_path, Blocktree}; use crate::cluster_info::{ClusterInfo, Node}; - use crate::entry::create_ticks; use crate::genesis_utils::create_genesis_block; - use crate::shred::max_ticks_per_n_shreds; + use solana_ledger::blocktree::{get_tmp_ledger_path, Blocktree}; + use solana_ledger::entry::create_ticks; + use solana_ledger::shred::max_ticks_per_n_shreds; use solana_runtime::bank::Bank; use solana_sdk::genesis_block::GenesisBlock; use solana_sdk::signature::{Keypair, KeypairUtil}; diff --git a/core/src/chacha.rs b/core/src/chacha.rs index c3147a7485..ad4d5911e8 100644 --- a/core/src/chacha.rs +++ b/core/src/chacha.rs @@ -1,4 +1,4 @@ -use crate::blocktree::Blocktree; +use solana_ledger::blocktree::Blocktree; use std::fs::File; use std::io; use std::io::{BufWriter, Write}; @@ -72,11 +72,11 @@ pub fn chacha_cbc_encrypt_ledger( #[cfg(test)] mod tests { - use crate::blocktree::get_tmp_ledger_path; - use crate::blocktree::Blocktree; use crate::chacha::chacha_cbc_encrypt_ledger; - use crate::entry::Entry; use crate::gen_keys::GenKeys; + use solana_ledger::blocktree::get_tmp_ledger_path; + use solana_ledger::blocktree::Blocktree; + use solana_ledger::entry::Entry; use solana_sdk::hash::{hash, Hash, Hasher}; use solana_sdk::signature::KeypairUtil; use solana_sdk::system_transaction; diff --git a/core/src/chacha_cuda.rs b/core/src/chacha_cuda.rs index 50c7a348ba..d5c1697088 100644 --- a/core/src/chacha_cuda.rs +++ b/core/src/chacha_cuda.rs @@ -1,8 +1,8 @@ // Module used by validators to approve storage mining proofs in parallel using the GPU -use crate::blocktree::Blocktree; use crate::chacha::{CHACHA_BLOCK_SIZE, CHACHA_KEY_SIZE}; -use crate::perf_libs; +use solana_ledger::blocktree::Blocktree; +use solana_ledger::perf_libs; use solana_sdk::hash::Hash; use std::io; use std::mem::size_of; @@ -113,10 +113,10 @@ pub fn chacha_cbc_encrypt_file_many_keys( #[cfg(test)] mod tests { use super::*; - use crate::blocktree::get_tmp_ledger_path; use crate::chacha::chacha_cbc_encrypt_ledger; - use crate::entry::create_ticks; use crate::replicator::sample_file; + use solana_ledger::blocktree::get_tmp_ledger_path; + use solana_ledger::entry::create_ticks; use solana_sdk::clock::DEFAULT_SLOTS_PER_SEGMENT; use solana_sdk::signature::{Keypair, KeypairUtil}; use std::fs::{remove_dir_all, remove_file}; diff --git a/core/src/cluster_info.rs b/core/src/cluster_info.rs index 417e3e66af..a4d4e3a55a 100644 --- a/core/src/cluster_info.rs +++ b/core/src/cluster_info.rs @@ -13,7 +13,6 @@ //! //! Bank needs to provide an interface for us to query the stake weight use crate::bank_forks::BankForks; -use crate::blocktree::Blocktree; use crate::contact_info::ContactInfo; use crate::crds_gossip::CrdsGossip; use crate::crds_gossip_error::CrdsGossipError; @@ -23,7 +22,6 @@ use crate::packet::{to_shared_blob, Blob, Packet, SharedBlob}; use crate::repair_service::RepairType; use crate::result::{Error, Result}; use crate::sendmmsg::{multicast, send_mmsg}; -use crate::staking_utils; use crate::streamer::{BlobReceiver, BlobSender}; use crate::weighted_shuffle::{weighted_best, weighted_shuffle}; use bincode::{deserialize, serialize, serialized_size}; @@ -32,6 +30,8 @@ use itertools::Itertools; use rand::SeedableRng; use rand::{thread_rng, Rng}; use rand_chacha::ChaChaRng; +use solana_ledger::blocktree::Blocktree; +use solana_ledger::staking_utils; use solana_metrics::{datapoint_debug, inc_new_counter_debug, inc_new_counter_error}; use solana_netutil::{ bind_common, bind_common_in_range, bind_in_range, find_available_port_in_range, @@ -1779,17 +1779,16 @@ fn report_time_spent(label: &str, time: &Duration, extra: &str) { #[cfg(test)] mod tests { use super::*; - use crate::blocktree::get_tmp_ledger_path; - use crate::blocktree::tests::make_many_slot_entries; - use crate::blocktree::Blocktree; use crate::blocktree_processor::tests::fill_blocktree_slot_with_ticks; use crate::crds_value::CrdsValueLabel; use crate::repair_service::RepairType; use crate::result::Error; - use crate::shred::max_ticks_per_n_shreds; - use crate::shred::{Shred, ShredHeader}; use crate::test_tx::test_tx; use rayon::prelude::*; + use solana_ledger::blocktree::get_tmp_ledger_path; + use solana_ledger::blocktree::make_many_slot_entries; + use solana_ledger::blocktree::Blocktree; + use solana_ledger::shred::{max_ticks_per_n_shreds, Shred, ShredHeader}; use solana_sdk::hash::Hash; use solana_sdk::signature::{Keypair, KeypairUtil}; use std::collections::HashSet; diff --git a/core/src/cluster_info_repair_listener.rs b/core/src/cluster_info_repair_listener.rs index 8a6761325b..75d55092bf 100644 --- a/core/src/cluster_info_repair_listener.rs +++ b/core/src/cluster_info_repair_listener.rs @@ -1,12 +1,13 @@ -use crate::blocktree::Blocktree; use crate::cluster_info::ClusterInfo; use crate::crds_value::EpochSlots; use crate::result::Result; +use crate::rooted_slot_iterator::RootedSlotIterator; use crate::service::Service; use byteorder::{ByteOrder, LittleEndian}; use rand::seq::SliceRandom; use rand::SeedableRng; use rand_chacha::ChaChaRng; +use solana_ledger::blocktree::Blocktree; use solana_metrics::datapoint; use solana_sdk::{epoch_schedule::EpochSchedule, pubkey::Pubkey}; use std::{ @@ -264,7 +265,7 @@ impl ClusterInfoRepairListener { num_slots_to_repair: usize, epoch_schedule: &EpochSchedule, ) -> Result<()> { - let slot_iter = blocktree.rooted_slot_iterator(repairee_epoch_slots.root); + let slot_iter = RootedSlotIterator::new(repairee_epoch_slots.root, &blocktree); if slot_iter.is_err() { info!( "Root for repairee is on different fork. My root: {}, repairee_root: {} repairee_pubkey: {:?}", @@ -479,11 +480,11 @@ impl Service for ClusterInfoRepairListener { #[cfg(test)] mod tests { use super::*; - use crate::blocktree::get_tmp_ledger_path; - use crate::blocktree::tests::make_many_slot_entries; use crate::cluster_info::Node; use crate::packet::{Blob, SharedBlob}; use crate::streamer; + use solana_ledger::blocktree::get_tmp_ledger_path; + use solana_ledger::blocktree::make_many_slot_entries; use std::collections::BTreeSet; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::mpsc::channel; diff --git a/core/src/cuda_runtime.rs b/core/src/cuda_runtime.rs index 27c2e38bbc..16ed4453a3 100644 --- a/core/src/cuda_runtime.rs +++ b/core/src/cuda_runtime.rs @@ -5,8 +5,8 @@ // copies from host memory to GPU memory unless the memory is page-pinned and // cannot be paged to disk. The cuda driver provides these interfaces to pin and unpin memory. -use crate::perf_libs; use crate::recycler::Reset; +use solana_ledger::perf_libs; use std::ops::{Deref, DerefMut}; #[cfg(feature = "pin_gpu_memory")] diff --git a/core/src/genesis_utils.rs b/core/src/genesis_utils.rs index 6901d263a6..2aae34143b 100644 --- a/core/src/genesis_utils.rs +++ b/core/src/genesis_utils.rs @@ -1,14 +1 @@ -pub use solana_runtime::genesis_utils::{ - create_genesis_block_with_leader, GenesisBlockInfo, BOOTSTRAP_LEADER_LAMPORTS, -}; -use solana_sdk::pubkey::Pubkey; - -// same as genesis_block::create_genesis_block, but with bootstrap_leader staking logic -// for the core crate tests -pub fn create_genesis_block(mint_lamports: u64) -> GenesisBlockInfo { - create_genesis_block_with_leader( - mint_lamports, - &Pubkey::new_rand(), - BOOTSTRAP_LEADER_LAMPORTS, - ) -} +pub use solana_ledger::genesis_utils::*; diff --git a/core/src/gossip_service.rs b/core/src/gossip_service.rs index c0cb25d138..c82f8f956f 100644 --- a/core/src/gossip_service.rs +++ b/core/src/gossip_service.rs @@ -1,13 +1,13 @@ //! The `gossip_service` module implements the network control plane. use crate::bank_forks::BankForks; -use crate::blocktree::Blocktree; use crate::cluster_info::{ClusterInfo, VALIDATOR_PORT_RANGE}; use crate::contact_info::ContactInfo; use crate::service::Service; use crate::streamer; use rand::{thread_rng, Rng}; use solana_client::thin_client::{create_client, ThinClient}; +use solana_ledger::blocktree::Blocktree; use solana_sdk::pubkey::Pubkey; use solana_sdk::signature::{Keypair, KeypairUtil}; use std::net::{IpAddr, SocketAddr, UdpSocket}; diff --git a/core/src/ledger_cleanup_service.rs b/core/src/ledger_cleanup_service.rs index ec8a5afd2f..418762725b 100644 --- a/core/src/ledger_cleanup_service.rs +++ b/core/src/ledger_cleanup_service.rs @@ -1,8 +1,8 @@ //! The `ledger_cleanup_service` drops older ledger data to limit disk space usage -use crate::blocktree::Blocktree; use crate::result::{Error, Result}; use crate::service::Service; +use solana_ledger::blocktree::Blocktree; use solana_sdk::clock::DEFAULT_SLOTS_PER_EPOCH; use solana_sdk::pubkey::Pubkey; use std::string::ToString; @@ -75,8 +75,8 @@ impl Service for LedgerCleanupService { #[cfg(test)] mod tests { use super::*; - use crate::blocktree::get_tmp_ledger_path; - use crate::blocktree::tests::make_many_slot_entries; + use solana_ledger::blocktree::get_tmp_ledger_path; + use solana_ledger::blocktree::make_many_slot_entries; use std::sync::mpsc::channel; #[test] diff --git a/core/src/lib.rs b/core/src/lib.rs index b5be3afc2c..3ee62676a5 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -7,14 +7,11 @@ pub mod bank_forks; pub mod banking_stage; -#[macro_use] -pub mod blocktree; pub mod broadcast_stage; pub mod chacha; pub mod chacha_cuda; pub mod cluster_info_vote_listener; pub mod confidence; -pub mod perf_libs; pub mod recycler; pub mod shred_fetch_stage; #[macro_use] @@ -32,19 +29,13 @@ pub mod crds_gossip_pull; pub mod crds_gossip_push; pub mod crds_value; pub mod cuda_runtime; -pub mod entry; -pub mod erasure; pub mod fetch_stage; pub mod gen_keys; pub mod genesis_utils; pub mod gossip_service; -pub mod leader_schedule; -pub mod leader_schedule_cache; -pub mod leader_schedule_utils; pub mod ledger_cleanup_service; pub mod local_vote_signer_service; pub mod packet; -pub mod poh; pub mod poh_recorder; pub mod poh_service; pub mod recvmmsg; @@ -53,6 +44,7 @@ pub mod replay_stage; pub mod replicator; pub mod result; pub mod retransmit_stage; +pub mod rooted_slot_iterator; pub mod rpc; pub mod rpc_pubsub; pub mod rpc_pubsub_service; @@ -60,12 +52,10 @@ pub mod rpc_service; pub mod rpc_subscriptions; pub mod sendmmsg; pub mod service; -pub mod shred; pub mod sigverify; pub mod sigverify_stage; pub mod snapshot_package; pub mod snapshot_utils; -pub mod staking_utils; pub mod storage_stage; pub mod streamer; pub mod test_tx; @@ -76,9 +66,6 @@ pub(crate) mod version; pub mod weighted_shuffle; pub mod window_service; -#[macro_use] -extern crate dlopen_derive; - #[macro_use] extern crate solana_budget_program; @@ -105,6 +92,9 @@ extern crate solana_metrics; #[macro_use] extern crate matches; +#[macro_use] +extern crate solana_ledger; + extern crate bzip2; extern crate crossbeam_channel; extern crate dir_diff; diff --git a/core/src/packet.rs b/core/src/packet.rs index 8f46fb30ac..1856040e3a 100644 --- a/core/src/packet.rs +++ b/core/src/packet.rs @@ -1,12 +1,12 @@ //! The `packet` module defines data structures and methods to pull data from the network. use crate::cuda_runtime::PinnedVec; -use crate::erasure::ErasureConfig; use crate::recvmmsg::{recv_mmsg, NUM_RCVMMSGS}; use crate::recycler::{Recycler, Reset}; use crate::result::{Error, Result}; use bincode; use byteorder::{ByteOrder, LittleEndian}; use serde::Serialize; +use solana_ledger::erasure::ErasureConfig; use solana_metrics::inc_new_counter_debug; pub use solana_sdk::packet::{Meta, Packet, PACKET_DATA_SIZE}; use solana_sdk::pubkey::Pubkey; diff --git a/core/src/poh_recorder.rs b/core/src/poh_recorder.rs index 72a098cc3c..c90314af87 100644 --- a/core/src/poh_recorder.rs +++ b/core/src/poh_recorder.rs @@ -10,11 +10,11 @@ //! For Entries: //! * recorded entry must be >= WorkingBank::min_tick_height && entry must be < WorkingBank::max_tick_height //! -use crate::blocktree::Blocktree; -use crate::entry::Entry; -use crate::leader_schedule_cache::LeaderScheduleCache; -use crate::poh::Poh; use crate::result::{Error, Result}; +use solana_ledger::blocktree::Blocktree; +use solana_ledger::entry::Entry; +use solana_ledger::leader_schedule_cache::LeaderScheduleCache; +use solana_ledger::poh::Poh; use solana_runtime::bank::Bank; pub use solana_sdk::clock::Slot; use solana_sdk::clock::NUM_CONSECUTIVE_LEADER_SLOTS; @@ -459,9 +459,9 @@ impl PohRecorder { #[cfg(test)] mod tests { use super::*; - use crate::blocktree::{get_tmp_ledger_path, Blocktree}; use crate::genesis_utils::{create_genesis_block, GenesisBlockInfo}; use crate::test_tx::test_tx; + use solana_ledger::blocktree::{get_tmp_ledger_path, Blocktree}; use solana_sdk::clock::DEFAULT_TICKS_PER_SLOT; use solana_sdk::hash::hash; use std::sync::mpsc::sync_channel; diff --git a/core/src/poh_service.rs b/core/src/poh_service.rs index d31aa037e0..948ff13f53 100644 --- a/core/src/poh_service.rs +++ b/core/src/poh_service.rs @@ -109,12 +109,12 @@ impl Service for PohService { #[cfg(test)] mod tests { use super::*; - use crate::blocktree::{get_tmp_ledger_path, Blocktree}; use crate::genesis_utils::{create_genesis_block, GenesisBlockInfo}; - use crate::leader_schedule_cache::LeaderScheduleCache; use crate::poh_recorder::WorkingBank; use crate::result::Result; use crate::test_tx::test_tx; + use solana_ledger::blocktree::{get_tmp_ledger_path, Blocktree}; + use solana_ledger::leader_schedule_cache::LeaderScheduleCache; use solana_runtime::bank::Bank; use solana_sdk::hash::hash; use solana_sdk::pubkey::Pubkey; diff --git a/core/src/repair_service.rs b/core/src/repair_service.rs index c2804996ca..8f173dffe1 100644 --- a/core/src/repair_service.rs +++ b/core/src/repair_service.rs @@ -1,13 +1,10 @@ //! The `repair_service` module implements the tools necessary to generate a thread which //! regularly finds missing blobs in the ledger and sends repair requests for those blobs use crate::{ - bank_forks::BankForks, - blocktree::{Blocktree, CompletedSlotsReceiver, SlotMeta}, - cluster_info::ClusterInfo, - cluster_info_repair_listener::ClusterInfoRepairListener, - result::Result, - service::Service, + bank_forks::BankForks, cluster_info::ClusterInfo, + cluster_info_repair_listener::ClusterInfoRepairListener, result::Result, service::Service, }; +use solana_ledger::blocktree::{Blocktree, CompletedSlotsReceiver, SlotMeta}; use solana_sdk::{epoch_schedule::EpochSchedule, pubkey::Pubkey}; use std::{ collections::BTreeSet, @@ -401,15 +398,15 @@ impl Service for RepairService { #[cfg(test)] mod test { use super::*; - use crate::blocktree::tests::{ - make_chaining_slot_entries, make_many_slot_entries, make_slot_entries, - }; - use crate::blocktree::{get_tmp_ledger_path, Blocktree}; use crate::cluster_info::Node; - use crate::shred::max_ticks_per_n_shreds; use itertools::Itertools; use rand::seq::SliceRandom; use rand::{thread_rng, Rng}; + use solana_ledger::blocktree::{get_tmp_ledger_path, Blocktree}; + use solana_ledger::blocktree::{ + make_chaining_slot_entries, make_many_slot_entries, make_slot_entries, + }; + use solana_ledger::shred::max_ticks_per_n_shreds; use std::sync::mpsc::channel; use std::thread::Builder; diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs index 1eee16a4df..1f7697ceb7 100644 --- a/core/src/replay_stage.rs +++ b/core/src/replay_stage.rs @@ -1,21 +1,21 @@ //! The `replay_stage` replays transactions broadcast by the leader. use crate::bank_forks::BankForks; -use crate::blocktree::{Blocktree, BlocktreeError}; use crate::blocktree_processor; use crate::cluster_info::ClusterInfo; use crate::confidence::{ AggregateConfidenceService, ConfidenceAggregationData, ForkConfidenceCache, }; use crate::consensus::{StakeLockout, Tower}; -use crate::entry::{Entry, EntrySlice}; -use crate::leader_schedule_cache::LeaderScheduleCache; use crate::packet::BlobError; use crate::poh_recorder::PohRecorder; use crate::result::{Error, Result}; use crate::rpc_subscriptions::RpcSubscriptions; use crate::service::Service; use crate::snapshot_package::SnapshotPackageSender; +use solana_ledger::blocktree::{Blocktree, BlocktreeError}; +use solana_ledger::entry::{Entry, EntrySlice}; +use solana_ledger::leader_schedule_cache::LeaderScheduleCache; use solana_metrics::{datapoint_warn, inc_new_counter_info}; use solana_runtime::bank::Bank; use solana_sdk::hash::Hash; @@ -868,13 +868,13 @@ impl Service for ReplayStage { #[cfg(test)] mod test { use super::*; - use crate::blocktree::tests::make_slot_entries; - use crate::blocktree::{entries_to_test_shreds, get_tmp_ledger_path, BlocktreeError}; use crate::confidence::BankConfidence; - use crate::entry; use crate::genesis_utils::{create_genesis_block, create_genesis_block_with_leader}; use crate::replay_stage::ReplayStage; - use crate::shred::{Shred, ShredHeader, DATA_COMPLETE_SHRED, SIZE_OF_SHRED_HEADER}; + use solana_ledger::blocktree::make_slot_entries; + use solana_ledger::blocktree::{entries_to_test_shreds, get_tmp_ledger_path, BlocktreeError}; + use solana_ledger::entry; + use solana_ledger::shred::{Shred, ShredHeader, DATA_COMPLETE_SHRED, SIZE_OF_SHRED_HEADER}; use solana_runtime::genesis_utils::GenesisBlockInfo; use solana_sdk::hash::{hash, Hash}; use solana_sdk::packet::PACKET_DATA_SIZE; diff --git a/core/src/replicator.rs b/core/src/replicator.rs index 5acb656414..b379c73410 100644 --- a/core/src/replicator.rs +++ b/core/src/replicator.rs @@ -1,16 +1,13 @@ -use crate::blocktree::Blocktree; use crate::chacha::{chacha_cbc_encrypt_ledger, CHACHA_BLOCK_SIZE}; use crate::cluster_info::{ClusterInfo, Node, VALIDATOR_PORT_RANGE}; use crate::contact_info::ContactInfo; use crate::gossip_service::GossipService; -use crate::leader_schedule_cache::LeaderScheduleCache; use crate::packet::to_shared_blob; use crate::recycler::Recycler; use crate::repair_service; use crate::repair_service::{RepairService, RepairSlotRange, RepairStrategy}; use crate::result::{Error, Result}; use crate::service::Service; -use crate::shred::Shred; use crate::shred_fetch_stage::ShredFetchStage; use crate::storage_stage::NUM_STORAGE_SAMPLES; use crate::streamer::{receiver, responder, PacketReceiver}; @@ -24,6 +21,9 @@ use solana_client::rpc_client::RpcClient; use solana_client::rpc_request::RpcRequest; use solana_client::thin_client::ThinClient; use solana_ed25519_dalek as ed25519_dalek; +use solana_ledger::blocktree::Blocktree; +use solana_ledger::leader_schedule_cache::LeaderScheduleCache; +use solana_ledger::shred::Shred; use solana_netutil::bind_in_range; use solana_sdk::account_utils::State; use solana_sdk::client::{AsyncClient, SyncClient}; diff --git a/core/src/result.rs b/core/src/result.rs index 6b1615da43..c1a0422259 100644 --- a/core/src/result.rs +++ b/core/src/result.rs @@ -1,11 +1,11 @@ //! The `result` module exposes a Result type that propagates one of many different Error types. -use crate::blocktree; use crate::cluster_info; use crate::packet; use crate::poh_recorder; use bincode; use serde_json; +use solana_ledger::blocktree; use solana_sdk::transaction; use std; use std::any::Any; diff --git a/core/src/retransmit_stage.rs b/core/src/retransmit_stage.rs index f4ad970e36..59c80d959a 100644 --- a/core/src/retransmit_stage.rs +++ b/core/src/retransmit_stage.rs @@ -2,18 +2,20 @@ use crate::{ bank_forks::BankForks, - blocktree::{Blocktree, CompletedSlotsReceiver}, cluster_info::{compute_retransmit_peers, ClusterInfo, DATA_PLANE_FANOUT}, - leader_schedule_cache::LeaderScheduleCache, repair_service::RepairStrategy, result::{Error, Result}, service::Service, - staking_utils, streamer::PacketReceiver, window_service::{should_retransmit_and_persist, WindowService}, }; use rand::SeedableRng; use rand_chacha::ChaChaRng; +use solana_ledger::{ + blocktree::{Blocktree, CompletedSlotsReceiver}, + leader_schedule_cache::LeaderScheduleCache, + staking_utils, +}; use solana_measure::measure::Measure; use solana_metrics::inc_new_counter_error; use solana_sdk::epoch_schedule::EpochSchedule; @@ -260,11 +262,11 @@ impl Service for RetransmitStage { #[cfg(test)] mod tests { use super::*; - use crate::blocktree::create_new_tmp_ledger; use crate::blocktree_processor::{process_blocktree, ProcessOptions}; use crate::contact_info::ContactInfo; use crate::genesis_utils::{create_genesis_block, GenesisBlockInfo}; use crate::packet::{Meta, Packet, Packets}; + use solana_ledger::blocktree::create_new_tmp_ledger; use solana_netutil::find_available_port_in_range; use solana_sdk::pubkey::Pubkey; diff --git a/core/src/blocktree/rooted_slot_iterator.rs b/core/src/rooted_slot_iterator.rs similarity index 94% rename from core/src/blocktree/rooted_slot_iterator.rs rename to core/src/rooted_slot_iterator.rs index d0f52be51b..3f76ed22db 100644 --- a/core/src/blocktree/rooted_slot_iterator.rs +++ b/core/src/rooted_slot_iterator.rs @@ -1,12 +1,12 @@ -use super::*; +use solana_ledger::blocktree::*; pub struct RootedSlotIterator<'a> { next_slots: Vec, - blocktree: &'a super::Blocktree, + blocktree: &'a Blocktree, } impl<'a> RootedSlotIterator<'a> { - pub fn new(start_slot: u64, blocktree: &'a super::Blocktree) -> Result { + pub fn new(start_slot: u64, blocktree: &'a Blocktree) -> Result { if blocktree.is_root(start_slot) { Ok(Self { next_slots: vec![start_slot], @@ -18,7 +18,7 @@ impl<'a> RootedSlotIterator<'a> { } } impl<'a> Iterator for RootedSlotIterator<'a> { - type Item = (u64, super::SlotMeta); + type Item = (u64, SlotMeta); fn next(&mut self) -> Option { // Clone b/c passing the closure to the map below requires exclusive access to @@ -53,6 +53,7 @@ impl<'a> Iterator for RootedSlotIterator<'a> { mod tests { use super::*; use crate::blocktree_processor::tests::fill_blocktree_slot_with_ticks; + use solana_sdk::hash::Hash; #[test] fn test_rooted_slot_iterator() { diff --git a/core/src/rpc.rs b/core/src/rpc.rs index c057944796..7388a1f184 100644 --- a/core/src/rpc.rs +++ b/core/src/rpc.rs @@ -505,7 +505,7 @@ impl RpcSol for RpcSolImpl { fn get_leader_schedule(&self, meta: Self::Metadata) -> Result>> { let bank = meta.request_processor.read().unwrap().bank(); Ok( - crate::leader_schedule_utils::leader_schedule(bank.epoch(), &bank).map( + solana_ledger::leader_schedule_utils::leader_schedule(bank.epoch(), &bank).map( |leader_schedule| { leader_schedule .get_slot_leaders() diff --git a/core/src/sigverify.rs b/core/src/sigverify.rs index 1aa3dbaaa1..11142d19da 100644 --- a/core/src/sigverify.rs +++ b/core/src/sigverify.rs @@ -6,11 +6,11 @@ use crate::cuda_runtime::PinnedVec; use crate::packet::{Packet, Packets}; -use crate::perf_libs; use crate::recycler::Recycler; use crate::result::Result; use bincode::serialized_size; use rayon::ThreadPool; +use solana_ledger::perf_libs; use solana_metrics::inc_new_counter_debug; use solana_sdk::message::MessageHeader; use solana_sdk::pubkey::Pubkey; diff --git a/core/src/sigverify_stage.rs b/core/src/sigverify_stage.rs index 79529108db..a9ee7a6cdb 100644 --- a/core/src/sigverify_stage.rs +++ b/core/src/sigverify_stage.rs @@ -7,7 +7,6 @@ use crate::cuda_runtime::PinnedVec; use crate::packet::Packets; -use crate::perf_libs; use crate::recycler::Recycler; use crate::result::{Error, Result}; use crate::service::Service; @@ -15,6 +14,7 @@ use crate::sigverify; use crate::sigverify::TxOffset; use crate::streamer::{self, PacketReceiver}; use crossbeam_channel::Sender as CrossbeamSender; +use solana_ledger::perf_libs; use solana_measure::measure::Measure; use solana_metrics::{datapoint_debug, inc_new_counter_info}; use solana_sdk::timing; diff --git a/core/src/storage_stage.rs b/core/src/storage_stage.rs index 14b30353fa..b00093d52f 100644 --- a/core/src/storage_stage.rs +++ b/core/src/storage_stage.rs @@ -3,13 +3,13 @@ // to submit its proof for mining to be rewarded. use crate::bank_forks::BankForks; -use crate::blocktree::Blocktree; use crate::chacha_cuda::chacha_cbc_encrypt_file_many_keys; use crate::cluster_info::ClusterInfo; use crate::result::{Error, Result}; use crate::service::Service; use rand::{Rng, SeedableRng}; use rand_chacha::ChaChaRng; +use solana_ledger::blocktree::Blocktree; use solana_runtime::bank::Bank; use solana_runtime::storage_utils::replicator_accounts; use solana_sdk::account::Account; @@ -411,7 +411,7 @@ impl StorageStage { // TODO: cuda required to generate the reference values // but if it is missing, then we need to take care not to // process storage mining results. - if crate::perf_libs::api().is_some() { + if solana_ledger::perf_libs::api().is_some() { // Lock the keys, since this is the IV memory, // it will be updated in-place by the encryption. // Should be overwritten by the proof signatures which replace the @@ -629,13 +629,14 @@ impl Service for StorageStage { #[cfg(test)] mod tests { use super::*; - use crate::blocktree::{create_new_tmp_ledger, Blocktree}; + use crate::blocktree_processor; use crate::cluster_info::ClusterInfo; use crate::contact_info::ContactInfo; use crate::genesis_utils::{create_genesis_block, GenesisBlockInfo}; use crate::service::Service; - use crate::{blocktree_processor, entry}; use rayon::prelude::*; + use solana_ledger::blocktree::{create_new_tmp_ledger, Blocktree}; + use solana_ledger::entry; use solana_runtime::bank::Bank; use solana_sdk::clock::DEFAULT_TICKS_PER_SLOT; use solana_sdk::hash::{Hash, Hasher}; @@ -750,7 +751,7 @@ mod tests { .collect::>(); bank_sender.send(rooted_banks).unwrap(); - if crate::perf_libs::api().is_some() { + if solana_ledger::perf_libs::api().is_some() { for _ in 0..5 { result = storage_state.get_mining_result(&signature); if result != Hash::default() { @@ -766,7 +767,7 @@ mod tests { exit.store(true, Ordering::Relaxed); storage_stage.join().unwrap(); - if crate::perf_libs::api().is_some() { + if solana_ledger::perf_libs::api().is_some() { assert_ne!(result, Hash::default()); } else { assert_eq!(result, Hash::default()); diff --git a/core/src/tpu.rs b/core/src/tpu.rs index 69167c11ef..1f9923df8a 100644 --- a/core/src/tpu.rs +++ b/core/src/tpu.rs @@ -2,7 +2,6 @@ //! multi-stage transaction processing pipeline in software. use crate::banking_stage::BankingStage; -use crate::blocktree::Blocktree; use crate::broadcast_stage::{BroadcastStage, BroadcastStageType}; use crate::cluster_info::ClusterInfo; use crate::cluster_info_vote_listener::ClusterInfoVoteListener; @@ -11,6 +10,7 @@ use crate::poh_recorder::{PohRecorder, WorkingBankEntry}; use crate::service::Service; use crate::sigverify_stage::SigVerifyStage; use crossbeam_channel::unbounded; +use solana_ledger::blocktree::Blocktree; use std::net::UdpSocket; use std::sync::atomic::AtomicBool; use std::sync::mpsc::{channel, Receiver}; diff --git a/core/src/tvu.rs b/core/src/tvu.rs index 245f8fc10c..21fe101b74 100644 --- a/core/src/tvu.rs +++ b/core/src/tvu.rs @@ -14,10 +14,8 @@ use crate::bank_forks::BankForks; use crate::blockstream_service::BlockstreamService; -use crate::blocktree::{Blocktree, CompletedSlotsReceiver}; use crate::cluster_info::ClusterInfo; use crate::confidence::ForkConfidenceCache; -use crate::leader_schedule_cache::LeaderScheduleCache; use crate::ledger_cleanup_service::LedgerCleanupService; use crate::poh_recorder::PohRecorder; use crate::replay_stage::ReplayStage; @@ -27,6 +25,8 @@ use crate::service::Service; use crate::shred_fetch_stage::ShredFetchStage; use crate::snapshot_package::SnapshotPackagerService; use crate::storage_stage::{StorageStage, StorageState}; +use solana_ledger::blocktree::{Blocktree, CompletedSlotsReceiver}; +use solana_ledger::leader_schedule_cache::LeaderScheduleCache; use solana_sdk::pubkey::Pubkey; use solana_sdk::signature::{Keypair, KeypairUtil}; use std::net::UdpSocket; @@ -226,9 +226,9 @@ impl Service for Tvu { pub mod tests { use super::*; use crate::banking_stage::create_test_recorder; - use crate::blocktree::create_new_tmp_ledger; use crate::cluster_info::{ClusterInfo, Node}; use crate::genesis_utils::{create_genesis_block, GenesisBlockInfo}; + use solana_ledger::blocktree::create_new_tmp_ledger; use solana_runtime::bank::Bank; use std::sync::atomic::Ordering; diff --git a/core/src/validator.rs b/core/src/validator.rs index e2ac6d9084..80ae536510 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -1,14 +1,12 @@ //! The `validator` module hosts all the validator microservices. use crate::bank_forks::{BankForks, SnapshotConfig}; -use crate::blocktree::{Blocktree, CompletedSlotsReceiver}; use crate::blocktree_processor::{self, BankForksInfo}; use crate::broadcast_stage::BroadcastStageType; use crate::cluster_info::{ClusterInfo, Node}; use crate::confidence::ForkConfidenceCache; use crate::contact_info::ContactInfo; use crate::gossip_service::{discover_cluster, GossipService}; -use crate::leader_schedule_cache::LeaderScheduleCache; use crate::poh_recorder::PohRecorder; use crate::poh_service::PohService; use crate::rpc::JsonRpcConfig; @@ -21,6 +19,8 @@ use crate::snapshot_utils; use crate::storage_stage::StorageState; use crate::tpu::Tpu; use crate::tvu::{Sockets, Tvu}; +use solana_ledger::blocktree::{Blocktree, CompletedSlotsReceiver}; +use solana_ledger::leader_schedule_cache::LeaderScheduleCache; use solana_metrics::datapoint_info; use solana_sdk::clock::{Slot, DEFAULT_SLOTS_PER_TURN}; use solana_sdk::genesis_block::GenesisBlock; @@ -120,7 +120,7 @@ impl Validator { warn!("vote pubkey: {:?}", vote_account); warn!( "CUDA is {}abled", - if crate::perf_libs::api().is_some() { + if solana_ledger::perf_libs::api().is_some() { "en" } else { "dis" @@ -575,8 +575,8 @@ impl Service for Validator { } pub fn new_validator_for_tests() -> (Validator, ContactInfo, Keypair, PathBuf) { - use crate::blocktree::create_new_tmp_ledger; use crate::genesis_utils::{create_genesis_block_with_leader, GenesisBlockInfo}; + use solana_ledger::blocktree::create_new_tmp_ledger; let node_keypair = Arc::new(Keypair::new()); let node = Node::new_localhost_with_pubkey(&node_keypair.pubkey()); @@ -616,8 +616,8 @@ pub fn new_validator_for_tests() -> (Validator, ContactInfo, Keypair, PathBuf) { #[cfg(test)] mod tests { use super::*; - use crate::blocktree::create_new_tmp_ledger; use crate::genesis_utils::create_genesis_block_with_leader; + use solana_ledger::blocktree::create_new_tmp_ledger; use std::fs::remove_dir_all; #[test] diff --git a/core/src/window_service.rs b/core/src/window_service.rs index ad7aa1d238..f7d8af68a6 100644 --- a/core/src/window_service.rs +++ b/core/src/window_service.rs @@ -1,16 +1,16 @@ //! `window_service` handles the data plane incoming blobs, storing them in //! blocktree and retransmitting where required //! -use crate::blocktree::{self, Blocktree}; use crate::cluster_info::ClusterInfo; -use crate::leader_schedule_cache::LeaderScheduleCache; use crate::repair_service::{RepairService, RepairStrategy}; use crate::result::{Error, Result}; use crate::service::Service; -use crate::shred::Shred; use crate::streamer::{PacketReceiver, PacketSender}; use rayon::iter::{IntoParallelRefMutIterator, ParallelIterator}; use rayon::ThreadPool; +use solana_ledger::blocktree::{self, Blocktree}; +use solana_ledger::leader_schedule_cache::LeaderScheduleCache; +use solana_ledger::shred::Shred; use solana_metrics::{inc_new_counter_debug, inc_new_counter_error}; use solana_rayon_threadlimit::get_thread_count; use solana_runtime::bank::Bank; @@ -267,19 +267,19 @@ impl Service for WindowService { mod test { use super::*; use crate::{ - blocktree::tests::make_many_slot_entries, - blocktree::{get_tmp_ledger_path, Blocktree}, cluster_info::ClusterInfo, contact_info::ContactInfo, - entry::{create_ticks, Entry}, genesis_utils::create_genesis_block_with_leader, packet::{Packet, Packets}, repair_service::RepairSlotRange, service::Service, - shred::Shredder, - shred::SIZE_OF_SHRED_TYPE, }; use rand::{seq::SliceRandom, thread_rng}; + use solana_ledger::{ + blocktree::{get_tmp_ledger_path, make_many_slot_entries, Blocktree}, + entry::{create_ticks, Entry}, + shred::{Shredder, SIZE_OF_SHRED_TYPE}, + }; use solana_sdk::{ epoch_schedule::MINIMUM_SLOTS_PER_EPOCH, hash::Hash, diff --git a/genesis/Cargo.toml b/genesis/Cargo.toml index ebda097c1f..5d32bc2375 100644 --- a/genesis/Cargo.toml +++ b/genesis/Cargo.toml @@ -15,8 +15,8 @@ serde = "1.0.101" serde_derive = "1.0.101" serde_json = "1.0.41" serde_yaml = "0.8.11" -solana-core = { path = "../core", version = "0.20.0" } solana-genesis-programs = { path = "../genesis_programs", version = "0.20.0" } +solana-ledger = { path = "../ledger", version = "0.20.0" } solana-sdk = { path = "../sdk", version = "0.20.0" } solana-stake-api = { path = "../programs/stake_api", version = "0.20.0" } solana-storage-api = { path = "../programs/storage_api", version = "0.20.0" } diff --git a/genesis/src/main.rs b/genesis/src/main.rs index 406a52789c..385948abd8 100644 --- a/genesis/src/main.rs +++ b/genesis/src/main.rs @@ -1,9 +1,9 @@ //! A command-line executable for generating the chain's genesis block. use clap::{crate_description, crate_name, crate_version, value_t_or_exit, App, Arg}; -use solana_core::blocktree::create_new_ledger; -use solana_core::poh::compute_hashes_per_tick; use solana_genesis::Base64Account; +use solana_ledger::blocktree::create_new_ledger; +use solana_ledger::poh::compute_hashes_per_tick; use solana_sdk::{ account::Account, clock, diff --git a/ledger-tool/Cargo.toml b/ledger-tool/Cargo.toml index f7941da4f2..68317858c3 100644 --- a/ledger-tool/Cargo.toml +++ b/ledger-tool/Cargo.toml @@ -16,6 +16,7 @@ serde_derive = "1.0.101" serde_json = "1.0.41" serde_yaml = "0.8.11" solana-core = { path = "../core", version = "0.20.0" } +solana-ledger = { path = "../ledger", version = "0.20.0" } solana-logger = { path = "../logger", version = "0.20.0" } solana-runtime = { path = "../runtime", version = "0.20.0" } solana-sdk = { path = "../sdk", version = "0.20.0" } diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index fbb50420f9..ea9c3b6cc7 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -1,6 +1,7 @@ use clap::{crate_description, crate_name, crate_version, value_t_or_exit, App, Arg, SubCommand}; -use solana_core::blocktree::Blocktree; use solana_core::blocktree_processor::{process_blocktree, ProcessOptions}; +use solana_core::rooted_slot_iterator::RootedSlotIterator; +use solana_ledger::blocktree::Blocktree; use solana_sdk::clock::Slot; use solana_sdk::genesis_block::GenesisBlock; use std::collections::BTreeMap; @@ -36,9 +37,8 @@ fn output_slot(blocktree: &Blocktree, slot: u64, method: &LedgerOutputMethod) { } fn output_ledger(blocktree: Blocktree, starting_slot: u64, method: LedgerOutputMethod) { - let rooted_slot_iterator = blocktree - .rooted_slot_iterator(starting_slot) - .unwrap_or_else(|err| { + let rooted_slot_iterator = + RootedSlotIterator::new(starting_slot, &blocktree).unwrap_or_else(|err| { eprintln!( "Failed to load entries starting from slot {}: {:?}", starting_slot, err @@ -188,9 +188,8 @@ fn main() { let slot_hashes: BTreeMap = serde_yaml::from_reader(prune_file).unwrap(); - let iter = blocktree - .rooted_slot_iterator(0) - .expect("Failed to get rooted slot"); + let iter = + RootedSlotIterator::new(0, &blocktree).expect("Failed to get rooted slot"); let potential_hashes: Vec<_> = iter .filter_map(|(slot, meta)| { @@ -231,9 +230,7 @@ fn main() { usize::from_str(DEFAULT_ROOT_COUNT).unwrap() }; - let iter = blocktree - .rooted_slot_iterator(0) - .expect("Failed to get rooted slot"); + let iter = RootedSlotIterator::new(0, &blocktree).expect("Failed to get rooted slot"); let slot_hash: Vec<_> = iter .filter_map(|(slot, meta)| { diff --git a/ledger-tool/tests/basic.rs b/ledger-tool/tests/basic.rs index 162c6137f4..35242fbd68 100644 --- a/ledger-tool/tests/basic.rs +++ b/ledger-tool/tests/basic.rs @@ -1,9 +1,9 @@ #[macro_use] -extern crate solana_core; +extern crate solana_ledger; use assert_cmd::prelude::*; -use solana_core::blocktree::create_new_tmp_ledger; use solana_core::genesis_utils::create_genesis_block; +use solana_ledger::blocktree::create_new_tmp_ledger; use std::process::Command; use std::process::Output; diff --git a/ledger/Cargo.toml b/ledger/Cargo.toml new file mode 100644 index 0000000000..4c6ee95250 --- /dev/null +++ b/ledger/Cargo.toml @@ -0,0 +1,49 @@ +[package] +name = "solana-ledger" +version = "0.20.0" +description = "Solana ledger" +authors = ["Solana Maintainers "] +repository = "https://github.com/solana-labs/solana" +license = "Apache-2.0" +homepage = "https://solana.com/" +edition = "2018" + +[dependencies] +bincode = "1.2.0" +byteorder = "1.3.2" +chrono = { version = "0.4.9", features = ["serde"] } +dlopen = "0.1.8" +dlopen_derive = "0.1.4" +itertools = "0.8.0" +lazy_static = "1.4.0" +log = { version = "0.4.8" } +rand = "0.6.5" +rand_chacha = "0.1.1" +rayon = "1.2.0" +reed-solomon-erasure = { package = "solana-reed-solomon-erasure", version = "4.0.1-3", features = ["simd-accel"] } +serde = "1.0.101" +serde_derive = "1.0.101" +solana-logger = { path = "../logger", version = "0.20.0" } +solana-merkle-tree = { path = "../merkle-tree", version = "0.20.0" } +solana-metrics = { path = "../metrics", version = "0.20.0" } +solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "0.20.0" } +solana-runtime = { path = "../runtime", version = "0.20.0" } +solana-sdk = { path = "../sdk", version = "0.20.0" } +solana-stake-api = { path = "../programs/stake_api", version = "0.20.0" } +solana-vote-api = { path = "../programs/vote_api", version = "0.20.0" } +sys-info = "0.5.8" + +[dependencies.rocksdb] +# Avoid the vendored bzip2 within rocksdb-sys that can cause linker conflicts +# when also using the bzip2 crate +version = "0.11.0" +default-features = false +features = ["lz4"] + +[dev-dependencies] +matches = "0.1.6" +solana-budget-api = { path = "../programs/budget_api", version = "0.20.0" } + +[lib] +crate-type = ["lib"] +name = "solana_ledger" diff --git a/core/src/blocktree.rs b/ledger/src/blocktree.rs similarity index 98% rename from core/src/blocktree.rs rename to ledger/src/blocktree.rs index 0fa555d8c0..28af4ca107 100644 --- a/core/src/blocktree.rs +++ b/ledger/src/blocktree.rs @@ -1,7 +1,7 @@ -//! The `block_tree` module provides functions for parallel verification of the +//! The `blocktree` module provides functions for parallel verification of the //! Proof of History ledger as well as iterative read, append write, and random //! access read to a persistent file-based ledger. -use crate::entry::Entry; +use crate::entry::{create_ticks, Entry}; use crate::erasure::ErasureConfig; use crate::shred::{Shred, Shredder}; @@ -9,7 +9,7 @@ use bincode::deserialize; use std::collections::HashMap; -use rocksdb; +use log::*; use solana_metrics::{datapoint_debug, datapoint_error}; @@ -28,13 +28,11 @@ use std::sync::{Arc, RwLock}; use std::time::Instant; pub use self::meta::*; -pub use self::rooted_slot_iterator::*; use crate::leader_schedule_cache::LeaderScheduleCache; use solana_sdk::clock::Slot; mod db; mod meta; -mod rooted_slot_iterator; pub use db::columns; use db::{columns as cf, Column, IteratorDirection, IteratorMode}; @@ -291,10 +289,6 @@ impl Blocktree { self.orphans_cf.get(slot) } - pub fn rooted_slot_iterator(&self, slot: u64) -> Result { - RootedSlotIterator::new(slot, self) - } - pub fn slot_meta_iterator(&self, slot: u64) -> Result> { let meta_iter = self .db @@ -1687,10 +1681,59 @@ pub fn entries_to_test_shreds( shredder.entries_to_shreds(&entries, is_full_slot, 0).0 } +pub fn make_slot_entries( + slot: u64, + parent_slot: u64, + num_entries: u64, +) -> (Vec, Vec) { + let entries = create_ticks(num_entries, Hash::default()); + let shreds = entries_to_test_shreds(entries.clone(), slot, parent_slot, true); + (shreds, entries) +} + +pub fn make_many_slot_entries( + start_slot: u64, + num_slots: u64, + entries_per_slot: u64, +) -> (Vec, Vec) { + let mut shreds = vec![]; + let mut entries = vec![]; + for slot in start_slot..start_slot + num_slots { + let parent_slot = if slot == 0 { 0 } else { slot - 1 }; + + let (slot_shreds, slot_entries) = make_slot_entries(slot, parent_slot, entries_per_slot); + shreds.extend(slot_shreds); + entries.extend(slot_entries); + } + + (shreds, entries) +} + +// Create shreds for slots that have a parent-child relationship defined by the input `chain` +pub fn make_chaining_slot_entries( + chain: &[u64], + entries_per_slot: u64, +) -> Vec<(Vec, Vec)> { + let mut slots_shreds_and_entries = vec![]; + for (i, slot) in chain.iter().enumerate() { + let parent_slot = { + if *slot == 0 || i == 0 { + 0 + } else { + chain[i - 1] + } + }; + + let result = make_slot_entries(*slot, parent_slot, entries_per_slot); + slots_shreds_and_entries.push(result); + } + + slots_shreds_and_entries +} + #[cfg(test)] pub mod tests { use super::*; - use crate::entry::{create_ticks, Entry}; use crate::genesis_utils::{create_genesis_block, GenesisBlockInfo}; use crate::shred::max_ticks_per_n_shreds; use itertools::Itertools; @@ -3507,55 +3550,4 @@ pub mod tests { drop(blocktree); Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction"); } - - pub fn make_slot_entries( - slot: u64, - parent_slot: u64, - num_entries: u64, - ) -> (Vec, Vec) { - let entries = create_ticks(num_entries, Hash::default()); - let shreds = entries_to_test_shreds(entries.clone(), slot, parent_slot, true); - (shreds, entries) - } - - pub fn make_many_slot_entries( - start_slot: u64, - num_slots: u64, - entries_per_slot: u64, - ) -> (Vec, Vec) { - let mut shreds = vec![]; - let mut entries = vec![]; - for slot in start_slot..start_slot + num_slots { - let parent_slot = if slot == 0 { 0 } else { slot - 1 }; - - let (slot_shreds, slot_entries) = - make_slot_entries(slot, parent_slot, entries_per_slot); - shreds.extend(slot_shreds); - entries.extend(slot_entries); - } - - (shreds, entries) - } - - // Create shreds for slots that have a parent-child relationship defined by the input `chain` - pub fn make_chaining_slot_entries( - chain: &[u64], - entries_per_slot: u64, - ) -> Vec<(Vec, Vec)> { - let mut slots_shreds_and_entries = vec![]; - for (i, slot) in chain.iter().enumerate() { - let parent_slot = { - if *slot == 0 || i == 0 { - 0 - } else { - chain[i - 1] - } - }; - - let result = make_slot_entries(*slot, parent_slot, entries_per_slot); - slots_shreds_and_entries.push(result); - } - - slots_shreds_and_entries - } } diff --git a/core/src/blocktree/db.rs b/ledger/src/blocktree/db.rs similarity index 99% rename from core/src/blocktree/db.rs rename to ledger/src/blocktree/db.rs index 6166e8dd8e..34d5b961a9 100644 --- a/core/src/blocktree/db.rs +++ b/ledger/src/blocktree/db.rs @@ -2,6 +2,7 @@ use crate::blocktree::{BlocktreeError, Result}; use bincode::{deserialize, serialize}; use byteorder::{BigEndian, ByteOrder}; +use log::*; use serde::de::DeserializeOwned; use serde::Serialize; diff --git a/core/src/blocktree/meta.rs b/ledger/src/blocktree/meta.rs similarity index 99% rename from core/src/blocktree/meta.rs rename to ledger/src/blocktree/meta.rs index cef855a961..f65e71ab05 100644 --- a/core/src/blocktree/meta.rs +++ b/ledger/src/blocktree/meta.rs @@ -1,4 +1,5 @@ use crate::erasure::ErasureConfig; +use serde::{Deserialize, Serialize}; use solana_metrics::datapoint; use std::cmp::Ordering; use std::{collections::BTreeSet, ops::Range, ops::RangeBounds}; diff --git a/core/src/entry.rs b/ledger/src/entry.rs similarity index 99% rename from core/src/entry.rs rename to ledger/src/entry.rs index 0517bfd7cc..55cb950dbe 100644 --- a/core/src/entry.rs +++ b/ledger/src/entry.rs @@ -4,10 +4,12 @@ //! represents an approximate amount of time since the last Entry was created. use crate::perf_libs; use crate::poh::Poh; +use log::*; use rayon::prelude::*; use rayon::ThreadPool; +use serde::{Deserialize, Serialize}; use solana_merkle_tree::MerkleTree; -use solana_metrics::inc_new_counter_warn; +use solana_metrics::*; use solana_rayon_threadlimit::get_thread_count; use solana_sdk::hash::Hash; use solana_sdk::timing; @@ -317,7 +319,6 @@ pub fn create_ticks(num_ticks: u64, mut hash: Hash) -> Vec { ticks } -#[cfg(test)] /// Creates the next Tick or Transaction Entry `num_hashes` after `start_hash`. pub fn next_entry(prev_hash: &Hash, num_hashes: u64, transactions: Vec) -> Entry { assert!(num_hashes > 0 || transactions.is_empty()); diff --git a/core/src/erasure.rs b/ledger/src/erasure.rs similarity index 99% rename from core/src/erasure.rs rename to ledger/src/erasure.rs index 2e4c5ed0cf..18c02f365e 100644 --- a/core/src/erasure.rs +++ b/ledger/src/erasure.rs @@ -43,6 +43,7 @@ use reed_solomon_erasure::galois_8::Field; use reed_solomon_erasure::ReedSolomon; +use serde::{Deserialize, Serialize}; //TODO(sakridge) pick these values /// Number of data blobs @@ -130,6 +131,7 @@ impl Default for Session { #[cfg(test)] pub mod test { use super::*; + use log::*; /// Specifies the contents of a 16-data-blob and 4-coding-blob erasure set /// Exists to be passed to `generate_blocktree_with_coding` diff --git a/ledger/src/genesis_utils.rs b/ledger/src/genesis_utils.rs new file mode 100644 index 0000000000..6901d263a6 --- /dev/null +++ b/ledger/src/genesis_utils.rs @@ -0,0 +1,14 @@ +pub use solana_runtime::genesis_utils::{ + create_genesis_block_with_leader, GenesisBlockInfo, BOOTSTRAP_LEADER_LAMPORTS, +}; +use solana_sdk::pubkey::Pubkey; + +// same as genesis_block::create_genesis_block, but with bootstrap_leader staking logic +// for the core crate tests +pub fn create_genesis_block(mint_lamports: u64) -> GenesisBlockInfo { + create_genesis_block_with_leader( + mint_lamports, + &Pubkey::new_rand(), + BOOTSTRAP_LEADER_LAMPORTS, + ) +} diff --git a/core/src/leader_schedule.rs b/ledger/src/leader_schedule.rs similarity index 98% rename from core/src/leader_schedule.rs rename to ledger/src/leader_schedule.rs index 855df2d20a..572c84a798 100644 --- a/core/src/leader_schedule.rs +++ b/ledger/src/leader_schedule.rs @@ -30,7 +30,7 @@ impl LeaderSchedule { Self { slot_leaders } } - pub(crate) fn get_slot_leaders(&self) -> &[Pubkey] { + pub fn get_slot_leaders(&self) -> &[Pubkey] { &self.slot_leaders } } diff --git a/core/src/leader_schedule_cache.rs b/ledger/src/leader_schedule_cache.rs similarity index 99% rename from core/src/leader_schedule_cache.rs rename to ledger/src/leader_schedule_cache.rs index f211d8db39..da4532d2c6 100644 --- a/core/src/leader_schedule_cache.rs +++ b/ledger/src/leader_schedule_cache.rs @@ -1,4 +1,5 @@ use crate::{blocktree::Blocktree, leader_schedule::LeaderSchedule, leader_schedule_utils}; +use log::*; use solana_runtime::bank::Bank; use solana_sdk::{epoch_schedule::EpochSchedule, pubkey::Pubkey}; use std::{ @@ -230,7 +231,7 @@ impl LeaderScheduleCache { mod tests { use super::*; use crate::{ - blocktree::{get_tmp_ledger_path, tests::make_slot_entries}, + blocktree::{get_tmp_ledger_path, make_slot_entries}, genesis_utils::{ create_genesis_block, create_genesis_block_with_leader, GenesisBlockInfo, BOOTSTRAP_LEADER_LAMPORTS, diff --git a/core/src/leader_schedule_utils.rs b/ledger/src/leader_schedule_utils.rs similarity index 99% rename from core/src/leader_schedule_utils.rs rename to ledger/src/leader_schedule_utils.rs index 604d8fd6bd..cf643d41ae 100644 --- a/core/src/leader_schedule_utils.rs +++ b/ledger/src/leader_schedule_utils.rs @@ -52,7 +52,6 @@ fn sort_stakes(stakes: &mut Vec<(Pubkey, u64)>) { #[cfg(test)] mod tests { use super::*; - use crate::staking_utils; use solana_runtime::genesis_utils::{ create_genesis_block_with_leader, BOOTSTRAP_LEADER_LAMPORTS, }; diff --git a/ledger/src/lib.rs b/ledger/src/lib.rs new file mode 100644 index 0000000000..e28e1605f7 --- /dev/null +++ b/ledger/src/lib.rs @@ -0,0 +1,12 @@ +#[macro_use] +pub mod blocktree; +pub mod entry; +pub mod erasure; +pub mod genesis_utils; +pub mod leader_schedule; +pub mod leader_schedule_cache; +pub mod leader_schedule_utils; +pub mod perf_libs; +pub mod poh; +pub mod shred; +pub mod staking_utils; diff --git a/core/src/perf_libs.rs b/ledger/src/perf_libs.rs similarity index 99% rename from core/src/perf_libs.rs rename to ledger/src/perf_libs.rs index 1c42474d64..63e798dd93 100644 --- a/core/src/perf_libs.rs +++ b/ledger/src/perf_libs.rs @@ -1,5 +1,7 @@ use core::ffi::c_void; use dlopen::symbor::{Container, SymBorApi, Symbol}; +use dlopen_derive::SymBorApi; +use log::*; use solana_sdk::packet::Packet; use std::env; use std::ffi::OsStr; diff --git a/core/src/poh.rs b/ledger/src/poh.rs similarity index 99% rename from core/src/poh.rs rename to ledger/src/poh.rs index d47afdb0b1..bce4020132 100644 --- a/core/src/poh.rs +++ b/ledger/src/poh.rs @@ -1,4 +1,5 @@ //! The `Poh` module provides an object for generating a Proof of History. +use log::*; use solana_sdk::hash::{hash, hashv, Hash}; use std::thread::{Builder, JoinHandle}; use std::time::{Duration, Instant}; @@ -116,6 +117,7 @@ pub fn compute_hashes_per_tick(duration: Duration, hashes_sample_size: u64) -> u #[cfg(test)] mod tests { use crate::poh::{Poh, PohEntry}; + use matches::assert_matches; use solana_sdk::hash::{hash, hashv, Hash}; fn verify(initial_hash: Hash, entries: &[(PohEntry, Option)]) -> bool { diff --git a/core/src/shred.rs b/ledger/src/shred.rs similarity index 99% rename from core/src/shred.rs rename to ledger/src/shred.rs index 37c08aa142..e502f48461 100644 --- a/core/src/shred.rs +++ b/ledger/src/shred.rs @@ -1,6 +1,5 @@ //! The `shred` module defines data structures and methods to pull MTU sized data frames from the network. -use crate::entry::create_ticks; -use crate::entry::Entry; +use crate::entry::{create_ticks, Entry}; use crate::erasure::Session; use bincode::serialized_size; use core::cell::RefCell; @@ -9,6 +8,7 @@ use rayon::iter::{IndexedParallelIterator, IntoParallelRefMutIterator, ParallelI use rayon::slice::ParallelSlice; use rayon::ThreadPool; use serde::{Deserialize, Serialize}; +use solana_metrics::datapoint_debug; use solana_rayon_threadlimit::get_thread_count; use solana_sdk::hash::Hash; use solana_sdk::packet::PACKET_DATA_SIZE; @@ -695,6 +695,7 @@ pub fn max_entries_per_n_shred(entry: &Entry, num_shreds: u64) -> u64 { #[cfg(test)] pub mod tests { use super::*; + use matches::assert_matches; use solana_sdk::system_transaction; use std::collections::HashSet; use std::convert::TryInto; diff --git a/core/src/staking_utils.rs b/ledger/src/staking_utils.rs similarity index 100% rename from core/src/staking_utils.rs rename to ledger/src/staking_utils.rs diff --git a/local_cluster/Cargo.toml b/local_cluster/Cargo.toml index 6548b470c6..a9215cb069 100644 --- a/local_cluster/Cargo.toml +++ b/local_cluster/Cargo.toml @@ -18,6 +18,7 @@ solana-client = { path = "../client", version = "0.20.0" } solana-drone = { path = "../drone", version = "0.20.0" } solana-exchange-api = { path = "../programs/exchange_api", version = "0.20.0" } solana-exchange-program = { path = "../programs/exchange_program", version = "0.20.0" } +solana-ledger = { path = "../ledger", version = "0.20.0" } solana-logger = { path = "../logger", version = "0.20.0" } solana-move-loader-api = { path = "../programs/move_loader_api", version = "0.20.0", optional = true } solana-move-loader-program = { path = "../programs/move_loader_program", version = "0.20.0", optional = true } diff --git a/local_cluster/src/cluster_tests.rs b/local_cluster/src/cluster_tests.rs index d81639f819..5319110b58 100644 --- a/local_cluster/src/cluster_tests.rs +++ b/local_cluster/src/cluster_tests.rs @@ -5,13 +5,13 @@ use solana_client::thin_client::create_client; /// All tests must start from an entry point and a funding keypair and /// discover the rest of the network. use solana_core::{ - blocktree::Blocktree, - cluster_info::VALIDATOR_PORT_RANGE, - consensus::VOTE_THRESHOLD_DEPTH, - contact_info::ContactInfo, - entry::{Entry, EntrySlice}, + cluster_info::VALIDATOR_PORT_RANGE, consensus::VOTE_THRESHOLD_DEPTH, contact_info::ContactInfo, gossip_service::discover_cluster, }; +use solana_ledger::{ + blocktree::Blocktree, + entry::{Entry, EntrySlice}, +}; use solana_sdk::{ client::SyncClient, clock::{DEFAULT_TICKS_PER_SECOND, DEFAULT_TICKS_PER_SLOT, NUM_CONSECUTIVE_LEADER_SLOTS}, diff --git a/local_cluster/src/lib.rs b/local_cluster/src/lib.rs index 52b0040542..ff0f7d25f7 100644 --- a/local_cluster/src/lib.rs +++ b/local_cluster/src/lib.rs @@ -19,7 +19,7 @@ extern crate solana_bench_tps; #[macro_use] #[cfg(test)] -extern crate solana_core; +extern crate solana_ledger; #[cfg(test)] extern crate solana_drone; diff --git a/local_cluster/src/local_cluster.rs b/local_cluster/src/local_cluster.rs index ea9c8d665e..c25e9baf68 100644 --- a/local_cluster/src/local_cluster.rs +++ b/local_cluster/src/local_cluster.rs @@ -1,7 +1,6 @@ use crate::cluster::{Cluster, ClusterValidatorInfo, ValidatorInfo}; use solana_client::thin_client::{create_client, ThinClient}; use solana_core::{ - blocktree::create_new_tmp_ledger, cluster_info::{Node, VALIDATOR_PORT_RANGE}, contact_info::ContactInfo, genesis_utils::{create_genesis_block_with_leader, GenesisBlockInfo}, @@ -10,6 +9,7 @@ use solana_core::{ service::Service, validator::{Validator, ValidatorConfig}, }; +use solana_ledger::blocktree::create_new_tmp_ledger; use solana_sdk::{ client::SyncClient, clock::{DEFAULT_SLOTS_PER_EPOCH, DEFAULT_SLOTS_PER_SEGMENT, DEFAULT_TICKS_PER_SLOT}, diff --git a/local_cluster/src/tests/local_cluster.rs b/local_cluster/src/tests/local_cluster.rs index c0b737787a..356e99dabd 100644 --- a/local_cluster/src/tests/local_cluster.rs +++ b/local_cluster/src/tests/local_cluster.rs @@ -6,9 +6,10 @@ use crate::{ use log::*; use serial_test_derive::serial; use solana_core::{ - bank_forks::SnapshotConfig, blocktree::Blocktree, broadcast_stage::BroadcastStageType, + bank_forks::SnapshotConfig, broadcast_stage::BroadcastStageType, gossip_service::discover_cluster, snapshot_utils, validator::ValidatorConfig, }; +use solana_ledger::blocktree::Blocktree; use solana_runtime::accounts_db::AccountsDB; use solana_sdk::{ client::SyncClient, diff --git a/local_cluster/src/tests/replicator.rs b/local_cluster/src/tests/replicator.rs index 0bce9ea6d1..585b00e062 100644 --- a/local_cluster/src/tests/replicator.rs +++ b/local_cluster/src/tests/replicator.rs @@ -1,13 +1,13 @@ use crate::local_cluster::{ClusterConfig, LocalCluster}; use serial_test_derive::serial; use solana_client::thin_client::create_client; -use solana_core::blocktree::{create_new_tmp_ledger, get_tmp_ledger_path, Blocktree}; use solana_core::cluster_info::{ClusterInfo, Node, VALIDATOR_PORT_RANGE}; use solana_core::contact_info::ContactInfo; use solana_core::gossip_service::discover_cluster; use solana_core::replicator::Replicator; use solana_core::storage_stage::SLOTS_PER_TURN_TEST; use solana_core::validator::ValidatorConfig; +use solana_ledger::blocktree::{create_new_tmp_ledger, get_tmp_ledger_path, Blocktree}; use solana_sdk::genesis_block::create_genesis_block; use solana_sdk::signature::{Keypair, KeypairUtil}; use std::fs::remove_dir_all; diff --git a/validator/Cargo.toml b/validator/Cargo.toml index 48d173cc64..dbc7645623 100644 --- a/validator/Cargo.toml +++ b/validator/Cargo.toml @@ -19,6 +19,7 @@ serde_json = "1.0.41" solana-client = { path = "../client", version = "0.20.0" } solana-core = { path = "../core", version = "0.20.0" } solana-drone = { path = "../drone", version = "0.20.0" } +solana-ledger = { path = "../ledger", version = "0.20.0" } solana-logger = { path = "../logger", version = "0.20.0" } solana-metrics = { path = "../metrics", version = "0.20.0" } solana-netutil = { path = "../netutil", version = "0.20.0" } diff --git a/validator/src/lib.rs b/validator/src/lib.rs index ef8c93a797..72276e10f8 100644 --- a/validator/src/lib.rs +++ b/validator/src/lib.rs @@ -414,7 +414,7 @@ pub fn main() { .get_matches(); if matches.is_present("cuda") { - solana_core::perf_libs::init_cuda(); + solana_ledger::perf_libs::init_cuda(); } let mut validator_config = ValidatorConfig::default();