diff --git a/Cargo.lock b/Cargo.lock index 59e0a3fa18..cd9b4e694a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2029,6 +2029,7 @@ dependencies = [ "solana-budget-program 0.13.0", "solana-client 0.13.0", "solana-drone 0.13.0", + "solana-kvstore 0.0.1", "solana-logger 0.13.0", "solana-metrics 0.13.0", "solana-netutil 0.13.0", @@ -2256,6 +2257,20 @@ dependencies = [ "solana-sdk 0.13.0", ] +[[package]] +name = "solana-kvstore" +version = "0.0.1" +dependencies = [ + "bincode 1.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "chrono 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "crc 1.8.1 (registry+https://github.com/rust-lang/crates.io-index)", + "memmap 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "solana-ledger-tool" version = "0.13.0" diff --git a/Cargo.toml b/Cargo.toml index afebefe1c6..c5b9e20e7b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,6 +67,7 @@ members = [ "genesis", "install", "keygen", + "kvstore", "ledger-tool", "logger", "metrics", diff --git a/ci/publish-crate.sh b/ci/publish-crate.sh index cc3bbea57d..a70aea2aa3 100755 --- a/ci/publish-crate.sh +++ b/ci/publish-crate.sh @@ -11,6 +11,7 @@ cd "$(dirname "$0")/.." # here. (TODO: figure the crate ordering dynamically) # CRATES=( + kvstore logger netutil sdk diff --git a/core/Cargo.toml b/core/Cargo.toml index 1abd63c9c0..174425b0e8 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -17,7 +17,7 @@ codecov = { repository = "solana-labs/solana", branch = "master", service = "git chacha = [] cuda = [] erasure = [] -kvstore = ["crc", "memmap"] +kvstore = ["solana-kvstore"] [dependencies] bincode = "1.1.2" @@ -48,6 +48,7 @@ serde_json = "1.0.39" solana-budget-api = { path = "../programs/budget_api", version = "0.13.0" } solana-client = { path = "../client", version = "0.13.0" } solana-drone = { path = "../drone", version = "0.13.0" } +solana-kvstore = { path = "../kvstore", version = "0.0.1", optional = true } solana-logger = { path = "../logger", version = "0.13.0" } solana-metrics = { path = "../metrics", version = "0.13.0" } solana-netutil = { path = "../netutil", version = "0.13.0" } diff --git a/core/src/blocktree.rs b/core/src/blocktree.rs index c1609140ba..bb40600d3a 100644 --- a/core/src/blocktree.rs +++ b/core/src/blocktree.rs @@ -3,10 +3,10 @@ //! access read to a persistent file-based ledger. use crate::entry::Entry; -#[cfg(feature = "kvstore")] -use crate::kvstore; use crate::packet::{Blob, SharedBlob, BLOB_HEADER_SIZE}; use crate::result::{Error, Result}; +#[cfg(feature = "kvstore")] +use solana_kvstore as kvstore; use bincode::{deserialize, serialize}; diff --git a/core/src/blocktree/kvs.rs b/core/src/blocktree/kvs.rs index 1b97214f4e..87f0d4e176 100644 --- a/core/src/blocktree/kvs.rs +++ b/core/src/blocktree/kvs.rs @@ -1,7 +1,7 @@ use crate::entry::Entry; -use crate::kvstore::{self, Key, KvStore}; use crate::packet::Blob; use crate::result::{Error, Result}; +use solana_kvstore::{self as kvstore, Key, KvStore}; use std::sync::Arc; diff --git a/core/src/lib.rs b/core/src/lib.rs index 0c53b506fa..df7e084854 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -37,8 +37,6 @@ pub mod fetch_stage; pub mod fullnode; pub mod gen_keys; pub mod gossip_service; -#[cfg(feature = "kvstore")] -pub mod kvstore; pub mod leader_confirmation_service; pub mod leader_schedule; pub mod leader_schedule_utils; diff --git a/kvstore/Cargo.toml b/kvstore/Cargo.toml new file mode 100644 index 0000000000..d2e694dace --- /dev/null +++ b/kvstore/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "solana-kvstore" +description = "Embedded Key-Value store for solana" +version = "0.0.1" +homepage = "https://solana.com/" +repository = "https://github.com/solana-labs/solana" +authors = ["Solana Maintainers "] +license = "Apache-2.0" +edition = "2018" + +[dependencies] +bincode = "1.1.2" +byteorder = "1.3.1" +chrono = "0.4.0" +crc = "1.8.1" +memmap = "0.7.0" +rand = "0.6.5" +serde = "1.0.89" +serde_derive = "1.0.88" + diff --git a/core/benches/kvstore.rs b/kvstore/benches/basic.rs similarity index 98% rename from core/benches/kvstore.rs rename to kvstore/benches/basic.rs index 516c305a0d..19aee83d9a 100644 --- a/core/benches/kvstore.rs +++ b/kvstore/benches/basic.rs @@ -1,4 +1,3 @@ -#![cfg(feature = "kvstore")] #![feature(test)] extern crate test; @@ -9,7 +8,7 @@ use rand::{self, thread_rng, Rng}; use test::Bencher; -use solana::kvstore::{Config, Key, KvStore}; +use solana_kvstore::{Config, Key, KvStore}; const SMALL_SIZE: usize = 512; const LARGE_SIZE: usize = 32 * 1024; diff --git a/core/src/kvstore/compactor.rs b/kvstore/src/compactor.rs similarity index 97% rename from core/src/kvstore/compactor.rs rename to kvstore/src/compactor.rs index 0e1c444847..8c1124337b 100644 --- a/core/src/kvstore/compactor.rs +++ b/kvstore/src/compactor.rs @@ -1,6 +1,6 @@ -use crate::kvstore::error::{Error, Result}; -use crate::kvstore::mapper::{Kind, Mapper}; -use crate::kvstore::sstable::{Key, Merged, SSTable}; +use crate::error::{Error, Result}; +use crate::mapper::{Kind, Mapper}; +use crate::sstable::{Key, Merged, SSTable}; use std::collections::BTreeMap; use std::path::PathBuf; diff --git a/core/src/kvstore/error.rs b/kvstore/src/error.rs similarity index 100% rename from core/src/kvstore/error.rs rename to kvstore/src/error.rs diff --git a/core/src/kvstore/io_utils.rs b/kvstore/src/io_utils.rs similarity index 100% rename from core/src/kvstore/io_utils.rs rename to kvstore/src/io_utils.rs diff --git a/core/src/kvstore.rs b/kvstore/src/lib.rs similarity index 98% rename from core/src/kvstore.rs rename to kvstore/src/lib.rs index e576f0cc65..3dd3ce86ab 100644 --- a/core/src/kvstore.rs +++ b/kvstore/src/lib.rs @@ -1,7 +1,7 @@ -use crate::kvstore::mapper::{Disk, Mapper, Memory}; -use crate::kvstore::sstable::SSTable; -use crate::kvstore::storage::WriteState; -use crate::kvstore::writelog::WriteLog; +use crate::mapper::{Disk, Mapper, Memory}; +use crate::sstable::SSTable; +use crate::storage::WriteState; +use crate::writelog::WriteLog; use std::collections::BTreeMap; use std::fs; use std::io; @@ -21,6 +21,9 @@ mod storage; mod writelog; mod writetx; +#[macro_use] +extern crate serde_derive; + pub use self::error::{Error, Result}; pub use self::readtx::ReadTx as Snapshot; pub use self::sstable::Key; diff --git a/core/src/kvstore/mapper.rs b/kvstore/src/mapper.rs similarity index 93% rename from core/src/kvstore/mapper.rs rename to kvstore/src/mapper.rs index fab8f4250b..87228f4810 100644 --- a/core/src/kvstore/mapper.rs +++ b/kvstore/src/mapper.rs @@ -1,6 +1,6 @@ -use crate::kvstore::io_utils::Writer; -use crate::kvstore::sstable::SSTable; -use crate::kvstore::Result; +use crate::io_utils::Writer; +use crate::sstable::SSTable; +use crate::Result; use std::path::Path; use std::sync::RwLock; diff --git a/core/src/kvstore/mapper/disk.rs b/kvstore/src/mapper/disk.rs similarity index 97% rename from core/src/kvstore/mapper/disk.rs rename to kvstore/src/mapper/disk.rs index 37d3125f48..7a6b1e37b0 100644 --- a/core/src/kvstore/mapper/disk.rs +++ b/kvstore/src/mapper/disk.rs @@ -1,7 +1,7 @@ -use crate::kvstore::io_utils::{MemMap, Writer}; -use crate::kvstore::mapper::{Kind, Mapper, RwLockExt}; -use crate::kvstore::sstable::SSTable; -use crate::kvstore::Result; +use crate::io_utils::{MemMap, Writer}; +use crate::mapper::{Kind, Mapper, RwLockExt}; +use crate::sstable::SSTable; +use crate::Result; use memmap::Mmap; diff --git a/core/src/kvstore/mapper/memory.rs b/kvstore/src/mapper/memory.rs similarity index 95% rename from core/src/kvstore/mapper/memory.rs rename to kvstore/src/mapper/memory.rs index 3db5c57b7e..42c643e21f 100644 --- a/core/src/kvstore/mapper/memory.rs +++ b/kvstore/src/mapper/memory.rs @@ -1,7 +1,7 @@ -use crate::kvstore::io_utils::{MemMap, SharedWriter, Writer}; -use crate::kvstore::mapper::{Kind, Mapper, RwLockExt}; -use crate::kvstore::sstable::SSTable; -use crate::kvstore::Result; +use crate::io_utils::{MemMap, SharedWriter, Writer}; +use crate::mapper::{Kind, Mapper, RwLockExt}; +use crate::sstable::SSTable; +use crate::Result; use rand::{rngs::SmallRng, FromEntropy, Rng}; diff --git a/core/src/kvstore/readtx.rs b/kvstore/src/readtx.rs similarity index 86% rename from core/src/kvstore/readtx.rs rename to kvstore/src/readtx.rs index 9d95e69458..0a0b1d0580 100644 --- a/core/src/kvstore/readtx.rs +++ b/kvstore/src/readtx.rs @@ -1,6 +1,6 @@ -use crate::kvstore::error::Result; -use crate::kvstore::sstable::{Key, SSTable, Value}; -use crate::kvstore::storage; +use crate::error::Result; +use crate::sstable::{Key, SSTable, Value}; +use crate::storage; use std::collections::BTreeMap; use std::ops::RangeInclusive; diff --git a/core/src/kvstore/sstable.rs b/kvstore/src/sstable.rs similarity index 99% rename from core/src/kvstore/sstable.rs rename to kvstore/src/sstable.rs index 0bcb404b56..55d8e50af3 100644 --- a/core/src/kvstore/sstable.rs +++ b/kvstore/src/sstable.rs @@ -1,5 +1,5 @@ -use crate::kvstore::error::Result; -use crate::kvstore::io_utils::{MemMap, Writer}; +use crate::error::Result; +use crate::io_utils::{MemMap, Writer}; use byteorder::{BigEndian, ByteOrder, WriteBytesExt}; diff --git a/core/src/kvstore/storage.rs b/kvstore/src/storage.rs similarity index 96% rename from core/src/kvstore/storage.rs rename to kvstore/src/storage.rs index 4c738f2b7d..43f0d9504b 100644 --- a/core/src/kvstore/storage.rs +++ b/kvstore/src/storage.rs @@ -1,7 +1,7 @@ -use crate::kvstore::error::Result; -use crate::kvstore::mapper::{Kind, Mapper}; -use crate::kvstore::sstable::{Key, Merged, SSTable, Value}; -use crate::kvstore::writelog::WriteLog; +use crate::error::Result; +use crate::mapper::{Kind, Mapper}; +use crate::sstable::{Key, Merged, SSTable, Value}; +use crate::writelog::WriteLog; use chrono::Utc; diff --git a/core/src/kvstore/writelog.rs b/kvstore/src/writelog.rs similarity index 97% rename from core/src/kvstore/writelog.rs rename to kvstore/src/writelog.rs index ba5a16b977..0310312626 100644 --- a/core/src/kvstore/writelog.rs +++ b/kvstore/src/writelog.rs @@ -1,7 +1,7 @@ -use crate::kvstore::error::Result; -use crate::kvstore::io_utils::{CRCReader, CRCWriter}; -use crate::kvstore::sstable::Value; -use crate::kvstore::Key; +use crate::error::Result; +use crate::io_utils::{CRCReader, CRCWriter}; +use crate::sstable::Value; +use crate::Key; use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt}; use memmap::Mmap; use std::collections::BTreeMap; @@ -138,7 +138,8 @@ impl Logger { impl LogWriter for CRCWriter> { fn sync(&mut self, _: bool) -> Result<()> { - Ok(self.flush()?) + self.flush()?; + Ok(()) } fn mmap(&self) -> Result { @@ -310,7 +311,7 @@ mod test { #[test] fn test_reset() { - use crate::kvstore::error::Error; + use crate::error::Error; let wal = WriteLog::memory(Config::default()); diff --git a/core/src/kvstore/writetx.rs b/kvstore/src/writetx.rs similarity index 80% rename from core/src/kvstore/writetx.rs rename to kvstore/src/writetx.rs index 8bd33739a1..44863e23c7 100644 --- a/core/src/kvstore/writetx.rs +++ b/kvstore/src/writetx.rs @@ -1,5 +1,5 @@ -use crate::kvstore::error::Result; -use crate::kvstore::sstable::Key; +use crate::error::Result; +use crate::sstable::Key; #[derive(Debug)] pub struct WriteTx<'a> { diff --git a/core/tests/kvstore.rs b/kvstore/tests/basic.rs similarity index 98% rename from core/tests/kvstore.rs rename to kvstore/tests/basic.rs index bec27ca40b..ad4a408682 100644 --- a/core/tests/kvstore.rs +++ b/kvstore/tests/basic.rs @@ -1,10 +1,9 @@ -#![cfg(feature = "kvstore")] use rand::{thread_rng, Rng}; use std::fs; use std::path::{Path, PathBuf}; -use solana::kvstore::{Config, Key, KvStore}; +use solana_kvstore::{Config, Key, KvStore}; const KB: usize = 1024; const HALF_KB: usize = 512;