Extract kvstore into separate crate (#3327)

* extract kvstore into new crate

* add kvstore crate to CI publishing list

Cargo.lock

@@ -2029,6 +2029,7 @@ dependencies = [
  "solana-budget-program 0.13.0",
  "solana-client 0.13.0",
  "solana-drone 0.13.0",
+ "solana-kvstore 0.0.1",
  "solana-logger 0.13.0",
  "solana-metrics 0.13.0",
  "solana-netutil 0.13.0",
@@ -2256,6 +2257,20 @@ dependencies = [
  "solana-sdk 0.13.0",
 ]
 
+[[package]]
+name = "solana-kvstore"
+version = "0.0.1"
+dependencies = [
+ "bincode 1.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "chrono 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "crc 1.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "memmap 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_derive 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "solana-ledger-tool"
 version = "0.13.0"

Cargo.toml

@@ -67,6 +67,7 @@ members = [
     "genesis",
     "install",
     "keygen",
+    "kvstore",
     "ledger-tool",
     "logger",
     "metrics",

ci/publish-crate.sh

@@ -11,6 +11,7 @@ cd "$(dirname "$0")/.."
 # here. (TODO: figure the crate ordering dynamically)
 #
 CRATES=(
+  kvstore
   logger
   netutil
   sdk

core/Cargo.toml

@@ -17,7 +17,7 @@ codecov = { repository = "solana-labs/solana", branch = "master", service = "git
 chacha = []
 cuda = []
 erasure = []
-kvstore = ["crc", "memmap"]
+kvstore = ["solana-kvstore"]
 
 [dependencies]
 bincode = "1.1.2"
@@ -48,6 +48,7 @@ serde_json = "1.0.39"
 solana-budget-api = { path = "../programs/budget_api", version = "0.13.0" }
 solana-client = { path = "../client", version = "0.13.0" }
 solana-drone = { path = "../drone", version = "0.13.0" }
+solana-kvstore = { path = "../kvstore", version = "0.0.1", optional = true }
 solana-logger = { path = "../logger", version = "0.13.0" }
 solana-metrics = { path = "../metrics", version = "0.13.0" }
 solana-netutil = { path = "../netutil", version = "0.13.0" }

core/src/blocktree.rs

@@ -3,10 +3,10 @@
 //! access read to a persistent file-based ledger.
 
 use crate::entry::Entry;
-#[cfg(feature = "kvstore")]
-use crate::kvstore;
 use crate::packet::{Blob, SharedBlob, BLOB_HEADER_SIZE};
 use crate::result::{Error, Result};
+#[cfg(feature = "kvstore")]
+use solana_kvstore as kvstore;
 
 use bincode::{deserialize, serialize};
 

core/src/blocktree/kvs.rs

@@ -1,7 +1,7 @@
 use crate::entry::Entry;
-use crate::kvstore::{self, Key, KvStore};
 use crate::packet::Blob;
 use crate::result::{Error, Result};
+use solana_kvstore::{self as kvstore, Key, KvStore};
 
 use std::sync::Arc;
 

core/src/lib.rs

@@ -37,8 +37,6 @@ pub mod fetch_stage;
 pub mod fullnode;
 pub mod gen_keys;
 pub mod gossip_service;
-#[cfg(feature = "kvstore")]
-pub mod kvstore;
 pub mod leader_confirmation_service;
 pub mod leader_schedule;
 pub mod leader_schedule_utils;

kvstore/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "solana-kvstore"
+description = "Embedded Key-Value store for solana"
+version = "0.0.1"
+homepage = "https://solana.com/"
+repository = "https://github.com/solana-labs/solana"
+authors = ["Solana Maintainers <maintainers@solana.com>"]
+license = "Apache-2.0"
+edition = "2018"
+
+[dependencies]
+bincode = "1.1.2"
+byteorder = "1.3.1"
+chrono = "0.4.0"
+crc = "1.8.1"
+memmap = "0.7.0"
+rand = "0.6.5"
+serde = "1.0.89"
+serde_derive = "1.0.88"
+

kvstore/benches/basic.rs

@@ -1,4 +1,3 @@
-#![cfg(feature = "kvstore")]
 #![feature(test)]
 extern crate test;
 
@@ -9,7 +8,7 @@ use rand::{self, thread_rng, Rng};
 
 use test::Bencher;
 
-use solana::kvstore::{Config, Key, KvStore};
+use solana_kvstore::{Config, Key, KvStore};
 
 const SMALL_SIZE: usize = 512;
 const LARGE_SIZE: usize = 32 * 1024;

kvstore/src/compactor.rs

@@ -1,6 +1,6 @@
-use crate::kvstore::error::{Error, Result};
-use crate::kvstore::mapper::{Kind, Mapper};
-use crate::kvstore::sstable::{Key, Merged, SSTable};
+use crate::error::{Error, Result};
+use crate::mapper::{Kind, Mapper};
+use crate::sstable::{Key, Merged, SSTable};
 
 use std::collections::BTreeMap;
 use std::path::PathBuf;

kvstore/src/lib.rs

@@ -1,7 +1,7 @@
-use crate::kvstore::mapper::{Disk, Mapper, Memory};
-use crate::kvstore::sstable::SSTable;
-use crate::kvstore::storage::WriteState;
-use crate::kvstore::writelog::WriteLog;
+use crate::mapper::{Disk, Mapper, Memory};
+use crate::sstable::SSTable;
+use crate::storage::WriteState;
+use crate::writelog::WriteLog;
 use std::collections::BTreeMap;
 use std::fs;
 use std::io;
@@ -21,6 +21,9 @@ mod storage;
 mod writelog;
 mod writetx;
 
+#[macro_use]
+extern crate serde_derive;
+
 pub use self::error::{Error, Result};
 pub use self::readtx::ReadTx as Snapshot;
 pub use self::sstable::Key;

kvstore/src/mapper.rs

@@ -1,6 +1,6 @@
-use crate::kvstore::io_utils::Writer;
-use crate::kvstore::sstable::SSTable;
-use crate::kvstore::Result;
+use crate::io_utils::Writer;
+use crate::sstable::SSTable;
+use crate::Result;
 
 use std::path::Path;
 use std::sync::RwLock;

kvstore/src/mapper/disk.rs

@@ -1,7 +1,7 @@
-use crate::kvstore::io_utils::{MemMap, Writer};
-use crate::kvstore::mapper::{Kind, Mapper, RwLockExt};
-use crate::kvstore::sstable::SSTable;
-use crate::kvstore::Result;
+use crate::io_utils::{MemMap, Writer};
+use crate::mapper::{Kind, Mapper, RwLockExt};
+use crate::sstable::SSTable;
+use crate::Result;
 
 use memmap::Mmap;
 

kvstore/src/mapper/memory.rs

@@ -1,7 +1,7 @@
-use crate::kvstore::io_utils::{MemMap, SharedWriter, Writer};
-use crate::kvstore::mapper::{Kind, Mapper, RwLockExt};
-use crate::kvstore::sstable::SSTable;
-use crate::kvstore::Result;
+use crate::io_utils::{MemMap, SharedWriter, Writer};
+use crate::mapper::{Kind, Mapper, RwLockExt};
+use crate::sstable::SSTable;
+use crate::Result;
 
 use rand::{rngs::SmallRng, FromEntropy, Rng};
 

kvstore/src/readtx.rs

@@ -1,6 +1,6 @@
-use crate::kvstore::error::Result;
-use crate::kvstore::sstable::{Key, SSTable, Value};
-use crate::kvstore::storage;
+use crate::error::Result;
+use crate::sstable::{Key, SSTable, Value};
+use crate::storage;
 
 use std::collections::BTreeMap;
 use std::ops::RangeInclusive;

kvstore/src/sstable.rs

@@ -1,5 +1,5 @@
-use crate::kvstore::error::Result;
-use crate::kvstore::io_utils::{MemMap, Writer};
+use crate::error::Result;
+use crate::io_utils::{MemMap, Writer};
 
 use byteorder::{BigEndian, ByteOrder, WriteBytesExt};
 

kvstore/src/storage.rs

@@ -1,7 +1,7 @@
-use crate::kvstore::error::Result;
-use crate::kvstore::mapper::{Kind, Mapper};
-use crate::kvstore::sstable::{Key, Merged, SSTable, Value};
-use crate::kvstore::writelog::WriteLog;
+use crate::error::Result;
+use crate::mapper::{Kind, Mapper};
+use crate::sstable::{Key, Merged, SSTable, Value};
+use crate::writelog::WriteLog;
 
 use chrono::Utc;
 

kvstore/src/writelog.rs

@@ -1,7 +1,7 @@
-use crate::kvstore::error::Result;
-use crate::kvstore::io_utils::{CRCReader, CRCWriter};
-use crate::kvstore::sstable::Value;
-use crate::kvstore::Key;
+use crate::error::Result;
+use crate::io_utils::{CRCReader, CRCWriter};
+use crate::sstable::Value;
+use crate::Key;
 use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt};
 use memmap::Mmap;
 use std::collections::BTreeMap;
@@ -138,7 +138,8 @@ impl Logger {
 
 impl LogWriter for CRCWriter<Vec<u8>> {
     fn sync(&mut self, _: bool) -> Result<()> {
-        Ok(self.flush()?)
+        self.flush()?;
+        Ok(())
     }
 
     fn mmap(&self) -> Result<Mmap> {
@@ -310,7 +311,7 @@ mod test {
 
     #[test]
     fn test_reset() {
-        use crate::kvstore::error::Error;
+        use crate::error::Error;
 
         let wal = WriteLog::memory(Config::default());
 

kvstore/src/writetx.rs

@@ -1,5 +1,5 @@
-use crate::kvstore::error::Result;
-use crate::kvstore::sstable::Key;
+use crate::error::Result;
+use crate::sstable::Key;
 
 #[derive(Debug)]
 pub struct WriteTx<'a> {

kvstore/tests/basic.rs

@@ -1,10 +1,9 @@
-#![cfg(feature = "kvstore")]
 use rand::{thread_rng, Rng};
 
 use std::fs;
 use std::path::{Path, PathBuf};
 
-use solana::kvstore::{Config, Key, KvStore};
+use solana_kvstore::{Config, Key, KvStore};
 
 const KB: usize = 1024;
 const HALF_KB: usize = 512;