Rename snapshot.tar.bz2 to snapshot-<slot>-<hash>.tar.bz2
This commit is contained in:
Michael Vines
parent
848c43a9ab
commit
fcabc6f799
|
|
@ -197,7 +197,7 @@ dependencies = [
|
|||
"peeking_take_while 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"quote 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"shlex 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"which 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
|
@ -584,7 +584,7 @@ dependencies = [
|
|||
"encode_unicode 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"termios 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
|
|
@ -900,7 +900,7 @@ version = "1.1.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"strsim 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
|
@ -998,7 +998,7 @@ dependencies = [
|
|||
"atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"humantime 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"termcolor 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
|
|
@ -1010,7 +1010,7 @@ dependencies = [
|
|||
"atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"humantime 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"termcolor 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
|
|
@ -1022,7 +1022,7 @@ dependencies = [
|
|||
"atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"humantime 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"termcolor 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
|
|
@ -1316,7 +1316,7 @@ dependencies = [
|
|||
"bstr 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1624,7 +1624,7 @@ dependencies = [
|
|||
"console 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"number_prefix 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1840,8 +1840,8 @@ dependencies = [
|
|||
"itertools 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"lalrpop-util 0.17.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"petgraph 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"regex-syntax 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde_derive 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"sha2 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
|
|
@ -2656,7 +2656,7 @@ dependencies = [
|
|||
"rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"regex-syntax 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rusty-fork 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
|
@ -2976,13 +2976,13 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.3.1"
|
||||
version = "1.3.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"aho-corasick 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"regex-syntax 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"thread_local 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -3000,7 +3000,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
|
||||
[[package]]
|
||||
name = "regex-syntax"
|
||||
version = "0.6.12"
|
||||
version = "0.6.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
|
|
@ -3831,6 +3831,7 @@ dependencies = [
|
|||
"rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rayon 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"reqwest 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde_derive 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
|
|
@ -3882,7 +3883,7 @@ dependencies = [
|
|||
"lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"regex-syntax 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"reqwest 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"syn 0.15.42 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
|
|
@ -4053,6 +4054,7 @@ dependencies = [
|
|||
"rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rayon 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rocksdb 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde_bytes 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
|
|
@ -4716,7 +4718,7 @@ dependencies = [
|
|||
"lalrpop 0.17.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"lalrpop-util 0.17.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"solana_libra_failure_ext 0.0.1-sol4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"solana_libra_types 0.0.1-sol4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
|
@ -5229,6 +5231,14 @@ dependencies = [
|
|||
"lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thread_local"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "threshold_crypto"
|
||||
version = "0.3.2"
|
||||
|
|
@ -5286,7 +5296,7 @@ version = "1.1.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -6376,10 +6386,10 @@ dependencies = [
|
|||
"checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84"
|
||||
"checksum redox_users 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3fe5204c3a17e97dde73f285d49be585df59ed84b50a872baf416e73b62c3828"
|
||||
"checksum regex 0.1.80 (registry+https://github.com/rust-lang/crates.io-index)" = "4fd4ace6a8cf7860714a2c2280d6c1f7e6a413486c13298bbc86fd3da019402f"
|
||||
"checksum regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dc220bd33bdce8f093101afe22a037b8eb0e5af33592e6a9caafff0d4cb81cbd"
|
||||
"checksum regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "322cf97724bea3ee221b78fe25ac9c46114ebb51747ad5babd51a2fc6a8235a8"
|
||||
"checksum regex-automata 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "92b73c2a1770c255c240eaa4ee600df1704a38dc3feaa6e949e7fcd4f8dc09f9"
|
||||
"checksum regex-syntax 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "f9ec002c35e86791825ed294b50008eea9ddfc8def4420124fbc6b08db834957"
|
||||
"checksum regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)" = "11a7e20d1cce64ef2fed88b66d347f88bd9babb82845b2b858f3edbf59a4f716"
|
||||
"checksum regex-syntax 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)" = "b28dfe3fe9badec5dbf0a79a9cccad2cfc2ab5484bdb3e44cbd1ae8b3ba2be06"
|
||||
"checksum remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4a83fa3702a688b9359eccba92d153ac33fd2e8462f9e0e3fdf155239ea7792e"
|
||||
"checksum rental 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)" = "01916ebd9fc2e81978a5dc9542a2fa47f5bb2ca3402e14c7cc42d6e3c5123e1f"
|
||||
"checksum rental-impl 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)" = "82260d54cf2cbe9608df161f7e7c98e81fae702aa13af9e4d5d39dc2ffb25ab6"
|
||||
|
|
@ -6493,6 +6503,7 @@ dependencies = [
|
|||
"checksum thread-scoped 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "bcbb6aa301e5d3b0b5ef639c9a9c7e2f1c944f177b460c04dc24c69b1fa2bd99"
|
||||
"checksum thread_local 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)" = "8576dbbfcaef9641452d5cf0df9b0e7eeab7694956dd33bb61515fb8f18cfdd5"
|
||||
"checksum thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b"
|
||||
"checksum thread_local 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14"
|
||||
"checksum threshold_crypto 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "95be1032c63011f20b01c5edb64930e2b51512782b43b458b1e3449613d70f87"
|
||||
"checksum time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f"
|
||||
"checksum tiny-bip39 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1cd1fb03fe8e07d17cd851a624a9fff74642a997b67fbd1ccd77533241640d92"
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ num-traits = "0.2"
|
|||
rand = "0.6.5"
|
||||
rand_chacha = "0.1.1"
|
||||
rayon = "1.2.0"
|
||||
regex = "1.3.4"
|
||||
serde = "1.0.104"
|
||||
serde_derive = "1.0.103"
|
||||
serde_json = "1.0.46"
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ use jsonrpc_http_server::{
|
|||
hyper, AccessControlAllowOrigin, CloseHandle, DomainsValidation, RequestMiddleware,
|
||||
RequestMiddlewareAction, ServerBuilder,
|
||||
};
|
||||
use regex::Regex;
|
||||
use solana_ledger::{bank_forks::BankForks, blockstore::Blockstore};
|
||||
use solana_sdk::hash::Hash;
|
||||
use std::{
|
||||
|
|
@ -28,13 +29,17 @@ pub struct JsonRpcService {
|
|||
close_handle: Option<CloseHandle>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct RpcRequestMiddleware {
|
||||
ledger_path: PathBuf,
|
||||
snapshot_archive_path_regex: Regex,
|
||||
}
|
||||
impl RpcRequestMiddleware {
|
||||
pub fn new(ledger_path: PathBuf) -> Self {
|
||||
Self { ledger_path }
|
||||
Self {
|
||||
ledger_path,
|
||||
snapshot_archive_path_regex: Regex::new(r"/snapshot-\d+-[[:alnum:]]+\.tar\.bz2$")
|
||||
.unwrap(),
|
||||
}
|
||||
}
|
||||
|
||||
fn not_found() -> hyper::Response<hyper::Body> {
|
||||
|
|
@ -51,9 +56,19 @@ impl RpcRequestMiddleware {
|
|||
.unwrap()
|
||||
}
|
||||
|
||||
fn get(&self, filename: &str) -> RequestMiddlewareAction {
|
||||
info!("get {}", filename);
|
||||
let filename = self.ledger_path.join(filename);
|
||||
fn is_get_path(&self, path: &str) -> bool {
|
||||
match path {
|
||||
"/genesis.tar.bz2" => true,
|
||||
_ => self.snapshot_archive_path_regex.is_match(path),
|
||||
}
|
||||
}
|
||||
|
||||
fn get(&self, path: &str) -> RequestMiddlewareAction {
|
||||
let filename = self.ledger_path.join(
|
||||
path.split_at(1).1, // Drop leading '/' from path
|
||||
);
|
||||
info!("get {} -> {:?}", path, filename);
|
||||
|
||||
RequestMiddlewareAction::Respond {
|
||||
should_validate_hosts: true,
|
||||
response: Box::new(
|
||||
|
|
@ -73,13 +88,14 @@ impl RpcRequestMiddleware {
|
|||
impl RequestMiddleware for RpcRequestMiddleware {
|
||||
fn on_request(&self, request: hyper::Request<hyper::Body>) -> RequestMiddlewareAction {
|
||||
trace!("request uri: {}", request.uri());
|
||||
match request.uri().path() {
|
||||
"/snapshot.tar.bz2" => self.get("snapshot.tar.bz2"),
|
||||
"/genesis.tar.bz2" => self.get("genesis.tar.bz2"),
|
||||
_ => RequestMiddlewareAction::Proceed {
|
||||
|
||||
if self.is_get_path(request.uri().path()) {
|
||||
self.get(request.uri().path())
|
||||
} else {
|
||||
RequestMiddlewareAction::Proceed {
|
||||
should_continue_on_invalid_cors: false,
|
||||
request,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -234,4 +250,25 @@ mod tests {
|
|||
rpc_service.exit();
|
||||
rpc_service.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_get_path() {
|
||||
let rrm = RpcRequestMiddleware::new(PathBuf::from("/"));
|
||||
|
||||
assert!(rrm.is_get_path("/genesis.tar.bz2"));
|
||||
assert!(!rrm.is_get_path("genesis.tar.bz2"));
|
||||
|
||||
assert!(!rrm.is_get_path("/snapshot.tar.bz2"));
|
||||
|
||||
assert!(
|
||||
rrm.is_get_path("/snapshot-100-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.bz2")
|
||||
);
|
||||
assert!(!rrm.is_get_path(
|
||||
"/snapshot-notaslotnumber-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.bz2"
|
||||
));
|
||||
|
||||
assert!(!rrm.is_get_path("/"));
|
||||
assert!(!rrm.is_get_path(".."));
|
||||
assert!(!rrm.is_get_path("🎣"));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -154,8 +154,10 @@ mod tests {
|
|||
}
|
||||
|
||||
// Create a packageable snapshot
|
||||
let output_tar_path =
|
||||
snapshot_utils::get_snapshot_archive_path(&snapshot_package_output_path);
|
||||
let output_tar_path = snapshot_utils::get_snapshot_archive_path(
|
||||
&snapshot_package_output_path,
|
||||
&(42, Hash::default()),
|
||||
);
|
||||
let snapshot_package = SnapshotPackage::new(
|
||||
5,
|
||||
vec![],
|
||||
|
|
|
|||
|
|
@ -915,7 +915,10 @@ fn main() {
|
|||
snapshot_utils::package_snapshot(
|
||||
&bank,
|
||||
&slot_snapshot_paths,
|
||||
snapshot_utils::get_snapshot_archive_path(output_directory),
|
||||
snapshot_utils::get_snapshot_archive_path(
|
||||
output_directory,
|
||||
&(bank.slot(), bank.hash()),
|
||||
),
|
||||
&temp_dir,
|
||||
&bank.src.roots(),
|
||||
storages,
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ rand = "0.6.5"
|
|||
rand_chacha = "0.1.1"
|
||||
rayon = "1.2.0"
|
||||
reed-solomon-erasure = { package = "solana-reed-solomon-erasure", version = "4.0.1-3", features = ["simd-accel"] }
|
||||
regex = "1.3.4"
|
||||
serde = "1.0.104"
|
||||
serde_bytes = "0.11.3"
|
||||
solana-client = { path = "../client", version = "1.1.0" }
|
||||
|
|
|
|||
|
|
@ -192,7 +192,10 @@ impl BankForks {
|
|||
root,
|
||||
&root_bank.src.roots(),
|
||||
snapshot_package_sender.as_ref().unwrap(),
|
||||
snapshot_utils::get_snapshot_archive_path(&config.snapshot_package_output_path),
|
||||
snapshot_utils::get_snapshot_archive_path(
|
||||
&config.snapshot_package_output_path,
|
||||
&(root_bank.slot(), root_bank.hash()),
|
||||
),
|
||||
);
|
||||
if r.is_err() {
|
||||
warn!("Error generating snapshot for bank: {}, err: {:?}", root, r);
|
||||
|
|
|
|||
|
|
@ -52,40 +52,44 @@ pub fn load(
|
|||
fs::create_dir_all(&snapshot_config.snapshot_path)
|
||||
.expect("Couldn't create snapshot directory");
|
||||
|
||||
let tar = snapshot_utils::get_snapshot_archive_path(
|
||||
match snapshot_utils::get_highest_snapshot_archive_path(
|
||||
&snapshot_config.snapshot_package_output_path,
|
||||
);
|
||||
if tar.exists() {
|
||||
info!("Loading snapshot package: {:?}", tar);
|
||||
// Fail hard here if snapshot fails to load, don't silently continue
|
||||
) {
|
||||
Some(tar) => {
|
||||
if tar.exists() {
|
||||
info!("Loading snapshot package: {:?}", tar);
|
||||
// Fail hard here if snapshot fails to load, don't silently continue
|
||||
|
||||
if account_paths.is_empty() {
|
||||
panic!("Account paths not present when booting from snapshot")
|
||||
if account_paths.is_empty() {
|
||||
panic!("Account paths not present when booting from snapshot")
|
||||
}
|
||||
|
||||
let deserialized_bank = snapshot_utils::bank_from_archive(
|
||||
&account_paths,
|
||||
&snapshot_config.snapshot_path,
|
||||
&tar,
|
||||
)
|
||||
.expect("Load from snapshot failed");
|
||||
|
||||
let snapshot_hash = (
|
||||
deserialized_bank.slot(),
|
||||
deserialized_bank.get_accounts_hash(),
|
||||
);
|
||||
return to_loadresult(
|
||||
blockstore_processor::process_blockstore_from_root(
|
||||
genesis_config,
|
||||
blockstore,
|
||||
Arc::new(deserialized_bank),
|
||||
&process_options,
|
||||
&VerifyRecyclers::default(),
|
||||
),
|
||||
Some(snapshot_hash),
|
||||
);
|
||||
} else {
|
||||
info!("Snapshot package does not exist: {:?}", tar);
|
||||
}
|
||||
}
|
||||
|
||||
let deserialized_bank = snapshot_utils::bank_from_archive(
|
||||
&account_paths,
|
||||
&snapshot_config.snapshot_path,
|
||||
&tar,
|
||||
)
|
||||
.expect("Load from snapshot failed");
|
||||
|
||||
let snapshot_hash = (
|
||||
deserialized_bank.slot(),
|
||||
deserialized_bank.get_accounts_hash(),
|
||||
);
|
||||
return to_loadresult(
|
||||
blockstore_processor::process_blockstore_from_root(
|
||||
genesis_config,
|
||||
blockstore,
|
||||
Arc::new(deserialized_bank),
|
||||
&process_options,
|
||||
&VerifyRecyclers::default(),
|
||||
),
|
||||
Some(snapshot_hash),
|
||||
);
|
||||
} else {
|
||||
info!("Snapshot package does not exist: {:?}", tar);
|
||||
None => info!("No snapshot package available"),
|
||||
}
|
||||
} else {
|
||||
info!("Snapshots disabled");
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ use bincode::serialize_into;
|
|||
use bzip2::bufread::BzDecoder;
|
||||
use fs_extra::dir::CopyOptions;
|
||||
use log::*;
|
||||
use regex::Regex;
|
||||
use solana_measure::measure::Measure;
|
||||
use solana_runtime::{
|
||||
accounts_db::{SnapshotStorage, SnapshotStorages},
|
||||
|
|
@ -11,7 +12,7 @@ use solana_runtime::{
|
|||
MAX_SNAPSHOT_DATA_FILE_SIZE,
|
||||
},
|
||||
};
|
||||
use solana_sdk::clock::Slot;
|
||||
use solana_sdk::{clock::Slot, hash::Hash};
|
||||
use std::{
|
||||
cmp::Ordering,
|
||||
env,
|
||||
|
|
@ -205,6 +206,13 @@ pub fn archive_snapshot_package(snapshot_package: &SnapshotPackage) -> Result<()
|
|||
let metadata = fs::metadata(&archive_path)?;
|
||||
fs::rename(&archive_path, &snapshot_package.tar_output_file)?;
|
||||
|
||||
// Keep around at most two snapshot archives
|
||||
let archives = get_snapshot_archives(snapshot_package.tar_output_file.parent().unwrap());
|
||||
for old_archive in archives.into_iter().skip(2) {
|
||||
fs::remove_file(old_archive.0)
|
||||
.unwrap_or_else(|err| info!("Failed to remove old snapshot: {:}", err));
|
||||
}
|
||||
|
||||
timer.stop();
|
||||
info!(
|
||||
"Successfully created tarball. slot: {}, elapsed ms: {}, size={}",
|
||||
|
|
@ -497,8 +505,59 @@ fn is_snapshot_compression_disabled() -> bool {
|
|||
}
|
||||
}
|
||||
|
||||
pub fn get_snapshot_archive_path<P: AsRef<Path>>(snapshot_output_dir: P) -> PathBuf {
|
||||
snapshot_output_dir.as_ref().join("snapshot.tar.bz2")
|
||||
pub fn get_snapshot_archive_path<P: AsRef<Path>>(
|
||||
snapshot_output_dir: P,
|
||||
snapshot_hash: &(Slot, Hash),
|
||||
) -> PathBuf {
|
||||
snapshot_output_dir.as_ref().join(format!(
|
||||
"snapshot-{}-{}.tar.bz2",
|
||||
snapshot_hash.0, snapshot_hash.1
|
||||
))
|
||||
}
|
||||
|
||||
fn snapshot_hash_of(archive_filename: &str) -> Option<(Slot, Hash)> {
|
||||
let snapshot_filename_regex = Regex::new(r"snapshot-(\d+)-([[:alnum:]]+)\.tar\.bz2$").unwrap();
|
||||
|
||||
if let Some(captures) = snapshot_filename_regex.captures(archive_filename) {
|
||||
let slot_str = captures.get(1).unwrap().as_str();
|
||||
let hash_str = captures.get(2).unwrap().as_str();
|
||||
|
||||
if let (Ok(slot), Ok(hash)) = (slot_str.parse::<Slot>(), hash_str.parse::<Hash>()) {
|
||||
return Some((slot, hash));
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn get_snapshot_archives<P: AsRef<Path>>(snapshot_output_dir: P) -> Vec<(PathBuf, (Slot, Hash))> {
|
||||
let files = fs::read_dir(&snapshot_output_dir)
|
||||
.unwrap_or_else(|err| panic!("Unable to read snapshot directory: {}", err));
|
||||
|
||||
let mut archives: Vec<_> = files
|
||||
.filter_map(|entry| {
|
||||
if let Ok(entry) = entry {
|
||||
let path = entry.path();
|
||||
if path.is_file() {
|
||||
if let Some(snapshot_hash) =
|
||||
snapshot_hash_of(path.file_name().unwrap().to_str().unwrap())
|
||||
{
|
||||
return Some((path, snapshot_hash));
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
})
|
||||
.collect();
|
||||
|
||||
archives.sort_by(|a, b| (b.1).0.cmp(&(a.1).0)); // reverse sort by slot
|
||||
archives
|
||||
}
|
||||
|
||||
pub fn get_highest_snapshot_archive_path<P: AsRef<Path>>(
|
||||
snapshot_output_dir: P,
|
||||
) -> Option<PathBuf> {
|
||||
let archives = get_snapshot_archives(snapshot_output_dir);
|
||||
archives.into_iter().next().map(|archive| archive.0)
|
||||
}
|
||||
|
||||
pub fn untar_snapshot_in<P: AsRef<Path>, Q: AsRef<Path>>(
|
||||
|
|
@ -733,4 +792,13 @@ mod tests {
|
|||
);
|
||||
assert_matches!(result, Err(SnapshotError::IO(ref message)) if message.to_string().starts_with("invalid snapshot data file"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_snapshot_hash_of() {
|
||||
assert_eq!(
|
||||
snapshot_hash_of(&format!("snapshot-42-{}.tar.bz2", Hash::default())),
|
||||
Some((42, Hash::default()))
|
||||
);
|
||||
assert!(snapshot_hash_of("invalid").is_none());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -80,7 +80,7 @@ fn new_spinner_progress_bar() -> ProgressBar {
|
|||
progress_bar
|
||||
}
|
||||
|
||||
fn download_file(url: &str, destination_file: &Path, not_found_ok: bool) -> Result<(), String> {
|
||||
fn download_file(url: &str, destination_file: &Path) -> Result<(), String> {
|
||||
if destination_file.is_file() {
|
||||
return Err(format!("{:?} already exists", destination_file));
|
||||
}
|
||||
|
|
@ -96,15 +96,9 @@ fn download_file(url: &str, destination_file: &Path, not_found_ok: bool) -> Resu
|
|||
let client = reqwest::blocking::Client::new();
|
||||
let response = client.get(url).send().map_err(|err| err.to_string())?;
|
||||
|
||||
if response.status() == reqwest::StatusCode::NOT_FOUND && not_found_ok {
|
||||
progress_bar.finish_and_clear();
|
||||
info!("Archive not found at {}", url);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let response = response
|
||||
.error_for_status()
|
||||
.map_err(|err| format!("Unable to get: {:?}", err))?;
|
||||
.map_err(|err| format!("Unable to download {}: {}", url, err))?;
|
||||
let download_size = {
|
||||
response
|
||||
.headers()
|
||||
|
|
@ -460,7 +454,6 @@ fn download_genesis(
|
|||
download_file(
|
||||
&format!("http://{}/{}", rpc_addr, "genesis.tar.bz2"),
|
||||
&tmp_genesis_package,
|
||||
false,
|
||||
)?;
|
||||
extract_archive(&tmp_genesis_package, &ledger_path)?;
|
||||
|
||||
|
|
@ -496,28 +489,22 @@ fn download_genesis(
|
|||
fn download_snapshot(
|
||||
rpc_addr: &SocketAddr,
|
||||
ledger_path: &Path,
|
||||
snapshot_hash: Option<(Slot, Hash)>,
|
||||
snapshot_hash: (Slot, Hash),
|
||||
) -> Result<(), String> {
|
||||
if snapshot_hash.is_none() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let snapshot_package = solana_ledger::snapshot_utils::get_snapshot_archive_path(ledger_path);
|
||||
let snapshot_package =
|
||||
solana_ledger::snapshot_utils::get_snapshot_archive_path(ledger_path, &snapshot_hash);
|
||||
if snapshot_package.exists() {
|
||||
fs::remove_file(&snapshot_package)
|
||||
.map_err(|err| format!("error removing {:?}: {}", snapshot_package, err))?;
|
||||
Ok(())
|
||||
} else {
|
||||
download_file(
|
||||
&format!(
|
||||
"http://{}/{}",
|
||||
rpc_addr,
|
||||
snapshot_package.file_name().unwrap().to_str().unwrap()
|
||||
),
|
||||
&snapshot_package,
|
||||
)
|
||||
}
|
||||
download_file(
|
||||
&format!(
|
||||
"http://{}/{}",
|
||||
rpc_addr,
|
||||
snapshot_package.file_name().unwrap().to_str().unwrap()
|
||||
),
|
||||
&snapshot_package,
|
||||
true,
|
||||
)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// This function is duplicated in ledger-tool/src/main.rs...
|
||||
|
|
@ -1144,7 +1131,13 @@ pub fn main() {
|
|||
}
|
||||
Ok(())
|
||||
})
|
||||
.and_then(|_| download_snapshot(&rpc_contact_info.rpc, &ledger_path, snapshot_hash))
|
||||
.and_then(|_| {
|
||||
if let Some(snapshot_hash) = snapshot_hash {
|
||||
download_snapshot(&rpc_contact_info.rpc, &ledger_path, snapshot_hash)
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
})
|
||||
.and_then(|_| {
|
||||
if !validator_config.voting_disabled && !no_check_vote_account {
|
||||
check_vote_account(
|
||||
|
|
|
|||
Loading…
Reference in New Issue