blockworks-foundation
/
solana-with-rpc-optimizations
mirror of https://github.com/blockworks-foundation/solana-with-rpc-optimizations.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Refactor genesis download/load/check functions (#17276) 

* Refactor genesis ingest functions

* Consolidate genesis.bin/genesis.tar.bz2 references
This commit is contained in:
sakridge 2021-05-24 16:45:36 +02:00 committed by GitHub
parent 9d112cf41f
commit a8dca3976b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 158 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
Cargo.lock generated
View File

@ -4549,6 +4549,15 @@ dependencies = [
"tempfile",
]
[[package]]
name = "solana-genesis-utils"
version = "1.7.0"
dependencies = [
"solana-download-utils",
"solana-runtime",
"solana-sdk",
]
[[package]]
name = "solana-gossip"
version = "1.7.0"
@ -5442,6 +5451,7 @@ dependencies = [
"solana-core",
"solana-download-utils",
"solana-faucet",
"solana-genesis-utils",
"solana-ledger",
"solana-logger 1.7.0",
"solana-metrics",

1
Cargo.toml
View File

@ -21,6 +21,7 @@ members = [
"perf",
"validator",
"genesis",
"genesis-utils",
"gossip",
"install",
"keygen",

26
core/src/rpc_service.rs
View File

@ -26,7 +26,10 @@ use solana_runtime::{
commitment::BlockCommitmentCache,
snapshot_utils,
};
use solana_sdk::{hash::Hash, native_token::lamports_to_sol, pubkey::Pubkey};
use solana_sdk::{
genesis_config::DEFAULT_GENESIS_DOWNLOAD_PATH, hash::Hash, native_token::lamports_to_sol,
pubkey::Pubkey,
};
use std::{
collections::HashSet,
net::SocketAddr,
@ -101,7 +104,7 @@ impl RpcRequestMiddleware {
fn is_file_get_path(&self, path: &str) -> bool {
match path {
"/genesis.tar.bz2" => true,
DEFAULT_GENESIS_DOWNLOAD_PATH => true,
_ => {
if self.snapshot_config.is_some() {
self.snapshot_archive_path_regex.is_match(path)
@ -136,7 +139,7 @@ impl RpcRequestMiddleware {
let stem = path.split_at(1).1; // Drop leading '/' from path
let filename = {
match path {
"/genesis.tar.bz2" => {
DEFAULT_GENESIS_DOWNLOAD_PATH => {
inc_new_counter_info!("rpc-get_genesis", 1);
self.ledger_path.join(stem)
}
@ -488,7 +491,10 @@ mod tests {
bank::Bank, bank_forks::ArchiveFormat, snapshot_utils::SnapshotVersion,
snapshot_utils::DEFAULT_MAX_SNAPSHOTS_TO_RETAIN,
};
use solana_sdk::{genesis_config::ClusterType, signature::Signer};
use solana_sdk::{
genesis_config::{ClusterType, DEFAULT_GENESIS_ARCHIVE},
signature::Signer,
};
use std::io::Write;
use std::net::{IpAddr, Ipv4Addr};
@ -592,8 +598,8 @@ mod tests {
RpcHealth::stub(),
);
assert!(rrm.is_file_get_path("/genesis.tar.bz2"));
assert!(!rrm.is_file_get_path("genesis.tar.bz2"));
assert!(rrm.is_file_get_path(DEFAULT_GENESIS_DOWNLOAD_PATH));
assert!(!rrm.is_file_get_path(DEFAULT_GENESIS_ARCHIVE));
assert!(!rrm.is_file_get_path("/snapshot.tar.bz2")); // This is a redirect
@ -629,7 +635,7 @@ mod tests {
let ledger_path = get_tmp_ledger_path!();
std::fs::create_dir(&ledger_path).unwrap();
let genesis_path = ledger_path.join("genesis.tar.bz2");
let genesis_path = ledger_path.join(DEFAULT_GENESIS_ARCHIVE);
let rrm = RpcRequestMiddleware::new(
ledger_path.clone(),
None,
@ -638,7 +644,7 @@ mod tests {
);
// File does not exist => request should fail.
let action = rrm.process_file_get("/genesis.tar.bz2");
let action = rrm.process_file_get(DEFAULT_GENESIS_DOWNLOAD_PATH);
if let RequestMiddlewareAction::Respond { response, .. } = action {
let response = runtime.block_on(response);
let response = response.unwrap();
@ -653,7 +659,7 @@ mod tests {
}
// Normal file exist => request should succeed.
let action = rrm.process_file_get("/genesis.tar.bz2");
let action = rrm.process_file_get(DEFAULT_GENESIS_DOWNLOAD_PATH);
if let RequestMiddlewareAction::Respond { response, .. } = action {
let response = runtime.block_on(response);
let response = response.unwrap();
@ -672,7 +678,7 @@ mod tests {
symlink::symlink_file("wrong", &genesis_path).unwrap();
// File is a symbolic link => request should fail.
let action = rrm.process_file_get("/genesis.tar.bz2");
let action = rrm.process_file_get(DEFAULT_GENESIS_DOWNLOAD_PATH);
if let RequestMiddlewareAction::Respond { response, .. } = action {
let response = runtime.block_on(response);
let response = response.unwrap();

7
download-utils/src/lib.rs
View File

@ -3,8 +3,7 @@ use console::Emoji;
use indicatif::{ProgressBar, ProgressStyle};
use log::*;
use solana_runtime::{bank_forks::ArchiveFormat, snapshot_utils};
use solana_sdk::clock::Slot;
use solana_sdk::hash::Hash;
use solana_sdk::{clock::Slot, genesis_config::DEFAULT_GENESIS_ARCHIVE, hash::Hash};
use std::fs::{self, File};
use std::io;
use std::io::Read;
@ -158,11 +157,11 @@ pub fn download_genesis_if_missing(
) -> Result<PathBuf, String> {
if !genesis_package.exists() {
let tmp_genesis_path = genesis_package.parent().unwrap().join("tmp-genesis");
let tmp_genesis_package = tmp_genesis_path.join("genesis.tar.bz2");
let tmp_genesis_package = tmp_genesis_path.join(DEFAULT_GENESIS_ARCHIVE);
let _ignored = fs::remove_dir_all(&tmp_genesis_path);
download_file(
&format!("http://{}/{}", rpc_addr, "genesis.tar.bz2"),
&format!("http://{}/{}", rpc_addr, DEFAULT_GENESIS_ARCHIVE),
&tmp_genesis_package,
use_progress_bar,
)?;

22
genesis-utils/Cargo.toml Normal file
View File

@ -0,0 +1,22 @@
[package]
name = "solana-genesis-utils"
version = "1.7.0"
description = "Solana Genesis Utils"
authors = ["Solana Maintainers <maintainers@solana.foundation>"]
repository = "https://github.com/solana-labs/solana"
license = "Apache-2.0"
homepage = "https://solana.com/"
documentation = "https://docs.rs/solana-download-utils"
edition = "2018"
[dependencies]
solana-sdk = { path = "../sdk", version = "=1.7.0" }
solana-download-utils = { path = "../download-utils", version = "=1.7.0" }
solana-runtime = { path = "../runtime", version = "=1.7.0" }
[lib]
crate-type = ["lib"]
name = "solana_genesis_utils"
[package.metadata.docs.rs]
targets = ["x86_64-unknown-linux-gnu"]

75
genesis-utils/src/lib.rs Normal file
View File

@ -0,0 +1,75 @@
use solana_download_utils::download_genesis_if_missing;
use solana_runtime::hardened_unpack::unpack_genesis_archive;
use solana_sdk::{
genesis_config::{GenesisConfig, DEFAULT_GENESIS_ARCHIVE},
hash::Hash,
};
use std::net::SocketAddr;
fn check_genesis_hash(
genesis_config: &GenesisConfig,
expected_genesis_hash: Option<Hash>,
) -> Result<(), String> {
let genesis_hash = genesis_config.hash();
if let Some(expected_genesis_hash) = expected_genesis_hash {
if expected_genesis_hash != genesis_hash {
return Err(format!(
"Genesis hash mismatch: expected {} but downloaded genesis hash is {}",
expected_genesis_hash, genesis_hash,
));
}
}
Ok(())
}
fn load_local_genesis(
ledger_path: &std::path::Path,
expected_genesis_hash: Option<Hash>,
) -> Result<GenesisConfig, String> {
let existing_genesis = GenesisConfig::load(&ledger_path)
.map_err(|err| format!("Failed to load genesis config: {}", err))?;
check_genesis_hash(&existing_genesis, expected_genesis_hash)?;
Ok(existing_genesis)
}
pub fn download_then_check_genesis_hash(
rpc_addr: &SocketAddr,
ledger_path: &std::path::Path,
expected_genesis_hash: Option<Hash>,
max_genesis_archive_unpacked_size: u64,
no_genesis_fetch: bool,
use_progress_bar: bool,
) -> Result<GenesisConfig, String> {
if no_genesis_fetch {
let genesis_config = load_local_genesis(ledger_path, expected_genesis_hash)?;
return Ok(genesis_config);
}
let genesis_package = ledger_path.join(DEFAULT_GENESIS_ARCHIVE);
let genesis_config = if let Ok(tmp_genesis_package) =
download_genesis_if_missing(rpc_addr, &genesis_package, use_progress_bar)
{
unpack_genesis_archive(
&tmp_genesis_package,
&ledger_path,
max_genesis_archive_unpacked_size,
)
.map_err(|err| format!("Failed to unpack downloaded genesis config: {}", err))?;
let downloaded_genesis = GenesisConfig::load(&ledger_path)
.map_err(|err| format!("Failed to load downloaded genesis config: {}", err))?;
check_genesis_hash(&downloaded_genesis, expected_genesis_hash)?;
std::fs::rename(tmp_genesis_package, genesis_package)
.map_err(|err| format!("Unable to rename: {:?}", err))?;
downloaded_genesis
} else {
load_local_genesis(ledger_path, expected_genesis_hash)?
};
Ok(genesis_config)
}

24
ledger/src/blockstore.rs
View File

@ -28,7 +28,7 @@ use solana_rayon_threadlimit::get_thread_count;
use solana_runtime::hardened_unpack::{unpack_genesis_archive, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE};
use solana_sdk::{
clock::{Slot, UnixTimestamp, DEFAULT_TICKS_PER_SECOND, MS_PER_TICK},
genesis_config::GenesisConfig,
genesis_config::{GenesisConfig, DEFAULT_GENESIS_ARCHIVE, DEFAULT_GENESIS_FILE},
hash::Hash,
pubkey::Pubkey,
sanitize::Sanitize,
@ -3434,13 +3434,13 @@ pub fn create_new_ledger(
// Explicitly close the blockstore before we create the archived genesis file
drop(blockstore);
let archive_path = ledger_path.join("genesis.tar.bz2");
let archive_path = ledger_path.join(DEFAULT_GENESIS_ARCHIVE);
let args = vec![
"jcfhS",
archive_path.to_str().unwrap(),
"-C",
ledger_path.to_str().unwrap(),
"genesis.bin",
DEFAULT_GENESIS_FILE,
"rocksdb",
];
let output = std::process::Command::new("tar")
@ -3478,18 +3478,24 @@ pub fn create_new_ledger(
let mut error_messages = String::new();
fs::rename(
&ledger_path.join("genesis.tar.bz2"),
ledger_path.join("genesis.tar.bz2.failed"),
&ledger_path.join(DEFAULT_GENESIS_ARCHIVE),
ledger_path.join(format!("{}.failed", DEFAULT_GENESIS_ARCHIVE)),
)
.unwrap_or_else(|e| {
error_messages += &format!("/failed to stash problematic genesis.tar.bz2: {}", e)
error_messages += &format!(
"/failed to stash problematic {}: {}",
DEFAULT_GENESIS_ARCHIVE, e
)
});
fs::rename(
&ledger_path.join("genesis.bin"),
ledger_path.join("genesis.bin.failed"),
&ledger_path.join(DEFAULT_GENESIS_FILE),
ledger_path.join(format!("{}.failed", DEFAULT_GENESIS_FILE)),
)
.unwrap_or_else(|e| {
error_messages += &format!("/failed to stash problematic genesis.bin: {}", e)
error_messages += &format!(
"/failed to stash problematic {}: {}",
DEFAULT_GENESIS_FILE, e
)
});
fs::rename(
&ledger_path.join("rocksdb"),

7
runtime/src/hardened_unpack.rs
View File

@ -1,3 +1,4 @@
use solana_sdk::genesis_config::{DEFAULT_GENESIS_ARCHIVE, DEFAULT_GENESIS_FILE};
use {
bzip2::bufread::BzDecoder,
log::*,
@ -280,7 +281,7 @@ pub fn open_genesis_config(
max_genesis_archive_unpacked_size: u64,
) -> GenesisConfig {
GenesisConfig::load(&ledger_path).unwrap_or_else(|load_err| {
let genesis_package = ledger_path.join("genesis.tar.bz2");
let genesis_package = ledger_path.join(DEFAULT_GENESIS_ARCHIVE);
unpack_genesis_archive(
&genesis_package,
ledger_path,
@ -348,8 +349,8 @@ fn is_valid_genesis_archive_entry(parts: &[&str], kind: tar::EntryType) -> bool
trace!("validating: {:?} {:?}", parts, kind);
#[allow(clippy::match_like_matches_macro)]
match (parts, kind) {
(["genesis.bin"], GNUSparse) => true,
(["genesis.bin"], Regular) => true,
([DEFAULT_GENESIS_FILE], GNUSparse) => true,
([DEFAULT_GENESIS_FILE], Regular) => true,
(["rocksdb"], Directory) => true,
(["rocksdb", _], GNUSparse) => true,
(["rocksdb", _], Regular) => true,

6
sdk/src/genesis_config.rs
View File

@ -32,6 +32,10 @@ use std::{
time::{SystemTime, UNIX_EPOCH},
};
pub const DEFAULT_GENESIS_FILE: &str = "genesis.bin";
pub const DEFAULT_GENESIS_ARCHIVE: &str = "genesis.tar.bz2";
pub const DEFAULT_GENESIS_DOWNLOAD_PATH: &str = "/genesis.tar.bz2";
// deprecated default that is no longer used
pub const UNUSED_DEFAULT: u64 = 1024;
@ -152,7 +156,7 @@ impl GenesisConfig {
}
fn genesis_filename(ledger_path: &Path) -> PathBuf {
Path::new(ledger_path).join("genesis.bin")
Path::new(ledger_path).join(DEFAULT_GENESIS_FILE)
}
pub fn load(ledger_path: &Path) -> Result<Self, std::io::Error> {

1
validator/Cargo.toml
View File

@ -34,6 +34,7 @@ solana-client = { path = "../client", version = "=1.7.0" }
solana-core = { path = "../core", version = "=1.7.0" }
solana-download-utils = { path = "../download-utils", version = "=1.7.0" }
solana-faucet = { path = "../faucet", version = "=1.7.0" }
solana-genesis-utils = { path = "../genesis-utils", version = "=1.7.0" }
solana-ledger = { path = "../ledger", version = "=1.7.0" }
solana-logger = { path = "../logger", version = "=1.7.0" }
solana-metrics = { path = "../metrics", version = "=1.7.0" }

79
validator/src/main.rs
View File

@ -34,7 +34,8 @@ use {
is_snapshot_config_invalid, Validator, ValidatorConfig, ValidatorStartProgress,
},
},
solana_download_utils::{download_genesis_if_missing, download_snapshot},
solana_download_utils::download_snapshot,
solana_genesis_utils::download_then_check_genesis_hash,
solana_ledger::blockstore_db::BlockstoreRecoveryMode,
solana_perf::recycler::enable_recycler_warming,
solana_rpc::rpc_pubsub_service::PubSubConfig,
@ -43,13 +44,12 @@ use {
AccountIndex, AccountSecondaryIndexes, AccountSecondaryIndexesIncludeExclude,
},
bank_forks::{ArchiveFormat, SnapshotConfig, SnapshotVersion},
hardened_unpack::{unpack_genesis_archive, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE},
hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
snapshot_utils::{get_highest_snapshot_archive_path, DEFAULT_MAX_SNAPSHOTS_TO_RETAIN},
},
solana_sdk::{
clock::{Slot, DEFAULT_S_PER_SLOT},
commitment_config::CommitmentConfig,
genesis_config::GenesisConfig,
hash::Hash,
pubkey::Pubkey,
signature::{Keypair, Signer},
@ -648,74 +648,6 @@ fn validators_set(
}
}
fn check_genesis_hash(
genesis_config: &GenesisConfig,
expected_genesis_hash: Option<Hash>,
) -> Result<(), String> {
let genesis_hash = genesis_config.hash();
if let Some(expected_genesis_hash) = expected_genesis_hash {
if expected_genesis_hash != genesis_hash {
return Err(format!(
"Genesis hash mismatch: expected {} but downloaded genesis hash is {}",
expected_genesis_hash, genesis_hash,
));
}
}
Ok(())
}
fn load_local_genesis(
ledger_path: &std::path::Path,
expected_genesis_hash: Option<Hash>,
) -> Result<GenesisConfig, String> {
let existing_genesis = GenesisConfig::load(&ledger_path)
.map_err(|err| format!("Failed to load genesis config: {}", err))?;
check_genesis_hash(&existing_genesis, expected_genesis_hash)?;
Ok(existing_genesis)
}
fn download_then_check_genesis_hash(
rpc_addr: &SocketAddr,
ledger_path: &std::path::Path,
expected_genesis_hash: Option<Hash>,
max_genesis_archive_unpacked_size: u64,
no_genesis_fetch: bool,
use_progress_bar: bool,
) -> Result<Hash, String> {
if no_genesis_fetch {
let genesis_config = load_local_genesis(ledger_path, expected_genesis_hash)?;
return Ok(genesis_config.hash());
}
let genesis_package = ledger_path.join("genesis.tar.bz2");
let genesis_config = if let Ok(tmp_genesis_package) =
download_genesis_if_missing(rpc_addr, &genesis_package, use_progress_bar)
{
unpack_genesis_archive(
&tmp_genesis_package,
&ledger_path,
max_genesis_archive_unpacked_size,
)
.map_err(|err| format!("Failed to unpack downloaded genesis config: {}", err))?;
let downloaded_genesis = GenesisConfig::load(&ledger_path)
.map_err(|err| format!("Failed to load downloaded genesis config: {}", err))?;
check_genesis_hash(&downloaded_genesis, expected_genesis_hash)?;
std::fs::rename(tmp_genesis_package, genesis_package)
.map_err(|err| format!("Unable to rename: {:?}", err))?;
downloaded_genesis
} else {
load_local_genesis(ledger_path, expected_genesis_hash)?
};
Ok(genesis_config.hash())
}
fn verify_reachable_ports(
node: &Node,
cluster_entrypoint: &ContactInfo,
@ -872,7 +804,7 @@ fn rpc_bootstrap(
Err(err) => Err(format!("Failed to get RPC node version: {}", err)),
}
.and_then(|_| {
let genesis_hash = download_then_check_genesis_hash(
let genesis_config = download_then_check_genesis_hash(
&rpc_contact_info.rpc,
&ledger_path,
validator_config.expected_genesis_hash,
@ -881,7 +813,8 @@ fn rpc_bootstrap(
use_progress_bar,
);
if let Ok(genesis_hash) = genesis_hash {
if let Ok(genesis_config) = genesis_config {
let genesis_hash = genesis_config.hash();
if validator_config.expected_genesis_hash.is_none() {
info!("Expected genesis hash set to {}", genesis_hash);
validator_config.expected_genesis_hash = Some(genesis_hash);