diff --git a/Cargo.lock b/Cargo.lock index 90b78bebb..96687cf02 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3576,6 +3576,7 @@ dependencies = [ "solana-clap-utils", "solana-genesis-programs", "solana-ledger", + "solana-logger", "solana-sdk", "solana-stake-program", "solana-storage-program", diff --git a/core/src/validator.rs b/core/src/validator.rs index 8bf9d7182..a56bb689d 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -30,7 +30,7 @@ use solana_ledger::{ blockstore::{Blockstore, CompletedSlotsReceiver}, blockstore_processor::{self, BankForksInfo}, create_new_tmp_ledger, - hardened_unpack::open_genesis_config, + hardened_unpack::{open_genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE}, leader_schedule::FixedSchedule, leader_schedule_cache::LeaderScheduleCache, }; @@ -82,6 +82,7 @@ pub struct ValidatorConfig { pub frozen_accounts: Vec, pub no_rocksdb_compaction: bool, pub accounts_hash_interval_slots: u64, + pub max_genesis_archive_unpacked_size: u64, } impl Default for ValidatorConfig { @@ -109,6 +110,7 @@ impl Default for ValidatorConfig { frozen_accounts: vec![], no_rocksdb_compaction: false, accounts_hash_interval_slots: std::u64::MAX, + max_genesis_archive_unpacked_size: MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, } } } @@ -569,7 +571,8 @@ fn new_banks_from_blockstore( LeaderScheduleCache, Option<(Slot, Hash)>, ) { - let genesis_config = open_genesis_config(blockstore_path); + let genesis_config = + open_genesis_config(blockstore_path, config.max_genesis_archive_unpacked_size); // This needs to be limited otherwise the state in the VoteAccount data // grows too large diff --git a/genesis/Cargo.toml b/genesis/Cargo.toml index eced6afe0..c18a3129e 100644 --- a/genesis/Cargo.toml +++ b/genesis/Cargo.toml @@ -18,6 +18,7 @@ serde_yaml = "0.8.11" solana-clap-utils = { path = "../clap-utils", version = "1.2.0" } solana-genesis-programs = { path = "../genesis-programs", version = "1.2.0" } solana-ledger = { path = "../ledger", version = "1.2.0" } +solana-logger = { path = "../logger", version = "1.2.0" } solana-sdk = { path = "../sdk", version = "1.2.0" } solana-stake-program = { path = "../programs/stake", version = "1.2.0" } solana-storage-program = { path = "../programs/storage", version = "1.2.0" } diff --git a/genesis/src/main.rs b/genesis/src/main.rs index e948c3787..8da1b8a85 100644 --- a/genesis/src/main.rs +++ b/genesis/src/main.rs @@ -6,7 +6,10 @@ use solana_clap_utils::{ input_validators::{is_pubkey_or_keypair, is_rfc3339_datetime, is_valid_percentage}, }; use solana_genesis::{genesis_accounts::add_genesis_accounts, Base64Account}; -use solana_ledger::{blockstore::create_new_ledger, poh::compute_hashes_per_tick}; +use solana_ledger::{ + blockstore::create_new_ledger, hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, + poh::compute_hashes_per_tick, +}; use solana_sdk::{ account::Account, clock, @@ -121,6 +124,7 @@ fn main() -> Result<(), Box> { timing::duration_as_us(&PohConfig::default().target_tick_duration); let default_ticks_per_slot = &clock::DEFAULT_TICKS_PER_SLOT.to_string(); let default_operating_mode = "stable"; + let default_genesis_archive_unpacked_size = MAX_GENESIS_ARCHIVE_UNPACKED_SIZE.to_string(); let matches = App::new(crate_name!()) .about(crate_description!()) @@ -327,6 +331,16 @@ fn main() -> Result<(), Box> { "Selects the features that will be enabled for the cluster" ), ) + .arg( + Arg::with_name("max_genesis_archive_unpacked_size") + .long("max-genesis-archive-unpacked-size") + .value_name("NUMBER") + .takes_value(true) + .default_value(&default_genesis_archive_unpacked_size) + .help( + "maximum total uncompressed file size of created genesis archive", + ), + ) .get_matches(); let faucet_lamports = value_t!(matches, "faucet_lamports", u64).unwrap_or(0); @@ -513,6 +527,9 @@ fn main() -> Result<(), Box> { } } + let max_genesis_archive_unpacked_size = + value_t_or_exit!(matches, "max_genesis_archive_unpacked_size", u64); + let issued_lamports = genesis_config .accounts .iter() @@ -521,7 +538,12 @@ fn main() -> Result<(), Box> { add_genesis_accounts(&mut genesis_config, issued_lamports - faucet_lamports); - create_new_ledger(&ledger_path, &genesis_config)?; + solana_logger::setup(); + create_new_ledger( + &ledger_path, + &genesis_config, + max_genesis_archive_unpacked_size, + )?; println!("{}", genesis_config); Ok(()) diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index 3018539df..4f0d73da2 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -11,7 +11,7 @@ use solana_ledger::{ blockstore::Blockstore, blockstore_db::{self, Column, Database}, blockstore_processor::{BankForksInfo, ProcessOptions}, - hardened_unpack::open_genesis_config, + hardened_unpack::{open_genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE}, rooted_slot_iterator::RootedSlotIterator, snapshot_utils, }; @@ -575,6 +575,12 @@ fn load_bank_forks( ) } +fn open_genesis_config_by(ledger_path: &Path, matches: &ArgMatches<'_>) -> GenesisConfig { + let max_genesis_archive_unpacked_size = + value_t_or_exit!(matches, "max_genesis_archive_unpacked_size", u64); + open_genesis_config(ledger_path, max_genesis_archive_unpacked_size) +} + #[allow(clippy::cognitive_complexity)] fn main() { const DEFAULT_ROOT_COUNT: &str = "1"; @@ -612,6 +618,13 @@ fn main() { .long("allow-dead-slots") .takes_value(false) .help("Output dead slots as well"); + let default_genesis_archive_unpacked_size = MAX_GENESIS_ARCHIVE_UNPACKED_SIZE.to_string(); + let max_genesis_archive_unpacked_size_arg = Arg::with_name("max_genesis_archive_unpacked_size") + .long("max-genesis-archive-unpacked-size") + .value_name("NUMBER") + .takes_value(true) + .default_value(&default_genesis_archive_unpacked_size) + .help("maximum total uncompressed size of unpacked genesis archive"); let matches = App::new(crate_name!()) .about(crate_description!()) @@ -663,15 +676,18 @@ fn main() { .subcommand( SubCommand::with_name("genesis") .about("Prints the ledger's genesis config") + .arg(&max_genesis_archive_unpacked_size_arg) ) .subcommand( SubCommand::with_name("genesis-hash") .about("Prints the ledger's genesis hash") + .arg(&max_genesis_archive_unpacked_size_arg) ) .subcommand( SubCommand::with_name("shred-version") .about("Prints the ledger's shred hash") .arg(&hard_forks_arg) + .arg(&max_genesis_archive_unpacked_size_arg) ) .subcommand( SubCommand::with_name("bounds") @@ -696,6 +712,7 @@ fn main() { .arg(&account_paths_arg) .arg(&halt_at_slot_arg) .arg(&hard_forks_arg) + .arg(&max_genesis_archive_unpacked_size_arg) .arg( Arg::with_name("skip_poh_verify") .long("skip-poh-verify") @@ -709,6 +726,7 @@ fn main() { .arg(&account_paths_arg) .arg(&halt_at_slot_arg) .arg(&hard_forks_arg) + .arg(&max_genesis_archive_unpacked_size_arg) .arg( Arg::with_name("include_all_votes") .long("include-all-votes") @@ -727,6 +745,7 @@ fn main() { .arg(&no_snapshot_arg) .arg(&account_paths_arg) .arg(&hard_forks_arg) + .arg(&max_genesis_archive_unpacked_size_arg) .arg( Arg::with_name("snapshot_slot") .index(1) @@ -755,6 +774,7 @@ fn main() { .takes_value(false) .help("Include sysvars too"), ) + .arg(&max_genesis_archive_unpacked_size_arg) ).subcommand( SubCommand::with_name("prune") .about("Prune the ledger from a yaml file containing a list of slots to prune.") @@ -847,11 +867,14 @@ fn main() { LedgerOutputMethod::Print, ); } - ("genesis", Some(_arg_matches)) => { - println!("{}", open_genesis_config(&ledger_path)); + ("genesis", Some(arg_matches)) => { + println!("{}", open_genesis_config_by(&ledger_path, arg_matches)); } - ("genesis-hash", Some(_arg_matches)) => { - println!("{}", open_genesis_config(&ledger_path).hash()); + ("genesis-hash", Some(arg_matches)) => { + println!( + "{}", + open_genesis_config_by(&ledger_path, arg_matches).hash() + ); } ("shred-version", Some(arg_matches)) => { let process_options = ProcessOptions { @@ -860,7 +883,7 @@ fn main() { poh_verify: false, ..ProcessOptions::default() }; - let genesis_config = open_genesis_config(&ledger_path); + let genesis_config = open_genesis_config_by(&ledger_path, arg_matches); match load_bank_forks(arg_matches, &ledger_path, &genesis_config, process_options) { Ok((bank_forks, bank_forks_info, _leader_schedule_cache, _snapshot_hash)) => { let bank_info = &bank_forks_info[0]; @@ -923,12 +946,15 @@ fn main() { poh_verify: !arg_matches.is_present("skip_poh_verify"), ..ProcessOptions::default() }; - println!("{}", open_genesis_config(&ledger_path).hash()); + println!( + "genesis hash: {}", + open_genesis_config_by(&ledger_path, arg_matches).hash() + ); load_bank_forks( arg_matches, &ledger_path, - &open_genesis_config(&ledger_path), + &open_genesis_config_by(&ledger_path, arg_matches), process_options, ) .unwrap_or_else(|err| { @@ -950,7 +976,7 @@ fn main() { match load_bank_forks( arg_matches, &ledger_path, - &open_genesis_config(&ledger_path), + &open_genesis_config_by(&ledger_path, arg_matches), process_options, ) { Ok((bank_forks, bank_forks_info, _leader_schedule_cache, _snapshot_hash)) => { @@ -991,7 +1017,7 @@ fn main() { poh_verify: false, ..ProcessOptions::default() }; - let genesis_config = open_genesis_config(&ledger_path); + let genesis_config = open_genesis_config_by(&ledger_path, arg_matches); match load_bank_forks(arg_matches, &ledger_path, &genesis_config, process_options) { Ok((bank_forks, _bank_forks_info, _leader_schedule_cache, _snapshot_hash)) => { let bank = bank_forks.get(snapshot_slot).unwrap_or_else(|| { @@ -1055,7 +1081,7 @@ fn main() { poh_verify: false, ..ProcessOptions::default() }; - let genesis_config = open_genesis_config(&ledger_path); + let genesis_config = open_genesis_config_by(&ledger_path, arg_matches); let include_sysvars = arg_matches.is_present("include_sysvars"); match load_bank_forks(arg_matches, &ledger_path, &genesis_config, process_options) { Ok((bank_forks, bank_forks_info, _leader_schedule_cache, _snapshot_hash)) => { diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index e7c35a8b7..fdbd9b14a 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -10,6 +10,7 @@ use crate::{ blockstore_meta::*, entry::{create_ticks, Entry}, erasure::ErasureConfig, + hardened_unpack::{unpack_genesis_archive, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE}, leader_schedule_cache::LeaderScheduleCache, next_slots_iterator::NextSlotsIterator, shred::{Result as ShredResult, Shred, Shredder}, @@ -45,6 +46,7 @@ use std::{ cmp, collections::HashMap, fs, + io::{Error as IOError, ErrorKind}, path::{Path, PathBuf}, rc::Rc, sync::{ @@ -2622,7 +2624,11 @@ fn calculate_stake_weighted_timestamp( // Creates a new ledger with slot 0 full of ticks (and only ticks). // // Returns the blockhash that can be used to append entries with. -pub fn create_new_ledger(ledger_path: &Path, genesis_config: &GenesisConfig) -> Result { +pub fn create_new_ledger( + ledger_path: &Path, + genesis_config: &GenesisConfig, + max_genesis_archive_unpacked_size: u64, +) -> Result { Blockstore::destroy(ledger_path)?; genesis_config.write(&ledger_path)?; @@ -2658,7 +2664,6 @@ pub fn create_new_ledger(ledger_path: &Path, genesis_config: &GenesisConfig) -> .output() .unwrap(); if !output.status.success() { - use std::io::{Error as IOError, ErrorKind}; use std::str::from_utf8; error!("tar stdout: {}", from_utf8(&output.stdout).unwrap_or("?")); error!("tar stderr: {}", from_utf8(&output.stderr).unwrap_or("?")); @@ -2672,6 +2677,54 @@ pub fn create_new_ledger(ledger_path: &Path, genesis_config: &GenesisConfig) -> ))); } + // ensure the genesis archive can be unpacked and it is under + // max_genesis_archive_unpacked_size, immedately after creating it above. + { + let temp_dir = tempfile::TempDir::new().unwrap(); + // unpack into a temp dir, while completely discarding the unpacked files + let unpack_check = unpack_genesis_archive( + &archive_path, + &temp_dir.into_path(), + max_genesis_archive_unpacked_size, + ); + if let Err(unpack_err) = unpack_check { + // stash problematic original archived genesis related files to + // examine them later and to prevent validator and ledger-tool from + // naively consuming them + let mut error_messages = String::new(); + + fs::rename( + &ledger_path.join("genesis.tar.bz2"), + ledger_path.join("genesis.tar.bz2.failed"), + ) + .unwrap_or_else(|e| { + error_messages += &format!("/failed to stash problematic genesis.tar.bz2: {}", e) + }); + fs::rename( + &ledger_path.join("genesis.bin"), + ledger_path.join("genesis.bin.failed"), + ) + .unwrap_or_else(|e| { + error_messages += &format!("/failed to stash problematic genesis.bin: {}", e) + }); + fs::rename( + &ledger_path.join("rocksdb"), + ledger_path.join("rocksdb.failed"), + ) + .unwrap_or_else(|e| { + error_messages += &format!("/failed to stash problematic rocksdb: {}", e) + }); + + return Err(BlockstoreError::IO(IOError::new( + ErrorKind::Other, + format!( + "Error checking to unpack genesis archive: {}{}", + unpack_err, error_messages + ), + ))); + } + } + Ok(last_hash) } @@ -2739,7 +2792,12 @@ pub fn verify_shred_slots(slot: Slot, parent_slot: Slot, last_root: Slot) -> boo // ticks) pub fn create_new_ledger_from_name(name: &str, genesis_config: &GenesisConfig) -> (PathBuf, Hash) { let ledger_path = get_ledger_path_from_name(name); - let blockhash = create_new_ledger(&ledger_path, genesis_config).unwrap(); + let blockhash = create_new_ledger( + &ledger_path, + genesis_config, + MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, + ) + .unwrap(); (ledger_path, blockhash) } diff --git a/ledger/src/blockstore_db.rs b/ledger/src/blockstore_db.rs index ae94c722f..131ad1eef 100644 --- a/ledger/src/blockstore_db.rs +++ b/ledger/src/blockstore_db.rs @@ -1,4 +1,4 @@ -use crate::blockstore_meta; +use crate::{blockstore_meta, hardened_unpack::UnpackError}; use bincode::{deserialize, serialize}; use byteorder::{BigEndian, ByteOrder}; use log::*; @@ -55,6 +55,7 @@ pub enum BlockstoreError { Serialize(#[from] Box), FsExtraError(#[from] fs_extra::error::Error), SlotCleanedUp, + UnpackError(#[from] UnpackError), } pub type Result = std::result::Result; diff --git a/ledger/src/hardened_unpack.rs b/ledger/src/hardened_unpack.rs index 3ba7d9345..7d1a41aa1 100644 --- a/ledger/src/hardened_unpack.rs +++ b/ledger/src/hardened_unpack.rs @@ -19,9 +19,9 @@ use thiserror::Error; #[derive(Error, Debug)] pub enum UnpackError { - #[error("IO error")] + #[error("IO error: {0}")] IO(#[from] std::io::Error), - #[error("Archive error")] + #[error("Archive error: {0}")] Archive(String), } @@ -29,15 +29,15 @@ pub type Result = std::result::Result; const MAX_SNAPSHOT_ARCHIVE_UNPACKED_SIZE: u64 = 500 * 1024 * 1024 * 1024; // 500 GiB const MAX_SNAPSHOT_ARCHIVE_UNPACKED_COUNT: u64 = 500_000; -const MAX_GENESIS_ARCHIVE_UNPACKED_SIZE: u64 = 1024 * 1024 * 1024; // 1024 MiB +pub const MAX_GENESIS_ARCHIVE_UNPACKED_SIZE: u64 = 10 * 1024 * 1024; // 10 MiB const MAX_GENESIS_ARCHIVE_UNPACKED_COUNT: u64 = 100; fn checked_total_size_sum(total_size: u64, entry_size: u64, limit_size: u64) -> Result { let total_size = total_size.saturating_add(entry_size); if total_size > limit_size { return Err(UnpackError::Archive(format!( - "too large snapshot: {:?}", - total_size + "too large archive: {} than limit: {}", + total_size, limit_size, ))); } Ok(total_size) @@ -151,10 +151,18 @@ fn is_valid_snapshot_archive_entry(parts: &[&str], kind: tar::EntryType) -> bool } } -pub fn open_genesis_config(ledger_path: &Path) -> GenesisConfig { +pub fn open_genesis_config( + ledger_path: &Path, + max_genesis_archive_unpacked_size: u64, +) -> GenesisConfig { GenesisConfig::load(&ledger_path).unwrap_or_else(|load_err| { let genesis_package = ledger_path.join("genesis.tar.bz2"); - unpack_genesis_archive(&genesis_package, ledger_path).unwrap_or_else(|unpack_err| { + unpack_genesis_archive( + &genesis_package, + ledger_path, + max_genesis_archive_unpacked_size, + ) + .unwrap_or_else(|unpack_err| { warn!( "Failed to open ledger genesis_config at {:?}: {}, {}", ledger_path, load_err, unpack_err, @@ -170,17 +178,20 @@ pub fn open_genesis_config(ledger_path: &Path) -> GenesisConfig { pub fn unpack_genesis_archive( archive_filename: &Path, destination_dir: &Path, -) -> std::result::Result<(), String> { + max_genesis_archive_unpacked_size: u64, +) -> std::result::Result<(), UnpackError> { info!("Extracting {:?}...", archive_filename); let extract_start = Instant::now(); - fs::create_dir_all(destination_dir).map_err(|err| err.to_string())?; - let tar_bz2 = File::open(&archive_filename) - .map_err(|err| format!("Unable to open {:?}: {:?}", archive_filename, err))?; + fs::create_dir_all(destination_dir)?; + let tar_bz2 = File::open(&archive_filename)?; let tar = BzDecoder::new(BufReader::new(tar_bz2)); let mut archive = Archive::new(tar); - unpack_genesis(&mut archive, destination_dir) - .map_err(|err| format!("Unable to unpack {:?}: {:?}", archive_filename, err))?; + unpack_genesis( + &mut archive, + destination_dir, + max_genesis_archive_unpacked_size, + )?; info!( "Extracted {:?} in {:?}", archive_filename, @@ -189,11 +200,15 @@ pub fn unpack_genesis_archive( Ok(()) } -fn unpack_genesis>(archive: &mut Archive, unpack_dir: P) -> Result<()> { +fn unpack_genesis>( + archive: &mut Archive, + unpack_dir: P, + max_genesis_archive_unpacked_size: u64, +) -> Result<()> { unpack_archive( archive, unpack_dir, - MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, + max_genesis_archive_unpacked_size, MAX_GENESIS_ARCHIVE_UNPACKED_COUNT, is_valid_genesis_archive_entry, ) @@ -311,7 +326,9 @@ mod tests { } fn finalize_and_unpack_genesis(archive: tar::Builder>) -> Result<()> { - with_finalize_and_unpack(archive, |a, b| unpack_genesis(a, b)) + with_finalize_and_unpack(archive, |a, b| { + unpack_genesis(a, b, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE) + }) } #[test] @@ -440,7 +457,7 @@ mod tests { let mut archive = Builder::new(Vec::new()); archive.append(&header, data).unwrap(); let result = finalize_and_unpack_snapshot(archive); - assert_matches!(result, Err(UnpackError::Archive(ref message)) if message.to_string() == *"too large snapshot: 1125899906842624"); + assert_matches!(result, Err(UnpackError::Archive(ref message)) if message.to_string() == format!("too large archive: 1125899906842624 than limit: {}", MAX_SNAPSHOT_ARCHIVE_UNPACKED_SIZE)); } #[test] @@ -456,7 +473,7 @@ mod tests { let result = checked_total_size_sum(u64::max_value() - 2, 2, MAX_SNAPSHOT_ARCHIVE_UNPACKED_SIZE); - assert_matches!(result, Err(UnpackError::Archive(ref message)) if message.to_string() == *"too large snapshot: 18446744073709551615"); + assert_matches!(result, Err(UnpackError::Archive(ref message)) if message.to_string() == format!("too large archive: 18446744073709551615 than limit: {}", MAX_SNAPSHOT_ARCHIVE_UNPACKED_SIZE)); } #[test] diff --git a/multinode-demo/setup.sh b/multinode-demo/setup.sh index c50d36bf3..b1b10cf3c 100755 --- a/multinode-demo/setup.sh +++ b/multinode-demo/setup.sh @@ -27,6 +27,7 @@ $solana_keygen new --no-passphrase -so "$SOLANA_CONFIG_DIR"/bootstrap-validator/ args=( "$@" + --max-genesis-archive-unpacked-size 1073741824 --enable-warmup-epochs --bootstrap-validator "$SOLANA_CONFIG_DIR"/bootstrap-validator/identity.json "$SOLANA_CONFIG_DIR"/bootstrap-validator/vote-account.json diff --git a/multinode-demo/validator.sh b/multinode-demo/validator.sh index 263b5f2f1..9c02c3c7b 100755 --- a/multinode-demo/validator.sh +++ b/multinode-demo/validator.sh @@ -6,7 +6,9 @@ here=$(dirname "$0") # shellcheck source=multinode-demo/common.sh source "$here"/common.sh -args=() +args=( + --max-genesis-archive-unpacked-size 1073741824 +) airdrops_enabled=1 node_sol=500 # 500 SOL: number of SOL to airdrop the node for transaction fees and vote account rent exemption (ignored if airdrops_enabled=0) label= @@ -147,6 +149,9 @@ while [[ -n $1 ]]; do elif [[ $1 = --halt-on-trusted-validators-accounts-hash-mismatch ]]; then args+=("$1") shift + elif [[ $1 = --max-genesis-archive-unpacked-size ]]; then + args+=("$1" "$2") + shift 2 elif [[ $1 = -h ]]; then usage "$@" else diff --git a/net/remote/remote-node.sh b/net/remote/remote-node.sh index 7b16a331b..c3e5ad04a 100755 --- a/net/remote/remote-node.sh +++ b/net/remote/remote-node.sh @@ -229,7 +229,7 @@ EOF fi multinode-demo/setup.sh "${args[@]}" - solana-ledger-tool -l config/bootstrap-validator shred-version | tee config/shred-version + solana-ledger-tool -l config/bootstrap-validator shred-version --max-genesis-archive-unpacked-size 1073741824 | tee config/shred-version fi args=( --gossip-host "$entrypointIp" diff --git a/validator/src/main.rs b/validator/src/main.rs index f3d0ef328..c9e547678 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -23,7 +23,7 @@ use solana_core::{ use solana_download_utils::{download_genesis_if_missing, download_snapshot}; use solana_ledger::{ bank_forks::{CompressionType, SnapshotConfig}, - hardened_unpack::unpack_genesis_archive, + hardened_unpack::{unpack_genesis_archive, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE}, }; use solana_perf::recycler::enable_recycler_warming; use solana_sdk::{ @@ -366,11 +366,17 @@ fn download_then_check_genesis_hash( rpc_addr: &SocketAddr, ledger_path: &std::path::Path, expected_genesis_hash: Option, + max_genesis_archive_unpacked_size: u64, ) -> Result { let genesis_package = ledger_path.join("genesis.tar.bz2"); let genesis_config = if let Ok(tmp_genesis_package) = download_genesis_if_missing(rpc_addr, &genesis_package) { - unpack_genesis_archive(&tmp_genesis_package, &ledger_path)?; + unpack_genesis_archive( + &tmp_genesis_package, + &ledger_path, + max_genesis_archive_unpacked_size, + ) + .map_err(|err| format!("Failed to unpack downloaded genesis config: {}", err))?; let downloaded_genesis = GenesisConfig::load(&ledger_path) .map_err(|err| format!("Failed to load downloaded genesis config: {}", err))?; @@ -463,6 +469,7 @@ pub fn main() { let default_dynamic_port_range = &format!("{}-{}", VALIDATOR_PORT_RANGE.0, VALIDATOR_PORT_RANGE.1); let default_limit_ledger_size = &DEFAULT_MAX_LEDGER_SHREDS.to_string(); + let default_genesis_archive_unpacked_size = &MAX_GENESIS_ARCHIVE_UNPACKED_SIZE.to_string(); let matches = App::new(crate_name!()).about(crate_description!()) .version(solana_clap_utils::version!()) @@ -636,14 +643,14 @@ pub fn main() { .help("Comma separated persistent accounts location"), ) .arg( - clap::Arg::with_name("gossip_port") + Arg::with_name("gossip_port") .long("gossip-port") .value_name("PORT") .takes_value(true) .help("Gossip port number for the node"), ) .arg( - clap::Arg::with_name("gossip_host") + Arg::with_name("gossip_host") .long("gossip-host") .value_name("HOST") .takes_value(true) @@ -652,7 +659,7 @@ pub fn main() { .help("IP address for the node to advertise in gossip when --entrypoint is not provided [default: 127.0.0.1]"), ) .arg( - clap::Arg::with_name("dynamic_port_range") + Arg::with_name("dynamic_port_range") .long("dynamic-port-range") .value_name("MIN_PORT-MAX_PORT") .takes_value(true) @@ -661,7 +668,7 @@ pub fn main() { .help("Range to use for dynamically assigned ports"), ) .arg( - clap::Arg::with_name("snapshot_interval_slots") + Arg::with_name("snapshot_interval_slots") .long("snapshot-interval-slots") .value_name("SNAPSHOT_INTERVAL_SLOTS") .takes_value(true) @@ -669,7 +676,7 @@ pub fn main() { .help("Number of slots between generating snapshots, 0 to disable snapshots"), ) .arg( - clap::Arg::with_name("accounts_hash_interval_slots") + Arg::with_name("accounts_hash_interval_slots") .long("accounts-hash-slots") .value_name("ACCOUNTS_HASH_INTERVAL_SLOTS") .takes_value(true) @@ -677,7 +684,7 @@ pub fn main() { .help("Number of slots between generating accounts hash."), ) .arg( - clap::Arg::with_name("limit_ledger_size") + Arg::with_name("limit_ledger_size") .long("limit-ledger-size") .value_name("SHRED_COUNT") .takes_value(true) @@ -687,13 +694,13 @@ pub fn main() { .help("Keep this amount of shreds in root slots."), ) .arg( - clap::Arg::with_name("skip_poh_verify") + Arg::with_name("skip_poh_verify") .long("skip-poh-verify") .takes_value(false) .help("Skip ledger verification at node bootup"), ) .arg( - clap::Arg::with_name("cuda") + Arg::with_name("cuda") .long("cuda") .takes_value(false) .help("Use CUDA"), @@ -762,7 +769,7 @@ pub fn main() { .help("Disable manual compaction of the ledger database. May increase storage requirements.") ) .arg( - clap::Arg::with_name("bind_address") + Arg::with_name("bind_address") .long("bind-address") .value_name("HOST") .takes_value(true) @@ -771,7 +778,7 @@ pub fn main() { .help("IP address to bind the validator ports"), ) .arg( - clap::Arg::with_name("rpc_bind_address") + Arg::with_name("rpc_bind_address") .long("rpc-bind-address") .value_name("HOST") .takes_value(true) @@ -779,14 +786,14 @@ pub fn main() { .help("IP address to bind the RPC port [default: use --bind-address]"), ) .arg( - clap::Arg::with_name("halt_on_trusted_validators_accounts_hash_mismatch") + Arg::with_name("halt_on_trusted_validators_accounts_hash_mismatch") .long("halt-on-trusted-validators-accounts-hash-mismatch") .requires("trusted_validators") .takes_value(false) .help("Abort the validator if a bank hash mismatch is detected within trusted validator set"), ) .arg( - clap::Arg::with_name("frozen_accounts") + Arg::with_name("frozen_accounts") .long("frozen-account") .validator(is_pubkey) .value_name("PUBKEY") @@ -804,6 +811,16 @@ pub fn main() { .takes_value(true) .help("Type of snapshot compression to use."), ) + .arg( + Arg::with_name("max_genesis_archive_unpacked_size") + .long("max-genesis-archive-unpacked-size") + .value_name("NUMBER") + .takes_value(true) + .default_value(&default_genesis_archive_unpacked_size) + .help( + "maximum total uncompressed file size of downloaded genesis archive", + ), + ) .get_matches(); let identity_keypair = Arc::new(keypair_of(&matches, "identity").unwrap_or_else(Keypair::new)); @@ -1058,6 +1075,8 @@ pub fn main() { ) }), ); + let max_genesis_archive_unpacked_size = + value_t_or_exit!(matches, "max_genesis_archive_unpacked_size", u64); let cluster_entrypoint = entrypoint_addr .as_ref() @@ -1141,6 +1160,7 @@ pub fn main() { &rpc_contact_info.rpc, &ledger_path, validator_config.expected_genesis_hash, + max_genesis_archive_unpacked_size, ); if let Ok(genesis_hash) = genesis_hash {