Pass around --max-genesis-archive-unpacked-size (#9161)

automerge

Cargo.lock

@@ -3576,6 +3576,7 @@ dependencies = [
  "solana-clap-utils",
  "solana-genesis-programs",
  "solana-ledger",
+ "solana-logger",
  "solana-sdk",
  "solana-stake-program",
  "solana-storage-program",

core/src/validator.rs

@@ -30,7 +30,7 @@ use solana_ledger::{
     blockstore::{Blockstore, CompletedSlotsReceiver},
     blockstore_processor::{self, BankForksInfo},
     create_new_tmp_ledger,
-    hardened_unpack::open_genesis_config,
+    hardened_unpack::{open_genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE},
     leader_schedule::FixedSchedule,
     leader_schedule_cache::LeaderScheduleCache,
 };
@@ -82,6 +82,7 @@ pub struct ValidatorConfig {
     pub frozen_accounts: Vec<Pubkey>,
     pub no_rocksdb_compaction: bool,
     pub accounts_hash_interval_slots: u64,
+    pub max_genesis_archive_unpacked_size: u64,
 }
 
 impl Default for ValidatorConfig {
@@ -109,6 +110,7 @@ impl Default for ValidatorConfig {
             frozen_accounts: vec![],
             no_rocksdb_compaction: false,
             accounts_hash_interval_slots: std::u64::MAX,
+            max_genesis_archive_unpacked_size: MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
         }
     }
 }
@@ -569,7 +571,8 @@ fn new_banks_from_blockstore(
     LeaderScheduleCache,
     Option<(Slot, Hash)>,
 ) {
-    let genesis_config = open_genesis_config(blockstore_path);
+    let genesis_config =
+        open_genesis_config(blockstore_path, config.max_genesis_archive_unpacked_size);
 
     // This needs to be limited otherwise the state in the VoteAccount data
     // grows too large

genesis/Cargo.toml

@@ -18,6 +18,7 @@ serde_yaml = "0.8.11"
 solana-clap-utils = { path = "../clap-utils", version = "1.2.0" }
 solana-genesis-programs = { path = "../genesis-programs", version = "1.2.0" }
 solana-ledger = { path = "../ledger", version = "1.2.0" }
+solana-logger = { path = "../logger", version = "1.2.0" }
 solana-sdk = { path = "../sdk", version = "1.2.0" }
 solana-stake-program = { path = "../programs/stake", version = "1.2.0" }
 solana-storage-program = { path = "../programs/storage", version = "1.2.0" }

genesis/src/main.rs

@@ -6,7 +6,10 @@ use solana_clap_utils::{
     input_validators::{is_pubkey_or_keypair, is_rfc3339_datetime, is_valid_percentage},
 };
 use solana_genesis::{genesis_accounts::add_genesis_accounts, Base64Account};
-use solana_ledger::{blockstore::create_new_ledger, poh::compute_hashes_per_tick};
+use solana_ledger::{
+    blockstore::create_new_ledger, hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
+    poh::compute_hashes_per_tick,
+};
 use solana_sdk::{
     account::Account,
     clock,
@@ -121,6 +124,7 @@ fn main() -> Result<(), Box<dyn error::Error>> {
         timing::duration_as_us(&PohConfig::default().target_tick_duration);
     let default_ticks_per_slot = &clock::DEFAULT_TICKS_PER_SLOT.to_string();
     let default_operating_mode = "stable";
+    let default_genesis_archive_unpacked_size = MAX_GENESIS_ARCHIVE_UNPACKED_SIZE.to_string();
 
     let matches = App::new(crate_name!())
         .about(crate_description!())
@@ -327,6 +331,16 @@ fn main() -> Result<(), Box<dyn error::Error>> {
                     "Selects the features that will be enabled for the cluster"
                 ),
         )
+        .arg(
+            Arg::with_name("max_genesis_archive_unpacked_size")
+                .long("max-genesis-archive-unpacked-size")
+                .value_name("NUMBER")
+                .takes_value(true)
+                .default_value(&default_genesis_archive_unpacked_size)
+                .help(
+                    "maximum total uncompressed file size of created genesis archive",
+                ),
+        )
         .get_matches();
 
     let faucet_lamports = value_t!(matches, "faucet_lamports", u64).unwrap_or(0);
@@ -513,6 +527,9 @@ fn main() -> Result<(), Box<dyn error::Error>> {
         }
     }
 
+    let max_genesis_archive_unpacked_size =
+        value_t_or_exit!(matches, "max_genesis_archive_unpacked_size", u64);
+
     let issued_lamports = genesis_config
         .accounts
         .iter()
@@ -521,7 +538,12 @@ fn main() -> Result<(), Box<dyn error::Error>> {
 
     add_genesis_accounts(&mut genesis_config, issued_lamports - faucet_lamports);
 
-    create_new_ledger(&ledger_path, &genesis_config)?;
+    solana_logger::setup();
+    create_new_ledger(
+        &ledger_path,
+        &genesis_config,
+        max_genesis_archive_unpacked_size,
+    )?;
 
     println!("{}", genesis_config);
     Ok(())

ledger-tool/src/main.rs

@@ -11,7 +11,7 @@ use solana_ledger::{
     blockstore::Blockstore,
     blockstore_db::{self, Column, Database},
     blockstore_processor::{BankForksInfo, ProcessOptions},
-    hardened_unpack::open_genesis_config,
+    hardened_unpack::{open_genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE},
     rooted_slot_iterator::RootedSlotIterator,
     snapshot_utils,
 };
@@ -575,6 +575,12 @@ fn load_bank_forks(
     )
 }
 
+fn open_genesis_config_by(ledger_path: &Path, matches: &ArgMatches<'_>) -> GenesisConfig {
+    let max_genesis_archive_unpacked_size =
+        value_t_or_exit!(matches, "max_genesis_archive_unpacked_size", u64);
+    open_genesis_config(ledger_path, max_genesis_archive_unpacked_size)
+}
+
 #[allow(clippy::cognitive_complexity)]
 fn main() {
     const DEFAULT_ROOT_COUNT: &str = "1";
@@ -612,6 +618,13 @@ fn main() {
         .long("allow-dead-slots")
         .takes_value(false)
         .help("Output dead slots as well");
+    let default_genesis_archive_unpacked_size = MAX_GENESIS_ARCHIVE_UNPACKED_SIZE.to_string();
+    let max_genesis_archive_unpacked_size_arg = Arg::with_name("max_genesis_archive_unpacked_size")
+        .long("max-genesis-archive-unpacked-size")
+        .value_name("NUMBER")
+        .takes_value(true)
+        .default_value(&default_genesis_archive_unpacked_size)
+        .help("maximum total uncompressed size of unpacked genesis archive");
 
     let matches = App::new(crate_name!())
         .about(crate_description!())
@@ -663,15 +676,18 @@ fn main() {
         .subcommand(
             SubCommand::with_name("genesis")
             .about("Prints the ledger's genesis config")
+            .arg(&max_genesis_archive_unpacked_size_arg)
         )
         .subcommand(
             SubCommand::with_name("genesis-hash")
             .about("Prints the ledger's genesis hash")
+            .arg(&max_genesis_archive_unpacked_size_arg)
         )
         .subcommand(
             SubCommand::with_name("shred-version")
             .about("Prints the ledger's shred hash")
             .arg(&hard_forks_arg)
+            .arg(&max_genesis_archive_unpacked_size_arg)
         )
         .subcommand(
             SubCommand::with_name("bounds")
@@ -696,6 +712,7 @@ fn main() {
             .arg(&account_paths_arg)
             .arg(&halt_at_slot_arg)
             .arg(&hard_forks_arg)
+            .arg(&max_genesis_archive_unpacked_size_arg)
             .arg(
                 Arg::with_name("skip_poh_verify")
                     .long("skip-poh-verify")
@@ -709,6 +726,7 @@ fn main() {
             .arg(&account_paths_arg)
             .arg(&halt_at_slot_arg)
             .arg(&hard_forks_arg)
+            .arg(&max_genesis_archive_unpacked_size_arg)
             .arg(
                 Arg::with_name("include_all_votes")
                     .long("include-all-votes")
@@ -727,6 +745,7 @@ fn main() {
             .arg(&no_snapshot_arg)
             .arg(&account_paths_arg)
             .arg(&hard_forks_arg)
+            .arg(&max_genesis_archive_unpacked_size_arg)
             .arg(
                 Arg::with_name("snapshot_slot")
                     .index(1)
@@ -755,6 +774,7 @@ fn main() {
                     .takes_value(false)
                     .help("Include sysvars too"),
             )
+            .arg(&max_genesis_archive_unpacked_size_arg)
         ).subcommand(
             SubCommand::with_name("prune")
             .about("Prune the ledger from a yaml file containing a list of slots to prune.")
@@ -847,11 +867,14 @@ fn main() {
                 LedgerOutputMethod::Print,
             );
         }
-        ("genesis", Some(_arg_matches)) => {
+        ("genesis", Some(arg_matches)) => {
-            println!("{}", open_genesis_config(&ledger_path));
+            println!("{}", open_genesis_config_by(&ledger_path, arg_matches));
         }
-        ("genesis-hash", Some(_arg_matches)) => {
+        ("genesis-hash", Some(arg_matches)) => {
-            println!("{}", open_genesis_config(&ledger_path).hash());
+            println!(
+                "{}",
+                open_genesis_config_by(&ledger_path, arg_matches).hash()
+            );
         }
         ("shred-version", Some(arg_matches)) => {
             let process_options = ProcessOptions {
@@ -860,7 +883,7 @@ fn main() {
                 poh_verify: false,
                 ..ProcessOptions::default()
             };
-            let genesis_config = open_genesis_config(&ledger_path);
+            let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
             match load_bank_forks(arg_matches, &ledger_path, &genesis_config, process_options) {
                 Ok((bank_forks, bank_forks_info, _leader_schedule_cache, _snapshot_hash)) => {
                     let bank_info = &bank_forks_info[0];
@@ -923,12 +946,15 @@ fn main() {
                 poh_verify: !arg_matches.is_present("skip_poh_verify"),
                 ..ProcessOptions::default()
             };
-            println!("{}", open_genesis_config(&ledger_path).hash());
+            println!(
+                "genesis hash: {}",
+                open_genesis_config_by(&ledger_path, arg_matches).hash()
+            );
 
             load_bank_forks(
                 arg_matches,
                 &ledger_path,
-                &open_genesis_config(&ledger_path),
+                &open_genesis_config_by(&ledger_path, arg_matches),
                 process_options,
             )
             .unwrap_or_else(|err| {
@@ -950,7 +976,7 @@ fn main() {
             match load_bank_forks(
                 arg_matches,
                 &ledger_path,
-                &open_genesis_config(&ledger_path),
+                &open_genesis_config_by(&ledger_path, arg_matches),
                 process_options,
             ) {
                 Ok((bank_forks, bank_forks_info, _leader_schedule_cache, _snapshot_hash)) => {
@@ -991,7 +1017,7 @@ fn main() {
                 poh_verify: false,
                 ..ProcessOptions::default()
             };
-            let genesis_config = open_genesis_config(&ledger_path);
+            let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
             match load_bank_forks(arg_matches, &ledger_path, &genesis_config, process_options) {
                 Ok((bank_forks, _bank_forks_info, _leader_schedule_cache, _snapshot_hash)) => {
                     let bank = bank_forks.get(snapshot_slot).unwrap_or_else(|| {
@@ -1055,7 +1081,7 @@ fn main() {
                 poh_verify: false,
                 ..ProcessOptions::default()
             };
-            let genesis_config = open_genesis_config(&ledger_path);
+            let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
             let include_sysvars = arg_matches.is_present("include_sysvars");
             match load_bank_forks(arg_matches, &ledger_path, &genesis_config, process_options) {
                 Ok((bank_forks, bank_forks_info, _leader_schedule_cache, _snapshot_hash)) => {

ledger/src/blockstore.rs

@@ -10,6 +10,7 @@ use crate::{
     blockstore_meta::*,
     entry::{create_ticks, Entry},
     erasure::ErasureConfig,
+    hardened_unpack::{unpack_genesis_archive, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE},
     leader_schedule_cache::LeaderScheduleCache,
     next_slots_iterator::NextSlotsIterator,
     shred::{Result as ShredResult, Shred, Shredder},
@@ -45,6 +46,7 @@ use std::{
     cmp,
     collections::HashMap,
     fs,
+    io::{Error as IOError, ErrorKind},
     path::{Path, PathBuf},
     rc::Rc,
     sync::{
@@ -2622,7 +2624,11 @@ fn calculate_stake_weighted_timestamp(
 // Creates a new ledger with slot 0 full of ticks (and only ticks).
 //
 // Returns the blockhash that can be used to append entries with.
-pub fn create_new_ledger(ledger_path: &Path, genesis_config: &GenesisConfig) -> Result<Hash> {
+pub fn create_new_ledger(
+    ledger_path: &Path,
+    genesis_config: &GenesisConfig,
+    max_genesis_archive_unpacked_size: u64,
+) -> Result<Hash> {
     Blockstore::destroy(ledger_path)?;
     genesis_config.write(&ledger_path)?;
 
@@ -2658,7 +2664,6 @@ pub fn create_new_ledger(ledger_path: &Path, genesis_config: &GenesisConfig) ->
         .output()
         .unwrap();
     if !output.status.success() {
-        use std::io::{Error as IOError, ErrorKind};
         use std::str::from_utf8;
         error!("tar stdout: {}", from_utf8(&output.stdout).unwrap_or("?"));
         error!("tar stderr: {}", from_utf8(&output.stderr).unwrap_or("?"));
@@ -2672,6 +2677,54 @@ pub fn create_new_ledger(ledger_path: &Path, genesis_config: &GenesisConfig) ->
         )));
     }
 
+    // ensure the genesis archive can be unpacked and it is under
+    // max_genesis_archive_unpacked_size, immedately after creating it above.
+    {
+        let temp_dir = tempfile::TempDir::new().unwrap();
+        // unpack into a temp dir, while completely discarding the unpacked files
+        let unpack_check = unpack_genesis_archive(
+            &archive_path,
+            &temp_dir.into_path(),
+            max_genesis_archive_unpacked_size,
+        );
+        if let Err(unpack_err) = unpack_check {
+            // stash problematic original archived genesis related files to
+            // examine them later and to prevent validator and ledger-tool from
+            // naively consuming them
+            let mut error_messages = String::new();
+
+            fs::rename(
+                &ledger_path.join("genesis.tar.bz2"),
+                ledger_path.join("genesis.tar.bz2.failed"),
+            )
+            .unwrap_or_else(|e| {
+                error_messages += &format!("/failed to stash problematic genesis.tar.bz2: {}", e)
+            });
+            fs::rename(
+                &ledger_path.join("genesis.bin"),
+                ledger_path.join("genesis.bin.failed"),
+            )
+            .unwrap_or_else(|e| {
+                error_messages += &format!("/failed to stash problematic genesis.bin: {}", e)
+            });
+            fs::rename(
+                &ledger_path.join("rocksdb"),
+                ledger_path.join("rocksdb.failed"),
+            )
+            .unwrap_or_else(|e| {
+                error_messages += &format!("/failed to stash problematic rocksdb: {}", e)
+            });
+
+            return Err(BlockstoreError::IO(IOError::new(
+                ErrorKind::Other,
+                format!(
+                    "Error checking to unpack genesis archive: {}{}",
+                    unpack_err, error_messages
+                ),
+            )));
+        }
+    }
+
     Ok(last_hash)
 }
 
@@ -2739,7 +2792,12 @@ pub fn verify_shred_slots(slot: Slot, parent_slot: Slot, last_root: Slot) -> boo
 // ticks)
 pub fn create_new_ledger_from_name(name: &str, genesis_config: &GenesisConfig) -> (PathBuf, Hash) {
     let ledger_path = get_ledger_path_from_name(name);
-    let blockhash = create_new_ledger(&ledger_path, genesis_config).unwrap();
+    let blockhash = create_new_ledger(
+        &ledger_path,
+        genesis_config,
+        MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
+    )
+    .unwrap();
     (ledger_path, blockhash)
 }
 

ledger/src/blockstore_db.rs

@@ -1,4 +1,4 @@
-use crate::blockstore_meta;
+use crate::{blockstore_meta, hardened_unpack::UnpackError};
 use bincode::{deserialize, serialize};
 use byteorder::{BigEndian, ByteOrder};
 use log::*;
@@ -55,6 +55,7 @@ pub enum BlockstoreError {
     Serialize(#[from] Box<bincode::ErrorKind>),
     FsExtraError(#[from] fs_extra::error::Error),
     SlotCleanedUp,
+    UnpackError(#[from] UnpackError),
 }
 pub type Result<T> = std::result::Result<T, BlockstoreError>;
 

ledger/src/hardened_unpack.rs

@@ -19,9 +19,9 @@ use thiserror::Error;
 
 #[derive(Error, Debug)]
 pub enum UnpackError {
-    #[error("IO error")]
+    #[error("IO error: {0}")]
     IO(#[from] std::io::Error),
-    #[error("Archive error")]
+    #[error("Archive error: {0}")]
     Archive(String),
 }
 
@@ -29,15 +29,15 @@ pub type Result<T> = std::result::Result<T, UnpackError>;
 
 const MAX_SNAPSHOT_ARCHIVE_UNPACKED_SIZE: u64 = 500 * 1024 * 1024 * 1024; // 500 GiB
 const MAX_SNAPSHOT_ARCHIVE_UNPACKED_COUNT: u64 = 500_000;
-const MAX_GENESIS_ARCHIVE_UNPACKED_SIZE: u64 = 1024 * 1024 * 1024; // 1024 MiB
+pub const MAX_GENESIS_ARCHIVE_UNPACKED_SIZE: u64 = 10 * 1024 * 1024; // 10 MiB
 const MAX_GENESIS_ARCHIVE_UNPACKED_COUNT: u64 = 100;
 
 fn checked_total_size_sum(total_size: u64, entry_size: u64, limit_size: u64) -> Result<u64> {
     let total_size = total_size.saturating_add(entry_size);
     if total_size > limit_size {
         return Err(UnpackError::Archive(format!(
-            "too large snapshot: {:?}",
+            "too large archive: {} than limit: {}",
-            total_size
+            total_size, limit_size,
         )));
     }
     Ok(total_size)
@@ -151,10 +151,18 @@ fn is_valid_snapshot_archive_entry(parts: &[&str], kind: tar::EntryType) -> bool
     }
 }
 
-pub fn open_genesis_config(ledger_path: &Path) -> GenesisConfig {
+pub fn open_genesis_config(
+    ledger_path: &Path,
+    max_genesis_archive_unpacked_size: u64,
+) -> GenesisConfig {
     GenesisConfig::load(&ledger_path).unwrap_or_else(|load_err| {
         let genesis_package = ledger_path.join("genesis.tar.bz2");
-        unpack_genesis_archive(&genesis_package, ledger_path).unwrap_or_else(|unpack_err| {
+        unpack_genesis_archive(
+            &genesis_package,
+            ledger_path,
+            max_genesis_archive_unpacked_size,
+        )
+        .unwrap_or_else(|unpack_err| {
             warn!(
                 "Failed to open ledger genesis_config at {:?}: {}, {}",
                 ledger_path, load_err, unpack_err,
@@ -170,17 +178,20 @@ pub fn open_genesis_config(ledger_path: &Path) -> GenesisConfig {
 pub fn unpack_genesis_archive(
     archive_filename: &Path,
     destination_dir: &Path,
-) -> std::result::Result<(), String> {
+    max_genesis_archive_unpacked_size: u64,
+) -> std::result::Result<(), UnpackError> {
     info!("Extracting {:?}...", archive_filename);
     let extract_start = Instant::now();
 
-    fs::create_dir_all(destination_dir).map_err(|err| err.to_string())?;
+    fs::create_dir_all(destination_dir)?;
-    let tar_bz2 = File::open(&archive_filename)
+    let tar_bz2 = File::open(&archive_filename)?;
-        .map_err(|err| format!("Unable to open {:?}: {:?}", archive_filename, err))?;
     let tar = BzDecoder::new(BufReader::new(tar_bz2));
     let mut archive = Archive::new(tar);
-    unpack_genesis(&mut archive, destination_dir)
+    unpack_genesis(
-        .map_err(|err| format!("Unable to unpack {:?}: {:?}", archive_filename, err))?;
+        &mut archive,
+        destination_dir,
+        max_genesis_archive_unpacked_size,
+    )?;
     info!(
         "Extracted {:?} in {:?}",
         archive_filename,
@@ -189,11 +200,15 @@ pub fn unpack_genesis_archive(
     Ok(())
 }
 
-fn unpack_genesis<A: Read, P: AsRef<Path>>(archive: &mut Archive<A>, unpack_dir: P) -> Result<()> {
+fn unpack_genesis<A: Read, P: AsRef<Path>>(
+    archive: &mut Archive<A>,
+    unpack_dir: P,
+    max_genesis_archive_unpacked_size: u64,
+) -> Result<()> {
     unpack_archive(
         archive,
         unpack_dir,
-        MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
+        max_genesis_archive_unpacked_size,
         MAX_GENESIS_ARCHIVE_UNPACKED_COUNT,
         is_valid_genesis_archive_entry,
     )
@@ -311,7 +326,9 @@ mod tests {
     }
 
     fn finalize_and_unpack_genesis(archive: tar::Builder<Vec<u8>>) -> Result<()> {
-        with_finalize_and_unpack(archive, |a, b| unpack_genesis(a, b))
+        with_finalize_and_unpack(archive, |a, b| {
+            unpack_genesis(a, b, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE)
+        })
     }
 
     #[test]
@@ -440,7 +457,7 @@ mod tests {
         let mut archive = Builder::new(Vec::new());
         archive.append(&header, data).unwrap();
         let result = finalize_and_unpack_snapshot(archive);
-        assert_matches!(result, Err(UnpackError::Archive(ref message)) if message.to_string() == *"too large snapshot: 1125899906842624");
+        assert_matches!(result, Err(UnpackError::Archive(ref message)) if message.to_string() == format!("too large archive: 1125899906842624 than limit: {}", MAX_SNAPSHOT_ARCHIVE_UNPACKED_SIZE));
     }
 
     #[test]
@@ -456,7 +473,7 @@ mod tests {
 
         let result =
             checked_total_size_sum(u64::max_value() - 2, 2, MAX_SNAPSHOT_ARCHIVE_UNPACKED_SIZE);
-        assert_matches!(result, Err(UnpackError::Archive(ref message)) if message.to_string() == *"too large snapshot: 18446744073709551615");
+        assert_matches!(result, Err(UnpackError::Archive(ref message)) if message.to_string() == format!("too large archive: 18446744073709551615 than limit: {}", MAX_SNAPSHOT_ARCHIVE_UNPACKED_SIZE));
     }
 
     #[test]

multinode-demo/setup.sh

@@ -27,6 +27,7 @@ $solana_keygen new --no-passphrase -so "$SOLANA_CONFIG_DIR"/bootstrap-validator/
 
 args=(
   "$@"
+  --max-genesis-archive-unpacked-size 1073741824
   --enable-warmup-epochs
   --bootstrap-validator "$SOLANA_CONFIG_DIR"/bootstrap-validator/identity.json
                         "$SOLANA_CONFIG_DIR"/bootstrap-validator/vote-account.json

multinode-demo/validator.sh

@@ -6,7 +6,9 @@ here=$(dirname "$0")
 # shellcheck source=multinode-demo/common.sh
 source "$here"/common.sh
 
-args=()
+args=(
+  --max-genesis-archive-unpacked-size 1073741824
+)
 airdrops_enabled=1
 node_sol=500 # 500 SOL: number of SOL to airdrop the node for transaction fees and vote account rent exemption (ignored if airdrops_enabled=0)
 label=
@@ -147,6 +149,9 @@ while [[ -n $1 ]]; do
     elif [[ $1 = --halt-on-trusted-validators-accounts-hash-mismatch ]]; then
       args+=("$1")
       shift
+    elif [[ $1 = --max-genesis-archive-unpacked-size ]]; then
+      args+=("$1" "$2")
+      shift 2
     elif [[ $1 = -h ]]; then
       usage "$@"
     else

net/remote/remote-node.sh

@@ -229,7 +229,7 @@ EOF
       fi
       multinode-demo/setup.sh "${args[@]}"
 
-      solana-ledger-tool -l config/bootstrap-validator shred-version | tee config/shred-version
+      solana-ledger-tool -l config/bootstrap-validator shred-version --max-genesis-archive-unpacked-size 1073741824 | tee config/shred-version
     fi
     args=(
       --gossip-host "$entrypointIp"

validator/src/main.rs

@@ -23,7 +23,7 @@ use solana_core::{
 use solana_download_utils::{download_genesis_if_missing, download_snapshot};
 use solana_ledger::{
     bank_forks::{CompressionType, SnapshotConfig},
-    hardened_unpack::unpack_genesis_archive,
+    hardened_unpack::{unpack_genesis_archive, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE},
 };
 use solana_perf::recycler::enable_recycler_warming;
 use solana_sdk::{
@@ -366,11 +366,17 @@ fn download_then_check_genesis_hash(
     rpc_addr: &SocketAddr,
     ledger_path: &std::path::Path,
     expected_genesis_hash: Option<Hash>,
+    max_genesis_archive_unpacked_size: u64,
 ) -> Result<Hash, String> {
     let genesis_package = ledger_path.join("genesis.tar.bz2");
     let genesis_config =
         if let Ok(tmp_genesis_package) = download_genesis_if_missing(rpc_addr, &genesis_package) {
-            unpack_genesis_archive(&tmp_genesis_package, &ledger_path)?;
+            unpack_genesis_archive(
+                &tmp_genesis_package,
+                &ledger_path,
+                max_genesis_archive_unpacked_size,
+            )
+            .map_err(|err| format!("Failed to unpack downloaded genesis config: {}", err))?;
 
             let downloaded_genesis = GenesisConfig::load(&ledger_path)
                 .map_err(|err| format!("Failed to load downloaded genesis config: {}", err))?;
@@ -463,6 +469,7 @@ pub fn main() {
     let default_dynamic_port_range =
         &format!("{}-{}", VALIDATOR_PORT_RANGE.0, VALIDATOR_PORT_RANGE.1);
     let default_limit_ledger_size = &DEFAULT_MAX_LEDGER_SHREDS.to_string();
+    let default_genesis_archive_unpacked_size = &MAX_GENESIS_ARCHIVE_UNPACKED_SIZE.to_string();
 
     let matches = App::new(crate_name!()).about(crate_description!())
         .version(solana_clap_utils::version!())
@@ -636,14 +643,14 @@ pub fn main() {
                 .help("Comma separated persistent accounts location"),
         )
         .arg(
-            clap::Arg::with_name("gossip_port")
+            Arg::with_name("gossip_port")
                 .long("gossip-port")
                 .value_name("PORT")
                 .takes_value(true)
                 .help("Gossip port number for the node"),
         )
         .arg(
-            clap::Arg::with_name("gossip_host")
+            Arg::with_name("gossip_host")
                 .long("gossip-host")
                 .value_name("HOST")
                 .takes_value(true)
@@ -652,7 +659,7 @@ pub fn main() {
                 .help("IP address for the node to advertise in gossip when --entrypoint is not provided [default: 127.0.0.1]"),
         )
         .arg(
-            clap::Arg::with_name("dynamic_port_range")
+            Arg::with_name("dynamic_port_range")
                 .long("dynamic-port-range")
                 .value_name("MIN_PORT-MAX_PORT")
                 .takes_value(true)
@@ -661,7 +668,7 @@ pub fn main() {
                 .help("Range to use for dynamically assigned ports"),
         )
         .arg(
-            clap::Arg::with_name("snapshot_interval_slots")
+            Arg::with_name("snapshot_interval_slots")
                 .long("snapshot-interval-slots")
                 .value_name("SNAPSHOT_INTERVAL_SLOTS")
                 .takes_value(true)
@@ -669,7 +676,7 @@ pub fn main() {
                 .help("Number of slots between generating snapshots, 0 to disable snapshots"),
         )
         .arg(
-            clap::Arg::with_name("accounts_hash_interval_slots")
+            Arg::with_name("accounts_hash_interval_slots")
                 .long("accounts-hash-slots")
                 .value_name("ACCOUNTS_HASH_INTERVAL_SLOTS")
                 .takes_value(true)
@@ -677,7 +684,7 @@ pub fn main() {
                 .help("Number of slots between generating accounts hash."),
         )
         .arg(
-            clap::Arg::with_name("limit_ledger_size")
+            Arg::with_name("limit_ledger_size")
                 .long("limit-ledger-size")
                 .value_name("SHRED_COUNT")
                 .takes_value(true)
@@ -687,13 +694,13 @@ pub fn main() {
                 .help("Keep this amount of shreds in root slots."),
         )
         .arg(
-            clap::Arg::with_name("skip_poh_verify")
+            Arg::with_name("skip_poh_verify")
                 .long("skip-poh-verify")
                 .takes_value(false)
                 .help("Skip ledger verification at node bootup"),
         )
         .arg(
-            clap::Arg::with_name("cuda")
+            Arg::with_name("cuda")
                 .long("cuda")
                 .takes_value(false)
                 .help("Use CUDA"),
@@ -762,7 +769,7 @@ pub fn main() {
                 .help("Disable manual compaction of the ledger database. May increase storage requirements.")
         )
         .arg(
-            clap::Arg::with_name("bind_address")
+            Arg::with_name("bind_address")
                 .long("bind-address")
                 .value_name("HOST")
                 .takes_value(true)
@@ -771,7 +778,7 @@ pub fn main() {
                 .help("IP address to bind the validator ports"),
         )
         .arg(
-            clap::Arg::with_name("rpc_bind_address")
+            Arg::with_name("rpc_bind_address")
                 .long("rpc-bind-address")
                 .value_name("HOST")
                 .takes_value(true)
@@ -779,14 +786,14 @@ pub fn main() {
                 .help("IP address to bind the RPC port [default: use --bind-address]"),
         )
         .arg(
-            clap::Arg::with_name("halt_on_trusted_validators_accounts_hash_mismatch")
+            Arg::with_name("halt_on_trusted_validators_accounts_hash_mismatch")
                 .long("halt-on-trusted-validators-accounts-hash-mismatch")
                 .requires("trusted_validators")
                 .takes_value(false)
                 .help("Abort the validator if a bank hash mismatch is detected within trusted validator set"),
         )
         .arg(
-            clap::Arg::with_name("frozen_accounts")
+            Arg::with_name("frozen_accounts")
                 .long("frozen-account")
                 .validator(is_pubkey)
                 .value_name("PUBKEY")
@@ -804,6 +811,16 @@ pub fn main() {
                 .takes_value(true)
                 .help("Type of snapshot compression to use."),
         )
+        .arg(
+            Arg::with_name("max_genesis_archive_unpacked_size")
+                .long("max-genesis-archive-unpacked-size")
+                .value_name("NUMBER")
+                .takes_value(true)
+                .default_value(&default_genesis_archive_unpacked_size)
+                .help(
+                    "maximum total uncompressed file size of downloaded genesis archive",
+                ),
+        )
         .get_matches();
 
     let identity_keypair = Arc::new(keypair_of(&matches, "identity").unwrap_or_else(Keypair::new));
@@ -1058,6 +1075,8 @@ pub fn main() {
             )
         }),
     );
+    let max_genesis_archive_unpacked_size =
+        value_t_or_exit!(matches, "max_genesis_archive_unpacked_size", u64);
 
     let cluster_entrypoint = entrypoint_addr
         .as_ref()
@@ -1141,6 +1160,7 @@ pub fn main() {
                         &rpc_contact_info.rpc,
                         &ledger_path,
                         validator_config.expected_genesis_hash,
+                        max_genesis_archive_unpacked_size,
                     );
 
                     if let Ok(genesis_hash) = genesis_hash {