Create genesis.tar.bz2 in solana-genesis (#7039)

* Use clap_utils

* Create genesis.tar.bz2 in solana-genesis

* Remove shell-based genesis.tar.bz2 generation

* Make Option=>Result conv more rusty

* stop using solana_logger

* Simplify by just using vec!

* clean up abit

core/src/snapshot_packager_service.rs

@@ -92,12 +92,14 @@ impl SnapshotPackagerService {
 
         // Tar the staging directory into the archive at `archive_path`
         let archive_path = tar_dir.join("new_state.tar.bz2");
-        let mut args = vec!["jcfhS"];
+        let args = vec![
-        args.push(archive_path.to_str().unwrap());
+            "jcfhS",
-        args.push("-C");
+            archive_path.to_str().unwrap(),
-        args.push(staging_dir.path().to_str().unwrap());
+            "-C",
-        args.push(TAR_ACCOUNTS_DIR);
+            staging_dir.path().to_str().unwrap(),
-        args.push(TAR_SNAPSHOTS_DIR);
+            TAR_ACCOUNTS_DIR,
+            TAR_SNAPSHOTS_DIR,
+        ];
 
         let output = std::process::Command::new("tar").args(&args).output()?;
         if !output.status.success() {

genesis/src/main.rs

@@ -3,7 +3,8 @@
 mod genesis_accounts;
 
 use crate::genesis_accounts::create_genesis_accounts;
-use clap::{crate_description, crate_name, value_t, value_t_or_exit, App, Arg};
+use clap::{crate_description, crate_name, value_t, value_t_or_exit, App, Arg, ArgMatches};
+use solana_clap_utils::input_parsers::pubkey_of;
 use solana_genesis::Base64Account;
 use solana_ledger::blocktree::create_new_ledger;
 use solana_ledger::poh::compute_hashes_per_tick;
@@ -15,9 +16,9 @@ use solana_sdk::{
     genesis_config::{GenesisConfig, OperatingMode},
     native_token::sol_to_lamports,
     poh_config::PohConfig,
-    pubkey::{read_pubkey_file, Pubkey},
+    pubkey::Pubkey,
     rent::Rent,
-    signature::{read_keypair_file, Keypair, KeypairUtil},
+    signature::{Keypair, KeypairUtil},
     system_program, timing,
 };
 use solana_stake_program::stake_state;
@@ -30,10 +31,14 @@ pub enum AccountFileFormat {
     Keypair,
 }
 
-fn pubkey_from_file(key_file: &str) -> Result<Pubkey, Box<dyn error::Error>> {
+fn required_pubkey(matches: &ArgMatches<'_>, name: &str) -> Result<Pubkey, Box<dyn error::Error>> {
-    read_pubkey_file(key_file)
+    pubkey_of(matches, name).ok_or_else(|| {
-        .or_else(|_| read_keypair_file(key_file).map(|keypair| keypair.pubkey()))
+        format!(
-        .map_err(|err| format!("Failed to read {}: {}", key_file, err).into())
+            "Invalid pubkey or file: {}",
+            matches.value_of(name).unwrap()
+        )
+        .into()
+    })
 }
 
 fn pubkey_from_str(key_str: &str) -> Result<Pubkey, Box<dyn error::Error>> {
@@ -305,26 +310,17 @@ fn main() -> Result<(), Box<dyn error::Error>> {
         )
         .get_matches();
 
-    let bootstrap_leader_pubkey_file = matches.value_of("bootstrap_leader_pubkey_file").unwrap();
-    let bootstrap_vote_pubkey_file = matches.value_of("bootstrap_vote_pubkey_file").unwrap();
-    let bootstrap_stake_pubkey_file = matches.value_of("bootstrap_stake_pubkey_file").unwrap();
-    let bootstrap_storage_pubkey_file = matches.value_of("bootstrap_storage_pubkey_file");
-    let faucet_pubkey_file = matches.value_of("faucet_pubkey_file");
     let faucet_lamports = value_t!(matches, "faucet_lamports", u64);
     let ledger_path = PathBuf::from(matches.value_of("ledger_path").unwrap());
     let bootstrap_leader_lamports = value_t_or_exit!(matches, "bootstrap_leader_lamports", u64);
     let bootstrap_leader_stake_lamports =
         value_t_or_exit!(matches, "bootstrap_leader_stake_lamports", u64);
 
-    let bootstrap_leader_pubkey = pubkey_from_file(bootstrap_leader_pubkey_file)?;
+    let bootstrap_leader_pubkey = required_pubkey(&matches, "bootstrap_leader_pubkey_file")?;
-    let bootstrap_vote_pubkey = pubkey_from_file(bootstrap_vote_pubkey_file)?;
+    let bootstrap_vote_pubkey = required_pubkey(&matches, "bootstrap_vote_pubkey_file")?;
-    let bootstrap_stake_pubkey = pubkey_from_file(bootstrap_stake_pubkey_file)?;
+    let bootstrap_stake_pubkey = required_pubkey(&matches, "bootstrap_stake_pubkey_file")?;
-    let bootstrap_storage_pubkey =
+    let bootstrap_storage_pubkey = pubkey_of(&matches, "bootstrap_storage_pubkey_file");
-        if let Some(bootstrap_storage_pubkey_file) = bootstrap_storage_pubkey_file {
+    let faucet_pubkey = pubkey_of(&matches, "faucet_pubkey_file");
-            Some(pubkey_from_file(bootstrap_storage_pubkey_file)?)
-        } else {
-            None
-        };
 
     let bootstrap_leader_vote_account =
         vote_state::create_account(&bootstrap_vote_pubkey, &bootstrap_leader_pubkey, 0, 1);
@@ -362,9 +358,9 @@ fn main() -> Result<(), Box<dyn error::Error>> {
         ));
     }
 
-    if let Some(faucet_pubkey_file) = faucet_pubkey_file {
+    if let Some(faucet_pubkey) = faucet_pubkey {
         accounts.push((
-            pubkey_from_file(faucet_pubkey_file)?,
+            faucet_pubkey,
             Account::new(faucet_lamports.unwrap(), 0, &system_program::id()),
         ));
     }

ledger/src/blocktree.rs

@@ -1856,6 +1856,36 @@ pub fn create_new_ledger(ledger_path: &Path, genesis_config: &GenesisConfig) ->
 
     blocktree.insert_shreds(shreds, None, false)?;
     blocktree.set_roots(&[0])?;
+    // Explicitly close the blocktree before we create the archived genesis file
+    drop(blocktree);
+
+    let archive_path = ledger_path.join("genesis.tar.bz2");
+    let args = vec![
+        "jcfhS",
+        archive_path.to_str().unwrap(),
+        "-C",
+        ledger_path.to_str().unwrap(),
+        "genesis.bin",
+        "rocksdb",
+    ];
+    let output = std::process::Command::new("tar")
+        .args(&args)
+        .output()
+        .unwrap();
+    if !output.status.success() {
+        use std::io::{Error as IOError, ErrorKind};
+        use std::str::from_utf8;
+        eprintln!("tar stdout: {}", from_utf8(&output.stdout).unwrap_or("?"));
+        eprintln!("tar stderr: {}", from_utf8(&output.stderr).unwrap_or("?"));
+
+        return Err(BlocktreeError::IO(IOError::new(
+            ErrorKind::Other,
+            format!(
+                "Error trying to generate snapshot archive: {}",
+                output.status
+            ),
+        )));
+    }
 
     Ok(last_hash)
 }

multinode-demo/setup.sh

@@ -37,9 +37,3 @@ default_arg --faucet-lamports 500000000000000000
 default_arg --hashes-per-tick auto
 default_arg --operating-mode development
 $solana_genesis "${args[@]}"
-
-(
-  cd "$SOLANA_CONFIG_DIR"/bootstrap-leader
-  set -x
-  tar jcvfS genesis.tar.bz2 genesis.bin rocksdb
-)

run.sh

@@ -87,7 +87,6 @@ solana-genesis \
   --bootstrap-storage-pubkey "$dataDir"/leader-storage-account-keypair.json \
   --ledger "$ledgerDir" \
   --operating-mode development
-tar jcfS "$ledgerDir/genesis.tar.bz2" -C "$ledgerDir" genesis.bin rocksdb
 
 abort() {
   set +e