From 8cbc450192e8036eca9fe86910159dc22d443040 Mon Sep 17 00:00:00 2001 From: Ryo Onodera Date: Fri, 22 Nov 2019 02:57:27 +0900 Subject: [PATCH] Create genesis.tar.bz2 in solana-genesis (#7039) * Use clap_utils * Create genesis.tar.bz2 in solana-genesis * Remove shell-based genesis.tar.bz2 generation * Make Option=>Result conv more rusty * stop using solana_logger * Simplify by just using vec! * clean up abit --- core/src/snapshot_packager_service.rs | 14 +++++---- genesis/src/main.rs | 42 ++++++++++++--------------- ledger/src/blocktree.rs | 30 +++++++++++++++++++ multinode-demo/setup.sh | 6 ---- run.sh | 1 - 5 files changed, 57 insertions(+), 36 deletions(-) diff --git a/core/src/snapshot_packager_service.rs b/core/src/snapshot_packager_service.rs index 6518c3352..f279bc70b 100644 --- a/core/src/snapshot_packager_service.rs +++ b/core/src/snapshot_packager_service.rs @@ -92,12 +92,14 @@ impl SnapshotPackagerService { // Tar the staging directory into the archive at `archive_path` let archive_path = tar_dir.join("new_state.tar.bz2"); - let mut args = vec!["jcfhS"]; - args.push(archive_path.to_str().unwrap()); - args.push("-C"); - args.push(staging_dir.path().to_str().unwrap()); - args.push(TAR_ACCOUNTS_DIR); - args.push(TAR_SNAPSHOTS_DIR); + let args = vec![ + "jcfhS", + archive_path.to_str().unwrap(), + "-C", + staging_dir.path().to_str().unwrap(), + TAR_ACCOUNTS_DIR, + TAR_SNAPSHOTS_DIR, + ]; let output = std::process::Command::new("tar").args(&args).output()?; if !output.status.success() { diff --git a/genesis/src/main.rs b/genesis/src/main.rs index 9ccbdf2e8..eb00b0f21 100644 --- a/genesis/src/main.rs +++ b/genesis/src/main.rs @@ -3,7 +3,8 @@ mod genesis_accounts; use crate::genesis_accounts::create_genesis_accounts; -use clap::{crate_description, crate_name, value_t, value_t_or_exit, App, Arg}; +use clap::{crate_description, crate_name, value_t, value_t_or_exit, App, Arg, ArgMatches}; +use solana_clap_utils::input_parsers::pubkey_of; use solana_genesis::Base64Account; use solana_ledger::blocktree::create_new_ledger; use solana_ledger::poh::compute_hashes_per_tick; @@ -15,9 +16,9 @@ use solana_sdk::{ genesis_config::{GenesisConfig, OperatingMode}, native_token::sol_to_lamports, poh_config::PohConfig, - pubkey::{read_pubkey_file, Pubkey}, + pubkey::Pubkey, rent::Rent, - signature::{read_keypair_file, Keypair, KeypairUtil}, + signature::{Keypair, KeypairUtil}, system_program, timing, }; use solana_stake_program::stake_state; @@ -30,10 +31,14 @@ pub enum AccountFileFormat { Keypair, } -fn pubkey_from_file(key_file: &str) -> Result> { - read_pubkey_file(key_file) - .or_else(|_| read_keypair_file(key_file).map(|keypair| keypair.pubkey())) - .map_err(|err| format!("Failed to read {}: {}", key_file, err).into()) +fn required_pubkey(matches: &ArgMatches<'_>, name: &str) -> Result> { + pubkey_of(matches, name).ok_or_else(|| { + format!( + "Invalid pubkey or file: {}", + matches.value_of(name).unwrap() + ) + .into() + }) } fn pubkey_from_str(key_str: &str) -> Result> { @@ -305,26 +310,17 @@ fn main() -> Result<(), Box> { ) .get_matches(); - let bootstrap_leader_pubkey_file = matches.value_of("bootstrap_leader_pubkey_file").unwrap(); - let bootstrap_vote_pubkey_file = matches.value_of("bootstrap_vote_pubkey_file").unwrap(); - let bootstrap_stake_pubkey_file = matches.value_of("bootstrap_stake_pubkey_file").unwrap(); - let bootstrap_storage_pubkey_file = matches.value_of("bootstrap_storage_pubkey_file"); - let faucet_pubkey_file = matches.value_of("faucet_pubkey_file"); let faucet_lamports = value_t!(matches, "faucet_lamports", u64); let ledger_path = PathBuf::from(matches.value_of("ledger_path").unwrap()); let bootstrap_leader_lamports = value_t_or_exit!(matches, "bootstrap_leader_lamports", u64); let bootstrap_leader_stake_lamports = value_t_or_exit!(matches, "bootstrap_leader_stake_lamports", u64); - let bootstrap_leader_pubkey = pubkey_from_file(bootstrap_leader_pubkey_file)?; - let bootstrap_vote_pubkey = pubkey_from_file(bootstrap_vote_pubkey_file)?; - let bootstrap_stake_pubkey = pubkey_from_file(bootstrap_stake_pubkey_file)?; - let bootstrap_storage_pubkey = - if let Some(bootstrap_storage_pubkey_file) = bootstrap_storage_pubkey_file { - Some(pubkey_from_file(bootstrap_storage_pubkey_file)?) - } else { - None - }; + let bootstrap_leader_pubkey = required_pubkey(&matches, "bootstrap_leader_pubkey_file")?; + let bootstrap_vote_pubkey = required_pubkey(&matches, "bootstrap_vote_pubkey_file")?; + let bootstrap_stake_pubkey = required_pubkey(&matches, "bootstrap_stake_pubkey_file")?; + let bootstrap_storage_pubkey = pubkey_of(&matches, "bootstrap_storage_pubkey_file"); + let faucet_pubkey = pubkey_of(&matches, "faucet_pubkey_file"); let bootstrap_leader_vote_account = vote_state::create_account(&bootstrap_vote_pubkey, &bootstrap_leader_pubkey, 0, 1); @@ -362,9 +358,9 @@ fn main() -> Result<(), Box> { )); } - if let Some(faucet_pubkey_file) = faucet_pubkey_file { + if let Some(faucet_pubkey) = faucet_pubkey { accounts.push(( - pubkey_from_file(faucet_pubkey_file)?, + faucet_pubkey, Account::new(faucet_lamports.unwrap(), 0, &system_program::id()), )); } diff --git a/ledger/src/blocktree.rs b/ledger/src/blocktree.rs index 85243321f..52bcac000 100644 --- a/ledger/src/blocktree.rs +++ b/ledger/src/blocktree.rs @@ -1856,6 +1856,36 @@ pub fn create_new_ledger(ledger_path: &Path, genesis_config: &GenesisConfig) -> blocktree.insert_shreds(shreds, None, false)?; blocktree.set_roots(&[0])?; + // Explicitly close the blocktree before we create the archived genesis file + drop(blocktree); + + let archive_path = ledger_path.join("genesis.tar.bz2"); + let args = vec![ + "jcfhS", + archive_path.to_str().unwrap(), + "-C", + ledger_path.to_str().unwrap(), + "genesis.bin", + "rocksdb", + ]; + let output = std::process::Command::new("tar") + .args(&args) + .output() + .unwrap(); + if !output.status.success() { + use std::io::{Error as IOError, ErrorKind}; + use std::str::from_utf8; + eprintln!("tar stdout: {}", from_utf8(&output.stdout).unwrap_or("?")); + eprintln!("tar stderr: {}", from_utf8(&output.stderr).unwrap_or("?")); + + return Err(BlocktreeError::IO(IOError::new( + ErrorKind::Other, + format!( + "Error trying to generate snapshot archive: {}", + output.status + ), + ))); + } Ok(last_hash) } diff --git a/multinode-demo/setup.sh b/multinode-demo/setup.sh index 4cd78a80e..5606f7b42 100755 --- a/multinode-demo/setup.sh +++ b/multinode-demo/setup.sh @@ -37,9 +37,3 @@ default_arg --faucet-lamports 500000000000000000 default_arg --hashes-per-tick auto default_arg --operating-mode development $solana_genesis "${args[@]}" - -( - cd "$SOLANA_CONFIG_DIR"/bootstrap-leader - set -x - tar jcvfS genesis.tar.bz2 genesis.bin rocksdb -) diff --git a/run.sh b/run.sh index 82c1c4efa..0a6ee826f 100755 --- a/run.sh +++ b/run.sh @@ -87,7 +87,6 @@ solana-genesis \ --bootstrap-storage-pubkey "$dataDir"/leader-storage-account-keypair.json \ --ledger "$ledgerDir" \ --operating-mode development -tar jcfS "$ledgerDir/genesis.tar.bz2" -C "$ledgerDir" genesis.bin rocksdb abort() { set +e