Add option to load accounts from file

This introduces the `--clone-from-file` option for
solana-test-validator. It allows specifying any number of files
(without extension) containing account info and data, which will be
loaded at genesis. This is similar to `--bpf-program` for programs
loading.

The files will be searched for in the CWD or in `tests/fixtures`.

Example: `solana-test-validator --clone-from-file SRM_token USD_token`

.gitignore

@@ -4,6 +4,7 @@
 /solana-metrics/
 /solana-metrics.tar.bz2
 /target/
+/test-ledger/
 
 **/*.rs.bk
 .cargo

Cargo.lock

@@ -5981,6 +5981,9 @@ version = "1.10.0"
 dependencies = [
  "base64 0.12.3",
  "log 0.4.14",
+ "serde_derive",
+ "serde_json",
+ "solana-cli-output",
  "solana-client",
  "solana-core",
  "solana-gossip",

cli-output/src/cli_output.rs

@@ -99,7 +99,7 @@ impl OutputFormat {
 pub struct CliAccount {
     #[serde(flatten)]
     pub keyed_account: RpcKeyedAccount,
-    #[serde(skip_serializing)]
+    #[serde(skip_serializing, skip_deserializing)]
     pub use_lamports_unit: bool,
 }
 

docs/src/developing/test-validator.md

@@ -14,6 +14,7 @@ starts a full-featured, single-node cluster on the developer's workstation.
 - Direct [on-chain program](on-chain-programs/overview) deployment
   (`--bpf-program ...`)
 - Clone accounts from a public cluster, including programs (`--clone ...`)
+- Load accounts from files
 - Configurable transaction history retention (`--limit-ledger-size ...`)
 - Configurable epoch length (`--slots-per-epoch ...`)
 - Jump to an arbitrary slot (`--warp-slot ...`)

test-validator/Cargo.toml

@@ -13,6 +13,9 @@ edition = "2021"
 [dependencies]
 base64 = "0.12.3"
 log = "0.4.14"
+serde_derive = "1.0.103"
+serde_json = "1.0.72"
+solana-cli-output = { path = "../cli-output", version = "=1.10.0" }
 solana-client = { path = "../client", version = "=1.10.0" }
 solana-core = { path = "../core", version = "=1.10.0" }
 solana-gossip = { path = "../gossip", version = "=1.10.0" }

test-validator/src/lib.rs

@@ -1,6 +1,7 @@
 #![allow(clippy::integer_arithmetic)]
 use {
     log::*,
+    solana_cli_output::CliAccount,
     solana_client::rpc_client::RpcClient,
     solana_core::{
         tower_storage::TowerStorage,
@@ -36,15 +37,23 @@ use {
     solana_streamer::socket::SocketAddrSpace,
     std::{
         collections::HashMap,
-        fs::remove_dir_all,
+        fs::{remove_dir_all, File},
+        io::Read,
         net::{IpAddr, Ipv4Addr, SocketAddr},
         path::{Path, PathBuf},
+        str::FromStr,
         sync::{Arc, RwLock},
         thread::sleep,
         time::Duration,
     },
 };
 
+#[derive(Clone)]
+pub struct AccountInfo<'a> {
+    pub address: Pubkey,
+    pub filename: &'a str,
+}
+
 #[derive(Clone)]
 pub struct ProgramInfo {
     pub program_id: Pubkey,
@@ -204,6 +213,41 @@ impl TestValidatorGenesis {
         self
     }
 
+    pub fn add_accounts_from_json_files(&mut self, accounts: &[AccountInfo]) -> &mut Self {
+        for account in accounts {
+            let account_path =
+                solana_program_test::find_file(account.filename).unwrap_or_else(|| {
+                    error!("Unable to locate {}", account.filename);
+                    solana_core::validator::abort();
+                });
+            let mut file = File::open(&account_path).unwrap();
+            let mut account_info_raw = String::new();
+            file.read_to_string(&mut account_info_raw).unwrap();
+
+            let result: serde_json::Result<CliAccount> = serde_json::from_str(&account_info_raw);
+            let account_info = match result {
+                Err(err) => {
+                    error!(
+                        "Unable to deserialize {}: {}",
+                        account_path.to_str().unwrap(),
+                        err
+                    );
+                    solana_core::validator::abort();
+                }
+                Ok(deserialized) => deserialized,
+            };
+            let address = Pubkey::from_str(account_info.keyed_account.pubkey.as_str()).unwrap();
+            let account = account_info
+                .keyed_account
+                .account
+                .decode::<AccountSharedData>()
+                .unwrap();
+
+            self.add_account(address, account);
+        }
+        self
+    }
+
     /// Add an account to the test environment with the account data in the provided `filename`
     pub fn add_account_with_file_data(
         &mut self,

validator/src/bin/solana-test-validator.rs

@@ -167,6 +167,19 @@ fn main() {
                        First argument can be a public key or path to file that can be parsed as a keypair",
                 ),
         )
+        .arg(
+            Arg::with_name("account")
+                .long("account")
+                .value_name("ADDRESS FILENAME.JSON")
+                .takes_value(true)
+                .number_of_values(2)
+                .multiple(true)
+                .help(
+                    "Load an account from the provided JSON file (see `solana account --help` on how to dump \
+                        an account to file). Files are searched for relatively to CWD and tests/fixtures. \
+                        If the ledger already exists then this parameter is silently ignored",
+                ),
+        )
         .arg(
             Arg::with_name("no_bpf_jit")
                 .long("no-bpf-jit")
@@ -404,7 +417,7 @@ fn main() {
         faucet_port,
     ));
 
-    let mut programs = vec![];
+    let mut programs_to_load = vec![];
     if let Some(values) = matches.values_of("bpf_program") {
         let values: Vec<&str> = values.collect::<Vec<_>>();
         for address_program in values.chunks(2) {
@@ -427,7 +440,7 @@ fn main() {
                         exit(1);
                     }
 
-                    programs.push(ProgramInfo {
+                    programs_to_load.push(ProgramInfo {
                         program_id: address,
                         loader: solana_sdk::bpf_loader::id(),
                         program_path,
@@ -438,7 +451,25 @@ fn main() {
         }
     }
 
-    let clone_accounts: HashSet<_> = pubkeys_of(&matches, "clone_account")
+    let mut accounts_to_load = vec![];
+    if let Some(values) = matches.values_of("account") {
+        let values: Vec<&str> = values.collect::<Vec<_>>();
+        for address_filename in values.chunks(2) {
+            match address_filename {
+                [address, filename] => {
+                    let address = address.parse::<Pubkey>().unwrap_or_else(|err| {
+                        println!("Error: invalid address {}: {}", address, err);
+                        exit(1);
+                    });
+
+                    accounts_to_load.push(AccountInfo { address, filename });
+                }
+                _ => unreachable!(),
+            }
+        }
+    }
+
+    let accounts_to_clone: HashSet<_> = pubkeys_of(&matches, "clone_account")
         .map(|v| v.into_iter().collect())
         .unwrap_or_default();
 
@@ -500,6 +531,7 @@ fn main() {
         for (name, long) in &[
             ("bpf_program", "--bpf-program"),
             ("clone_account", "--clone"),
+            ("clone_account_from_file", "--clone-from-file"),
             ("mint_address", "--mint"),
             ("slots_per_epoch", "--slots-per-epoch"),
             ("faucet_sol", "--faucet-sol"),
@@ -565,11 +597,12 @@ fn main() {
         })
         .bpf_jit(!matches.is_present("no_bpf_jit"))
         .rpc_port(rpc_port)
-        .add_programs_with_path(&programs);
+        .add_programs_with_path(&programs_to_load)
+        .add_accounts_from_json_files(&accounts_to_load);
 
-    if !clone_accounts.is_empty() {
+    if !accounts_to_clone.is_empty() {
         genesis.clone_accounts(
-            clone_accounts,
+            accounts_to_clone,
             cluster_rpc_client
                 .as_ref()
                 .expect("bug: --url argument missing?"),