cli: Add solana-test-validator control to Anchor.toml (#834)

CHANGELOG.md

@@ -13,6 +13,7 @@ incremented for features.
 
 ### Features
 
+* cli: Add support for configuration options for `solana-test-validator` in Anchor.toml ([#834](https://github.com/project-serum/anchor/pull/834)).
 * cli: `target/types` directory now created on build to store a TypeScript types file for each program's IDL ([#795](https://github.com/project-serum/anchor/pull/795)).
 * ts: `Program<T>` can now be typed with an IDL type ([#795](https://github.com/project-serum/anchor/pull/795)).
 * lang: Add `mint::freeze_authority` keyword for mint initialization within `#[derive(Accounts)]` ([#835](https://github.com/project-serum/anchor/pull/835)).

cli/src/config.rs

@@ -459,7 +459,9 @@ fn deser_programs(
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct Test {
-    pub genesis: Vec<GenesisEntry>,
+    pub genesis: Option<Vec<GenesisEntry>>,
+    pub clone: Option<Vec<CloneEntry>>,
+    pub validator: Option<Validator>,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -470,6 +472,64 @@ pub struct GenesisEntry {
     pub program: String,
 }
 
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct CloneEntry {
+    // Base58 pubkey string.
+    pub address: String,
+}
+
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
+pub struct Validator {
+    // IP address to bind the validator ports. [default: 0.0.0.0]
+    #[serde(default = "default_bind_address")]
+    pub bind_address: String,
+    // Range to use for dynamically assigned ports. [default: 1024-65535]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub dynamic_port_range: Option<String>,
+    // Enable the faucet on this port [deafult: 9900].
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub faucet_port: Option<u16>,
+    // Give the faucet address this much SOL in genesis. [default: 1000000]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub faucet_sol: Option<String>,
+    // Gossip DNS name or IP address for the validator to advertise in gossip. [default: 127.0.0.1]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub gossip_host: Option<String>,
+    // Gossip port number for the validator
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub gossip_port: Option<u16>,
+    // URL for Solana's JSON RPC or moniker.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub url: Option<String>,
+    // Use DIR as ledger location
+    #[serde(default = "default_ledger_path")]
+    pub ledger: String,
+    // Keep this amount of shreds in root slots. [default: 10000]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub limit_ledger_size: Option<String>,
+    // Enable JSON RPC on this port, and the next port for the RPC websocket. [default: 8899]
+    #[serde(default = "default_rpc_port")]
+    pub rpc_port: u16,
+    // Override the number of slots in an epoch.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub slots_per_epoch: Option<String>,
+    // Warp the ledger to WARP_SLOT after starting the validator.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub warp_slot: Option<String>,
+}
+
+fn default_ledger_path() -> String {
+    ".anchor/test-ledger".to_string()
+}
+
+fn default_bind_address() -> String {
+    "0.0.0.0".to_string()
+}
+
+fn default_rpc_port() -> u16 {
+    8899
+}
+
 #[derive(Debug, Clone)]
 pub struct Program {
     pub lib_name: String,

cli/src/lib.rs

@@ -1,5 +1,6 @@
 use crate::config::{
-    AnchorPackage, Config, ConfigOverride, Manifest, ProgramDeployment, ProgramWorkspace, WithPath,
+    AnchorPackage, Config, ConfigOverride, Manifest, ProgramDeployment, ProgramWorkspace, Test,
+    WithPath,
 };
 use anchor_client::Cluster;
 use anchor_lang::idl::{IdlAccount, IdlInstruction};
@@ -955,7 +956,8 @@ fn verify(
         .join("target/verifiable/")
         .join(format!("{}.so", binary_name));
 
-    let bin_ver = verify_bin(program_id, &bin_path, cfg.provider.cluster.url())?;
+    let url = cluster_url(&cfg);
+    let bin_ver = verify_bin(program_id, &bin_path, &url)?;
     if !bin_ver.is_verified {
         println!("Error: Binaries don't match");
         std::process::exit(1);
@@ -1080,7 +1082,8 @@ pub enum BinVerificationState {
 // Fetches an IDL for the given program_id.
 fn fetch_idl(cfg_override: &ConfigOverride, idl_addr: Pubkey) -> Result<Idl> {
     let cfg = Config::discover(cfg_override)?.expect("Inside a workspace");
-    let client = RpcClient::new(cfg.provider.cluster.url().to_string());
+    let url = cluster_url(&cfg);
+    let client = RpcClient::new(url);
 
     let mut account = client
         .get_account_with_commitment(&idl_addr, CommitmentConfig::processed())?
@@ -1177,7 +1180,8 @@ fn idl_set_buffer(cfg_override: &ConfigOverride, program_id: Pubkey, buffer: Pub
     with_workspace(cfg_override, |cfg| {
         let keypair = solana_sdk::signature::read_keypair_file(&cfg.provider.wallet.to_string())
             .map_err(|_| anyhow!("Unable to read keypair file"))?;
-        let client = RpcClient::new(cfg.provider.cluster.url().to_string());
+        let url = cluster_url(cfg);
+        let client = RpcClient::new(url);
 
         // Instruction to set the buffer onto the IdlAccount.
         let set_buffer_ix = {
@@ -1229,7 +1233,8 @@ fn idl_upgrade(
 
 fn idl_authority(cfg_override: &ConfigOverride, program_id: Pubkey) -> Result<()> {
     with_workspace(cfg_override, |cfg| {
-        let client = RpcClient::new(cfg.provider.cluster.url().to_string());
+        let url = cluster_url(cfg);
+        let client = RpcClient::new(url);
         let idl_address = {
             let account = client
                 .get_account_with_commitment(&program_id, CommitmentConfig::processed())?
@@ -1266,7 +1271,8 @@ fn idl_set_authority(
         };
         let keypair = solana_sdk::signature::read_keypair_file(&cfg.provider.wallet.to_string())
             .map_err(|_| anyhow!("Unable to read keypair file"))?;
-        let client = RpcClient::new(cfg.provider.cluster.url().to_string());
+        let url = cluster_url(cfg);
+        let client = RpcClient::new(url);
 
         // Instruction data.
         let data =
@@ -1336,7 +1342,8 @@ fn idl_write(cfg: &Config, program_id: &Pubkey, idl: &Idl, idl_address: Pubkey)
     // Misc.
     let keypair = solana_sdk::signature::read_keypair_file(&cfg.provider.wallet.to_string())
         .map_err(|_| anyhow!("Unable to read keypair file"))?;
-    let client = RpcClient::new(cfg.provider.cluster.url().to_string());
+    let url = cluster_url(cfg);
+    let client = RpcClient::new(url);
 
     // Serialize and compress the idl.
     let idl_data = {
@@ -1462,13 +1469,15 @@ fn test(
         if is_localnet && (!skip_local_validator) {
             let flags = match skip_deploy {
                 true => None,
-                false => Some(genesis_flags(cfg)?),
+                false => Some(validator_flags(cfg)?),
             };
             validator_handle = Some(start_test_validator(cfg, flags, true)?);
         }
 
+        let url = cluster_url(cfg);
+
         // Setup log reader.
-        let log_streams = stream_logs(cfg);
+        let log_streams = stream_logs(cfg, &url);
 
         // Run the tests.
         let test_result: Result<_> = {
@@ -1485,7 +1494,7 @@ fn test(
 
             std::process::Command::new(program)
                 .args(args)
-                .env("ANCHOR_PROVIDER_URL", cfg.provider.cluster.url())
+                .env("ANCHOR_PROVIDER_URL", url)
                 .env("ANCHOR_WALLET", cfg.provider.wallet.to_string())
                 .stdout(Stdio::inherit())
                 .stderr(Stdio::inherit())
@@ -1528,9 +1537,10 @@ fn test(
     })
 }
 
-// Returns the solana-test-validator flags to embed the workspace programs
+// Returns the solana-test-validator flags. This will embed the workspace
-// in the genesis block. This allows us to run tests without every deploying.
+// programs in the genesis block so we don't have to deploy every time. It also
-fn genesis_flags(cfg: &WithPath<Config>) -> Result<Vec<String>> {
+// allows control of other solana-test-validator features.
+fn validator_flags(cfg: &WithPath<Config>) -> Result<Vec<String>> {
     let programs = cfg.programs.get(&Cluster::Localnet);
 
     let mut flags = Vec::new();
@@ -1559,17 +1569,40 @@ fn genesis_flags(cfg: &WithPath<Config>) -> Result<Vec<String>> {
             write_idl(idl, OutFile::File(idl_out))?;
         }
     }
+
     if let Some(test) = cfg.test.as_ref() {
-        for entry in &test.genesis {
+        if let Some(genesis) = &test.genesis {
-            flags.push("--bpf-program".to_string());
+            for entry in genesis {
-            flags.push(entry.address.clone());
+                flags.push("--bpf-program".to_string());
-            flags.push(entry.program.clone());
+                flags.push(entry.address.clone());
+                flags.push(entry.program.clone());
+            }
+        }
+        if let Some(clone) = &test.clone {
+            for entry in clone {
+                flags.push("--clone".to_string());
+                flags.push(entry.address.clone());
+            }
+        }
+        if let Some(validator) = &test.validator {
+            for (key, value) in serde_json::to_value(validator)?.as_object().unwrap() {
+                if key == "ledger" {
+                    continue;
+                };
+                flags.push(format!("--{}", key.replace("_", "-")));
+                if let serde_json::Value::String(v) = value {
+                    flags.push(v.to_string());
+                } else {
+                    flags.push(value.to_string());
+                }
+            }
         }
     }
+
     Ok(flags)
 }
 
-fn stream_logs(config: &WithPath<Config>) -> Result<Vec<std::process::Child>> {
+fn stream_logs(config: &WithPath<Config>, rpc_url: &str) -> Result<Vec<std::process::Child>> {
     let program_logs_dir = ".anchor/program-logs";
     if Path::new(program_logs_dir).exists() {
         fs::remove_dir_all(program_logs_dir)?;
@@ -1595,23 +1628,25 @@ fn stream_logs(config: &WithPath<Config>) -> Result<Vec<std::process::Child>> {
             .arg("logs")
             .arg(metadata.address)
             .arg("--url")
-            .arg(config.provider.cluster.url())
+            .arg(rpc_url)
             .stdout(stdio)
             .spawn()?;
         handles.push(child);
     }
     if let Some(test) = config.test.as_ref() {
-        for entry in &test.genesis {
+        if let Some(genesis) = &test.genesis {
-            let log_file = File::create(format!("{}/{}.log", program_logs_dir, entry.address))?;
+            for entry in genesis {
-            let stdio = std::process::Stdio::from(log_file);
+                let log_file = File::create(format!("{}/{}.log", program_logs_dir, entry.address))?;
-            let child = std::process::Command::new("solana")
+                let stdio = std::process::Stdio::from(log_file);
-                .arg("logs")
+                let child = std::process::Command::new("solana")
-                .arg(entry.address.clone())
+                    .arg("logs")
-                .arg("--url")
+                    .arg(entry.address.clone())
-                .arg(config.provider.cluster.url())
+                    .arg("--url")
-                .stdout(stdio)
+                    .arg(rpc_url)
-                .spawn()?;
+                    .stdout(stdio)
-            handles.push(child);
+                    .spawn()?;
+                handles.push(child);
+            }
         }
     }
 
@@ -1628,21 +1663,13 @@ fn start_test_validator(
     flags: Option<Vec<String>>,
     test_log_stdout: bool,
 ) -> Result<Child> {
-    fs::create_dir_all(".anchor")?;
+    //
-    let test_ledger_filename = ".anchor/test-ledger";
+    let (test_ledger_directory, test_ledger_log_filename) = test_validator_file_paths(cfg);
-    let test_ledger_log_filename = ".anchor/test-ledger-log.txt";
-
-    if Path::new(test_ledger_filename).exists() {
-        fs::remove_dir_all(test_ledger_filename)?;
-    }
-    if Path::new(test_ledger_log_filename).exists() {
-        fs::remove_file(test_ledger_log_filename)?;
-    }
 
     // Start a validator for testing.
     let (test_validator_stdout, test_validator_stderr) = match test_log_stdout {
         true => {
-            let test_validator_stdout_file = File::create(test_ledger_log_filename)?;
+            let test_validator_stdout_file = File::create(&test_ledger_log_filename)?;
             let test_validator_sterr_file = test_validator_stdout_file.try_clone()?;
             (
                 Stdio::from(test_validator_stdout_file),
@@ -1651,9 +1678,12 @@ fn start_test_validator(
         }
         false => (Stdio::inherit(), Stdio::inherit()),
     };
+
+    let rpc_url = test_validator_rpc_url(cfg);
+
     let mut validator_handle = std::process::Command::new("solana-test-validator")
         .arg("--ledger")
-        .arg(test_ledger_filename)
+        .arg(test_ledger_directory)
         .arg("--mint")
         .arg(cfg.wallet_kp()?.pubkey().to_string())
         .args(flags.unwrap_or_default())
@@ -1663,7 +1693,7 @@ fn start_test_validator(
         .map_err(|e| anyhow::format_err!("{}", e.to_string()))?;
 
     // Wait for the validator to be ready.
-    let client = RpcClient::new("http://localhost:8899".to_string());
+    let client = RpcClient::new(rpc_url);
     let mut count = 0;
     let ms_wait = 5000;
     while count < ms_wait {
@@ -1682,13 +1712,63 @@ fn start_test_validator(
         validator_handle.kill()?;
         std::process::exit(1);
     }
-
     Ok(validator_handle)
 }
 
+// Return the URL that solana-test-validator should be running on given the
+// configuration
+fn test_validator_rpc_url(cfg: &Config) -> String {
+    match &cfg.test.as_ref() {
+        Some(Test {
+            validator: Some(validator),
+            ..
+        }) => format!("http://{}:{}", validator.bind_address, validator.rpc_port),
+        _ => "http://localhost:8899".to_string(),
+    }
+}
+
+// Setup and return paths to the solana-test-validator ledger directory and log
+// files given the configuration
+fn test_validator_file_paths(cfg: &Config) -> (String, String) {
+    let ledger_directory = match &cfg.test.as_ref() {
+        Some(Test {
+            validator: Some(validator),
+            ..
+        }) => &validator.ledger,
+        _ => ".anchor/test-ledger",
+    };
+
+    if !Path::new(&ledger_directory).is_relative() {
+        // Prevent absolute paths to avoid someone using / or similar, as the
+        // directory gets removed
+        eprintln!("Ledger directory {} must be relative", ledger_directory);
+        std::process::exit(1);
+    }
+    if Path::new(&ledger_directory).exists() {
+        fs::remove_dir_all(&ledger_directory).unwrap();
+    }
+
+    fs::create_dir_all(&ledger_directory).unwrap();
+
+    (
+        ledger_directory.to_string(),
+        format!("{}/test-ledger-log.txt", ledger_directory),
+    )
+}
+
+fn cluster_url(cfg: &Config) -> String {
+    let is_localnet = cfg.provider.cluster == Cluster::Localnet;
+    match is_localnet {
+        // Cluster is Localnet, assume the intent is to use the configuration
+        // for solana-test-validator
+        true => test_validator_rpc_url(cfg),
+        false => cfg.provider.cluster.url().to_string(),
+    }
+}
+
 fn deploy(cfg_override: &ConfigOverride, program_str: Option<String>) -> Result<()> {
     with_workspace(cfg_override, |cfg| {
-        let url = cfg.provider.cluster.url().to_string();
+        let url = cluster_url(cfg);
         let keypair = cfg.provider.wallet.to_string();
 
         // Deploy the programs.
@@ -1762,11 +1842,12 @@ fn upgrade(
     let program_filepath = path.canonicalize()?.display().to_string();
 
     with_workspace(cfg_override, |cfg| {
+        let url = cluster_url(cfg);
         let exit = std::process::Command::new("solana")
             .arg("program")
             .arg("deploy")
             .arg("--url")
-            .arg(cfg.provider.cluster.url())
+            .arg(url)
             .arg("--keypair")
             .arg(&cfg.provider.wallet.to_string())
             .arg("--program-id")
@@ -1794,7 +1875,8 @@ fn create_idl_account(
     let idl_address = IdlAccount::address(program_id);
     let keypair = solana_sdk::signature::read_keypair_file(keypair_path)
         .map_err(|_| anyhow!("Unable to read keypair file"))?;
-    let client = RpcClient::new(cfg.provider.cluster.url().to_string());
+    let url = cluster_url(cfg);
+    let client = RpcClient::new(url);
     let idl_data = serialize_idl(idl)?;
 
     // Run `Create instruction.
@@ -1847,7 +1929,8 @@ fn create_idl_buffer(
 ) -> Result<Pubkey> {
     let keypair = solana_sdk::signature::read_keypair_file(keypair_path)
         .map_err(|_| anyhow!("Unable to read keypair file"))?;
-    let client = RpcClient::new(cfg.provider.cluster.url().to_string());
+    let url = cluster_url(cfg);
+    let client = RpcClient::new(url);
 
     let buffer = Keypair::generate(&mut OsRng);
 
@@ -1920,7 +2003,7 @@ fn migrate(cfg_override: &ConfigOverride) -> Result<()> {
     with_workspace(cfg_override, |cfg| {
         println!("Running migration deploy script");
 
-        let url = cfg.provider.cluster.url().to_string();
+        let url = cluster_url(cfg);
         let cur_dir = std::env::current_dir()?;
 
         let use_ts =
@@ -1967,8 +2050,8 @@ fn migrate(cfg_override: &ConfigOverride) -> Result<()> {
 
 fn set_workspace_dir_or_exit() {
     let d = match Config::discover(&ConfigOverride::default()) {
-        Err(_) => {
+        Err(err) => {
-            println!("Not in anchor workspace.");
+            println!("Workspace configuration error: {}", err);
             std::process::exit(1);
         }
         Ok(d) => d,
@@ -2074,11 +2157,8 @@ fn shell(cfg_override: &ConfigOverride) -> Result<()> {
                     .collect::<Vec<ProgramWorkspace>>(),
             }
         };
-        let js_code = template::node_shell(
+        let url = cluster_url(cfg);
-            cfg.provider.cluster.url(),
+        let js_code = template::node_shell(&url, &cfg.provider.wallet.to_string(), programs)?;
-            &cfg.provider.wallet.to_string(),
-            programs,
-        )?;
         let mut child = std::process::Command::new("node")
             .args(&["-e", &js_code, "-i", "--experimental-repl-await"])
             .stdout(Stdio::inherit())
@@ -2096,6 +2176,7 @@ fn shell(cfg_override: &ConfigOverride) -> Result<()> {
 
 fn run(cfg_override: &ConfigOverride, script: String) -> Result<()> {
     with_workspace(cfg_override, |cfg| {
+        let url = cluster_url(cfg);
         let script = cfg
             .scripts
             .get(&script)
@@ -2103,7 +2184,7 @@ fn run(cfg_override: &ConfigOverride, script: String) -> Result<()> {
         let exit = std::process::Command::new("bash")
             .arg("-c")
             .arg(&script)
-            .env("ANCHOR_PROVIDER_URL", cfg.provider.cluster.url())
+            .env("ANCHOR_PROVIDER_URL", url)
             .env("ANCHOR_WALLET", cfg.provider.wallet.to_string())
             .stdout(Stdio::inherit())
             .stderr(Stdio::inherit())
@@ -2351,15 +2432,17 @@ fn localnet(
             )?;
         }
 
-        // Setup log reader.
-        let log_streams = stream_logs(cfg);
-
         let flags = match skip_deploy {
             true => None,
-            false => Some(genesis_flags(cfg)?),
+            false => Some(validator_flags(cfg)?),
         };
+
         let validator_handle = &mut start_test_validator(cfg, flags, false)?;
 
+        // Setup log reader.
+        let url = test_validator_rpc_url(cfg);
+        let log_streams = stream_logs(cfg, &url);
+
         std::io::stdin().lock().lines().next().unwrap().unwrap();
 
         // Check all errors and shut down.