bigtable: allow custom instance names

Cargo.lock

@@ -6077,6 +6077,7 @@ dependencies = [
  "solana-runtime",
  "solana-sdk",
  "solana-send-transaction-service",
+ "solana-storage-bigtable",
  "solana-streamer",
  "solana-test-validator",
  "solana-version",

ledger-tool/src/bigtable.rs

@@ -322,6 +322,15 @@ impl BigTableSubCommand for App<'_, '_> {
                 .about("Ledger data on a BigTable instance")
                 .setting(AppSettings::InferSubcommands)
                 .setting(AppSettings::SubcommandRequiredElseHelp)
+                .arg(
+                    Arg::with_name("rpc_bigtable_instance_name")
+                        .global(true)
+                        .long("rpc-bigtable-instance-name")
+                        .takes_value(true)
+                        .value_name("INSTANCE_NAME")
+                        .default_value(solana_storage_bigtable::DEFAULT_INSTANCE_NAME)
+                        .help("Name of the target Bigtable instance")
+                )
                 .subcommand(
                     SubCommand::with_name("upload")
                         .about("Upload the ledger to BigTable")
@@ -431,7 +440,8 @@ impl BigTableSubCommand for App<'_, '_> {
                                 .required(true)
                                 .default_value("1000")
                                 .help("Maximum number of slots to check"),
-                        ).arg(
+                        )
+                        .arg(
                             Arg::with_name("reference_credential")
                                 .long("reference-credential")
                                 .short("c")
@@ -439,6 +449,14 @@ impl BigTableSubCommand for App<'_, '_> {
                                 .takes_value(true)
                                 .required(true)
                                 .help("File path for a credential to a reference bigtable"),
+                        )
+                        .arg(
+                            Arg::with_name("reference_instance_name")
+                                .long("reference-instance-name")
+                                .takes_value(true)
+                                .value_name("INSTANCE_NAME")
+                                .default_value(solana_storage_bigtable::DEFAULT_INSTANCE_NAME)
+                                .help("Name of the reference Bigtable instance to compare to")
                         ),
                 )
                 .subcommand(
@@ -535,7 +553,28 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) {
     let verbose = matches.is_present("verbose");
     let output_format = OutputFormat::from_matches(matches, "output_format", verbose);
 
-    let future = match matches.subcommand() {
+    // this is kinda stupid, but there seems to be a bug in clap when a subcommand
+    // arg is marked both `global(true)` and `default_value("default_value")`.
+    // despite the "global", when the arg is specified on the subcommand, its value
+    // is not propagated down to the (sub)subcommand args, resulting in the default
+    // value when queried there. similarly, if the arg is specified on the
+    // (sub)subcommand, the value is not propagated back up to the subcommand args,
+    // again resulting in the default value. the arg having declared a
+    // `default_value()` obviates `is_present(...)` tests since they will always
+    // return true. so we consede and compare against the expected default. :/
+    let (subcommand, sub_matches) = matches.subcommand();
+    let on_command = matches
+        .value_of("rpc_bigtable_instance_name")
+        .map(|v| v != solana_storage_bigtable::DEFAULT_INSTANCE_NAME)
+        .unwrap_or(false);
+    let instance_name = if on_command {
+        value_t_or_exit!(matches, "rpc_bigtable_instance_name", String)
+    } else {
+        let sub_matches = sub_matches.as_ref().unwrap();
+        value_t_or_exit!(sub_matches, "rpc_bigtable_instance_name", String)
+    };
+
+    let future = match (subcommand, sub_matches) {
         ("upload", Some(arg_matches)) => {
             let starting_slot = value_t!(arg_matches, "starting_slot", Slot).unwrap_or(0);
             let ending_slot = value_t!(arg_matches, "ending_slot", Slot).ok();
@@ -547,6 +586,7 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) {
             );
             let config = solana_storage_bigtable::LedgerStorageConfig {
                 read_only: false,
+                instance_name,
                 ..solana_storage_bigtable::LedgerStorageConfig::default()
             };
             runtime.block_on(upload(
@@ -561,6 +601,7 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) {
             let slots = values_t_or_exit!(arg_matches, "slots", Slot);
             let config = solana_storage_bigtable::LedgerStorageConfig {
                 read_only: !arg_matches.is_present("force"),
+                instance_name,
                 ..solana_storage_bigtable::LedgerStorageConfig::default()
             };
             runtime.block_on(delete_slots(slots, config))
@@ -568,6 +609,7 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) {
         ("first-available-block", Some(_arg_matches)) => {
             let config = solana_storage_bigtable::LedgerStorageConfig {
                 read_only: true,
+                instance_name,
                 ..solana_storage_bigtable::LedgerStorageConfig::default()
             };
             runtime.block_on(first_available_block(config))
@@ -576,6 +618,7 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) {
             let slot = value_t_or_exit!(arg_matches, "slot", Slot);
             let config = solana_storage_bigtable::LedgerStorageConfig {
                 read_only: false,
+                instance_name,
                 ..solana_storage_bigtable::LedgerStorageConfig::default()
             };
             runtime.block_on(block(slot, output_format, config))
@@ -585,6 +628,7 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) {
             let limit = value_t_or_exit!(arg_matches, "limit", usize);
             let config = solana_storage_bigtable::LedgerStorageConfig {
                 read_only: false,
+                instance_name,
                 ..solana_storage_bigtable::LedgerStorageConfig::default()
             };
 
@@ -595,6 +639,7 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) {
             let limit = value_t_or_exit!(arg_matches, "limit", usize);
             let config = solana_storage_bigtable::LedgerStorageConfig {
                 read_only: false,
+                instance_name,
                 ..solana_storage_bigtable::LedgerStorageConfig::default()
             };
             let credential_path = Some(value_t_or_exit!(
@@ -602,9 +647,12 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) {
                 "reference_credential",
                 String
             ));
+            let ref_instance_name =
+                value_t_or_exit!(arg_matches, "reference_instance_name", String);
             let ref_config = solana_storage_bigtable::LedgerStorageConfig {
                 read_only: false,
                 credential_path,
+                instance_name: ref_instance_name,
                 ..solana_storage_bigtable::LedgerStorageConfig::default()
             };
 
@@ -618,6 +666,7 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) {
                 .expect("Invalid signature");
             let config = solana_storage_bigtable::LedgerStorageConfig {
                 read_only: false,
+                instance_name,
                 ..solana_storage_bigtable::LedgerStorageConfig::default()
             };
 
@@ -636,6 +685,7 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) {
             let show_transactions = arg_matches.is_present("show_transactions");
             let config = solana_storage_bigtable::LedgerStorageConfig {
                 read_only: true,
+                instance_name,
                 ..solana_storage_bigtable::LedgerStorageConfig::default()
             };
 

rpc/src/rpc.rs

@@ -165,12 +165,24 @@ impl JsonRpcConfig {
     }
 }
 
-#[derive(Debug, Default, Clone)]
+#[derive(Debug, Clone)]
 pub struct RpcBigtableConfig {
     pub enable_bigtable_ledger_upload: bool,
+    pub bigtable_instance_name: String,
     pub timeout: Option<Duration>,
 }
 
+impl Default for RpcBigtableConfig {
+    fn default() -> Self {
+        let bigtable_instance_name = solana_storage_bigtable::DEFAULT_INSTANCE_NAME.to_string();
+        Self {
+            enable_bigtable_ledger_upload: false,
+            bigtable_instance_name,
+            timeout: None,
+        }
+    }
+}
+
 #[derive(Clone)]
 pub struct JsonRpcRequestProcessor {
     bank_forks: Arc<RwLock<BankForks>>,

rpc/src/rpc_service.rs

@@ -378,13 +378,15 @@ impl JsonRpcService {
         let (bigtable_ledger_storage, _bigtable_ledger_upload_service) =
             if let Some(RpcBigtableConfig {
                 enable_bigtable_ledger_upload,
+                ref bigtable_instance_name,
                 timeout,
             }) = config.rpc_bigtable_config
             {
                 let bigtable_config = solana_storage_bigtable::LedgerStorageConfig {
-                    read_only: !config.enable_bigtable_ledger_upload,
+                    read_only: !enable_bigtable_ledger_upload,
-                    timeout: config.rpc_bigtable_timeout,
+                    timeout,
                     credential_path: None,
+                    instance_name: bigtable_instance_name.clone(),
                 };
                 runtime
                     .block_on(solana_storage_bigtable::LedgerStorage::new_with_config(

storage-bigtable/src/lib.rs

@@ -360,11 +360,14 @@ impl From<LegacyTransactionByAddrInfo> for TransactionByAddrInfo {
     }
 }
 
+pub const DEFAULT_INSTANCE_NAME: &str = "solana-ledger";
+
 #[derive(Debug)]
 pub struct LedgerStorageConfig {
     pub read_only: bool,
     pub timeout: Option<std::time::Duration>,
     pub credential_path: Option<String>,
+    pub instance_name: String,
 }
 
 impl Default for LedgerStorageConfig {
@@ -373,6 +376,7 @@ impl Default for LedgerStorageConfig {
             read_only: true,
             timeout: None,
             credential_path: None,
+            instance_name: DEFAULT_INSTANCE_NAME.to_string(),
         }
     }
 }
@@ -392,6 +396,7 @@ impl LedgerStorage {
             read_only,
             timeout,
             credential_path,
+            ..LedgerStorageConfig::default()
         })
         .await
     }
@@ -401,9 +406,14 @@ impl LedgerStorage {
             read_only,
             timeout,
             credential_path,
+            instance_name,
         } = config;
-        let connection =
+        let connection = bigtable::BigTableConnection::new(
-            bigtable::BigTableConnection::new("solana-ledger", read_only, timeout, credential_path)
+            instance_name.as_str(),
+            read_only,
+            timeout,
+            credential_path,
+        )
         .await?;
         Ok(Self { connection })
     }

validator/Cargo.toml

@@ -48,6 +48,7 @@ solana-rpc = { path = "../rpc", version = "=1.10.4" }
 solana-runtime = { path = "../runtime", version = "=1.10.4" }
 solana-sdk = { path = "../sdk", version = "=1.10.4" }
 solana-send-transaction-service = { path = "../send-transaction-service", version = "=1.10.4" }
+solana-storage-bigtable = { path = "../storage-bigtable", version = "=1.10.4" }
 solana-streamer = { path = "../streamer", version = "=1.10.4" }
 solana-test-validator = { path = "../test-validator", version = "=1.10.4" }
 solana-version = { path = "../version", version = "=1.10.4" }

validator/src/main.rs

@@ -1209,6 +1209,14 @@ pub fn main() {
                 .default_value("30")
                 .help("Number of seconds before timing out RPC requests backed by BigTable"),
         )
+        .arg(
+            Arg::with_name("rpc_bigtable_instance_name")
+                .long("rpc-bigtable-instance-name")
+                .takes_value(true)
+                .value_name("INSTANCE_NAME")
+                .default_value(solana_storage_bigtable::DEFAULT_INSTANCE_NAME)
+                .help("Name of the Bigtable instance to upload to")
+        )
         .arg(
             Arg::with_name("rpc_pubsub_worker_threads")
                 .long("rpc-pubsub-worker-threads")
@@ -2282,6 +2290,7 @@ pub fn main() {
     {
         Some(RpcBigtableConfig {
             enable_bigtable_ledger_upload: matches.is_present("enable_bigtable_ledger_upload"),
+            bigtable_instance_name: value_t_or_exit!(matches, "rpc_bigtable_instance_name", String),
             timeout: value_t!(matches, "rpc_bigtable_timeout", u64)
                 .ok()
                 .map(Duration::from_secs),