blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

bigtable: allow custom instance names

This commit is contained in:
Trent Nelson 2022-03-19 01:04:17 -06:00 committed by Trent Nelson
parent f513195468
commit 9b32b72990
7 changed files with 93 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Cargo.lock generated
View File

@ -6077,6 +6077,7 @@ dependencies = [
"solana-runtime",
"solana-sdk",
"solana-send-transaction-service",
"solana-storage-bigtable",
"solana-streamer",
"solana-test-validator",
"solana-version",

54
ledger-tool/src/bigtable.rs
View File

@ -322,6 +322,15 @@ impl BigTableSubCommand for App<'_, '_> {
.about("Ledger data on a BigTable instance")
.setting(AppSettings::InferSubcommands)
.setting(AppSettings::SubcommandRequiredElseHelp)
.arg(
Arg::with_name("rpc_bigtable_instance_name")
.global(true)
.long("rpc-bigtable-instance-name")
.takes_value(true)
.value_name("INSTANCE_NAME")
.default_value(solana_storage_bigtable::DEFAULT_INSTANCE_NAME)
.help("Name of the target Bigtable instance")
)
.subcommand(
SubCommand::with_name("upload")
.about("Upload the ledger to BigTable")
@ -431,7 +440,8 @@ impl BigTableSubCommand for App<'_, '_> {
.required(true)
.default_value("1000")
.help("Maximum number of slots to check"),
).arg(
)
.arg(
Arg::with_name("reference_credential")
.long("reference-credential")
.short("c")
@ -439,6 +449,14 @@ impl BigTableSubCommand for App<'_, '_> {
.takes_value(true)
.required(true)
.help("File path for a credential to a reference bigtable"),
)
.arg(
Arg::with_name("reference_instance_name")
.long("reference-instance-name")
.takes_value(true)
.value_name("INSTANCE_NAME")
.default_value(solana_storage_bigtable::DEFAULT_INSTANCE_NAME)
.help("Name of the reference Bigtable instance to compare to")
),
)
.subcommand(
@ -535,7 +553,28 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) {
let verbose = matches.is_present("verbose");
let output_format = OutputFormat::from_matches(matches, "output_format", verbose);
let future = match matches.subcommand() {
// this is kinda stupid, but there seems to be a bug in clap when a subcommand
// arg is marked both `global(true)` and `default_value("default_value")`.
// despite the "global", when the arg is specified on the subcommand, its value
// is not propagated down to the (sub)subcommand args, resulting in the default
// value when queried there. similarly, if the arg is specified on the
// (sub)subcommand, the value is not propagated back up to the subcommand args,
// again resulting in the default value. the arg having declared a
// `default_value()` obviates `is_present(...)` tests since they will always
// return true. so we consede and compare against the expected default. :/
let (subcommand, sub_matches) = matches.subcommand();
let on_command = matches
.value_of("rpc_bigtable_instance_name")
.map(|v| v != solana_storage_bigtable::DEFAULT_INSTANCE_NAME)
.unwrap_or(false);
let instance_name = if on_command {
value_t_or_exit!(matches, "rpc_bigtable_instance_name", String)
} else {
let sub_matches = sub_matches.as_ref().unwrap();
value_t_or_exit!(sub_matches, "rpc_bigtable_instance_name", String)
};
let future = match (subcommand, sub_matches) {
("upload", Some(arg_matches)) => {
let starting_slot = value_t!(arg_matches, "starting_slot", Slot).unwrap_or(0);
let ending_slot = value_t!(arg_matches, "ending_slot", Slot).ok();
@ -547,6 +586,7 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) {
);
let config = solana_storage_bigtable::LedgerStorageConfig {
read_only: false,
instance_name,
..solana_storage_bigtable::LedgerStorageConfig::default()
};
runtime.block_on(upload(
@ -561,6 +601,7 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) {
let slots = values_t_or_exit!(arg_matches, "slots", Slot);
let config = solana_storage_bigtable::LedgerStorageConfig {
read_only: !arg_matches.is_present("force"),
instance_name,
..solana_storage_bigtable::LedgerStorageConfig::default()
};
runtime.block_on(delete_slots(slots, config))
@ -568,6 +609,7 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) {
("first-available-block", Some(_arg_matches)) => {
let config = solana_storage_bigtable::LedgerStorageConfig {
read_only: true,
instance_name,
..solana_storage_bigtable::LedgerStorageConfig::default()
};
runtime.block_on(first_available_block(config))
@ -576,6 +618,7 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) {
let slot = value_t_or_exit!(arg_matches, "slot", Slot);
let config = solana_storage_bigtable::LedgerStorageConfig {
read_only: false,
instance_name,
..solana_storage_bigtable::LedgerStorageConfig::default()
};
runtime.block_on(block(slot, output_format, config))
@ -585,6 +628,7 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) {
let limit = value_t_or_exit!(arg_matches, "limit", usize);
let config = solana_storage_bigtable::LedgerStorageConfig {
read_only: false,
instance_name,
..solana_storage_bigtable::LedgerStorageConfig::default()
};
@ -595,6 +639,7 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) {
let limit = value_t_or_exit!(arg_matches, "limit", usize);
let config = solana_storage_bigtable::LedgerStorageConfig {
read_only: false,
instance_name,
..solana_storage_bigtable::LedgerStorageConfig::default()
};
let credential_path = Some(value_t_or_exit!(
@ -602,9 +647,12 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) {
"reference_credential",
String
));
let ref_instance_name =
value_t_or_exit!(arg_matches, "reference_instance_name", String);
let ref_config = solana_storage_bigtable::LedgerStorageConfig {
read_only: false,
credential_path,
instance_name: ref_instance_name,
..solana_storage_bigtable::LedgerStorageConfig::default()
};
@ -618,6 +666,7 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) {
.expect("Invalid signature");
let config = solana_storage_bigtable::LedgerStorageConfig {
read_only: false,
instance_name,
..solana_storage_bigtable::LedgerStorageConfig::default()
};
@ -636,6 +685,7 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) {
let show_transactions = arg_matches.is_present("show_transactions");
let config = solana_storage_bigtable::LedgerStorageConfig {
read_only: true,
instance_name,
..solana_storage_bigtable::LedgerStorageConfig::default()
};

14
rpc/src/rpc.rs
View File

@ -165,12 +165,24 @@ impl JsonRpcConfig {
}
}
#[derive(Debug, Default, Clone)]
#[derive(Debug, Clone)]
pub struct RpcBigtableConfig {
pub enable_bigtable_ledger_upload: bool,
pub bigtable_instance_name: String,
pub timeout: Option<Duration>,
}
impl Default for RpcBigtableConfig {
fn default() -> Self {
let bigtable_instance_name = solana_storage_bigtable::DEFAULT_INSTANCE_NAME.to_string();
Self {
enable_bigtable_ledger_upload: false,
bigtable_instance_name,
timeout: None,
}
}
}
#[derive(Clone)]
pub struct JsonRpcRequestProcessor {
bank_forks: Arc<RwLock<BankForks>>,

6
rpc/src/rpc_service.rs
View File

@ -378,13 +378,15 @@ impl JsonRpcService {
let (bigtable_ledger_storage, _bigtable_ledger_upload_service) =
if let Some(RpcBigtableConfig {
enable_bigtable_ledger_upload,
ref bigtable_instance_name,
timeout,
}) = config.rpc_bigtable_config
{
let bigtable_config = solana_storage_bigtable::LedgerStorageConfig {
read_only: !config.enable_bigtable_ledger_upload,
timeout: config.rpc_bigtable_timeout,
read_only: !enable_bigtable_ledger_upload,
timeout,
credential_path: None,
instance_name: bigtable_instance_name.clone(),
};
runtime
.block_on(solana_storage_bigtable::LedgerStorage::new_with_config(

16
storage-bigtable/src/lib.rs
View File

@ -360,11 +360,14 @@ impl From<LegacyTransactionByAddrInfo> for TransactionByAddrInfo {
}
}
pub const DEFAULT_INSTANCE_NAME: &str = "solana-ledger";
#[derive(Debug)]
pub struct LedgerStorageConfig {
pub read_only: bool,
pub timeout: Option<std::time::Duration>,
pub credential_path: Option<String>,
pub instance_name: String,
}
impl Default for LedgerStorageConfig {
@ -373,6 +376,7 @@ impl Default for LedgerStorageConfig {
read_only: true,
timeout: None,
credential_path: None,
instance_name: DEFAULT_INSTANCE_NAME.to_string(),
}
}
}
@ -392,6 +396,7 @@ impl LedgerStorage {
read_only,
timeout,
credential_path,
..LedgerStorageConfig::default()
})
.await
}
@ -401,10 +406,15 @@ impl LedgerStorage {
read_only,
timeout,
credential_path,
instance_name,
} = config;
let connection =
bigtable::BigTableConnection::new("solana-ledger", read_only, timeout, credential_path)
.await?;
let connection = bigtable::BigTableConnection::new(
instance_name.as_str(),
read_only,
timeout,
credential_path,
)
.await?;
Ok(Self { connection })
}

1
validator/Cargo.toml
View File

@ -48,6 +48,7 @@ solana-rpc = { path = "../rpc", version = "=1.10.4" }
solana-runtime = { path = "../runtime", version = "=1.10.4" }
solana-sdk = { path = "../sdk", version = "=1.10.4" }
solana-send-transaction-service = { path = "../send-transaction-service", version = "=1.10.4" }
solana-storage-bigtable = { path = "../storage-bigtable", version = "=1.10.4" }
solana-streamer = { path = "../streamer", version = "=1.10.4" }
solana-test-validator = { path = "../test-validator", version = "=1.10.4" }
solana-version = { path = "../version", version = "=1.10.4" }

9
validator/src/main.rs
View File

@ -1209,6 +1209,14 @@ pub fn main() {
.default_value("30")
.help("Number of seconds before timing out RPC requests backed by BigTable"),
)
.arg(
Arg::with_name("rpc_bigtable_instance_name")
.long("rpc-bigtable-instance-name")
.takes_value(true)
.value_name("INSTANCE_NAME")
.default_value(solana_storage_bigtable::DEFAULT_INSTANCE_NAME)
.help("Name of the Bigtable instance to upload to")
)
.arg(
Arg::with_name("rpc_pubsub_worker_threads")
.long("rpc-pubsub-worker-threads")
@ -2282,6 +2290,7 @@ pub fn main() {
{
Some(RpcBigtableConfig {
enable_bigtable_ledger_upload: matches.is_present("enable_bigtable_ledger_upload"),
bigtable_instance_name: value_t_or_exit!(matches, "rpc_bigtable_instance_name", String),
timeout: value_t!(matches, "rpc_bigtable_timeout", u64)
.ok()
.map(Duration::from_secs),