Add --force arg for bigtable upload
This commit is contained in:
parent
f5c564bc6c
commit
98e3e570d2
|
@ -74,6 +74,7 @@ impl BigTableUploadService {
|
||||||
start_slot,
|
start_slot,
|
||||||
Some(end_slot),
|
Some(end_slot),
|
||||||
true,
|
true,
|
||||||
|
false,
|
||||||
exit.clone(),
|
exit.clone(),
|
||||||
));
|
));
|
||||||
|
|
||||||
|
|
|
@ -20,6 +20,7 @@ async fn upload(
|
||||||
starting_slot: Slot,
|
starting_slot: Slot,
|
||||||
ending_slot: Option<Slot>,
|
ending_slot: Option<Slot>,
|
||||||
allow_missing_metadata: bool,
|
allow_missing_metadata: bool,
|
||||||
|
force_reupload: bool,
|
||||||
) -> Result<(), Box<dyn std::error::Error>> {
|
) -> Result<(), Box<dyn std::error::Error>> {
|
||||||
let bigtable = solana_storage_bigtable::LedgerStorage::new(false, None)
|
let bigtable = solana_storage_bigtable::LedgerStorage::new(false, None)
|
||||||
.await
|
.await
|
||||||
|
@ -31,6 +32,7 @@ async fn upload(
|
||||||
starting_slot,
|
starting_slot,
|
||||||
ending_slot,
|
ending_slot,
|
||||||
allow_missing_metadata,
|
allow_missing_metadata,
|
||||||
|
force_reupload,
|
||||||
Arc::new(AtomicBool::new(false)),
|
Arc::new(AtomicBool::new(false)),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
|
@ -247,6 +249,16 @@ impl BigTableSubCommand for App<'_, '_> {
|
||||||
.long("allow-missing-metadata")
|
.long("allow-missing-metadata")
|
||||||
.takes_value(false)
|
.takes_value(false)
|
||||||
.help("Don't panic if transaction metadata is missing"),
|
.help("Don't panic if transaction metadata is missing"),
|
||||||
|
)
|
||||||
|
.arg(
|
||||||
|
Arg::with_name("force_reupload")
|
||||||
|
.long("force")
|
||||||
|
.takes_value(false)
|
||||||
|
.help(
|
||||||
|
"Force reupload of any blocks already present in BigTable instance\
|
||||||
|
Note: reupload will *not* delete any data from the tx-by-addr table;\
|
||||||
|
Use with care.",
|
||||||
|
),
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
.subcommand(
|
.subcommand(
|
||||||
|
@ -389,6 +401,7 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) {
|
||||||
let starting_slot = value_t!(arg_matches, "starting_slot", Slot).unwrap_or(0);
|
let starting_slot = value_t!(arg_matches, "starting_slot", Slot).unwrap_or(0);
|
||||||
let ending_slot = value_t!(arg_matches, "ending_slot", Slot).ok();
|
let ending_slot = value_t!(arg_matches, "ending_slot", Slot).ok();
|
||||||
let allow_missing_metadata = arg_matches.is_present("allow_missing_metadata");
|
let allow_missing_metadata = arg_matches.is_present("allow_missing_metadata");
|
||||||
|
let force_reupload = arg_matches.is_present("force_reupload");
|
||||||
let blockstore =
|
let blockstore =
|
||||||
crate::open_blockstore(&ledger_path, AccessType::TryPrimaryThenSecondary, None);
|
crate::open_blockstore(&ledger_path, AccessType::TryPrimaryThenSecondary, None);
|
||||||
|
|
||||||
|
@ -397,6 +410,7 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) {
|
||||||
starting_slot,
|
starting_slot,
|
||||||
ending_slot,
|
ending_slot,
|
||||||
allow_missing_metadata,
|
allow_missing_metadata,
|
||||||
|
force_reupload,
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
("first-available-block", Some(_arg_matches)) => runtime.block_on(first_available_block()),
|
("first-available-block", Some(_arg_matches)) => runtime.block_on(first_available_block()),
|
||||||
|
|
|
@ -24,6 +24,7 @@ pub async fn upload_confirmed_blocks(
|
||||||
starting_slot: Slot,
|
starting_slot: Slot,
|
||||||
ending_slot: Option<Slot>,
|
ending_slot: Option<Slot>,
|
||||||
allow_missing_metadata: bool,
|
allow_missing_metadata: bool,
|
||||||
|
force_reupload: bool,
|
||||||
exit: Arc<AtomicBool>,
|
exit: Arc<AtomicBool>,
|
||||||
) -> Result<(), Box<dyn std::error::Error>> {
|
) -> Result<(), Box<dyn std::error::Error>> {
|
||||||
let mut measure = Measure::start("entire upload");
|
let mut measure = Measure::start("entire upload");
|
||||||
|
@ -63,7 +64,7 @@ pub async fn upload_confirmed_blocks(
|
||||||
);
|
);
|
||||||
|
|
||||||
// Gather the blocks that are already present in bigtable, by slot
|
// Gather the blocks that are already present in bigtable, by slot
|
||||||
let bigtable_slots = {
|
let bigtable_slots = if !force_reupload {
|
||||||
let mut bigtable_slots = vec![];
|
let mut bigtable_slots = vec![];
|
||||||
let first_blockstore_slot = *blockstore_slots.first().unwrap();
|
let first_blockstore_slot = *blockstore_slots.first().unwrap();
|
||||||
let last_blockstore_slot = *blockstore_slots.last().unwrap();
|
let last_blockstore_slot = *blockstore_slots.last().unwrap();
|
||||||
|
@ -94,6 +95,8 @@ pub async fn upload_confirmed_blocks(
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.filter(|slot| *slot <= last_blockstore_slot)
|
.filter(|slot| *slot <= last_blockstore_slot)
|
||||||
.collect::<Vec<_>>()
|
.collect::<Vec<_>>()
|
||||||
|
} else {
|
||||||
|
Vec::new()
|
||||||
};
|
};
|
||||||
|
|
||||||
// The blocks that still need to be uploaded is the difference between what's already in the
|
// The blocks that still need to be uploaded is the difference between what's already in the
|
||||||
|
|
Loading…
Reference in New Issue