Add mechanism to load v0.22.3 snapshots on newer Solana versions
This commit is contained in:
parent
9bd6be779f
commit
934c32cbc6
|
@ -1,6 +1,6 @@
|
||||||
use solana_ledger::snapshot_package::{SnapshotPackage, SnapshotPackageReceiver};
|
use solana_ledger::snapshot_package::{SnapshotPackage, SnapshotPackageReceiver};
|
||||||
use solana_ledger::snapshot_utils::{
|
use solana_ledger::snapshot_utils::{
|
||||||
serialize_status_cache, SnapshotError, TAR_ACCOUNTS_DIR, TAR_SNAPSHOTS_DIR,
|
serialize_status_cache, SnapshotError, TAR_ACCOUNTS_DIR, TAR_SNAPSHOTS_DIR, TAR_VERSION_FILE,
|
||||||
};
|
};
|
||||||
use solana_measure::measure::Measure;
|
use solana_measure::measure::Measure;
|
||||||
use solana_metrics::datapoint_info;
|
use solana_metrics::datapoint_info;
|
||||||
|
@ -91,6 +91,7 @@ impl SnapshotPackagerService {
|
||||||
let staging_dir = TempDir::new()?;
|
let staging_dir = TempDir::new()?;
|
||||||
let staging_accounts_dir = staging_dir.path().join(TAR_ACCOUNTS_DIR);
|
let staging_accounts_dir = staging_dir.path().join(TAR_ACCOUNTS_DIR);
|
||||||
let staging_snapshots_dir = staging_dir.path().join(TAR_SNAPSHOTS_DIR);
|
let staging_snapshots_dir = staging_dir.path().join(TAR_SNAPSHOTS_DIR);
|
||||||
|
let staging_version_file = staging_dir.path().join(TAR_VERSION_FILE);
|
||||||
fs::create_dir_all(&staging_accounts_dir)?;
|
fs::create_dir_all(&staging_accounts_dir)?;
|
||||||
|
|
||||||
// Add the snapshots to the staging directory
|
// Add the snapshots to the staging directory
|
||||||
|
@ -119,6 +120,15 @@ impl SnapshotPackagerService {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Write version file
|
||||||
|
{
|
||||||
|
use std::io::Write;
|
||||||
|
let snapshot_version = format!("{}\n", env!("CARGO_PKG_VERSION"));
|
||||||
|
let mut f = std::fs::File::create(staging_version_file)?;
|
||||||
|
//f.write_all(&snapshot_version.to_string().into_bytes())?;
|
||||||
|
f.write_all(&snapshot_version.into_bytes())?;
|
||||||
|
}
|
||||||
|
|
||||||
// Tar the staging directory into the archive at `archive_path`
|
// Tar the staging directory into the archive at `archive_path`
|
||||||
let archive_path = tar_dir.join("new_state.tar.bz2");
|
let archive_path = tar_dir.join("new_state.tar.bz2");
|
||||||
let args = vec![
|
let args = vec![
|
||||||
|
@ -128,6 +138,7 @@ impl SnapshotPackagerService {
|
||||||
staging_dir.path().to_str().unwrap(),
|
staging_dir.path().to_str().unwrap(),
|
||||||
TAR_ACCOUNTS_DIR,
|
TAR_ACCOUNTS_DIR,
|
||||||
TAR_SNAPSHOTS_DIR,
|
TAR_SNAPSHOTS_DIR,
|
||||||
|
TAR_VERSION_FILE,
|
||||||
];
|
];
|
||||||
|
|
||||||
let output = std::process::Command::new("tar").args(&args).output()?;
|
let output = std::process::Command::new("tar").args(&args).output()?;
|
||||||
|
|
|
@ -24,6 +24,7 @@ use thiserror::Error;
|
||||||
pub const SNAPSHOT_STATUS_CACHE_FILE_NAME: &str = "status_cache";
|
pub const SNAPSHOT_STATUS_CACHE_FILE_NAME: &str = "status_cache";
|
||||||
pub const TAR_SNAPSHOTS_DIR: &str = "snapshots";
|
pub const TAR_SNAPSHOTS_DIR: &str = "snapshots";
|
||||||
pub const TAR_ACCOUNTS_DIR: &str = "accounts";
|
pub const TAR_ACCOUNTS_DIR: &str = "accounts";
|
||||||
|
pub const TAR_VERSION_FILE: &str = "version";
|
||||||
|
|
||||||
#[derive(PartialEq, Ord, Eq, Debug)]
|
#[derive(PartialEq, Ord, Eq, Debug)]
|
||||||
pub struct SlotSnapshotPaths {
|
pub struct SlotSnapshotPaths {
|
||||||
|
@ -328,7 +329,20 @@ pub fn bank_from_archive<P: AsRef<Path>>(
|
||||||
let mut measure = Measure::start("bank rebuild from snapshot");
|
let mut measure = Measure::start("bank rebuild from snapshot");
|
||||||
let unpacked_accounts_dir = unpack_dir.as_ref().join(TAR_ACCOUNTS_DIR);
|
let unpacked_accounts_dir = unpack_dir.as_ref().join(TAR_ACCOUNTS_DIR);
|
||||||
let unpacked_snapshots_dir = unpack_dir.as_ref().join(TAR_SNAPSHOTS_DIR);
|
let unpacked_snapshots_dir = unpack_dir.as_ref().join(TAR_SNAPSHOTS_DIR);
|
||||||
|
let unpacked_version_file = unpack_dir.as_ref().join(TAR_VERSION_FILE);
|
||||||
|
|
||||||
|
let snapshot_version = if let Ok(mut f) = File::open(unpacked_version_file) {
|
||||||
|
let mut snapshot_version = String::new();
|
||||||
|
f.read_to_string(&mut snapshot_version)?;
|
||||||
|
snapshot_version
|
||||||
|
} else {
|
||||||
|
// Once v0.23.x is deployed, this default can be removed and snapshots without a version
|
||||||
|
// file can be rejected
|
||||||
|
String::from("v0.22.3")
|
||||||
|
};
|
||||||
|
|
||||||
let bank = rebuild_bank_from_snapshots(
|
let bank = rebuild_bank_from_snapshots(
|
||||||
|
snapshot_version.trim(),
|
||||||
account_paths,
|
account_paths,
|
||||||
&unpacked_snapshots_dir,
|
&unpacked_snapshots_dir,
|
||||||
unpacked_accounts_dir,
|
unpacked_accounts_dir,
|
||||||
|
@ -376,6 +390,7 @@ pub fn untar_snapshot_in<P: AsRef<Path>, Q: AsRef<Path>>(
|
||||||
}
|
}
|
||||||
|
|
||||||
fn rebuild_bank_from_snapshots<P>(
|
fn rebuild_bank_from_snapshots<P>(
|
||||||
|
snapshot_version: &str,
|
||||||
local_account_paths: &[PathBuf],
|
local_account_paths: &[PathBuf],
|
||||||
unpacked_snapshots_dir: &PathBuf,
|
unpacked_snapshots_dir: &PathBuf,
|
||||||
append_vecs_path: P,
|
append_vecs_path: P,
|
||||||
|
@ -383,6 +398,8 @@ fn rebuild_bank_from_snapshots<P>(
|
||||||
where
|
where
|
||||||
P: AsRef<Path>,
|
P: AsRef<Path>,
|
||||||
{
|
{
|
||||||
|
info!("snapshot version: {}", snapshot_version);
|
||||||
|
|
||||||
let mut snapshot_paths = get_snapshot_paths(&unpacked_snapshots_dir);
|
let mut snapshot_paths = get_snapshot_paths(&unpacked_snapshots_dir);
|
||||||
if snapshot_paths.len() > 1 {
|
if snapshot_paths.len() > 1 {
|
||||||
return Err(get_io_error("invalid snapshot format"));
|
return Err(get_io_error("invalid snapshot format"));
|
||||||
|
@ -391,13 +408,25 @@ where
|
||||||
.pop()
|
.pop()
|
||||||
.ok_or_else(|| get_io_error("No snapshots found in snapshots directory"))?;
|
.ok_or_else(|| get_io_error("No snapshots found in snapshots directory"))?;
|
||||||
|
|
||||||
info!("Loading from {:?}", &root_paths.snapshot_file_path);
|
info!("Loading bank from {:?}", &root_paths.snapshot_file_path);
|
||||||
let bank = deserialize_snapshot_data_file(
|
let bank = deserialize_snapshot_data_file(
|
||||||
&root_paths.snapshot_file_path,
|
&root_paths.snapshot_file_path,
|
||||||
MAX_SNAPSHOT_DATA_FILE_SIZE,
|
MAX_SNAPSHOT_DATA_FILE_SIZE,
|
||||||
|stream| {
|
|stream| {
|
||||||
// Rebuild the root bank
|
let bank: Bank = match snapshot_version {
|
||||||
let bank: Bank = deserialize_from_snapshot(stream.by_ref())?;
|
env!("CARGO_PKG_VERSION") => deserialize_from_snapshot(stream.by_ref())?,
|
||||||
|
"v0.22.3" => {
|
||||||
|
let bank0223: solana_runtime::bank::LegacyBank0223 =
|
||||||
|
deserialize_from_snapshot(stream.by_ref())?;
|
||||||
|
bank0223.into()
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
return Err(get_io_error(&format!(
|
||||||
|
"unsupported snapshot version: {}",
|
||||||
|
snapshot_version
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
};
|
||||||
// Rebuild accounts
|
// Rebuild accounts
|
||||||
bank.rc.accounts_from_stream(
|
bank.rc.accounts_from_stream(
|
||||||
stream.by_ref(),
|
stream.by_ref(),
|
||||||
|
@ -415,7 +444,7 @@ where
|
||||||
|stream| {
|
|stream| {
|
||||||
// Rebuild status cache
|
// Rebuild status cache
|
||||||
let slot_deltas: Vec<SlotDelta<transaction::Result<()>>> =
|
let slot_deltas: Vec<SlotDelta<transaction::Result<()>>> =
|
||||||
deserialize_from_snapshot(stream).unwrap_or_default();
|
deserialize_from_snapshot(stream)?;
|
||||||
|
|
||||||
Ok(slot_deltas)
|
Ok(slot_deltas)
|
||||||
},
|
},
|
||||||
|
|
|
@ -2043,6 +2043,121 @@ where
|
||||||
.deserialize_from(reader)
|
.deserialize_from(reader)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The `bank` struct as defined in Solana v0.22.3
|
||||||
|
#[derive(Default, Deserialize, Serialize)]
|
||||||
|
pub struct LegacyBank0223 {
|
||||||
|
#[serde(skip)]
|
||||||
|
pub rc: BankRc,
|
||||||
|
|
||||||
|
#[serde(skip)]
|
||||||
|
pub src: StatusCacheRc,
|
||||||
|
|
||||||
|
pub blockhash_queue: RwLock<BlockhashQueue>,
|
||||||
|
pub ancestors: HashMap<Slot, usize>,
|
||||||
|
pub hash: RwLock<Hash>,
|
||||||
|
pub parent_hash: Hash,
|
||||||
|
pub parent_slot: Slot,
|
||||||
|
|
||||||
|
#[serde(serialize_with = "serialize_atomicu64")]
|
||||||
|
#[serde(deserialize_with = "deserialize_atomicu64")]
|
||||||
|
pub transaction_count: AtomicU64,
|
||||||
|
|
||||||
|
#[serde(serialize_with = "serialize_atomicu64")]
|
||||||
|
#[serde(deserialize_with = "deserialize_atomicu64")]
|
||||||
|
pub tick_height: AtomicU64,
|
||||||
|
|
||||||
|
#[serde(serialize_with = "serialize_atomicu64")]
|
||||||
|
#[serde(deserialize_with = "deserialize_atomicu64")]
|
||||||
|
pub signature_count: AtomicU64,
|
||||||
|
|
||||||
|
#[serde(serialize_with = "serialize_atomicu64")]
|
||||||
|
#[serde(deserialize_with = "deserialize_atomicu64")]
|
||||||
|
pub capitalization: AtomicU64,
|
||||||
|
|
||||||
|
pub max_tick_height: u64,
|
||||||
|
pub hashes_per_tick: Option<u64>,
|
||||||
|
pub ticks_per_slot: u64,
|
||||||
|
pub ns_per_slot: u128,
|
||||||
|
pub genesis_creation_time: UnixTimestamp,
|
||||||
|
pub slots_per_year: f64,
|
||||||
|
pub slots_per_segment: u64,
|
||||||
|
pub slot: Slot,
|
||||||
|
pub epoch: Epoch,
|
||||||
|
pub block_height: u64,
|
||||||
|
pub collector_id: Pubkey,
|
||||||
|
|
||||||
|
#[serde(serialize_with = "serialize_atomicu64")]
|
||||||
|
#[serde(deserialize_with = "deserialize_atomicu64")]
|
||||||
|
pub collector_fees: AtomicU64,
|
||||||
|
|
||||||
|
pub fee_calculator: FeeCalculator,
|
||||||
|
|
||||||
|
#[serde(serialize_with = "serialize_atomicu64")]
|
||||||
|
#[serde(deserialize_with = "deserialize_atomicu64")]
|
||||||
|
pub collected_rent: AtomicU64,
|
||||||
|
|
||||||
|
pub rent_collector: RentCollector,
|
||||||
|
pub epoch_schedule: EpochSchedule,
|
||||||
|
pub inflation: Arc<RwLock<Inflation>>,
|
||||||
|
pub stakes: RwLock<Stakes>,
|
||||||
|
pub storage_accounts: RwLock<StorageAccounts>,
|
||||||
|
pub epoch_stakes: HashMap<Epoch, Stakes>,
|
||||||
|
|
||||||
|
#[serde(serialize_with = "serialize_atomicbool")]
|
||||||
|
#[serde(deserialize_with = "deserialize_atomicbool")]
|
||||||
|
pub is_delta: AtomicBool,
|
||||||
|
|
||||||
|
pub message_processor: MessageProcessor,
|
||||||
|
|
||||||
|
#[serde(skip)]
|
||||||
|
pub entered_epoch_callback: Arc<RwLock<Option<EnteredEpochCallback>>>,
|
||||||
|
|
||||||
|
#[serde(skip)]
|
||||||
|
pub last_vote_sync: AtomicU64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<LegacyBank0223> for Bank {
|
||||||
|
fn from(bank: LegacyBank0223) -> Self {
|
||||||
|
Bank {
|
||||||
|
rc: bank.rc,
|
||||||
|
src: bank.src,
|
||||||
|
slot: bank.slot,
|
||||||
|
epoch: bank.epoch,
|
||||||
|
blockhash_queue: bank.blockhash_queue,
|
||||||
|
hashes_per_tick: bank.hashes_per_tick,
|
||||||
|
ticks_per_slot: bank.ticks_per_slot,
|
||||||
|
ns_per_slot: bank.ns_per_slot,
|
||||||
|
genesis_creation_time: bank.genesis_creation_time,
|
||||||
|
slots_per_segment: bank.slots_per_segment,
|
||||||
|
slots_per_year: bank.slots_per_year,
|
||||||
|
epoch_schedule: bank.epoch_schedule,
|
||||||
|
collected_rent: bank.collected_rent,
|
||||||
|
rent_collector: bank.rent_collector,
|
||||||
|
max_tick_height: bank.max_tick_height,
|
||||||
|
block_height: bank.block_height,
|
||||||
|
fee_calculator: bank.fee_calculator,
|
||||||
|
capitalization: bank.capitalization,
|
||||||
|
inflation: bank.inflation,
|
||||||
|
transaction_count: bank.transaction_count,
|
||||||
|
stakes: bank.stakes,
|
||||||
|
epoch_stakes: bank.epoch_stakes,
|
||||||
|
storage_accounts: bank.storage_accounts,
|
||||||
|
parent_hash: bank.parent_hash,
|
||||||
|
parent_slot: bank.parent_slot,
|
||||||
|
collector_id: bank.collector_id,
|
||||||
|
collector_fees: bank.collector_fees,
|
||||||
|
ancestors: bank.ancestors,
|
||||||
|
hash: bank.hash,
|
||||||
|
is_delta: bank.is_delta,
|
||||||
|
tick_height: bank.tick_height,
|
||||||
|
signature_count: bank.signature_count,
|
||||||
|
message_processor: bank.message_processor,
|
||||||
|
entered_epoch_callback: bank.entered_epoch_callback,
|
||||||
|
last_vote_sync: bank.last_vote_sync,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
Loading…
Reference in New Issue