2021-11-01 13:48:17 -07:00
|
|
|
use jsonrpc_core::futures::StreamExt;
|
|
|
|
use jsonrpc_core_client::transports::http;
|
|
|
|
|
|
|
|
use solana_account_decoder::UiAccountEncoding;
|
|
|
|
use solana_client::rpc_config::{RpcAccountInfoConfig, RpcProgramAccountsConfig};
|
2021-11-02 06:35:45 -07:00
|
|
|
use solana_client::rpc_response::{Response, RpcKeyedAccount};
|
2022-03-22 04:54:38 -07:00
|
|
|
use solana_rpc::{rpc::rpc_accounts::AccountsDataClient, rpc::OptionalContext};
|
2021-11-02 06:35:45 -07:00
|
|
|
use solana_sdk::{account::Account, commitment_config::CommitmentConfig, pubkey::Pubkey};
|
2021-11-01 13:48:17 -07:00
|
|
|
|
2021-11-07 01:52:38 -08:00
|
|
|
use futures::{future, future::FutureExt};
|
2021-11-18 05:21:17 -08:00
|
|
|
use tonic::transport::{Certificate, ClientTlsConfig, Endpoint, Identity};
|
2021-11-02 05:22:13 -07:00
|
|
|
|
2021-11-02 00:55:39 -07:00
|
|
|
use log::*;
|
2021-11-08 06:31:35 -08:00
|
|
|
use std::{collections::HashMap, str::FromStr, time::Duration};
|
2021-11-01 13:48:17 -07:00
|
|
|
|
2022-03-22 01:36:59 -07:00
|
|
|
pub mod geyser_proto {
|
2022-03-23 00:16:02 -07:00
|
|
|
tonic::include_proto!("accountsdb");
|
2021-11-01 13:48:17 -07:00
|
|
|
}
|
2022-03-22 04:54:38 -07:00
|
|
|
use geyser_proto::accounts_db_client::AccountsDbClient;
|
2021-11-01 13:48:17 -07:00
|
|
|
|
2021-11-08 06:48:50 -08:00
|
|
|
use crate::{
|
2021-11-08 11:15:46 -08:00
|
|
|
metrics, AccountWrite, AnyhowWrap, Config, GrpcSourceConfig, SlotStatus, SlotUpdate,
|
2021-11-18 05:21:17 -08:00
|
|
|
SnapshotSourceConfig, TlsConfig,
|
2021-11-08 06:48:50 -08:00
|
|
|
};
|
2021-11-01 13:48:17 -07:00
|
|
|
|
2021-11-07 01:52:38 -08:00
|
|
|
type SnapshotData = Response<Vec<RpcKeyedAccount>>;
|
|
|
|
|
2021-11-02 06:35:45 -07:00
|
|
|
enum Message {
|
2022-03-22 01:36:59 -07:00
|
|
|
GrpcUpdate(geyser_proto::Update),
|
2021-11-07 01:52:38 -08:00
|
|
|
Snapshot(SnapshotData),
|
2021-11-02 06:35:45 -07:00
|
|
|
}
|
|
|
|
|
2021-11-08 00:57:56 -08:00
|
|
|
async fn get_snapshot(
|
|
|
|
rpc_http_url: String,
|
|
|
|
program_id: Pubkey,
|
2021-12-13 07:46:36 -08:00
|
|
|
) -> anyhow::Result<OptionalContext<Vec<RpcKeyedAccount>>> {
|
2022-03-22 01:36:59 -07:00
|
|
|
let rpc_client = http::connect_with_options::<AccountsDataClient>(&rpc_http_url, true)
|
2021-11-01 13:48:17 -07:00
|
|
|
.await
|
|
|
|
.map_err_anyhow()?;
|
|
|
|
|
|
|
|
let account_info_config = RpcAccountInfoConfig {
|
|
|
|
encoding: Some(UiAccountEncoding::Base64),
|
2021-12-13 07:46:36 -08:00
|
|
|
commitment: Some(CommitmentConfig::finalized()),
|
2021-11-01 13:48:17 -07:00
|
|
|
data_slice: None,
|
|
|
|
};
|
|
|
|
let program_accounts_config = RpcProgramAccountsConfig {
|
2021-11-07 01:52:38 -08:00
|
|
|
filters: None,
|
2021-11-01 13:48:17 -07:00
|
|
|
with_context: Some(true),
|
|
|
|
account_config: account_info_config.clone(),
|
|
|
|
};
|
|
|
|
|
2021-11-02 06:35:45 -07:00
|
|
|
info!("requesting snapshot");
|
2021-11-01 13:48:17 -07:00
|
|
|
let account_snapshot = rpc_client
|
|
|
|
.get_program_accounts(
|
|
|
|
program_id.to_string(),
|
|
|
|
Some(program_accounts_config.clone()),
|
|
|
|
)
|
|
|
|
.await
|
|
|
|
.map_err_anyhow()?;
|
2021-12-13 07:46:36 -08:00
|
|
|
info!("snapshot received");
|
|
|
|
Ok(account_snapshot)
|
2021-11-07 01:52:38 -08:00
|
|
|
}
|
|
|
|
|
2022-03-22 01:36:59 -07:00
|
|
|
async fn feed_data_geyser(
|
2021-11-08 06:48:50 -08:00
|
|
|
grpc_config: &GrpcSourceConfig,
|
2021-11-18 05:21:17 -08:00
|
|
|
tls_config: Option<ClientTlsConfig>,
|
2021-11-08 06:48:50 -08:00
|
|
|
snapshot_config: &SnapshotSourceConfig,
|
2021-11-07 01:52:38 -08:00
|
|
|
sender: async_channel::Sender<Message>,
|
2021-11-09 05:27:09 -08:00
|
|
|
) -> anyhow::Result<()> {
|
2021-11-08 06:48:50 -08:00
|
|
|
let program_id = Pubkey::from_str(&snapshot_config.program_id)?;
|
2021-11-08 00:57:56 -08:00
|
|
|
|
2021-11-18 05:21:17 -08:00
|
|
|
let endpoint = Endpoint::from_str(&grpc_config.connection_string)?;
|
|
|
|
let channel = if let Some(tls) = tls_config {
|
|
|
|
endpoint.tls_config(tls)?
|
|
|
|
} else {
|
|
|
|
endpoint
|
|
|
|
}
|
|
|
|
.connect()
|
|
|
|
.await?;
|
2022-03-22 04:54:38 -07:00
|
|
|
let mut client = AccountsDbClient::new(channel);
|
2021-11-07 01:52:38 -08:00
|
|
|
|
|
|
|
let mut update_stream = client
|
2022-03-22 01:36:59 -07:00
|
|
|
.subscribe(geyser_proto::SubscribeRequest {})
|
2021-11-07 01:52:38 -08:00
|
|
|
.await?
|
|
|
|
.into_inner();
|
|
|
|
|
2021-12-13 07:46:36 -08:00
|
|
|
// We can't get a snapshot immediately since the finalized snapshot would be for a
|
|
|
|
// slot in the past and we'd be missing intermediate updates.
|
|
|
|
//
|
2022-01-03 00:13:32 -08:00
|
|
|
// Delay the request until the first slot we received all writes for becomes rooted
|
2021-12-13 07:46:36 -08:00
|
|
|
// to avoid that problem - partially. The rooted slot will still be larger than the
|
|
|
|
// finalized slot, so add a number of slots as a buffer.
|
|
|
|
//
|
|
|
|
// If that buffer isn't sufficient, there'll be a retry.
|
|
|
|
|
2022-01-03 00:13:32 -08:00
|
|
|
// The first slot that we will receive _all_ account writes for
|
|
|
|
let mut first_full_slot: u64 = u64::MAX;
|
|
|
|
|
2021-12-13 07:46:36 -08:00
|
|
|
// If a snapshot should be performed when ready.
|
|
|
|
let mut snapshot_needed = true;
|
|
|
|
|
2022-01-03 00:13:32 -08:00
|
|
|
// The highest "rooted" slot that has been seen.
|
|
|
|
let mut max_rooted_slot = 0;
|
|
|
|
|
|
|
|
// Data for slots will arrive out of order. This value defines how many
|
|
|
|
// slots after a slot was marked "rooted" we assume it'll not receive
|
|
|
|
// any more account write information.
|
|
|
|
//
|
|
|
|
// This is important for the write_version mapping (to know when slots can
|
|
|
|
// be dropped).
|
|
|
|
let max_out_of_order_slots = 40;
|
2021-12-13 07:46:36 -08:00
|
|
|
|
|
|
|
// Number of slots that we expect "finalized" commitment to lag
|
2022-01-03 00:13:32 -08:00
|
|
|
// behind "rooted". This matters for getProgramAccounts based snapshots,
|
|
|
|
// which will have "finalized" commitment.
|
2021-12-13 07:46:36 -08:00
|
|
|
let mut rooted_to_finalized_slots = 30;
|
|
|
|
|
2021-11-07 01:52:38 -08:00
|
|
|
let mut snapshot_future = future::Fuse::terminated();
|
2021-11-01 13:48:17 -07:00
|
|
|
|
2021-11-08 02:42:22 -08:00
|
|
|
// The plugin sends a ping every 5s or so
|
|
|
|
let fatal_idle_timeout = Duration::from_secs(60);
|
|
|
|
|
2022-01-03 00:13:32 -08:00
|
|
|
// Highest slot that an account write came in for.
|
|
|
|
let mut newest_write_slot: u64 = 0;
|
|
|
|
|
|
|
|
// map slot -> (pubkey -> write count)
|
|
|
|
//
|
|
|
|
// Since the write_version is a private indentifier per node it can't be used
|
|
|
|
// to deduplicate events from multiple nodes. Here we rewrite it such that each
|
|
|
|
// pubkey and each slot has a consecutive numbering of writes starting at 1.
|
|
|
|
//
|
|
|
|
// That number will be consistent for each node.
|
|
|
|
let mut slot_pubkey_writes = HashMap::<u64, HashMap<[u8; 32], u32>>::new();
|
2021-12-20 06:06:23 -08:00
|
|
|
|
2022-01-03 00:13:32 -08:00
|
|
|
// Keep track of write version from RPC node, to check the assumption that it
|
|
|
|
// increases monotonically
|
2021-12-20 06:06:23 -08:00
|
|
|
let mut last_write_version: u64 = 0;
|
|
|
|
|
2021-11-01 13:48:17 -07:00
|
|
|
loop {
|
|
|
|
tokio::select! {
|
|
|
|
update = update_stream.next() => {
|
2022-03-22 01:36:59 -07:00
|
|
|
use geyser_proto::{update::UpdateOneof, slot_update::Status};
|
|
|
|
let mut update = update.ok_or(anyhow::anyhow!("geyser plugin has closed the stream"))??;
|
2021-12-20 06:06:23 -08:00
|
|
|
match update.update_oneof.as_mut().expect("invalid grpc") {
|
2022-01-03 00:13:32 -08:00
|
|
|
UpdateOneof::SubscribeResponse(subscribe_response) => {
|
|
|
|
first_full_slot = subscribe_response.highest_write_slot + 1;
|
|
|
|
},
|
2021-12-13 07:46:36 -08:00
|
|
|
UpdateOneof::SlotUpdate(slot_update) => {
|
|
|
|
let status = slot_update.status;
|
2022-01-03 00:13:32 -08:00
|
|
|
if status == Status::Rooted as i32 {
|
|
|
|
if slot_update.slot > max_rooted_slot {
|
|
|
|
max_rooted_slot = slot_update.slot;
|
|
|
|
|
|
|
|
// drop data for slots that are well beyond rooted
|
|
|
|
slot_pubkey_writes.retain(|&k, _| k >= max_rooted_slot - max_out_of_order_slots);
|
|
|
|
}
|
|
|
|
if snapshot_needed && max_rooted_slot - rooted_to_finalized_slots > first_full_slot {
|
|
|
|
snapshot_needed = false;
|
|
|
|
snapshot_future = tokio::spawn(get_snapshot(snapshot_config.rpc_http_url.clone(), program_id)).fuse();
|
|
|
|
}
|
2021-11-07 01:52:38 -08:00
|
|
|
}
|
2021-11-01 13:48:17 -07:00
|
|
|
},
|
2021-12-13 07:46:36 -08:00
|
|
|
UpdateOneof::AccountWrite(write) => {
|
2022-01-03 00:13:32 -08:00
|
|
|
if write.slot < first_full_slot {
|
|
|
|
// Don't try to process data for slots where we may have missed writes:
|
|
|
|
// We could not map the write_version correctly for them.
|
2021-12-20 06:06:23 -08:00
|
|
|
continue;
|
2021-12-13 07:46:36 -08:00
|
|
|
}
|
2021-12-20 06:06:23 -08:00
|
|
|
|
2022-01-03 00:13:32 -08:00
|
|
|
if write.slot > newest_write_slot {
|
|
|
|
newest_write_slot = write.slot;
|
2022-01-04 01:14:30 -08:00
|
|
|
} else if max_rooted_slot > 0 && write.slot < max_rooted_slot - max_out_of_order_slots {
|
2022-01-03 00:13:32 -08:00
|
|
|
anyhow::bail!("received write {} slots back from max rooted slot {}", max_rooted_slot - write.slot, max_rooted_slot);
|
|
|
|
}
|
|
|
|
|
2021-12-20 06:06:23 -08:00
|
|
|
// We assume we will receive write versions in sequence.
|
|
|
|
// If this is not the case, logic here does not work correctly because
|
|
|
|
// a later write could arrive first.
|
2022-01-03 00:13:32 -08:00
|
|
|
if write.write_version <= last_write_version {
|
|
|
|
anyhow::bail!("unexpected write version: {} expected > {}", write.write_version, last_write_version);
|
|
|
|
}
|
2021-12-20 06:06:23 -08:00
|
|
|
last_write_version = write.write_version;
|
|
|
|
|
2022-01-03 00:13:32 -08:00
|
|
|
let pubkey_writes = slot_pubkey_writes.entry(write.slot).or_default();
|
|
|
|
|
|
|
|
let pubkey_bytes = Pubkey::new(&write.pubkey).to_bytes();
|
|
|
|
let writes = pubkey_writes.entry(pubkey_bytes).or_insert(1); // write version 0 is reserved for snapshots
|
|
|
|
write.write_version = *writes as u64;
|
|
|
|
*writes += 1;
|
2021-11-01 13:48:17 -07:00
|
|
|
},
|
2022-03-22 01:36:59 -07:00
|
|
|
geyser_proto::update::UpdateOneof::Ping(_) => {},
|
2021-11-01 13:48:17 -07:00
|
|
|
}
|
2021-12-13 07:46:36 -08:00
|
|
|
sender.send(Message::GrpcUpdate(update)).await.expect("send success");
|
2021-11-01 13:48:17 -07:00
|
|
|
},
|
2021-11-07 01:52:38 -08:00
|
|
|
snapshot = &mut snapshot_future => {
|
2021-12-13 07:46:36 -08:00
|
|
|
let snapshot = snapshot??;
|
|
|
|
if let OptionalContext::Context(snapshot_data) = snapshot {
|
2022-01-03 00:13:32 -08:00
|
|
|
info!("snapshot is for slot {}, first full slot was {}", snapshot_data.context.slot, first_full_slot);
|
|
|
|
if snapshot_data.context.slot >= first_full_slot {
|
2021-12-13 07:46:36 -08:00
|
|
|
sender
|
|
|
|
.send(Message::Snapshot(snapshot_data))
|
|
|
|
.await
|
|
|
|
.expect("send success");
|
|
|
|
} else {
|
|
|
|
info!(
|
|
|
|
"snapshot is too old: has slot {}, expected {} minimum",
|
|
|
|
snapshot_data.context.slot,
|
2022-01-03 00:13:32 -08:00
|
|
|
first_full_slot
|
2021-12-13 07:46:36 -08:00
|
|
|
);
|
|
|
|
// try again in another 10 slots
|
|
|
|
snapshot_needed = true;
|
|
|
|
rooted_to_finalized_slots += 10;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
anyhow::bail!("bad snapshot format");
|
|
|
|
}
|
2021-11-07 01:52:38 -08:00
|
|
|
},
|
2021-11-08 02:42:22 -08:00
|
|
|
_ = tokio::time::sleep(fatal_idle_timeout) => {
|
2022-03-22 01:36:59 -07:00
|
|
|
anyhow::bail!("geyser plugin hasn't sent a message in too long");
|
2021-11-01 13:48:17 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-11-18 05:21:17 -08:00
|
|
|
fn make_tls_config(config: &TlsConfig) -> ClientTlsConfig {
|
|
|
|
let server_root_ca_cert =
|
|
|
|
std::fs::read(&config.ca_cert_path).expect("reading server root ca cert");
|
|
|
|
let server_root_ca_cert = Certificate::from_pem(server_root_ca_cert);
|
|
|
|
let client_cert = std::fs::read(&config.client_cert_path).expect("reading client cert");
|
|
|
|
let client_key = std::fs::read(&config.client_key_path).expect("reading client key");
|
|
|
|
let client_identity = Identity::from_pem(client_cert, client_key);
|
|
|
|
ClientTlsConfig::new()
|
|
|
|
.ca_certificate(server_root_ca_cert)
|
|
|
|
.identity(client_identity)
|
2021-11-30 08:15:41 -08:00
|
|
|
.domain_name(&config.domain_name)
|
2021-11-18 05:21:17 -08:00
|
|
|
}
|
|
|
|
|
2021-11-03 04:17:49 -07:00
|
|
|
pub async fn process_events(
|
2021-11-02 05:22:13 -07:00
|
|
|
config: Config,
|
2021-11-03 04:17:49 -07:00
|
|
|
account_write_queue_sender: async_channel::Sender<AccountWrite>,
|
|
|
|
slot_queue_sender: async_channel::Sender<SlotUpdate>,
|
2021-11-08 11:15:46 -08:00
|
|
|
metrics_sender: metrics::Metrics,
|
2021-11-01 13:48:17 -07:00
|
|
|
) {
|
2022-03-22 01:36:59 -07:00
|
|
|
// Subscribe to geyser
|
2022-03-16 03:04:30 -07:00
|
|
|
let (msg_sender, msg_receiver) =
|
|
|
|
async_channel::bounded::<Message>(config.postgres_target.account_write_max_queue_size);
|
2021-11-08 06:48:50 -08:00
|
|
|
for grpc_source in config.grpc_sources {
|
|
|
|
let msg_sender = msg_sender.clone();
|
|
|
|
let snapshot_source = config.snapshot_source.clone();
|
2021-11-08 11:15:46 -08:00
|
|
|
let metrics_sender = metrics_sender.clone();
|
2021-11-18 05:21:17 -08:00
|
|
|
|
|
|
|
// Make TLS config if configured
|
|
|
|
let tls_config = grpc_source.tls.as_ref().map(make_tls_config);
|
|
|
|
|
2021-11-08 06:48:50 -08:00
|
|
|
tokio::spawn(async move {
|
2021-11-09 05:23:42 -08:00
|
|
|
let mut metric_retries = metrics_sender.register_u64(format!(
|
2021-11-08 11:15:46 -08:00
|
|
|
"grpc_source_{}_connection_retries",
|
|
|
|
grpc_source.name
|
|
|
|
));
|
|
|
|
let metric_status =
|
2021-11-09 05:23:42 -08:00
|
|
|
metrics_sender.register_string(format!("grpc_source_{}_status", grpc_source.name));
|
2021-11-08 11:15:46 -08:00
|
|
|
|
2021-11-08 06:48:50 -08:00
|
|
|
// Continuously reconnect on failure
|
|
|
|
loop {
|
2021-11-08 11:15:46 -08:00
|
|
|
metric_status.set("connected".into());
|
2022-03-22 01:36:59 -07:00
|
|
|
let out = feed_data_geyser(
|
2021-11-18 05:21:17 -08:00
|
|
|
&grpc_source,
|
|
|
|
tls_config.clone(),
|
|
|
|
&snapshot_source,
|
|
|
|
msg_sender.clone(),
|
|
|
|
);
|
2021-11-08 06:48:50 -08:00
|
|
|
let result = out.await;
|
|
|
|
assert!(result.is_err());
|
|
|
|
if let Err(err) = result {
|
|
|
|
warn!(
|
2022-03-22 01:36:59 -07:00
|
|
|
"error during communication with the geyser plugin. retrying. {:?}",
|
2021-11-08 06:48:50 -08:00
|
|
|
err
|
|
|
|
);
|
|
|
|
}
|
2021-11-08 11:15:46 -08:00
|
|
|
|
|
|
|
metric_status.set("disconnected".into());
|
|
|
|
metric_retries.increment();
|
|
|
|
|
2021-11-08 06:48:50 -08:00
|
|
|
tokio::time::sleep(std::time::Duration::from_secs(
|
|
|
|
grpc_source.retry_connection_sleep_secs,
|
|
|
|
))
|
|
|
|
.await;
|
2021-11-01 13:48:17 -07:00
|
|
|
}
|
2021-11-08 06:48:50 -08:00
|
|
|
});
|
|
|
|
}
|
2021-11-01 13:48:17 -07:00
|
|
|
|
2022-01-03 00:13:32 -08:00
|
|
|
// slot -> (pubkey -> write_version)
|
|
|
|
//
|
|
|
|
// To avoid unnecessarily sending requests to SQL, we track the latest write_version
|
|
|
|
// for each (slot, pubkey). If an already-seen write_version comes in, it can be safely
|
|
|
|
// discarded.
|
|
|
|
let mut latest_write = HashMap::<u64, HashMap<[u8; 32], u64>>::new();
|
|
|
|
|
|
|
|
// Number of slots to retain in latest_write
|
|
|
|
let latest_write_retention = 50;
|
|
|
|
|
2021-11-09 05:23:42 -08:00
|
|
|
let mut metric_account_writes = metrics_sender.register_u64("grpc_account_writes".into());
|
|
|
|
let mut metric_account_queue = metrics_sender.register_u64("account_write_queue".into());
|
|
|
|
let mut metric_slot_queue = metrics_sender.register_u64("slot_update_queue".into());
|
|
|
|
let mut metric_slot_updates = metrics_sender.register_u64("grpc_slot_updates".into());
|
|
|
|
let mut metric_snapshots = metrics_sender.register_u64("grpc_snapshots".into());
|
2021-11-08 11:15:46 -08:00
|
|
|
let mut metric_snapshot_account_writes =
|
2021-11-09 05:23:42 -08:00
|
|
|
metrics_sender.register_u64("grpc_snapshot_account_writes".into());
|
2021-11-08 06:31:35 -08:00
|
|
|
|
2021-11-01 13:48:17 -07:00
|
|
|
loop {
|
2021-11-07 01:52:38 -08:00
|
|
|
let msg = msg_receiver.recv().await.expect("sender must not close");
|
2021-11-01 13:48:17 -07:00
|
|
|
|
2021-11-02 06:35:45 -07:00
|
|
|
match msg {
|
|
|
|
Message::GrpcUpdate(update) => {
|
2021-11-07 01:52:38 -08:00
|
|
|
match update.update_oneof.expect("invalid grpc") {
|
2022-03-22 01:36:59 -07:00
|
|
|
geyser_proto::update::UpdateOneof::AccountWrite(update) => {
|
2021-11-02 06:35:45 -07:00
|
|
|
assert!(update.pubkey.len() == 32);
|
|
|
|
assert!(update.owner.len() == 32);
|
2021-11-08 06:31:35 -08:00
|
|
|
|
2021-11-08 11:15:46 -08:00
|
|
|
metric_account_writes.increment();
|
|
|
|
metric_account_queue.set(account_write_queue_sender.len() as u64);
|
|
|
|
|
2022-01-03 00:13:32 -08:00
|
|
|
// Skip writes that a different server has already sent
|
|
|
|
let pubkey_writes = latest_write.entry(update.slot).or_default();
|
|
|
|
let pubkey_bytes = Pubkey::new(&update.pubkey).to_bytes();
|
|
|
|
let writes = pubkey_writes.entry(pubkey_bytes).or_insert(0);
|
|
|
|
if update.write_version <= *writes {
|
|
|
|
continue;
|
2021-11-08 06:31:35 -08:00
|
|
|
}
|
2022-01-03 00:13:32 -08:00
|
|
|
*writes = update.write_version;
|
|
|
|
latest_write.retain(|&k, _| k >= update.slot - latest_write_retention);
|
2021-11-08 06:31:35 -08:00
|
|
|
|
2021-11-02 06:35:45 -07:00
|
|
|
account_write_queue_sender
|
|
|
|
.send(AccountWrite {
|
|
|
|
pubkey: Pubkey::new(&update.pubkey),
|
2022-03-23 00:18:02 -07:00
|
|
|
slot: update.slot,
|
|
|
|
write_version: update.write_version,
|
|
|
|
lamports: update.lamports,
|
2021-11-02 06:35:45 -07:00
|
|
|
owner: Pubkey::new(&update.owner),
|
|
|
|
executable: update.executable,
|
2022-03-23 00:18:02 -07:00
|
|
|
rent_epoch: update.rent_epoch,
|
2021-11-02 06:35:45 -07:00
|
|
|
data: update.data,
|
2022-01-26 01:06:03 -08:00
|
|
|
is_selected: update.is_selected,
|
2021-11-02 06:35:45 -07:00
|
|
|
})
|
2021-11-03 04:17:49 -07:00
|
|
|
.await
|
|
|
|
.expect("send success");
|
2021-11-02 06:35:45 -07:00
|
|
|
}
|
2022-03-22 01:36:59 -07:00
|
|
|
geyser_proto::update::UpdateOneof::SlotUpdate(update) => {
|
2021-11-08 11:15:46 -08:00
|
|
|
metric_slot_updates.increment();
|
|
|
|
metric_slot_queue.set(slot_queue_sender.len() as u64);
|
|
|
|
|
2022-03-22 01:36:59 -07:00
|
|
|
use geyser_proto::slot_update::Status;
|
2021-11-08 03:39:56 -08:00
|
|
|
let status = Status::from_i32(update.status).map(|v| match v {
|
|
|
|
Status::Processed => SlotStatus::Processed,
|
|
|
|
Status::Confirmed => SlotStatus::Confirmed,
|
|
|
|
Status::Rooted => SlotStatus::Rooted,
|
|
|
|
});
|
|
|
|
if status.is_none() {
|
2021-11-02 06:35:45 -07:00
|
|
|
error!("unexpected slot status: {}", update.status);
|
|
|
|
continue;
|
|
|
|
}
|
2021-11-15 06:25:04 -08:00
|
|
|
let slot_update = SlotUpdate {
|
2022-03-23 00:18:02 -07:00
|
|
|
slot: update.slot,
|
|
|
|
parent: update.parent,
|
2021-11-15 06:25:04 -08:00
|
|
|
status: status.expect("qed"),
|
|
|
|
};
|
|
|
|
|
2021-11-02 06:35:45 -07:00
|
|
|
slot_queue_sender
|
2021-11-15 06:25:04 -08:00
|
|
|
.send(slot_update)
|
2021-11-03 04:17:49 -07:00
|
|
|
.await
|
|
|
|
.expect("send success");
|
2021-11-02 06:35:45 -07:00
|
|
|
}
|
2022-03-22 01:36:59 -07:00
|
|
|
geyser_proto::update::UpdateOneof::Ping(_) => {}
|
|
|
|
geyser_proto::update::UpdateOneof::SubscribeResponse(_) => {}
|
2021-11-01 13:48:17 -07:00
|
|
|
}
|
2021-11-03 04:17:49 -07:00
|
|
|
}
|
2021-11-02 06:35:45 -07:00
|
|
|
Message::Snapshot(update) => {
|
2021-11-08 11:15:46 -08:00
|
|
|
metric_snapshots.increment();
|
2021-11-02 06:35:45 -07:00
|
|
|
info!("processing snapshot...");
|
|
|
|
for keyed_account in update.value {
|
2021-11-08 11:15:46 -08:00
|
|
|
metric_snapshot_account_writes.increment();
|
|
|
|
metric_account_queue.set(account_write_queue_sender.len() as u64);
|
|
|
|
|
2021-11-07 01:52:38 -08:00
|
|
|
// TODO: Resnapshot on invalid data?
|
2021-11-02 06:35:45 -07:00
|
|
|
let account: Account = keyed_account.account.decode().unwrap();
|
|
|
|
let pubkey = Pubkey::from_str(&keyed_account.pubkey).unwrap();
|
2021-11-03 04:17:49 -07:00
|
|
|
account_write_queue_sender
|
|
|
|
.send(AccountWrite::from(pubkey, update.context.slot, 0, account))
|
|
|
|
.await
|
|
|
|
.expect("send success");
|
2021-11-02 06:35:45 -07:00
|
|
|
}
|
|
|
|
info!("processing snapshot done");
|
2021-11-03 04:17:49 -07:00
|
|
|
}
|
2021-11-01 13:48:17 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|