From 12d74789efd6c23bcb503adcfc8f64b702d6b3f7 Mon Sep 17 00:00:00 2001 From: Christian Kamm Date: Wed, 8 Nov 2023 10:27:55 +0100 Subject: [PATCH] cli: save-snapshot command (#773) --- Cargo.lock | 7 +- Cargo.toml | 2 +- bin/cli/Cargo.toml | 3 + bin/cli/src/main.rs | 16 ++ bin/cli/src/save_snapshot.rs | 173 ++++++++++++++++++ programs/mango-v4/src/state/mango_account.rs | 12 ++ .../src/state/mango_account_components.rs | 11 +- programs/mango-v4/src/state/mint_info.rs | 6 +- programs/mango-v4/src/state/oracle.rs | 5 +- programs/mango-v4/src/state/perp_market.rs | 11 +- programs/mango-v4/src/state/stable_price.rs | 3 +- 11 files changed, 241 insertions(+), 8 deletions(-) create mode 100644 bin/cli/src/save_snapshot.rs diff --git a/Cargo.lock b/Cargo.lock index f7c189946..825be7af7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3246,9 +3246,9 @@ dependencies = [ [[package]] name = "mango-feeds-connector" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c51f9dd65f5c5ce13a41aa099c7fd6f2462ad9bc0184333a348a4b683139f9d" +checksum = "0fcd440ee3dd5090a6f36bf8d9392ce7f9cc705828fdacf88b6022ddb7aeb895" dependencies = [ "anyhow", "async-channel", @@ -3347,6 +3347,8 @@ dependencies = [ "anchor-lang", "anchor-spl", "anyhow", + "async-channel", + "base64 0.21.4", "clap 3.2.25", "dotenv", "fixed 1.11.0 (git+https://github.com/blockworks-foundation/fixed.git?branch=v1.11.0-borsh0_10-mango)", @@ -3355,6 +3357,7 @@ dependencies = [ "mango-v4", "mango-v4-client", "pyth-sdk-solana", + "serde_json", "serum_dex 0.5.10 (git+https://github.com/openbook-dex/program.git)", "solana-client", "solana-sdk", diff --git a/Cargo.toml b/Cargo.toml index d5942cf0b..c1f5514b5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ fixed = { git = "https://github.com/blockworks-foundation/fixed.git", branch = " pyth-sdk-solana = "0.8.0" # commit c85e56d (0.5.10 plus depedency updates) serum_dex = { git = "https://github.com/openbook-dex/program.git", default-features=false } -mango-feeds-connector = "0.2.0" +mango-feeds-connector = "0.2.1" # 1.16.7+ is required due to this: https://github.com/blockworks-foundation/mango-v4/issues/712 solana-address-lookup-table-program = "~1.16.7" diff --git a/bin/cli/Cargo.toml b/bin/cli/Cargo.toml index c638296c9..5078f4190 100644 --- a/bin/cli/Cargo.toml +++ b/bin/cli/Cargo.toml @@ -12,6 +12,8 @@ anchor-client = { workspace = true } anchor-lang = { workspace = true } anchor-spl = { workspace = true } anyhow = "1.0" +async-channel = "1.6" +base64 = "0.21" clap = { version = "3.1.8", features = ["derive", "env"] } dotenv = "0.15.0" fixed = { workspace = true, features = ["serde", "borsh"] } @@ -19,6 +21,7 @@ futures = "0.3.21" mango-v4 = { path = "../../programs/mango-v4", features = ["client"] } mango-v4-client = { path = "../../lib/client" } pyth-sdk-solana = { workspace = true } +serde_json = "1.0" serum_dex = { workspace = true, features = ["no-entrypoint", "program"] } solana-client = { workspace = true } solana-sdk = { workspace = true } diff --git a/bin/cli/src/main.rs b/bin/cli/src/main.rs index 0eb44c6c0..9dd1c5d84 100644 --- a/bin/cli/src/main.rs +++ b/bin/cli/src/main.rs @@ -6,6 +6,7 @@ use solana_sdk::pubkey::Pubkey; use std::str::FromStr; use std::sync::Arc; +mod save_snapshot; mod test_oracles; #[derive(Parser, Debug, Clone)] @@ -117,6 +118,16 @@ enum Command { #[clap(flatten)] rpc: Rpc, }, + SaveSnapshot { + #[clap(short, long)] + group: String, + + #[clap(flatten)] + rpc: Rpc, + + #[clap(short, long)] + output: String, + }, } impl Rpc { @@ -229,6 +240,11 @@ async fn main() -> Result<(), anyhow::Error> { let group = pubkey_from_cli(&group); test_oracles::run(&client, group).await?; } + Command::SaveSnapshot { group, rpc, output } => { + let mango_group = pubkey_from_cli(&group); + let client = rpc.client(None)?; + save_snapshot::save_snapshot(mango_group, client, output).await? + } }; Ok(()) diff --git a/bin/cli/src/save_snapshot.rs b/bin/cli/src/save_snapshot.rs new file mode 100644 index 000000000..c318c99c3 --- /dev/null +++ b/bin/cli/src/save_snapshot.rs @@ -0,0 +1,173 @@ +use anchor_lang::AccountDeserialize; +use itertools::Itertools; +use mango_v4::accounts_zerocopy::LoadZeroCopy; +use mango_v4::serum3_cpi::{load_open_orders_bytes, OpenOrdersSlim}; +use mango_v4_client::{ + account_update_stream, chain_data, snapshot_source, websocket_source, Client, MangoGroupContext, +}; +use solana_sdk::account::{AccountSharedData, ReadableAccount}; +use solana_sdk::pubkey::Pubkey; + +use std::fs; +use std::path::Path; +use std::time::Duration; + +pub async fn save_snapshot( + mango_group: Pubkey, + client: Client, + output: String, +) -> anyhow::Result<()> { + let out_path = Path::new(&output); + if out_path.exists() { + anyhow::bail!("path {output} exists already"); + } + fs::create_dir_all(out_path).unwrap(); + + let rpc_url = client.cluster.url().to_string(); + let ws_url = client.cluster.ws_url().to_string(); + + let group_context = MangoGroupContext::new_from_rpc(&client.rpc_async(), mango_group).await?; + + let oracles_and_vaults = group_context + .tokens + .values() + .map(|value| value.mint_info.oracle) + .chain(group_context.perp_markets.values().map(|p| p.market.oracle)) + .chain( + group_context + .tokens + .values() + .flat_map(|value| value.mint_info.vaults), + ) + .unique() + .filter(|pk| *pk != Pubkey::default()) + .collect::>(); + + let serum_programs = group_context + .serum3_markets + .values() + .map(|s3| s3.market.serum_program) + .unique() + .collect_vec(); + + let (account_update_sender, account_update_receiver) = + async_channel::unbounded::(); + + // Sourcing account and slot data from solana via websockets + websocket_source::start( + websocket_source::Config { + rpc_ws_url: ws_url.clone(), + serum_programs, + open_orders_authority: mango_group, + }, + oracles_and_vaults.clone(), + account_update_sender.clone(), + ); + + let first_websocket_slot = websocket_source::get_next_create_bank_slot( + account_update_receiver.clone(), + Duration::from_secs(10), + ) + .await?; + + // Getting solana account snapshots via jsonrpc + snapshot_source::start( + snapshot_source::Config { + rpc_http_url: rpc_url.clone(), + mango_group, + get_multiple_accounts_count: 100, + parallel_rpc_requests: 10, + snapshot_interval: Duration::from_secs(6000), + min_slot: first_websocket_slot + 10, + }, + oracles_and_vaults, + account_update_sender, + ); + + let mut chain_data = chain_data::ChainData::new(); + + use account_update_stream::Message; + loop { + let message = account_update_receiver + .recv() + .await + .expect("channel not closed"); + + message.update_chain_data(&mut chain_data); + + match message { + Message::Account(_) => {} + Message::Snapshot(snapshot) => { + for slot in snapshot.iter().map(|a| a.slot).unique() { + chain_data.update_slot(chain_data::SlotData { + slot, + parent: None, + status: chain_data::SlotStatus::Rooted, + chain: 0, + }); + } + break; + } + _ => {} + } + } + + // Write out all the data + use base64::Engine; + use serde_json::json; + let b64 = base64::engine::general_purpose::STANDARD; + for (pk, account) in chain_data.iter_accounts_rooted() { + let debug = to_debug(&account.account); + let data = json!({ + "address": pk.to_string(), + "slot": account.slot, + // mimic an rpc response + "account": { + "owner": account.account.owner().to_string(), + "data": [b64.encode(account.account.data()), "base64"], + "lamports": account.account.lamports(), + "executable": account.account.executable(), + "rentEpoch": account.account.rent_epoch(), + "size": account.account.data().len(), + }, + "debug": debug, + }) + .to_string(); + fs::write(out_path.join(format!("{}.json", pk)), data)?; + } + + Ok(()) +} + +fn to_debug(account: &AccountSharedData) -> Option { + use mango_v4::state::*; + if account.owner() == &mango_v4::ID { + let mut bytes = account.data(); + if let Ok(mango_account) = MangoAccount::try_deserialize(&mut bytes) { + return Some(format!("{mango_account:?}")); + } + } + if let Ok(d) = account.load::() { + return Some(format!("{d:?}")); + } + if let Ok(d) = account.load::() { + return Some(format!("{d:?}")); + } + if let Ok(d) = account.load::() { + return Some(format!("{d:?}")); + } + if let Ok(d) = account.load::() { + return Some(format!("{d:?}")); + } + if let Ok(d) = account.load::() { + return Some(format!("{d:?}")); + } + // TODO: owner check... + if &account.data()[0..5] == b"serum" { + if let Ok(oo) = load_open_orders_bytes(account.data()) { + return Some(format!("{:?}", OpenOrdersSlim::from_oo(oo))); + } + } + // BookSide? EventQueue? + None +} diff --git a/programs/mango-v4/src/state/mango_account.rs b/programs/mango-v4/src/state/mango_account.rs index 18dcb952f..2403a4d1f 100644 --- a/programs/mango-v4/src/state/mango_account.rs +++ b/programs/mango-v4/src/state/mango_account.rs @@ -4,6 +4,7 @@ use std::mem::size_of; use anchor_lang::prelude::*; use anchor_lang::Discriminator; use arrayref::array_ref; +use derivative::Derivative; use fixed::types::I80F48; @@ -13,6 +14,7 @@ use static_assertions::const_assert_eq; use crate::error::*; use crate::health::{HealthCache, HealthType}; use crate::logs::{DeactivatePerpPositionLog, DeactivateTokenPositionLog}; +use crate::util; use super::BookSideOrderTree; use super::FillEvent; @@ -83,6 +85,8 @@ impl MangoAccountPdaSeeds { // MangoAccount binary data is backwards compatible: when ignoring trailing bytes, a v2 account can // be read as a v1 account and a v3 account can be read as v1 or v2 etc. #[account] +#[derive(Derivative)] +#[derivative(Debug)] pub struct MangoAccount { // fixed // note: keep MangoAccountFixed in sync with changes here @@ -92,6 +96,7 @@ pub struct MangoAccount { // ABI: Clients rely on this being at offset 40 pub owner: Pubkey, + #[derivative(Debug(format_with = "util::format_zero_terminated_utf8_bytes"))] pub name: [u8; 32], // Alternative authority/signer of transactions for a mango account @@ -117,6 +122,7 @@ pub struct MangoAccount { pub bump: u8, + #[derivative(Debug = "ignore")] pub padding: [u8; 1], // (Display only) @@ -144,22 +150,28 @@ pub struct MangoAccount { /// Next id to use when adding a token condition swap pub next_token_conditional_swap_id: u64, + #[derivative(Debug = "ignore")] pub reserved: [u8; 200], // dynamic pub header_version: u8, + #[derivative(Debug = "ignore")] pub padding3: [u8; 7], // note: padding is required for TokenPosition, etc. to be aligned + #[derivative(Debug = "ignore")] pub padding4: u32, // Maps token_index -> deposit/borrow account for each token // that is active on this MangoAccount. pub tokens: Vec, + #[derivative(Debug = "ignore")] pub padding5: u32, // Maps serum_market_index -> open orders for each serum market // that is active on this MangoAccount. pub serum3: Vec, + #[derivative(Debug = "ignore")] pub padding6: u32, pub perps: Vec, + #[derivative(Debug = "ignore")] pub padding7: u32, pub perp_open_orders: Vec, // WARNING: This does not have further fields, like tcs, intentionally: diff --git a/programs/mango-v4/src/state/mango_account_components.rs b/programs/mango-v4/src/state/mango_account_components.rs index e701ddc2b..0bcbb466d 100644 --- a/programs/mango-v4/src/state/mango_account_components.rs +++ b/programs/mango-v4/src/state/mango_account_components.rs @@ -824,14 +824,23 @@ impl PerpPosition { } #[zero_copy] -#[derive(AnchorSerialize, AnchorDeserialize, Debug)] +#[derive(AnchorSerialize, AnchorDeserialize, Derivative)] +#[derivative(Debug)] pub struct PerpOpenOrder { pub side_and_tree: u8, // SideAndOrderTree -- enums aren't POD + + #[derivative(Debug = "ignore")] pub padding1: [u8; 1], + pub market: PerpMarketIndex, + + #[derivative(Debug = "ignore")] pub padding2: [u8; 4], + pub client_id: u64, pub id: u128, + + #[derivative(Debug = "ignore")] pub reserved: [u8; 64], } const_assert_eq!(size_of::(), 1 + 1 + 2 + 4 + 8 + 16 + 64); diff --git a/programs/mango-v4/src/state/mint_info.rs b/programs/mango-v4/src/state/mint_info.rs index c32f7947d..73ae555a8 100644 --- a/programs/mango-v4/src/state/mint_info.rs +++ b/programs/mango-v4/src/state/mint_info.rs @@ -1,4 +1,5 @@ use anchor_lang::prelude::*; +use derivative::Derivative; use static_assertions::const_assert_eq; use std::mem::size_of; @@ -13,7 +14,8 @@ pub const MAX_BANKS: usize = 6; // can load this account to figure out which address maps to use when calling // instructions that need banks/oracles for all active positions. #[account(zero_copy)] -#[derive(Debug)] +#[derive(Derivative)] +#[derivative(Debug)] pub struct MintInfo { // ABI: Clients rely on this being at offset 8 pub group: Pubkey, @@ -22,6 +24,7 @@ pub struct MintInfo { pub token_index: TokenIndex, pub group_insurance_fund: u8, + #[derivative(Debug = "ignore")] pub padding1: [u8; 5], pub mint: Pubkey, pub banks: [Pubkey; MAX_BANKS], @@ -30,6 +33,7 @@ pub struct MintInfo { pub registration_time: u64, + #[derivative(Debug = "ignore")] pub reserved: [u8; 2560], } const_assert_eq!( diff --git a/programs/mango-v4/src/state/oracle.rs b/programs/mango-v4/src/state/oracle.rs index e795de270..65c483100 100644 --- a/programs/mango-v4/src/state/oracle.rs +++ b/programs/mango-v4/src/state/oracle.rs @@ -2,6 +2,7 @@ use std::mem::size_of; use anchor_lang::prelude::*; use anchor_lang::Discriminator; +use derivative::Derivative; use fixed::types::I80F48; use static_assertions::const_assert_eq; @@ -57,10 +58,12 @@ pub mod switchboard_v2_mainnet_oracle { } #[zero_copy] -#[derive(AnchorDeserialize, AnchorSerialize, Debug)] +#[derive(AnchorDeserialize, AnchorSerialize, Derivative)] +#[derivative(Debug)] pub struct OracleConfig { pub conf_filter: I80F48, pub max_staleness_slots: i64, + #[derivative(Debug = "ignore")] pub reserved: [u8; 72], } const_assert_eq!(size_of::(), 16 + 8 + 72); diff --git a/programs/mango-v4/src/state/perp_market.rs b/programs/mango-v4/src/state/perp_market.rs index e8adf27a6..8aa550254 100644 --- a/programs/mango-v4/src/state/perp_market.rs +++ b/programs/mango-v4/src/state/perp_market.rs @@ -1,6 +1,7 @@ use std::mem::size_of; use anchor_lang::prelude::*; +use derivative::Derivative; use fixed::types::I80F48; use static_assertions::const_assert_eq; @@ -10,13 +11,15 @@ use crate::error::MangoError; use crate::logs::PerpUpdateFundingLogV2; use crate::state::orderbook::Side; use crate::state::{oracle, TokenIndex}; +use crate::util; use super::{orderbook, OracleConfig, OracleState, Orderbook, StablePriceModel, DAY_I80F48}; pub type PerpMarketIndex = u16; #[account(zero_copy)] -#[derive(Debug)] +#[derive(Derivative)] +#[derivative(Debug)] pub struct PerpMarket { // ABI: Clients rely on this being at offset 8 pub group: Pubkey, @@ -46,6 +49,7 @@ pub struct PerpMarket { pub base_decimals: u8, /// Name. Trailing zero bytes are ignored. + #[derivative(Debug(format_with = "util::format_zero_terminated_utf8_bytes"))] pub name: [u8; 16], /// Address of the BookSide account for bids @@ -156,7 +160,10 @@ pub struct PerpMarket { /// /// See also PerpPosition::settle_pnl_limit_realized_trade pub settle_pnl_limit_factor: f32, + + #[derivative(Debug = "ignore")] pub padding3: [u8; 4], + /// Window size in seconds for the perp settlement limit pub settle_pnl_limit_window_size_ts: u64, @@ -165,6 +172,7 @@ pub struct PerpMarket { pub reduce_only: u8, pub force_close: u8, + #[derivative(Debug = "ignore")] pub padding4: [u8; 6], /// Weights for full perp market health, if positive @@ -177,6 +185,7 @@ pub struct PerpMarket { // This ensures that fees_settled is strictly increasing for stats gathering purposes pub fees_withdrawn: u64, + #[derivative(Debug = "ignore")] pub reserved: [u8; 1880], } diff --git a/programs/mango-v4/src/state/stable_price.rs b/programs/mango-v4/src/state/stable_price.rs index ebf363f5f..c70caa169 100644 --- a/programs/mango-v4/src/state/stable_price.rs +++ b/programs/mango-v4/src/state/stable_price.rs @@ -15,7 +15,8 @@ use std::mem::size_of; /// price over every `delay_interval_seconds` (assume 1h) and then applying the /// `delay_growth_limit` between intervals. #[zero_copy] -#[derive(Derivative, Debug)] +#[derive(Derivative)] +#[derivative(Debug)] pub struct StablePriceModel { /// Current stable price to use in health pub stable_price: f64,