cli: save-snapshot command (#773)

Cargo.lock

@@ -3246,9 +3246,9 @@ dependencies = [
 
 [[package]]
 name = "mango-feeds-connector"
-version = "0.2.0"
+version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c51f9dd65f5c5ce13a41aa099c7fd6f2462ad9bc0184333a348a4b683139f9d"
+checksum = "0fcd440ee3dd5090a6f36bf8d9392ce7f9cc705828fdacf88b6022ddb7aeb895"
 dependencies = [
  "anyhow",
  "async-channel",
@@ -3347,6 +3347,8 @@ dependencies = [
  "anchor-lang",
  "anchor-spl",
  "anyhow",
+ "async-channel",
+ "base64 0.21.4",
  "clap 3.2.25",
  "dotenv",
  "fixed 1.11.0 (git+https://github.com/blockworks-foundation/fixed.git?branch=v1.11.0-borsh0_10-mango)",
@@ -3355,6 +3357,7 @@ dependencies = [
  "mango-v4",
  "mango-v4-client",
  "pyth-sdk-solana",
+ "serde_json",
  "serum_dex 0.5.10 (git+https://github.com/openbook-dex/program.git)",
  "solana-client",
  "solana-sdk",

Cargo.toml

@@ -13,7 +13,7 @@ fixed = { git = "https://github.com/blockworks-foundation/fixed.git", branch = "
 pyth-sdk-solana = "0.8.0"
 # commit c85e56d (0.5.10 plus depedency updates)
 serum_dex = { git = "https://github.com/openbook-dex/program.git", default-features=false }
-mango-feeds-connector = "0.2.0"
+mango-feeds-connector = "0.2.1"
 
 # 1.16.7+ is required due to this: https://github.com/blockworks-foundation/mango-v4/issues/712
 solana-address-lookup-table-program = "~1.16.7"

bin/cli/Cargo.toml

@@ -12,6 +12,8 @@ anchor-client = { workspace = true }
 anchor-lang = { workspace = true }
 anchor-spl = { workspace = true }
 anyhow = "1.0"
+async-channel = "1.6"
+base64 = "0.21"
 clap = { version = "3.1.8", features = ["derive", "env"] }
 dotenv = "0.15.0"
 fixed = { workspace = true, features = ["serde", "borsh"] }
@@ -19,6 +21,7 @@ futures = "0.3.21"
 mango-v4 = { path = "../../programs/mango-v4", features = ["client"] }
 mango-v4-client = { path = "../../lib/client" }
 pyth-sdk-solana = { workspace = true }
+serde_json = "1.0"
 serum_dex = { workspace = true, features = ["no-entrypoint", "program"] }
 solana-client = { workspace = true }
 solana-sdk = { workspace = true }

bin/cli/src/main.rs

@@ -6,6 +6,7 @@ use solana_sdk::pubkey::Pubkey;
 use std::str::FromStr;
 use std::sync::Arc;
 
+mod save_snapshot;
 mod test_oracles;
 
 #[derive(Parser, Debug, Clone)]
@@ -117,6 +118,16 @@ enum Command {
         #[clap(flatten)]
         rpc: Rpc,
     },
+    SaveSnapshot {
+        #[clap(short, long)]
+        group: String,
+
+        #[clap(flatten)]
+        rpc: Rpc,
+
+        #[clap(short, long)]
+        output: String,
+    },
 }
 
 impl Rpc {
@@ -229,6 +240,11 @@ async fn main() -> Result<(), anyhow::Error> {
             let group = pubkey_from_cli(&group);
             test_oracles::run(&client, group).await?;
         }
+        Command::SaveSnapshot { group, rpc, output } => {
+            let mango_group = pubkey_from_cli(&group);
+            let client = rpc.client(None)?;
+            save_snapshot::save_snapshot(mango_group, client, output).await?
+        }
     };
 
     Ok(())

bin/cli/src/save_snapshot.rs

@@ -0,0 +1,173 @@
+use anchor_lang::AccountDeserialize;
+use itertools::Itertools;
+use mango_v4::accounts_zerocopy::LoadZeroCopy;
+use mango_v4::serum3_cpi::{load_open_orders_bytes, OpenOrdersSlim};
+use mango_v4_client::{
+    account_update_stream, chain_data, snapshot_source, websocket_source, Client, MangoGroupContext,
+};
+use solana_sdk::account::{AccountSharedData, ReadableAccount};
+use solana_sdk::pubkey::Pubkey;
+
+use std::fs;
+use std::path::Path;
+use std::time::Duration;
+
+pub async fn save_snapshot(
+    mango_group: Pubkey,
+    client: Client,
+    output: String,
+) -> anyhow::Result<()> {
+    let out_path = Path::new(&output);
+    if out_path.exists() {
+        anyhow::bail!("path {output} exists already");
+    }
+    fs::create_dir_all(out_path).unwrap();
+
+    let rpc_url = client.cluster.url().to_string();
+    let ws_url = client.cluster.ws_url().to_string();
+
+    let group_context = MangoGroupContext::new_from_rpc(&client.rpc_async(), mango_group).await?;
+
+    let oracles_and_vaults = group_context
+        .tokens
+        .values()
+        .map(|value| value.mint_info.oracle)
+        .chain(group_context.perp_markets.values().map(|p| p.market.oracle))
+        .chain(
+            group_context
+                .tokens
+                .values()
+                .flat_map(|value| value.mint_info.vaults),
+        )
+        .unique()
+        .filter(|pk| *pk != Pubkey::default())
+        .collect::<Vec<Pubkey>>();
+
+    let serum_programs = group_context
+        .serum3_markets
+        .values()
+        .map(|s3| s3.market.serum_program)
+        .unique()
+        .collect_vec();
+
+    let (account_update_sender, account_update_receiver) =
+        async_channel::unbounded::<account_update_stream::Message>();
+
+    // Sourcing account and slot data from solana via websockets
+    websocket_source::start(
+        websocket_source::Config {
+            rpc_ws_url: ws_url.clone(),
+            serum_programs,
+            open_orders_authority: mango_group,
+        },
+        oracles_and_vaults.clone(),
+        account_update_sender.clone(),
+    );
+
+    let first_websocket_slot = websocket_source::get_next_create_bank_slot(
+        account_update_receiver.clone(),
+        Duration::from_secs(10),
+    )
+    .await?;
+
+    // Getting solana account snapshots via jsonrpc
+    snapshot_source::start(
+        snapshot_source::Config {
+            rpc_http_url: rpc_url.clone(),
+            mango_group,
+            get_multiple_accounts_count: 100,
+            parallel_rpc_requests: 10,
+            snapshot_interval: Duration::from_secs(6000),
+            min_slot: first_websocket_slot + 10,
+        },
+        oracles_and_vaults,
+        account_update_sender,
+    );
+
+    let mut chain_data = chain_data::ChainData::new();
+
+    use account_update_stream::Message;
+    loop {
+        let message = account_update_receiver
+            .recv()
+            .await
+            .expect("channel not closed");
+
+        message.update_chain_data(&mut chain_data);
+
+        match message {
+            Message::Account(_) => {}
+            Message::Snapshot(snapshot) => {
+                for slot in snapshot.iter().map(|a| a.slot).unique() {
+                    chain_data.update_slot(chain_data::SlotData {
+                        slot,
+                        parent: None,
+                        status: chain_data::SlotStatus::Rooted,
+                        chain: 0,
+                    });
+                }
+                break;
+            }
+            _ => {}
+        }
+    }
+
+    // Write out all the data
+    use base64::Engine;
+    use serde_json::json;
+    let b64 = base64::engine::general_purpose::STANDARD;
+    for (pk, account) in chain_data.iter_accounts_rooted() {
+        let debug = to_debug(&account.account);
+        let data = json!({
+            "address": pk.to_string(),
+            "slot": account.slot,
+            // mimic an rpc response
+            "account": {
+                "owner": account.account.owner().to_string(),
+                "data": [b64.encode(account.account.data()), "base64"],
+                "lamports": account.account.lamports(),
+                "executable": account.account.executable(),
+                "rentEpoch": account.account.rent_epoch(),
+                "size": account.account.data().len(),
+            },
+            "debug": debug,
+        })
+        .to_string();
+        fs::write(out_path.join(format!("{}.json", pk)), data)?;
+    }
+
+    Ok(())
+}
+
+fn to_debug(account: &AccountSharedData) -> Option<String> {
+    use mango_v4::state::*;
+    if account.owner() == &mango_v4::ID {
+        let mut bytes = account.data();
+        if let Ok(mango_account) = MangoAccount::try_deserialize(&mut bytes) {
+            return Some(format!("{mango_account:?}"));
+        }
+    }
+    if let Ok(d) = account.load::<Bank>() {
+        return Some(format!("{d:?}"));
+    }
+    if let Ok(d) = account.load::<Group>() {
+        return Some(format!("{d:?}"));
+    }
+    if let Ok(d) = account.load::<MintInfo>() {
+        return Some(format!("{d:?}"));
+    }
+    if let Ok(d) = account.load::<PerpMarket>() {
+        return Some(format!("{d:?}"));
+    }
+    if let Ok(d) = account.load::<Serum3Market>() {
+        return Some(format!("{d:?}"));
+    }
+    // TODO: owner check...
+    if &account.data()[0..5] == b"serum" {
+        if let Ok(oo) = load_open_orders_bytes(account.data()) {
+            return Some(format!("{:?}", OpenOrdersSlim::from_oo(oo)));
+        }
+    }
+    // BookSide? EventQueue?
+    None
+}

programs/mango-v4/src/state/mango_account.rs

@@ -4,6 +4,7 @@ use std::mem::size_of;
 use anchor_lang::prelude::*;
 use anchor_lang::Discriminator;
 use arrayref::array_ref;
+use derivative::Derivative;
 
 use fixed::types::I80F48;
 
@@ -13,6 +14,7 @@ use static_assertions::const_assert_eq;
 use crate::error::*;
 use crate::health::{HealthCache, HealthType};
 use crate::logs::{DeactivatePerpPositionLog, DeactivateTokenPositionLog};
+use crate::util;
 
 use super::BookSideOrderTree;
 use super::FillEvent;
@@ -83,6 +85,8 @@ impl MangoAccountPdaSeeds {
 // MangoAccount binary data is backwards compatible: when ignoring trailing bytes, a v2 account can
 // be read as a v1 account and a v3 account can be read as v1 or v2 etc.
 #[account]
+#[derive(Derivative)]
+#[derivative(Debug)]
 pub struct MangoAccount {
     // fixed
     // note: keep MangoAccountFixed in sync with changes here
@@ -92,6 +96,7 @@ pub struct MangoAccount {
     // ABI: Clients rely on this being at offset 40
     pub owner: Pubkey,
 
+    #[derivative(Debug(format_with = "util::format_zero_terminated_utf8_bytes"))]
     pub name: [u8; 32],
 
     // Alternative authority/signer of transactions for a mango account
@@ -117,6 +122,7 @@ pub struct MangoAccount {
 
     pub bump: u8,
 
+    #[derivative(Debug = "ignore")]
     pub padding: [u8; 1],
 
     // (Display only)
@@ -144,22 +150,28 @@ pub struct MangoAccount {
     /// Next id to use when adding a token condition swap
     pub next_token_conditional_swap_id: u64,
 
+    #[derivative(Debug = "ignore")]
     pub reserved: [u8; 200],
 
     // dynamic
     pub header_version: u8,
+    #[derivative(Debug = "ignore")]
     pub padding3: [u8; 7],
     // note: padding is required for TokenPosition, etc. to be aligned
+    #[derivative(Debug = "ignore")]
     pub padding4: u32,
     // Maps token_index -> deposit/borrow account for each token
     // that is active on this MangoAccount.
     pub tokens: Vec<TokenPosition>,
+    #[derivative(Debug = "ignore")]
     pub padding5: u32,
     // Maps serum_market_index -> open orders for each serum market
     // that is active on this MangoAccount.
     pub serum3: Vec<Serum3Orders>,
+    #[derivative(Debug = "ignore")]
     pub padding6: u32,
     pub perps: Vec<PerpPosition>,
+    #[derivative(Debug = "ignore")]
     pub padding7: u32,
     pub perp_open_orders: Vec<PerpOpenOrder>,
     // WARNING: This does not have further fields, like tcs, intentionally:

programs/mango-v4/src/state/mango_account_components.rs

@@ -824,14 +824,23 @@ impl PerpPosition {
 }
 
 #[zero_copy]
-#[derive(AnchorSerialize, AnchorDeserialize, Debug)]
+#[derive(AnchorSerialize, AnchorDeserialize, Derivative)]
+#[derivative(Debug)]
 pub struct PerpOpenOrder {
     pub side_and_tree: u8, // SideAndOrderTree -- enums aren't POD
+
+    #[derivative(Debug = "ignore")]
     pub padding1: [u8; 1],
+
     pub market: PerpMarketIndex,
+
+    #[derivative(Debug = "ignore")]
     pub padding2: [u8; 4],
+
     pub client_id: u64,
     pub id: u128,
+
+    #[derivative(Debug = "ignore")]
     pub reserved: [u8; 64],
 }
 const_assert_eq!(size_of::<PerpOpenOrder>(), 1 + 1 + 2 + 4 + 8 + 16 + 64);

programs/mango-v4/src/state/mint_info.rs

@@ -1,4 +1,5 @@
 use anchor_lang::prelude::*;
+use derivative::Derivative;
 use static_assertions::const_assert_eq;
 use std::mem::size_of;
 
@@ -13,7 +14,8 @@ pub const MAX_BANKS: usize = 6;
 // can load this account to figure out which address maps to use when calling
 // instructions that need banks/oracles for all active positions.
 #[account(zero_copy)]
-#[derive(Debug)]
+#[derive(Derivative)]
+#[derivative(Debug)]
 pub struct MintInfo {
     // ABI: Clients rely on this being at offset 8
     pub group: Pubkey,
@@ -22,6 +24,7 @@ pub struct MintInfo {
     pub token_index: TokenIndex,
 
     pub group_insurance_fund: u8,
+    #[derivative(Debug = "ignore")]
     pub padding1: [u8; 5],
     pub mint: Pubkey,
     pub banks: [Pubkey; MAX_BANKS],
@@ -30,6 +33,7 @@ pub struct MintInfo {
 
     pub registration_time: u64,
 
+    #[derivative(Debug = "ignore")]
     pub reserved: [u8; 2560],
 }
 const_assert_eq!(

programs/mango-v4/src/state/oracle.rs

@@ -2,6 +2,7 @@ use std::mem::size_of;
 
 use anchor_lang::prelude::*;
 use anchor_lang::Discriminator;
+use derivative::Derivative;
 use fixed::types::I80F48;
 
 use static_assertions::const_assert_eq;
@@ -57,10 +58,12 @@ pub mod switchboard_v2_mainnet_oracle {
 }
 
 #[zero_copy]
-#[derive(AnchorDeserialize, AnchorSerialize, Debug)]
+#[derive(AnchorDeserialize, AnchorSerialize, Derivative)]
+#[derivative(Debug)]
 pub struct OracleConfig {
     pub conf_filter: I80F48,
     pub max_staleness_slots: i64,
+    #[derivative(Debug = "ignore")]
     pub reserved: [u8; 72],
 }
 const_assert_eq!(size_of::<OracleConfig>(), 16 + 8 + 72);

programs/mango-v4/src/state/perp_market.rs

@@ -1,6 +1,7 @@
 use std::mem::size_of;
 
 use anchor_lang::prelude::*;
+use derivative::Derivative;
 use fixed::types::I80F48;
 
 use static_assertions::const_assert_eq;
@@ -10,13 +11,15 @@ use crate::error::MangoError;
 use crate::logs::PerpUpdateFundingLogV2;
 use crate::state::orderbook::Side;
 use crate::state::{oracle, TokenIndex};
+use crate::util;
 
 use super::{orderbook, OracleConfig, OracleState, Orderbook, StablePriceModel, DAY_I80F48};
 
 pub type PerpMarketIndex = u16;
 
 #[account(zero_copy)]
-#[derive(Debug)]
+#[derive(Derivative)]
+#[derivative(Debug)]
 pub struct PerpMarket {
     // ABI: Clients rely on this being at offset 8
     pub group: Pubkey,
@@ -46,6 +49,7 @@ pub struct PerpMarket {
     pub base_decimals: u8,
 
     /// Name. Trailing zero bytes are ignored.
+    #[derivative(Debug(format_with = "util::format_zero_terminated_utf8_bytes"))]
     pub name: [u8; 16],
 
     /// Address of the BookSide account for bids
@@ -156,7 +160,10 @@ pub struct PerpMarket {
     ///
     /// See also PerpPosition::settle_pnl_limit_realized_trade
     pub settle_pnl_limit_factor: f32,
+
+    #[derivative(Debug = "ignore")]
     pub padding3: [u8; 4],
+
     /// Window size in seconds for the perp settlement limit
     pub settle_pnl_limit_window_size_ts: u64,
 
@@ -165,6 +172,7 @@ pub struct PerpMarket {
     pub reduce_only: u8,
     pub force_close: u8,
 
+    #[derivative(Debug = "ignore")]
     pub padding4: [u8; 6],
 
     /// Weights for full perp market health, if positive
@@ -177,6 +185,7 @@ pub struct PerpMarket {
     // This ensures that fees_settled is strictly increasing for stats gathering purposes
     pub fees_withdrawn: u64,
 
+    #[derivative(Debug = "ignore")]
     pub reserved: [u8; 1880],
 }
 

programs/mango-v4/src/state/stable_price.rs

@@ -15,7 +15,8 @@ use std::mem::size_of;
 /// price over every `delay_interval_seconds` (assume 1h) and then applying the
 /// `delay_growth_limit` between intervals.
 #[zero_copy]
-#[derive(Derivative, Debug)]
+#[derive(Derivative)]
+#[derivative(Debug)]
 pub struct StablePriceModel {
     /// Current stable price to use in health
     pub stable_price: f64,