From 7e4f42dc4d22cdc63445546c32908bfaa9e82d7c Mon Sep 17 00:00:00 2001 From: waterquarks Date: Thu, 29 Sep 2022 16:07:57 +0200 Subject: [PATCH] Basic L2 differ --- .DS_Store | Bin 0 -> 6148 bytes .idea/.gitignore | 8 + .idea/inspectionProfiles/Project_Default.xml | 7 + .../inspectionProfiles/profiles_settings.xml | 6 + .idea/misc.xml | 4 + .idea/modules.xml | 8 + .idea/solana-accountsdb-connector.iml | 19 ++ .idea/vcs.xml | 6 + Cargo.lock | 160 ++++++++++++--- Cargo.toml | 3 +- config-devnet.toml | 53 +++++ connector-trace/src/main.rs | 103 ---------- scratch/Cargo.toml | 22 ++ scratch/src/main.rs | 13 ++ {connector-trace => trackoor}/Cargo.toml | 6 +- .../example-config.toml | 0 trackoor/src/main.rs | 194 ++++++++++++++++++ 17 files changed, 482 insertions(+), 130 deletions(-) create mode 100644 .DS_Store create mode 100644 .idea/.gitignore create mode 100644 .idea/inspectionProfiles/Project_Default.xml create mode 100644 .idea/inspectionProfiles/profiles_settings.xml create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/solana-accountsdb-connector.iml create mode 100644 .idea/vcs.xml create mode 100644 config-devnet.toml delete mode 100644 connector-trace/src/main.rs create mode 100644 scratch/Cargo.toml create mode 100644 scratch/src/main.rs rename {connector-trace => trackoor}/Cargo.toml (77%) rename {connector-trace => trackoor}/example-config.toml (100%) create mode 100644 trackoor/src/main.rs diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..269709724fb9c5a81a6a91226dfba13cbec221bf GIT binary patch literal 6148 zcmeHKUrWO<5Wj5Gt_)$I!X690PTXWF;!BzH3s}(ymD$=ki(Mn@{ydC9pY;p*N&GzC zC7C#;;Dd;hyK?zmE_X@uOGz345Y1s&2PgwT5tT4!WATYlJL!VtjE7LDJ|eh<4s1gY z{Do+C{6z+6?bad1RqMkDKCWK^U68nzcj%4rNrQ3v7{oHbjF;dsj?zJ`_9}Au!pf>; z*;dKA_U`4_8~B4^+VuOER6CP04raC=oJXB;yS#BMlYt*4oxVzl!VZR9T|`MJ$4xm* z!c^sYy1}-KcDr1eOzMq;s?*r3Ppi)4aJPo`eq%bdin1Iz$3uU0olWbDU`EX)l> zsL|2ysBjRjMlP8FW?-Iyysnn%{6G2n{y(3@HD-Vr_*V>wLd$D4aZ2WFU6>r5wGQeH rDhb798sAgUP)9Mw(otMNRf2wp3`EyprV%|T{3D=f;DQ + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..105ce2d --- /dev/null +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..d1e22ec --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..2a5f775 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/solana-accountsdb-connector.iml b/.idea/solana-accountsdb-connector.iml new file mode 100644 index 0000000..7f539ef --- /dev/null +++ b/.idea/solana-accountsdb-connector.iml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index e118897..b2474a3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2784,9 +2784,37 @@ dependencies = [ "enumflags2", "fixed", "fixed-macro", - "mango-common", - "mango-logs", - "mango-macro", + "mango-common 3.0.0 (git+https://github.com/blockworks-foundation/mango-v3?branch=pan/solana-1.10)", + "mango-logs 0.1.0 (git+https://github.com/blockworks-foundation/mango-v3?branch=pan/solana-1.10)", + "mango-macro 3.0.0 (git+https://github.com/blockworks-foundation/mango-v3?branch=pan/solana-1.10)", + "num_enum", + "pyth-client", + "safe-transmute", + "serde", + "serum_dex", + "solana-program", + "spl-token", + "static_assertions", + "switchboard-program", + "switchboard-utils", + "thiserror", +] + +[[package]] +name = "mango" +version = "3.6.1" +dependencies = [ + "anchor-lang", + "arrayref", + "bincode", + "bs58 0.4.0", + "bytemuck", + "enumflags2", + "fixed", + "fixed-macro", + "mango-common 3.0.0", + "mango-logs 0.1.0", + "mango-macro 3.0.0", "num_enum", "pyth-client", "safe-transmute", @@ -2800,6 +2828,14 @@ dependencies = [ "thiserror", ] +[[package]] +name = "mango-common" +version = "3.0.0" +dependencies = [ + "bytemuck", + "solana-program", +] + [[package]] name = "mango-common" version = "3.0.0" @@ -2809,6 +2845,14 @@ dependencies = [ "solana-program", ] +[[package]] +name = "mango-logs" +version = "0.1.0" +dependencies = [ + "anchor-lang", + "base64 0.13.0", +] + [[package]] name = "mango-logs" version = "0.1.0" @@ -2818,13 +2862,25 @@ dependencies = [ "base64 0.13.0", ] +[[package]] +name = "mango-macro" +version = "3.0.0" +dependencies = [ + "bytemuck", + "mango-common 3.0.0", + "quote 1.0.21", + "safe-transmute", + "solana-program", + "syn 1.0.99", +] + [[package]] name = "mango-macro" version = "3.0.0" source = "git+https://github.com/blockworks-foundation/mango-v3?branch=pan/solana-1.10#64892f4ab0837c7fd260956867ee30a1a941d625" dependencies = [ "bytemuck", - "mango-common", + "mango-common 3.0.0 (git+https://github.com/blockworks-foundation/mango-v3?branch=pan/solana-1.10)", "quote 1.0.21", "safe-transmute", "solana-program", @@ -4529,6 +4585,26 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "scratch" +version = "0.1.0" +dependencies = [ + "anyhow", + "arrayref", + "async-channel", + "bytemuck", + "log 0.4.17", + "mango 3.6.1", + "serde", + "serde_derive", + "solana-geyser-connector-lib", + "solana-logger", + "solana-sdk", + "sqlite", + "tokio", + "toml", +] + [[package]] name = "scroll" version = "0.10.2" @@ -4751,8 +4827,8 @@ dependencies = [ "fixed", "jsonrpsee", "log 0.4.17", - "mango", - "mango-common", + "mango 3.5.1", + "mango-common 3.0.0 (git+https://github.com/blockworks-foundation/mango-v3?branch=pan/solana-1.10)", "serde", "serde_derive", "solana-geyser-connector-lib", @@ -5278,7 +5354,7 @@ dependencies = [ "jsonrpc-core 18.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "jsonrpc-core-client", "log 0.4.17", - "mango", + "mango 3.5.1", "native-tls", "postgres-native-tls", "postgres-types", @@ -5309,8 +5385,8 @@ dependencies = [ "bs58 0.3.1", "fixed", "log 0.4.17", - "mango", - "mango-common", + "mango 3.5.1", + "mango-common 3.0.0 (git+https://github.com/blockworks-foundation/mango-v3?branch=pan/solana-1.10)", "postgres-types", "postgres_query", "solana-geyser-connector-lib", @@ -5358,22 +5434,6 @@ dependencies = [ "toml", ] -[[package]] -name = "solana-geyser-connector-trace" -version = "0.1.0" -dependencies = [ - "anyhow", - "async-channel", - "log 0.4.17", - "serde", - "serde_derive", - "solana-geyser-connector-lib", - "solana-logger", - "solana-sdk", - "tokio", - "toml", -] - [[package]] name = "solana-geyser-plugin-interface" version = "1.10.35" @@ -6214,6 +6274,36 @@ dependencies = [ "thiserror", ] +[[package]] +name = "sqlite" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2df8edd55685048550daaaf2be9024182f3523086cc86f7d50c136e55173e8c" +dependencies = [ + "libc", + "sqlite3-sys", +] + +[[package]] +name = "sqlite3-src" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1815a7a02c996eb8e5c64f61fcb6fd9b12e593ce265c512c5853b2513635691" +dependencies = [ + "cc", + "pkg-config", +] + +[[package]] +name = "sqlite3-sys" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d47c99824fc55360ba00caf28de0b8a0458369b832e016a64c13af0ad9fbb9ee" +dependencies = [ + "libc", + "sqlite3-src", +] + [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -7050,6 +7140,26 @@ dependencies = [ "tracing", ] +[[package]] +name = "trackoor" +version = "0.1.0" +dependencies = [ + "anyhow", + "arrayref", + "async-channel", + "bytemuck", + "itertools 0.10.3", + "log 0.4.17", + "mango 3.6.1", + "serde", + "serde_derive", + "solana-geyser-connector-lib", + "solana-logger", + "solana-sdk", + "tokio", + "toml", +] + [[package]] name = "traitobject" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 23c6c15..bd4ef07 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,9 +4,10 @@ members = [ "lib", "connector-raw", "connector-mango", - "connector-trace", + "trackoor", "service-mango-fills", "service-mango-pnl", + "scratch" ] diff --git a/config-devnet.toml b/config-devnet.toml new file mode 100644 index 0000000..45edf71 --- /dev/null +++ b/config-devnet.toml @@ -0,0 +1,53 @@ +bind_ws_addr = "0.0.0.0:2082" + +[source] +dedup_queue_size = 50000 +rpc_ws_url = "" + +[[source.grpc_sources]] +name = "accountsdb-client" +connection_string = "http://chi5.rpcpool.com:10001" +retry_connection_sleep_secs = 30 + +[source.grpc_sources.tls] +ca_cert_path = "ca-devnet.pem" +client_cert_path = "client-devnet.pem" +client_key_path = "client-devnet.pem" +domain_name = "mango-accountsdb.rpcpool.com" + +[source.snapshot] +rpc_http_url = "http://mango.devnet.rpcpool.com/" +program_id = "4skJ85cdxQAFVKbcGgfun8iZPL7BadVYXG3kGEGkufqA" + +[[markets]] +name = "MNGO-PERP" +event_queue = "uaUCSQejWYrDeYSuvn4As4kaCwJ2rLnRQSsSjY3ogZk" + +[[markets]] +name = "ETH-PERP" +event_queue = "8WLv5fKLYkyZpFG74kRmp2RALHQFcNKmH7eJn8ebHC13" + +[[markets]] +name = "SOL-PERP" +event_queue = "CZ5MCRvkN38d5pnZDDEEyMiED3drgDUVpEUjkuJq31Kf" + +[[markets]] +name = "ADA-PERP" +event_queue = "5v5fz2cCSy2VvrgVf5Vu7PF23RiZjv6BL36bgg48bA1c" + +[[markets]] +name = "FTT-PERP" +event_queue = "7rswj7FVZcMYUKxcTLndZhWBmuVNc2GuxqjuXU8KcPWv" + +[[markets]] +name = "AVAX-PERP" +event_queue = "4b7NqjqWoQoQh9V3dubfjkLPQVNJijwAwr7D9q6vTqqd" + +[[markets]] +name = "BNB-PERP" +event_queue = "96Y87LTz5Mops7wdT9EJo1eM79XToKYJJmRZxNatV85d" + +[[markets]] +name = "MATIC-PERP" +event_queue = "77maU5zdfYayqhqjBi2ocosM4PXvPXxbps2Up7dxDsMR" + diff --git a/connector-trace/src/main.rs b/connector-trace/src/main.rs deleted file mode 100644 index d0bb0ca..0000000 --- a/connector-trace/src/main.rs +++ /dev/null @@ -1,103 +0,0 @@ -use { - log::*, - serde::Deserialize, - solana_geyser_connector_lib::{chain_data::*, *}, - solana_sdk::{account::WritableAccount, clock::Epoch, pubkey::Pubkey}, - std::{fs::File, io::Read, str::FromStr}, -}; - -#[derive(Clone, Debug, Deserialize)] -pub struct Config { - pub source: SourceConfig, -} - -#[tokio::main] -async fn main() -> anyhow::Result<()> { - let args: Vec = std::env::args().collect(); - if args.len() < 2 { - println!("requires a config file argument"); - return Ok(()); - } - - let config: Config = { - let mut file = File::open(&args[1])?; - let mut contents = String::new(); - file.read_to_string(&mut contents)?; - toml::from_str(&contents).unwrap() - }; - - solana_logger::setup_with_default("info"); - info!("startup"); - - let metrics_tx = metrics::start(); - - let (account_write_queue_sender, account_write_queue_receiver) = - async_channel::unbounded::(); - let (slot_queue_sender, slot_queue_receiver) = async_channel::unbounded::(); - let mut chain_cache = ChainData::new(); - // SOL-PERP event q - let events_pk = Pubkey::from_str("31cKs646dt1YkA3zPyxZ7rUAkxTBz279w4XEobFXcAKP").unwrap(); - let mut last_ev_q_version = (0u64, 0u64); - - tokio::spawn(async move { - loop { - tokio::select! { - Ok(account_write) = account_write_queue_receiver.recv() => { - trace!("account write slot:{} pk:{:?} wv:{}", account_write.slot, account_write.pubkey, account_write.write_version); - - if account_write.pubkey == events_pk { - info!("account write slot:{} pk:{:?} wv:{}", account_write.slot, account_write.pubkey, account_write.write_version); - - } - - chain_cache.update_account( - account_write.pubkey, - AccountData { - slot: account_write.slot, - write_version: account_write.write_version, - account: WritableAccount::create( - account_write.lamports, - account_write.data.clone(), - account_write.owner, - account_write.executable, - account_write.rent_epoch as Epoch, - ), - }, - ); - } - Ok(slot_update) = slot_queue_receiver.recv() => { - - chain_cache.update_slot(SlotData { - slot: slot_update.slot, - parent: slot_update.parent, - status: slot_update.status, - chain: 0, - }); - info!("slot update {} {:?} {:?}", slot_update.slot, slot_update.parent, slot_update.status); - - } - }; - - let events_acc = chain_cache.account(&events_pk); - if events_acc.is_ok() { - let acc = events_acc.unwrap(); - - let ev_q_version = (acc.slot, acc.write_version); - if ev_q_version != last_ev_q_version { - info!("chain cache slot:{} wv:{}", acc.slot, acc.write_version); - last_ev_q_version = ev_q_version; - } - } - } - }); - - grpc_plugin_source::process_events( - &config.source, - account_write_queue_sender, - slot_queue_sender, - metrics_tx, - ) - .await; - - Ok(()) -} diff --git a/scratch/Cargo.toml b/scratch/Cargo.toml new file mode 100644 index 0000000..c88a2c6 --- /dev/null +++ b/scratch/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "scratch" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +solana-geyser-connector-lib = { path = "../lib" } +solana-logger = "=1.10.35" +solana-sdk = "=1.10.35" +log = "0.4" +tokio = { version = "1", features = ["full"] } +anyhow = "1.0" +async-channel = "1.6" +serde = "1.0" +serde_derive = "1.0.130" +toml = "0.5" +mango = { path = "../../mango-v3/program" } +bytemuck = "*" +arrayref = "*" +sqlite = "*" \ No newline at end of file diff --git a/scratch/src/main.rs b/scratch/src/main.rs new file mode 100644 index 0000000..898d003 --- /dev/null +++ b/scratch/src/main.rs @@ -0,0 +1,13 @@ +fn main() { + let connection = sqlite::open(":memory:").unwrap(); + + connection + .execute( + " + CREATE TABLE users (name TEXT, age INTEGER); + INSERT INTO users VALUES ('Alice', 42); + INSERT INTO users VALUES ('Bob', 69); + ", + ) + .unwrap(); +} diff --git a/connector-trace/Cargo.toml b/trackoor/Cargo.toml similarity index 77% rename from connector-trace/Cargo.toml rename to trackoor/Cargo.toml index e08ba3c..644c9ef 100644 --- a/connector-trace/Cargo.toml +++ b/trackoor/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "solana-geyser-connector-trace" +name = "trackoor" version = "0.1.0" authors = ["Maximilian Schneider "] edition = "2018" @@ -15,3 +15,7 @@ async-channel = "1.6" serde = "1.0" serde_derive = "1.0.130" toml = "0.5" +mango = { path = "../../mango-v3/program" } +bytemuck = "*" +arrayref = "*" +itertools = "*" diff --git a/connector-trace/example-config.toml b/trackoor/example-config.toml similarity index 100% rename from connector-trace/example-config.toml rename to trackoor/example-config.toml diff --git a/trackoor/src/main.rs b/trackoor/src/main.rs new file mode 100644 index 0000000..297fd48 --- /dev/null +++ b/trackoor/src/main.rs @@ -0,0 +1,194 @@ +use std::any::Any; +use std::ops::Div; +use std::time::{SystemTime, UNIX_EPOCH}; + +use arrayref::array_ref; +use itertools::Itertools; +use mango::matching::{AnyNode, BookSide, LeafNode, NodeHandle}; +use mango::queue::{AnyEvent, EventQueueHeader, EventType, FillEvent}; + +use { + log::*, + serde::Deserialize, + solana_geyser_connector_lib::{*, chain_data::*}, + solana_sdk::{account::{ReadableAccount, WritableAccount}, clock::Epoch, pubkey::Pubkey}, + std::{fs::File, io::Read, mem::size_of, str::FromStr}, +}; + +#[derive(Clone, Debug, Deserialize)] +pub struct Config { + pub source: SourceConfig, +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let args: Vec = std::env::args().collect(); + + solana_logger::setup_with_default("info"); + + if args.len() < 2 { + eprintln!("Please enter a config file path argument."); + return Ok(()); + } + + let config: Config = { + let mut file = File::open(&args[1])?; + let mut contents = String::new(); + file.read_to_string(&mut contents)?; + toml::from_str(&contents).unwrap() + }; + + let metrics_tx = metrics::start(); + + let (account_write_queue_sender, account_write_queue_receiver) = + async_channel::unbounded::(); + + let (slot_queue_sender, slot_queue_receiver) = async_channel::unbounded::(); + + let mut chain_cache = ChainData::new(); + + let events_pk = Pubkey::from_str("Fu8q5EiFunGwSRrjFKjRUoMABj5yCoMEPccMbUiAT6PD").unwrap(); + + tokio::spawn(async move { + let mut trailing_l2_snapshot: Option> = None; + + loop { + tokio::select! { + Ok(account_write) = account_write_queue_receiver.recv() => { + if account_write.pubkey != events_pk { + continue; + } + + chain_cache.update_account( + account_write.pubkey, + AccountData { + slot: account_write.slot, + write_version: account_write.write_version, + account: WritableAccount::create( + account_write.lamports, + account_write.data.clone(), + account_write.owner, + account_write.executable, + account_write.rent_epoch as Epoch, + ), + }, + ); + + info!("account write slot:{} pk:{:?} wv:{}", account_write.slot, account_write.pubkey, account_write.write_version); + + let cache = chain_cache.account(&events_pk); + + if cache.is_err() { + continue; + } + + let book_side: &BookSide = bytemuck::from_bytes(cache.unwrap().account.data()); + + let current_l2_snapshot: Vec<(i64, i64)> = book_side + .iter_valid(SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs()) + .map(|(node_handle, leaf_node)| (leaf_node.price(), leaf_node.quantity)) + .group_by(|(price, quantity)| *price) + .into_iter() + .map(|(price, group)| (price, group.map(|(price, quantity)| quantity).fold(0, |acc, x| acc + x))) + .collect(); + + // TODO: Figure out how to run the diff only at the end of each block instead of at every write + + info!("{:?}", ¤t_l2_snapshot[0..3]); + + let mut diff: Vec<(i64, i64)> = vec!(); + + if let Some(ref previous_l2_snapshot) = trailing_l2_snapshot { + println!("Previous snapshot - diff and send deltas."); + + for previous_order in previous_l2_snapshot { + let (previous_order_price, previous_order_size) = *previous_order; + + let peer = current_l2_snapshot + .iter() + .find(|&(current_order_price, current_order_size)| previous_order_price == *current_order_price); + + match peer { + None => diff.push((previous_order_price, 0)), + _ => continue + } + } + + for current_order in ¤t_l2_snapshot { + let (current_order_price, current_order_size) = *current_order; + + let peer = previous_l2_snapshot + .iter() + .find(|&(previous_order_price, previous_order_size)| *previous_order_price == current_order_price); + + match peer { + Some(previous_order) => { + let (previous_order_price, previous_order_size) = previous_order; + + if *previous_order_size == current_order_size { + continue; + } + + diff.push(current_order.clone()); + }, + None => diff.push(current_order.clone()) + } + } + } else { + println!("No previous snapshot - just send over the current one."); + + for current_order in ¤t_l2_snapshot { + diff.push(current_order.clone()); + } + } + + let base_decimals = 9 as f64; + + let quote_decimals = 6 as f64; + + let quote_lot_size = 100 as f64; + + let base_lot_size = 10000000 as f64; + + let price_lots_to_ui_convertor = (10 as f64) + .powf(base_decimals - quote_decimals) + * quote_lot_size + / base_lot_size; + + let base_lots_to_ui_convertor = base_lot_size / (10 as f64).powf(base_decimals); + + // let price_ui = leaf_node.price() as f64 * price_lots_to_ui_convertor; + // + // let quantity_ui = leaf_node.quantity as f64 * base_lots_to_ui_convertor; + + println!( + "{:#?}", + diff.iter().map(|(price, quantity)| (*price as f64 * price_lots_to_ui_convertor, *quantity as f64 * base_lots_to_ui_convertor)).collect::>() + ); + + trailing_l2_snapshot = Some(current_l2_snapshot); + } + Ok(slot_update) = slot_queue_receiver.recv() => { + chain_cache.update_slot(SlotData { + slot: slot_update.slot, + parent: slot_update.parent, + status: slot_update.status, + chain: 0, + }); + + info!("slot update {} {:?} {:?}", slot_update.slot, slot_update.parent, slot_update.status); + } + }; + } + }); + + grpc_plugin_source::process_events( + &config.source, + account_write_queue_sender, + slot_queue_sender, + metrics_tx, + ) + .await; + + Ok(()) +}