diff --git a/.DS_Store b/.DS_Store
new file mode 100644
index 0000000..2697097
Binary files /dev/null and b/.DS_Store differ
diff --git a/.idea/.gitignore b/.idea/.gitignore
new file mode 100644
index 0000000..13566b8
--- /dev/null
+++ b/.idea/.gitignore
@@ -0,0 +1,8 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml
diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml
new file mode 100644
index 0000000..c73d219
--- /dev/null
+++ b/.idea/inspectionProfiles/Project_Default.xml
@@ -0,0 +1,7 @@
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml
new file mode 100644
index 0000000..105ce2d
--- /dev/null
+++ b/.idea/inspectionProfiles/profiles_settings.xml
@@ -0,0 +1,6 @@
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/misc.xml b/.idea/misc.xml
new file mode 100644
index 0000000..d1e22ec
--- /dev/null
+++ b/.idea/misc.xml
@@ -0,0 +1,4 @@
+
+
+
+
\ No newline at end of file
diff --git a/.idea/modules.xml b/.idea/modules.xml
new file mode 100644
index 0000000..2a5f775
--- /dev/null
+++ b/.idea/modules.xml
@@ -0,0 +1,8 @@
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/solana-accountsdb-connector.iml b/.idea/solana-accountsdb-connector.iml
new file mode 100644
index 0000000..7f539ef
--- /dev/null
+++ b/.idea/solana-accountsdb-connector.iml
@@ -0,0 +1,19 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
new file mode 100644
index 0000000..94a25f7
--- /dev/null
+++ b/.idea/vcs.xml
@@ -0,0 +1,6 @@
+
+
+
+
+
+
\ No newline at end of file
diff --git a/Cargo.lock b/Cargo.lock
index e118897..b2474a3 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2784,9 +2784,37 @@ dependencies = [
"enumflags2",
"fixed",
"fixed-macro",
- "mango-common",
- "mango-logs",
- "mango-macro",
+ "mango-common 3.0.0 (git+https://github.com/blockworks-foundation/mango-v3?branch=pan/solana-1.10)",
+ "mango-logs 0.1.0 (git+https://github.com/blockworks-foundation/mango-v3?branch=pan/solana-1.10)",
+ "mango-macro 3.0.0 (git+https://github.com/blockworks-foundation/mango-v3?branch=pan/solana-1.10)",
+ "num_enum",
+ "pyth-client",
+ "safe-transmute",
+ "serde",
+ "serum_dex",
+ "solana-program",
+ "spl-token",
+ "static_assertions",
+ "switchboard-program",
+ "switchboard-utils",
+ "thiserror",
+]
+
+[[package]]
+name = "mango"
+version = "3.6.1"
+dependencies = [
+ "anchor-lang",
+ "arrayref",
+ "bincode",
+ "bs58 0.4.0",
+ "bytemuck",
+ "enumflags2",
+ "fixed",
+ "fixed-macro",
+ "mango-common 3.0.0",
+ "mango-logs 0.1.0",
+ "mango-macro 3.0.0",
"num_enum",
"pyth-client",
"safe-transmute",
@@ -2800,6 +2828,14 @@ dependencies = [
"thiserror",
]
+[[package]]
+name = "mango-common"
+version = "3.0.0"
+dependencies = [
+ "bytemuck",
+ "solana-program",
+]
+
[[package]]
name = "mango-common"
version = "3.0.0"
@@ -2809,6 +2845,14 @@ dependencies = [
"solana-program",
]
+[[package]]
+name = "mango-logs"
+version = "0.1.0"
+dependencies = [
+ "anchor-lang",
+ "base64 0.13.0",
+]
+
[[package]]
name = "mango-logs"
version = "0.1.0"
@@ -2818,13 +2862,25 @@ dependencies = [
"base64 0.13.0",
]
+[[package]]
+name = "mango-macro"
+version = "3.0.0"
+dependencies = [
+ "bytemuck",
+ "mango-common 3.0.0",
+ "quote 1.0.21",
+ "safe-transmute",
+ "solana-program",
+ "syn 1.0.99",
+]
+
[[package]]
name = "mango-macro"
version = "3.0.0"
source = "git+https://github.com/blockworks-foundation/mango-v3?branch=pan/solana-1.10#64892f4ab0837c7fd260956867ee30a1a941d625"
dependencies = [
"bytemuck",
- "mango-common",
+ "mango-common 3.0.0 (git+https://github.com/blockworks-foundation/mango-v3?branch=pan/solana-1.10)",
"quote 1.0.21",
"safe-transmute",
"solana-program",
@@ -4529,6 +4585,26 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
+[[package]]
+name = "scratch"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "arrayref",
+ "async-channel",
+ "bytemuck",
+ "log 0.4.17",
+ "mango 3.6.1",
+ "serde",
+ "serde_derive",
+ "solana-geyser-connector-lib",
+ "solana-logger",
+ "solana-sdk",
+ "sqlite",
+ "tokio",
+ "toml",
+]
+
[[package]]
name = "scroll"
version = "0.10.2"
@@ -4751,8 +4827,8 @@ dependencies = [
"fixed",
"jsonrpsee",
"log 0.4.17",
- "mango",
- "mango-common",
+ "mango 3.5.1",
+ "mango-common 3.0.0 (git+https://github.com/blockworks-foundation/mango-v3?branch=pan/solana-1.10)",
"serde",
"serde_derive",
"solana-geyser-connector-lib",
@@ -5278,7 +5354,7 @@ dependencies = [
"jsonrpc-core 18.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
"jsonrpc-core-client",
"log 0.4.17",
- "mango",
+ "mango 3.5.1",
"native-tls",
"postgres-native-tls",
"postgres-types",
@@ -5309,8 +5385,8 @@ dependencies = [
"bs58 0.3.1",
"fixed",
"log 0.4.17",
- "mango",
- "mango-common",
+ "mango 3.5.1",
+ "mango-common 3.0.0 (git+https://github.com/blockworks-foundation/mango-v3?branch=pan/solana-1.10)",
"postgres-types",
"postgres_query",
"solana-geyser-connector-lib",
@@ -5358,22 +5434,6 @@ dependencies = [
"toml",
]
-[[package]]
-name = "solana-geyser-connector-trace"
-version = "0.1.0"
-dependencies = [
- "anyhow",
- "async-channel",
- "log 0.4.17",
- "serde",
- "serde_derive",
- "solana-geyser-connector-lib",
- "solana-logger",
- "solana-sdk",
- "tokio",
- "toml",
-]
-
[[package]]
name = "solana-geyser-plugin-interface"
version = "1.10.35"
@@ -6214,6 +6274,36 @@ dependencies = [
"thiserror",
]
+[[package]]
+name = "sqlite"
+version = "0.27.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2df8edd55685048550daaaf2be9024182f3523086cc86f7d50c136e55173e8c"
+dependencies = [
+ "libc",
+ "sqlite3-sys",
+]
+
+[[package]]
+name = "sqlite3-src"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d1815a7a02c996eb8e5c64f61fcb6fd9b12e593ce265c512c5853b2513635691"
+dependencies = [
+ "cc",
+ "pkg-config",
+]
+
+[[package]]
+name = "sqlite3-sys"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d47c99824fc55360ba00caf28de0b8a0458369b832e016a64c13af0ad9fbb9ee"
+dependencies = [
+ "libc",
+ "sqlite3-src",
+]
+
[[package]]
name = "stable_deref_trait"
version = "1.2.0"
@@ -7050,6 +7140,26 @@ dependencies = [
"tracing",
]
+[[package]]
+name = "trackoor"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "arrayref",
+ "async-channel",
+ "bytemuck",
+ "itertools 0.10.3",
+ "log 0.4.17",
+ "mango 3.6.1",
+ "serde",
+ "serde_derive",
+ "solana-geyser-connector-lib",
+ "solana-logger",
+ "solana-sdk",
+ "tokio",
+ "toml",
+]
+
[[package]]
name = "traitobject"
version = "0.1.0"
diff --git a/Cargo.toml b/Cargo.toml
index 23c6c15..bd4ef07 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -4,9 +4,10 @@ members = [
"lib",
"connector-raw",
"connector-mango",
- "connector-trace",
+ "trackoor",
"service-mango-fills",
"service-mango-pnl",
+ "scratch"
]
diff --git a/config-devnet.toml b/config-devnet.toml
new file mode 100644
index 0000000..45edf71
--- /dev/null
+++ b/config-devnet.toml
@@ -0,0 +1,53 @@
+bind_ws_addr = "0.0.0.0:2082"
+
+[source]
+dedup_queue_size = 50000
+rpc_ws_url = ""
+
+[[source.grpc_sources]]
+name = "accountsdb-client"
+connection_string = "http://chi5.rpcpool.com:10001"
+retry_connection_sleep_secs = 30
+
+[source.grpc_sources.tls]
+ca_cert_path = "ca-devnet.pem"
+client_cert_path = "client-devnet.pem"
+client_key_path = "client-devnet.pem"
+domain_name = "mango-accountsdb.rpcpool.com"
+
+[source.snapshot]
+rpc_http_url = "http://mango.devnet.rpcpool.com/"
+program_id = "4skJ85cdxQAFVKbcGgfun8iZPL7BadVYXG3kGEGkufqA"
+
+[[markets]]
+name = "MNGO-PERP"
+event_queue = "uaUCSQejWYrDeYSuvn4As4kaCwJ2rLnRQSsSjY3ogZk"
+
+[[markets]]
+name = "ETH-PERP"
+event_queue = "8WLv5fKLYkyZpFG74kRmp2RALHQFcNKmH7eJn8ebHC13"
+
+[[markets]]
+name = "SOL-PERP"
+event_queue = "CZ5MCRvkN38d5pnZDDEEyMiED3drgDUVpEUjkuJq31Kf"
+
+[[markets]]
+name = "ADA-PERP"
+event_queue = "5v5fz2cCSy2VvrgVf5Vu7PF23RiZjv6BL36bgg48bA1c"
+
+[[markets]]
+name = "FTT-PERP"
+event_queue = "7rswj7FVZcMYUKxcTLndZhWBmuVNc2GuxqjuXU8KcPWv"
+
+[[markets]]
+name = "AVAX-PERP"
+event_queue = "4b7NqjqWoQoQh9V3dubfjkLPQVNJijwAwr7D9q6vTqqd"
+
+[[markets]]
+name = "BNB-PERP"
+event_queue = "96Y87LTz5Mops7wdT9EJo1eM79XToKYJJmRZxNatV85d"
+
+[[markets]]
+name = "MATIC-PERP"
+event_queue = "77maU5zdfYayqhqjBi2ocosM4PXvPXxbps2Up7dxDsMR"
+
diff --git a/connector-trace/src/main.rs b/connector-trace/src/main.rs
deleted file mode 100644
index d0bb0ca..0000000
--- a/connector-trace/src/main.rs
+++ /dev/null
@@ -1,103 +0,0 @@
-use {
- log::*,
- serde::Deserialize,
- solana_geyser_connector_lib::{chain_data::*, *},
- solana_sdk::{account::WritableAccount, clock::Epoch, pubkey::Pubkey},
- std::{fs::File, io::Read, str::FromStr},
-};
-
-#[derive(Clone, Debug, Deserialize)]
-pub struct Config {
- pub source: SourceConfig,
-}
-
-#[tokio::main]
-async fn main() -> anyhow::Result<()> {
- let args: Vec = std::env::args().collect();
- if args.len() < 2 {
- println!("requires a config file argument");
- return Ok(());
- }
-
- let config: Config = {
- let mut file = File::open(&args[1])?;
- let mut contents = String::new();
- file.read_to_string(&mut contents)?;
- toml::from_str(&contents).unwrap()
- };
-
- solana_logger::setup_with_default("info");
- info!("startup");
-
- let metrics_tx = metrics::start();
-
- let (account_write_queue_sender, account_write_queue_receiver) =
- async_channel::unbounded::();
- let (slot_queue_sender, slot_queue_receiver) = async_channel::unbounded::();
- let mut chain_cache = ChainData::new();
- // SOL-PERP event q
- let events_pk = Pubkey::from_str("31cKs646dt1YkA3zPyxZ7rUAkxTBz279w4XEobFXcAKP").unwrap();
- let mut last_ev_q_version = (0u64, 0u64);
-
- tokio::spawn(async move {
- loop {
- tokio::select! {
- Ok(account_write) = account_write_queue_receiver.recv() => {
- trace!("account write slot:{} pk:{:?} wv:{}", account_write.slot, account_write.pubkey, account_write.write_version);
-
- if account_write.pubkey == events_pk {
- info!("account write slot:{} pk:{:?} wv:{}", account_write.slot, account_write.pubkey, account_write.write_version);
-
- }
-
- chain_cache.update_account(
- account_write.pubkey,
- AccountData {
- slot: account_write.slot,
- write_version: account_write.write_version,
- account: WritableAccount::create(
- account_write.lamports,
- account_write.data.clone(),
- account_write.owner,
- account_write.executable,
- account_write.rent_epoch as Epoch,
- ),
- },
- );
- }
- Ok(slot_update) = slot_queue_receiver.recv() => {
-
- chain_cache.update_slot(SlotData {
- slot: slot_update.slot,
- parent: slot_update.parent,
- status: slot_update.status,
- chain: 0,
- });
- info!("slot update {} {:?} {:?}", slot_update.slot, slot_update.parent, slot_update.status);
-
- }
- };
-
- let events_acc = chain_cache.account(&events_pk);
- if events_acc.is_ok() {
- let acc = events_acc.unwrap();
-
- let ev_q_version = (acc.slot, acc.write_version);
- if ev_q_version != last_ev_q_version {
- info!("chain cache slot:{} wv:{}", acc.slot, acc.write_version);
- last_ev_q_version = ev_q_version;
- }
- }
- }
- });
-
- grpc_plugin_source::process_events(
- &config.source,
- account_write_queue_sender,
- slot_queue_sender,
- metrics_tx,
- )
- .await;
-
- Ok(())
-}
diff --git a/scratch/Cargo.toml b/scratch/Cargo.toml
new file mode 100644
index 0000000..c88a2c6
--- /dev/null
+++ b/scratch/Cargo.toml
@@ -0,0 +1,22 @@
+[package]
+name = "scratch"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+solana-geyser-connector-lib = { path = "../lib" }
+solana-logger = "=1.10.35"
+solana-sdk = "=1.10.35"
+log = "0.4"
+tokio = { version = "1", features = ["full"] }
+anyhow = "1.0"
+async-channel = "1.6"
+serde = "1.0"
+serde_derive = "1.0.130"
+toml = "0.5"
+mango = { path = "../../mango-v3/program" }
+bytemuck = "*"
+arrayref = "*"
+sqlite = "*"
\ No newline at end of file
diff --git a/scratch/src/main.rs b/scratch/src/main.rs
new file mode 100644
index 0000000..898d003
--- /dev/null
+++ b/scratch/src/main.rs
@@ -0,0 +1,13 @@
+fn main() {
+ let connection = sqlite::open(":memory:").unwrap();
+
+ connection
+ .execute(
+ "
+ CREATE TABLE users (name TEXT, age INTEGER);
+ INSERT INTO users VALUES ('Alice', 42);
+ INSERT INTO users VALUES ('Bob', 69);
+ ",
+ )
+ .unwrap();
+}
diff --git a/connector-trace/Cargo.toml b/trackoor/Cargo.toml
similarity index 77%
rename from connector-trace/Cargo.toml
rename to trackoor/Cargo.toml
index e08ba3c..644c9ef 100644
--- a/connector-trace/Cargo.toml
+++ b/trackoor/Cargo.toml
@@ -1,5 +1,5 @@
[package]
-name = "solana-geyser-connector-trace"
+name = "trackoor"
version = "0.1.0"
authors = ["Maximilian Schneider "]
edition = "2018"
@@ -15,3 +15,7 @@ async-channel = "1.6"
serde = "1.0"
serde_derive = "1.0.130"
toml = "0.5"
+mango = { path = "../../mango-v3/program" }
+bytemuck = "*"
+arrayref = "*"
+itertools = "*"
diff --git a/connector-trace/example-config.toml b/trackoor/example-config.toml
similarity index 100%
rename from connector-trace/example-config.toml
rename to trackoor/example-config.toml
diff --git a/trackoor/src/main.rs b/trackoor/src/main.rs
new file mode 100644
index 0000000..297fd48
--- /dev/null
+++ b/trackoor/src/main.rs
@@ -0,0 +1,194 @@
+use std::any::Any;
+use std::ops::Div;
+use std::time::{SystemTime, UNIX_EPOCH};
+
+use arrayref::array_ref;
+use itertools::Itertools;
+use mango::matching::{AnyNode, BookSide, LeafNode, NodeHandle};
+use mango::queue::{AnyEvent, EventQueueHeader, EventType, FillEvent};
+
+use {
+ log::*,
+ serde::Deserialize,
+ solana_geyser_connector_lib::{*, chain_data::*},
+ solana_sdk::{account::{ReadableAccount, WritableAccount}, clock::Epoch, pubkey::Pubkey},
+ std::{fs::File, io::Read, mem::size_of, str::FromStr},
+};
+
+#[derive(Clone, Debug, Deserialize)]
+pub struct Config {
+ pub source: SourceConfig,
+}
+
+#[tokio::main]
+async fn main() -> anyhow::Result<()> {
+ let args: Vec = std::env::args().collect();
+
+ solana_logger::setup_with_default("info");
+
+ if args.len() < 2 {
+ eprintln!("Please enter a config file path argument.");
+ return Ok(());
+ }
+
+ let config: Config = {
+ let mut file = File::open(&args[1])?;
+ let mut contents = String::new();
+ file.read_to_string(&mut contents)?;
+ toml::from_str(&contents).unwrap()
+ };
+
+ let metrics_tx = metrics::start();
+
+ let (account_write_queue_sender, account_write_queue_receiver) =
+ async_channel::unbounded::();
+
+ let (slot_queue_sender, slot_queue_receiver) = async_channel::unbounded::();
+
+ let mut chain_cache = ChainData::new();
+
+ let events_pk = Pubkey::from_str("Fu8q5EiFunGwSRrjFKjRUoMABj5yCoMEPccMbUiAT6PD").unwrap();
+
+ tokio::spawn(async move {
+ let mut trailing_l2_snapshot: Option> = None;
+
+ loop {
+ tokio::select! {
+ Ok(account_write) = account_write_queue_receiver.recv() => {
+ if account_write.pubkey != events_pk {
+ continue;
+ }
+
+ chain_cache.update_account(
+ account_write.pubkey,
+ AccountData {
+ slot: account_write.slot,
+ write_version: account_write.write_version,
+ account: WritableAccount::create(
+ account_write.lamports,
+ account_write.data.clone(),
+ account_write.owner,
+ account_write.executable,
+ account_write.rent_epoch as Epoch,
+ ),
+ },
+ );
+
+ info!("account write slot:{} pk:{:?} wv:{}", account_write.slot, account_write.pubkey, account_write.write_version);
+
+ let cache = chain_cache.account(&events_pk);
+
+ if cache.is_err() {
+ continue;
+ }
+
+ let book_side: &BookSide = bytemuck::from_bytes(cache.unwrap().account.data());
+
+ let current_l2_snapshot: Vec<(i64, i64)> = book_side
+ .iter_valid(SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs())
+ .map(|(node_handle, leaf_node)| (leaf_node.price(), leaf_node.quantity))
+ .group_by(|(price, quantity)| *price)
+ .into_iter()
+ .map(|(price, group)| (price, group.map(|(price, quantity)| quantity).fold(0, |acc, x| acc + x)))
+ .collect();
+
+ // TODO: Figure out how to run the diff only at the end of each block instead of at every write
+
+ info!("{:?}", ¤t_l2_snapshot[0..3]);
+
+ let mut diff: Vec<(i64, i64)> = vec!();
+
+ if let Some(ref previous_l2_snapshot) = trailing_l2_snapshot {
+ println!("Previous snapshot - diff and send deltas.");
+
+ for previous_order in previous_l2_snapshot {
+ let (previous_order_price, previous_order_size) = *previous_order;
+
+ let peer = current_l2_snapshot
+ .iter()
+ .find(|&(current_order_price, current_order_size)| previous_order_price == *current_order_price);
+
+ match peer {
+ None => diff.push((previous_order_price, 0)),
+ _ => continue
+ }
+ }
+
+ for current_order in ¤t_l2_snapshot {
+ let (current_order_price, current_order_size) = *current_order;
+
+ let peer = previous_l2_snapshot
+ .iter()
+ .find(|&(previous_order_price, previous_order_size)| *previous_order_price == current_order_price);
+
+ match peer {
+ Some(previous_order) => {
+ let (previous_order_price, previous_order_size) = previous_order;
+
+ if *previous_order_size == current_order_size {
+ continue;
+ }
+
+ diff.push(current_order.clone());
+ },
+ None => diff.push(current_order.clone())
+ }
+ }
+ } else {
+ println!("No previous snapshot - just send over the current one.");
+
+ for current_order in ¤t_l2_snapshot {
+ diff.push(current_order.clone());
+ }
+ }
+
+ let base_decimals = 9 as f64;
+
+ let quote_decimals = 6 as f64;
+
+ let quote_lot_size = 100 as f64;
+
+ let base_lot_size = 10000000 as f64;
+
+ let price_lots_to_ui_convertor = (10 as f64)
+ .powf(base_decimals - quote_decimals)
+ * quote_lot_size
+ / base_lot_size;
+
+ let base_lots_to_ui_convertor = base_lot_size / (10 as f64).powf(base_decimals);
+
+ // let price_ui = leaf_node.price() as f64 * price_lots_to_ui_convertor;
+ //
+ // let quantity_ui = leaf_node.quantity as f64 * base_lots_to_ui_convertor;
+
+ println!(
+ "{:#?}",
+ diff.iter().map(|(price, quantity)| (*price as f64 * price_lots_to_ui_convertor, *quantity as f64 * base_lots_to_ui_convertor)).collect::>()
+ );
+
+ trailing_l2_snapshot = Some(current_l2_snapshot);
+ }
+ Ok(slot_update) = slot_queue_receiver.recv() => {
+ chain_cache.update_slot(SlotData {
+ slot: slot_update.slot,
+ parent: slot_update.parent,
+ status: slot_update.status,
+ chain: 0,
+ });
+
+ info!("slot update {} {:?} {:?}", slot_update.slot, slot_update.parent, slot_update.status);
+ }
+ };
+ }
+ });
+
+ grpc_plugin_source::process_events(
+ &config.source,
+ account_write_queue_sender,
+ slot_queue_sender,
+ metrics_tx,
+ )
+ .await;
+
+ Ok(())
+}