This is now mango-fills

- mango-fills-connector: Is about getting data from rpc, websockets,
  geyser. Shall be reused in v4/client and other places.
- mango-fills-lib: is a library we'll probably remove
- service-* is the individual services

Cargo.lock

@@ -3040,6 +3040,97 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "mango-feeds-connector"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "arrayref",
+ "async-channel",
+ "async-stream 0.2.1",
+ "async-trait",
+ "base64 0.21.0",
+ "bs58 0.4.0",
+ "bytemuck",
+ "bytes 1.4.0",
+ "chrono",
+ "fixed",
+ "futures 0.3.26",
+ "futures-core",
+ "futures-util",
+ "itertools 0.10.5",
+ "jsonrpc-core 18.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "jsonrpc-core-client",
+ "log 0.4.17",
+ "native-tls",
+ "prost 0.9.0",
+ "rand 0.7.3",
+ "rustls 0.20.8",
+ "rustls-pemfile 1.0.2",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "solana-account-decoder",
+ "solana-client",
+ "solana-rpc",
+ "solana-sdk",
+ "tokio",
+ "tokio-stream",
+ "tonic 0.6.2",
+ "tonic-build 0.6.2",
+ "warp",
+]
+
+[[package]]
+name = "mango-feeds-lib"
+version = "0.1.0"
+dependencies = [
+ "anchor-lang",
+ "anyhow",
+ "arrayref",
+ "async-channel",
+ "async-stream 0.2.1",
+ "async-trait",
+ "base64 0.21.0",
+ "bs58 0.4.0",
+ "bytemuck",
+ "bytes 1.4.0",
+ "chrono",
+ "fixed",
+ "futures 0.3.26",
+ "futures-core",
+ "futures-util",
+ "itertools 0.10.5",
+ "jsonrpc-core 18.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "jsonrpc-core-client",
+ "log 0.4.17",
+ "mango-feeds-connector",
+ "mango-v4",
+ "native-tls",
+ "postgres-native-tls",
+ "postgres-types",
+ "postgres_query",
+ "prost 0.9.0",
+ "rand 0.7.3",
+ "rustls 0.20.8",
+ "rustls-pemfile 1.0.2",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "serum_dex 0.5.10 (git+https://github.com/jup-ag/openbook-program?branch=feat/expose-things)",
+ "solana-account-decoder",
+ "solana-client",
+ "solana-rpc",
+ "solana-sdk",
+ "tokio",
+ "tokio-postgres",
+ "tokio-postgres-rustls",
+ "tokio-stream",
+ "tonic 0.6.2",
+ "tonic-build 0.6.2",
+ "warp",
+]
+
 [[package]]
 name = "mango-v4"
 version = "0.6.0"
@@ -5221,24 +5312,24 @@ dependencies = [
  "anyhow",
  "async-channel",
  "async-trait",
- "bs58 0.3.1",
+ "bs58 0.4.0",
  "bytemuck",
  "client",
  "futures-channel",
  "futures-util",
- "log 0.3.9",
+ "log 0.4.17",
+ "mango-feeds-lib",
  "mango-v4",
  "serde",
  "serde_derive",
  "serde_json",
  "serum_dex 0.5.10 (git+https://github.com/openbook-dex/program)",
  "solana-client",
- "solana-geyser-connector-lib",
  "solana-logger",
  "solana-sdk",
  "tokio",
  "tokio-tungstenite",
- "toml 0.5.11",
+ "toml 0.7.1",
  "ws",
 ]
 
@@ -5258,12 +5349,12 @@ dependencies = [
  "futures-util",
  "jemallocator",
  "log 0.4.17",
+ "mango-feeds-lib",
  "mango-v4",
  "serde",
  "serde_derive",
  "serde_json",
  "serum_dex 0.5.10 (git+https://github.com/jup-ag/openbook-program?branch=feat/expose-things)",
- "solana-geyser-connector-lib",
  "solana-logger",
  "tokio",
  "tokio-tungstenite",
@@ -5286,12 +5377,12 @@ dependencies = [
  "futures-channel",
  "futures-util",
  "log 0.4.17",
+ "mango-feeds-lib",
  "mango-v4",
  "serde",
  "serde_derive",
  "serde_json",
  "serum_dex 0.5.10 (git+https://github.com/openbook-dex/program?branch=master)",
- "solana-geyser-connector-lib",
  "solana-logger",
  "tokio",
  "tokio-tungstenite",
@@ -5313,10 +5404,10 @@ dependencies = [
  "fixed",
  "jsonrpsee",
  "log 0.4.17",
+ "mango-feeds-lib",
  "mango-v4",
  "serde",
  "serde_derive",
- "solana-geyser-connector-lib",
  "solana-logger",
  "solana-sdk",
  "tokio",
@@ -5843,97 +5934,6 @@ dependencies = [
  "syn 1.0.107",
 ]
 
-[[package]]
-name = "solana-geyser-connector-data-streams"
-version = "0.1.0"
-dependencies = [
- "anyhow",
- "arrayref",
- "async-channel",
- "async-stream 0.2.1",
- "async-trait",
- "base64 0.9.3",
- "bs58 0.3.1",
- "bytemuck",
- "bytes 1.4.0",
- "chrono",
- "fixed",
- "futures 0.3.26",
- "futures-core",
- "futures-util",
- "itertools 0.10.5",
- "jsonrpc-core 18.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "jsonrpc-core-client",
- "log 0.4.17",
- "native-tls",
- "prost 0.9.0",
- "rand 0.7.3",
- "rustls 0.20.8",
- "rustls-pemfile 1.0.2",
- "serde",
- "serde_derive",
- "serde_json",
- "solana-account-decoder",
- "solana-client",
- "solana-rpc",
- "solana-sdk",
- "tokio",
- "tokio-stream",
- "tonic 0.6.2",
- "tonic-build 0.6.2",
- "warp",
-]
-
-[[package]]
-name = "solana-geyser-connector-lib"
-version = "0.1.0"
-dependencies = [
- "anchor-lang",
- "anyhow",
- "arrayref",
- "async-channel",
- "async-stream 0.2.1",
- "async-trait",
- "base64 0.21.0",
- "bs58 0.4.0",
- "bytemuck",
- "bytes 1.4.0",
- "chrono",
- "fixed",
- "futures 0.3.26",
- "futures-core",
- "futures-util",
- "itertools 0.10.5",
- "jsonrpc-core 18.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "jsonrpc-core-client",
- "log 0.4.17",
- "mango-v4",
- "native-tls",
- "postgres-native-tls",
- "postgres-types",
- "postgres_query",
- "prost 0.9.0",
- "rand 0.7.3",
- "rustls 0.20.8",
- "rustls-pemfile 1.0.2",
- "serde",
- "serde_derive",
- "serde_json",
- "serum_dex 0.5.10 (git+https://github.com/jup-ag/openbook-program?branch=feat/expose-things)",
- "solana-account-decoder",
- "solana-client",
- "solana-geyser-connector-data-streams",
- "solana-rpc",
- "solana-sdk",
- "tokio",
- "tokio-postgres",
- "tokio-postgres-rustls",
- "tokio-stream",
- "tonic 0.6.2",
- "tonic-build 0.6.2",
- "warp",
-]
-
 [[package]]
 name = "solana-gossip"
 version = "1.14.13"

Cargo.toml

@@ -1,6 +1,6 @@
 [workspace]
 members = [
-        "data-streams",
+        "connector",
         "lib",
         "service-mango-crank",
         "service-mango-fills",

connector/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "solana-geyser-connector-data-streams"
+name = "mango-feeds-connector"
 version = "0.1.0"
 authors = ["Christian Kamm <mail@ckamm.de>"]
 edition = "2021"

lib/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "solana-geyser-connector-lib"
+name = "mango-feeds-lib"
 version = "0.1.0"
 authors = ["Christian Kamm <mail@ckamm.de>"]
 edition = "2021"
@@ -8,7 +8,7 @@ edition = "2021"
 
 
 [dependencies]
-solana-geyser-connector-data-streams = { file = "0.1.0", path = "../data-streams" }
+mango-feeds-connector = { path = "../connector" }
 
 jsonrpc-core = "18.0.0"
 jsonrpc-core-client = { version = "18.0.0", features = ["ws", "http"] }

lib/src/lib.rs

@@ -8,7 +8,7 @@ pub mod serum;
 use serde::{ser::SerializeStruct, Serialize, Serializer};
 use serde_derive::Deserialize;
 
-pub use solana_geyser_connector_data_streams::*;
+pub use mango_feeds_connector::*;
 
 #[derive(Clone, Debug, Deserialize)]
 pub struct PostgresConfig {

service-mango-crank/Cargo.toml

@@ -5,7 +5,7 @@ authors = ["Maximilian Schneider <max@mango.markets>"]
 edition = "2018"
 
 [dependencies]
-solana-geyser-connector-lib = { path = "../lib" }
+mango-feeds-lib = { path = "../lib" }
 solana-client = "1"
 solana-logger = "1"
 solana-sdk = "1"

service-mango-crank/src/main.rs

@@ -15,11 +15,9 @@ use log::*;
 use solana_client::nonblocking::rpc_client::RpcClient;
 use std::{collections::HashSet, fs::File, io::Read, str::FromStr, sync::Arc, time::Duration};
 
+use mango_feeds_lib::FilterConfig;
+use mango_feeds_lib::{grpc_plugin_source, metrics, websocket_source, MetricsConfig, SourceConfig};
 use serde::Deserialize;
-use solana_geyser_connector_lib::FilterConfig;
-use solana_geyser_connector_lib::{
-    grpc_plugin_source, metrics, websocket_source, MetricsConfig, SourceConfig,
-};
 #[derive(Clone, Debug, Deserialize)]
 pub struct Config {
     pub source: SourceConfig,

service-mango-crank/src/mango_v4_perp_crank_sink.rs

@@ -7,9 +7,7 @@ use std::{
 use async_channel::Sender;
 use async_trait::async_trait;
 use log::*;
-use solana_geyser_connector_lib::{
+use mango_feeds_lib::{account_write_filter::AccountWriteSink, chain_data::AccountData};
-    account_write_filter::AccountWriteSink, chain_data::AccountData,
-};
 use solana_sdk::{
     account::ReadableAccount,
     instruction::{AccountMeta, Instruction},

service-mango-crank/src/openbook_crank_sink.rs

@@ -6,10 +6,10 @@ use std::{
 use async_channel::Sender;
 use async_trait::async_trait;
 use log::*;
-use serum_dex::{instruction::MarketInstruction, state::EventView};
+use mango_feeds_lib::{
-use solana_geyser_connector_lib::{
     account_write_filter::AccountWriteSink, chain_data::AccountData, serum::SerumEventQueueHeader,
 };
+use serum_dex::{instruction::MarketInstruction, state::EventView};
 use solana_sdk::{
     account::ReadableAccount,
     instruction::{AccountMeta, Instruction},

service-mango-crank/src/transaction_builder.rs

@@ -1,4 +1,4 @@
-use solana_geyser_connector_lib::{
+use mango_feeds_lib::{
     account_write_filter::{self, AccountWriteRoute},
     metrics::Metrics,
     AccountWrite, SlotUpdate,

service-mango-fills/Cargo.toml

@@ -5,7 +5,7 @@ authors = ["Christian Kamm <mail@ckamm.de>", "Maximilian Schneider <max@mango.ma
 edition = "2018"
 
 [dependencies]
-solana-geyser-connector-lib = { path = "../lib" }
+mango-feeds-lib = { path = "../lib" }
 solana-logger = "*"
 bs58 = "*"
 log = "*"

service-mango-fills/src/main.rs

@@ -26,18 +26,18 @@ use tokio::{
 };
 use tokio_tungstenite::tungstenite::{protocol::Message, Error};
 
-use serde::Deserialize;
+use mango_feeds_lib::{
-use solana_geyser_connector_lib::{
     fill_event_filter::FillEventType,
     fill_event_postgres_target,
     metrics::{MetricType, MetricU64},
     orderbook_filter::MarketConfig,
     FilterConfig, PostgresConfig, PostgresTlsConfig, StatusResponse,
 };
-use solana_geyser_connector_lib::{
+use mango_feeds_lib::{
     fill_event_filter::{self, FillCheckpoint, FillEventFilterMessage},
     grpc_plugin_source, metrics, websocket_source, MetricsConfig, SourceConfig,
 };
+use serde::Deserialize;
 
 type CheckpointMap = Arc<Mutex<HashMap<String, FillCheckpoint>>>;
 type PeerMap = Arc<Mutex<HashMap<SocketAddr, Peer>>>;

service-mango-orderbook/Cargo.toml

@@ -5,7 +5,7 @@ authors = ["Riordan Panayides <riordan@panayid.es>"]
 edition = "2021"
 
 [dependencies]
-solana-geyser-connector-lib = { path = "../lib" }
+mango-feeds-lib = { path = "../lib" }
 solana-logger = "*"
 bs58 = "*"
 log = "*"

service-mango-orderbook/src/main.rs

@@ -26,15 +26,13 @@ use tokio::{
 };
 use tokio_tungstenite::tungstenite::{protocol::Message, Error};
 
-use serde::Deserialize;
+use mango_feeds_lib::{grpc_plugin_source, metrics, websocket_source, MetricsConfig, SourceConfig};
-use solana_geyser_connector_lib::{
+use mango_feeds_lib::{
-    grpc_plugin_source, metrics, websocket_source, MetricsConfig, SourceConfig,
-};
-use solana_geyser_connector_lib::{
     metrics::{MetricType, MetricU64},
     orderbook_filter::{self, MarketConfig, OrderbookCheckpoint, OrderbookFilterMessage},
     FilterConfig, StatusResponse,
 };
+use serde::Deserialize;
 
 type CheckpointMap = Arc<Mutex<HashMap<String, OrderbookCheckpoint>>>;
 type PeerMap = Arc<Mutex<HashMap<SocketAddr, Peer>>>;

service-mango-pnl/Cargo.toml

@@ -5,7 +5,7 @@ authors = ["Christian Kamm <mail@ckamm.de>"]
 edition = "2021"
 
 [dependencies]
-solana-geyser-connector-lib = { path = "../lib" }
+mango-feeds-lib = { path = "../lib" }
 solana-logger = "~1.14.9"
 solana-sdk = "~1.14.9"
 log = "0.4"

service-mango-pnl/src/main.rs

@@ -1,8 +1,8 @@
 use {
     log::*,
+    mango_feeds_lib::chain_data::ChainData,
+    mango_feeds_lib::*,
     serde_derive::{Deserialize, Serialize},
-    solana_geyser_connector_lib::chain_data::ChainData,
-    solana_geyser_connector_lib::*,
     solana_sdk::pubkey::Pubkey,
     std::str::FromStr,
     std::{
@@ -18,8 +18,8 @@ use anchor_client::Cluster;
 use anchor_lang::Discriminator;
 use client::{chain_data, health_cache, AccountFetcher, Client, MangoGroupContext};
 use fixed::types::I80F48;
+use mango_feeds_lib::metrics::*;
 use mango_v4::state::{MangoAccount, MangoAccountValue, PerpMarketIndex};
-use solana_geyser_connector_lib::metrics::*;
 use solana_sdk::commitment_config::CommitmentConfig;
 use solana_sdk::{account::ReadableAccount, signature::Keypair};
 #[derive(Clone, Debug, Deserialize)]