feat: add non-orderbook coingecko routes

.gitignore

@@ -1 +1,2 @@
 /target
+.env

markets.json

@@ -14,5 +14,9 @@
     {
         "name" : "BONK/SOL",
         "address" : "Hs97TCZeuYiJxooo3U73qEHXg3dKpRL4uYKYRryEK9CF"
+    },
+    {
+        "name" : "BTC/USDC",
+        "address" : "3BAKsQd3RuhZKES2DGysMhjBdwjZYKYmxRqnSMtZ4KSN"
     }
 ]

sqlx-data.json

@@ -44,6 +44,36 @@
     },
     "query": "CREATE TABLE IF NOT EXISTS fills (\n            id numeric PRIMARY KEY,\n            time timestamptz not null,\n            market text not null,\n            open_orders text not null,\n            open_orders_owner text not null,\n            bid bool not null,\n            maker bool not null,\n            native_qty_paid numeric not null,\n            native_qty_received numeric not null,\n            native_fee_or_rebate numeric not null,\n            fee_tier text not null,\n            order_id text not null\n        )"
   },
+  "409267a0a1c925b3723396b1534b17ccdd7a27aac7bbcfaef159dbc6e3005625": {
+    "describe": {
+      "columns": [
+        {
+          "name": "address!",
+          "ordinal": 0,
+          "type_info": "Text"
+        },
+        {
+          "name": "raw_quote_size!",
+          "ordinal": 1,
+          "type_info": "Numeric"
+        },
+        {
+          "name": "raw_base_size!",
+          "ordinal": 2,
+          "type_info": "Numeric"
+        }
+      ],
+      "nullable": [
+        false,
+        null,
+        null
+      ],
+      "parameters": {
+        "Left": []
+      }
+    },
+    "query": "select market as \"address!\",\n        sum(native_qty_paid) as \"raw_quote_size!\",\n        sum(native_qty_received) as \"raw_base_size!\"\n        from fills \n        where \"time\" >= current_timestamp - interval '1 day' \n        and bid = true\n        group by market"
+  },
   "4bab7d4329b2969b2ba610546c660207740c9bafe644df55fa57101df30e4899": {
     "describe": {
       "columns": [],
@@ -74,6 +104,42 @@
     },
     "query": "CREATE INDEX IF NOT EXISTS idx_market_time ON fills (market, time)"
   },
+  "81d619e2680874afff756031aa4fef16678a7ea226a259e1bdb316bf52478939": {
+    "describe": {
+      "columns": [
+        {
+          "name": "market_name!",
+          "ordinal": 0,
+          "type_info": "Text"
+        },
+        {
+          "name": "high!",
+          "ordinal": 1,
+          "type_info": "Numeric"
+        },
+        {
+          "name": "low!",
+          "ordinal": 2,
+          "type_info": "Numeric"
+        },
+        {
+          "name": "close!",
+          "ordinal": 3,
+          "type_info": "Numeric"
+        }
+      ],
+      "nullable": [
+        true,
+        null,
+        null,
+        true
+      ],
+      "parameters": {
+        "Left": []
+      }
+    },
+    "query": "select \n        g.market_name as \"market_name!\", \n        g.high as \"high!\", \n        g.low as \"low!\", \n        c.\"close\" as \"close!\"\n      from \n        (\n          SELECT \n            market_name, \n            max(start_time) as \"start_time\", \n            max(high) as \"high\", \n            min(low) as \"low\" \n          from \n            candles \n          where \n            \"resolution\" = '1M' \n            and \"start_time\" >= current_timestamp - interval '1 day' \n          group by \n            market_name\n        ) as g \n        join candles c on g.market_name = c.market_name \n        and g.start_time = c.start_time \n      where \n        c.resolution = '1M'"
+  },
   "866773102b03b002e9d0535d3173f36264e1d30a46a5ec8240b0ea8076d6d1c5": {
     "describe": {
       "columns": [

src/database/fetch.rs

@@ -2,7 +2,13 @@ use chrono::{DateTime, Utc};
 use sqlx::{pool::PoolConnection, Postgres};
 
 use crate::{
-    structs::{candle::Candle, openbook::PgOpenBookFill, resolution::Resolution, trader::PgTrader},
+    structs::{
+        candle::Candle,
+        coingecko::{PgCoinGecko24HighLow, PgCoinGecko24HourVolume},
+        openbook::PgOpenBookFill,
+        resolution::Resolution,
+        trader::PgTrader,
+    },
     utils::AnyhowWrap,
 };
 
@@ -182,7 +188,7 @@ pub async fn fetch_tradingview_candles(
         and resolution = $2
         and start_time >= $3
         and end_time <= $4
-        ORDER BY start_time asc"#, // TODO: order?
+        ORDER BY start_time asc"#,
         market_name,
         resolution.to_string(),
         start_time,
@@ -264,3 +270,56 @@ LIMIT 10000"#,
     .await
     .map_err_anyhow()
 }
+
+pub async fn fetch_coingecko_24h_volume(
+    conn: &mut PoolConnection<Postgres>,
+) -> anyhow::Result<Vec<PgCoinGecko24HourVolume>> {
+    sqlx::query_as!(
+        PgCoinGecko24HourVolume,
+        r#"select market as "address!",
+        sum(native_qty_paid) as "raw_quote_size!",
+        sum(native_qty_received) as "raw_base_size!"
+        from fills 
+        where "time" >= current_timestamp - interval '1 day' 
+        and bid = true
+        group by market"#
+    )
+    .fetch_all(conn)
+    .await
+    .map_err_anyhow()
+}
+
+pub async fn fetch_coingecko_24h_high_low(
+    conn: &mut PoolConnection<Postgres>,
+) -> anyhow::Result<Vec<PgCoinGecko24HighLow>> {
+    sqlx::query_as!(
+        PgCoinGecko24HighLow,
+        r#"select 
+        g.market_name as "market_name!", 
+        g.high as "high!", 
+        g.low as "low!", 
+        c."close" as "close!"
+      from 
+        (
+          SELECT 
+            market_name, 
+            max(start_time) as "start_time", 
+            max(high) as "high", 
+            min(low) as "low" 
+          from 
+            candles 
+          where 
+            "resolution" = '1M' 
+            and "start_time" >= current_timestamp - interval '1 day' 
+          group by 
+            market_name
+        ) as g 
+        join candles c on g.market_name = c.market_name 
+        and g.start_time = c.start_time 
+      where 
+        c.resolution = '1M'"#
+    )
+    .fetch_all(conn)
+    .await
+    .map_err_anyhow()
+}

src/server/coingecko.rs

@@ -0,0 +1,123 @@
+use std::str::FromStr;
+
+use crate::server_error::ServerError;
+use actix_web::{get, web, HttpResponse, Scope};
+use anchor_lang::prelude::Pubkey;
+use futures::join;
+use num_traits::ToPrimitive;
+use openbook_candles::{
+    database::fetch::{fetch_coingecko_24h_high_low, fetch_coingecko_24h_volume},
+    structs::coingecko::{
+        CoinGecko24HourVolume, CoinGeckoPair, CoinGeckoTicker, PgCoinGecko24HighLow,
+    },
+    utils::WebContext,
+};
+use solana_client::nonblocking::rpc_client::RpcClient;
+
+pub fn service() -> Scope {
+    web::scope("/coingecko")
+        .service(pairs)
+        .service(tickers)
+        .service(orderbook)
+}
+
+#[get("/pairs")]
+pub async fn pairs(context: web::Data<WebContext>) -> Result<HttpResponse, ServerError> {
+    let markets = context.markets.clone();
+
+    let pairs = markets
+        .iter()
+        .map(|m| CoinGeckoPair {
+            ticker_id: m.name.clone(),
+            base: m.base_mint_key.clone(),
+            target: m.quote_mint_key.clone(),
+            pool_id: m.address.clone(),
+        })
+        .collect::<Vec<CoinGeckoPair>>();
+
+    Ok(HttpResponse::Ok().json(pairs))
+}
+
+#[get("/tickers")]
+pub async fn tickers(context: web::Data<WebContext>) -> Result<HttpResponse, ServerError> {
+    let markets = context.markets.clone();
+
+    // rpc get bid ask liquidity
+
+    let mut conn = context.pool.acquire().await.unwrap();
+    let raw_volumes = match fetch_coingecko_24h_volume(&mut conn).await {
+        Ok(c) => c,
+        Err(_) => return Err(ServerError::DbQueryError),
+    };
+    let high_low = match fetch_coingecko_24h_high_low(&mut conn).await {
+        Ok(c) => c,
+        Err(_) => return Err(ServerError::DbQueryError),
+    };
+
+    let default_hl = PgCoinGecko24HighLow::default();
+    let default_volume = CoinGecko24HourVolume::default();
+    let volumes: Vec<CoinGecko24HourVolume> = raw_volumes
+        .into_iter()
+        .map(|v| v.convert_to_readable(&markets))
+        .collect();
+    let tickers = markets
+        .iter()
+        .map(|m| {
+            let name = m.name.clone();
+            let high_low = high_low
+                .iter()
+                .find(|x| x.market_name == name)
+                .unwrap_or(&default_hl);
+            let volume = volumes
+                .iter()
+                .find(|x| x.market_name == name)
+                .unwrap_or(&default_volume);
+            CoinGeckoTicker {
+                ticker_id: m.name.clone(),
+                base_currency: m.base_mint_key.clone(),
+                target_currency: m.quote_mint_key.clone(),
+                last_price: high_low.close.to_f64().unwrap(),
+                base_volume: volume.base_volume.to_f64().unwrap(),
+                target_volume: volume.target_volume.to_f64().unwrap(),
+                liquidity_in_usd: 0.0,
+                bid: 0.0,
+                ask: 0.0,
+                high: high_low.high.to_f64().unwrap(),
+                low: high_low.low.to_f64().unwrap(),
+            }
+        })
+        .collect::<Vec<CoinGeckoTicker>>();
+
+    Ok(HttpResponse::Ok().json(tickers))
+}
+
+#[get("/orderbook")]
+pub async fn orderbook(context: web::Data<WebContext>) -> Result<HttpResponse, ServerError> {
+    let client = RpcClient::new(context.rpc_url.clone());
+
+    let markets = context.markets.clone();
+    let bid_keys = markets
+        .iter()
+        .map(|m| Pubkey::from_str(&m.bids_key).unwrap())
+        .collect::<Vec<Pubkey>>();
+    let ask_keys = markets
+        .iter()
+        .map(|m| Pubkey::from_str(&m.asks_key).unwrap())
+        .collect::<Vec<Pubkey>>();
+
+    // client.get_multiple_accounts(&bid_keys)
+
+    let (bid_results, _ask_results) = join!(
+        client.get_multiple_accounts(&bid_keys),
+        client.get_multiple_accounts(&ask_keys)
+    );
+
+    let x = bid_results.unwrap();
+
+    println!("{:?}", x);
+
+    // decode results
+
+    let markets = context.markets.clone();
+    Ok(HttpResponse::Ok().json(markets))
+}

src/server/main.rs

@@ -11,10 +11,11 @@ use openbook_candles::{
     structs::markets::{fetch_market_infos, load_markets},
     utils::{Config, WebContext},
 };
-use traders::{get_top_traders_by_base_volume, get_top_traders_by_quote_volume};
 use std::env;
+use traders::{get_top_traders_by_base_volume, get_top_traders_by_quote_volume};
 
 mod candles;
+mod coingecko;
 mod markets;
 mod server_error;
 mod traders;
@@ -45,6 +46,7 @@ async fn main() -> std::io::Result<()> {
     let pool = connect_to_database(&config).await.unwrap();
 
     let context = Data::new(WebContext {
+        rpc_url,
         pool,
         markets: market_infos,
     });
@@ -59,7 +61,8 @@ async fn main() -> std::io::Result<()> {
                     .service(get_candles)
                     .service(get_top_traders_by_base_volume)
                     .service(get_top_traders_by_quote_volume)
-                    .service(get_markets),
+                    .service(get_markets)
+                    .service(coingecko::service()),
             )
     })
     .bind(("127.0.0.1", 8080))?

src/server/mod.rs

@@ -1,3 +1,4 @@
 pub mod candles;
 pub mod traders;
-pub mod markets;
+pub mod markets;
+pub mod coingecko;

src/structs/coingecko.rs

@@ -0,0 +1,65 @@
+use num_traits::ToPrimitive;
+use serde::Serialize;
+use sqlx::types::Decimal;
+
+use super::{markets::MarketInfo, openbook::token_factor};
+
+#[derive(Debug, Clone, Serialize)]
+pub struct CoinGeckoPair {
+    pub ticker_id: String,
+    pub base: String,
+    pub target: String,
+    pub pool_id: String,
+}
+
+#[derive(Debug, Clone, Serialize)]
+pub struct CoinGeckoTicker {
+    pub ticker_id: String,
+    pub base_currency: String,
+    pub target_currency: String,
+    pub last_price: f64,
+    pub base_volume: f64,
+    pub target_volume: f64,
+    pub liquidity_in_usd: f64,
+    pub bid: f64,
+    pub ask: f64,
+    pub high: f64,
+    pub low: f64,
+}
+
+pub struct PgCoinGecko24HourVolume {
+    pub address: String,
+    pub raw_base_size: Decimal,
+    pub raw_quote_size: Decimal,
+}
+impl PgCoinGecko24HourVolume {
+    pub fn convert_to_readable(&self, markets: &Vec<MarketInfo>) -> CoinGecko24HourVolume {
+        let market = markets.iter().find(|m| m.address == self.address).unwrap();
+        let base_volume = (self.raw_base_size / token_factor(market.base_decimals))
+            .to_f64()
+            .unwrap();
+        let target_volume = (self.raw_quote_size / token_factor(market.quote_decimals))
+            .to_f64()
+            .unwrap();
+        CoinGecko24HourVolume {
+            market_name: market.name.clone(),
+            base_volume,
+            target_volume,
+        }
+    }
+}
+
+#[derive(Debug, Default)]
+pub struct CoinGecko24HourVolume {
+    pub market_name: String,
+    pub base_volume: f64,
+    pub target_volume: f64,
+}
+
+#[derive(Debug, Default)]
+pub struct PgCoinGecko24HighLow {
+    pub market_name: String,
+    pub high: Decimal,
+    pub low: Decimal,
+    pub close: Decimal,
+}

src/structs/markets.rs

@@ -18,6 +18,8 @@ pub struct MarketInfo {
     pub quote_decimals: u8,
     pub base_mint_key: String,
     pub quote_mint_key: String,
+    pub bids_key: String,
+    pub asks_key: String,
     pub base_lot_size: u64,
     pub quote_lot_size: u64,
 }
@@ -72,6 +74,8 @@ pub async fn fetch_market_infos(
                 AnchorDeserialize::deserialize(&mut market_bytes).unwrap();
 
             let market_address_string = serum_bytes_to_pubkey(raw_market.own_address).to_string();
+            let bids_key = serum_bytes_to_pubkey(raw_market.bids);
+            let asks_key = serum_bytes_to_pubkey(raw_market.asks);
             let base_mint_key = serum_bytes_to_pubkey(raw_market.coin_mint);
             let quote_mint_key = serum_bytes_to_pubkey(raw_market.pc_mint);
             mint_key_map.insert(base_mint_key, 0);
@@ -91,6 +95,8 @@ pub async fn fetch_market_infos(
                 quote_decimals: 0,
                 base_mint_key: base_mint_key.to_string(),
                 quote_mint_key: quote_mint_key.to_string(),
+                bids_key: bids_key.to_string(),
+                asks_key: asks_key.to_string(),
                 base_lot_size: raw_market.coin_lot_size,
                 quote_lot_size: raw_market.pc_lot_size,
             }

src/structs/mod.rs

@@ -1,4 +1,5 @@
 pub mod candle;
+pub mod coingecko;
 pub mod markets;
 pub mod openbook;
 pub mod resolution;

src/structs/trader.rs

@@ -41,6 +41,7 @@ pub struct TraderResponse {
     pub traders: Vec<Trader>,
 }
 
+// Note that the Postgres queries only return volumes in base or quote
 pub fn calculate_trader_volume(trader: PgTrader, decimals: u8) -> Trader {
     let bid_size = trader.raw_bid_size / token_factor(decimals);
     let ask_size = trader.raw_ask_size / token_factor(decimals);

src/utils/mod.rs

@@ -24,10 +24,12 @@ pub struct Config {
 }
 
 pub struct WebContext {
+    pub rpc_url: String,
     pub markets: Vec<MarketInfo>,
     pub pool: Pool<Postgres>,
 }
 
+#[allow(deprecated)]
 pub fn to_timestampz(seconds: u64) -> chrono::DateTime<Utc> {
     chrono::DateTime::<Utc>::from_utc(NaiveDateTime::from_timestamp(seconds as i64, 0), Utc)
 }