2021-04-27 06:27:38 -07:00
|
|
|
import { Account, Connection, PublicKey } from "@solana/web3.js"
|
|
|
|
import { Market, decodeEventQueue } from "@project-serum/serum"
|
|
|
|
import cors from "cors"
|
|
|
|
import express from "express"
|
|
|
|
import { Tedis, TedisPool } from "tedis"
|
|
|
|
import { URL } from "url"
|
|
|
|
import { Order, Trade, TradeSide } from "./interfaces"
|
|
|
|
import { RedisConfig, RedisStore, createRedisStore } from "./redis"
|
|
|
|
import { encodeEvents } from "./serum"
|
2021-02-15 04:42:34 -08:00
|
|
|
|
|
|
|
function sleep(ms: number) {
|
2021-04-27 06:27:38 -07:00
|
|
|
return new Promise((resolve) => setTimeout(resolve, ms))
|
2021-02-15 04:42:34 -08:00
|
|
|
}
|
|
|
|
|
2021-04-27 06:27:38 -07:00
|
|
|
const MINUTES = 60 * 1000
|
2021-02-15 04:42:34 -08:00
|
|
|
|
|
|
|
class OrderBuffer {
|
2021-04-27 06:27:38 -07:00
|
|
|
cache: Map<string, number>
|
|
|
|
cleanupInterval: number
|
|
|
|
lastCleanup: number
|
|
|
|
timeToLive: number
|
|
|
|
|
|
|
|
constructor(timeToLive = 10 * MINUTES, cleanupInterval = 30 * MINUTES) {
|
|
|
|
this.cache = new Map()
|
|
|
|
this.cleanupInterval = cleanupInterval
|
|
|
|
this.lastCleanup = Date.now()
|
|
|
|
this.timeToLive = timeToLive
|
2021-02-15 04:42:34 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
// returns a list of unique trades that have not been observed by the order buffer
|
|
|
|
// guarantees to not emit a new trade even if the same fills have been supplied twice
|
2021-04-27 06:27:38 -07:00
|
|
|
filterNewTrades(fills: Order[]): Trade[] {
|
|
|
|
const now = Date.now()
|
|
|
|
const takerOrders = fills.filter((o) => !o.eventFlags.maker)
|
|
|
|
const allTrades = takerOrders.map((o) => {
|
|
|
|
return {
|
|
|
|
id: o.orderId.toString(16),
|
|
|
|
price: o.price,
|
|
|
|
side: o.side === "buy" ? TradeSide.Buy : TradeSide.Sell,
|
|
|
|
size: o.size,
|
|
|
|
ts: now,
|
|
|
|
}
|
|
|
|
})
|
|
|
|
const newTrades = allTrades.filter((t) => !this.cache.has(t.id))
|
2021-02-15 04:42:34 -08:00
|
|
|
|
|
|
|
// store newTrades in cache
|
2021-04-27 06:27:38 -07:00
|
|
|
newTrades.forEach((t) => this.cache.set(t.id, now))
|
2021-02-15 04:42:34 -08:00
|
|
|
|
|
|
|
// cleanup cache
|
|
|
|
if (now > this.lastCleanup + this.cleanupInterval) {
|
2021-04-27 06:27:38 -07:00
|
|
|
let staleCacheEntries: string[] = []
|
|
|
|
this.cache.forEach((ts: number, key: string, _) => {
|
2021-02-15 04:42:34 -08:00
|
|
|
if (ts > now + this.timeToLive) {
|
2021-04-27 06:27:38 -07:00
|
|
|
staleCacheEntries.push(key)
|
2021-02-15 04:42:34 -08:00
|
|
|
}
|
2021-04-27 06:27:38 -07:00
|
|
|
})
|
2021-02-15 04:42:34 -08:00
|
|
|
|
|
|
|
staleCacheEntries.forEach((key) => {
|
2021-04-27 06:27:38 -07:00
|
|
|
this.cache.delete(key)
|
|
|
|
})
|
2021-02-15 04:42:34 -08:00
|
|
|
|
2021-04-27 06:27:38 -07:00
|
|
|
this.lastCleanup = now
|
2021-02-15 04:42:34 -08:00
|
|
|
}
|
|
|
|
|
2021-04-27 06:27:38 -07:00
|
|
|
return newTrades
|
2021-02-15 04:42:34 -08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-04-06 15:10:20 -07:00
|
|
|
interface MarketConfig {
|
2021-04-27 06:27:38 -07:00
|
|
|
clusterUrl: string
|
|
|
|
programId: string
|
|
|
|
marketName: string
|
|
|
|
marketPk: string
|
|
|
|
}
|
2021-02-15 04:42:34 -08:00
|
|
|
|
2021-04-06 15:10:20 -07:00
|
|
|
async function collectTrades(m: MarketConfig, r: RedisConfig) {
|
2021-04-27 06:27:38 -07:00
|
|
|
const store = await createRedisStore(r, m.marketName)
|
|
|
|
const marketAddress = new PublicKey(m.marketPk)
|
|
|
|
const programKey = new PublicKey(m.programId)
|
|
|
|
const connection = new Connection(m.clusterUrl)
|
|
|
|
const market = await Market.load(connection, marketAddress, undefined, programKey)
|
2021-04-06 15:10:20 -07:00
|
|
|
|
|
|
|
async function storeTrades(ts: Trade[]) {
|
|
|
|
if (ts.length > 0) {
|
2021-04-27 06:27:38 -07:00
|
|
|
console.log(m.marketName, ts.length)
|
2021-04-06 15:10:20 -07:00
|
|
|
for (let i = 0; i < ts.length; i += 1) {
|
2021-04-27 06:27:38 -07:00
|
|
|
await store.storeTrade(ts[i])
|
2021-04-06 15:10:20 -07:00
|
|
|
}
|
|
|
|
}
|
2021-04-27 06:27:38 -07:00
|
|
|
}
|
2021-02-15 04:42:34 -08:00
|
|
|
|
2021-04-27 06:27:38 -07:00
|
|
|
const orderBuffer = new OrderBuffer()
|
2021-02-15 04:42:34 -08:00
|
|
|
while (true) {
|
|
|
|
try {
|
2021-04-27 06:27:38 -07:00
|
|
|
let fills = await market.loadFills(connection)
|
|
|
|
let trades = orderBuffer.filterNewTrades(fills)
|
|
|
|
storeTrades(trades)
|
2021-02-15 04:42:34 -08:00
|
|
|
} catch (err) {
|
2021-04-27 06:27:38 -07:00
|
|
|
const error = err.toString().split("\n", 1)[0]
|
|
|
|
console.error(m.marketName, { error })
|
2021-02-15 04:42:34 -08:00
|
|
|
}
|
|
|
|
|
2021-04-27 06:27:38 -07:00
|
|
|
await sleep(10000)
|
2021-02-15 04:42:34 -08:00
|
|
|
}
|
2021-04-27 06:27:38 -07:00
|
|
|
}
|
2021-04-06 15:10:20 -07:00
|
|
|
|
|
|
|
async function collectEventQueue(m: MarketConfig, r: RedisConfig) {
|
2021-04-27 06:27:38 -07:00
|
|
|
const store = await createRedisStore(r, m.marketName)
|
|
|
|
const marketAddress = new PublicKey(m.marketPk)
|
|
|
|
const programKey = new PublicKey(m.programId)
|
|
|
|
const connection = new Connection(m.clusterUrl)
|
|
|
|
const market = await Market.load(connection, marketAddress, undefined, programKey)
|
2021-04-06 15:10:20 -07:00
|
|
|
|
|
|
|
while (true) {
|
|
|
|
try {
|
2021-04-27 06:27:38 -07:00
|
|
|
const accountInfo = await connection.getAccountInfo(market["_decoded"].eventQueue)
|
2021-04-06 15:10:20 -07:00
|
|
|
if (accountInfo === null) {
|
2021-04-27 06:27:38 -07:00
|
|
|
throw new Error(`Event queue account for market ${m.marketName} not found`)
|
2021-04-06 15:10:20 -07:00
|
|
|
}
|
2021-04-27 06:27:38 -07:00
|
|
|
const events = decodeEventQueue(accountInfo.data, 1000).filter((e) => e.eventFlags.fill)
|
2021-04-06 15:10:20 -07:00
|
|
|
if (events.length > 0) {
|
2021-04-27 06:27:38 -07:00
|
|
|
const encoded = encodeEvents(events)
|
|
|
|
store.storeBuffer(Date.now(), encoded)
|
2021-04-06 15:10:20 -07:00
|
|
|
}
|
|
|
|
} catch (err) {
|
2021-04-27 06:27:38 -07:00
|
|
|
const error = err.toString().split("\n", 1)[0]
|
|
|
|
console.error(m.marketName, { error })
|
2021-02-15 04:42:34 -08:00
|
|
|
}
|
2021-04-27 06:27:38 -07:00
|
|
|
await sleep(10000)
|
2021-04-06 15:10:20 -07:00
|
|
|
}
|
2021-04-27 06:27:38 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
const redisUrl = new URL(process.env.REDISCLOUD_URL || "redis://localhost:6379")
|
|
|
|
const host = redisUrl.hostname
|
|
|
|
const port = parseInt(redisUrl.port)
|
|
|
|
let password: string | undefined
|
|
|
|
if (redisUrl.password !== "") {
|
|
|
|
password = redisUrl.password
|
2021-04-06 15:10:20 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
const network = "mainnet-beta"
|
2021-05-04 05:24:34 -07:00
|
|
|
const clusterUrl = process.env.RPC_ENDPOINT_URL || "https://solana-api.projectserum.com"
|
2021-04-27 06:27:38 -07:00
|
|
|
const programIdV3 = "9xQeWvG816bUx9EPjHmaT23yvVM2ZWbrrpZb9PusVFin"
|
2021-04-06 15:10:20 -07:00
|
|
|
|
|
|
|
const marketsV3: Record<string, string> = {
|
|
|
|
"BTC/USDT": "5r8FfnbNYcQbS1m4CYmoHYGjBtu6bxfo6UJHNRfzPiYH",
|
|
|
|
"ETH/USDT": "71CtEComq2XdhGNbXBuYPmosAjMCPSedcgbNi5jDaGbR",
|
2021-04-27 06:27:38 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
const nativeMarketsV3: Record<string, string> = {
|
|
|
|
"BTC/USDT": "C1EuT9VokAKLiW7i2ASnZUvxDoKuKkCpDDeNxAptuNe4",
|
|
|
|
"ETH/USDT": "7dLVkUfBVfCGkFhSXDCq1ukM9usathSgS716t643iFGF",
|
|
|
|
"BTC/USDC": "A8YFbxQYFVqKZaoYJLLUVcQiWP7G2MeEgW5wsAQgMvFw",
|
|
|
|
"ETH/USDC": "4tSvZvnbyzHXLMTiFonMyxZoHmFqau1XArcRCVHLZ5gX",
|
2021-04-26 20:32:30 -07:00
|
|
|
"SOL/USDC": "9wFFyRfZBsuAha4YcuxcXLKwMxJR43S7fPfQLusDBzvT",
|
|
|
|
"SOL/USDT": "HWHvQhFmJB3NUcu1aihKmrKegfVxBEHzwVX6yZCKEsi1",
|
|
|
|
"SRM/USDC": "ByRys5tuUWDgL73G8JBAEfkdFf8JWBzPBDHsBVQ5vbQA",
|
|
|
|
"SRM/USDT": "AtNnsY1AyRERWJ8xCskfz38YdvruWVJQUVXgScC1iPb",
|
|
|
|
"RAY/USDT": "teE55QrL4a4QSfydR9dnHF97jgCfptpuigbb53Lo95g",
|
2021-04-07 14:44:21 -07:00
|
|
|
"MCAPS/USDC": "GgzXqy6agt7nnfoPjAEAFpWqnUwLBK5r2acaAQqXiEM8",
|
2021-04-27 06:27:38 -07:00
|
|
|
}
|
2021-04-06 15:10:20 -07:00
|
|
|
|
2021-04-27 06:27:38 -07:00
|
|
|
const symbolsByPk = Object.assign(
|
|
|
|
{},
|
|
|
|
...Object.entries(marketsV3).map(([a, b]) => ({ [b]: a })),
|
|
|
|
...Object.entries(nativeMarketsV3).map(([a, b]) => ({ [b]: a }))
|
|
|
|
)
|
2021-04-06 15:10:20 -07:00
|
|
|
|
|
|
|
function collectMarketData(programId: string, markets: Record<string, string>) {
|
2021-04-27 06:27:38 -07:00
|
|
|
Object.entries(markets).forEach((e) => {
|
|
|
|
const [marketName, marketPk] = e
|
|
|
|
const marketConfig = { clusterUrl, programId, marketName, marketPk } as MarketConfig
|
|
|
|
collectTrades(marketConfig, { host, port, password, db: 0 })
|
2021-04-06 15:10:20 -07:00
|
|
|
//collectEventQueue(marketConfig, { host, port, password, db: 1});
|
2021-04-27 06:27:38 -07:00
|
|
|
})
|
|
|
|
}
|
2021-04-06 15:10:20 -07:00
|
|
|
|
2021-04-27 06:27:38 -07:00
|
|
|
collectMarketData(programIdV3, marketsV3)
|
|
|
|
collectMarketData(programIdV3, nativeMarketsV3)
|
2021-04-06 15:10:20 -07:00
|
|
|
|
|
|
|
interface TradingViewHistory {
|
2021-04-27 06:27:38 -07:00
|
|
|
s: string
|
|
|
|
t: number[]
|
|
|
|
c: number[]
|
|
|
|
o: number[]
|
|
|
|
h: number[]
|
|
|
|
l: number[]
|
|
|
|
v: number[]
|
|
|
|
}
|
|
|
|
|
|
|
|
const app = express()
|
|
|
|
app.use(cors())
|
|
|
|
|
2021-05-04 03:36:37 -07:00
|
|
|
const max_conn = parseInt(process.env.REDIS_MAX_CONN || "") || 200;
|
|
|
|
const redisConfig = { host, port, password, db: 0, max_conn }
|
2021-04-27 06:27:38 -07:00
|
|
|
const pool = new TedisPool(redisConfig)
|
|
|
|
|
|
|
|
const HOURS = 60 * MINUTES
|
|
|
|
const resolutions: { [id: string]: number | undefined } = {
|
2021-04-06 15:10:20 -07:00
|
|
|
"1": 1 * MINUTES,
|
2021-04-23 12:41:26 -07:00
|
|
|
"3": 3 * MINUTES,
|
|
|
|
"5": 5 * MINUTES,
|
|
|
|
"15": 15 * MINUTES,
|
|
|
|
"30": 30 * MINUTES,
|
2021-04-06 15:10:20 -07:00
|
|
|
"60": 1 * HOURS,
|
2021-04-23 12:41:26 -07:00
|
|
|
"120": 2 * HOURS,
|
|
|
|
"180": 3 * HOURS,
|
2021-04-06 15:10:20 -07:00
|
|
|
"240": 4 * HOURS,
|
2021-04-27 06:27:38 -07:00
|
|
|
"1D": 24 * HOURS,
|
|
|
|
}
|
|
|
|
|
|
|
|
app.get("/tv/config", async (req, res) => {
|
|
|
|
const response = {
|
|
|
|
supported_resolutions: Object.keys(resolutions),
|
|
|
|
supports_group_request: false,
|
|
|
|
supports_marks: false,
|
|
|
|
supports_search: true,
|
|
|
|
supports_timescale_marks: false,
|
|
|
|
}
|
|
|
|
res.send(response)
|
|
|
|
})
|
|
|
|
|
|
|
|
app.get("/tv/symbols", async (req, res) => {
|
|
|
|
const symbol = req.query.symbol as string
|
|
|
|
const response = {
|
|
|
|
name: symbol,
|
|
|
|
ticker: symbol,
|
|
|
|
description: symbol,
|
|
|
|
type: "Spot",
|
|
|
|
session: "24x7",
|
|
|
|
exchange: "Mango",
|
|
|
|
listed_exchange: "Mango",
|
|
|
|
timezone: "Etc/UTC",
|
|
|
|
has_intraday: true,
|
|
|
|
supported_resolutions: Object.keys(resolutions),
|
|
|
|
minmov: 1,
|
|
|
|
pricescale: 100,
|
|
|
|
}
|
|
|
|
res.send(response)
|
|
|
|
})
|
|
|
|
|
|
|
|
app.get("/tv/history", async (req, res) => {
|
2021-04-06 15:10:20 -07:00
|
|
|
// parse
|
2021-04-27 06:27:38 -07:00
|
|
|
const marketName = req.query.symbol as string
|
2021-04-26 20:32:30 -07:00
|
|
|
const marketPk = nativeMarketsV3[marketName] || marketsV3[marketName]
|
2021-04-27 06:27:38 -07:00
|
|
|
const resolution = resolutions[req.query.resolution as string] as number
|
|
|
|
let from = parseInt(req.query.from as string) * 1000
|
|
|
|
let to = parseInt(req.query.to as string) * 1000
|
2021-04-06 15:10:20 -07:00
|
|
|
|
|
|
|
// validate
|
2021-04-27 06:27:38 -07:00
|
|
|
const validSymbol = marketPk != undefined
|
|
|
|
const validResolution = resolution != undefined
|
|
|
|
const validFrom = true || new Date(from).getFullYear() >= 2021
|
2021-04-06 15:10:20 -07:00
|
|
|
|
|
|
|
// respond
|
|
|
|
if (!(validSymbol && validResolution && validFrom)) {
|
2021-04-27 06:27:38 -07:00
|
|
|
const error = { s: "error", validSymbol, validResolution, validFrom }
|
|
|
|
console.error({ req, error })
|
|
|
|
res.status(500).send(error)
|
|
|
|
return
|
2021-04-06 15:10:20 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
try {
|
2021-04-27 06:27:38 -07:00
|
|
|
const conn = await pool.getTedis()
|
2021-04-06 15:10:20 -07:00
|
|
|
try {
|
2021-04-27 06:27:38 -07:00
|
|
|
const store = new RedisStore(conn, marketName)
|
2021-04-06 15:10:20 -07:00
|
|
|
|
|
|
|
// snap candle boundaries to exact hours
|
2021-04-27 06:27:38 -07:00
|
|
|
from = Math.floor(from / resolution) * resolution
|
|
|
|
to = Math.ceil(to / resolution) * resolution
|
2021-04-06 15:10:20 -07:00
|
|
|
|
|
|
|
// ensure the candle is at least one period in length
|
|
|
|
if (from == to) {
|
2021-04-27 06:27:38 -07:00
|
|
|
to += resolution
|
|
|
|
}
|
|
|
|
const candles = await store.loadCandles(resolution, from, to)
|
|
|
|
const response = {
|
|
|
|
s: "ok",
|
|
|
|
t: candles.map((c) => c.start / 1000),
|
|
|
|
c: candles.map((c) => c.close),
|
|
|
|
o: candles.map((c) => c.open),
|
|
|
|
h: candles.map((c) => c.high),
|
|
|
|
l: candles.map((c) => c.low),
|
|
|
|
v: candles.map((c) => c.volume),
|
2021-04-06 15:10:20 -07:00
|
|
|
}
|
2021-04-27 06:27:38 -07:00
|
|
|
res.send(response)
|
|
|
|
return
|
2021-04-06 15:10:20 -07:00
|
|
|
} finally {
|
2021-04-27 06:27:38 -07:00
|
|
|
pool.putTedis(conn)
|
2021-04-06 15:10:20 -07:00
|
|
|
}
|
|
|
|
} catch (e) {
|
2021-04-27 06:27:38 -07:00
|
|
|
console.error({ req, e })
|
|
|
|
const error = { s: "error" }
|
|
|
|
res.status(500).send(error)
|
2021-04-06 15:10:20 -07:00
|
|
|
}
|
2021-04-27 06:27:38 -07:00
|
|
|
})
|
2021-02-15 04:42:34 -08:00
|
|
|
|
2021-04-27 06:27:38 -07:00
|
|
|
app.get("/trades/address/:marketPk", async (req, res) => {
|
2021-04-06 15:10:20 -07:00
|
|
|
try {
|
2021-04-27 06:27:38 -07:00
|
|
|
const conn = await pool.getTedis()
|
2021-04-06 15:10:20 -07:00
|
|
|
try {
|
2021-04-27 06:27:38 -07:00
|
|
|
const marketPk = req.params.marketPk as string
|
|
|
|
const marketName = symbolsByPk[marketPk]
|
|
|
|
const store = new RedisStore(conn, marketName)
|
|
|
|
const trades = await store.loadRecentTrades()
|
|
|
|
const response = {
|
|
|
|
success: true,
|
|
|
|
data: trades.map((t) => {
|
|
|
|
return {
|
|
|
|
market: marketName,
|
|
|
|
marketAddress: marketPk,
|
|
|
|
price: t.price,
|
|
|
|
size: t.size,
|
|
|
|
side: t.side == TradeSide.Buy ? "buy" : "sell",
|
|
|
|
time: t.ts,
|
|
|
|
orderId: "",
|
|
|
|
feeCost: 0,
|
|
|
|
}
|
|
|
|
}),
|
|
|
|
}
|
|
|
|
res.send(response)
|
|
|
|
return
|
2021-04-06 15:10:20 -07:00
|
|
|
} finally {
|
2021-04-27 06:27:38 -07:00
|
|
|
pool.putTedis(conn)
|
2021-04-06 15:10:20 -07:00
|
|
|
}
|
|
|
|
} catch (e) {
|
2021-04-27 06:27:38 -07:00
|
|
|
console.error({ req, e })
|
|
|
|
const error = { s: "error" }
|
|
|
|
res.status(500).send(error)
|
2021-04-06 15:10:20 -07:00
|
|
|
}
|
2021-04-27 06:27:38 -07:00
|
|
|
})
|
2021-04-06 15:10:20 -07:00
|
|
|
|
2021-04-27 06:27:38 -07:00
|
|
|
const httpPort = parseInt(process.env.PORT || "5000")
|
|
|
|
app.listen(httpPort)
|