blockworks-foundation
/
serum-history
mirror of https://github.com/blockworks-foundation/serum-history.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Trade Collector for serum 

* collects from spot_markets defined in @mango/client IDS
* fetches loadFills endpoint every 5 seconds
* stores trades as base64 encoded blobs in redis lists per symbol&day
This commit is contained in:
Maximilian Schneider 2021-02-15 13:42:34 +01:00
commit 3c7c56b371
12 changed files with 2326 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

119
.gitignore vendored Normal file
View File

@ -0,0 +1,119 @@
# Created by https://www.toptal.com/developers/gitignore/api/node
# Edit at https://www.toptal.com/developers/gitignore?templates=node
### Node ###
# Logs
logs
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
lerna-debug.log*
# Diagnostic reports (https://nodejs.org/api/report.html)
report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
# Runtime data
pids
*.pid
*.seed
*.pid.lock
# Directory for instrumented libs generated by jscoverage/JSCover
lib-cov
# Coverage directory used by tools like istanbul
coverage
*.lcov
# nyc test coverage
.nyc_output
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
.grunt
# Bower dependency directory (https://bower.io/)
bower_components
# node-waf configuration
.lock-wscript
# Compiled binary addons (https://nodejs.org/api/addons.html)
build/Release
# Dependency directories
node_modules/
jspm_packages/
# TypeScript v1 declaration files
typings/
# TypeScript cache
*.tsbuildinfo
# Optional npm cache directory
.npm
# Optional eslint cache
.eslintcache
# Optional stylelint cache
.stylelintcache
# Microbundle cache
.rpt2_cache/
.rts2_cache_cjs/
.rts2_cache_es/
.rts2_cache_umd/
# Optional REPL history
.node_repl_history
# Output of 'npm pack'
*.tgz
# Yarn Integrity file
.yarn-integrity
# dotenv environment variables file
.env
.env.test
.env*.local
# parcel-bundler cache (https://parceljs.org/)
.cache
.parcel-cache
# Next.js build output
.next
# Nuxt.js build / generate output
.nuxt
dist
# Gatsby files
.cache/
# Comment in the public line in if your project uses Gatsby and not Next.js
# https://nextjs.org/blog/next-9-1#public-directory-support
# public
# vuepress build output
.vuepress/dist
# Serverless directories
.serverless/
# FuseBox cache
.fusebox/
# DynamoDB Local files
.dynamodb/
# TernJS port file
.tern-port
# Stores VSCode versions used for testing VSCode extensions
.vscode-test
# End of https://www.toptal.com/developers/gitignore/api/node

13
jasmine.json Normal file
View File

@ -0,0 +1,13 @@
{
"reporters": [
{
"name": "jasmine-spec-reporter#SpecReporter",
"options": {
"displayStacktrace": "all"
}
}
],
"spec_dir": "src/test",
"spec_files": ["**/*_spec.ts"],
"stopSpecOnExpectationFailure": false
}

34
package.json Normal file
View File

@ -0,0 +1,34 @@
{
"name": "serum-history",
"version": "1.0.0",
"description": "Aggregate Serum DEX history for display in TradingView charts.",
"main": "index.js",
"repository": "https://github.com/blockworks-foundation/serum-history",
"author": "Blockworks Foundation",
"license": "MIT",
"private": true,
"devDependencies": {
"@types/bn.js": "^5.1.0",
"@types/jasmine": "^3.6.3",
"jasmine": "^3.6.4",
"jasmine-spec-reporter": "^6.0.0",
"jasmine-ts": "^0.3.0",
"ts-node": "^9.1.1",
"typescript": "^4.1.4"
},
"dependencies": {
"@mango/client": "git+ssh://git@github.com:blockworks-foundation/mango-client-ts.git",
"@project-serum/serum": "^0.13.20",
"@solana/web3.js": "^0.91.0",
"tedis": "^0.1.12"
},
"scripts": {
"build": "tsc",
"start": "ts-node src/index.ts",
"watch": "tsc --watch",
"clean": "rm -rf dist",
"prepare": "run-s clean build",
"shell": "node -e \"$(< shell)\" -i --experimental-repl-await",
"test": "jasmine-ts --config=jasmine.json"
}
}

59
src/base64.ts Normal file
View File

@ -0,0 +1,59 @@
import { Candle, Trade, Coder } from './interfaces';
export class Base64TradeCoder implements Coder<Trade> {
constructor() {};
encode(t: Trade): string {
const buf = Buffer.alloc(14);
buf.writeFloatLE(t.price, 0);
buf.writeFloatLE(t.size, 4);
buf.writeUIntLE(t.ts, 8, 6);
const base64 = buf.toString('base64');
return base64;
};
decode(s: string): Trade {
const buf = Buffer.from(s, 'base64');
const trade = {
price: buf.readFloatLE(0),
size: buf.readFloatLE(4),
ts: buf.readUIntLE(8, 6)
};
return trade;
};
};
export class Base64CandleCoder implements Coder<Candle> {
constructor() {};
encode(c: Candle): string {
const buf = Buffer.alloc(36);
buf.writeFloatLE(c.open, 0);
buf.writeFloatLE(c.close, 4);
buf.writeFloatLE(c.high, 8);
buf.writeFloatLE(c.low, 12);
buf.writeFloatLE(c.volume, 16);
buf.writeFloatLE(c.vwap, 20);
buf.writeUIntLE(c.start, 24, 6);
buf.writeUIntLE(c.end, 30, 6);
const base64 = buf.toString('base64');
return base64;
};
decode(s: string): Candle {
const buf = Buffer.from(s, 'base64');
const candle = {
open: buf.readFloatLE(0),
close: buf.readFloatLE(4),
high: buf.readFloatLE(8),
low: buf.readFloatLE(12),
volume: buf.readFloatLE(16),
vwap: buf.readFloatLE(20),
start: buf.readUIntLE(24, 6),
end: buf.readUIntLE(30, 6)
};
return candle;
};
}

33
src/candle.ts Normal file
View File

@ -0,0 +1,33 @@
import { Candle, Trade } from "./interfaces";
export function batch(ts: Trade[], start: number, end: number): Candle | undefined {
const batchTrades = ts.filter(t => t.ts >= start && t.ts < end);
if (batchTrades.length == 0) {
return undefined;
} else {
let t0 = batchTrades[0];
let c = { open: t0.price,
close: t0.price,
high: t0.price,
low: t0.price,
volume: t0.size,
vwap: t0.price * t0.size,
start, end };
batchTrades.slice(1).forEach(t => {
c.close = t.price;
c.high = Math.max(c.high, t.price);
c.low = Math.min(c.low, t.price);
c.volume += t.size;
c.vwap += t.price * t.size;
});
c.vwap /= c.volume;
return c;
}
}

110
src/index.ts Normal file
View File

@ -0,0 +1,110 @@
import { Account, Connection, PublicKey } from '@solana/web3.js';
import { Market } from '@project-serum/serum';
import { IDS } from '@mango/client';
import { Tedis } from 'tedis';
import { Order, Trade } from './interfaces';
import { RedisStore } from './redis';
function sleep(ms: number) {
return new Promise(resolve => setTimeout(resolve, ms));
}
const MINUTES = 60*1000;
class OrderBuffer {
cache: Map<string, number>;
cleanupInterval: number;
lastCleanup: number;
timeToLive: number;
constructor(timeToLive = 10*MINUTES, cleanupInterval = 30*MINUTES) {
this.cache = new Map();
this.cleanupInterval = cleanupInterval;
this.lastCleanup = Date.now();
this.timeToLive = timeToLive;
}
// returns a list of unique trades that have not been observed by the order buffer
// guarantees to not emit a new trade even if the same fills have been supplied twice
filterNewTrades(fills: Order[]) : Trade[] {
const now = Date.now();
const takerOrders = fills.filter(o => !o.eventFlags.maker);
const allTrades = takerOrders.map( o => {
return { id: o.orderId.toString(16), price: o.price, size: o.size, ts: now };
});
const newTrades = allTrades.filter(t => !this.cache.has(t.id));
// store newTrades in cache
newTrades.forEach(t => this.cache.set(t.id, now));
// cleanup cache
if (now > this.lastCleanup + this.cleanupInterval) {
let staleCacheEntries: string[] = [];
this.cache.forEach((ts: number, key: string, _) => {
if (ts > now + this.timeToLive) {
staleCacheEntries.push(key);
}
});
staleCacheEntries.forEach((key) => {
this.cache.delete(key);
});
this.lastCleanup = now;
}
return newTrades;
}
}
// process data from cluster as it arrives
async function observeMarket(clusterUrl: string, programId: string, marketName:string, marketPk: string, tradeCb: (trades: Trade[]) => void) {
const marketAddress = new PublicKey(marketPk);
const programKey = new PublicKey(programId);
const connection = new Connection(clusterUrl);
console.log({ marketName, connection });
const market = await Market.load(connection, marketAddress, undefined, programKey);
console.log({ marketName, market });
const orderBuffer = new OrderBuffer();
while (true) {
try {
let fills = await market.loadFills(connection);
let trades = orderBuffer.filterNewTrades(fills);
if (trades.length > 0) {
tradeCb(trades);
}
} catch (err) {
const error = err.toString().split('\n', 1)[0];
console.error({ marketName, error });
}
await sleep(5000);
}
}
const { log, error } = console;
console.log = (...args: any[]) => log.bind(console)(new Date(), ...args);
console.error = (...args: any[]) => log.bind(error)(new Date(), ...args);
let network = "mainnet-beta"
let clusterUrl = IDS['cluster_urls'][network];
let programId = IDS[network]['dex_program_id'];
Object.entries(IDS[network]['spot_markets']).forEach(e => {
const [marketName, marketPk] = e;
console.log('start processing', {network, clusterUrl, marketName, marketPk});
const connection = new Tedis({
port: 6379,
host: "127.0.0.1"});
const store = new RedisStore(connection, marketName);
observeMarket(clusterUrl, programId, marketName as string, marketPk as string, async (trades) => {
console.log({marketName, trades});
for (let i = 0; i < trades.length; i += 1) {
await store.store(trades[i]);
}
});
});

37
src/interfaces.ts Normal file
View File

@ -0,0 +1,37 @@
import BN from 'bn.js';
export interface Order {
orderId: BN;
price: number;
size: number;
eventFlags: { maker: boolean };
};
export interface Trade {
id?: string;
price: number;
size: number;
ts: number;
};
export interface Coder<T> {
encode: (t: T) => string;
decode: (s: string) => T;
};
export interface Candle {
open: number;
close: number;
high: number;
low: number;
volume: number;
vwap: number;
start: number;
end: number;
};
export interface CandleStore {
store: (t: Trade) => Promise<void>;
load: (resolution: number, from: number, to:number) => Promise<Candle[]>;
};

55
src/redis.ts Normal file
View File

@ -0,0 +1,55 @@
import { Base64TradeCoder } from './base64';
const coder = new Base64TradeCoder();
import { batch } from './candle';
import { Candle, CandleStore, Trade } from './interfaces';
import { Tedis } from 'tedis';
export class RedisStore implements CandleStore {
connection: Tedis;
symbol: string;
constructor(connection: Tedis, symbol: string) {
this.connection = connection;
this.symbol = symbol;
};
async store(t: Trade): Promise<void> {
await this.connection.rpush(this.keyForTrade(t), coder.encode(t));
};
async load(resolution: number, from: number, to: number): Promise<Candle[]> {
const keys = this.keysForCandles(resolution, from, to);
const tradeRequests = keys.map(k => this.connection.lrange(k, 0, -1));
const tradeResponses = await Promise.all(tradeRequests);
const trades = tradeResponses.flat().map(t => coder.decode(t));
const candles: Candle[] = [];
while (from + resolution < to) {
let candle = batch(trades, from, from+resolution);
if (candle) {
candles.push(candle);
}
from += resolution;
}
return candles;
};
keyForTime(ts: number): string {
const d = new Date(ts);
return `${this.symbol}-${d.getUTCFullYear()}-${d.getUTCMonth()}-${d.getUTCDate()}`;
};
keyForTrade(t: Trade): string {
return this.keyForTime(t.ts);
};
keysForCandles(resolution: number, from: number, to: number): string[] {
const keys = new Set<string>();
while (from < to) {
keys.add(this.keyForTime(from));
from += resolution
};
keys.add(this.keyForTime(to));
return Array.from(keys);
};
};

43
src/test/base64_spec.ts Normal file
View File

@ -0,0 +1,43 @@
import { Base64TradeCoder, Base64CandleCoder } from '../base64';
const t = {id: '1234567890', price: 0.1234567, size: 1234567, ts: Date.now()};
const c = {open: 0.12345, close: 0.123456, high: 0.1234567, low: 0.12345678,
volume: 1234567, vwap: 0.123456789, start: 1234567890, end: 1234567899};
describe('Base64TradeCoder', () => {
it('encodes to 20 bytes', () => {
let e = new Base64TradeCoder().encode(t);
expect(e.length).toBe(20);
});
it('preserves price,size,ts', () => {
let e = new Base64TradeCoder().encode(t);
let d = new Base64TradeCoder().decode(e);
expect(d.price).toBeCloseTo(t.price, 7);
expect(d.size).toBeCloseTo(t.size, 7);
expect(d.ts).toBe(t.ts);
});
});
describe('Base64CandleCoder', () => {
it('encodes to 48 bytes', () => {
let e = new Base64CandleCoder().encode(c);
expect(e.length).toBe(48);
});
it('preserves data', () => {
let e = new Base64CandleCoder().encode(c);
let d = new Base64CandleCoder().decode(e);
expect(d.open).toBeCloseTo(c.open, 7);
expect(d.close).toBeCloseTo(c.close, 7);
expect(d.high).toBeCloseTo(c.high, 7);
expect(d.low).toBeCloseTo(c.low, 7);
expect(d.volume).toBeCloseTo(c.volume, 7);
expect(d.vwap).toBeCloseTo(c.vwap, 7);
expect(d.start).toBe(c.start);
expect(d.end).toBe(c.end);
});
});

40
src/test/redis_spec.ts Normal file
View File

@ -0,0 +1,40 @@
import { RedisStore } from '../redis';
import { Tedis } from "tedis";
const DAYS = 86400000;
const YEARS = 365*DAYS;
const ts = [{id: '0', price: 1.2, size: 3.4, ts: 1234567890000},
{id: '1', price: 2.3, size: 4.5, ts: 1234567890000-0.8*DAYS},
{id: '2', price: 3.4, size: 5.6, ts: 1234567890000+1*DAYS},
{id: '3', price: 4.5, size: 6.7, ts: 1234567890000+3*DAYS}];
const s = new RedisStore({} as Tedis, 'ABC/DEF');
describe('RedisStore', () => {
it('stores trades in buckets per day', () => {
let keys = ts.map(t => s.keyForTrade(t));
expect(keys[0]).toBe('ABC/DEF-2009-1-13');
expect(keys[0]).toBe(keys[1]);
expect(keys[0]).not.toBe(keys[2]);
expect(keys[0]).not.toBe(keys[3]);
expect(keys[2]).not.toBe(keys[3]);
});
it('iterates buckets per day', () => {
let from = ts[0].ts;
let to = from + 1.2*DAYS;
let keys = s.keysForCandles(DAYS, from, to);
expect(keys).toEqual(['ABC/DEF-2009-1-13',
'ABC/DEF-2009-1-14',
'ABC/DEF-2009-1-15']);
});
it('iterates preserving order', () => {
let from = ts[0].ts;
let to = from + 1*YEARS;
let keys = s.keysForCandles(DAYS, from, to);
expect(keys[0]).toEqual('ABC/DEF-2009-1-13');
expect(keys[keys.length-1]).toEqual('ABC/DEF-2010-1-13');
});
});

70
tsconfig.json Normal file
View File

@ -0,0 +1,70 @@
{
"compilerOptions": {
/* Visit https://aka.ms/tsconfig.json to read more about this file */
/* Basic Options */
// "incremental": true, /* Enable incremental compilation */
"target": "es5", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */
"module": "commonjs", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */
"lib": ["es2019"], /* Specify library files to be included in the compilation. */
// "allowJs": true, /* Allow javascript files to be compiled. */
// "checkJs": true, /* Report errors in .js files. */
// "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */
// "declaration": true, /* Generates corresponding '.d.ts' file. */
// "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */
// "sourceMap": true, /* Generates corresponding '.map' file. */
// "outFile": "./", /* Concatenate and emit output to single file. */
"outDir": "dist", /* Redirect output structure to the directory. */
"rootDir": "src", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */
// "composite": true, /* Enable project compilation */
// "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */
// "removeComments": true, /* Do not emit comments to output. */
// "noEmit": true, /* Do not emit outputs. */
// "importHelpers": true, /* Import emit helpers from 'tslib'. */
// "downlevelIteration": true, /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */
// "isolatedModules": true, /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */
/* Strict Type-Checking Options */
"strict": true, /* Enable all strict type-checking options. */
// "noImplicitAny": true, /* Raise error on expressions and declarations with an implied 'any' type. */
// "strictNullChecks": true, /* Enable strict null checks. */
// "strictFunctionTypes": true, /* Enable strict checking of function types. */
// "strictBindCallApply": true, /* Enable strict 'bind', 'call', and 'apply' methods on functions. */
// "strictPropertyInitialization": true, /* Enable strict checking of property initialization in classes. */
// "noImplicitThis": true, /* Raise error on 'this' expressions with an implied 'any' type. */
// "alwaysStrict": true, /* Parse in strict mode and emit "use strict" for each source file. */
/* Additional Checks */
// "noUnusedLocals": true, /* Report errors on unused locals. */
// "noUnusedParameters": true, /* Report errors on unused parameters. */
// "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */
// "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */
// "noUncheckedIndexedAccess": true, /* Include 'undefined' in index signature results */
/* Module Resolution Options */
// "moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */
// "baseUrl": "./", /* Base directory to resolve non-absolute module names. */
// "paths": {}, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */
// "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */
// "typeRoots": [], /* List of folders to include type definitions from. */
// "types": [], /* Type declaration files to be included in compilation. */
// "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */
"esModuleInterop": true, /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */
// "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */
// "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */
/* Source Map Options */
// "sourceRoot": "", /* Specify the location where debugger should locate TypeScript files instead of source locations. */
// "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */
// "inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */
// "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */
/* Experimental Options */
// "experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */
// "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */
/* Advanced Options */
"skipLibCheck": true, /* Skip type checking of declaration files. */
"forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */
}
}

1713
yarn.lock Normal file
View File

File diff suppressed because it is too large Load Diff