Add api server (#1)

* Add

* progress

* Order placement

* Everything implemented

.gitignore

@@ -117,3 +117,5 @@ dist
 
 .idea/
 lib/
+
+secrets.json

.prettierignore

@@ -0,0 +1,2 @@
+lib/
+node_modules/

shell

@@ -1,3 +1,5 @@
 const lib = require('./lib/index');
 const solana = require('@solana/web3.js');
 const serum = require('@project-serum/serum');
+
+const SerumApi = lib.exchange.SerumApi;

src/config.ts

@@ -1,6 +1,6 @@
 import dotenv from "dotenv";
 
-// use passed port if sepcified otherwise default to the .env file
+// use passed port if specified otherwise default to the .env file
 const PASSED_PORT = process.env.PORT;
 
 dotenv.config();
@@ -14,3 +14,10 @@ export const LOGGING_DIR = process.env.LOGGING_DIR || "";
 export const RESTART_INTERVAL_SEC = parseInt(
   process.env.RESTART_INTERVAL_SEC || "0"
 );
+
+export const HARD_CODED_MINTS = process.env.HARD_CODED_MINTS || {};
+export const DEFAULT_TIMEOUT = 15000;
+export const BLOCKHASH_CACHE_TIME = 30;
+export const NUM_CONNECTIONS = 1;
+export const SOLANA_URL =
+  process.env.SOLANA_URL || "http://validator-lb.wirelesstable.net";

src/exchange/config.ts

@@ -0,0 +1,40 @@
+import { getLayoutVersion, MARKETS, TOKEN_MINTS } from "@project-serum/serum";
+import { HARD_CODED_MINTS } from "../config";
+import { Pair } from "./types";
+import { PublicKey } from "@solana/web3.js";
+
+export const MARKET_PARAMS = MARKETS.map((marketInfo) => {
+  const [coin, priceCurrency] = marketInfo.name.split("/");
+  return {
+    address: marketInfo.address,
+    market: new Pair(coin, priceCurrency),
+    programId: marketInfo.programId,
+    version: getLayoutVersion(marketInfo.programId),
+  };
+});
+
+export const HARD_CODED_COINS = new Set(Object.keys(HARD_CODED_MINTS));
+
+export const COIN_MINTS: { [coin: string]: string } = Object.fromEntries(
+  TOKEN_MINTS.filter((mint) => !(mint.name in HARD_CODED_MINTS))
+    .map((mint) => [mint.name, mint.address.toBase58()])
+    .concat(Object.entries(HARD_CODED_MINTS))
+);
+
+export const MINT_COINS: { [mint: string]: string } = Object.assign(
+  {},
+  ...Object.entries(COIN_MINTS).map(([coin, mint]) => ({
+    [mint]: coin,
+  }))
+);
+
+export const EXCHANGE_ENABLED_MARKETS: {
+  [exchange: string]: {
+    address: PublicKey;
+    market: Pair;
+    programId: PublicKey;
+    version: number;
+  }[];
+} = {
+  serum: MARKET_PARAMS,
+};

src/exchange/index.ts

@@ -0,0 +1,4 @@
+export * from "./api";
+export * from "./config";
+export * from "./solana";
+export * from "./utils";

src/exchange/solana.ts

@@ -0,0 +1,110 @@
+import { Account, Blockhash, Connection, Transaction } from "@solana/web3.js";
+import fetch, { Response } from "node-fetch";
+import jayson from "jayson/lib/client/browser";
+import { sleep } from "../utils";
+import { struct } from "superstruct";
+
+export async function signAndSerializeTransaction(
+  connection: Connection,
+  transaction: Transaction,
+  signers: Array<Account>,
+  blockhash: Blockhash
+): Promise<Buffer> {
+  transaction.recentBlockhash = blockhash;
+  transaction.sign(...signers);
+  return transaction.serialize();
+}
+
+export type RpcRequest = (methodName: string, args: Array<any>) => any;
+
+function jsonRpcResult(resultDescription: any) {
+  const jsonRpcVersion = struct.literal("2.0");
+  return struct.union([
+    struct({
+      jsonrpc: jsonRpcVersion,
+      id: "string",
+      error: "any",
+    }),
+    struct({
+      jsonrpc: jsonRpcVersion,
+      id: "string",
+      error: "null?",
+      result: resultDescription,
+    }),
+  ]);
+}
+
+function jsonRpcResultAndContext(resultDescription: any) {
+  return jsonRpcResult({
+    context: struct({
+      slot: "number",
+    }),
+    value: resultDescription,
+  });
+}
+
+const AccountInfoResult = struct({
+  executable: "boolean",
+  owner: "string",
+  lamports: "number",
+  data: "any",
+  rentEpoch: "number?",
+});
+
+export const GetMultipleAccountsAndContextRpcResult = jsonRpcResultAndContext(
+  struct.array([struct.union(["null", AccountInfoResult])])
+);
+
+export function createRpcRequest(url: string): RpcRequest {
+  const server = new jayson(async (request, callback) => {
+    const options = {
+      method: "POST",
+      body: request,
+      headers: {
+        "Content-Type": "application/json",
+      },
+    };
+
+    try {
+      let too_many_requests_retries = 5;
+      let res: Response = {};
+      let waitTime = 500;
+      for (;;) {
+        res = await fetch(url, options);
+        if (res.status !== 429 /* Too many requests */) {
+          break;
+        }
+        too_many_requests_retries -= 1;
+        if (too_many_requests_retries === 0) {
+          break;
+        }
+        console.log(
+          `Server responded with ${res.status} ${res.statusText}.  Retrying after ${waitTime}ms delay...`
+        );
+        await sleep(waitTime);
+        waitTime *= 2;
+      }
+
+      const text = await res.text();
+      if (res.ok) {
+        callback(null, text);
+      } else {
+        callback(new Error(`${res.status} ${res.statusText}: ${text}`));
+      }
+    } catch (err) {
+      callback(err);
+    }
+  }, {});
+
+  return (method, args) => {
+    return new Promise((resolve, reject) => {
+      server.request(method, args, (err, response) => {
+        if (err) {
+          reject(err);
+          return;
+        }
+        resolve(response);
+      });
+    });
+  };
+}

src/exchange/types.ts

@@ -0,0 +1,225 @@
+import { PublicKey } from "@solana/web3.js";
+import BN from "bn.js";
+import { Order as SerumOrder } from "@project-serum/serum/lib/market";
+
+export type Coin = string;
+export type Exchange = string;
+
+export class Pair {
+  coin;
+  priceCurrency;
+
+  constructor(coin: Coin, priceCurrency: Coin) {
+    this.coin = coin;
+    this.priceCurrency = priceCurrency;
+  }
+
+  static key(coin: Coin, priceCurrency: Coin): string {
+    return `${coin}/${priceCurrency}`;
+  }
+
+  key(): string {
+    return Pair.key(this.coin, this.priceCurrency);
+  }
+
+  static fromKey(key: string): Pair {
+    const [coin, priceCurrency] = key.split("/");
+    return new Pair(coin, priceCurrency);
+  }
+
+  equals(other: Pair): boolean {
+    return (
+      other.coin === this.coin && other.priceCurrency === this.priceCurrency
+    );
+  }
+}
+
+export enum Dir {
+  B = 1,
+  S = -1,
+}
+
+export enum OrderType {
+  limit = "limit",
+  ioc = "ioc",
+  postOnly = "postOnly",
+}
+
+export enum Liquidity {
+  T = "T",
+  M = "M",
+}
+
+export class Order<T = any> {
+  exchange: Exchange;
+  coin: Coin;
+  priceCurrency: Coin;
+  side: Dir;
+  price: number;
+  quantity: number;
+  info: T;
+
+  constructor(
+    exchange: Exchange,
+    coin: Coin,
+    priceCurrency: Coin,
+    side: Dir,
+    price: number,
+    quantity: number,
+    info: T
+  ) {
+    this.exchange = exchange;
+    this.coin = coin;
+    this.priceCurrency = priceCurrency;
+    this.side = side;
+    this.price = price;
+    this.quantity = quantity;
+    this.info = info;
+  }
+}
+
+export class OrderInfo {
+  orderId: string;
+  openOrdersAddress: string;
+  openOrdersSlot: number;
+  price: number;
+  priceLots: string;
+  size: number;
+  sizeLots: string;
+  side: "buy" | "sell";
+  clientId: string;
+  feeTier: number;
+
+  constructor(
+    orderId: string,
+    openOrdersAddress: string,
+    openOrdersSlot: number,
+    price: number,
+    priceLots: string,
+    size: number,
+    sizeLots: string,
+    side: "buy" | "sell",
+    clientId: string,
+    feeTier: number
+  ) {
+    this.orderId = orderId;
+    this.openOrdersAddress = openOrdersAddress;
+    this.openOrdersSlot = openOrdersSlot;
+    this.price = price;
+    this.priceLots = priceLots;
+    this.size = size;
+    this.sizeLots = sizeLots;
+    this.side = side;
+    this.clientId = clientId;
+    this.feeTier = feeTier;
+  }
+
+  static fromSerumOrder(order: SerumOrder): OrderInfo {
+    return new OrderInfo(
+      order.orderId.toString(),
+      order.openOrdersAddress.toBase58(),
+      order.openOrdersSlot,
+      order.price,
+      order.priceLots.toString(),
+      order.size,
+      order.sizeLots.toString(),
+      order.side,
+      order.clientId ? order.clientId.toString() : "",
+      order.feeTier
+    );
+  }
+
+  toSerumOrder(): SerumOrder {
+    return {
+      orderId: new BN(this.orderId),
+      openOrdersAddress: new PublicKey(this.openOrdersAddress),
+      openOrdersSlot: this.openOrdersSlot,
+      price: this.price,
+      priceLots: new BN(this.priceLots),
+      size: this.size,
+      sizeLots: new BN(this.sizeLots),
+      side: this.side,
+      clientId: new BN(this.clientId),
+      feeTier: this.feeTier,
+    };
+  }
+}
+
+export interface Trade<T = any> {
+  exchange: Exchange;
+  coin: Coin;
+  priceCurrency: Coin;
+  id: string;
+  orderId: string;
+  price: number;
+  quantity: number;
+  time: number;
+  side: Dir;
+  info?: T;
+}
+
+export interface Fill<T = any> {
+  exchange: Exchange;
+  coin: Coin;
+  priceCurrency: Coin;
+  side: Dir;
+  price: number;
+  quantity: number;
+  time: number;
+  orderId: string;
+  fee: number;
+  info?: T;
+}
+
+export interface L2OrderBook {
+  bids: [number, number][];
+  asks: [number, number][];
+  market: Pair;
+  validAt: number;
+  receivedAt: number;
+}
+
+export interface OwnOrders<T = Order> {
+  [orderId: string]: T;
+}
+
+export interface MarketInfo {
+  address: PublicKey;
+  baseMint: PublicKey;
+  quoteMint: PublicKey;
+  minOrderSize: number;
+  tickSize: number;
+  programId: PublicKey;
+  [propName: string]: unknown;
+}
+
+export interface RawTrade {
+  size: number;
+  price: number;
+  side: string;
+  eventFlags: {
+    fill: boolean;
+    out: boolean;
+    bid: boolean;
+    maker: boolean;
+  };
+  orderId: BN;
+  openOrders: PublicKey;
+  openOrdersSlot: number;
+  feeTier: number;
+  nativeQuantityReleased: BN;
+  nativeQuantityPaid: BN;
+  nativeFeeOrRebate: BN;
+}
+
+export interface TimestampedL2Levels {
+  orderbook: [number, number][];
+  receivedAt: number;
+}
+
+export type TokenAccountInfo = {
+  pubkey: PublicKey;
+  mint: PublicKey;
+  owner: PublicKey;
+  amount: number;
+};

src/exchange/utils.ts

@@ -0,0 +1,97 @@
+import {
+  Account,
+  Connection,
+  PublicKey,
+  SystemProgram,
+  Transaction,
+} from "@solana/web3.js";
+import BufferLayout from "buffer-layout";
+import { TokenInstructions } from "@project-serum/serum";
+import BN from "bn.js";
+
+export const ACCOUNT_LAYOUT = BufferLayout.struct([
+  BufferLayout.blob(32, "mint"),
+  BufferLayout.blob(32, "owner"),
+  BufferLayout.nu64("amount"),
+  BufferLayout.blob(93),
+]);
+
+export function parseTokenAccountData(
+  data: Buffer
+): { mint: PublicKey; owner: PublicKey; amount: number } {
+  const { mint, owner, amount } = ACCOUNT_LAYOUT.decode(data);
+  return {
+    mint: new PublicKey(mint),
+    owner: new PublicKey(owner),
+    amount,
+  };
+}
+
+export const MINT_LAYOUT = BufferLayout.struct([
+  BufferLayout.blob(4),
+  BufferLayout.blob(32, "mintAuthority"),
+  BufferLayout.blob(8, "supply"),
+  BufferLayout.u8("decimals"),
+  BufferLayout.u8("isInitialized"),
+  BufferLayout.blob(4, "freezeAuthorityOption"),
+  BufferLayout.blob(32, "freezeAuthority"),
+]);
+
+export function parseMintData(
+  data: Buffer
+): { mintAuthority: PublicKey; supply: number; decimals: number } {
+  const { mintAuthority, supply, decimals } = MINT_LAYOUT.decode(data);
+  return {
+    mintAuthority: new PublicKey(mintAuthority),
+    supply,
+    decimals,
+  };
+}
+
+export async function createAndInitializeTokenAccount({
+  connection,
+  payer,
+  mintPublicKey,
+  newAccount,
+}: {
+  connection: Connection;
+  payer: Account;
+  mintPublicKey: PublicKey;
+  newAccount: Account;
+}): Promise<string> {
+  const transaction = new Transaction();
+  const createAccountInstr = SystemProgram.createAccount({
+    fromPubkey: payer.publicKey,
+    newAccountPubkey: newAccount.publicKey,
+    lamports: await connection.getMinimumBalanceForRentExemption(
+      ACCOUNT_LAYOUT.span
+    ),
+    space: ACCOUNT_LAYOUT.span,
+    programId: TokenInstructions.TOKEN_PROGRAM_ID,
+  });
+  transaction.add(createAccountInstr);
+  transaction.add(
+    TokenInstructions.initializeAccount({
+      account: newAccount.publicKey,
+      mint: mintPublicKey,
+      owner: payer.publicKey,
+    })
+  );
+  const signers = [payer, newAccount];
+  return await connection.sendTransaction(transaction, signers);
+}
+
+export function makeClientOrderId(bits = 64): BN {
+  let binaryString = "1";
+  for (let i = 1; i < bits; i++) {
+    binaryString += Math.max(
+      Math.min(Math.floor(Math.random() * 2), 1),
+      0
+    ).toString();
+  }
+  return new BN(binaryString, 2);
+}
+
+export function getTokenMultiplierFromDecimals(decimals: number): BN {
+  return new BN(10).pow(new BN(decimals));
+}

src/index.ts

@@ -1,4 +1,5 @@
 import * as utils from "./utils";
 import * as configs from "./config";
+import * as exchange from "./exchange/index";
 
-export { utils, configs };
+export { utils, configs, exchange };

src/routes.ts

@@ -1,11 +1,35 @@
 import express from "express";
+import { SerumApi } from "./exchange/api";
+import expressAsyncHandler from "express-async-handler";
+import { logger } from "./utils";
 
 const router = express.Router();
+let api: SerumApi;
 
 router.get("/", (req, res, next) => {
-  res.send(
-    "Hello from the Serum rest server!"
-  );
+  res.send("Hello from the Serum rest server!");
 });
 
+router.use(
+  "/",
+  expressAsyncHandler(async (req, res, next) => {
+    if (!api) {
+      logger.debug("Creating api.");
+      api = await SerumApi.create();
+    }
+    next();
+  })
+);
+
+router.get(
+  "/market_info",
+  expressAsyncHandler(async (req, res, next) => {
+    logger.info("Received request to get market_info");
+    api
+      .getMarketInfo()
+      .then((marketInfo) => res.send({ status: "ok", data: marketInfo }))
+      .catch((err) => next(err));
+  })
+);
+
 export { router as default };

src/utils.ts

@@ -5,6 +5,7 @@ import winston, { format } from "winston";
 import "winston-daily-rotate-file";
 const { combine, timestamp, printf } = format;
 import fs from "fs";
+import { Dir } from "./exchange/types";
 // Logging
 if (
   LOGGING_DIR &&
@@ -71,9 +72,7 @@ class MorganStream {
 }
 export const morganStream = new MorganStream();
 
-export const getKeys = (
-  keys: string[]
-): string[] => {
+export const getKeys = (keys: string[]): any[] => {
   const allSecrets = JSON.parse(readFileSync(SECRETS_FILE, "utf-8"));
   const secrets: string[] = [];
   for (const key of keys) {
@@ -96,3 +95,30 @@ export function divideBnToNumber(numerator: BN, denominator: BN): number {
   const gcd = rem.gcd(denominator);
   return quotient + rem.div(gcd).toNumber() / denominator.div(gcd).toNumber();
 }
+
+export class DirUtil {
+  public static buySell = (dir: Dir): "buy" | "sell" => {
+    return dir === 1 ? "buy" : "sell";
+  };
+
+  public static parse = (raw: string | bigint | Dir): Dir => {
+    if (raw === Dir.B) {
+      return Dir.B;
+    } else if (raw === Dir.S) {
+      return Dir.S;
+    } else if (
+      typeof raw === "string" &&
+      ["bid", "buy", "b", "create", "long"].includes(raw.toLowerCase())
+    ) {
+      return Dir.B;
+    } else if (
+      typeof raw === "string" &&
+      ["ask", "sell", "sale", "a", "s", "redeem", "short"].includes(
+        raw.toLowerCase()
+      )
+    ) {
+      return Dir.S;
+    }
+    throw TypeError(`Cannot parse Dir from ${raw}`);
+  };
+}