event-scraper-v3/src/utils.ts

import crypto from 'crypto';
import { blob, struct, u32 } from 'buffer-layout';
import { accountFlagsLayout, zeros } from '@project-serum/serum/lib/layout';
import { AccountInfo, Commitment, Connection, PublicKey } from '@solana/web3.js';
import { SERUM_DEX_V3_PK } from '.';

export const TEST_TRACKED_MARKETS = ['BTC/USDT', 'BTC/USDC'];

export const SOURCES = {
  websocket: 1,
  loadEventQueue: 2,
};

export const WEBSOCKET_MARKETS = [
  { name: 'BTC/USDT', address: 'C1EuT9VokAKLiW7i2ASnZUvxDoKuKkCpDDeNxAptuNe4' },
  { name: 'BTC/USDC', address: 'A8YFbxQYFVqKZaoYJLLUVcQiWP7G2MeEgW5wsAQgMvFw' },
  { name: 'ETH/USDT', address: '7dLVkUfBVfCGkFhSXDCq1ukM9usathSgS716t643iFGF' },
  { name: 'ETH/USDC', address: '4tSvZvnbyzHXLMTiFonMyxZoHmFqau1XArcRCVHLZ5gX' },
  { name: 'SOL/USDC', address: '9wFFyRfZBsuAha4YcuxcXLKwMxJR43S7fPfQLusDBzvT' },
  { name: 'SRM/USDC', address: 'ByRys5tuUWDgL73G8JBAEfkdFf8JWBzPBDHsBVQ5vbQA' },
  { name: 'MNGO/USDC', address: '3d4rzwpy9iGdCZvgxcu7B1YocYffVLsQXPXkBZKt2zLc' },
];

export const TRACKED_MARKETS = [
  // { name: 'BTC/WUSDT', address: '5r8FfnbNYcQbS1m4CYmoHYGjBtu6bxfo6UJHNRfzPiYH' },
  // { name: 'ETH/WUSDT', address: '71CtEComq2XdhGNbXBuYPmosAjMCPSedcgbNi5jDaGbR' },

  { name: 'BTC/USDT', address: 'C1EuT9VokAKLiW7i2ASnZUvxDoKuKkCpDDeNxAptuNe4' },
  { name: 'ETH/USDT', address: '7dLVkUfBVfCGkFhSXDCq1ukM9usathSgS716t643iFGF' },
  // { name: 'FIDA/USDT', address: 'EbV7pPpEvheLizuYX3gUCvWM8iySbSRAhu2mQ5Vz2Mxf' },
  // { name: 'HGET/USDT', address: 'ErQXxiNfJgd4fqQ58PuEw5xY35TZG84tHT6FXf5s4UxY' },
  // { name: 'KIN/USDT', address: '4nCFQr8sahhhL4XJ7kngGFBmpkmyf3xLzemuMhn6mWTm' },
  { name: 'RAY/USDT', address: 'teE55QrL4a4QSfydR9dnHF97jgCfptpuigbb53Lo95g' },
  { name: 'SOL/USDT', address: 'HWHvQhFmJB3NUcu1aihKmrKegfVxBEHzwVX6yZCKEsi1' },
  { name: 'SRM/USDT', address: 'AtNnsY1AyRERWJ8xCskfz38YdvruWVJQUVXgScC1iPb' },
  // { name: 'SUSHI/USDT', address: '6DgQRTpJTnAYBSShngAVZZDq7j9ogRN1GfSQ3cq9tubW' },

  { name: 'ALEPH/USDC', address: 'GcoKtAmTy5QyuijXSmJKBtFdt99e6Buza18Js7j9AJ6e' },
  { name: 'APEX/USDC', address: 'GX26tyJyDxiFj5oaKvNB9npAHNgdoV9ZYHs5ijs5yG2U' },
  // { name: 'BOP/USDC', address: '7MmPwD1K56DthW14P1PnWZ4zPCbPWemGs3YggcT1KzsM' },
  { name: 'BTC/USDC', address: 'A8YFbxQYFVqKZaoYJLLUVcQiWP7G2MeEgW5wsAQgMvFw' },
  // { name: 'CATO/USDC', address: '9fe1MWiKqUdwift3dEpxuRHWftG72rysCRHbxDy6i9xB' },
  // { name: 'CCAI/USDC', address: '7gZNLDbWE73ueAoHuAeFoSu7JqmorwCLpNTBXHtYSFTa' },
  { name: 'COPE/USDC', address: '6fc7v3PmjZG9Lk2XTot6BywGyYLkBQuzuFKd4FpCsPxk' },
  { name: 'ETH/USDC', address: '4tSvZvnbyzHXLMTiFonMyxZoHmFqau1XArcRCVHLZ5gX' },
  // { name: 'FAB/USDC', address: 'Cud48DK2qoxsWNzQeTL5D8sAiHsGwG8Ev1VMNcYLayxt' },
  { name: 'FIDA/USDC', address: 'E14BKBhDWD4EuTkWj1ooZezesGxMW8LPCps4W5PuzZJo' },
  { name: 'FTT/USDC', address: '2Pbh1CvRVku1TgewMfycemghf6sU9EyuFDcNXqvRmSxc' },
  // { name: 'HXRO/USDC', address: '6Pn1cSiRos3qhBf54uBP9ZQg8x3JTardm1dL3n4p29tA' },
  // { name: 'KIN/USDC', address: 'Bn6NPyr6UzrFAwC4WmvPvDr2Vm8XSUnFykM2aQroedgn' },
  // { name: 'LINK/USDC', address: '3hwH1txjJVS8qv588tWrjHfRxdqNjBykM1kMcit484up' },
  // { name: 'LIQ/USDC', address: 'FLKUQGh9VAG4otn4njLPUf5gaUPx5aAZ2Q6xWiD3hH5u' },
  // { name: 'MAPS/USDC', address: '3A8XQRWXC7BjLpgLDDBhQJLT5yPCzS16cGYRKHkKxvYo' },
  // { name: 'MEDIA/USDC', address: 'FfiqqvJcVL7oCCu8WQUMHLUC2dnHQPAPjTdSzsERFWjb' },
  // { name: 'MER/USDC', address: 'G4LcexdCzzJUKZfqyVDQFzpkjhB1JoCNL8Kooxi9nJz5' },
  { name: 'MNGO/USDC', address: '3d4rzwpy9iGdCZvgxcu7B1YocYffVLsQXPXkBZKt2zLc' },
  // { name: 'MOLA/USDC', address: 'HSpeWWRqBJ4HH2FPyfDhoN1AUq3gYoDenQGZASSqzYW1' },
  // { name: 'NINJA/USDC', address: 'J4oPt5Q3FYxrznkXLkbosAWrJ4rZLqJpGqz7vZUL4eMM' },
  // { name: 'OXY/USDC', address: 'GZ3WBFsqntmERPwumFEYgrX2B7J7G11MzNZAy7Hje27X' },
  // { name: 'OXYPOOL/USDC', address: 'G1uoNqQzdasMUvXV66Eki5dwjWv5N9YU8oHKJrE4mfka' },
  { name: 'RAY/USDC', address: '2xiv8A5xrJ7RnGdxXB42uFEkYHJjszEhaJyKKt4WaLep' },
  // { name: 'ROPE/USDC', address: '4Sg1g8U2ZuGnGYxAhc6MmX9MX7yZbrrraPkCQ9MdCPtF' },
  // { name: 'renBTC/USDC', address: '74Ciu5yRzhe8TFTHvQuEVbFZJrbnCMRoohBK33NNiPtv' },
  // { name: 'renDOGE/USDC', address: '5FpKCWYXgHWZ9CdDMHjwxAfqxJLdw2PRXuAmtECkzADk' },
  { name: 'SBR/USDC', address: 'HXBi8YBwbh4TXF6PjVw81m8Z3Cc4WBofvauj5SBFdgUs' },
  { name: 'SAMO/USDC', address: 'FR3SPJmgfRSKKQ2ysUZBu7vJLpzTixXnjzb84bY3Diif' },
  { name: 'SLRS/USDC', address: '2Gx3UfV831BAh8uQv1FKSPKS9yajfeeD8GJ4ZNb2o2YP' },
  { name: 'SNY/USDC', address: 'DPfj2jYwPaezkCmUNm5SSYfkrkz8WFqwGLcxDDUsN3gA' },
  { name: 'SOL/USDC', address: '9wFFyRfZBsuAha4YcuxcXLKwMxJR43S7fPfQLusDBzvT', interval: '4000' },
  // { name: 'SOLAPE/USDC', address: '4zffJaPyeXZ2wr4whHgP39QyTfurqZ2BEd4M5W6SEuon' },
  { name: 'SRM/USDC', address: 'ByRys5tuUWDgL73G8JBAEfkdFf8JWBzPBDHsBVQ5vbQA' },
  { name: 'STEP/USDC', address: '97qCB4cAVSTthvJu3eNoEx6AY6DLuRDtCoPm5Tdyg77S' },
  { name: 'SUSHI/USDC', address: 'A1Q9iJDVVS8Wsswr9ajeZugmj64bQVCYLZQLra2TMBMo' },
  // { name: 'TOMO/USDC', address: '8BdpjpSD5n3nk8DQLqPUyTZvVqFu6kcff5bzUX5dqDpy' },
  // { name: 'TULIP/USDC', address: '8GufnKq7YnXKhnB3WNhgy5PzU9uvHbaaRrZWQK6ixPxW' },
  // { name: 'UNI/USDC', address: '6JYHjaQBx6AtKSSsizDMwozAEDEZ5KBsSUzH7kRjGJon' },
  { name: 'USDT/USDC', address: '77quYg4MGneUdjgXCunt9GgM1usmrxKY31twEy3WHwcS' },
  // { name: 'WOO/USDC', address: '2Ux1EYeWsxywPKouRCNiALCZ1y3m563Tc4hq1kQganiq' },
  // { name: 'YFI/USDC', address: '7qcCo8jqepnjjvB5swP4Afsr3keVBs6gNpBTNubd1Kr2' },
  { name: 'MSOL/USDC', address: '6oGsL2puUgySccKzn9XA9afqF217LfxP5ocq4B3LWsjy' },
  { name: 'BNB/USDC', address: '3zzTxtDCt9PimwzGrgWJEbxZfSLetDMkdYegPanGNpMf' },
  { name: 'AVAX/USDC', address: 'E8JQstcwjuqN5kdMyUJLNuaectymnhffkvfg1j286UCr' },
  { name: 'LUNA/USDC', address: 'HBTu8hNaoT3VyiSSzJYa8jwt9sDGKtJviSwFa11iXdmE' },
];

export const wait = async (ms) => {
  return new Promise((resolve) => setTimeout(resolve, ms));
};

export const formatEvent = (uuid: string, event, marketMeta, loadTimestamp, source) => {
  return {
    address: marketMeta.address,
    programId: SERUM_DEX_V3_PK,
    baseCurrency: marketMeta.baseSymbol,
    quoteCurrency: marketMeta.quoteSymbol,
    fill: event.eventFlags['fill'],
    out: event.eventFlags['out'],
    bid: event.eventFlags['bid'],
    maker: event.eventFlags['maker'],
    openOrderSlot: event.openOrdersSlot.toString(),
    feeTier: event.feeTier.toString(),
    nativeQuantityReleased: parseInt(event.nativeQuantityReleased.toString()),
    nativeQuantityPaid: parseInt(event.nativeQuantityPaid.toString()),
    nativeFeeOrRebate: event.nativeFeeOrRebate.toString(),
    orderId: event.orderId.toString(),
    openOrders: event.openOrders.toString(),
    clientOrderId: event.clientOrderId.toString(),
    seqNum: event.seqNum,
    uuid,
    source,
    loadTimestamp,
  };
};

export const formatFilledEvents = (events, marketMeta, loadTimestamp, source) => {
  const newRecords = events.map((event) => {
    const uuid = crypto.createHash('sha256').update(JSON.stringify(event)).digest('hex');
    return formatEvent(uuid, event, marketMeta, loadTimestamp, source);
  });
  return newRecords;
};

const EVENT_QUEUE_HEADER = struct([
  blob(5),

  accountFlagsLayout('accountFlags'),
  u32('head'),
  zeros(4),
  u32('count'),
  zeros(4),
  u32('seqNum'),
  zeros(4),
]);

export const decodeEventHeader = (eventDataBuffer) => {
  return EVENT_QUEUE_HEADER.decode(eventDataBuffer);
};

export async function getMultipleAccounts(
  connection: Connection,
  publicKeys: PublicKey[],
  commitment?: Commitment
): Promise<{ publicKey: PublicKey; accountInfo: AccountInfo<Buffer> }[]> {
  const publicKeyStrs = publicKeys.map((pk) => pk.toBase58());
  // load connection commitment as a default
  commitment ||= connection.commitment;

  const args = commitment ? [publicKeyStrs, { commitment }] : [publicKeyStrs];
  // @ts-ignore
  const resp = await connection._rpcRequest('getMultipleAccounts', args);
  if (resp.error) {
    throw new Error(resp.error.message);
  }

  return resp.result.value.map(({ data, executable, lamports, owner }, i: number) => ({
    publicKey: publicKeys[i],
    accountInfo: {
      data: Buffer.from(data[0], 'base64'),
      executable,
      owner: new PublicKey(owner),
      lamports,
    },
  }));
}