event-scraper-v3/src/scrapePerpQueue.ts

import {
  getMultipleAccounts,
  IDS,
  PerpMarket,
  PerpMarketLayout,
  ZERO_BN,
} from '@blockworks-foundation/mango-client';
import BN from 'bn.js';
import { Connection as DbConnection } from 'typeorm';
import { Connection, PublicKey } from '@solana/web3.js';
import { wait } from './utils';
import { PerpEvent, PerpLiquidationEvent, PerpSequenceNumber } from './entity';

export async function getAllPerpMarkets(connection: Connection) {
  const DEFAULT_MANGO_GROUP_NAME = process.env.GROUP || 'mainnet.1';
  const defaultMangoGroupIds = IDS['groups'].find(
    (group) => group.name === DEFAULT_MANGO_GROUP_NAME
  );
  const perpMarketInfos = defaultMangoGroupIds.perpMarkets;
  const perpMarketPks = perpMarketInfos.map((mkt) => new PublicKey(mkt.publicKey));
  const allMarketAccountInfos = await getMultipleAccounts(connection, perpMarketPks);

  return perpMarketInfos.map((perpInfo, i) => {
    const decoded = PerpMarketLayout.decode(allMarketAccountInfos[i].accountInfo.data);
    return new PerpMarket(
      new PublicKey(perpInfo.publicKey),
      perpInfo.baseDecimals,
      perpInfo.quoteDecimals,
      decoded
    );
  });
}

function formatFillEvents(address, fillEvents) {
  return fillEvents.map((fillEvent) => {
    return {
      loadTimestamp: new Date(fillEvent.timestamp.toNumber() * 1000).toISOString(),
      address,
      takerSide: fillEvent.takerSide,
      seqNum: fillEvent.seqNum.toNumber(),
      makerFee: fillEvent.makerFee.toNumber(),
      takerFee: fillEvent.takerFee.toNumber(),
      maker: fillEvent.maker.toString(),
      makerOrderId: fillEvent.makerOrderId.toString(),
      taker: fillEvent.taker.toString(),
      takerOrderId: fillEvent.takerOrderId.toString(),
      price: fillEvent.price,
      quantity: fillEvent.quantity,
    };
  });
}

function formatLiquidateEvent(address, liquidationEvents) {
  return liquidationEvents.map((event) => {
    return {
      seqNum: event.seqNum.toNumber(),
      loadTimestamp: new Date(event.timestamp.toNumber() * 1000).toISOString(),
      address,
      liqee: event.liqee.toString(),
      liqor: event.liqor.toString(),
      price: event.price,
      quantity: event.quantity,
      liquidationFee: event.liquidationFee.toNumber(),
    };
  });
}

const getLastSeqNumForMarket = async (address, db) => {
  try {
    return await db
      .getRepository(PerpSequenceNumber)
      .createQueryBuilder('perp_sequence_number')
      .where('perp_sequence_number.address = :address', { address })
      .addOrderBy('perp_sequence_number.seqNum', 'DESC')
      .select('"seqNum"')
      .limit(1)
      .execute();
  } catch (error) {
    console.error('Unable to fetch perp_sequence_number', error);
  }
};

async function insertPerpEvents(perpMarketPk, allEvents, fillEvents, liquidateEvents, db) {
  if (allEvents.length === 0) return;

  if (fillEvents.length) {
    const newRecords = formatFillEvents(perpMarketPk, fillEvents);
    await db.getRepository(PerpEvent).createQueryBuilder().insert().values(newRecords).execute();
  }

  if (liquidateEvents.length) {
    const newRecords = formatLiquidateEvent(perpMarketPk, liquidateEvents);

    await db
      .getRepository(PerpLiquidationEvent)
      .createQueryBuilder()
      .insert()
      .values(newRecords)
      .execute();
  }

  const sequenceNumbers = allEvents
    .map((e) => e.fill || e.liquidate || e.out)
    .map((event) => ({
      address: perpMarketPk,
      seqNum: event.seqNum.toNumber(),
      loadTimestamp: new Date().toISOString(),
    }));

  await db
    .getRepository(PerpSequenceNumber)
    .createQueryBuilder()
    .insert()
    .values(sequenceNumbers)
    .execute();
}

async function captureEventsForPerpMarket(
  db: DbConnection,
  connection: Connection,
  perpMarket: PerpMarket
) {
  let lastSeqNum;
  const perpMarketPk = perpMarket.publicKey.toString();
  const lastSeqNumRecord = await getLastSeqNumForMarket(perpMarketPk, db);
  if (lastSeqNumRecord.length) {
    lastSeqNum = new BN(lastSeqNumRecord[0]?.seqNum);
  }
  const eventQueue = await perpMarket.loadEventQueue(connection);
  const allEvents = eventQueue.eventsSince(lastSeqNum);

  const fillEvents = allEvents
    .map((e) => e.fill)
    .filter((e) => !!e)
    .map((e) => perpMarket.parseFillEvent(e));
  const liquidateEvents = allEvents
    .map((e) => e.liquidate)
    .filter((e) => !!e)
    .map((e) => ({
      ...e,
      price: e.price.toNumber(),
      quantity: perpMarket.baseLotsToNumber(e.quantity),
    }));

  console.log(
    `market ${perpMarketPk} lastSeqNum: ${lastSeqNum?.toNumber()}, fill events: ${
      fillEvents.length
    }, all events: ${allEvents.length}`
  );

  try {
    await insertPerpEvents(perpMarketPk, allEvents, fillEvents, liquidateEvents, db);
  } catch (error) {
    console.error('Error inserting event queue data:', error);
  }
}

export const startPerpEventParsing = async (
  db: DbConnection,
  connection: Connection,
  perpMarket,
  waitTime
) => {
  while (true) {
    try {
      await captureEventsForPerpMarket(db, connection, perpMarket);
    } catch (e) {
      console.log(`Error in startPerpEventParsing() ${e}`);
    }

    await wait(waitTime);
  }
};