blockworks-foundation
/
mango-transaction-scraper-v3
mirror of https://github.com/blockworks-foundation/mango-transaction-scraper-v3.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Various bugfixes to anchor parsers - net balances, fill events, openOrdersBalance.

This commit is contained in:
Nicholas Clarke 2021-10-18 22:45:06 -07:00
parent ff8a3c0f37
commit 2fc8605bc9
5 changed files with 90 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

88
src/anchorParsers.ts
View File

@ -8,6 +8,7 @@ import {
PerpMarketConfig,
} from "@blockworks-foundation/mango-client";
import { Coder } from "@project-serum/anchor";
import { getLatestObjPerCombination } from './utils';
// Unfortunately ids.json does not correspond to the token indexes in the log - so keep a map here for reference
// mango group -> token index -> mint key
@ -57,6 +58,21 @@ var tokenIndexesMap = {
},
};
// Lot sizes also not available on ids.json
// mango group -> perp name -> lot sizes
var perpLotSizes = {
'98pjRuQjK3qA6gXts96PqZT4Ze5QmnCmt3QYjhbUSPue': {
'BTC-PERP': {
'baseLotSize': 100,
'quoteLotSize': 10
},
'SOL-PERP': {
'baseLotSize': 10000000,
'quoteLotSize': 100
}
}
}
var ids = IDS;
export function anchorParser(parsedTransactions, result, signature, blockTime, slot, blockDatetime) {
@ -79,13 +95,12 @@ export function anchorParser(parsedTransactions, result, signature, blockTime, s
// Can have multiple inserts per signature so add instructionNum column to allow a primary key
let eventNum = 1;
let allNetBalances: any = [];
for (const log of serializedLogMessages) {
const decodedEvent = coder.events.decode(log);
const eventName = decodedEvent?.name;
const eventData = decodedEvent?.data as any;
console.log("decodedEvent", decodedEvent);
if (eventName === "CachePricesLog") {
parsedTransactions.cache_prices.push(
...parseCachePrices(
@ -121,7 +136,9 @@ export function anchorParser(parsedTransactions, result, signature, blockTime, s
)
);
} else if (eventName === "TokenBalanceLog") {
parsedTransactions.net_balances.push(
// Net balances is a special case - we only want to keep the latest change to net balance for each
// (mangoAccount, symbol) pair in each transaction
allNetBalances.push(
parseTokenBalance(
eventNum,
eventData,
@ -216,7 +233,7 @@ export function anchorParser(parsedTransactions, result, signature, blockTime, s
)
);
} else if (eventName === "SettleFeesLog") {
parsedTransactions.settle_pnl.push(
parsedTransactions.settle_fees.push(
parseSettleFees(
eventNum,
eventData,
@ -284,7 +301,13 @@ export function anchorParser(parsedTransactions, result, signature, blockTime, s
} else {
throw new Error("Unknown anchor event: " + eventName);
}
eventNum++;
}
// Extract the latest (mangoAccount, symbol) pair from net balances changes in the transaction
parsedTransactions.net_balances.push(...getLatestObjPerCombination(allNetBalances, ['mango_account','symbol']))
}
function parseMngoAccrual(
@ -307,11 +330,7 @@ function parseMngoAccrual(
) as PerpMarketConfig;
let mangoDecimals = groupConfig.tokens.find((e) => e["symbol"] === 'MNGO')!.decimals
let mangoAccrualNative = new I80F48(eventData.mngoAccrual)
let mangoAccrualUi = mangoAccrualNative
.div(I80F48.fromNumber(Math.pow(10, mangoDecimals)))
.toNumber();
let mangoAccrualUi = eventData.mngoAccrual.toNumber() / Math.pow(10, mangoDecimals)
return {
margin_account: eventData.mangoAccount.toString(),
@ -352,21 +371,21 @@ function parseOpenOrdersBalance(
// Assuming that quote currency is always USDC
let quoteDecimals = tokens.find((e) => e.symbol === 'USDC').decimals;
let baseFree = new I80F48(eventData.baseFree).div(I80F48.fromNumber(Math.pow(10, baseDecimals))).toNumber()
let baseTotal = new I80F48(eventData.baseTotal).div(I80F48.fromNumber(Math.pow(10, baseDecimals))).toNumber()
let quoteFree = new I80F48(eventData.quoteFree).div(I80F48.fromNumber(Math.pow(10, quoteDecimals))).toNumber()
let quoteTotal = new I80F48(eventData.quoteTotal).div(I80F48.fromNumber(Math.pow(10, quoteDecimals))).toNumber()
let referrerRebatesAccrued = new I80F48(eventData.referrerRebatesAccrued).div(I80F48.fromNumber(Math.pow(10, quoteDecimals))).toNumber()
let baseFree = eventData.baseFree.toNumber() / Math.pow(10, baseDecimals)
let baseTotal = eventData.baseTotal.toNumber() / Math.pow(10, baseDecimals)
let quoteFree = eventData.quoteFree.toNumber() / Math.pow(10, quoteDecimals)
let quoteTotal = eventData.quoteTotal.toNumber() / Math.pow(10, quoteDecimals)
let referrerRebatesAccrued = eventData.referrerRebatesAccrued.toNumber() / Math.pow(10, quoteDecimals)
let marginAccount = eventData.mangoAccount.toString()
return {
margin_account: marginAccount,
symbol: symbol,
baseFree: baseFree,
baseTotal: baseTotal,
quoteFree: quoteFree,
quoteTotal: quoteTotal,
referrerRebatesAccrued: referrerRebatesAccrued,
base_free: baseFree,
base_total: baseTotal,
quote_free: quoteFree,
quote_total: quoteTotal,
referrer_rebates_accrued: referrerRebatesAccrued,
instruction_num: instructionNum,
mango_group: mangoGroup,
@ -468,32 +487,35 @@ function parseFillLog(
];
let marketIndex = eventData.marketIndex.toNumber();
let perpMarket = perpMarkets.find((e) => e["marketIndex"] === marketIndex);
let lotSizes = perpLotSizes[mangoGroupPk][perpMarket.name]
const fill = {
event_num: instructionNum,
maker: eventData.maker.toString(),
maker_fee: new I80F48(eventData.makerFee).toNumber(),
maker_order_id: eventData.makerOrderId.toNumber(),
price: eventData.price.toNumber() / perpMarket.quoteLotSize,
maker_order_id: eventData.makerOrderId.toString(),
price: eventData.price.toNumber() / lotSizes.quoteLotSize,
// Storing both price and quantity in UI terms to be consistent with db
quantity: eventData.quantity.toNumber() * perpMarket.baseLotSize / Math.pow(10, perpMarket.quoteDecimals),
quantity: eventData.quantity.toNumber() * lotSizes.baseLotSize / Math.pow(10, perpMarket.baseDecimals),
seq_num: eventData.seqNum.toNumber(),
taker: eventData.taker.toString(),
taker_fee: new I80F48(eventData.takerFee).toNumber(),
taker_order_id: eventData.takerOrderId.toNumber(),
taker_side: eventData.takerSide.toNumber(),
taker_order_id: eventData.takerOrderId.toString(),
taker_side: eventData.takerSide === 0 ? 'bid' : 'ask',
perp_market: perpMarket.name,
base_symbol: perpMarket.baseSymbol,
mango_group: mangoGroupPk,
block_datetime: blockDatetime,
slot: slot,
signature: signature,
blocktime: blockTime,
};
// TODO: verify i80f48 conversions
return fill;
}
@ -1032,13 +1054,7 @@ function parseFillLog(
let tokens = ids["groups"].find((e) => e["publicKey"] === mangoGroupPk)[
"tokens"
];
let quoteSymbol = ids["groups"].find((e) => e["publicKey"] === mangoGroupPk)[
"quoteSymbol"
];
let quoteDecimals = ids["groups"]
.find((e) => e["publicKey"] === mangoGroupPk)
["tokens"].find((e) => e.symbol === quoteSymbol).decimals;
let mangoAccountPk = eventData.mangoAccount.toString();
let tokenIndex = eventData.tokenIndex.toNumber();

3
src/index.ts
View File

@ -62,6 +62,8 @@ async function insertMangoTransactions(
await client.query('COMMIT');
} catch (e) {
await client.query('ROLLBACK');
console.log('transaction rolled back')
// TODO: check settle fees
throw e;
} finally {
client.release();
@ -91,6 +93,7 @@ async function insertMangoTransactions(
await client.query('COMMIT');
} catch (e) {
await client.query('ROLLBACK');
console.log('transaction rolled back')
throw e;
} finally {
client.release();

35
src/jsonParsers.ts
View File

@ -6,6 +6,8 @@ import {
GroupConfig,
PerpMarketConfig,
} from '@blockworks-foundation/mango-client';
import { getLatestObjPerCombination } from './utils';
// Unfortunately ids.json does not correspond to the token indexes in the log - so keep a map here for reference
// mango group -> token index -> mint key
@ -55,6 +57,21 @@ var oracleIndexesMap = {
},
};
// Lot sizes also not available on ids.json
// mango group -> perp name -> lot sizes
var perpLotSizes = {
'98pjRuQjK3qA6gXts96PqZT4Ze5QmnCmt3QYjhbUSPue': {
'BTC-PERP': {
'baseLotSize': 100,
'quoteLotSize': 10
},
'SOL-PERP': {
'baseLotSize': 10000000,
'quoteLotSize': 100
}
}
}
var ids = IDS;
export function jsonParser(parsedTransactions, result, instructions, signature, blockTime, slot, blockDatetime) {
@ -428,18 +445,6 @@ function extractSettleFees(
return out
}
function getLatestObjPerCombination(arr, combinationFields) {
// Utility function - iterates over arr and return the element with the highest index per set of combinationFields
let latestCombinations = {};
for (let values of arr) {
let combination = combinationFields.map((e) => values[e]);
latestCombinations[combination] = values;
}
return Object.values(latestCombinations);
}
function insertQuotesAroundField(jsonString, field) {
// Utility function to fix malformed json (json with quotes around strings)
// Assumes fields have a single space before the key
@ -491,6 +496,8 @@ function extractSettleFees(
let perpMarket = perpMarkets.find(
(e) => e['publicKey'] === perpMarketPk,
);
let lotSizes = perpLotSizes[mangoGroupPk][perpMarket.name]
let events: any = [];
let startDetailsStr = 'Program log: FillEvent details: ';
@ -519,8 +526,8 @@ function extractSettleFees(
maker_order_id: eventDetails['maker_order_id'],
// Storing both price and quantity in UI terms to be consistent with db
price: eventDetails['price'] / perpMarket.quoteLotSize,
quantity: eventDetails['quantity'] * perpMarket.baseLotSize / Math.pow(10, perpMarket.quoteDecimals),
price: eventDetails['price'] / lotSizes.quoteLotSize,
quantity: eventDetails['quantity'] * lotSizes.baseLotSize / Math.pow(10, perpMarket.baseDecimals),
seq_num: eventDetails['seq_num'],
taker: eventDetails['taker'],

4
src/tests/testParseTransactions.ts
View File

@ -41,6 +41,10 @@ async function processMangoTransactions(rawTransactionsPool, schema, limit) {
"4JuHNZy2GcFJYFCYoTrekyCa2eBSq8cECVUvQp32LbFshAVFaVrvypwoKNjpfZ2rK9N7AG5PHsqqhHSWuGU3rAFk", // MngoAccrualLog
"5MuJyoTsTzMGhGKDvN21BaZrfjaw1SeRgfTxz67U7zm7aVokrMqqvMhHvRCANGbjhvAeNFT1Zmk7vZoGp32b4H7t", // OpenOrdersBalanceLog, TokenBalanceLog
"Ztxy2398k8zDPh1VFiErvAQAnQdMuUok7Z772yCST9oZpagmqjggRuzU8ScrBEpcX4X4BknC7gxvfX128BGxkXg", // WithdrawLog
"3KzZ5jYfwp6LyPUt5LFbbPLEEsYNYtdAA5hs2KAdZxqarFiVVUKJ7WFxYXf8xeFnbWwypEWpNRCeE1d3y4e5Sxrk", // FillLog
"NGgdZu6YwNnFGoSMxc6ZgfUhSVy2rkB9BDY9yvYRLYZoteDSArWn5hDUGZuYCktC81jzGb6u9W6u7ab9mWutfQe", // SettlePnl
"67axUM8Q6tvCCNa6Cq8JfR9pLCAf5wyqc7gy1eafEtA4kcYE9Gte1LT8b9ziEWhtLfaGim5t5ChYU5uaQLfteGTJ", // SettlePnl - Multiple
"24hCHGXrf9nCnkioWf6HxtRqnQHGhhtTkARgA9tqkEmWb3uUzuE2oUU3so1JGNGZftCaBMjWek5rqAHRU3VAyQEJ" // Multiple net balances changes to the same (mangoAccount, token) pair
];
let signaturesSql = signatures.map((e) => "'" + e + "'").join(',');

10
src/utils.ts
View File

@ -47,4 +47,14 @@ export function notify(content) {
}
}
export function getLatestObjPerCombination(arr, combinationFields) {
// Utility function - iterates over arr and return the element with the highest index per set of combinationFields
let latestCombinations = {};
for (let values of arr) {
let combination = combinationFields.map((e) => values[e]);
latestCombinations[combination] = values;
}
return Object.values(latestCombinations);
}