diff --git a/src/index.ts b/src/index.ts index 30552be..5c81787 100644 --- a/src/index.ts +++ b/src/index.ts @@ -56,7 +56,7 @@ async function insertMangoTransactions( ); await client.query(insertsSql); } - console.log(tableName + ' inserted'); + console.log(inserts.length + ' records inserted into ' + tableName); } } await client.query('COMMIT'); @@ -74,28 +74,31 @@ async function insertMangoTransactions( { table: table }, ); - client = await rawTransactionsPool.connect(); - try { - await client.query('BEGIN'); + if (processStates.length > 0) { + client = await rawTransactionsPool.connect(); + try { + await client.query('BEGIN'); - for (let i = 0, j = processStates.length; i < j; i += batchSize) { - let updatesBatch = processStates.slice(i, i + batchSize); - let updatedSql = - pgp.helpers.update(updatesBatch, processStateCs) + - ' WHERE v.signature = t.signature'; - await client.query(updatedSql); + for (let i = 0, j = processStates.length; i < j; i += batchSize) { + let updatesBatch = processStates.slice(i, i + batchSize); + let updatedSql = + pgp.helpers.update(updatesBatch, processStateCs) + + ' WHERE v.signature = t.signature'; + await client.query(updatedSql); + } + console.log(processStates.length + ' process states updated'); + + await client.query('COMMIT'); + } catch (e) { + await client.query('ROLLBACK'); + throw e; + } finally { + client.release(); } - - console.log('process states updated'); - await client.query('COMMIT'); - } catch (e) { - await client.query('ROLLBACK'); - throw e; - } finally { - client.release(); } } + async function processMangoTransactions( address, rawTransactionsPool, @@ -109,13 +112,15 @@ async function processMangoTransactions( res = await client.query( 'select transaction, signature from ' + schema + - ".transactions where process_state = 'ready for parsing' and program_pk = $1 order by id asc limit $2", + ".transactions where process_state = 'ready for parsing' and program_pk = $1 limit $2", [address, limit], ); } finally { client.release(); } + console.log('Fetched ' + res.rows.length + ' records to parse.') + let transactions = res.rows.map((e) => [e.transaction, e.signature]); let [processStates, parsedTransactions] = parseTransactions( transactions, diff --git a/src/parseTransactions.ts b/src/parseTransactions.ts index 9c34f55..ffeb363 100644 --- a/src/parseTransactions.ts +++ b/src/parseTransactions.ts @@ -84,7 +84,7 @@ export function parseTransactions(transactionsResult, mangoProgramId) { net_balances: [], redeem_mngo: [], - update_funding: [], + funding: [], // @clarkeni: new log added OpenOrdersBalanceLog // @clarkeni: new log added MngoAccrual // Mango: PlacePerpOrder @@ -136,16 +136,50 @@ export function parseTransactions(transactionsResult, mangoProgramId) { } // Go through all instructions and get the update funding stuff first - extractUpdateFundings( + parsedTransactions.funding.push( + ...extractUpdateFundings( result, instructions, - parsedTransactions, signature, blockTime, slot, blockDatetime, + ) ); + parsedTransactions.net_balances.push( + ...extractNetBalances( + result.transaction.message.accountKeys, + result.meta.logMessages, + signature, + blockTime, + slot, + blockDatetime, + ) + ); + + parsedTransactions.settle_fees.push( + ...extractSettleFees( + result.transaction.message.accountKeys, + result.meta.logMessages, + signature, + blockTime, + slot, + blockDatetime, + ) + ); + + parsedTransactions.settle_pnl.push( + ...extractSettlePnl( + result.transaction.message.accountKeys, + result.meta.logMessages, + signature, + blockTime, + slot, + blockDatetime, + ) + ); + // Can have multiple inserts per signature so add instructionNum column to allow a primary key for (let instruction of instructions) { const instructionName = instruction.instructionName; @@ -252,31 +286,9 @@ export function parseTransactions(transactionsResult, mangoProgramId) { blockDatetime, ), ); - } else if (instructionName === 'SettlePnl') { - parsedTransactions.settle_pnl.push( - ...parseSettlePnl( - instructionNum, - result.meta.logMessages, - instruction.accounts, - signature, - blockTime, - slot, - blockDatetime, - ), - ); - } else if (instructionName === 'SettleFees') { - parsedTransactions.settle_fees.push( - ...parseSettleFees( - instructionNum, - result.meta.logMessages, - instruction.accounts, - signature, - blockTime, - slot, - blockDatetime, - ), - ); } else if (instructionName === 'ConsumeEvents') { + // if (instructionName === 'ConsumeEvents') { + parsedTransactions.fill_events.push( ...parseConsumeEvents( result.meta.logMessages, @@ -287,6 +299,7 @@ export function parseTransactions(transactionsResult, mangoProgramId) { blockDatetime, ), ); + // } } else if (instructionName === 'RedeemMngo') { parsedTransactions.redeem_mngo.push( ...parseRedeemMngo( @@ -305,48 +318,6 @@ export function parseTransactions(transactionsResult, mangoProgramId) { } } - // Need to know the mango group pk of the transaction for information not tied to instructions - // Transaction will only have one mango group - so check which one it is by iterating over mango group pks in IDS - // TODO: add mango group to appropriate log messages and remove this workaround - let ids = IDS; - console.log(); - let mangoGroupPk; - let accountKeys = result.transaction.message.accountKeys.map( - (e) => e.pubkey, - ); - for (let pk of ids.groups.map((e) => e.publicKey)) { - if (accountKeys.includes(pk)) { - mangoGroupPk = pk; - break; - } - } - - let allNetBalances: any = []; - // Some information is not tied to instructions specifically - for (let logMessage of result.meta.logMessages) { - if ( - logMessage.startsWith('Program log: checked_sub_net details: ') || - logMessage.startsWith('Program log: checked_add_net details: ') - ) { - let parsedNetAmounts = parseNetAmounts( - logMessage, - mangoGroupPk, - signature, - blockTime, - slot, - blockDatetime, - ); - allNetBalances.push(...parsedNetAmounts); - } - } - // Only want to store the latest deposit/borrow amounts per marginAccount/symbol pair for each instruction - parsedTransactions.net_balances.push( - ...getLatestObjPerCombination(allNetBalances, [ - 'mango_account', - 'symbol', - ]), - ); - processStates.push({ signature: signature, process_state: 'processed', @@ -364,6 +335,189 @@ export function parseTransactions(transactionsResult, mangoProgramId) { return [processStates, parsedTransactions]; } +function extractSettleFees( + allAccounts, + logMessages, + signature, + blockTime, + slot, + blockDatetime, +) { + + // Need to know the mango group pk of the transaction for information not tied to instructions + // Transaction will only have one mango group - so check which one it is by iterating over mango group pks in IDS + // TODO: add mango group to appropriate log messages and remove this workaround + let ids = IDS; + console.log(); + let mangoGroupPk; + let accountKeys = allAccounts.map( + (e) => e.pubkey, + ); + for (let pk of ids.groups.map((e) => e.publicKey)) { + if (accountKeys.includes(pk)) { + mangoGroupPk = pk; + break; + } + } + + let startDetailsStr = 'Program log: settle_fees details: ' + const filteredLogs = logMessages.filter((line) => + line.startsWith(startDetailsStr), + ); + + let instructionNum = 1; + let out: any = [] + for (let log of filteredLogs) { + let perpMarkets = ids['groups'].find((e) => e['publicKey'] === mangoGroupPk)[ + 'perpMarkets' + ]; + + // Log JSON is missing quotes around mango accounts + // Also trailing comma at end of json + // TODO: fix this + let settlePnlDetails; + try { + settlePnlDetails = JSON.parse(log.slice(startDetailsStr.length)); + } catch { + let jsonString = log.slice(startDetailsStr.length); + jsonString = insertQuotesAroundField(jsonString, 'mango_account'); + + jsonString = jsonString.replace(', }', ' }'); + + settlePnlDetails = JSON.parse(jsonString); + } + + let mangoAccount = settlePnlDetails['mango_account']; + + let marketIndex = settlePnlDetails['market_index']; + let perpMarket = perpMarkets.find( + (e) => e['marketIndex'] === marketIndex, + ); + let perpMarketName = perpMarket.name; + + let settlement = + settlePnlDetails['settlement'] / Math.pow(10, perpMarket.quoteDecimals); + + out.push(...[ + { + margin_account: mangoAccount, + settlement: settlement, + perp_market_name: perpMarketName, + instruction_num: instructionNum, + mango_group: mangoGroupPk, + block_datetime: blockDatetime, + slot: slot, + signature: signature, + blocktime: blockTime, + }, + ]) + + instructionNum++; + } + + return out +} + +function extractSettlePnl( + allAccounts, + logMessages, + signature, + blockTime, + slot, + blockDatetime, +) { + + // Need to know the mango group pk of the transaction for information not tied to instructions + // Transaction will only have one mango group - so check which one it is by iterating over mango group pks in IDS + // TODO: add mango group to appropriate log messages and remove this workaround + let ids = IDS; + console.log(); + let mangoGroupPk; + let accountKeys = allAccounts.map( + (e) => e.pubkey, + ); + for (let pk of ids.groups.map((e) => e.publicKey)) { + if (accountKeys.includes(pk)) { + mangoGroupPk = pk; + break; + } + } + + let startDetailsStr = 'Program log: settle_pnl details: ' + const filteredLogs = logMessages.filter((line) => + line.startsWith(startDetailsStr), + ); + + let instructionNum = 1; + let out: any = [] + for (let log of filteredLogs) { + let perpMarkets = ids['groups'].find((e) => e['publicKey'] === mangoGroupPk)[ + 'perpMarkets' + ]; + + // Log JSON is missing quotes around mango accounts + // TODO: fix this + let settlePnlDetails; + try { + settlePnlDetails = JSON.parse(log.slice(startDetailsStr.length)); + } catch { + let jsonString = log.slice(startDetailsStr.length); + jsonString = insertQuotesAroundField(jsonString, 'mango_account_a'); + jsonString = insertQuotesAroundField(jsonString, 'mango_account_b'); + + settlePnlDetails = JSON.parse(jsonString); + } + + let mangoAccountA = settlePnlDetails['mango_account_a']; + let mangoAccountB = settlePnlDetails['mango_account_b']; + + let marketIndex = settlePnlDetails['market_index']; + let perpMarket = perpMarkets.find( + (e) => e['marketIndex'] === marketIndex, + ); + let perpMarketName = perpMarket.name; + + let settlement = settlePnlDetails['settlement']; + + // A's quote position is reduced by settlement and B's quote position is increased by settlement + let settlementA = (-1 * settlement) / Math.pow(10, perpMarket.quoteDecimals); + let settlementB = settlement / Math.pow(10, perpMarket.quoteDecimals); + + out.push(...[ + { + margin_account: mangoAccountA, + settlement: settlementA, + perp_market_name: perpMarketName, + counterparty: mangoAccountB, + instruction_num: instructionNum, + + mango_group: mangoGroupPk, + block_datetime: blockDatetime, + slot: slot, + signature: signature, + blocktime: blockTime, + }, + { + margin_account: mangoAccountB, + settlement: settlementB, + perp_market_name: perpMarketName, + counterparty: mangoAccountA, + instruction_num: instructionNum, + + mango_group: mangoGroupPk, + block_datetime: blockDatetime, + slot: slot, + signature: signature, + blocktime: blockTime, + }, + ]); + + instructionNum++; + } + + return out +} + function getLatestObjPerCombination(arr, combinationFields) { // Utility function - iterates over arr and return the element with the highest index per set of combinationFields @@ -415,8 +569,19 @@ function parseConsumeEvents( ) { // instructionNum is used here to form a primary key on the db table (with signature) + let mangoGroupPk = accounts[0]; + let perpMarkets = ids['groups'].find((e) => e['publicKey'] === mangoGroupPk)[ + 'perpMarkets' + ]; + + let perpMarketPk = accounts[2]; + + let perpMarket = perpMarkets.find( + (e) => e['publicKey'] === perpMarketPk, + ); + let events: any = []; let startDetailsStr = 'Program log: FillEvent details: '; let eventNum = 1; @@ -458,6 +623,9 @@ function parseConsumeEvents( slot: slot, signature: signature, blocktime: blockTime, + + perp_market: perpMarket.name, + base_symbol: perpMarket.baseSymbol }); eventNum++; @@ -467,169 +635,6 @@ function parseConsumeEvents( return events; } -function parseSettleFees( - instructionNum, - logMessages, - accounts, - signature, - blockTime, - slot, - blockDatetime, -) { - let mangoGroupPk = accounts[0]; - - let perpMarkets = ids['groups'].find((e) => e['publicKey'] === mangoGroupPk)[ - 'perpMarkets' - ]; - - let mangoAccount; - let perpMarketName; - let settlement; - let startDetailsStr = 'Program log: settle_fees details: '; - // Sometimes SettleFees is called but nothing is settled - see mxK5eEiEUeCcQtHwmUKziyYUZJ3NXdjmZigAR6npWgcpQoJtqvikt5A7osD4y6oiLZJhzYFvqAqDznFjHm77K8V - let detailsFound = false; - for (let logMessage of logMessages) { - if (logMessage.startsWith(startDetailsStr)) { - detailsFound = true; - - // Log JSON is missing quotes around mango accounts - // Also trailing comma at end of json - // TODO: fix this - let settlePnlDetails; - try { - settlePnlDetails = JSON.parse(logMessage.slice(startDetailsStr.length)); - } catch { - let jsonString = logMessage.slice(startDetailsStr.length); - jsonString = insertQuotesAroundField(jsonString, 'mango_account'); - - jsonString = jsonString.replace(', }', ' }'); - - settlePnlDetails = JSON.parse(jsonString); - } - - mangoAccount = settlePnlDetails['mango_account']; - - let marketIndex = settlePnlDetails['market_index']; - let perpMarket = perpMarkets.find( - (e) => e['marketIndex'] === marketIndex, - ); - perpMarketName = perpMarket.name; - - settlement = - settlePnlDetails['settlement'] / Math.pow(10, perpMarket.quoteDecimals); - } - } - - if (detailsFound) { - return [ - { - margin_account: mangoAccount, - settlement: settlement, - perp_market_name: perpMarketName, - instruction_num: instructionNum, - mango_group: mangoGroupPk, - block_datetime: blockDatetime, - slot: slot, - signature: signature, - blocktime: blockTime, - }, - ]; - } else { - return []; - } -} - -function parseSettlePnl( - instructionNum, - logMessages, - accounts, - signature, - blockTime, - slot, - blockDatetime, -) { - let mangoGroupPk = accounts[0]; - - let perpMarkets = ids['groups'].find((e) => e['publicKey'] === mangoGroupPk)[ - 'perpMarkets' - ]; - - let mangoAccountA; - let mangoAccountB; - let perpMarketName; - let settlementA; - let settlementB; - let startDetailsStr = 'Program log: settle_pnl details: '; - // Sometimes SettlePnl is called but nothing is settled - see 5fWGMQECxDgvvffBuzVCsig7WcREP7bqcFN7Y5ndAC5tdxhFHJ8oSQkHcLzuPoVmMMs3o1V8pr7T7sAHTnRzMoan - let detailsFound = false; - for (let logMessage of logMessages) { - if (logMessage.startsWith(startDetailsStr)) { - detailsFound = true; - - // Log JSON is missing quotes around mango accounts - // TODO: fix this - let settlePnlDetails; - try { - settlePnlDetails = JSON.parse(logMessage.slice(startDetailsStr.length)); - } catch { - let jsonString = logMessage.slice(startDetailsStr.length); - jsonString = insertQuotesAroundField(jsonString, 'mango_account_a'); - jsonString = insertQuotesAroundField(jsonString, 'mango_account_b'); - - settlePnlDetails = JSON.parse(jsonString); - } - - mangoAccountA = settlePnlDetails['mango_account_a']; - mangoAccountB = settlePnlDetails['mango_account_b']; - - let marketIndex = settlePnlDetails['market_index']; - let perpMarket = perpMarkets.find( - (e) => e['marketIndex'] === marketIndex, - ); - perpMarketName = perpMarket.name; - - let settlement = settlePnlDetails['settlement']; - - // A's quote position is reduced by settlement and B's quote position is increased by settlement - settlementA = (-1 * settlement) / Math.pow(10, perpMarket.quoteDecimals); - settlementB = settlement / Math.pow(10, perpMarket.quoteDecimals); - } - } - - if (detailsFound) { - return [ - { - margin_account: mangoAccountA, - settlement: settlementA, - perp_market_name: perpMarketName, - counterparty: mangoAccountB, - instruction_num: instructionNum, - - mango_group: mangoGroupPk, - block_datetime: blockDatetime, - slot: slot, - signature: signature, - blocktime: blockTime, - }, - { - margin_account: mangoAccountB, - settlement: settlementB, - perp_market_name: perpMarketName, - counterparty: mangoAccountA, - instruction_num: instructionNum, - - mango_group: mangoGroupPk, - block_datetime: blockDatetime, - slot: slot, - signature: signature, - blocktime: blockTime, - }, - ]; - } else { - return []; - } -} - function parseResolvePerpBankruptcy( logMessages, accounts, @@ -1359,7 +1364,54 @@ function parseCachePrices( return cachePrices; } -// *** @clarkeni take a look +function extractNetBalances( + allAccounts, + logMessages, + signature, + blockTime, + slot, + blockDatetime, +) { + + // Need to know the mango group pk of the transaction for information not tied to instructions + // Transaction will only have one mango group - so check which one it is by iterating over mango group pks in IDS + // TODO: add mango group to appropriate log messages and remove this workaround + let ids = IDS; + console.log(); + let mangoGroupPk; + let accountKeys = allAccounts.map( + (e) => e.pubkey, + ); + for (let pk of ids.groups.map((e) => e.publicKey)) { + if (accountKeys.includes(pk)) { + mangoGroupPk = pk; + break; + } + } + + let allNetBalances: any = []; + // Some information is not tied to instructions specifically + for (let logMessage of logMessages) { + if ( + logMessage.startsWith('Program log: checked_sub_net details: ') || + logMessage.startsWith('Program log: checked_add_net details: ') + ) { + let parsedNetAmounts = parseNetAmounts( + logMessage, + mangoGroupPk, + signature, + blockTime, + slot, + blockDatetime, + ); + allNetBalances.push(...parsedNetAmounts); + } + } + + // Only want to store the latest deposit/borrow amounts per marginAccount/symbol pair for each instruction + return getLatestObjPerCombination(allNetBalances, ['mango_account','symbol']) +} + function parseUpdateFunding( instructionNum: number, logMessage: string, @@ -1402,7 +1454,6 @@ function parseUpdateFunding( function extractUpdateFundings( result, instructions, - parsedTransactions, signature, blockTime, slot, @@ -1419,10 +1470,11 @@ function extractUpdateFundings( throw new Error("funding logs don't match length of funding instructions"); } + let out: any = [] for (let i = 0; i < fundingLogs.length; i++) { const ix = fundingInstructions[i]; const logMessage = fundingLogs[i]; - parsedTransactions.update_funding.push( + out.push( ...parseUpdateFunding( ix.instructionNum, logMessage, @@ -1434,4 +1486,6 @@ function extractUpdateFundings( ), ); } + + return out } diff --git a/src/tests/testParseTransactions.ts b/src/tests/testParseTransactions.ts index 5ee2066..d1d74a2 100644 --- a/src/tests/testParseTransactions.ts +++ b/src/tests/testParseTransactions.ts @@ -24,6 +24,7 @@ async function processMangoTransactions(rawTransactionsPool, schema, limit) { '4Xx7gVesMQQZqprJYLu5gNEYRLA5GTXKURrkc8jG3CLKynRwhEM93MownyAMhxpdFTvfXQ9kkxkRcemZ8Fn5WHyk', // ResolvePerpBankruptcy '3bzj3KkA3FSZHJuCmRgHhSgqeaEzD32sCHkYdRLcZm1vcuB4ra5NbU5EZqBhW6QjeKRV9QRWC4SHxK2hS54s79Zx', // settle_pnl '5TmhvKQJmjUD9dZgCszBF8gNKUohpxwjrYu1RngZVh1hEToGMtjPtXJF89QLHXzANMWWQRfMomsgCg8353CpYgBb', // settle_fees + 'mxK5eEiEUeCcQtHwmUKziyYUZJ3NXdjmZigAR6npWgcpQoJtqvikt5A7osD4y6oiLZJhzYFvqAqDznFjHm77K8V', // settle_fees called but nothing settled '4qV6PTD1nGj5qq89FQK8QKwN231pGgtayD7uX4B6y83b19gcVXB5ByLCvApSJjCRrboiCg7RVT2p2e1CtP3zuXDb', // force_settle_quote_positions '5qDPBrFjCcaZthjRCqisHRw1mFEkHFHRFWi5jbKCmpAgpAXNdEkSv8L472D12VB5AukYaGsWhAy5bcvvUGJ1Sgtv', // FillEvent '3YXaEG95w5eG7jBBjz8hW9auXVAv9z2MH8yw51tL8nqSqmKgXtrD1hgE7LCqK2hpFwcrpjeWtBeVqGsbCHLh3kSe', // redeem mango