blockworks-foundation
/
mango-transaction-scraper-v3
mirror of https://github.com/blockworks-foundation/mango-transaction-scraper-v3.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Parsing fixes for settle pnl and settle fees

This commit is contained in:
Nicholas Clarke 2021-10-17 14:43:15 -07:00
parent 0928b4e5de
commit cfbe9923ba
3 changed files with 314 additions and 254 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
src/index.ts
View File

@ -56,7 +56,7 @@ async function insertMangoTransactions(
);
await client.query(insertsSql);
}
console.log(tableName + ' inserted');
console.log(inserts.length + ' records inserted into ' + tableName);
}
}
await client.query('COMMIT');
@ -74,28 +74,31 @@ async function insertMangoTransactions(
{ table: table },
);
client = await rawTransactionsPool.connect();
try {
await client.query('BEGIN');
if (processStates.length > 0) {
client = await rawTransactionsPool.connect();
try {
await client.query('BEGIN');
for (let i = 0, j = processStates.length; i < j; i += batchSize) {
let updatesBatch = processStates.slice(i, i + batchSize);
let updatedSql =
pgp.helpers.update(updatesBatch, processStateCs) +
' WHERE v.signature = t.signature';
await client.query(updatedSql);
for (let i = 0, j = processStates.length; i < j; i += batchSize) {
let updatesBatch = processStates.slice(i, i + batchSize);
let updatedSql =
pgp.helpers.update(updatesBatch, processStateCs) +
' WHERE v.signature = t.signature';
await client.query(updatedSql);
}
console.log(processStates.length + ' process states updated');
await client.query('COMMIT');
} catch (e) {
await client.query('ROLLBACK');
throw e;
} finally {
client.release();
}
console.log('process states updated');
await client.query('COMMIT');
} catch (e) {
await client.query('ROLLBACK');
throw e;
} finally {
client.release();
}
}
async function processMangoTransactions(
address,
rawTransactionsPool,
@ -109,13 +112,15 @@ async function processMangoTransactions(
res = await client.query(
'select transaction, signature from ' +
schema +
".transactions where process_state = 'ready for parsing' and program_pk = $1 order by id asc limit $2",
".transactions where process_state = 'ready for parsing' and program_pk = $1 limit $2",
[address, limit],
);
} finally {
client.release();
}
console.log('Fetched ' + res.rows.length + ' records to parse.')
let transactions = res.rows.map((e) => [e.transaction, e.signature]);
let [processStates, parsedTransactions] = parseTransactions(
transactions,

524
src/parseTransactions.ts
View File

@ -84,7 +84,7 @@ export function parseTransactions(transactionsResult, mangoProgramId) {
net_balances: [],
redeem_mngo: [],
update_funding: [],
funding: [],
// @clarkeni: new log added OpenOrdersBalanceLog
// @clarkeni: new log added MngoAccrual
// Mango: PlacePerpOrder
@ -136,16 +136,50 @@ export function parseTransactions(transactionsResult, mangoProgramId) {
}
// Go through all instructions and get the update funding stuff first
extractUpdateFundings(
parsedTransactions.funding.push(
...extractUpdateFundings(
result,
instructions,
parsedTransactions,
signature,
blockTime,
slot,
blockDatetime,
)
);
parsedTransactions.net_balances.push(
...extractNetBalances(
result.transaction.message.accountKeys,
result.meta.logMessages,
signature,
blockTime,
slot,
blockDatetime,
)
);
parsedTransactions.settle_fees.push(
...extractSettleFees(
result.transaction.message.accountKeys,
result.meta.logMessages,
signature,
blockTime,
slot,
blockDatetime,
)
);
parsedTransactions.settle_pnl.push(
...extractSettlePnl(
result.transaction.message.accountKeys,
result.meta.logMessages,
signature,
blockTime,
slot,
blockDatetime,
)
);
// Can have multiple inserts per signature so add instructionNum column to allow a primary key
for (let instruction of instructions) {
const instructionName = instruction.instructionName;
@ -252,31 +286,9 @@ export function parseTransactions(transactionsResult, mangoProgramId) {
blockDatetime,
),
);
} else if (instructionName === 'SettlePnl') {
parsedTransactions.settle_pnl.push(
...parseSettlePnl(
instructionNum,
result.meta.logMessages,
instruction.accounts,
signature,
blockTime,
slot,
blockDatetime,
),
);
} else if (instructionName === 'SettleFees') {
parsedTransactions.settle_fees.push(
...parseSettleFees(
instructionNum,
result.meta.logMessages,
instruction.accounts,
signature,
blockTime,
slot,
blockDatetime,
),
);
} else if (instructionName === 'ConsumeEvents') {
// if (instructionName === 'ConsumeEvents') {
parsedTransactions.fill_events.push(
...parseConsumeEvents(
result.meta.logMessages,
@ -287,6 +299,7 @@ export function parseTransactions(transactionsResult, mangoProgramId) {
blockDatetime,
),
);
// }
} else if (instructionName === 'RedeemMngo') {
parsedTransactions.redeem_mngo.push(
...parseRedeemMngo(
@ -305,48 +318,6 @@ export function parseTransactions(transactionsResult, mangoProgramId) {
}
}
// Need to know the mango group pk of the transaction for information not tied to instructions
// Transaction will only have one mango group - so check which one it is by iterating over mango group pks in IDS
// TODO: add mango group to appropriate log messages and remove this workaround
let ids = IDS;
console.log();
let mangoGroupPk;
let accountKeys = result.transaction.message.accountKeys.map(
(e) => e.pubkey,
);
for (let pk of ids.groups.map((e) => e.publicKey)) {
if (accountKeys.includes(pk)) {
mangoGroupPk = pk;
break;
}
}
let allNetBalances: any = [];
// Some information is not tied to instructions specifically
for (let logMessage of result.meta.logMessages) {
if (
logMessage.startsWith('Program log: checked_sub_net details: ') ||
logMessage.startsWith('Program log: checked_add_net details: ')
) {
let parsedNetAmounts = parseNetAmounts(
logMessage,
mangoGroupPk,
signature,
blockTime,
slot,
blockDatetime,
);
allNetBalances.push(...parsedNetAmounts);
}
}
// Only want to store the latest deposit/borrow amounts per marginAccount/symbol pair for each instruction
parsedTransactions.net_balances.push(
...getLatestObjPerCombination(allNetBalances, [
'mango_account',
'symbol',
]),
);
processStates.push({
signature: signature,
process_state: 'processed',
@ -364,6 +335,189 @@ export function parseTransactions(transactionsResult, mangoProgramId) {
return [processStates, parsedTransactions];
}
function extractSettleFees(
allAccounts,
logMessages,
signature,
blockTime,
slot,
blockDatetime,
) {
// Need to know the mango group pk of the transaction for information not tied to instructions
// Transaction will only have one mango group - so check which one it is by iterating over mango group pks in IDS
// TODO: add mango group to appropriate log messages and remove this workaround
let ids = IDS;
console.log();
let mangoGroupPk;
let accountKeys = allAccounts.map(
(e) => e.pubkey,
);
for (let pk of ids.groups.map((e) => e.publicKey)) {
if (accountKeys.includes(pk)) {
mangoGroupPk = pk;
break;
}
}
let startDetailsStr = 'Program log: settle_fees details: '
const filteredLogs = logMessages.filter((line) =>
line.startsWith(startDetailsStr),
);
let instructionNum = 1;
let out: any = []
for (let log of filteredLogs) {
let perpMarkets = ids['groups'].find((e) => e['publicKey'] === mangoGroupPk)[
'perpMarkets'
];
// Log JSON is missing quotes around mango accounts
// Also trailing comma at end of json
// TODO: fix this
let settlePnlDetails;
try {
settlePnlDetails = JSON.parse(log.slice(startDetailsStr.length));
} catch {
let jsonString = log.slice(startDetailsStr.length);
jsonString = insertQuotesAroundField(jsonString, 'mango_account');
jsonString = jsonString.replace(', }', ' }');
settlePnlDetails = JSON.parse(jsonString);
}
let mangoAccount = settlePnlDetails['mango_account'];
let marketIndex = settlePnlDetails['market_index'];
let perpMarket = perpMarkets.find(
(e) => e['marketIndex'] === marketIndex,
);
let perpMarketName = perpMarket.name;
let settlement =
settlePnlDetails['settlement'] / Math.pow(10, perpMarket.quoteDecimals);
out.push(...[
{
margin_account: mangoAccount,
settlement: settlement,
perp_market_name: perpMarketName,
instruction_num: instructionNum,
mango_group: mangoGroupPk,
block_datetime: blockDatetime,
slot: slot,
signature: signature,
blocktime: blockTime,
},
])
instructionNum++;
}
return out
}
function extractSettlePnl(
allAccounts,
logMessages,
signature,
blockTime,
slot,
blockDatetime,
) {
// Need to know the mango group pk of the transaction for information not tied to instructions
// Transaction will only have one mango group - so check which one it is by iterating over mango group pks in IDS
// TODO: add mango group to appropriate log messages and remove this workaround
let ids = IDS;
console.log();
let mangoGroupPk;
let accountKeys = allAccounts.map(
(e) => e.pubkey,
);
for (let pk of ids.groups.map((e) => e.publicKey)) {
if (accountKeys.includes(pk)) {
mangoGroupPk = pk;
break;
}
}
let startDetailsStr = 'Program log: settle_pnl details: '
const filteredLogs = logMessages.filter((line) =>
line.startsWith(startDetailsStr),
);
let instructionNum = 1;
let out: any = []
for (let log of filteredLogs) {
let perpMarkets = ids['groups'].find((e) => e['publicKey'] === mangoGroupPk)[
'perpMarkets'
];
// Log JSON is missing quotes around mango accounts
// TODO: fix this
let settlePnlDetails;
try {
settlePnlDetails = JSON.parse(log.slice(startDetailsStr.length));
} catch {
let jsonString = log.slice(startDetailsStr.length);
jsonString = insertQuotesAroundField(jsonString, 'mango_account_a');
jsonString = insertQuotesAroundField(jsonString, 'mango_account_b');
settlePnlDetails = JSON.parse(jsonString);
}
let mangoAccountA = settlePnlDetails['mango_account_a'];
let mangoAccountB = settlePnlDetails['mango_account_b'];
let marketIndex = settlePnlDetails['market_index'];
let perpMarket = perpMarkets.find(
(e) => e['marketIndex'] === marketIndex,
);
let perpMarketName = perpMarket.name;
let settlement = settlePnlDetails['settlement'];
// A's quote position is reduced by settlement and B's quote position is increased by settlement
let settlementA = (-1 * settlement) / Math.pow(10, perpMarket.quoteDecimals);
let settlementB = settlement / Math.pow(10, perpMarket.quoteDecimals);
out.push(...[
{
margin_account: mangoAccountA,
settlement: settlementA,
perp_market_name: perpMarketName,
counterparty: mangoAccountB,
instruction_num: instructionNum,
mango_group: mangoGroupPk,
block_datetime: blockDatetime,
slot: slot,
signature: signature,
blocktime: blockTime,
},
{
margin_account: mangoAccountB,
settlement: settlementB,
perp_market_name: perpMarketName,
counterparty: mangoAccountA,
instruction_num: instructionNum,
mango_group: mangoGroupPk,
block_datetime: blockDatetime,
slot: slot,
signature: signature,
blocktime: blockTime,
},
]);
instructionNum++;
}
return out
}
function getLatestObjPerCombination(arr, combinationFields) {
// Utility function - iterates over arr and return the element with the highest index per set of combinationFields
@ -415,8 +569,19 @@ function parseConsumeEvents(
) {
// instructionNum is used here to form a primary key on the db table (with signature)
let mangoGroupPk = accounts[0];
let perpMarkets = ids['groups'].find((e) => e['publicKey'] === mangoGroupPk)[
'perpMarkets'
];
let perpMarketPk = accounts[2];
let perpMarket = perpMarkets.find(
(e) => e['publicKey'] === perpMarketPk,
);
let events: any = [];
let startDetailsStr = 'Program log: FillEvent details: ';
let eventNum = 1;
@ -458,6 +623,9 @@ function parseConsumeEvents(
slot: slot,
signature: signature,
blocktime: blockTime,
perp_market: perpMarket.name,
base_symbol: perpMarket.baseSymbol
});
eventNum++;
@ -467,169 +635,6 @@ function parseConsumeEvents(
return events;
}
function parseSettleFees(
instructionNum,
logMessages,
accounts,
signature,
blockTime,
slot,
blockDatetime,
) {
let mangoGroupPk = accounts[0];
let perpMarkets = ids['groups'].find((e) => e['publicKey'] === mangoGroupPk)[
'perpMarkets'
];
let mangoAccount;
let perpMarketName;
let settlement;
let startDetailsStr = 'Program log: settle_fees details: ';
// Sometimes SettleFees is called but nothing is settled - see mxK5eEiEUeCcQtHwmUKziyYUZJ3NXdjmZigAR6npWgcpQoJtqvikt5A7osD4y6oiLZJhzYFvqAqDznFjHm77K8V
let detailsFound = false;
for (let logMessage of logMessages) {
if (logMessage.startsWith(startDetailsStr)) {
detailsFound = true;
// Log JSON is missing quotes around mango accounts
// Also trailing comma at end of json
// TODO: fix this
let settlePnlDetails;
try {
settlePnlDetails = JSON.parse(logMessage.slice(startDetailsStr.length));
} catch {
let jsonString = logMessage.slice(startDetailsStr.length);
jsonString = insertQuotesAroundField(jsonString, 'mango_account');
jsonString = jsonString.replace(', }', ' }');
settlePnlDetails = JSON.parse(jsonString);
}
mangoAccount = settlePnlDetails['mango_account'];
let marketIndex = settlePnlDetails['market_index'];
let perpMarket = perpMarkets.find(
(e) => e['marketIndex'] === marketIndex,
);
perpMarketName = perpMarket.name;
settlement =
settlePnlDetails['settlement'] / Math.pow(10, perpMarket.quoteDecimals);
}
}
if (detailsFound) {
return [
{
margin_account: mangoAccount,
settlement: settlement,
perp_market_name: perpMarketName,
instruction_num: instructionNum,
mango_group: mangoGroupPk,
block_datetime: blockDatetime,
slot: slot,
signature: signature,
blocktime: blockTime,
},
];
} else {
return [];
}
}
function parseSettlePnl(
instructionNum,
logMessages,
accounts,
signature,
blockTime,
slot,
blockDatetime,
) {
let mangoGroupPk = accounts[0];
let perpMarkets = ids['groups'].find((e) => e['publicKey'] === mangoGroupPk)[
'perpMarkets'
];
let mangoAccountA;
let mangoAccountB;
let perpMarketName;
let settlementA;
let settlementB;
let startDetailsStr = 'Program log: settle_pnl details: ';
// Sometimes SettlePnl is called but nothing is settled - see 5fWGMQECxDgvvffBuzVCsig7WcREP7bqcFN7Y5ndAC5tdxhFHJ8oSQkHcLzuPoVmMMs3o1V8pr7T7sAHTnRzMoan
let detailsFound = false;
for (let logMessage of logMessages) {
if (logMessage.startsWith(startDetailsStr)) {
detailsFound = true;
// Log JSON is missing quotes around mango accounts
// TODO: fix this
let settlePnlDetails;
try {
settlePnlDetails = JSON.parse(logMessage.slice(startDetailsStr.length));
} catch {
let jsonString = logMessage.slice(startDetailsStr.length);
jsonString = insertQuotesAroundField(jsonString, 'mango_account_a');
jsonString = insertQuotesAroundField(jsonString, 'mango_account_b');
settlePnlDetails = JSON.parse(jsonString);
}
mangoAccountA = settlePnlDetails['mango_account_a'];
mangoAccountB = settlePnlDetails['mango_account_b'];
let marketIndex = settlePnlDetails['market_index'];
let perpMarket = perpMarkets.find(
(e) => e['marketIndex'] === marketIndex,
);
perpMarketName = perpMarket.name;
let settlement = settlePnlDetails['settlement'];
// A's quote position is reduced by settlement and B's quote position is increased by settlement
settlementA = (-1 * settlement) / Math.pow(10, perpMarket.quoteDecimals);
settlementB = settlement / Math.pow(10, perpMarket.quoteDecimals);
}
}
if (detailsFound) {
return [
{
margin_account: mangoAccountA,
settlement: settlementA,
perp_market_name: perpMarketName,
counterparty: mangoAccountB,
instruction_num: instructionNum,
mango_group: mangoGroupPk,
block_datetime: blockDatetime,
slot: slot,
signature: signature,
blocktime: blockTime,
},
{
margin_account: mangoAccountB,
settlement: settlementB,
perp_market_name: perpMarketName,
counterparty: mangoAccountA,
instruction_num: instructionNum,
mango_group: mangoGroupPk,
block_datetime: blockDatetime,
slot: slot,
signature: signature,
blocktime: blockTime,
},
];
} else {
return [];
}
}
function parseResolvePerpBankruptcy(
logMessages,
accounts,
@ -1359,7 +1364,54 @@ function parseCachePrices(
return cachePrices;
}
// *** @clarkeni take a look
function extractNetBalances(
allAccounts,
logMessages,
signature,
blockTime,
slot,
blockDatetime,
) {
// Need to know the mango group pk of the transaction for information not tied to instructions
// Transaction will only have one mango group - so check which one it is by iterating over mango group pks in IDS
// TODO: add mango group to appropriate log messages and remove this workaround
let ids = IDS;
console.log();
let mangoGroupPk;
let accountKeys = allAccounts.map(
(e) => e.pubkey,
);
for (let pk of ids.groups.map((e) => e.publicKey)) {
if (accountKeys.includes(pk)) {
mangoGroupPk = pk;
break;
}
}
let allNetBalances: any = [];
// Some information is not tied to instructions specifically
for (let logMessage of logMessages) {
if (
logMessage.startsWith('Program log: checked_sub_net details: ') ||
logMessage.startsWith('Program log: checked_add_net details: ')
) {
let parsedNetAmounts = parseNetAmounts(
logMessage,
mangoGroupPk,
signature,
blockTime,
slot,
blockDatetime,
);
allNetBalances.push(...parsedNetAmounts);
}
}
// Only want to store the latest deposit/borrow amounts per marginAccount/symbol pair for each instruction
return getLatestObjPerCombination(allNetBalances, ['mango_account','symbol'])
}
function parseUpdateFunding(
instructionNum: number,
logMessage: string,
@ -1402,7 +1454,6 @@ function parseUpdateFunding(
function extractUpdateFundings(
result,
instructions,
parsedTransactions,
signature,
blockTime,
slot,
@ -1419,10 +1470,11 @@ function extractUpdateFundings(
throw new Error("funding logs don't match length of funding instructions");
}
let out: any = []
for (let i = 0; i < fundingLogs.length; i++) {
const ix = fundingInstructions[i];
const logMessage = fundingLogs[i];
parsedTransactions.update_funding.push(
out.push(
...parseUpdateFunding(
ix.instructionNum,
logMessage,
@ -1434,4 +1486,6 @@ function extractUpdateFundings(
),
);
}
return out
}

1
src/tests/testParseTransactions.ts
View File

@ -24,6 +24,7 @@ async function processMangoTransactions(rawTransactionsPool, schema, limit) {
'4Xx7gVesMQQZqprJYLu5gNEYRLA5GTXKURrkc8jG3CLKynRwhEM93MownyAMhxpdFTvfXQ9kkxkRcemZ8Fn5WHyk', // ResolvePerpBankruptcy
'3bzj3KkA3FSZHJuCmRgHhSgqeaEzD32sCHkYdRLcZm1vcuB4ra5NbU5EZqBhW6QjeKRV9QRWC4SHxK2hS54s79Zx', // settle_pnl
'5TmhvKQJmjUD9dZgCszBF8gNKUohpxwjrYu1RngZVh1hEToGMtjPtXJF89QLHXzANMWWQRfMomsgCg8353CpYgBb', // settle_fees
'mxK5eEiEUeCcQtHwmUKziyYUZJ3NXdjmZigAR6npWgcpQoJtqvikt5A7osD4y6oiLZJhzYFvqAqDznFjHm77K8V', // settle_fees called but nothing settled
'4qV6PTD1nGj5qq89FQK8QKwN231pGgtayD7uX4B6y83b19gcVXB5ByLCvApSJjCRrboiCg7RVT2p2e1CtP3zuXDb', // force_settle_quote_positions
'5qDPBrFjCcaZthjRCqisHRw1mFEkHFHRFWi5jbKCmpAgpAXNdEkSv8L472D12VB5AukYaGsWhAy5bcvvUGJ1Sgtv', // FillEvent
'3YXaEG95w5eG7jBBjz8hW9auXVAv9z2MH8yw51tL8nqSqmKgXtrD1hgE7LCqK2hpFwcrpjeWtBeVqGsbCHLh3kSe', // redeem mango