Fix testing code to work with recent refactor

This commit is contained in:
Nicholas Clarke 2021-12-16 16:37:08 -08:00
parent b7e3424275
commit 5d8cb0e52a
3 changed files with 83 additions and 85 deletions

View File

@ -4,22 +4,12 @@ const pgp = require('pg-promise')({
import { sleep } from '@blockworks-foundation/mango-client';
export async function getUnprocessedTransactions(
export async function getTransactions(
signaturesToProcess,
connection,
address,
requestWaitTime,
pool,
schema,
numTransactions
requestWaitTime
) {
let signaturesToProcess = await getUnprocessedSignatures(
pool,
address,
schema,
numTransactions,
);
let promises: Promise<void>[] = [];
let transactions: any[] = [];
let counter = 1;
@ -49,7 +39,7 @@ export async function getUnprocessedTransactions(
return transactions;
}
async function getUnprocessedSignatures(
export async function getUnprocessedSignatures(
pool,
programPk,
schema,

View File

@ -3,7 +3,7 @@ import { Cluster, sleep } from '@blockworks-foundation/mango-client';
import { insertNewSignatures } from './signatures';
import { Pool } from 'pg';
import { notify } from './utils';
import { getUnprocessedTransactions } from './getTransactions';
import { getTransactions, getUnprocessedSignatures } from './getTransactions';
import { parseTransactions } from './parseTransactions';
const pgp = require('pg-promise')({
@ -123,7 +123,18 @@ async function processMangoTransactions(
parsingBatchSize,
) {
let transactions = await getUnprocessedTransactions(clusterConnection, address, requestWaitTime, rawTransactionsPool, schema, parsingBatchSize)
let signaturesToProcess = await getUnprocessedSignatures(
rawTransactionsPool,
address,
schema,
parsingBatchSize
);
let transactions = await getTransactions(
signaturesToProcess,
clusterConnection,
requestWaitTime
);
let [transactionSummaries, parsedTransactions] = parseTransactions(
transactions,

View File

@ -1,70 +1,75 @@
import { Pool } from 'pg';
import { parseTransactions } from '../parseTransactions';
import { getTransactions } from '../getTransactions'
import { Connection} from '@solana/web3.js';
const mangoProgramId =
process.env.MANGO_PROGRAM_ID || 'mv3ekLzLbnVPNxjSKvqBpU3ZeZXPQdEC3bp5MDEBG68';
async function processMangoTransactions(rawTransactionsPool, schema, limit) {
const client = await rawTransactionsPool.connect();
let res;
try {
// Below are examples of different types of transactions
let signatures = [
'4cYaxHEMAycyjRBfD2Va1cnaNcKSCxsLaYF47EnpUH4aqCEsZR9c9BJMeMr9NiPR7grN6puBYqWL6FdeX3ZgHrjj', // Deposit
'49ajp59Gtpr5Q4kD4y2rMZrQSgkFNrferoC5nUq6mAZsveFf3e12DXtCR1GMFSyhR6Aypd6tJG7SKveNwGHGBDYQ', // Withdraw
async function processMangoTransactions(clusterConnection, requestWaitTime) {
'5E9Jqz3nPtTx5rHTHmWyvw5RGTodPp4jRBx158dpe24b4QtpyAEVcEPuwvL6cwQ5MAcKHDiukU45L5xJ8fLR9Y1J', // CachePrices and CacheRootBanks
'2o3hH59r6Pggg8oTTEjtXyJ9xHmx2z1pPXG4SWoPeogW95sjQoxps5fu4LbaTWfTyMvK1epsvz1nLiZ9CTPAgyD1', // LiquidateTokenAndToken
'1F47ZsgeLNpDsHCqnNRbG8hi6C2rEeyT8YME61g84HSTRHrheXX69Zya2Dz3fW14SC1y84cAbLVg7jDae38Vp3a', // LiquidateTokenAndPerp
'4RBxzncCHW8XSHR1mFJrxtUMitYyYiSc9jvHQ4CPA8mqCukgw3dQNoYpgm5GRozyxMJP1j4ew9gNPkzCnrnkmaud', // LiquidatePerpMarket
// Below are examples of different types of transactions
let signatures = [
// '4cYaxHEMAycyjRBfD2Va1cnaNcKSCxsLaYF47EnpUH4aqCEsZR9c9BJMeMr9NiPR7grN6puBYqWL6FdeX3ZgHrjj', // Deposit
// '49ajp59Gtpr5Q4kD4y2rMZrQSgkFNrferoC5nUq6mAZsveFf3e12DXtCR1GMFSyhR6Aypd6tJG7SKveNwGHGBDYQ', // Withdraw
'56y2iUGUyQ8BcqLpL5N1QwR3QQF37NtaWRnd5XjQ25BNWKfNxvpnhwwqtD7B88dAvJEykTHNCLvUKdxY4jhQ2uhQ', //LiquidateTokenAndPerp example 2
'4ErqDTV11imwDnoA6AAj3VMZC9FQvyq2GA4emfpiSxVBifAQE9EDwPnHxoTMxULHfyyCf8GBBzoczGSqYJMFEV5D', //LiquidateTokenAndPerp example 3
'59DFSbsN1DbnqUiMsStS2xQn4tBncG8w3gg8aikBvQZKnWhf1yRt87EU4WMvXtnnVe18zWaRDFVfnVYp3ASgF7tV', // ResolveTokenBankruptcy
'4Xx7gVesMQQZqprJYLu5gNEYRLA5GTXKURrkc8jG3CLKynRwhEM93MownyAMhxpdFTvfXQ9kkxkRcemZ8Fn5WHyk', // ResolvePerpBankruptcy
'3bzj3KkA3FSZHJuCmRgHhSgqeaEzD32sCHkYdRLcZm1vcuB4ra5NbU5EZqBhW6QjeKRV9QRWC4SHxK2hS54s79Zx', // settle_pnl
'5TmhvKQJmjUD9dZgCszBF8gNKUohpxwjrYu1RngZVh1hEToGMtjPtXJF89QLHXzANMWWQRfMomsgCg8353CpYgBb', // settle_fees
'mxK5eEiEUeCcQtHwmUKziyYUZJ3NXdjmZigAR6npWgcpQoJtqvikt5A7osD4y6oiLZJhzYFvqAqDznFjHm77K8V', // settle_fees called but nothing settled
// '4qV6PTD1nGj5qq89FQK8QKwN231pGgtayD7uX4B6y83b19gcVXB5ByLCvApSJjCRrboiCg7RVT2p2e1CtP3zuXDb', // force_settle_quote_positions
'5qDPBrFjCcaZthjRCqisHRw1mFEkHFHRFWi5jbKCmpAgpAXNdEkSv8L472D12VB5AukYaGsWhAy5bcvvUGJ1Sgtv', // FillEvent
'3YXaEG95w5eG7jBBjz8hW9auXVAv9z2MH8yw51tL8nqSqmKgXtrD1hgE7LCqK2hpFwcrpjeWtBeVqGsbCHLh3kSe', // redeem mango
'2HNnZmThkFUsG1pw9bNaJoeeGUZJun3hkcpwBJt3ZU9FKe3CY17wrJgk1BZ8txm13RJ512ThbZVZxaqsxNFn4xVs', // checked_add_net details
'4ebib6h5kQHpcpK4A4UpH7ThJVEtui2X7vVvTfCW8iuJgjHMocH7nymN3zVrrbwZL9HQYJY1tHdnGjo7ZSgrL7M6', // error example
'59DFSbsN1DbnqUiMsStS2xQn4tBncG8w3gg8aikBvQZKnWhf1yRt87EU4WMvXtnnVe18zWaRDFVfnVYp3ASgF7tV', //token socialized loss
'4RCvRY8BWPB6FixyfufYKojUdnE91DiqmFE2b8e4FCyuWCdT1ipSzPBaWUgaajKucFr1jsveiMvTft5iiWbctCFk', // settle_pnl_multiple
'KDv62AKFqyrvULdUUeShdYK9zCeUDX8yb6kuUadoa6dJKrqjDvKU3JQj9t8e4H1FTEaEMsyLnYAT4HbTr8ikSGq', // multiple update funding
// '5E9Jqz3nPtTx5rHTHmWyvw5RGTodPp4jRBx158dpe24b4QtpyAEVcEPuwvL6cwQ5MAcKHDiukU45L5xJ8fLR9Y1J', // CachePrices and CacheRootBanks
// '2o3hH59r6Pggg8oTTEjtXyJ9xHmx2z1pPXG4SWoPeogW95sjQoxps5fu4LbaTWfTyMvK1epsvz1nLiZ9CTPAgyD1', // LiquidateTokenAndToken
// '1F47ZsgeLNpDsHCqnNRbG8hi6C2rEeyT8YME61g84HSTRHrheXX69Zya2Dz3fW14SC1y84cAbLVg7jDae38Vp3a', // LiquidateTokenAndPerp
// '4RBxzncCHW8XSHR1mFJrxtUMitYyYiSc9jvHQ4CPA8mqCukgw3dQNoYpgm5GRozyxMJP1j4ew9gNPkzCnrnkmaud', // LiquidatePerpMarket
// 3.1 transactions:
"2nyoFpqeopBuFgjSXntSUbJUYtxxFXTEWrf2t1715Bm4jTrYKNDwBkZKBuDN6mWt1vDcCuyPq56RGKv9SxuLJWwe", // UpdateRootBank
"3CPWZM49NXPmpB5GKkrJHLknfSN595Gkd3MxK3AdFvFJTFHsA9ShjAqyWjfSkxYpBAc3ju19maaECzb18ey3qwK4", // CachePrices and CacheRootBanks
"bf5kGknwbfzoMyz8cFHQAoK6ajwfqeNdFeWgaSGcy19poQPBbxpeW5BiLQFLK4DUswEwUvg2TZEusyqazT4WS4Y", // UpdateFunding
"4JuHNZy2GcFJYFCYoTrekyCa2eBSq8cECVUvQp32LbFshAVFaVrvypwoKNjpfZ2rK9N7AG5PHsqqhHSWuGU3rAFk", // MngoAccrualLog
"5MuJyoTsTzMGhGKDvN21BaZrfjaw1SeRgfTxz67U7zm7aVokrMqqvMhHvRCANGbjhvAeNFT1Zmk7vZoGp32b4H7t", // OpenOrdersBalanceLog, TokenBalanceLog
"Ztxy2398k8zDPh1VFiErvAQAnQdMuUok7Z772yCST9oZpagmqjggRuzU8ScrBEpcX4X4BknC7gxvfX128BGxkXg", // WithdrawLog
"3KzZ5jYfwp6LyPUt5LFbbPLEEsYNYtdAA5hs2KAdZxqarFiVVUKJ7WFxYXf8xeFnbWwypEWpNRCeE1d3y4e5Sxrk", // FillLog
"NGgdZu6YwNnFGoSMxc6ZgfUhSVy2rkB9BDY9yvYRLYZoteDSArWn5hDUGZuYCktC81jzGb6u9W6u7ab9mWutfQe", // SettlePnl
"67axUM8Q6tvCCNa6Cq8JfR9pLCAf5wyqc7gy1eafEtA4kcYE9Gte1LT8b9ziEWhtLfaGim5t5ChYU5uaQLfteGTJ", // SettlePnl - Multiple
"24hCHGXrf9nCnkioWf6HxtRqnQHGhhtTkARgA9tqkEmWb3uUzuE2oUU3so1JGNGZftCaBMjWek5rqAHRU3VAyQEJ", // Multiple net balances changes to the same (mangoAccount, token) pair
"eu5yMAACP41tvHAMmhuRAqu9tW4ZJeVY37TqbJLYVaqavKNh1rQEH97QGq6j3Zm721dgjMs94qmLm9PiSPDcwfX", // force_settle_quote_positions
"3twdGBA3Fz9Sbn4hS8AbHEcLomj8KyLVcB9P7ToiVP83cnHEHS7WFKgcsAa2TaDvYJ8emuhmBGWDybbtxVjgZXGt", // LiquidateTokenAndPerp
"3nQBUy8naBUj7US3PiVkcUt75jV4bszuQEV2vYMgrbYJGTJAbseuQJ7HvCQBhpWZqUUWPCeT72px1ijWdd4zC5ZG" // netbalances
];
// '56y2iUGUyQ8BcqLpL5N1QwR3QQF37NtaWRnd5XjQ25BNWKfNxvpnhwwqtD7B88dAvJEykTHNCLvUKdxY4jhQ2uhQ', //LiquidateTokenAndPerp example 2
// '4ErqDTV11imwDnoA6AAj3VMZC9FQvyq2GA4emfpiSxVBifAQE9EDwPnHxoTMxULHfyyCf8GBBzoczGSqYJMFEV5D', //LiquidateTokenAndPerp example 3
// '59DFSbsN1DbnqUiMsStS2xQn4tBncG8w3gg8aikBvQZKnWhf1yRt87EU4WMvXtnnVe18zWaRDFVfnVYp3ASgF7tV', // ResolveTokenBankruptcy
// '4Xx7gVesMQQZqprJYLu5gNEYRLA5GTXKURrkc8jG3CLKynRwhEM93MownyAMhxpdFTvfXQ9kkxkRcemZ8Fn5WHyk', // ResolvePerpBankruptcy
// '3bzj3KkA3FSZHJuCmRgHhSgqeaEzD32sCHkYdRLcZm1vcuB4ra5NbU5EZqBhW6QjeKRV9QRWC4SHxK2hS54s79Zx', // settle_pnl
// '5TmhvKQJmjUD9dZgCszBF8gNKUohpxwjrYu1RngZVh1hEToGMtjPtXJF89QLHXzANMWWQRfMomsgCg8353CpYgBb', // settle_fees
// 'mxK5eEiEUeCcQtHwmUKziyYUZJ3NXdjmZigAR6npWgcpQoJtqvikt5A7osD4y6oiLZJhzYFvqAqDznFjHm77K8V', // settle_fees called but nothing settled
// // '4qV6PTD1nGj5qq89FQK8QKwN231pGgtayD7uX4B6y83b19gcVXB5ByLCvApSJjCRrboiCg7RVT2p2e1CtP3zuXDb', // force_settle_quote_positions
// '5qDPBrFjCcaZthjRCqisHRw1mFEkHFHRFWi5jbKCmpAgpAXNdEkSv8L472D12VB5AukYaGsWhAy5bcvvUGJ1Sgtv', // FillEvent
// '3YXaEG95w5eG7jBBjz8hW9auXVAv9z2MH8yw51tL8nqSqmKgXtrD1hgE7LCqK2hpFwcrpjeWtBeVqGsbCHLh3kSe', // redeem mango
// '2HNnZmThkFUsG1pw9bNaJoeeGUZJun3hkcpwBJt3ZU9FKe3CY17wrJgk1BZ8txm13RJ512ThbZVZxaqsxNFn4xVs', // checked_add_net details
// '4ebib6h5kQHpcpK4A4UpH7ThJVEtui2X7vVvTfCW8iuJgjHMocH7nymN3zVrrbwZL9HQYJY1tHdnGjo7ZSgrL7M6', // error example
// '59DFSbsN1DbnqUiMsStS2xQn4tBncG8w3gg8aikBvQZKnWhf1yRt87EU4WMvXtnnVe18zWaRDFVfnVYp3ASgF7tV', //token socialized loss
// '4RCvRY8BWPB6FixyfufYKojUdnE91DiqmFE2b8e4FCyuWCdT1ipSzPBaWUgaajKucFr1jsveiMvTft5iiWbctCFk', // settle_pnl_multiple
// 'KDv62AKFqyrvULdUUeShdYK9zCeUDX8yb6kuUadoa6dJKrqjDvKU3JQj9t8e4H1FTEaEMsyLnYAT4HbTr8ikSGq', // multiple update funding
let signaturesSql = signatures.map((e) => "'" + e + "'").join(',');
// // 3.1 transactions:
// "2nyoFpqeopBuFgjSXntSUbJUYtxxFXTEWrf2t1715Bm4jTrYKNDwBkZKBuDN6mWt1vDcCuyPq56RGKv9SxuLJWwe", // UpdateRootBank
// "3CPWZM49NXPmpB5GKkrJHLknfSN595Gkd3MxK3AdFvFJTFHsA9ShjAqyWjfSkxYpBAc3ju19maaECzb18ey3qwK4", // CachePrices and CacheRootBanks
// "bf5kGknwbfzoMyz8cFHQAoK6ajwfqeNdFeWgaSGcy19poQPBbxpeW5BiLQFLK4DUswEwUvg2TZEusyqazT4WS4Y", // UpdateFunding
// "4JuHNZy2GcFJYFCYoTrekyCa2eBSq8cECVUvQp32LbFshAVFaVrvypwoKNjpfZ2rK9N7AG5PHsqqhHSWuGU3rAFk", // MngoAccrualLog
// "5MuJyoTsTzMGhGKDvN21BaZrfjaw1SeRgfTxz67U7zm7aVokrMqqvMhHvRCANGbjhvAeNFT1Zmk7vZoGp32b4H7t", // OpenOrdersBalanceLog, TokenBalanceLog
// "Ztxy2398k8zDPh1VFiErvAQAnQdMuUok7Z772yCST9oZpagmqjggRuzU8ScrBEpcX4X4BknC7gxvfX128BGxkXg", // WithdrawLog
// "3KzZ5jYfwp6LyPUt5LFbbPLEEsYNYtdAA5hs2KAdZxqarFiVVUKJ7WFxYXf8xeFnbWwypEWpNRCeE1d3y4e5Sxrk", // FillLog
// "NGgdZu6YwNnFGoSMxc6ZgfUhSVy2rkB9BDY9yvYRLYZoteDSArWn5hDUGZuYCktC81jzGb6u9W6u7ab9mWutfQe", // SettlePnl
// "67axUM8Q6tvCCNa6Cq8JfR9pLCAf5wyqc7gy1eafEtA4kcYE9Gte1LT8b9ziEWhtLfaGim5t5ChYU5uaQLfteGTJ", // SettlePnl - Multiple
// "24hCHGXrf9nCnkioWf6HxtRqnQHGhhtTkARgA9tqkEmWb3uUzuE2oUU3so1JGNGZftCaBMjWek5rqAHRU3VAyQEJ", // Multiple net balances changes to the same (mangoAccount, token) pair
// "eu5yMAACP41tvHAMmhuRAqu9tW4ZJeVY37TqbJLYVaqavKNh1rQEH97QGq6j3Zm721dgjMs94qmLm9PiSPDcwfX", // force_settle_quote_positions
// "3twdGBA3Fz9Sbn4hS8AbHEcLomj8KyLVcB9P7ToiVP83cnHEHS7WFKgcsAa2TaDvYJ8emuhmBGWDybbtxVjgZXGt", // LiquidateTokenAndPerp
// "3nQBUy8naBUj7US3PiVkcUt75jV4bszuQEV2vYMgrbYJGTJAbseuQJ7HvCQBhpWZqUUWPCeT72px1ijWdd4zC5ZG", // netbalances
// "3Y3mK95f2Za1K3fwoXLzB2LrTPjyyojCGg26x9aiJGCHmshtDy1Q8WDpkcv7v9PcU7VsbU2mduktMhJcT3mZTRac", // update funding
// "3YL7DHcULvjPPKFiDMUvrKuDG8zRT89sWuy5UNn7y6f6fdHQpNFTFNgg1JPZXpvaAELBp4EUdCkzQtjP9HQGmK6f"
// "3zZsXZdiT3afhDmPVzG5caJvhx3BonzeGuwWL9BLKedvMBEJdJdhSG3s1b9VM5HtXwMtLNysC2x2QNyH2tL6JfxU"
// "2ghyYdskT3Pcx11CqbjNuXscrsj7A4hNWfJ9fHm7vjZFrd6fbrhJxtc7icron3mP5JQ8HaLLex1uscHR5YvNwpyM"
// "5aWmePQwvT2RGd4uUfTTKX76S5PHC9Lo5nTuPWWQcbyWDVR5SFDJDALK6GAeJyVbeF21M3tu1mVqHxBxo7Ut4ch9"
// "46dyFA53Q8ha5JEYin9vcaHrQJL2qbwD4rT5DWkwBnbfB35VNcKCx4VBA25nEHnYG46Q1aCKQecWinYJK6KfXwgB", // Log truncated
// "5UuX55VW7puEFeULrBkPn9oHgeZzRCmMeekw8To9pHscnC9LkwgcXKqYw6RnHYbEwd9Zn74AJUkSEbUmFxqK1J4K" // Log truncated
res = await client.query(
'select transaction, signature from ' +
schema +
'.transactions where signature in (' +
signaturesSql +
') limit $1',
[limit],
);
} finally {
client.release();
}
// "2WYkCZnzxkVcpFgVEBhUpnRYyBNZNHmx8oEpL59UL8KbL7jPftRvaDmjtG7uBfLnANWghsAuuSbQWP9JuYcPCXcg",
// "61kQNnDJkekFP5SyitE4UgmvHTnKKs8yuc5Kosro5xxXEvmMZaLd5kB9NdDLGPvYQRMfUriMDje4rNiXKbKk1KJw"
let transactions = res.rows.map((e) => [e.transaction, e.signature]);
// "46Rf29aBtjeUUQYsLms2SdJ3nSCASdsFsGpgynZX4d9aMjTwYGk87FZEKM5CVGHK8M8tbpeSiX6xhg8V7FLLr9tY",
// "3tq9iqZuxHoV2xZ1NpqvQRWhd26tijetnZ3jdJGjBP2dWK41YAt79S9yQeX93ta1qEyCz2v6HQamSNan9HtN75i8",
// CancelAllPerpOrders
'67hoPW6rHJHqgCgZ2HnUgHrdnTE7gpdcvsZCo6vZYpkuQyrcyNKzxj8XTGb8b2fXQHGPJME8Y38REWu1cwFELES6',
];
let transactions = await getTransactions(signatures, clusterConnection, requestWaitTime);
let [processStates, parsedTransactions] = parseTransactions(
transactions,
mangoProgramId,
@ -73,28 +78,20 @@ async function processMangoTransactions(rawTransactionsPool, schema, limit) {
let errorProcessStates = processStates.filter(e => e.process_state === 'parsing error');
console.log(errorProcessStates.length + ' parsing errors')
// Set a breakpoint here to examine parsed transactions
console.log(parsedTransactions);
}
async function consumeTransactions() {
const rawConnectionString = process.env.CONNECTION_STRING_RAW;
const clusterUrl =
process.env.CLUSTER_URL || 'https://api.mainnet-beta.solana.com';
let requestWaitTime = parseInt(process.env.REQUEST_WAIT_TIME!) || 500;
let schema = 'transactions_v3';
let clusterConnection = new Connection(clusterUrl, 'finalized');
const rawTransactionsPool = new Pool({
connectionString: rawConnectionString,
ssl: {
rejectUnauthorized: false,
},
});
let transactionsParsingLimit = 50000;
await processMangoTransactions(
rawTransactionsPool,
schema,
transactionsParsingLimit,
clusterConnection,
requestWaitTime
);
}