blockworks-foundation
/
mango-transaction-scraper-v3
mirror of https://github.com/blockworks-foundation/mango-transaction-scraper-v3.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add PARSING_BATCH_SIZE as an environmental variable.

This commit is contained in:
Nicholas Clarke 2021-12-14 09:25:46 -08:00
parent 319153140d
commit 3b9412cc14
2 changed files with 9 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/getTransactions.ts
View File

@ -32,7 +32,9 @@ export async function getUnprocessedTransactions(
transactions.push([signature, confirmedTransaction]),
);
console.log('requested ', counter, ' of ', signaturesToProcess.length);
if (counter % 100 === 0) {
console.log('requested ', counter, ' of ', signaturesToProcess.length);
}
counter++;
promises.push(promise);
@ -40,6 +42,8 @@ export async function getUnprocessedTransactions(
// Limit request frequency to avoid request failures due to rate limiting
await sleep(requestWaitTime);
}
console.log('requested ', counter, ' of ', signaturesToProcess.length);
await (Promise as any).allSettled(promises);
return transactions;

8
src/index.ts
View File

@ -120,10 +120,10 @@ async function processMangoTransactions(
rawTransactionsPool,
parsedTransactionsPool,
schema,
limit,
parsingBatchSize,
) {
let transactions = await getUnprocessedTransactions(clusterConnection, address, requestWaitTime, rawTransactionsPool, schema, limit)
let transactions = await getUnprocessedTransactions(clusterConnection, address, requestWaitTime, rawTransactionsPool, schema, parsingBatchSize)
let [transactionSummaries, parsedTransactions] = parseTransactions(
transactions,
@ -142,6 +142,7 @@ async function consumeTransactions() {
const clusterUrl =
process.env.CLUSTER_URL || 'https://api.mainnet-beta.solana.com';
let requestWaitTime = parseInt(process.env.REQUEST_WAIT_TIME!) || 500;
let parsingBatchSize = parseInt(process.env.PARSING_BATCH_SIZE!) || 10000;
const rawConnectionString = process.env.CONNECTION_STRING_RAW;
const parsedConnectionString = process.env.CONNECTION_STRING_PARSED;
@ -175,7 +176,6 @@ async function consumeTransactions() {
schema,
);
let transactionsParsingLimit = 50000;
await processMangoTransactions(
clusterConnection,
requestWaitTime,
@ -183,7 +183,7 @@ async function consumeTransactions() {
rawTransactionsPool,
parsedTransactionsPool,
schema,
transactionsParsingLimit,
parsingBatchSize,
);
console.log('Refresh complete');