Add new command "init" to populate some rows into the transactions table (required before the scraper can start).
This commit is contained in:
parent
44b3b7027a
commit
5bf322ef00
|
@ -8,6 +8,7 @@
|
||||||
"clean": "rm -rf dist",
|
"clean": "rm -rf dist",
|
||||||
"prepare": "yarn clean && yarn build",
|
"prepare": "yarn clean && yarn build",
|
||||||
"start": "node dist/index.js",
|
"start": "node dist/index.js",
|
||||||
|
"init": "node dist/init.js",
|
||||||
"test": "echo \"Error: no test specified\" && exit 1",
|
"test": "echo \"Error: no test specified\" && exit 1",
|
||||||
"noop": ""
|
"noop": ""
|
||||||
},
|
},
|
||||||
|
|
23
src/index.ts
23
src/index.ts
|
@ -1,6 +1,6 @@
|
||||||
import { Connection, PublicKey } from '@solana/web3.js';
|
import { Connection } from '@solana/web3.js';
|
||||||
import { Cluster, sleep } from '@blockworks-foundation/mango-client';
|
import { sleep } from '@blockworks-foundation/mango-client';
|
||||||
import { insertNewSignatures } from './signatures';
|
import { insertSignatures, getNewSignatures } from './signatures';
|
||||||
import { Pool } from 'pg';
|
import { Pool } from 'pg';
|
||||||
import { notify } from './utils';
|
import { notify } from './utils';
|
||||||
import { getTransactions, getUnprocessedSignatures } from './getTransactions';
|
import { getTransactions, getUnprocessedSignatures } from './getTransactions';
|
||||||
|
@ -170,7 +170,7 @@ async function consumeTransactions() {
|
||||||
const workerPartitionEnd = parseInt(process.env.WORKER_PARTITION_END!);
|
const workerPartitionEnd = parseInt(process.env.WORKER_PARTITION_END!);
|
||||||
|
|
||||||
// Only one process should insert new signatures. 1 for true, 0 for false.
|
// Only one process should insert new signatures. 1 for true, 0 for false.
|
||||||
const insertSignatures = parseInt(process.env.INSERT_SIGNATURES!) === 1;
|
const insertSignaturesBool = parseInt(process.env.INSERT_SIGNATURES!) === 1;
|
||||||
|
|
||||||
let schema = 'transactions_v3';
|
let schema = 'transactions_v3';
|
||||||
|
|
||||||
|
@ -194,14 +194,19 @@ async function consumeTransactions() {
|
||||||
while (true) {
|
while (true) {
|
||||||
console.log('Refreshing transactions ' + Date());
|
console.log('Refreshing transactions ' + Date());
|
||||||
|
|
||||||
if (insertSignatures) {
|
if (insertSignaturesBool) {
|
||||||
await insertNewSignatures(
|
let newSignatures = await getNewSignatures(clusterConnection,
|
||||||
mangoProgramId,
|
mangoProgramId,
|
||||||
clusterConnection,
|
|
||||||
rawTransactionsPool,
|
|
||||||
requestWaitTime,
|
requestWaitTime,
|
||||||
|
rawTransactionsPool,
|
||||||
|
schema)
|
||||||
|
|
||||||
|
await insertSignatures(
|
||||||
|
mangoProgramId,
|
||||||
|
rawTransactionsPool,
|
||||||
schema,
|
schema,
|
||||||
);
|
newSignatures
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
await processMangoTransactions(
|
await processMangoTransactions(
|
||||||
|
|
|
@ -0,0 +1,43 @@
|
||||||
|
import { Connection, PublicKey } from '@solana/web3.js';
|
||||||
|
import { insertSignatures } from './signatures';
|
||||||
|
import { Pool } from 'pg';
|
||||||
|
|
||||||
|
|
||||||
|
const pgp = require('pg-promise')({
|
||||||
|
capSQL: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
const mangoProgramId =
|
||||||
|
process.env.MANGO_PROGRAM_ID ||
|
||||||
|
'5fNfvyp5czQVX77yoACa3JJVEhdRaWjPuazuWgjhTqEH';
|
||||||
|
|
||||||
|
async function initTransactions() {
|
||||||
|
const clusterUrl =
|
||||||
|
process.env.CLUSTER_URL || 'https://api.mainnet-beta.solana.com';
|
||||||
|
const rawConnectionString = process.env.CONNECTION_STRING_RAW;
|
||||||
|
|
||||||
|
let schema = 'transactions_v3';
|
||||||
|
|
||||||
|
console.log(clusterUrl);
|
||||||
|
let clusterConnection = new Connection(clusterUrl, 'finalized');
|
||||||
|
const rawTransactionsPool = new Pool({
|
||||||
|
connectionString: rawConnectionString,
|
||||||
|
ssl: {
|
||||||
|
rejectUnauthorized: false,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
let signaturesInfo = await clusterConnection.getConfirmedSignaturesForAddress2(new PublicKey(mangoProgramId));
|
||||||
|
|
||||||
|
await insertSignatures(
|
||||||
|
mangoProgramId,
|
||||||
|
rawTransactionsPool,
|
||||||
|
schema,
|
||||||
|
signaturesInfo
|
||||||
|
)
|
||||||
|
|
||||||
|
console.log('transactions table sucessfully initialised')
|
||||||
|
}
|
||||||
|
|
||||||
|
initTransactions();
|
||||||
|
|
|
@ -3,12 +3,24 @@ import { sleep } from '@blockworks-foundation/mango-client';
|
||||||
import { bulkBatchInsert } from './utils';
|
import { bulkBatchInsert } from './utils';
|
||||||
|
|
||||||
export async function getNewSignatures(
|
export async function getNewSignatures(
|
||||||
afterSlot: number,
|
|
||||||
connection: Connection,
|
connection: Connection,
|
||||||
addressPk: PublicKey,
|
address: string,
|
||||||
requestWaitTime: number,
|
requestWaitTime: number,
|
||||||
|
pool,
|
||||||
|
schema
|
||||||
) {
|
) {
|
||||||
|
|
||||||
|
let client = await pool.connect();
|
||||||
|
let latestSlotRows = await client.query(
|
||||||
|
'select max(slot) as max_slot from ' +
|
||||||
|
schema +
|
||||||
|
'.transactions where program_pk = $1',
|
||||||
|
[address],
|
||||||
|
);
|
||||||
|
client.release();
|
||||||
|
|
||||||
|
let afterSlot = latestSlotRows.rows[0]['max_slot'];
|
||||||
|
|
||||||
console.log('Fetching all signatures after slot ' + afterSlot)
|
console.log('Fetching all signatures after slot ' + afterSlot)
|
||||||
|
|
||||||
// Fetches all signatures associated with the account - working backwards in time until it encounters the "afterSlot" slot
|
// Fetches all signatures associated with the account - working backwards in time until it encounters the "afterSlot" slot
|
||||||
|
@ -26,7 +38,7 @@ export async function getNewSignatures(
|
||||||
}
|
}
|
||||||
|
|
||||||
let signaturesInfo = await connection.getConfirmedSignaturesForAddress2(
|
let signaturesInfo = await connection.getConfirmedSignaturesForAddress2(
|
||||||
addressPk,
|
new PublicKey(address),
|
||||||
options,
|
options,
|
||||||
);
|
);
|
||||||
signatures = signaturesInfo.map((x) => x['signature']);
|
signatures = signaturesInfo.map((x) => x['signature']);
|
||||||
|
@ -34,7 +46,8 @@ export async function getNewSignatures(
|
||||||
|
|
||||||
if (signaturesInfo[0].slot < afterSlot) {
|
if (signaturesInfo[0].slot < afterSlot) {
|
||||||
// Possibly this happens when validators are switched at the endpoint?
|
// Possibly this happens when validators are switched at the endpoint?
|
||||||
throw 'most recent fetched signature has slot less than afterSlot'
|
// In the absense of throwing this error the code will keep fetching signatures until to reaches the end of the signatures stored by the rpc node
|
||||||
|
throw ('most recent fetched signature has slot less than afterSlot')
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log('slots ' + signaturesInfo[0].slot + ' to ' + signaturesInfo[signaturesInfo.length - 1].slot)
|
console.log('slots ' + signaturesInfo[0].slot + ' to ' + signaturesInfo[signaturesInfo.length - 1].slot)
|
||||||
|
@ -73,31 +86,12 @@ export async function getNewSignatures(
|
||||||
return allSignaturesInfo;
|
return allSignaturesInfo;
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function insertNewSignatures(
|
export async function insertSignatures(
|
||||||
address: string,
|
address: string,
|
||||||
connection: Connection,
|
|
||||||
pool,
|
pool,
|
||||||
requestWaitTime: number,
|
|
||||||
schema: string,
|
schema: string,
|
||||||
|
newSignatures: ConfirmedSignatureInfo[]
|
||||||
) {
|
) {
|
||||||
let client = await pool.connect();
|
|
||||||
let latestSlotRows = await client.query(
|
|
||||||
'select max(slot) as max_slot from ' +
|
|
||||||
schema +
|
|
||||||
'.transactions where program_pk = $1',
|
|
||||||
[address],
|
|
||||||
);
|
|
||||||
|
|
||||||
client.release();
|
|
||||||
|
|
||||||
let latestSlot = latestSlotRows.rows[0]['max_slot'];
|
|
||||||
|
|
||||||
let newSignatures = await getNewSignatures(
|
|
||||||
latestSlot,
|
|
||||||
connection,
|
|
||||||
new PublicKey(address),
|
|
||||||
requestWaitTime,
|
|
||||||
);
|
|
||||||
|
|
||||||
// By default the signatures returned by getConfirmedSignaturesForAddress2 will be ordered newest -> oldest
|
// By default the signatures returned by getConfirmedSignaturesForAddress2 will be ordered newest -> oldest
|
||||||
// We reverse the order to oldest -> newest here
|
// We reverse the order to oldest -> newest here
|
||||||
|
|
Loading…
Reference in New Issue