From bc2d98d8895b8fc6fdbfa795e8c7f807b40dc6e0 Mon Sep 17 00:00:00 2001 From: Julian <52217955+julianmerlo95@users.noreply.github.com> Date: Tue, 20 Feb 2024 13:30:08 -0300 Subject: [PATCH] [Blockchain Watcher] (RPC QUERIES) Improve rpc queries (#1131) * Improve evm block search * Resolve test * Add more rpc for base * Add new rpc for avalanche * Improve logs --------- Co-authored-by: julian merlo --- blockchain-watcher/config/mainnet.json | 13 +++++++++++-- .../src/domain/actions/evm/GetEvmLogs.ts | 2 +- .../src/domain/actions/evm/GetEvmTransactions.ts | 13 +++++++++---- blockchain-watcher/src/domain/repositories.ts | 6 +++++- .../evm/evmRedeemedTransactionFoundMapper.ts | 2 +- .../mappers/solana/solanaTransferRedeemedMapper.ts | 2 +- .../sui/suiRedeemedTransactionFoundMapper.ts | 2 +- .../repositories/evm/EvmJsonRPCBlockRepository.ts | 8 ++++++-- .../evm/RateLimitedEvmJsonRPCBlockRepository.ts | 10 ++++++++-- .../domain/actions/evm/GetEvmTransactions.test.ts | 12 ++++++------ .../test/domain/actions/evm/PollEvm.test.ts | 4 ++-- 11 files changed, 51 insertions(+), 23 deletions(-) diff --git a/blockchain-watcher/config/mainnet.json b/blockchain-watcher/config/mainnet.json index 61f89fc9..a5b9e597 100644 --- a/blockchain-watcher/config/mainnet.json +++ b/blockchain-watcher/config/mainnet.json @@ -33,7 +33,11 @@ }, "avalanche": { "network": "mainnet", - "rpcs": ["https://api.avax.network/ext/bc/C/rpc", "https://avalanche.public-rpc.com"] + "rpcs": [ + "https://avalanche.blockpi.network/v1/rpc/public", + "https://api.avax.network/ext/bc/C/rpc", + "https://avalanche.public-rpc.com" + ] }, "oasis": { "network": "mainnet", @@ -89,7 +93,12 @@ }, "base": { "network": "mainnet", - "rpcs": ["https://base.publicnode.com", "https://mainnet.base.org"] + "rpcs": [ + "https://base-pokt.nodies.app", + "https://base.publicnode.com", + "https://mainnet.base.org", + "https://rpc.notadegen.com/base" + ] }, "sui": { "network": "mainnet", diff --git a/blockchain-watcher/src/domain/actions/evm/GetEvmLogs.ts b/blockchain-watcher/src/domain/actions/evm/GetEvmLogs.ts index e9df67a9..7f6d8648 100644 --- a/blockchain-watcher/src/domain/actions/evm/GetEvmLogs.ts +++ b/blockchain-watcher/src/domain/actions/evm/GetEvmLogs.ts @@ -28,7 +28,7 @@ export class GetEvmLogs { }); const blockNumbers = new Set(logs.map((log) => log.blockNumber)); - const blocks = await this.blockRepo.getBlocks(opts.chain, blockNumbers); + const blocks = await this.blockRepo.getBlocks(opts.chain, blockNumbers, false); logs.forEach((log) => { const block = blocks[log.blockHash]; log.blockTime = block.timestamp; diff --git a/blockchain-watcher/src/domain/actions/evm/GetEvmTransactions.ts b/blockchain-watcher/src/domain/actions/evm/GetEvmTransactions.ts index 17eb35e4..98f01f56 100644 --- a/blockchain-watcher/src/domain/actions/evm/GetEvmTransactions.ts +++ b/blockchain-watcher/src/domain/actions/evm/GetEvmTransactions.ts @@ -30,8 +30,15 @@ export class GetEvmTransactions { this.logger.info( `[${chain}][exec] Processing blocks [fromBlock: ${fromBlock} - toBlock: ${toBlock}]` ); + + const blockNumbers: Set = new Set(); for (let block = fromBlock; block <= toBlock; block++) { - const evmBlock = await this.blockRepo.getBlock(chain, block, isTransactionsPresent); + blockNumbers.add(block); + } + const evmBlocks = await this.blockRepo.getBlocks(chain, blockNumbers, isTransactionsPresent); + + for (const blockKey in evmBlocks) { + const evmBlock = evmBlocks[blockKey]; const transactions = evmBlock.transactions ?? []; // Only process transactions to the contract address configured @@ -42,9 +49,7 @@ export class GetEvmTransactions { ); if (transactionsByAddressConfigured.length > 0) { - const hashNumbers = new Set( - transactionsByAddressConfigured.map((transaction) => transaction.hash) - ); + const hashNumbers = new Set(transactionsByAddressConfigured.map((tx) => tx.hash)); const receiptTransactions = await this.blockRepo.getTransactionReceipt(chain, hashNumbers); const filterTransactions = this.filterTransactions( diff --git a/blockchain-watcher/src/domain/repositories.ts b/blockchain-watcher/src/domain/repositories.ts index 56271aea..aa53f3f1 100644 --- a/blockchain-watcher/src/domain/repositories.ts +++ b/blockchain-watcher/src/domain/repositories.ts @@ -21,7 +21,11 @@ import { SuiTransactionBlockReceipt } from "./entities/sui"; export interface EvmBlockRepository { getBlockHeight(chain: string, finality: string): Promise; - getBlocks(chain: string, blockNumbers: Set): Promise>; + getBlocks( + chain: string, + blockNumbers: Set, + isTransactionsPresent: boolean + ): Promise>; getFilteredLogs(chain: string, filter: EvmLogFilter): Promise; getTransactionReceipt( chain: string, diff --git a/blockchain-watcher/src/infrastructure/mappers/evm/evmRedeemedTransactionFoundMapper.ts b/blockchain-watcher/src/infrastructure/mappers/evm/evmRedeemedTransactionFoundMapper.ts index fdbd75c0..60fa813f 100644 --- a/blockchain-watcher/src/infrastructure/mappers/evm/evmRedeemedTransactionFoundMapper.ts +++ b/blockchain-watcher/src/infrastructure/mappers/evm/evmRedeemedTransactionFoundMapper.ts @@ -36,7 +36,7 @@ export const evmRedeemedTransactionFoundMapper = ( if (protocol && protocol.type && protocol.method) { logger.debug( - `[${transaction.chain}] Transaction info: [hash: ${transaction.hash}][VAA: ${emitterChain}/${emitterAddress}/${sequence}][protocol: ${protocol.type}/${protocol.method}]` + `[${transaction.chain}] Redeemed transaction info: [hash: ${transaction.hash}][VAA: ${emitterChain}/${emitterAddress}/${sequence}][protocol: ${protocol.type}/${protocol.method}]` ); return { diff --git a/blockchain-watcher/src/infrastructure/mappers/solana/solanaTransferRedeemedMapper.ts b/blockchain-watcher/src/infrastructure/mappers/solana/solanaTransferRedeemedMapper.ts index e3677ca2..d6270e68 100644 --- a/blockchain-watcher/src/infrastructure/mappers/solana/solanaTransferRedeemedMapper.ts +++ b/blockchain-watcher/src/infrastructure/mappers/solana/solanaTransferRedeemedMapper.ts @@ -59,7 +59,7 @@ export const solanaTransferRedeemedMapper = async ( const protocol = findProtocol(instruction, programIdIndex, programId, chain); logger.debug( - `[${chain}}] Transaction info: [hash: ${txHash}][VAA: ${emitterChain}/${emitterAddress.toString( + `[${chain}}] Redeemed transaction info: [hash: ${txHash}][VAA: ${emitterChain}/${emitterAddress.toString( "hex" )}/${sequence}]` ); diff --git a/blockchain-watcher/src/infrastructure/mappers/sui/suiRedeemedTransactionFoundMapper.ts b/blockchain-watcher/src/infrastructure/mappers/sui/suiRedeemedTransactionFoundMapper.ts index ce6ba48d..de1bd25f 100644 --- a/blockchain-watcher/src/infrastructure/mappers/sui/suiRedeemedTransactionFoundMapper.ts +++ b/blockchain-watcher/src/infrastructure/mappers/sui/suiRedeemedTransactionFoundMapper.ts @@ -21,7 +21,7 @@ export const suiRedeemedTransactionFoundMapper = ( const { emitterAddress, emitterChainId: emitterChain, sequence } = vaa; logger.info( - `[sui] Redeemed Transfer info: [digest: ${receipt.digest}][VAA: ${emitterChain}/${emitterAddress}/${sequence}]` + `[sui] Redeemed transaction info: [digest: ${receipt.digest}][VAA: ${emitterChain}/${emitterAddress}/${sequence}]` ); return { diff --git a/blockchain-watcher/src/infrastructure/repositories/evm/EvmJsonRPCBlockRepository.ts b/blockchain-watcher/src/infrastructure/repositories/evm/EvmJsonRPCBlockRepository.ts index 879f63b6..4beb8309 100644 --- a/blockchain-watcher/src/infrastructure/repositories/evm/EvmJsonRPCBlockRepository.ts +++ b/blockchain-watcher/src/infrastructure/repositories/evm/EvmJsonRPCBlockRepository.ts @@ -49,7 +49,11 @@ export class EvmJsonRPCBlockRepository implements EvmBlockRepository { * @param blockNumbers * @returns a record of block hash -> EvmBlock */ - async getBlocks(chain: string, blockNumbers: Set): Promise> { + async getBlocks( + chain: string, + blockNumbers: Set, + isTransactionsPresent: boolean = false + ): Promise> { if (!blockNumbers.size) return {}; let combinedResults: ResultBlocks[] = []; @@ -66,7 +70,7 @@ export class EvmJsonRPCBlockRepository implements EvmBlockRepository { jsonrpc: "2.0", id: blockNumberStrId, method: "eth_getBlockByNumber", - params: [blockNumberStrParam, false], + params: [blockNumberStrParam, isTransactionsPresent], }); } diff --git a/blockchain-watcher/src/infrastructure/repositories/evm/RateLimitedEvmJsonRPCBlockRepository.ts b/blockchain-watcher/src/infrastructure/repositories/evm/RateLimitedEvmJsonRPCBlockRepository.ts index 7c1ccebd..08a98d9b 100644 --- a/blockchain-watcher/src/infrastructure/repositories/evm/RateLimitedEvmJsonRPCBlockRepository.ts +++ b/blockchain-watcher/src/infrastructure/repositories/evm/RateLimitedEvmJsonRPCBlockRepository.ts @@ -23,8 +23,14 @@ export class RateLimitedEvmJsonRPCBlockRepository return this.breaker.fn(() => this.delegate.getBlockHeight(chain, finality)).execute(); } - getBlocks(chain: string, blockNumbers: Set): Promise> { - return this.breaker.fn(() => this.delegate.getBlocks(chain, blockNumbers)).execute(); + getBlocks( + chain: string, + blockNumbers: Set, + isTransactionsPresent: boolean + ): Promise> { + return this.breaker + .fn(() => this.delegate.getBlocks(chain, blockNumbers, isTransactionsPresent)) + .execute(); } getFilteredLogs(chain: string, filter: EvmLogFilter): Promise { diff --git a/blockchain-watcher/test/domain/actions/evm/GetEvmTransactions.test.ts b/blockchain-watcher/test/domain/actions/evm/GetEvmTransactions.test.ts index fc00888d..c4f3c2ef 100644 --- a/blockchain-watcher/test/domain/actions/evm/GetEvmTransactions.test.ts +++ b/blockchain-watcher/test/domain/actions/evm/GetEvmTransactions.test.ts @@ -4,7 +4,7 @@ import { EvmBlockRepository } from "../../../../src/domain/repositories"; import { EvmBlock, EvmLog, ReceiptTransaction } from "../../../../src/domain/entities/evm"; let getTransactionReceipt: jest.SpiedFunction; -let getBlockSpy: jest.SpiedFunction; +let getBlocksSpy: jest.SpiedFunction; let getEvmTransactions: GetEvmTransactions; let evmBlockRepo: EvmBlockRepository; @@ -68,7 +68,7 @@ describe("GetEvmTransactions", () => { // Then result.then((response) => { expect(response).toEqual([]); - expect(getBlockSpy).toHaveReturnedTimes(1); + expect(getBlocksSpy).toHaveReturnedTimes(1); }); }); @@ -101,7 +101,7 @@ describe("GetEvmTransactions", () => { expect(response[0].from).toEqual("0x3ee123456786797000d974cf647e7c347e8fa585"); expect(response[0].to).toEqual("0x3ee18b2214aff97000d974cf647e7c347e8fa585"); expect(getTransactionReceipt).toHaveReturnedTimes(1); - expect(getBlockSpy).toHaveReturnedTimes(1); + expect(getBlocksSpy).toHaveReturnedTimes(1); }); }); @@ -161,7 +161,7 @@ describe("GetEvmTransactions", () => { expect(response[0].from).toEqual("0x3ee123456786797000d974cf647e7c347e8fa585"); expect(response[0].to).toEqual("0x4cb69fae7e7af841e44e1a1c30af640739378bb2"); expect(getTransactionReceipt).toHaveReturnedTimes(2); - expect(getBlockSpy).toHaveReturnedTimes(2); + expect(getBlocksSpy).toHaveReturnedTimes(1); }); }); }); @@ -192,7 +192,7 @@ const givenEvmBlockRepository = ( const blocksResponse: Record = {}; const receiptResponse: Record = {}; if (height) { - for (let index = 0n; index <= (blocksAhead ?? 1n); index++) { + for (let index = height; index <= (blocksAhead ?? 1n); index++) { logsResponse.push({ blockNumber: height + index, blockHash: `0x0${index}`, @@ -254,7 +254,7 @@ const givenEvmBlockRepository = ( getBlock: () => Promise.resolve(blocksResponse[`0x01`]), }; - getBlockSpy = jest.spyOn(evmBlockRepo, "getBlock"); + getBlocksSpy = jest.spyOn(evmBlockRepo, "getBlocks"); getTransactionReceipt = jest.spyOn(evmBlockRepo, "getTransactionReceipt"); }; diff --git a/blockchain-watcher/test/domain/actions/evm/PollEvm.test.ts b/blockchain-watcher/test/domain/actions/evm/PollEvm.test.ts index 3e05f192..96ecc7e1 100644 --- a/blockchain-watcher/test/domain/actions/evm/PollEvm.test.ts +++ b/blockchain-watcher/test/domain/actions/evm/PollEvm.test.ts @@ -1,5 +1,4 @@ import { afterEach, describe, it, expect, jest } from "@jest/globals"; -import { setTimeout } from "timers/promises"; import { PollEvmLogsMetadata, PollEvm, PollEvmLogsConfig } from "../../../../src/domain/actions"; import { EvmBlockRepository, @@ -46,7 +45,8 @@ describe("PollEvm", () => { () => expect(getBlocksSpy).toHaveBeenCalledWith( "acala", - new Set([currentHeight, currentHeight + 1n]) + new Set([currentHeight, currentHeight + 1n]), + false ), () => expect(getLogsSpy).toBeCalledWith("acala", {