From 44e236fc61d5cb548cedf4e3436471f1c5f8d686 Mon Sep 17 00:00:00 2001 From: agodnic Date: Tue, 18 Jul 2023 12:00:04 -0300 Subject: [PATCH] Update tx tracker backfiller (#540) ### Description Related issues: https://github.com/wormhole-foundation/wormhole-explorer/issues/525, https://github.com/wormhole-foundation/wormhole-explorer/issues/524 Final updates before running the tx-tracker backfiller on staging and production. --- .../tx-tracker-backfiller/env/production.env | 18 ++++--- deploy/tx-tracker-backfiller/env/staging.env | 6 ++- deploy/tx-tracker-backfiller/env/test.env | 50 +++++++++++-------- tx-tracker/.env.example | 28 ----------- tx-tracker/cmd/backfiller/main.go | 1 + 5 files changed, 45 insertions(+), 58 deletions(-) delete mode 100644 tx-tracker/.env.example diff --git a/deploy/tx-tracker-backfiller/env/production.env b/deploy/tx-tracker-backfiller/env/production.env index 7d179b3f..3ec985c8 100644 --- a/deploy/tx-tracker-backfiller/env/production.env +++ b/deploy/tx-tracker-backfiller/env/production.env @@ -2,10 +2,16 @@ ENVIRONMENT=production NAMESPACE=wormscan NAME=wormscan-tx-tracker-backfiller IMAGE_NAME= -RESOURCES_LIMITS_MEMORY=256Mi +RESOURCES_LIMITS_MEMORY=128Mi RESOURCES_LIMITS_CPU=500m -RESOURCES_REQUESTS_MEMORY=128Mi -RESOURCES_REQUESTS_CPU=250m +RESOURCES_REQUESTS_MEMORY=64Mi +RESOURCES_REQUESTS_CPU=100m + +NUM_WORKERS=50 +BULK_SIZE=1000 +STRATEGY_NAME=time_range +STRATEGY_TIMESTAMP_AFTER=2023-01-01T00:00:00.000Z +STRATEGY_TIMESTAMP_BEFORE=2024-01-01T00:00:00.000Z ACALA_BASE_URL=https://eth-rpc-acala.aca-api.network ACALA_REQUESTS_PER_MINUTE=1 @@ -14,13 +20,13 @@ ALGORAND_BASE_URL=https://mainnet-idx.algonode.cloud ALGORAND_REQUESTS_PER_MINUTE=1 APTOS_BASE_URL=https://fullnode.mainnet.aptoslabs.com/v1 -APTOS_REQUESTS_PER_MINUTE=1 +APTOS_REQUESTS_PER_MINUTE=2 ARBITRUM_BASE_URL=https://rpc.ankr.com/arbitrum ARBITRUM_REQUESTS_PER_MINUTE=1 AVALANCHE_BASE_URL=https://api.avax.network/ext/bc/C/rpc -AVALANCHE_REQUESTS_PER_MINUTE=1 +AVALANCHE_REQUESTS_PER_MINUTE=2 BSC_BASE_URL=https://bsc-dataseed2.defibit.io BSC_REQUESTS_PER_MINUTE=2 @@ -32,7 +38,7 @@ ETHEREUM_BASE_URL=https://rpc.ankr.com/eth ETHEREUM_REQUESTS_PER_MINUTE=2 FANTOM_BASE_URL=https://rpc.ankr.com/fantom -FANTOM_REQUESTS_PER_MINUTE=1 +FANTOM_REQUESTS_PER_MINUTE=2 # Currently, the main URL for injective RPC (https://k8s.mainnet.lcd.injective.network) is down. # We're temporally using a different one. diff --git a/deploy/tx-tracker-backfiller/env/staging.env b/deploy/tx-tracker-backfiller/env/staging.env index d973194c..a45280ab 100644 --- a/deploy/tx-tracker-backfiller/env/staging.env +++ b/deploy/tx-tracker-backfiller/env/staging.env @@ -5,11 +5,13 @@ IMAGE_NAME= RESOURCES_LIMITS_MEMORY=128Mi RESOURCES_LIMITS_CPU=500m RESOURCES_REQUESTS_MEMORY=64Mi -RESOURCES_REQUESTS_CPU=250m +RESOURCES_REQUESTS_CPU=100m +NUM_WORKERS=50 +BULK_SIZE=1000 STRATEGY_NAME=time_range STRATEGY_TIMESTAMP_AFTER=2023-01-01T00:00:00.000Z -STRATEGY_TIMESTAMP_BEFORE=2023-04-01T00:00:00.000Z +STRATEGY_TIMESTAMP_BEFORE=2024-01-01T00:00:00.000Z ACALA_BASE_URL=https://eth-rpc-acala.aca-api.network ACALA_REQUESTS_PER_MINUTE=1 diff --git a/deploy/tx-tracker-backfiller/env/test.env b/deploy/tx-tracker-backfiller/env/test.env index d8946757..8210d4fe 100644 --- a/deploy/tx-tracker-backfiller/env/test.env +++ b/deploy/tx-tracker-backfiller/env/test.env @@ -3,69 +3,75 @@ NAMESPACE=wormscan-testnet NAME=wormscan-tx-tracker-backfiller IMAGE_NAME= RESOURCES_LIMITS_MEMORY=128Mi -RESOURCES_LIMITS_CPU=200m +RESOURCES_LIMITS_CPU=500m RESOURCES_REQUESTS_MEMORY=64Mi RESOURCES_REQUESTS_CPU=100m +NUM_WORKERS=50 +BULK_SIZE=1000 +STRATEGY_NAME=time_range +STRATEGY_TIMESTAMP_AFTER=2023-01-01T00:00:00.000Z +STRATEGY_TIMESTAMP_BEFORE=2024-01-01T00:00:00.000Z + ACALA_BASE_URL=https://acala-dev.aca-dev.network/eth/http -ACALA_REQUESTS_PER_MINUTE=1 +ACALA_REQUESTS_PER_MINUTE=2 ALGORAND_BASE_URL=https://testnet-idx.algonode.cloud -ALGORAND_REQUESTS_PER_MINUTE=1 +ALGORAND_REQUESTS_PER_MINUTE=2 APTOS_BASE_URL=https://fullnode.testnet.aptoslabs.com/v1 -APTOS_REQUESTS_PER_MINUTE=1 +APTOS_REQUESTS_PER_MINUTE=2 ARBITRUM_BASE_URL=https://goerli-rollup.arbitrum.io/rpc -ARBITRUM_REQUESTS_PER_MINUTE=1 +ARBITRUM_REQUESTS_PER_MINUTE=2 AVALANCHE_BASE_URL=https://rpc.ankr.com/avalanche_fuji -AVALANCHE_REQUESTS_PER_MINUTE=1 +AVALANCHE_REQUESTS_PER_MINUTE=2 BSC_BASE_URL=https://data-seed-prebsc-1-s1.binance.org:8545 -BSC_REQUESTS_PER_MINUTE=1 +BSC_REQUESTS_PER_MINUTE=2 CELO_BASE_URL=https://alfajores-forno.celo-testnet.org -CELO_REQUESTS_PER_MINUTE=1 +CELO_REQUESTS_PER_MINUTE=2 ETHEREUM_BASE_URL=https://rpc.ankr.com/eth_goerli -ETHEREUM_REQUESTS_PER_MINUTE=1 +ETHEREUM_REQUESTS_PER_MINUTE=2 FANTOM_BASE_URL=https://rpc.testnet.fantom.network -FANTOM_REQUESTS_PER_MINUTE=1 +FANTOM_REQUESTS_PER_MINUTE=2 INJECTIVE_BASE_URL=https://k8s.testnet.tm.injective.network:443 -INJECTIVE_REQUESTS_PER_MINUTE=4 +INJECTIVE_REQUESTS_PER_MINUTE=2 KARURA_BASE_URL=https://karura-dev.aca-dev.network/eth/http -KARURA_REQUESTS_PER_MINUTE=1 +KARURA_REQUESTS_PER_MINUTE=2 KLAYTN_BASE_URL=https://api.baobab.klaytn.net:8651 -KLAYTN_REQUESTS_PER_MINUTE=1 +KLAYTN_REQUESTS_PER_MINUTE=2 MOONBEAM_BASE_URL=https://rpc.api.moonbase.moonbeam.network -MOONBEAM_REQUESTS_PER_MINUTE=1 +MOONBEAM_REQUESTS_PER_MINUTE=2 OASIS_BASE_URL=https://testnet.emerald.oasis.dev -OASIS_REQUESTS_PER_MINUTE=1 +OASIS_REQUESTS_PER_MINUTE=2 OPTIMISM_BASE_URL=https://goerli.optimism.io -OPTIMISM_REQUESTS_PER_MINUTE=1 +OPTIMISM_REQUESTS_PER_MINUTE=2 POLYGON_BASE_URL=https://rpc.ankr.com/polygon_mumbai -POLYGON_REQUESTS_PER_MINUTE=1 +POLYGON_REQUESTS_PER_MINUTE=2 SOLANA_BASE_URL=https://api.devnet.solana.com -SOLANA_REQUESTS_PER_MINUTE=1 +SOLANA_REQUESTS_PER_MINUTE=2 SUI_BASE_URL=https://fullnode.testnet.sui.io:443 -SUI_REQUESTS_PER_MINUTE=1 +SUI_REQUESTS_PER_MINUTE=2 TERRA_BASE_URL=https://bombay-lcd.terra.dev -TERRA_REQUESTS_PER_MINUTE=1 +TERRA_REQUESTS_PER_MINUTE=2 TERRA2_BASE_URL=https://pisco-lcd.terra.dev -TERRA2_REQUESTS_PER_MINUTE=1 +TERRA2_REQUESTS_PER_MINUTE=2 XPLA_BASE_URL=https://cube-lcd.xpla.dev:443 -XPLA_REQUESTS_PER_MINUTE=1 +XPLA_REQUESTS_PER_MINUTE=2 diff --git a/tx-tracker/.env.example b/tx-tracker/.env.example deleted file mode 100644 index 7568d246..00000000 --- a/tx-tracker/.env.example +++ /dev/null @@ -1,28 +0,0 @@ -MONITORING_PORT= -LOG_LEVEL=INFO -PPROF_ENABLED=false -AWS_ENDPOINT= -AWS_ACCESS_KEY_ID= -AWS_SECRET_ACCESS_KEY= -AWS_REGION= -SQS_URL= -MONGODB_URI=mongodb://127.0.0.1:27017 -MONGODB_DATABASE= -VAA_PAYLOAD_PARSER_URL= -VAA_PAYLOAD_PARSER_TIMEOUT= -NUM_WORKERS=10 -BULK_SIZE=500 - -ANKR_BASE_URL=https://rpc.ankr.com/multichain -ANKR_API_KEY= -ANKR_REQUESTS_PER_MINUTE=30 - -CELO_BASE_URL=https://rpc.ankr.com/celo -CELO_API_KEY= -CELO_REQUESTS_PER_MINUTE=6 - -SOLANA_BASE_URL=https://api.mainnet-beta.solana.com -SOLANA_REQUESTS_PER_MINUTE=8 - -TERRA_BASE_URL=https://lcd.terra.dev -TERRA_REQUESTS_PER_MINUTE=6 \ No newline at end of file diff --git a/tx-tracker/cmd/backfiller/main.go b/tx-tracker/cmd/backfiller/main.go index 9825a9af..3fc5cc82 100644 --- a/tx-tracker/cmd/backfiller/main.go +++ b/tx-tracker/cmd/backfiller/main.go @@ -292,6 +292,7 @@ func consume(ctx context.Context, params *consumerParams) { // 2. Persisting source tx details in the database. v := globalTx.Vaas[0] p := consumer.ProcessSourceTxParams{ + Timestamp: v.Timestamp, VaaId: v.ID, ChainId: v.EmitterChain, Emitter: v.EmitterAddr,