wormhole-explorer/blockchain-watcher/src/infrastructure/repositories/SnsEventRepository.ts

164 lines
4.3 KiB
TypeScript
Raw Normal View History

import { LogFoundEvent } from "../../domain/entities";
import crypto from "node:crypto";
import {
SNSClient,
PublishBatchCommand,
PublishBatchCommandInput,
PublishBatchRequestEntry,
} from "@aws-sdk/client-sns";
import winston from "../log";
const CHUNK_SIZE = 10;
export class SnsEventRepository {
private client: SNSClient;
private cfg: SnsConfig;
private logger: winston.Logger;
constructor(snsClient: SNSClient, cfg: SnsConfig) {
this.client = snsClient;
this.cfg = cfg;
this.logger = winston.child({ module: "SnsEventRepository" });
this.logger.info(`Created for topic ${cfg.topicArn}`);
}
2024-02-14 06:14:25 -08:00
async publish(events: LogFoundEvent<any>[], chain: string): Promise<SnsPublishResult> {
if (!events.length) {
2024-02-14 06:14:25 -08:00
this.logger.debug(`[publish][${chain}] No events to publish, continuing...`);
return {
status: "success",
};
}
const batches: PublishBatchCommandInput[] = [];
const inputs: PublishBatchRequestEntry[] = events
.map(SnsEvent.fromLogFoundEvent)
.map((event) => ({
Id: crypto.randomUUID(),
Subject: this.cfg.subject ?? "blockchain-watcher",
Message: JSON.stringify(event),
MessageGroupId: this.cfg.groupId ?? "blockchain-watcher",
MessageDeduplicationId: event.trackId,
}));
// PublishBatchCommand: only supports max 10 items per batch
for (let i = 0; i < inputs.length; i += CHUNK_SIZE) {
const batch: PublishBatchCommandInput = {
TopicArn: this.cfg.topicArn,
PublishBatchRequestEntries: inputs.slice(i, i + CHUNK_SIZE),
};
batches.push(batch);
}
try {
const promises = [];
const errors = [];
for (const batch of batches) {
const command = new PublishBatchCommand(batch);
promises.push(this.client.send(command));
}
const results = await Promise.allSettled(promises);
for (const result of results) {
if (result.status !== "fulfilled") {
2024-02-14 06:14:25 -08:00
this.logger.error(`[publish][${chain}] ${result.reason}`);
errors.push(result.reason);
}
}
if (errors.length > 0) {
return {
status: "error",
reasons: errors,
};
}
} catch (error: unknown) {
[Blockchain Watcher] Emit LogMessagePublished for Solana (#809) * adding gen-relayer watcher code to branch * folder rename * adding some new abstract types for processor redesign * added first cut of new process manager logic * large refactoring to environment configuration * Add basic default config * prettier run * modifying event handler interface * abstract handler typing changes * Blockchain watcher: adding domain and infra layers (#786) * reorg domain-infra * watch evm blocks action * adding evm log parser * wider prettier * renaming watch action * adding doc * persist latest metadata * gh action for blockchain-watcher * adding log-message-published mapper * deps: remove peers and nodemon * adding handler for LogMessagePublished * added parser for log message published --------- Co-authored-by: chase-45 <chasemoran45@gmail.com> * Blockchain Watcher: ethereum -> sns implementation (#790) * evm block repo implementation * adding sns publisher * adding external config and initial runner * fix: start from latest if no fromBlock configured * feat: add dryRun option * fix: handler filtering and eth_getLogs filter payload * local e2e * actual sns arns for testnet * smaller docker image * deployment changes * [Blockchain Watcher] Apply missing formatting for log message published (#791) * Apply missing formatting for log message published * keep extracting * emitterAddress -> emitter * using configured default winston logger * wait for new block * adding basic metrics * staging mainnet config * send hexa block number * simpler log format * group entities in different files * [Blockchain watcher] Adding light observability (#793) * using configured default winston logger * wait for new block * adding basic metrics * staging mainnet config * send hexa block number * simpler log format * better error logs * wait when no block available * grouping watchers * adding 429 handling http client * abstracting polling iteration * Load jobs dinamically * changing deployment strategy to jobs per pod * group actions per platform * [Blockchain Watcher] Add StartJob action + rate limit handling (#799) * using configured default winston logger * wait for new block * adding basic metrics * staging mainnet config * send hexa block number * simpler log format * group entities in different files * grouping watchers * adding 429 handling http client * abstracting polling iteration * Load jobs dinamically * changing deployment strategy to jobs per pod * adding poll solana txs action * adding solana repo implementation * solana: mapping to logmsgpublished * fixing solana mapper test * updating readme * adding some metrics * adding solana worker * adding finality param to get solana block * minor changes - using confirmed for solana log extraction * updating version * add solana env var * [Blockchain Watcher] Feature-813/fix-coverage-report (#815) * Fix coverage report * Run prettier * Rollback global coverage value * Set global coverage value to 55 * Add github PR template * Run prettier * Rename github template file --------- Co-authored-by: Julian Merlo <julianmerlo@MacBook-Pro-de-Julian.local> * private solana urls as secret * adding prod cfg files * Change repository name and cretae error foulder (#819) * Change repository name and cretae error foulder * Rename SolanaFailure file * Rename SolanaFailure file --------- Co-authored-by: Julian Merlo <julianmerlo@MacBook-Pro-de-Julian.local> * passing node_options for solana * adding missing envs templates * cru * ratelimited solana client + increase coverage * [Blockchain Watcher] Feature 817/add hexagonal prefix (#824) * Put hexadecimal prefix * Create id variable * Add error log * Add error log * Remove bigint in getBlock params --------- Co-authored-by: Julian Merlo <julianmerlo@MacBook-Pro-de-Julian.local> * fix never ending solana jobs * adding job counter metric * solana: add retry handling for 429s * cleanup * remove duplicated tests * cru + item count * minor readme update + do not use private solana rpc * fix items counter * fix solana job programID for devnet * log basic repo info * solana: passing finality everywhere * delete unnecesary files * adjusting solana cfgs * [Blockchain Watcher] Fix/re order some files (#828) * fix/re-order-some-files * run prettier * Run prettier * Resolve comment in PR * Remove import * Resolve comment * Remove prefix in sns log --------- Co-authored-by: Julian Merlo <julianmerlo@MacBook-Pro-de-Julian.local> --------- Co-authored-by: chase-45 <chasemoran45@gmail.com> Co-authored-by: Julian <52217955+julianmerlo95@users.noreply.github.com> Co-authored-by: Julian Merlo <julianmerlo@MacBook-Pro-de-Julian.local>
2023-11-30 07:05:43 -08:00
this.logger.error(`[publish] ${error}`);
return {
status: "error",
};
}
2024-02-14 06:14:25 -08:00
this.logger.info(`[publish][${chain}] Published ${events.length} events to SNS`);
return {
status: "success",
};
}
2024-02-14 06:14:25 -08:00
async asTarget(): Promise<(events: LogFoundEvent<any>[], chain: string) => Promise<void>> {
return async (events: LogFoundEvent<any>[], chain: string) => {
const result = await this.publish(events, chain);
if (result.status === "error") {
[Blockchain Watcher] Emit LogMessagePublished for Solana (#809) * adding gen-relayer watcher code to branch * folder rename * adding some new abstract types for processor redesign * added first cut of new process manager logic * large refactoring to environment configuration * Add basic default config * prettier run * modifying event handler interface * abstract handler typing changes * Blockchain watcher: adding domain and infra layers (#786) * reorg domain-infra * watch evm blocks action * adding evm log parser * wider prettier * renaming watch action * adding doc * persist latest metadata * gh action for blockchain-watcher * adding log-message-published mapper * deps: remove peers and nodemon * adding handler for LogMessagePublished * added parser for log message published --------- Co-authored-by: chase-45 <chasemoran45@gmail.com> * Blockchain Watcher: ethereum -> sns implementation (#790) * evm block repo implementation * adding sns publisher * adding external config and initial runner * fix: start from latest if no fromBlock configured * feat: add dryRun option * fix: handler filtering and eth_getLogs filter payload * local e2e * actual sns arns for testnet * smaller docker image * deployment changes * [Blockchain Watcher] Apply missing formatting for log message published (#791) * Apply missing formatting for log message published * keep extracting * emitterAddress -> emitter * using configured default winston logger * wait for new block * adding basic metrics * staging mainnet config * send hexa block number * simpler log format * group entities in different files * [Blockchain watcher] Adding light observability (#793) * using configured default winston logger * wait for new block * adding basic metrics * staging mainnet config * send hexa block number * simpler log format * better error logs * wait when no block available * grouping watchers * adding 429 handling http client * abstracting polling iteration * Load jobs dinamically * changing deployment strategy to jobs per pod * group actions per platform * [Blockchain Watcher] Add StartJob action + rate limit handling (#799) * using configured default winston logger * wait for new block * adding basic metrics * staging mainnet config * send hexa block number * simpler log format * group entities in different files * grouping watchers * adding 429 handling http client * abstracting polling iteration * Load jobs dinamically * changing deployment strategy to jobs per pod * adding poll solana txs action * adding solana repo implementation * solana: mapping to logmsgpublished * fixing solana mapper test * updating readme * adding some metrics * adding solana worker * adding finality param to get solana block * minor changes - using confirmed for solana log extraction * updating version * add solana env var * [Blockchain Watcher] Feature-813/fix-coverage-report (#815) * Fix coverage report * Run prettier * Rollback global coverage value * Set global coverage value to 55 * Add github PR template * Run prettier * Rename github template file --------- Co-authored-by: Julian Merlo <julianmerlo@MacBook-Pro-de-Julian.local> * private solana urls as secret * adding prod cfg files * Change repository name and cretae error foulder (#819) * Change repository name and cretae error foulder * Rename SolanaFailure file * Rename SolanaFailure file --------- Co-authored-by: Julian Merlo <julianmerlo@MacBook-Pro-de-Julian.local> * passing node_options for solana * adding missing envs templates * cru * ratelimited solana client + increase coverage * [Blockchain Watcher] Feature 817/add hexagonal prefix (#824) * Put hexadecimal prefix * Create id variable * Add error log * Add error log * Remove bigint in getBlock params --------- Co-authored-by: Julian Merlo <julianmerlo@MacBook-Pro-de-Julian.local> * fix never ending solana jobs * adding job counter metric * solana: add retry handling for 429s * cleanup * remove duplicated tests * cru + item count * minor readme update + do not use private solana rpc * fix items counter * fix solana job programID for devnet * log basic repo info * solana: passing finality everywhere * delete unnecesary files * adjusting solana cfgs * [Blockchain Watcher] Fix/re order some files (#828) * fix/re-order-some-files * run prettier * Run prettier * Resolve comment in PR * Remove import * Resolve comment * Remove prefix in sns log --------- Co-authored-by: Julian Merlo <julianmerlo@MacBook-Pro-de-Julian.local> --------- Co-authored-by: chase-45 <chasemoran45@gmail.com> Co-authored-by: Julian <52217955+julianmerlo95@users.noreply.github.com> Co-authored-by: Julian Merlo <julianmerlo@MacBook-Pro-de-Julian.local>
2023-11-30 07:05:43 -08:00
this.logger.error(
2024-02-14 06:14:25 -08:00
`[asTarget][${chain}] Error publishing events to SNS: ${result.reason ?? result.reasons}`
[Blockchain Watcher] Emit LogMessagePublished for Solana (#809) * adding gen-relayer watcher code to branch * folder rename * adding some new abstract types for processor redesign * added first cut of new process manager logic * large refactoring to environment configuration * Add basic default config * prettier run * modifying event handler interface * abstract handler typing changes * Blockchain watcher: adding domain and infra layers (#786) * reorg domain-infra * watch evm blocks action * adding evm log parser * wider prettier * renaming watch action * adding doc * persist latest metadata * gh action for blockchain-watcher * adding log-message-published mapper * deps: remove peers and nodemon * adding handler for LogMessagePublished * added parser for log message published --------- Co-authored-by: chase-45 <chasemoran45@gmail.com> * Blockchain Watcher: ethereum -> sns implementation (#790) * evm block repo implementation * adding sns publisher * adding external config and initial runner * fix: start from latest if no fromBlock configured * feat: add dryRun option * fix: handler filtering and eth_getLogs filter payload * local e2e * actual sns arns for testnet * smaller docker image * deployment changes * [Blockchain Watcher] Apply missing formatting for log message published (#791) * Apply missing formatting for log message published * keep extracting * emitterAddress -> emitter * using configured default winston logger * wait for new block * adding basic metrics * staging mainnet config * send hexa block number * simpler log format * group entities in different files * [Blockchain watcher] Adding light observability (#793) * using configured default winston logger * wait for new block * adding basic metrics * staging mainnet config * send hexa block number * simpler log format * better error logs * wait when no block available * grouping watchers * adding 429 handling http client * abstracting polling iteration * Load jobs dinamically * changing deployment strategy to jobs per pod * group actions per platform * [Blockchain Watcher] Add StartJob action + rate limit handling (#799) * using configured default winston logger * wait for new block * adding basic metrics * staging mainnet config * send hexa block number * simpler log format * group entities in different files * grouping watchers * adding 429 handling http client * abstracting polling iteration * Load jobs dinamically * changing deployment strategy to jobs per pod * adding poll solana txs action * adding solana repo implementation * solana: mapping to logmsgpublished * fixing solana mapper test * updating readme * adding some metrics * adding solana worker * adding finality param to get solana block * minor changes - using confirmed for solana log extraction * updating version * add solana env var * [Blockchain Watcher] Feature-813/fix-coverage-report (#815) * Fix coverage report * Run prettier * Rollback global coverage value * Set global coverage value to 55 * Add github PR template * Run prettier * Rename github template file --------- Co-authored-by: Julian Merlo <julianmerlo@MacBook-Pro-de-Julian.local> * private solana urls as secret * adding prod cfg files * Change repository name and cretae error foulder (#819) * Change repository name and cretae error foulder * Rename SolanaFailure file * Rename SolanaFailure file --------- Co-authored-by: Julian Merlo <julianmerlo@MacBook-Pro-de-Julian.local> * passing node_options for solana * adding missing envs templates * cru * ratelimited solana client + increase coverage * [Blockchain Watcher] Feature 817/add hexagonal prefix (#824) * Put hexadecimal prefix * Create id variable * Add error log * Add error log * Remove bigint in getBlock params --------- Co-authored-by: Julian Merlo <julianmerlo@MacBook-Pro-de-Julian.local> * fix never ending solana jobs * adding job counter metric * solana: add retry handling for 429s * cleanup * remove duplicated tests * cru + item count * minor readme update + do not use private solana rpc * fix items counter * fix solana job programID for devnet * log basic repo info * solana: passing finality everywhere * delete unnecesary files * adjusting solana cfgs * [Blockchain Watcher] Fix/re order some files (#828) * fix/re-order-some-files * run prettier * Run prettier * Resolve comment in PR * Remove import * Resolve comment * Remove prefix in sns log --------- Co-authored-by: Julian Merlo <julianmerlo@MacBook-Pro-de-Julian.local> --------- Co-authored-by: chase-45 <chasemoran45@gmail.com> Co-authored-by: Julian <52217955+julianmerlo95@users.noreply.github.com> Co-authored-by: Julian Merlo <julianmerlo@MacBook-Pro-de-Julian.local>
2023-11-30 07:05:43 -08:00
);
throw new Error(`Error publishing events to SNS: ${result.reason}`);
}
};
}
}
export class SnsEvent {
trackId: string;
source: string;
event: string;
timestamp: string;
version: string;
data: Record<string, any>;
constructor(
trackId: string,
source: string,
event: string,
timestamp: string,
version: string,
data: Record<string, any>
) {
this.trackId = trackId;
this.source = source;
this.event = event;
this.timestamp = timestamp;
this.version = version;
this.data = data;
}
static fromLogFoundEvent<T>(logFoundEvent: LogFoundEvent<T>): SnsEvent {
return new SnsEvent(
`chain-event-${logFoundEvent.txHash}-${logFoundEvent.blockHeight}`,
"blockchain-watcher",
logFoundEvent.name,
new Date().toISOString(),
"1",
{
chainId: logFoundEvent.chainId,
emitter: logFoundEvent.address,
txHash: logFoundEvent.txHash,
blockHeight: logFoundEvent.blockHeight.toString(),
blockTime: new Date(logFoundEvent.blockTime * 1000).toISOString(),
attributes: logFoundEvent.attributes,
}
);
}
}
export type SnsConfig = {
region: string;
topicArn: string;
subject?: string;
groupId: string;
credentials?: {
accessKeyId: string;
secretAccessKey: string;
url: string;
};
};
export type SnsPublishResult = {
status: "success" | "error";
reason?: string;
reasons?: string[];
};