From 57b478cf665c0c6cb2b0489f6f03688149abd62f Mon Sep 17 00:00:00 2001 From: ckeun Date: Fri, 28 Oct 2022 15:44:10 -0500 Subject: [PATCH] Added docker container for onchain data (#7) --- Tiltfile | 17 ++++++++++- devnet/onchain-data.yaml | 44 +++++++++++++++++++++++++++ onchain_data/Dockerfile | 12 ++++++++ onchain_data/package.json | 7 ++--- onchain_data/src/allowList.json | 1 - onchain_data/src/getCustodyData.ts | 11 +++++-- onchain_data/src/getTerraCustody.ts | 46 ++++++++++++++++++----------- onchain_data/src/main.ts | 30 +++++++++++++++++++ onchain_data/tsconfig.json | 1 + web/src/components/Custody.tsx | 5 ++++ web/src/hooks/useCustodyData.ts | 1 + 11 files changed, 149 insertions(+), 26 deletions(-) create mode 100644 devnet/onchain-data.yaml create mode 100644 onchain_data/Dockerfile create mode 100644 onchain_data/src/main.ts diff --git a/Tiltfile b/Tiltfile index a4619368..ad86e0d9 100644 --- a/Tiltfile +++ b/Tiltfile @@ -15,6 +15,7 @@ config.define_bool("fly", False, "Enable fly component") config.define_bool("server", False, "Enable server component") config.define_bool("web", False, "Enable web component") config.define_bool("web_hot", False, "Enable how web component") +config.define_bool("onchain-data", False, "Enable onchain_data component") cfg = config.parse() webHost = cfg.get("webHost", "localhost") @@ -24,7 +25,7 @@ fly = cfg.get("fly", True) server = cfg.get("server", True) web = cfg.get("web", True) web_hot = cfg.get("web_hot", True) - +onchain_data = cfg.get("onchain-data", True) if mongo: k8s_yaml("devnet/mongo-pvc.yaml") k8s_yaml("devnet/mongo-pv.yaml") @@ -107,3 +108,17 @@ if web: port_forward(3000, name = "Web [:3000]", host = webHost), ] ) + +if onchain_data: + docker_build( + ref = "onchain-data", + context = "onchain_data", + dockerfile = "onchain_data/Dockerfile" + ) + + k8s_yaml("devnet/onchain-data.yaml") + + k8s_resource( + "onchain-data", + resource_deps = ["mongo"], + ) diff --git a/devnet/onchain-data.yaml b/devnet/onchain-data.yaml new file mode 100644 index 00000000..635a8406 --- /dev/null +++ b/devnet/onchain-data.yaml @@ -0,0 +1,44 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: onchain-data + labels: + app: onchain-data +spec: + clusterIP: None + selector: + app: onchain-data +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: onchain-data +spec: + selector: + matchLabels: + app: onchain-data + serviceName: onchain-data + template: + metadata: + labels: + app: onchain-data + spec: + restartPolicy: Always + terminationGracePeriodSeconds: 0 + containers: + - name: onchain-data + image: onchain-data:latest + env: + - name: MONGODB_URI + value: mongodb://root:example@mongo:27017/ + - name: allowlist + value: "true" + readinessProbe: + exec: + command: + - test + - -e + - "/app/ready" + periodSeconds: 15 + failureThreshold: 300 diff --git a/onchain_data/Dockerfile b/onchain_data/Dockerfile new file mode 100644 index 00000000..759a9686 --- /dev/null +++ b/onchain_data/Dockerfile @@ -0,0 +1,12 @@ +FROM node:16-alpine@sha256:004dbac84fed48e20f9888a23e32fa7cf83c2995e174a78d41d9a9dd1e051a20 + +RUN mkdir -p /app +WORKDIR /app + +COPY package.json package-lock.json ./ +RUN --mount=type=cache,uid=1000,gid=1000,target=/home/node/.npm \ + npm ci +COPY . . +RUN npm run build + +ENTRYPOINT node lib/main.js diff --git a/onchain_data/package.json b/onchain_data/package.json index 26531537..4cf394fb 100644 --- a/onchain_data/package.json +++ b/onchain_data/package.json @@ -5,12 +5,11 @@ "@terra-money/terra.js": "^3.1.3", "coingecko-api": "^1.0.10", "mongodb": "^4.10.0", - "ts-node": "^10.9.1" + "typescript": "^4.5.2", + "serve": "^11.0.0" }, "scripts": { "build": "tsc" }, - "devDependencies": { - "typescript": "^4.8.4" - } + "devDependencies": {} } diff --git a/onchain_data/src/allowList.json b/onchain_data/src/allowList.json index a75490ac..79cd8a78 100644 --- a/onchain_data/src/allowList.json +++ b/onchain_data/src/allowList.json @@ -126,7 +126,6 @@ "0x4988a896b1227218e4a686fde5eabdcabd91571f": "tether", "0x5183e1b1091804bc2602586919e6880ac1cf2896": "usn", "0xc9bdeed33cd01541e1eed10f90519d2c06fe3feb": "weth", - "0xC9BdeEd33CD01541e1eeD10f90519d2C06Fe3feB": "ethereum", "0xc4bdd27c33ec7daa6fcfd8532ddb524bf4038096": "wrapped-terra" }, "10": { diff --git a/onchain_data/src/getCustodyData.ts b/onchain_data/src/getCustodyData.ts index f4d6cea3..4db0dc3d 100644 --- a/onchain_data/src/getCustodyData.ts +++ b/onchain_data/src/getCustodyData.ts @@ -15,6 +15,7 @@ interface Token { interface CustodyInfo { _id: string; + updatedAt: string; chainName: string; chainId: number; emitterAddress: string; @@ -24,6 +25,9 @@ interface CustodyInfo { async function updateTable(chainInfo, client: MongoClient) { const custodyList = chainInfo.balances; + if (custodyList.length === 0) { + return; + } try { const totalCustodyUSD = custodyList .map((x) => x.tokenBalanceUSD) @@ -40,6 +44,7 @@ async function updateTable(chainInfo, client: MongoClient) { { _id: `${chainId}/${emitterAddress}` }, { $set: { + updatedAt: new Date().toISOString(), chainName: chainInfo.name, chainId: chainId, emitterAddress: emitterAddress, @@ -59,7 +64,7 @@ async function updateTable(chainInfo, client: MongoClient) { const useAllowListstr = process.env.allowlist || "false"; -(async () => { +export async function getCustodyData() { const uri = process.env.MONGODB_URI; if (uri === "" || uri === undefined) { console.log("No mongodb uri supplied"); @@ -102,4 +107,6 @@ const useAllowListstr = process.env.allowlist || "false"; } finally { await client.close(); } -})(); +} + +export default getCustodyData; diff --git a/onchain_data/src/getTerraCustody.ts b/onchain_data/src/getTerraCustody.ts index 2be8d50e..c10bacc6 100644 --- a/onchain_data/src/getTerraCustody.ts +++ b/onchain_data/src/getTerraCustody.ts @@ -261,22 +261,22 @@ async function getTokenValues(chainInfo, tokenInfos: any[], useAllowList) { // filter list by those with coin gecko prices const filteredBalances = custodyFiltered.filter((x) => - Object.keys(tokenPrices).includes(x.tokenAddress) + Object.keys(tokenPrices).includes(x?.tokenAddress) ); // calculate usd balances. add price and usd balance to tokenInfos const balancesUSD = filteredBalances.map((tokenInfo) => ({ ...tokenInfo, - tokenPrice: tokenPrices[tokenInfo.tokenAddress]["usd"], + tokenPrice: tokenPrices[tokenInfo?.tokenAddress]["usd"], tokenBalanceUSD: - tokenInfo.qty * tokenPrices[tokenInfo.tokenAddress]["usd"], + tokenInfo.qty * tokenPrices[tokenInfo?.tokenAddress]["usd"], })); // filter out disallowlist addresses const balancesUSDFiltered = balancesUSD.filter( - (x) => !DISALLOWLISTED_ADDRESSES.includes(x.tokenAddress) + (x) => !DISALLOWLISTED_ADDRESSES.includes(x?.tokenAddress) ); const sorted = balancesUSDFiltered.sort((a, b) => - a.tokenBalanceUSD < b.tokenBalanceUSD ? 1 : -1 + a?.tokenBalanceUSD < b?.tokenBalanceUSD ? 1 : -1 ); return sorted; @@ -288,26 +288,36 @@ async function getTokenValues(chainInfo, tokenInfos: any[], useAllowList) { export async function getTerraCustody(chainInfo, useAllowList) { const tokenAccounts = await getTerraTokenAccounts(chainInfo, useAllowList); console.log( - `Num of ${chainInfo.platform} token accounts=${tokenAccounts.length}` - ); - const custody = await getTokenValues(chainInfo, tokenAccounts, useAllowList); - console.log( - `Num of filtered ${chainInfo.platform} token accounts=${custody.length}` + `Num of ${chainInfo?.platform} token accounts=${tokenAccounts?.length}` ); + let custody = undefined; + try { + custody = await getTokenValues(chainInfo, tokenAccounts, useAllowList); + console.log( + `Num of filtered ${chainInfo?.platform} token accounts=${custody?.length}` + ); + } catch (e) { + console.log("could not fetch terra prices"); + } return custody; } export async function grabTerraCustodyData(chain, useAllowList) { const chainInfo = CHAIN_INFO_MAP[chain]; const balances = await getTerraCustody(chainInfo, useAllowList); - const chainInfo_ = { - ...chainInfo, - emitter_address: await getEmitterAddressTerra( - chainInfo.token_bridge_address - ), - balances: balances, - }; - return chainInfo_; + if (balances === undefined) { + console.log("could not pull terra balances"); + return { balances: [] }; + } else { + const chainInfo_ = { + ...chainInfo, + emitter_address: await getEmitterAddressTerra( + chainInfo.token_bridge_address + ), + balances: balances, + }; + return chainInfo_; + } } // const chain = process.env.chain; diff --git a/onchain_data/src/main.ts b/onchain_data/src/main.ts new file mode 100644 index 00000000..2a7780a3 --- /dev/null +++ b/onchain_data/src/main.ts @@ -0,0 +1,30 @@ +import getCustodyData from "./getCustodyData"; +import { open } from "fs/promises"; +let retryTimeout = 5 * 60 * 1000; +if (process.env.RETRY_TIMEOUT) { + try { + retryTimeout = parseInt(process.env.RETRY_TIMEOUT); + } catch (e) { + console.log( + `could not parseInt ${process.env.RETRY_TIMEOUT}. Using default timeout=${retryTimeout}` + ); + } +} +const filename = "/app/ready"; +let firstRun = true; + +async function main() { + while (true) { + console.log(`${new Date().toISOString()} - fetching custody data`); + await getCustodyData(); + if (firstRun) { + let fh = await open(filename, "a"); + await fh.close(); + firstRun = false; + } + console.log(`${new Date().toISOString()} - sleeping for ${retryTimeout}`); + await new Promise((resolve) => setTimeout(resolve, Number(retryTimeout))); + } +} + +main(); diff --git a/onchain_data/tsconfig.json b/onchain_data/tsconfig.json index c3b270d0..017035d9 100644 --- a/onchain_data/tsconfig.json +++ b/onchain_data/tsconfig.json @@ -4,6 +4,7 @@ "target": "es5", "module": "commonjs", "moduleResolution": "node", + "esModuleInterop": true, "lib": ["es2020"], "skipLibCheck": true, "allowJs": true, diff --git a/web/src/components/Custody.tsx b/web/src/components/Custody.tsx index 15ddad0c..6084a1e2 100644 --- a/web/src/components/Custody.tsx +++ b/web/src/components/Custody.tsx @@ -61,6 +61,11 @@ const columns = [ return `${value.length} Token` + (value.length == 1 ? "" : `s`); }, }), + columnHelper.accessor("updatedAt", { + header: () => "Last Updated", + cell: (info) => + info.getValue() ? new Date(info.getValue()).toLocaleString() : null, + }), ]; /* diff --git a/web/src/hooks/useCustodyData.ts b/web/src/hooks/useCustodyData.ts index bee03027..a47050ef 100644 --- a/web/src/hooks/useCustodyData.ts +++ b/web/src/hooks/useCustodyData.ts @@ -16,6 +16,7 @@ export type Token = { export type CustodyDataResponse = { _id: string; + updatedAt: string; chainId: number; chainName: string; custodyUSD: number;