Added docker container for onchain data (#7)

Tiltfile

@@ -15,6 +15,7 @@ config.define_bool("fly", False, "Enable fly component")
 config.define_bool("server", False, "Enable server component")
 config.define_bool("web", False, "Enable web component")
 config.define_bool("web_hot", False, "Enable how web component")
+config.define_bool("onchain-data", False, "Enable onchain_data component")
 
 cfg = config.parse()
 webHost = cfg.get("webHost", "localhost")
@@ -24,7 +25,7 @@ fly = cfg.get("fly", True)
 server = cfg.get("server", True)
 web = cfg.get("web", True)
 web_hot = cfg.get("web_hot", True)
-
+onchain_data = cfg.get("onchain-data", True)
 if mongo:
     k8s_yaml("devnet/mongo-pvc.yaml")
     k8s_yaml("devnet/mongo-pv.yaml")
@@ -107,3 +108,17 @@ if web:
             port_forward(3000, name = "Web [:3000]", host = webHost),
         ]
     )
+
+if onchain_data:
+    docker_build(
+        ref = "onchain-data",
+        context = "onchain_data",
+        dockerfile = "onchain_data/Dockerfile"
+    )
+
+    k8s_yaml("devnet/onchain-data.yaml")
+
+    k8s_resource(
+        "onchain-data",
+        resource_deps = ["mongo"],
+    )

devnet/onchain-data.yaml

@@ -0,0 +1,44 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: onchain-data
+  labels:
+    app: onchain-data
+spec:
+  clusterIP: None
+  selector:
+    app: onchain-data
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: onchain-data
+spec:
+  selector:
+    matchLabels:
+      app: onchain-data
+  serviceName: onchain-data
+  template:
+    metadata:
+      labels:
+        app: onchain-data
+    spec:
+      restartPolicy: Always
+      terminationGracePeriodSeconds: 0
+      containers:
+        - name: onchain-data
+          image: onchain-data:latest
+          env:
+            - name: MONGODB_URI
+              value: mongodb://root:example@mongo:27017/
+            - name: allowlist
+              value: "true"
+          readinessProbe:
+            exec:
+              command:
+                - test
+                - -e
+                - "/app/ready"
+            periodSeconds: 15
+            failureThreshold: 300

onchain_data/Dockerfile

@@ -0,0 +1,12 @@
+FROM node:16-alpine@sha256:004dbac84fed48e20f9888a23e32fa7cf83c2995e174a78d41d9a9dd1e051a20
+
+RUN mkdir -p /app
+WORKDIR /app
+
+COPY package.json package-lock.json ./
+RUN --mount=type=cache,uid=1000,gid=1000,target=/home/node/.npm \
+    npm ci
+COPY . .
+RUN npm run build
+
+ENTRYPOINT node lib/main.js

onchain_data/package.json

@@ -5,12 +5,11 @@
     "@terra-money/terra.js": "^3.1.3",
     "coingecko-api": "^1.0.10",
     "mongodb": "^4.10.0",
-    "ts-node": "^10.9.1"
+    "typescript": "^4.5.2",
+    "serve": "^11.0.0"
   },
   "scripts": {
     "build": "tsc"
   },
-  "devDependencies": {
-    "typescript": "^4.8.4"
-  }
+  "devDependencies": {}
 }

onchain_data/src/allowList.json

@@ -126,7 +126,6 @@
         "0x4988a896b1227218e4a686fde5eabdcabd91571f": "tether",
         "0x5183e1b1091804bc2602586919e6880ac1cf2896": "usn",
         "0xc9bdeed33cd01541e1eed10f90519d2c06fe3feb": "weth",
-        "0xC9BdeEd33CD01541e1eeD10f90519d2C06Fe3feB": "ethereum",
         "0xc4bdd27c33ec7daa6fcfd8532ddb524bf4038096": "wrapped-terra"
     },
     "10": {

onchain_data/src/getCustodyData.ts

@@ -15,6 +15,7 @@ interface Token {
 
 interface CustodyInfo {
   _id: string;
+  updatedAt: string;
   chainName: string;
   chainId: number;
   emitterAddress: string;
@@ -24,6 +25,9 @@ interface CustodyInfo {
 
 async function updateTable(chainInfo, client: MongoClient) {
   const custodyList = chainInfo.balances;
+  if (custodyList.length === 0) {
+    return;
+  }
   try {
     const totalCustodyUSD = custodyList
       .map((x) => x.tokenBalanceUSD)
@@ -40,6 +44,7 @@ async function updateTable(chainInfo, client: MongoClient) {
       { _id: `${chainId}/${emitterAddress}` },
       {
         $set: {
+          updatedAt: new Date().toISOString(),
           chainName: chainInfo.name,
           chainId: chainId,
           emitterAddress: emitterAddress,
@@ -59,7 +64,7 @@ async function updateTable(chainInfo, client: MongoClient) {
 
 const useAllowListstr = process.env.allowlist || "false";
 
-(async () => {
+export async function getCustodyData() {
   const uri = process.env.MONGODB_URI;
   if (uri === "" || uri === undefined) {
     console.log("No mongodb uri supplied");
@@ -102,4 +107,6 @@ const useAllowListstr = process.env.allowlist || "false";
   } finally {
     await client.close();
   }
-})();
+}
+
+export default getCustodyData;

onchain_data/src/getTerraCustody.ts

@@ -261,22 +261,22 @@ async function getTokenValues(chainInfo, tokenInfos: any[], useAllowList) {
 
     // filter list by those with coin gecko prices
     const filteredBalances = custodyFiltered.filter((x) =>
-      Object.keys(tokenPrices).includes(x.tokenAddress)
+      Object.keys(tokenPrices).includes(x?.tokenAddress)
     );
     // calculate usd balances. add price and usd balance to tokenInfos
     const balancesUSD = filteredBalances.map((tokenInfo) => ({
       ...tokenInfo,
-      tokenPrice: tokenPrices[tokenInfo.tokenAddress]["usd"],
+      tokenPrice: tokenPrices[tokenInfo?.tokenAddress]["usd"],
       tokenBalanceUSD:
-        tokenInfo.qty * tokenPrices[tokenInfo.tokenAddress]["usd"],
+        tokenInfo.qty * tokenPrices[tokenInfo?.tokenAddress]["usd"],
     }));
 
     // filter out disallowlist addresses
     const balancesUSDFiltered = balancesUSD.filter(
-      (x) => !DISALLOWLISTED_ADDRESSES.includes(x.tokenAddress)
+      (x) => !DISALLOWLISTED_ADDRESSES.includes(x?.tokenAddress)
     );
     const sorted = balancesUSDFiltered.sort((a, b) =>
-      a.tokenBalanceUSD < b.tokenBalanceUSD ? 1 : -1
+      a?.tokenBalanceUSD < b?.tokenBalanceUSD ? 1 : -1
     );
 
     return sorted;
@@ -288,26 +288,36 @@ async function getTokenValues(chainInfo, tokenInfos: any[], useAllowList) {
 export async function getTerraCustody(chainInfo, useAllowList) {
   const tokenAccounts = await getTerraTokenAccounts(chainInfo, useAllowList);
   console.log(
-    `Num of ${chainInfo.platform} token accounts=${tokenAccounts.length}`
-  );
-  const custody = await getTokenValues(chainInfo, tokenAccounts, useAllowList);
-  console.log(
-    `Num of filtered ${chainInfo.platform} token accounts=${custody.length}`
+    `Num of ${chainInfo?.platform} token accounts=${tokenAccounts?.length}`
   );
+  let custody = undefined;
+  try {
+    custody = await getTokenValues(chainInfo, tokenAccounts, useAllowList);
+    console.log(
+      `Num of filtered ${chainInfo?.platform} token accounts=${custody?.length}`
+    );
+  } catch (e) {
+    console.log("could not fetch terra prices");
+  }
   return custody;
 }
 
 export async function grabTerraCustodyData(chain, useAllowList) {
   const chainInfo = CHAIN_INFO_MAP[chain];
   const balances = await getTerraCustody(chainInfo, useAllowList);
-  const chainInfo_ = {
-    ...chainInfo,
-    emitter_address: await getEmitterAddressTerra(
-      chainInfo.token_bridge_address
-    ),
-    balances: balances,
-  };
-  return chainInfo_;
+  if (balances === undefined) {
+    console.log("could not pull terra balances");
+    return { balances: [] };
+  } else {
+    const chainInfo_ = {
+      ...chainInfo,
+      emitter_address: await getEmitterAddressTerra(
+        chainInfo.token_bridge_address
+      ),
+      balances: balances,
+    };
+    return chainInfo_;
+  }
 }
 
 // const chain = process.env.chain;

onchain_data/src/main.ts

@@ -0,0 +1,30 @@
+import getCustodyData from "./getCustodyData";
+import { open } from "fs/promises";
+let retryTimeout = 5 * 60 * 1000;
+if (process.env.RETRY_TIMEOUT) {
+  try {
+    retryTimeout = parseInt(process.env.RETRY_TIMEOUT);
+  } catch (e) {
+    console.log(
+      `could not parseInt ${process.env.RETRY_TIMEOUT}. Using default timeout=${retryTimeout}`
+    );
+  }
+}
+const filename = "/app/ready";
+let firstRun = true;
+
+async function main() {
+  while (true) {
+    console.log(`${new Date().toISOString()} - fetching custody data`);
+    await getCustodyData();
+    if (firstRun) {
+      let fh = await open(filename, "a");
+      await fh.close();
+      firstRun = false;
+    }
+    console.log(`${new Date().toISOString()} - sleeping for ${retryTimeout}`);
+    await new Promise((resolve) => setTimeout(resolve, Number(retryTimeout)));
+  }
+}
+
+main();

onchain_data/tsconfig.json

@@ -4,6 +4,7 @@
     "target": "es5",
     "module": "commonjs",
     "moduleResolution": "node",
+    "esModuleInterop": true,
     "lib": ["es2020"],
     "skipLibCheck": true,
     "allowJs": true,

web/src/components/Custody.tsx

@@ -61,6 +61,11 @@ const columns = [
       return `${value.length} Token` + (value.length == 1 ? "" : `s`);
     },
   }),
+  columnHelper.accessor("updatedAt", {
+    header: () => "Last Updated",
+    cell: (info) =>
+      info.getValue() ? new Date(info.getValue()).toLocaleString() : null,
+  }),
 ];
 
 /*

web/src/hooks/useCustodyData.ts

@@ -16,6 +16,7 @@ export type Token = {
 
 export type CustodyDataResponse = {
   _id: string;
+  updatedAt: string;
   chainId: number;
   chainName: string;
   custodyUSD: number;