Added docker container for onchain data (#7)
This commit is contained in:
parent
cc01fb6456
commit
57b478cf66
17
Tiltfile
17
Tiltfile
|
@ -15,6 +15,7 @@ config.define_bool("fly", False, "Enable fly component")
|
||||||
config.define_bool("server", False, "Enable server component")
|
config.define_bool("server", False, "Enable server component")
|
||||||
config.define_bool("web", False, "Enable web component")
|
config.define_bool("web", False, "Enable web component")
|
||||||
config.define_bool("web_hot", False, "Enable how web component")
|
config.define_bool("web_hot", False, "Enable how web component")
|
||||||
|
config.define_bool("onchain-data", False, "Enable onchain_data component")
|
||||||
|
|
||||||
cfg = config.parse()
|
cfg = config.parse()
|
||||||
webHost = cfg.get("webHost", "localhost")
|
webHost = cfg.get("webHost", "localhost")
|
||||||
|
@ -24,7 +25,7 @@ fly = cfg.get("fly", True)
|
||||||
server = cfg.get("server", True)
|
server = cfg.get("server", True)
|
||||||
web = cfg.get("web", True)
|
web = cfg.get("web", True)
|
||||||
web_hot = cfg.get("web_hot", True)
|
web_hot = cfg.get("web_hot", True)
|
||||||
|
onchain_data = cfg.get("onchain-data", True)
|
||||||
if mongo:
|
if mongo:
|
||||||
k8s_yaml("devnet/mongo-pvc.yaml")
|
k8s_yaml("devnet/mongo-pvc.yaml")
|
||||||
k8s_yaml("devnet/mongo-pv.yaml")
|
k8s_yaml("devnet/mongo-pv.yaml")
|
||||||
|
@ -107,3 +108,17 @@ if web:
|
||||||
port_forward(3000, name = "Web [:3000]", host = webHost),
|
port_forward(3000, name = "Web [:3000]", host = webHost),
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if onchain_data:
|
||||||
|
docker_build(
|
||||||
|
ref = "onchain-data",
|
||||||
|
context = "onchain_data",
|
||||||
|
dockerfile = "onchain_data/Dockerfile"
|
||||||
|
)
|
||||||
|
|
||||||
|
k8s_yaml("devnet/onchain-data.yaml")
|
||||||
|
|
||||||
|
k8s_resource(
|
||||||
|
"onchain-data",
|
||||||
|
resource_deps = ["mongo"],
|
||||||
|
)
|
||||||
|
|
|
@ -0,0 +1,44 @@
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: onchain-data
|
||||||
|
labels:
|
||||||
|
app: onchain-data
|
||||||
|
spec:
|
||||||
|
clusterIP: None
|
||||||
|
selector:
|
||||||
|
app: onchain-data
|
||||||
|
---
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: StatefulSet
|
||||||
|
metadata:
|
||||||
|
name: onchain-data
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: onchain-data
|
||||||
|
serviceName: onchain-data
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: onchain-data
|
||||||
|
spec:
|
||||||
|
restartPolicy: Always
|
||||||
|
terminationGracePeriodSeconds: 0
|
||||||
|
containers:
|
||||||
|
- name: onchain-data
|
||||||
|
image: onchain-data:latest
|
||||||
|
env:
|
||||||
|
- name: MONGODB_URI
|
||||||
|
value: mongodb://root:example@mongo:27017/
|
||||||
|
- name: allowlist
|
||||||
|
value: "true"
|
||||||
|
readinessProbe:
|
||||||
|
exec:
|
||||||
|
command:
|
||||||
|
- test
|
||||||
|
- -e
|
||||||
|
- "/app/ready"
|
||||||
|
periodSeconds: 15
|
||||||
|
failureThreshold: 300
|
|
@ -0,0 +1,12 @@
|
||||||
|
FROM node:16-alpine@sha256:004dbac84fed48e20f9888a23e32fa7cf83c2995e174a78d41d9a9dd1e051a20
|
||||||
|
|
||||||
|
RUN mkdir -p /app
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
COPY package.json package-lock.json ./
|
||||||
|
RUN --mount=type=cache,uid=1000,gid=1000,target=/home/node/.npm \
|
||||||
|
npm ci
|
||||||
|
COPY . .
|
||||||
|
RUN npm run build
|
||||||
|
|
||||||
|
ENTRYPOINT node lib/main.js
|
|
@ -5,12 +5,11 @@
|
||||||
"@terra-money/terra.js": "^3.1.3",
|
"@terra-money/terra.js": "^3.1.3",
|
||||||
"coingecko-api": "^1.0.10",
|
"coingecko-api": "^1.0.10",
|
||||||
"mongodb": "^4.10.0",
|
"mongodb": "^4.10.0",
|
||||||
"ts-node": "^10.9.1"
|
"typescript": "^4.5.2",
|
||||||
|
"serve": "^11.0.0"
|
||||||
},
|
},
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"build": "tsc"
|
"build": "tsc"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {}
|
||||||
"typescript": "^4.8.4"
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -126,7 +126,6 @@
|
||||||
"0x4988a896b1227218e4a686fde5eabdcabd91571f": "tether",
|
"0x4988a896b1227218e4a686fde5eabdcabd91571f": "tether",
|
||||||
"0x5183e1b1091804bc2602586919e6880ac1cf2896": "usn",
|
"0x5183e1b1091804bc2602586919e6880ac1cf2896": "usn",
|
||||||
"0xc9bdeed33cd01541e1eed10f90519d2c06fe3feb": "weth",
|
"0xc9bdeed33cd01541e1eed10f90519d2c06fe3feb": "weth",
|
||||||
"0xC9BdeEd33CD01541e1eeD10f90519d2C06Fe3feB": "ethereum",
|
|
||||||
"0xc4bdd27c33ec7daa6fcfd8532ddb524bf4038096": "wrapped-terra"
|
"0xc4bdd27c33ec7daa6fcfd8532ddb524bf4038096": "wrapped-terra"
|
||||||
},
|
},
|
||||||
"10": {
|
"10": {
|
||||||
|
|
|
@ -15,6 +15,7 @@ interface Token {
|
||||||
|
|
||||||
interface CustodyInfo {
|
interface CustodyInfo {
|
||||||
_id: string;
|
_id: string;
|
||||||
|
updatedAt: string;
|
||||||
chainName: string;
|
chainName: string;
|
||||||
chainId: number;
|
chainId: number;
|
||||||
emitterAddress: string;
|
emitterAddress: string;
|
||||||
|
@ -24,6 +25,9 @@ interface CustodyInfo {
|
||||||
|
|
||||||
async function updateTable(chainInfo, client: MongoClient) {
|
async function updateTable(chainInfo, client: MongoClient) {
|
||||||
const custodyList = chainInfo.balances;
|
const custodyList = chainInfo.balances;
|
||||||
|
if (custodyList.length === 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
try {
|
try {
|
||||||
const totalCustodyUSD = custodyList
|
const totalCustodyUSD = custodyList
|
||||||
.map((x) => x.tokenBalanceUSD)
|
.map((x) => x.tokenBalanceUSD)
|
||||||
|
@ -40,6 +44,7 @@ async function updateTable(chainInfo, client: MongoClient) {
|
||||||
{ _id: `${chainId}/${emitterAddress}` },
|
{ _id: `${chainId}/${emitterAddress}` },
|
||||||
{
|
{
|
||||||
$set: {
|
$set: {
|
||||||
|
updatedAt: new Date().toISOString(),
|
||||||
chainName: chainInfo.name,
|
chainName: chainInfo.name,
|
||||||
chainId: chainId,
|
chainId: chainId,
|
||||||
emitterAddress: emitterAddress,
|
emitterAddress: emitterAddress,
|
||||||
|
@ -59,7 +64,7 @@ async function updateTable(chainInfo, client: MongoClient) {
|
||||||
|
|
||||||
const useAllowListstr = process.env.allowlist || "false";
|
const useAllowListstr = process.env.allowlist || "false";
|
||||||
|
|
||||||
(async () => {
|
export async function getCustodyData() {
|
||||||
const uri = process.env.MONGODB_URI;
|
const uri = process.env.MONGODB_URI;
|
||||||
if (uri === "" || uri === undefined) {
|
if (uri === "" || uri === undefined) {
|
||||||
console.log("No mongodb uri supplied");
|
console.log("No mongodb uri supplied");
|
||||||
|
@ -102,4 +107,6 @@ const useAllowListstr = process.env.allowlist || "false";
|
||||||
} finally {
|
} finally {
|
||||||
await client.close();
|
await client.close();
|
||||||
}
|
}
|
||||||
})();
|
}
|
||||||
|
|
||||||
|
export default getCustodyData;
|
||||||
|
|
|
@ -261,22 +261,22 @@ async function getTokenValues(chainInfo, tokenInfos: any[], useAllowList) {
|
||||||
|
|
||||||
// filter list by those with coin gecko prices
|
// filter list by those with coin gecko prices
|
||||||
const filteredBalances = custodyFiltered.filter((x) =>
|
const filteredBalances = custodyFiltered.filter((x) =>
|
||||||
Object.keys(tokenPrices).includes(x.tokenAddress)
|
Object.keys(tokenPrices).includes(x?.tokenAddress)
|
||||||
);
|
);
|
||||||
// calculate usd balances. add price and usd balance to tokenInfos
|
// calculate usd balances. add price and usd balance to tokenInfos
|
||||||
const balancesUSD = filteredBalances.map((tokenInfo) => ({
|
const balancesUSD = filteredBalances.map((tokenInfo) => ({
|
||||||
...tokenInfo,
|
...tokenInfo,
|
||||||
tokenPrice: tokenPrices[tokenInfo.tokenAddress]["usd"],
|
tokenPrice: tokenPrices[tokenInfo?.tokenAddress]["usd"],
|
||||||
tokenBalanceUSD:
|
tokenBalanceUSD:
|
||||||
tokenInfo.qty * tokenPrices[tokenInfo.tokenAddress]["usd"],
|
tokenInfo.qty * tokenPrices[tokenInfo?.tokenAddress]["usd"],
|
||||||
}));
|
}));
|
||||||
|
|
||||||
// filter out disallowlist addresses
|
// filter out disallowlist addresses
|
||||||
const balancesUSDFiltered = balancesUSD.filter(
|
const balancesUSDFiltered = balancesUSD.filter(
|
||||||
(x) => !DISALLOWLISTED_ADDRESSES.includes(x.tokenAddress)
|
(x) => !DISALLOWLISTED_ADDRESSES.includes(x?.tokenAddress)
|
||||||
);
|
);
|
||||||
const sorted = balancesUSDFiltered.sort((a, b) =>
|
const sorted = balancesUSDFiltered.sort((a, b) =>
|
||||||
a.tokenBalanceUSD < b.tokenBalanceUSD ? 1 : -1
|
a?.tokenBalanceUSD < b?.tokenBalanceUSD ? 1 : -1
|
||||||
);
|
);
|
||||||
|
|
||||||
return sorted;
|
return sorted;
|
||||||
|
@ -288,26 +288,36 @@ async function getTokenValues(chainInfo, tokenInfos: any[], useAllowList) {
|
||||||
export async function getTerraCustody(chainInfo, useAllowList) {
|
export async function getTerraCustody(chainInfo, useAllowList) {
|
||||||
const tokenAccounts = await getTerraTokenAccounts(chainInfo, useAllowList);
|
const tokenAccounts = await getTerraTokenAccounts(chainInfo, useAllowList);
|
||||||
console.log(
|
console.log(
|
||||||
`Num of ${chainInfo.platform} token accounts=${tokenAccounts.length}`
|
`Num of ${chainInfo?.platform} token accounts=${tokenAccounts?.length}`
|
||||||
);
|
|
||||||
const custody = await getTokenValues(chainInfo, tokenAccounts, useAllowList);
|
|
||||||
console.log(
|
|
||||||
`Num of filtered ${chainInfo.platform} token accounts=${custody.length}`
|
|
||||||
);
|
);
|
||||||
|
let custody = undefined;
|
||||||
|
try {
|
||||||
|
custody = await getTokenValues(chainInfo, tokenAccounts, useAllowList);
|
||||||
|
console.log(
|
||||||
|
`Num of filtered ${chainInfo?.platform} token accounts=${custody?.length}`
|
||||||
|
);
|
||||||
|
} catch (e) {
|
||||||
|
console.log("could not fetch terra prices");
|
||||||
|
}
|
||||||
return custody;
|
return custody;
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function grabTerraCustodyData(chain, useAllowList) {
|
export async function grabTerraCustodyData(chain, useAllowList) {
|
||||||
const chainInfo = CHAIN_INFO_MAP[chain];
|
const chainInfo = CHAIN_INFO_MAP[chain];
|
||||||
const balances = await getTerraCustody(chainInfo, useAllowList);
|
const balances = await getTerraCustody(chainInfo, useAllowList);
|
||||||
const chainInfo_ = {
|
if (balances === undefined) {
|
||||||
...chainInfo,
|
console.log("could not pull terra balances");
|
||||||
emitter_address: await getEmitterAddressTerra(
|
return { balances: [] };
|
||||||
chainInfo.token_bridge_address
|
} else {
|
||||||
),
|
const chainInfo_ = {
|
||||||
balances: balances,
|
...chainInfo,
|
||||||
};
|
emitter_address: await getEmitterAddressTerra(
|
||||||
return chainInfo_;
|
chainInfo.token_bridge_address
|
||||||
|
),
|
||||||
|
balances: balances,
|
||||||
|
};
|
||||||
|
return chainInfo_;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// const chain = process.env.chain;
|
// const chain = process.env.chain;
|
||||||
|
|
|
@ -0,0 +1,30 @@
|
||||||
|
import getCustodyData from "./getCustodyData";
|
||||||
|
import { open } from "fs/promises";
|
||||||
|
let retryTimeout = 5 * 60 * 1000;
|
||||||
|
if (process.env.RETRY_TIMEOUT) {
|
||||||
|
try {
|
||||||
|
retryTimeout = parseInt(process.env.RETRY_TIMEOUT);
|
||||||
|
} catch (e) {
|
||||||
|
console.log(
|
||||||
|
`could not parseInt ${process.env.RETRY_TIMEOUT}. Using default timeout=${retryTimeout}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const filename = "/app/ready";
|
||||||
|
let firstRun = true;
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
while (true) {
|
||||||
|
console.log(`${new Date().toISOString()} - fetching custody data`);
|
||||||
|
await getCustodyData();
|
||||||
|
if (firstRun) {
|
||||||
|
let fh = await open(filename, "a");
|
||||||
|
await fh.close();
|
||||||
|
firstRun = false;
|
||||||
|
}
|
||||||
|
console.log(`${new Date().toISOString()} - sleeping for ${retryTimeout}`);
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, Number(retryTimeout)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
main();
|
|
@ -4,6 +4,7 @@
|
||||||
"target": "es5",
|
"target": "es5",
|
||||||
"module": "commonjs",
|
"module": "commonjs",
|
||||||
"moduleResolution": "node",
|
"moduleResolution": "node",
|
||||||
|
"esModuleInterop": true,
|
||||||
"lib": ["es2020"],
|
"lib": ["es2020"],
|
||||||
"skipLibCheck": true,
|
"skipLibCheck": true,
|
||||||
"allowJs": true,
|
"allowJs": true,
|
||||||
|
|
|
@ -61,6 +61,11 @@ const columns = [
|
||||||
return `${value.length} Token` + (value.length == 1 ? "" : `s`);
|
return `${value.length} Token` + (value.length == 1 ? "" : `s`);
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
|
columnHelper.accessor("updatedAt", {
|
||||||
|
header: () => "Last Updated",
|
||||||
|
cell: (info) =>
|
||||||
|
info.getValue() ? new Date(info.getValue()).toLocaleString() : null,
|
||||||
|
}),
|
||||||
];
|
];
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
|
|
@ -16,6 +16,7 @@ export type Token = {
|
||||||
|
|
||||||
export type CustodyDataResponse = {
|
export type CustodyDataResponse = {
|
||||||
_id: string;
|
_id: string;
|
||||||
|
updatedAt: string;
|
||||||
chainId: number;
|
chainId: number;
|
||||||
chainName: string;
|
chainName: string;
|
||||||
custodyUSD: number;
|
custodyUSD: number;
|
||||||
|
|
Loading…
Reference in New Issue