[price-service] add /get_vaa endpoint that serves cache and db vaas (#482)

* add /get_vaa endpoint that serves cache and db vaas

* update env vars

* fix precommit errors

* fix precommit errors

* rename removeExpiredValuesFromVaasCache to runCacheCleanupLoop

* move initialized envOrErr to constructor or func arg

* use setInterval

* update pyth-price-service env vars on tilt devnet

* use undefined instead of null

* use status code

* make web-api an env var

* update env var

* fix precommit

* fix precommit

* fix changes

* remove env vars from tilt-devnet

* address changes

* fix lint issues

* fix linting issues again

* bump package version

* remove unused imports

third_party/pyth/price-service/.env.sample

@@ -17,3 +17,6 @@ PROM_PORT=8081
 
 # The default is to log with level info.
 #LOG_LEVEL=debug
+
+REMOVE_EXPIRED_VALUES_INTERVAL_SECONDS=60
+CACHE_TTL_SECONDS=300

third_party/pyth/price-service/docker-compose.mainnet.yaml

@@ -37,6 +37,10 @@ services:
       READINESS_SPY_SYNC_TIME_SECONDS: "20"
       READINESS_NUM_LOADED_SYMBOLS: "50"
       LOG_LEVEL: warning
+      DB_API_CLUSTER: pythnet
+      REMOVE_EXPIRED_VALUES_INTERVAL_SECONDS: "60"
+      CACHE_TTL_SECONDS: "300"
+      DB_API_ENDPOINT: "https://web-api.pyth.network"
     healthcheck:
       test:
         [

third_party/pyth/price-service/docker-compose.testnet.yaml

@@ -37,6 +37,10 @@ services:
       READINESS_SPY_SYNC_TIME_SECONDS: "20"
       READINESS_NUM_LOADED_SYMBOLS: "50"
       LOG_LEVEL: warning
+      DB_API_CLUSTER: devnet
+      REMOVE_EXPIRED_VALUES_INTERVAL_SECONDS: "60"
+      CACHE_TTL_SECONDS: "300"
+      DB_API_ENDPOINT: "https://web-api.pyth.network"
     healthcheck:
       test:
         [

third_party/pyth/price-service/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@pythnetwork/pyth-price-service",
-  "version": "2.2.4",
+  "version": "2.3.0",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "@pythnetwork/pyth-price-service",
-      "version": "2.2.4",
+      "version": "2.3.0",
       "license": "Apache-2.0",
       "dependencies": {
         "@certusone/wormhole-sdk": "^0.1.4",
@@ -27,6 +27,7 @@
         "joi": "^17.6.0",
         "lru-cache": "^7.14.1",
         "morgan": "^1.10.0",
+        "node-fetch": "^2.6.1",
         "prom-client": "^14.0.1",
         "response-time": "^2.3.2",
         "winston": "^3.3.3",
@@ -36,6 +37,7 @@
         "@types/jest": "^27.5.0",
         "@types/long": "^4.0.1",
         "@types/node": "^16.6.1",
+        "@types/node-fetch": "^2.6.2",
         "@types/supertest": "^2.0.12",
         "jest": "^28.0.3",
         "prettier": "^2.3.2",
@@ -634,6 +636,14 @@
         "rxjs": "^7.3.0"
       }
     },
+    "node_modules/@certusone/wormhole-sdk/node_modules/axios": {
+      "version": "0.24.0",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-0.24.0.tgz",
+      "integrity": "sha512-Q6cWsys88HoPgAaFAVUb0WpPk0O8iTeisR9IMqy9G8AbO4NlpVknrnQS03zzF9PGAWgO3cgletO3VjV/P7VztA==",
+      "dependencies": {
+        "follow-redirects": "^1.14.4"
+      }
+    },
     "node_modules/@certusone/wormhole-spydk": {
       "version": "0.0.1",
       "resolved": "https://registry.npmjs.org/@certusone/wormhole-spydk/-/wormhole-spydk-0.0.1.tgz",
@@ -2908,6 +2918,30 @@
       "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.27.tgz",
       "integrity": "sha512-C1pD3kgLoZ56Uuy5lhfOxie4aZlA3UMGLX9rXteq4WitEZH6Rl80mwactt9QG0w0gLFlN/kLBTFnGXtDVWvWQw=="
     },
+    "node_modules/@types/node-fetch": {
+      "version": "2.6.2",
+      "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.2.tgz",
+      "integrity": "sha512-DHqhlq5jeESLy19TYhLakJ07kNumXWjcDdxXsLUMJZ6ue8VZJj4kLPQVE/2mdHh3xZziNF1xppu5lwmS53HR+A==",
+      "dev": true,
+      "dependencies": {
+        "@types/node": "*",
+        "form-data": "^3.0.0"
+      }
+    },
+    "node_modules/@types/node-fetch/node_modules/form-data": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz",
+      "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==",
+      "dev": true,
+      "dependencies": {
+        "asynckit": "^0.4.0",
+        "combined-stream": "^1.0.8",
+        "mime-types": "^2.1.12"
+      },
+      "engines": {
+        "node": ">= 6"
+      }
+    },
     "node_modules/@types/prettier": {
       "version": "2.6.0",
       "resolved": "https://registry.npmjs.org/@types/prettier/-/prettier-2.6.0.tgz",
@@ -3242,14 +3276,6 @@
       "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=",
       "dev": true
     },
-    "node_modules/axios": {
-      "version": "0.24.0",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-0.24.0.tgz",
-      "integrity": "sha512-Q6cWsys88HoPgAaFAVUb0WpPk0O8iTeisR9IMqy9G8AbO4NlpVknrnQS03zzF9PGAWgO3cgletO3VjV/P7VztA==",
-      "dependencies": {
-        "follow-redirects": "^1.14.4"
-      }
-    },
     "node_modules/babel-jest": {
       "version": "28.0.3",
       "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-28.0.3.tgz",
@@ -4033,6 +4059,25 @@
         "node-fetch": "2.6.7"
       }
     },
+    "node_modules/cross-fetch/node_modules/node-fetch": {
+      "version": "2.6.7",
+      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz",
+      "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==",
+      "dependencies": {
+        "whatwg-url": "^5.0.0"
+      },
+      "engines": {
+        "node": "4.x || >=6.0.0"
+      },
+      "peerDependencies": {
+        "encoding": "^0.1.0"
+      },
+      "peerDependenciesMeta": {
+        "encoding": {
+          "optional": true
+        }
+      }
+    },
     "node_modules/cross-spawn": {
       "version": "7.0.3",
       "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
@@ -4567,9 +4612,9 @@
       "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw=="
     },
     "node_modules/follow-redirects": {
-      "version": "1.14.9",
+      "version": "1.15.2",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.9.tgz",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz",
-      "integrity": "sha512-MQDfihBQYMcyy5dhRDJUHcw7lb2Pv/TuE6xP1vyraLukNDHKbDxDNaOE3NbCAdKQApno+GPRyo1YAp89yCjK4w==",
+      "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==",
       "funding": [
         {
           "type": "individual",
@@ -7241,22 +7286,11 @@
       "integrity": "sha512-Ntyt4AIXyaLIuMHF6IOoTakB3K+RWxwtsHNRxllEoA6vPwP9o4866g6YWDLUdnucilZhmkxiHwHr11gAENw+QA=="
     },
     "node_modules/node-fetch": {
-      "version": "2.6.7",
+      "version": "2.6.1",
-      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz",
+      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz",
-      "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==",
+      "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==",
-      "dependencies": {
-        "whatwg-url": "^5.0.0"
-      },
       "engines": {
         "node": "4.x || >=6.0.0"
-      },
-      "peerDependencies": {
-        "encoding": "^0.1.0"
-      },
-      "peerDependenciesMeta": {
-        "encoding": {
-          "optional": true
-        }
       }
     },
     "node_modules/node-gyp-build": {
@@ -8589,7 +8623,7 @@
     "node_modules/tr46": {
       "version": "0.0.3",
       "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
-      "integrity": "sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o="
+      "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="
     },
     "node_modules/triple-beam": {
       "version": "1.3.0",
@@ -8877,12 +8911,12 @@
     "node_modules/webidl-conversions": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
-      "integrity": "sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE="
+      "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="
     },
     "node_modules/whatwg-url": {
       "version": "5.0.0",
       "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
-      "integrity": "sha1-lmRU6HZUYuN2RNNib2dCzotwll0=",
+      "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",
       "dependencies": {
         "tr46": "~0.0.3",
         "webidl-conversions": "^3.0.0"
@@ -9483,6 +9517,16 @@
         "js-base64": "^3.6.1",
         "protobufjs": "^6.11.2",
         "rxjs": "^7.3.0"
+      },
+      "dependencies": {
+        "axios": {
+          "version": "0.24.0",
+          "resolved": "https://registry.npmjs.org/axios/-/axios-0.24.0.tgz",
+          "integrity": "sha512-Q6cWsys88HoPgAaFAVUb0WpPk0O8iTeisR9IMqy9G8AbO4NlpVknrnQS03zzF9PGAWgO3cgletO3VjV/P7VztA==",
+          "requires": {
+            "follow-redirects": "^1.14.4"
+          }
+        }
       }
     },
     "@certusone/wormhole-spydk": {
@@ -11124,6 +11168,29 @@
       "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.27.tgz",
       "integrity": "sha512-C1pD3kgLoZ56Uuy5lhfOxie4aZlA3UMGLX9rXteq4WitEZH6Rl80mwactt9QG0w0gLFlN/kLBTFnGXtDVWvWQw=="
     },
+    "@types/node-fetch": {
+      "version": "2.6.2",
+      "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.2.tgz",
+      "integrity": "sha512-DHqhlq5jeESLy19TYhLakJ07kNumXWjcDdxXsLUMJZ6ue8VZJj4kLPQVE/2mdHh3xZziNF1xppu5lwmS53HR+A==",
+      "dev": true,
+      "requires": {
+        "@types/node": "*",
+        "form-data": "^3.0.0"
+      },
+      "dependencies": {
+        "form-data": {
+          "version": "3.0.1",
+          "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz",
+          "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==",
+          "dev": true,
+          "requires": {
+            "asynckit": "^0.4.0",
+            "combined-stream": "^1.0.8",
+            "mime-types": "^2.1.12"
+          }
+        }
+      }
+    },
     "@types/prettier": {
       "version": "2.6.0",
       "resolved": "https://registry.npmjs.org/@types/prettier/-/prettier-2.6.0.tgz",
@@ -11430,14 +11497,6 @@
       "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=",
       "dev": true
     },
-    "axios": {
-      "version": "0.24.0",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-0.24.0.tgz",
-      "integrity": "sha512-Q6cWsys88HoPgAaFAVUb0WpPk0O8iTeisR9IMqy9G8AbO4NlpVknrnQS03zzF9PGAWgO3cgletO3VjV/P7VztA==",
-      "requires": {
-        "follow-redirects": "^1.14.4"
-      }
-    },
     "babel-jest": {
       "version": "28.0.3",
       "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-28.0.3.tgz",
@@ -12067,6 +12126,16 @@
       "integrity": "sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw==",
       "requires": {
         "node-fetch": "2.6.7"
+      },
+      "dependencies": {
+        "node-fetch": {
+          "version": "2.6.7",
+          "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz",
+          "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==",
+          "requires": {
+            "whatwg-url": "^5.0.0"
+          }
+        }
       }
     },
     "cross-spawn": {
@@ -12506,9 +12575,9 @@
       "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw=="
     },
     "follow-redirects": {
-      "version": "1.14.9",
+      "version": "1.15.2",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.9.tgz",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz",
-      "integrity": "sha512-MQDfihBQYMcyy5dhRDJUHcw7lb2Pv/TuE6xP1vyraLukNDHKbDxDNaOE3NbCAdKQApno+GPRyo1YAp89yCjK4w=="
+      "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA=="
     },
     "form-data": {
       "version": "4.0.0",
@@ -14499,12 +14568,9 @@
       "integrity": "sha512-Ntyt4AIXyaLIuMHF6IOoTakB3K+RWxwtsHNRxllEoA6vPwP9o4866g6YWDLUdnucilZhmkxiHwHr11gAENw+QA=="
     },
     "node-fetch": {
-      "version": "2.6.7",
+      "version": "2.6.1",
-      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz",
+      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz",
-      "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==",
+      "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw=="
-      "requires": {
-        "whatwg-url": "^5.0.0"
-      }
     },
     "node-gyp-build": {
       "version": "4.4.0",
@@ -15491,7 +15557,7 @@
     "tr46": {
       "version": "0.0.3",
       "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
-      "integrity": "sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o="
+      "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="
     },
     "triple-beam": {
       "version": "1.3.0",
@@ -15692,12 +15758,12 @@
     "webidl-conversions": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
-      "integrity": "sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE="
+      "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="
     },
     "whatwg-url": {
       "version": "5.0.0",
       "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
-      "integrity": "sha1-lmRU6HZUYuN2RNNib2dCzotwll0=",
+      "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",
       "requires": {
         "tr46": "~0.0.3",
         "webidl-conversions": "^3.0.0"

third_party/pyth/price-service/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pythnetwork/pyth-price-service",
-  "version": "2.2.4",
+  "version": "2.3.0",
   "description": "Pyth Price Service",
   "main": "index.js",
   "scripts": {
@@ -18,6 +18,7 @@
     "@types/jest": "^27.5.0",
     "@types/long": "^4.0.1",
     "@types/node": "^16.6.1",
+    "@types/node-fetch": "^2.6.2",
     "@types/supertest": "^2.0.12",
     "jest": "^28.0.3",
     "prettier": "^2.3.2",
@@ -46,6 +47,7 @@
     "joi": "^17.6.0",
     "lru-cache": "^7.14.1",
     "morgan": "^1.10.0",
+    "node-fetch": "^2.6.1",
     "prom-client": "^14.0.1",
     "response-time": "^2.3.2",
     "winston": "^3.3.3",

third_party/pyth/price-service/src/__tests__/rest.test.ts

@@ -2,11 +2,12 @@ import { HexString, Price, PriceFeed } from "@pythnetwork/pyth-sdk-js";
 import { Express } from "express";
 import { StatusCodes } from "http-status-codes";
 import request from "supertest";
-import { PriceInfo, PriceStore } from "../listen";
+import { PriceInfo, PriceStore, VaaCache } from "../listen";
 import { RestAPI } from "../rest";
 
 let app: Express;
 let priceInfoMap: Map<string, PriceInfo>;
+let vaasCache: VaaCache;
 
 function expandTo64Len(id: string): string {
   return id.repeat(64).substring(0, 64);
@@ -56,6 +57,12 @@ beforeAll(async () => {
     dummyPriceInfoPair(expandTo64Len("3456"), 2, "bad01bad"),
     dummyPriceInfoPair(expandTo64Len("10101"), 3, "bidbidbid"),
   ]);
+  vaasCache = new VaaCache();
+  vaasCache.set(
+    expandTo64Len("abcd"),
+    1,
+    Buffer.from("a1b2c3d4", "hex").toString("base64")
+  );
 
   const priceInfo: PriceStore = {
     getLatestPriceInfo: (priceFeedId: string) => {
@@ -63,6 +70,9 @@ beforeAll(async () => {
     },
     addUpdateListener: (_callback: (priceInfo: PriceInfo) => any) => undefined,
     getPriceIds: () => new Set(),
+    getVaa: (vaasCacheKey: string, publishTime: number) => {
+      return vaasCache.get(vaasCacheKey, publishTime);
+    },
   };
 
   const api = new RestAPI({ port: 8889 }, priceInfo, () => true);

third_party/pyth/price-service/src/__tests__/ws.test.ts

@@ -108,6 +108,7 @@ beforeAll(async () => {
     getLatestPriceInfo: (_priceFeedId: string) => undefined,
     addUpdateListener: (_callback: (priceInfo: PriceInfo) => any) => undefined,
     getPriceIds: () => new Set(priceInfos.map((info) => info.priceFeed.id)),
+    getVaa: (_vaasCacheKey: string) => undefined,
   };
 
   api = new WebSocketAPI(priceInfo);

third_party/pyth/price-service/src/index.ts

@@ -52,6 +52,8 @@ async function run() {
   const restAPI = new RestAPI(
     {
       port: parseInt(envOrErr("REST_PORT"), 10),
+      dbApiEndpoint: process.env.DB_API_ENDPOINT,
+      dbApiCluster: process.env.DB_API_CLUSTER,
     },
     listener,
     isReady,
@@ -61,6 +63,7 @@ async function run() {
   const wsAPI = new WebSocketAPI(listener, promClient);
 
   listener.run();
+  listener.runCacheCleanupLoop();
   const server = await restAPI.run();
   wsAPI.run(server);
 }

third_party/pyth/price-service/src/listen.ts

@@ -7,21 +7,21 @@ import {
 
 import { importCoreWasm } from "@certusone/wormhole-sdk/lib/cjs/solana/wasm";
 
-import {
-  getBatchSummary,
-  parseBatchPriceAttestation,
-  priceAttestationToPriceFeed,
-} from "@pythnetwork/p2w-sdk-js";
 import {
   FilterEntry,
   SubscribeSignedVAAResponse,
 } from "@certusone/wormhole-spydk/lib/cjs/proto/spy/v1/spy";
 import { ClientReadableStream } from "@grpc/grpc-js";
+import {
+  getBatchSummary,
+  parseBatchPriceAttestation,
+  priceAttestationToPriceFeed,
+} from "@pythnetwork/p2w-sdk-js";
 import { HexString, PriceFeed } from "@pythnetwork/pyth-sdk-js";
+import LRUCache from "lru-cache";
 import { sleep, TimestampInSec } from "./helpers";
 import { logger } from "./logging";
 import { PromClient } from "./promClient";
-import LRUCache from "lru-cache";
 
 export type PriceInfo = {
   vaa: Buffer;
@@ -37,6 +37,7 @@ export interface PriceStore {
   getPriceIds(): Set<HexString>;
   getLatestPriceInfo(priceFeedId: HexString): PriceInfo | undefined;
   addUpdateListener(callback: (priceInfo: PriceInfo) => any): void;
+  getVaa(priceFeedId: string, publishTime: number): VaaConfig | undefined;
 }
 
 type ListenerReadinessConfig = {
@@ -48,10 +49,74 @@ type ListenerConfig = {
   spyServiceHost: string;
   filtersRaw?: string;
   readiness: ListenerReadinessConfig;
+  webApiEndpoint?: string;
+  webApiCluster?: string;
 };
 
 type VaaKey = string;
 
+type VaaConfig = {
+  publishTime: number;
+  vaa: string;
+};
+
+export class VaaCache {
+  private cache: Map<string, VaaConfig[]>;
+  private ttl: number;
+
+  constructor(ttl: number = 300) {
+    this.cache = new Map();
+    this.ttl = ttl;
+  }
+
+  set(key: VaaKey, publishTime: number, vaa: string): void {
+    if (this.cache.has(key)) {
+      this.cache.get(key)!.push({ publishTime, vaa });
+    } else {
+      this.cache.set(key, [{ publishTime, vaa }]);
+    }
+  }
+
+  get(key: VaaKey, publishTime: number): VaaConfig | undefined {
+    if (!this.cache.has(key)) {
+      return undefined;
+    } else {
+      const vaaConf = this.find(this.cache.get(key)!, publishTime);
+      return vaaConf;
+    }
+  }
+
+  find(arr: VaaConfig[], publishTime: number): VaaConfig | undefined {
+    if (arr.length === 0 || publishTime < arr[0].publishTime) {
+      return undefined;
+    }
+    let left = 0;
+    let right = arr.length - 1;
+    let nextLargest = -1;
+
+    while (left <= right) {
+      const middle = Math.floor((left + right) / 2);
+      if (arr[middle].publishTime === publishTime) {
+        return arr[middle];
+      } else if (arr[middle].publishTime < publishTime) {
+        left = middle + 1;
+      } else {
+        nextLargest = middle;
+        right = middle - 1;
+      }
+    }
+
+    return nextLargest !== -1 ? arr[nextLargest] : undefined;
+  }
+
+  async removeExpiredValues() {
+    const now = Math.floor(Date.now() / 1000);
+    for (const arr of this.cache.values()) {
+      arr.filter((vaaConf) => now - vaaConf.publishTime < this.ttl);
+    }
+  }
+}
+
 export class Listener implements PriceStore {
   // Mapping of Price Feed Id to Vaa
   private priceFeedVaaMap = new Map<string, PriceInfo>();
@@ -62,6 +127,7 @@ export class Listener implements PriceStore {
   private readinessConfig: ListenerReadinessConfig;
   private updateCallbacks: ((priceInfo: PriceInfo) => any)[];
   private observedVaas: LRUCache<VaaKey, boolean>;
+  private vaasCache: VaaCache;
 
   constructor(config: ListenerConfig, promClient?: PromClient) {
     this.promClient = promClient;
@@ -73,6 +139,7 @@ export class Listener implements PriceStore {
       max: 10000, // At most 10000 items
       ttl: 60 * 1000, // 60 seconds
     });
+    this.vaasCache = new VaaCache();
   }
 
   private loadFilters(filtersRaw?: string) {
@@ -105,6 +172,10 @@ export class Listener implements PriceStore {
     logger.info("loaded " + this.filters.length + " filters");
   }
 
+  async runCacheCleanupLoop(interval: number = 60) {
+    setInterval(this.vaasCache.removeExpiredValues, interval * 1000);
+  }
+
   async run() {
     logger.info(
       "pyth_relay starting up, will listen for signed VAAs from " +
@@ -185,13 +256,13 @@ export class Listener implements PriceStore {
     const vaaEmitterAddressHex = Buffer.from(
       parsedVaa.emitter_address
     ).toString("hex");
-    const vaaKey: VaaKey = `${parsedVaa.emitter_chain}#${vaaEmitterAddressHex}#${parsedVaa.sequence}`;
+    const observedVaasKey: VaaKey = `${parsedVaa.emitter_chain}#${vaaEmitterAddressHex}#${parsedVaa.sequence}`;
 
-    if (this.observedVaas.has(vaaKey)) {
+    if (this.observedVaas.has(observedVaasKey)) {
       return;
     }
 
-    this.observedVaas.set(vaaKey, true);
+    this.observedVaas.set(observedVaasKey, true);
     this.promClient?.incReceivedVaa();
 
     let batchAttestation;
@@ -223,6 +294,11 @@ export class Listener implements PriceStore {
       const cachedPriceInfo = this.priceFeedVaaMap.get(key);
 
       if (this.isNewPriceInfo(cachedPriceInfo, priceInfo)) {
+        this.vaasCache.set(
+          priceInfo.priceFeed.id,
+          priceInfo.publishTime,
+          priceInfo.vaa.toString("base64")
+        );
         this.priceFeedVaaMap.set(key, priceInfo);
 
         if (cachedPriceInfo !== undefined) {
@@ -252,6 +328,10 @@ export class Listener implements PriceStore {
     );
   }
 
+  getVaa(priceFeedId: string, publishTime: number): VaaConfig | undefined {
+    return this.vaasCache.get(priceFeedId, publishTime);
+  }
+
   getLatestPriceInfo(priceFeedId: string): PriceInfo | undefined {
     return this.priceFeedVaaMap.get(priceFeedId);
   }

third_party/pyth/price-service/src/rest.ts

@@ -5,6 +5,7 @@ import { Joi, schema, validate, ValidationError } from "express-validation";
 import { Server } from "http";
 import { StatusCodes } from "http-status-codes";
 import morgan from "morgan";
+import fetch from "node-fetch";
 import { TimestampInSec } from "./helpers";
 import { PriceStore } from "./listen";
 import { logger } from "./logging";
@@ -36,14 +37,18 @@ export class RestAPI {
   private priceFeedVaaInfo: PriceStore;
   private isReady: (() => boolean) | undefined;
   private promClient: PromClient | undefined;
+  private dbApiEndpoint?: string;
+  private dbApiCluster?: string;
 
   constructor(
-    config: { port: number },
+    config: { port: number; dbApiEndpoint?: string; dbApiCluster?: string },
     priceFeedVaaInfo: PriceStore,
     isReady?: () => boolean,
     promClient?: PromClient
   ) {
     this.port = config.port;
+    this.dbApiEndpoint = config.dbApiEndpoint;
+    this.dbApiCluster = config.dbApiCluster;
     this.priceFeedVaaInfo = priceFeedVaaInfo;
     this.isReady = isReady;
     this.promClient = promClient;
@@ -113,6 +118,51 @@ export class RestAPI {
       "api/latest_vaas?ids[]=<price_feed_id>&ids[]=<price_feed_id_2>&.."
     );
 
+    const getVaaInputSchema: schema = {
+      query: Joi.object({
+        id: Joi.string()
+          .regex(/^(0x)?[a-f0-9]{64}$/)
+          .required(),
+        publish_time: Joi.number().required(),
+      }).required(),
+    };
+    app.get(
+      "/api/get_vaa",
+      validate(getVaaInputSchema),
+      (req: Request, res: Response) => {
+        const priceFeedId = req.query.id as string;
+        const publishTime = Number(req.query.publish_time as string);
+        const vaa = this.priceFeedVaaInfo.getVaa(priceFeedId, publishTime);
+        // if publishTime is older than cache ttl or vaa is not found, fetch from db
+        if (!vaa) {
+          // cache miss
+          if (this.dbApiEndpoint && this.dbApiCluster) {
+            fetch(
+              `${this.dbApiEndpoint}/vaa?id=${priceFeedId}&publishTime=${publishTime}&cluster=${this.dbApiCluster}`
+            )
+              .then((r: any) => r.json())
+              .then((arr: any) => {
+                if (arr.length > 0 && arr[0]) {
+                  res.json(arr[0]);
+                } else {
+                  res.status(StatusCodes.NOT_FOUND).send("VAA not found");
+                }
+              });
+          }
+        } else {
+          // cache hit
+          const processedVaa = {
+            publishTime: new Date(vaa.publishTime),
+            vaa: vaa.vaa,
+          };
+          res.json(processedVaa);
+        }
+      }
+    );
+    endpoints.push(
+      "api/get_vaa?id=<price_feed_id>&publish_time=<publish_time_in_unix_timestamp>"
+    );
+
     const latestPriceFeedsInputSchema: schema = {
       query: Joi.object({
         ids: Joi.array()

tilt-devnet/k8s/pyth-price-service.yaml

@@ -72,3 +72,7 @@ spec:
               value: "6"
             - name: LOG_LEVEL
               value: debug
+            - name: REMOVE_EXPIRED_VALUES_INTERVAL_SECONDS
+              value: "60"
+            - name: CACHE_TTL_SECONDS
+              value: "300"