From 2306d103d879ad7ffb475c9a14e3823916249f57 Mon Sep 17 00:00:00 2001 From: gipsh Date: Wed, 28 Jun 2023 15:04:36 -0300 Subject: [PATCH] 321 replace redis to elasticcache (#467) * redis prefix support for caches * fly support for prefix * unit tests * redis prefix for notional cache updater * fix test * fix tests * use redis-prfix from config map --- api/go.mod | 23 +++--- api/go.sum | 58 ++++++++++++---- api/internal/config/config.go | 2 + api/main.go | 81 +++++++++++++--------- common/client/cache/cache.go | 14 +++- common/client/cache/notional/cache.go | 19 ++++- common/client/cache/notional/cache_test.go | 38 ++++++++++ deploy/jobs/env/production.env | 1 + deploy/jobs/env/staging.env | 1 + deploy/jobs/env/test.env | 1 + deploy/jobs/notional.yaml | 5 ++ fly/config/config.go | 9 +++ fly/config/config_test.go | 27 ++++++++ fly/go.mod | 1 + fly/go.sum | 1 + fly/main.go | 4 +- fly/notifier/vaa_last_sequence.go | 12 +++- fly/notifier/vaa_last_sequence_test.go | 37 ++++++++++ jobs/cmd/main.go | 2 +- jobs/config/config.go | 1 + jobs/jobs/notional/notional.go | 13 +++- 21 files changed, 287 insertions(+), 63 deletions(-) create mode 100644 common/client/cache/notional/cache_test.go create mode 100644 fly/config/config_test.go create mode 100644 fly/notifier/vaa_last_sequence_test.go diff --git a/api/go.mod b/api/go.mod index 6c263ee2..9adee994 100644 --- a/api/go.mod +++ b/api/go.mod @@ -8,7 +8,7 @@ require ( github.com/ethereum/go-ethereum v1.10.21 github.com/gagliardetto/solana-go v1.7.1 github.com/gofiber/adaptor/v2 v2.1.29 - github.com/gofiber/fiber/v2 v2.39.0 + github.com/gofiber/fiber/v2 v2.47.0 github.com/improbable-eng/grpc-web v0.15.0 github.com/influxdata/influxdb-client-go/v2 v2.12.2 github.com/ipfs/go-log/v2 v2.5.1 @@ -30,14 +30,21 @@ require ( github.com/test-go/testify v1.1.4 ) -require github.com/redis/go-redis/v9 v9.0.5 // indirect +require ( + github.com/google/uuid v1.3.0 // indirect + github.com/philhofer/fwd v1.1.2 // indirect + github.com/redis/go-redis/v9 v9.0.5 // indirect + github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94 // indirect + github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee // indirect + github.com/tinylib/msgp v1.1.8 // indirect +) require ( contrib.go.opencensus.io/exporter/stackdriver v0.13.14 // indirect filippo.io/edwards25519 v1.0.0 // indirect github.com/KyleBanks/depth v1.2.1 // indirect github.com/XLabs/fiber-redis-storage v0.2.0 - github.com/andybalholm/brotli v1.0.4 // indirect + github.com/andybalholm/brotli v1.0.5 // indirect github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blendle/zapdriver v1.3.1 // indirect @@ -71,7 +78,7 @@ require ( github.com/ipfs/go-cid v0.2.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.15.12 // indirect + github.com/klauspost/compress v1.16.3 // indirect github.com/klauspost/cpuid/v2 v2.1.0 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-libp2p v0.22.0 // indirect @@ -80,7 +87,7 @@ require ( github.com/magiconair/properties v1.8.6 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.16 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect github.com/mattn/go-pointer v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect @@ -120,7 +127,7 @@ require ( github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect - github.com/valyala/fasthttp v1.41.0 // indirect + github.com/valyala/fasthttp v1.47.0 // indirect github.com/valyala/tcplisten v1.0.0 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.1 // indirect @@ -129,10 +136,10 @@ require ( go.opencensus.io v0.23.0 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.8.0 // indirect - golang.org/x/crypto v0.2.0 // indirect + golang.org/x/crypto v0.7.0 // indirect golang.org/x/net v0.10.0 // indirect golang.org/x/sync v0.2.0 // indirect - golang.org/x/sys v0.8.0 // indirect + golang.org/x/sys v0.9.0 // indirect golang.org/x/term v0.8.0 // indirect golang.org/x/text v0.9.0 // indirect golang.org/x/tools v0.9.3 // indirect diff --git a/api/go.sum b/api/go.sum index 47c6ab80..8b7a7a36 100644 --- a/api/go.sum +++ b/api/go.sum @@ -68,8 +68,9 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= -github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= +github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/ansrivas/fiberprometheus/v2 v2.4.1 h1:V87ahTcU/I4c8tD6GKiuyyB0Z82dw2VVqLDgBtUcUgc= github.com/ansrivas/fiberprometheus/v2 v2.4.1/go.mod h1:ATJ3l0sufyoZBz+TEohAyQJqbgUSQaPwCHNL/L67Wnw= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= @@ -242,8 +243,9 @@ github.com/gofiber/adaptor/v2 v2.1.25/go.mod h1:gOxtwMVqUStB5goAYtKd+hSvGupdd+aR github.com/gofiber/adaptor/v2 v2.1.29 h1:JnYd6fbqVM9D4zPchk+kg89PfxyuKqZKhBWGQDHfKH4= github.com/gofiber/adaptor/v2 v2.1.29/go.mod h1:z4mAV9mMsUgIEVGGS5Ii6ZMTJq4VdV1KWL1JAbsZdUA= github.com/gofiber/fiber/v2 v2.36.0/go.mod h1:tgCr+lierLwLoVHHO/jn3Niannv34WRkQETU8wiL9fQ= -github.com/gofiber/fiber/v2 v2.39.0 h1:uhWpYQ6EHN8J7FOPYbI2hrdBD/KNZBC5CjbuOd4QUt4= github.com/gofiber/fiber/v2 v2.39.0/go.mod h1:Cmuu+elPYGqlvQvdKyjtYsjGMi69PDp8a1AY2I5B2gM= +github.com/gofiber/fiber/v2 v2.47.0 h1:EN5lHVCc+Pyqh5OEsk8fzRiifgwpbrP0rulQ4iNf3fs= +github.com/gofiber/fiber/v2 v2.47.0/go.mod h1:mbFMVN1lQuzziTkkakgtKKdjfsXSw9BKR5lmcNksUoU= github.com/gofiber/utils v1.1.0 h1:vdEBpn7AzIUJRhe+CiTOJdUcTg4Q9RK+pEa0KPbLdrM= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -317,6 +319,8 @@ github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLe github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= @@ -410,9 +414,8 @@ github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYs github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/klauspost/compress v1.15.12 h1:YClS/PImqYbn+UILDnqxQCZ3RehC9N318SU3kElDUEM= -github.com/klauspost/compress v1.15.12/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= +github.com/klauspost/compress v1.16.3 h1:XuJt9zzcnaz6a16/OU53ZjWp/v7/42WcR5t2a0PcNQY= +github.com/klauspost/compress v1.16.3/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.1.0 h1:eyi1Ad2aNJMW95zcSbmGg7Cg6cq3ADwLpMAP96d8rF0= @@ -467,8 +470,9 @@ github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2y github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-pointer v0.0.1 h1:n+XhsuGeVO6MEAp7xyEukFINEa+Quek5psIR/ylA6o0= github.com/mattn/go-pointer v0.0.1/go.mod h1:2zXcozF6qYGgmsG+SeTZz3oAbFLdD3OWqnUbNvJZAlc= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= @@ -564,6 +568,9 @@ github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCko github.com/pelletier/go-toml/v2 v2.0.5 h1:ipoSadvV8oGUjnUbMub59IDPPwfxF694nG/jwbMiyQg= github.com/pelletier/go-toml/v2 v2.0.5/go.mod h1:OMHamSCAODeSsVrwwvcJOaoN0LIUIaFVNZzmWyNfXas= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= +github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= +github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw= +github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -635,6 +642,11 @@ github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= +github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94 h1:rmMl4fXJhKMNWl+K+r/fq4FbbKI+Ia2m9hYBLm2h4G4= +github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94/go.mod h1:90zrgN3D/WJsDd1iXHT96alCoN2KJo6/4x1DZC3wZs8= +github.com/savsgio/gotils v0.0.0-20220530130905-52f3993e8d6d/go.mod h1:Gy+0tqhJvgGlqnTF8CVGP0AaGRjwBtXs/a5PA0Y3+A4= +github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee h1:8Iv5m6xEo1NR1AvpV+7XmhI4r39LGNzwUL4YpMuL5vk= +github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee/go.mod h1:qwtSXrKuJh/zsFQ12yEE89xfCrGKK63Rr7ctU/uCo4g= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= @@ -711,6 +723,9 @@ github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhV github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tinylib/msgp v1.1.6/go.mod h1:75BAfg2hauQhs3qedfdDZmWAPcFMAvJE5b9rGOMufyw= +github.com/tinylib/msgp v1.1.8 h1:FCXC1xanKO4I8plpHGH2P7koL/RzZs12l/+r7vakfm0= +github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= @@ -723,8 +738,8 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.38.0/go.mod h1:t/G+3rLek+CyY9bnIE+YlMRddxVAAGjhxndDB4i4C0I= github.com/valyala/fasthttp v1.40.0/go.mod h1:t/G+3rLek+CyY9bnIE+YlMRddxVAAGjhxndDB4i4C0I= -github.com/valyala/fasthttp v1.41.0 h1:zeR0Z1my1wDHTRiamBCXVglQdbUwgb9uWG3k1HQz6jY= -github.com/valyala/fasthttp v1.41.0/go.mod h1:f6VbjjoI3z1NDOZOv17o6RvtRSWxC77seBFc2uWtgiY= +github.com/valyala/fasthttp v1.47.0 h1:y7moDoxYzMooFpT5aHgNgVOQDrS3qlkfiP9mDtGGK9c= +github.com/valyala/fasthttp v1.47.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= @@ -745,6 +760,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= @@ -798,11 +814,12 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.2.0 h1:BRXPfhNivWL5Yq0BGQ39a2sW6t44aODpfxkWjYdzewE= -golang.org/x/crypto v0.2.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= +golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= +golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -839,6 +856,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -890,7 +909,8 @@ golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220906165146-f3363e06e74c/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -916,6 +936,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -984,16 +1006,20 @@ golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1005,6 +1031,7 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1063,6 +1090,7 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -1072,6 +1100,8 @@ golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/api/internal/config/config.go b/api/internal/config/config.go index 13562634..06c8e065 100644 --- a/api/internal/config/config.go +++ b/api/internal/config/config.go @@ -38,6 +38,7 @@ type AppConfig struct { TvlExpiration int Enabled bool MetricExpiration int + Prefix string } PORT int LogLevel string @@ -75,6 +76,7 @@ func defaulConfig() *AppConfig { TvlExpiration int Enabled bool MetricExpiration int + Prefix string }{ MetricExpiration: 10, }, diff --git a/api/main.go b/api/main.go index 63c0432b..f1f65118 100644 --- a/api/main.go +++ b/api/main.go @@ -171,39 +171,11 @@ func main() { // Configure rate limiter if cfg.RateLimit.Enabled { - - store, err := frs.New( - frs.Config{URL: cfg.Cache.URL, Prefix: cfg.RateLimit.Prefix}) + rl, err := NewRateLimiter(appCtx, cfg, rootLogger) if err != nil { - rootLogger.Error("failed to initialize rate limiter", - zap.String("url", cfg.Cache.URL), - zap.String("prefix", cfg.RateLimit.Prefix), - zap.Error(err)) panic(err) } - - // default to 60 requests per minute - if cfg.RateLimit.Max == 0 { - cfg.RateLimit.Max = 60 - } - - rootLogger.Info("rate limit enabled", zap.Int("max requests per minute", cfg.RateLimit.Max)) - app.Use(limiter.New(limiter.Config{ - Next: func(c *fiber.Ctx) bool { - - ip := utils.GetRealIp(c) - return utils.IsPrivateIPAsString(ip) - }, - Max: cfg.RateLimit.Max, - Expiration: 60 * time.Second, - KeyGenerator: func(c *fiber.Ctx) string { - return utils.GetRealIp(c) - }, - LimitReached: func(c *fiber.Ctx) error { - return c.SendStatus(fiber.StatusTooManyRequests) - }, - Storage: store, - })) + app.Use(rl) } // Set up route handlers @@ -267,7 +239,7 @@ func NewCache(ctx context.Context, cfg *config.AppConfig, logger *zap.Logger) (w redisClient := redis.NewClient(&redis.Options{Addr: cfg.Cache.URL}) // get cache client - cacheClient, _ := wormscanCache.NewCacheClient(redisClient, cfg.Cache.Enabled, logger) + cacheClient, _ := wormscanCache.NewCacheClient(redisClient, cfg.Cache.Enabled, cfg.Cache.Prefix, logger) // get notional cache client and init load to local cache notionalCache, _ := wormscanNotionalCache.NewNotionalCache(ctx, redisClient, cfg.Cache.Channel, logger) @@ -279,3 +251,50 @@ func NewCache(ctx context.Context, cfg *config.AppConfig, logger *zap.Logger) (w func newInfluxClient(url, token string) influxdb2.Client { return influxdb2.NewClient(url, token) } + +func NewRateLimiter(ctx context.Context, cfg *config.AppConfig, logger *zap.Logger) (func(*fiber.Ctx) error, error) { + + if cfg.RateLimit.Prefix != "" { + cfg.RateLimit.Prefix += ":rate-limiter:" + } else { + cfg.RateLimit.Prefix = "rate-limiter:" + } + + // initialize rate limiter + store, err := frs.New( + frs.Config{URL: cfg.Cache.URL, Prefix: cfg.RateLimit.Prefix}) + if err != nil { + logger.Error("failed to initialize rate limiter", + zap.String("url", cfg.Cache.URL), + zap.String("prefix", cfg.RateLimit.Prefix), + zap.Error(err)) + return nil, err + } + + // default to 60 requests per minute + if cfg.RateLimit.Max == 0 { + cfg.RateLimit.Max = 60 + } + + logger.Info("rate limit enabled", zap.Int("max requests per minute", cfg.RateLimit.Max)) + + router := limiter.New(limiter.Config{ + Next: func(c *fiber.Ctx) bool { + + ip := utils.GetRealIp(c) + return utils.IsPrivateIPAsString(ip) + }, + Max: cfg.RateLimit.Max, + Expiration: 60 * time.Second, + KeyGenerator: func(c *fiber.Ctx) string { + return utils.GetRealIp(c) + }, + LimitReached: func(c *fiber.Ctx) error { + return c.SendStatus(fiber.StatusTooManyRequests) + }, + Storage: store, + }) + + return router, nil + +} diff --git a/common/client/cache/cache.go b/common/client/cache/cache.go index 3f92d776..495fdd9d 100644 --- a/common/client/cache/cache.go +++ b/common/client/cache/cache.go @@ -24,6 +24,7 @@ type CacheClient struct { Client *redis.Client Enabled bool logger *zap.Logger + Prefix string } // Cache is the interface for cache client. @@ -46,11 +47,11 @@ type CacheReadable interface { type CacheGetFunc func(ctx context.Context, key string) (string, error) // NewCacheClient init a new cache client. -func NewCacheClient(redisClient *redis.Client, enabled bool, log *zap.Logger) (*CacheClient, error) { +func NewCacheClient(redisClient *redis.Client, enabled bool, prefix string, log *zap.Logger) (*CacheClient, error) { if redisClient == nil { return nil, errors.New("redis client is nil") } - return &CacheClient{Client: redisClient, Enabled: enabled, logger: log}, nil + return &CacheClient{Client: redisClient, Enabled: enabled, logger: log, Prefix: prefix}, nil } // Get get a cache value or error from a key. @@ -61,6 +62,7 @@ func (c *CacheClient) Get(ctx context.Context, key string) (string, error) { if !c.Enabled { return "", ErrCacheNotEnabled } + key = c.renderKey(key) value, err := c.Client.Get(ctx, key).Result() if err != nil { requestID := fmt.Sprintf("%v", ctx.Value("requestid")) @@ -86,6 +88,7 @@ func (c *CacheClient) Set(ctx context.Context, key string, value interface{}, ex if !c.Enabled { return ErrCacheNotEnabled } + key = c.renderKey(key) err := c.Client.Set(ctx, key, value, expiration).Err() if err != nil { requestID := fmt.Sprintf("%v", ctx.Value("requestid")) @@ -98,3 +101,10 @@ func (c *CacheClient) Set(ctx context.Context, key string, value interface{}, ex } return nil } + +func (c *CacheClient) renderKey(key string) string { + if c.Prefix != "" { + return fmt.Sprintf("%s:%s", c.Prefix, key) + } + return key +} diff --git a/common/client/cache/notional/cache.go b/common/client/cache/notional/cache.go index 0cc9fbde..f381fd5b 100644 --- a/common/client/cache/notional/cache.go +++ b/common/client/cache/notional/cache.go @@ -16,7 +16,7 @@ import ( const ( wormscanNotionalUpdated = "NOTIONAL_UPDATED" - wormscanNotionalCacheKeyRegex = "*WORMSCAN:NOTIONAL:SYMBOL:*" + wormscanNotionalCacheKeyRegex = "WORMSCAN:NOTIONAL:SYMBOL:*" KeyFormatString = "WORMSCAN:NOTIONAL:SYMBOL:%s" ) @@ -50,6 +50,7 @@ type NotionalCache struct { pubSub *redis.PubSub channel string notionalMap sync.Map + prefix string logger *zap.Logger } @@ -66,6 +67,7 @@ func NewNotionalCache(ctx context.Context, redisClient *redis.Client, channel st pubSub: pubsub, channel: channel, notionalMap: sync.Map{}, + prefix: "", logger: log}, nil } @@ -92,7 +94,7 @@ func (c *NotionalCache) loadCache(ctx context.Context) error { for { // Get a page of results from the cursor var keys []string - scanCmd := c.client.Scan(ctx, cursor, wormscanNotionalCacheKeyRegex, 100) + scanCmd := c.client.Scan(ctx, cursor, c.renderRegExp(), 100) if scanCmd.Err() != nil { c.logger.Error("redis.ScanCmd has errors", zap.Error(err)) return fmt.Errorf("redis.ScanCmd has errors: %w", err) @@ -148,6 +150,7 @@ func (c *NotionalCache) Get(symbol domain.Symbol) (PriceData, error) { // get notional cache key key := fmt.Sprintf(KeyFormatString, symbol) + key = c.renderKey(key) // get notional cache value field, ok := c.notionalMap.Load(key) @@ -165,3 +168,15 @@ func (c *NotionalCache) Get(symbol domain.Symbol) (PriceData, error) { } return notional, nil } + +func (c *NotionalCache) renderKey(key string) string { + if c.prefix != "" { + return fmt.Sprintf("%s:%s", c.prefix, key) + } else { + return key + } +} + +func (c *NotionalCache) renderRegExp() string { + return "*" + c.renderKey(wormscanNotionalCacheKeyRegex) +} diff --git a/common/client/cache/notional/cache_test.go b/common/client/cache/notional/cache_test.go new file mode 100644 index 00000000..b40ca403 --- /dev/null +++ b/common/client/cache/notional/cache_test.go @@ -0,0 +1,38 @@ +package notional + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNotionalCache_renderKey(t *testing.T) { + + nc := &NotionalCache{ + client: nil, + prefix: "staging-mainnet", + } + + key := nc.renderKey("BTC/USDT") + assert.Equal(t, "staging-mainnet:BTC/USDT", key) + +} + +func TestNotionalCache_renderRegexp(t *testing.T) { + + nc := &NotionalCache{ + client: nil, + prefix: "staging-mainnet", + } + + key := nc.renderRegExp() + assert.Equal(t, "*staging-mainnet:WORMSCAN:NOTIONAL:SYMBOL:*", key) + + nc = &NotionalCache{ + client: nil, + prefix: "", + } + key = nc.renderRegExp() + assert.Equal(t, "*WORMSCAN:NOTIONAL:SYMBOL:*", key) + +} diff --git a/deploy/jobs/env/production.env b/deploy/jobs/env/production.env index 458fc95c..6d031dc1 100644 --- a/deploy/jobs/env/production.env +++ b/deploy/jobs/env/production.env @@ -11,3 +11,4 @@ COINGECKO_URL=https://api.coingecko.com/api/v3 NOTIONAL_CHANNEL=WORMSCAN:NOTIONAL LOG_LEVEL=INFO CRONTAB_SCHEDULE=*/5 * * * * +CACHE_PREFIX=mainnet-production \ No newline at end of file diff --git a/deploy/jobs/env/staging.env b/deploy/jobs/env/staging.env index b16392c4..9e9f6c06 100644 --- a/deploy/jobs/env/staging.env +++ b/deploy/jobs/env/staging.env @@ -11,3 +11,4 @@ COINGECKO_URL=https://api.coingecko.com/api/v3 NOTIONAL_CHANNEL=WORMSCAN:NOTIONAL LOG_LEVEL=INFO CRONTAB_SCHEDULE=*/5 * * * * +CACHE_PREFIX=mainnet-staging \ No newline at end of file diff --git a/deploy/jobs/env/test.env b/deploy/jobs/env/test.env index be3e648d..4550ae1a 100644 --- a/deploy/jobs/env/test.env +++ b/deploy/jobs/env/test.env @@ -11,3 +11,4 @@ COINGECKO_URL=https://api.coingecko.com/api/v3 NOTIONAL_CHANNEL=WORMSCAN:NOTIONAL LOG_LEVEL=INFO CRONTAB_SCHEDULE=*/5 * * * * +CACHE_PREFIX=mainnet-test diff --git a/deploy/jobs/notional.yaml b/deploy/jobs/notional.yaml index 97e02f09..c4b23941 100644 --- a/deploy/jobs/notional.yaml +++ b/deploy/jobs/notional.yaml @@ -31,4 +31,9 @@ spec: configMapKeyRef: name: config key: redis-uri + - name: CACHE_PREFIX + valueFrom: + configMapKeyRef: + name: config + key: redis-prefix restartPolicy: OnFailure diff --git a/fly/config/config.go b/fly/config/config.go index cab2ecd8..aa16f2d7 100644 --- a/fly/config/config.go +++ b/fly/config/config.go @@ -115,3 +115,12 @@ func GetMetricsEnabled() bool { } return metricsEnabled } + +func GetPrefix() string { + p2pNetwork, err := GetP2pNetwork() + if err != nil { + return "" + } + prefix := p2pNetwork.Enviroment + "-" + GetEnviroment() + return prefix +} diff --git a/fly/config/config_test.go b/fly/config/config_test.go new file mode 100644 index 00000000..6a954ca9 --- /dev/null +++ b/fly/config/config_test.go @@ -0,0 +1,27 @@ +package config + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetPrefix(t *testing.T) { + os.Clearenv() + os.Setenv("P2P_NETWORK", "mainnet") + os.Setenv("ENVIROMENT", "staging") + + prefix := GetPrefix() + + assert.Equal(t, "mainnet-staging", prefix) +} + +func TestGetPrefixNoP2P(t *testing.T) { + os.Clearenv() + os.Setenv("ENVIROMENT", "staging") + + prefix := GetPrefix() + + assert.Equal(t, "", prefix) +} diff --git a/fly/go.mod b/fly/go.mod index 693301ac..5433c398 100644 --- a/fly/go.mod +++ b/fly/go.mod @@ -18,6 +18,7 @@ require ( github.com/libp2p/go-libp2p-core v0.20.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/stretchr/testify v1.8.1 + github.com/test-go/testify v1.1.4 github.com/wormhole-foundation/wormhole/sdk v0.0.0-20230426150516-e695fad0bed8 go.mongodb.org/mongo-driver v1.11.2 go.uber.org/zap v1.24.0 diff --git a/fly/go.sum b/fly/go.sum index 14cd4243..67b1b6c2 100644 --- a/fly/go.sum +++ b/fly/go.sum @@ -2923,6 +2923,7 @@ github.com/tendermint/tm-db v0.6.7/go.mod h1:byQDzFkZV1syXr/ReXS808NxA2xvyuuVgXO github.com/tenntenn/modver v1.0.1/go.mod h1:bePIyQPb7UeioSRkw3Q0XeMhYZSMx9B8ePqg6SAMGH0= github.com/tenntenn/text/transform v0.0.0-20200319021203-7eef512accb3/go.mod h1:ON8b8w4BN/kE1EOhwT0o+d62W65a6aPw1nouo9LMgyY= github.com/test-go/testify v1.1.4 h1:Tf9lntrKUMHiXQ07qBScBTSA0dhYQlu83hswqelv1iE= +github.com/test-go/testify v1.1.4/go.mod h1:rH7cfJo/47vWGdi4GPj16x3/t1xGOj2YxzmNQzk2ghU= github.com/tetafro/godot v0.3.7/go.mod h1:/7NLHhv08H1+8DNj0MElpAACw1ajsCuf3TKNQxA5S+0= github.com/tetafro/godot v0.4.2/go.mod h1:/7NLHhv08H1+8DNj0MElpAACw1ajsCuf3TKNQxA5S+0= github.com/tetafro/godot v1.4.11/go.mod h1:LR3CJpxDVGlYOWn3ZZg1PgNZdTUvzsZWu8xaEohUpn8= diff --git a/fly/main.go b/fly/main.go index e0167f80..6b9c8788 100644 --- a/fly/main.go +++ b/fly/main.go @@ -175,13 +175,15 @@ func newVAANotifierFunc(isLocal bool, logger *zap.Logger) processor.VAANotifyFun } redisUri, err := getenv("REDIS_URI") + prefix := strings.ToLower(config.GetPrefix()) if err != nil { logger.Fatal("could not create vaa notifier ", zap.Error(err)) } + logger.Info("using redis notifier", zap.String("prefix", prefix)) client := redis.NewClient(&redis.Options{Addr: redisUri}) - return notifier.NewLastSequenceNotifier(client).Notify + return notifier.NewLastSequenceNotifier(client, prefix).Notify } func newAlertClient() (alert.AlertClient, error) { diff --git a/fly/notifier/vaa_last_sequence.go b/fly/notifier/vaa_last_sequence.go index f3981b25..62ee36c1 100644 --- a/fly/notifier/vaa_last_sequence.go +++ b/fly/notifier/vaa_last_sequence.go @@ -36,17 +36,25 @@ end type LastSequenceNotifier struct { client *redis.Client script *redis.Script + prefix string } -func NewLastSequenceNotifier(c *redis.Client) *LastSequenceNotifier { +func NewLastSequenceNotifier(c *redis.Client, prefix string) *LastSequenceNotifier { + if prefix == "" { + prefix = "wormscan:vaa-max-sequence" + } else { + prefix = fmt.Sprintf("%s-wormscan:vaa-max-sequence", prefix) + } + return &LastSequenceNotifier{ client: c, script: redis.NewScript(LUA_SCRIPT), + prefix: prefix, } } func (l *LastSequenceNotifier) Notify(ctx context.Context, v *vaa.VAA, _ []byte) error { - key := fmt.Sprintf("wormscan:vaa-max-sequence:%d:%s", v.EmitterChain, v.EmitterAddress.String()) + key := fmt.Sprintf("%s:%d:%s", l.prefix, v.EmitterChain, v.EmitterAddress.String()) sequence := strconv.FormatUint(v.Sequence, 10) _, err := l.script.Run(ctx, l.client, []string{key}, sequence).Result() return err diff --git a/fly/notifier/vaa_last_sequence_test.go b/fly/notifier/vaa_last_sequence_test.go new file mode 100644 index 00000000..797b114b --- /dev/null +++ b/fly/notifier/vaa_last_sequence_test.go @@ -0,0 +1,37 @@ +package notifier + +import ( + "os" + "testing" + + "github.com/test-go/testify/assert" + "github.com/wormhole-foundation/wormhole-explorer/fly/config" +) + +func TestNewLastSequenceNotifier(t *testing.T) { + + l := NewLastSequenceNotifier(nil, "mainnet-staging") + + assert.Equal(t, "mainnet-staging-wormscan:vaa-max-sequence", l.prefix) +} + +func TestNewLastSequenceNotifierBackwardsCompat(t *testing.T) { + + prefix := config.GetPrefix() + + l := NewLastSequenceNotifier(nil, prefix) + + assert.Equal(t, "wormscan:vaa-max-sequence", l.prefix) +} + +func TestNewLastSequenceNotifierWithPrefix(t *testing.T) { + + os.Setenv("P2P_NETWORK", "mainnet") + os.Setenv("ENVIROMENT", "staging") + + prefix := config.GetPrefix() + + l := NewLastSequenceNotifier(nil, prefix) + + assert.Equal(t, "mainnet-staging-wormscan:vaa-max-sequence", l.prefix) +} diff --git a/jobs/cmd/main.go b/jobs/cmd/main.go index 1dd708b7..415d124c 100644 --- a/jobs/cmd/main.go +++ b/jobs/cmd/main.go @@ -53,7 +53,7 @@ func initNotionalJob(ctx context.Context, cfg *config.Configuration, logger *zap // init redis client. redisClient := redis.NewClient(&redis.Options{Addr: cfg.CacheURL}) // create notional job. - notionalJob := notional.NewNotionalJob(api, redisClient, cfg.P2pNetwork, cfg.NotionalChannel, logger) + notionalJob := notional.NewNotionalJob(api, redisClient, cfg.CachePrefix, cfg.P2pNetwork, cfg.NotionalChannel, logger) return notionalJob } diff --git a/jobs/config/config.go b/jobs/config/config.go index 179c9a8a..3e988413 100644 --- a/jobs/config/config.go +++ b/jobs/config/config.go @@ -16,6 +16,7 @@ type Configuration struct { JobID string `env:"JOB_ID,required"` CoingeckoURL string `env:"COINGECKO_URL,required"` CacheURL string `env:"CACHE_URL,required"` + CachePrefix string `env:"CACHE_PREFIX,required"` NotionalChannel string `env:"NOTIONAL_CHANNEL,required"` P2pNetwork string `env:"P2P_NETWORK,required"` } diff --git a/jobs/jobs/notional/notional.go b/jobs/jobs/notional/notional.go index e77ca758..934fad35 100644 --- a/jobs/jobs/notional/notional.go +++ b/jobs/jobs/notional/notional.go @@ -16,16 +16,18 @@ import ( type NotionalJob struct { coingeckoAPI *coingecko.CoingeckoAPI cacheClient *redis.Client + cachePrefix string cacheChannel string p2pNetwork string logger *zap.Logger } // NewNotionalJob creates a new notional job. -func NewNotionalJob(api *coingecko.CoingeckoAPI, cacheClient *redis.Client, p2pNetwork, cacheChannel string, logger *zap.Logger) *NotionalJob { +func NewNotionalJob(api *coingecko.CoingeckoAPI, cacheClient *redis.Client, cachePrefix string, p2pNetwork, cacheChannel string, logger *zap.Logger) *NotionalJob { return &NotionalJob{ coingeckoAPI: api, cacheClient: cacheClient, + cachePrefix: cachePrefix, cacheChannel: cacheChannel, p2pNetwork: p2pNetwork, logger: logger, @@ -76,7 +78,7 @@ func (j *NotionalJob) Run() error { func (j *NotionalJob) updateNotionalCache(notionals map[domain.Symbol]notional.PriceData) error { for chainID, n := range notionals { - key := fmt.Sprintf(notional.KeyFormatString, chainID) + key := j.renderKey(fmt.Sprintf(notional.KeyFormatString, chainID)) err := j.cacheClient.Set(key, n, 0).Err() if err != nil { return err @@ -113,3 +115,10 @@ func convertToSymbols(m map[string]coingecko.NotionalUSD) map[domain.Symbol]noti return w } + +func (j *NotionalJob) renderKey(key string) string { + if j.cachePrefix != "" { + return fmt.Sprintf("%s-%s", j.cachePrefix, key) + } + return key +}