diff --git a/Makefile b/Makefile index c2e18598..9b1aa31b 100644 --- a/Makefile +++ b/Makefile @@ -8,9 +8,9 @@ help: @sed -n 's/^##//p' ${MAKEFILE_LIST} | column -t -s ':' | sed -e 's/^/ /' build: + make -C analytics/ build make -C api/ build make -C fly/ build - make -C influx-backfiller/ build make -C spy/ build make -C parser/ build make -C tx-tracker/ build @@ -20,9 +20,9 @@ doc: swag init -pd test: + cd analytics && go test -v -cover ./... cd api && go test -v -cover ./... cd fly && go test -v -cover ./... - cd influx-backfiller && go test -v -cover ./... cd spy && go test -v -cover ./... cd parser && go test -v -cover ./... cd tx-tracker && go test -v -cover ./... diff --git a/analytic/.env.example b/analytics/.env.example similarity index 100% rename from analytic/.env.example rename to analytics/.env.example diff --git a/analytic/.gitignore b/analytics/.gitignore similarity index 65% rename from analytic/.gitignore rename to analytics/.gitignore index 6c0c3f40..4c997942 100644 --- a/analytic/.gitignore +++ b/analytics/.gitignore @@ -1,3 +1,3 @@ __debug_bin .env -analytic +bin/* diff --git a/analytic/Dockerfile b/analytics/Dockerfile similarity index 70% rename from analytic/Dockerfile rename to analytics/Dockerfile index 737b4818..56957b7a 100644 --- a/analytic/Dockerfile +++ b/analytics/Dockerfile @@ -3,11 +3,11 @@ FROM --platform=linux/amd64 docker.io/golang:1.19.2@sha256:0467d7d12d170ed8d998a WORKDIR /app -COPY analytic analytic +COPY analytics analytics COPY common common # Build the Go app -RUN cd analytic && CGO_ENABLED=0 GOOS=linux go build -o "./analytic-pipeline" cmd/main.go +RUN cd analytics && CGO_ENABLED=0 GOOS=linux go build -o "./analytics-pipeline" cmd/main.go ############################ # STEP 2 build a small image @@ -16,6 +16,6 @@ FROM alpine #Copy certificates COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ # Copy our static executable. -COPY --from=build "/app/analytic/analytic-pipeline" "/analytic-pipeline" +COPY --from=build "/app/analytics/analytics-pipeline" "/analytics-pipeline" # Run the binary. -ENTRYPOINT ["/analytic-pipeline"] \ No newline at end of file +ENTRYPOINT ["/analytics-pipeline"] \ No newline at end of file diff --git a/analytic/Makefile b/analytics/Makefile similarity index 86% rename from analytic/Makefile rename to analytics/Makefile index 30c2018c..9f757993 100644 --- a/analytic/Makefile +++ b/analytics/Makefile @@ -8,7 +8,7 @@ help: @sed -n 's/^##//p' ${MAKEFILE_LIST} | column -t -s ':' | sed -e 's/^/ /' build: - go build -o analytic cmd/main.go + go build -o bin/analytics cmd/main.go doc: swag init -pd diff --git a/analytic/README.md b/analytics/README.md similarity index 100% rename from analytic/README.md rename to analytics/README.md diff --git a/influx-backfiller/cmd/commands.go b/analytics/cmd/main.go similarity index 66% rename from influx-backfiller/cmd/commands.go rename to analytics/cmd/main.go index 51148f3d..d9a12a8b 100644 --- a/influx-backfiller/cmd/commands.go +++ b/analytics/cmd/main.go @@ -1,26 +1,44 @@ -package cmd +package main import ( - "os" - "github.com/spf13/cobra" - "github.com/xlabs/influx-backfiller/cmd/metrics" - "github.com/xlabs/influx-backfiller/cmd/prices" + "github.com/wormhole-foundation/wormhole-explorer/analytics/cmd/metrics" + "github.com/wormhole-foundation/wormhole-explorer/analytics/cmd/prices" + "github.com/wormhole-foundation/wormhole-explorer/analytics/cmd/service" ) -// Execute executes the root command. -func Execute() error { +func main() { + execute() +} +func execute() error { root := &cobra.Command{ - Use: "backfiller", + Use: "analytics", Run: func(cmd *cobra.Command, args []string) { if len(args) == 0 { - _ = cmd.Help() - os.Exit(0) + service.Run() } }, } + addServiceCommand(root) + addBackfiller(root) + + return root.Execute() +} + +func addServiceCommand(root *cobra.Command) { + serviceCommand := &cobra.Command{ + Use: "service", + Short: "Run analytics as service", + Run: func(_ *cobra.Command, _ []string) { + service.Run() + }, + } + root.AddCommand(serviceCommand) +} + +func addBackfiller(root *cobra.Command) { metrics := &cobra.Command{ Use: "metrics", } @@ -33,8 +51,6 @@ func Execute() error { } addPricesCommand(prices) root.AddCommand(prices) - - return root.Execute() } func addVaaCountCommand(parent *cobra.Command) { @@ -46,30 +62,33 @@ func addVaaCountCommand(parent *cobra.Command) { metrics.RunVaaCount(input, output) }, } - //input flag + // input flag vaaCountCmd.Flags().StringVar(&input, "input", "", "path to input vaa file") vaaCountCmd.MarkFlagRequired("input") - //output flag + // output flag vaaCountCmd.Flags().StringVar(&output, "output", "", "path to output file") vaaCountCmd.MarkFlagRequired("output") parent.AddCommand(vaaCountCmd) } func addVaaVolumeCommand(parent *cobra.Command) { - var input, output string + var input, output, prices string vaaVolumeCmd := &cobra.Command{ Use: "vaa-volume", Short: "Generate volume metrics from a VAA csv file", Run: func(_ *cobra.Command, _ []string) { - metrics.RunVaaVolume(input, output) + metrics.RunVaaVolume(input, output, prices) }, } - //input flag + // input flag vaaVolumeCmd.Flags().StringVar(&input, "input", "", "path to input vaa file") vaaVolumeCmd.MarkFlagRequired("input") - //output flag + // output flag vaaVolumeCmd.Flags().StringVar(&output, "output", "", "path to output file") vaaVolumeCmd.MarkFlagRequired("output") + // prices flag + vaaVolumeCmd.Flags().StringVar(&prices, "prices", "prices.csv", "path to prices file") + parent.AddCommand(vaaVolumeCmd) } @@ -82,7 +101,7 @@ func addPricesCommand(root *cobra.Command) { prices.RunPrices(output) }, } - //output flag + // output flag vaaCountCmd.Flags().StringVar(&output, "output", "", "path to output file") vaaCountCmd.MarkFlagRequired("output") root.AddCommand(vaaCountCmd) diff --git a/influx-backfiller/cmd/metrics/common.go b/analytics/cmd/metrics/common.go similarity index 74% rename from influx-backfiller/cmd/metrics/common.go rename to analytics/cmd/metrics/common.go index 3bf2a4af..dca5fc13 100644 --- a/influx-backfiller/cmd/metrics/common.go +++ b/analytics/cmd/metrics/common.go @@ -24,7 +24,14 @@ func convertPointToLineProtocol(point *write.Point) string { } var tmp []string for _, f := range point.FieldList() { - tmp = append(tmp, fmt.Sprintf("%s=%v", f.Key, f.Value)) + switch f.Value.(type) { + case string: + tmp = append(tmp, fmt.Sprintf("%s=\"%v\"", f.Key, f.Value)) + case uint64, uint32, uint16, uint8: + tmp = append(tmp, fmt.Sprintf("%s=%vu", f.Key, f.Value)) + default: + tmp = append(tmp, fmt.Sprintf("%s=%v", f.Key, f.Value)) + } } fields := strings.Join(tmp, ",") diff --git a/influx-backfiller/cmd/metrics/vaa_count.go b/analytics/cmd/metrics/vaa_count.go similarity index 87% rename from influx-backfiller/cmd/metrics/vaa_count.go rename to analytics/cmd/metrics/vaa_count.go index 9a6bff74..f88de2b0 100644 --- a/influx-backfiller/cmd/metrics/vaa_count.go +++ b/analytics/cmd/metrics/vaa_count.go @@ -4,9 +4,9 @@ import ( "context" "os" - "github.com/wormhole-foundation/wormhole-explorer/analytic/metric" + "github.com/wormhole-foundation/wormhole-explorer/analytics/metric" + "github.com/wormhole-foundation/wormhole-explorer/analytics/parser" sdk "github.com/wormhole-foundation/wormhole/sdk/vaa" - "github.com/xlabs/influx-backfiller/parser" ) func RunVaaCount(inputFile, outputFile string) { diff --git a/influx-backfiller/cmd/metrics/volume.go b/analytics/cmd/metrics/volume.go similarity index 79% rename from influx-backfiller/cmd/metrics/volume.go rename to analytics/cmd/metrics/volume.go index 57c93d2b..bef6e2d2 100755 --- a/influx-backfiller/cmd/metrics/volume.go +++ b/analytics/cmd/metrics/volume.go @@ -6,17 +6,18 @@ import ( "errors" "fmt" "io" - "log" "os" "strings" "time" "github.com/influxdata/influxdb-client-go/v2/api/write" "github.com/shopspring/decimal" - "github.com/wormhole-foundation/wormhole-explorer/analytic/metric" + "github.com/wormhole-foundation/wormhole-explorer/analytics/metric" + "github.com/wormhole-foundation/wormhole-explorer/analytics/prices" "github.com/wormhole-foundation/wormhole-explorer/common/domain" + "github.com/wormhole-foundation/wormhole-explorer/common/logger" sdk "github.com/wormhole-foundation/wormhole/sdk/vaa" - "github.com/xlabs/influx-backfiller/prices" + "go.uber.org/zap" ) type LineParser struct { @@ -27,12 +28,17 @@ type LineParser struct { // read a csv file with VAAs and convert into a decoded csv file // ready to upload to the database -func RunVaaVolume(inputFile, outputFile string) { +func RunVaaVolume(inputFile, outputFile, pricesFile string) { + + // build logger + logger := logger.New("wormhole-explorer-analytics") + + logger.Info("starting wormhole-explorer-analytics ...") // open input file f, err := os.Open(inputFile) if err != nil { - panic(err) + logger.Fatal("opening input file", zap.Error(err)) } defer f.Close() @@ -40,22 +46,22 @@ func RunVaaVolume(inputFile, outputFile string) { missingTokensFile := "missing_tokens.csv" fmissingTokens, err := os.Create(missingTokensFile) if err != nil { - panic(err) + logger.Fatal("creating missing tokens file", zap.Error(err)) } defer fmissingTokens.Close() //open output file for writing fout, err := os.Create(outputFile) if err != nil { - panic(err) + logger.Fatal("creating output file", zap.Error(err)) } defer fout.Close() // init price cache! - fmt.Println("loading historical prices...") - lp := NewLineParser() + logger.Info("loading historical prices...") + lp := NewLineParser(pricesFile) lp.PriceCache.InitCache() - fmt.Println("done!") + logger.Info("loaded historical prices") r := bufio.NewReader(f) @@ -68,7 +74,7 @@ func RunVaaVolume(inputFile, outputFile string) { if err == io.EOF { break } - log.Fatalf("a real error happened here: %v\n", err) + logger.Fatal("a real error happened here", zap.Error(err)) } nl, err := lp.ParseLine(line) if err != nil { @@ -91,17 +97,18 @@ func RunVaaVolume(inputFile, outputFile string) { } - for k, v := range lp.MissingTokensCounter { - fmt.Printf("missing token %s %s %d\n", k.String(), lp.MissingTokens[k], v) + for k := range lp.MissingTokensCounter { fmissingTokens.WriteString(fmt.Sprintf("%s,%s,%d\n", k.String(), lp.MissingTokens[k], lp.MissingTokensCounter[k])) } - fmt.Println("done!") + logger.Info("missing tokens", zap.Int("count", len(lp.MissingTokens))) + + logger.Info("finished wormhole-explorer-analytics") } -func NewLineParser() *LineParser { - priceCache := prices.NewCoinPricesCache("prices.csv") +func NewLineParser(filename string) *LineParser { + priceCache := prices.NewCoinPricesCache(filename) return &LineParser{ MissingTokens: make(map[sdk.Address]sdk.ChainID), MissingTokensCounter: make(map[sdk.Address]int), diff --git a/influx-backfiller/cmd/prices/main.go b/analytics/cmd/prices/run.go similarity index 50% rename from influx-backfiller/cmd/prices/main.go rename to analytics/cmd/prices/run.go index aed04008..6d31ba59 100644 --- a/influx-backfiller/cmd/prices/main.go +++ b/analytics/cmd/prices/run.go @@ -5,8 +5,10 @@ import ( "os" "time" + "github.com/wormhole-foundation/wormhole-explorer/analytics/coingecko" "github.com/wormhole-foundation/wormhole-explorer/common/domain" - "github.com/xlabs/influx-backfiller/coingecko" + "github.com/wormhole-foundation/wormhole-explorer/common/logger" + "go.uber.org/zap" ) // go througth the symbol list provided by wormhole @@ -14,16 +16,27 @@ import ( // and save it to a file func RunPrices(output string) { + // build logger + logger := logger.New("wormhole-explorer-analytics") + + logger.Info("starting wormhole-explorer-analytics ...") + cg := coingecko.NewCoinGeckoAPI("") pricesOutput, err := os.Create(output) if err != nil { - panic(err) + logger.Fatal("creating file", zap.Error(err)) } defer pricesOutput.Close() - for _, token := range domain.GetAllTokens() { - fmt.Printf("%s [%s]\n", token.CoingeckoID, token.Symbol) + tokens := domain.GetAllTokens() + logger.Info("found tokens", zap.Int("count", len(tokens))) + for index, token := range tokens { + logger.Info("processing token", + zap.String("coingeckoID", token.CoingeckoID), + zap.Stringer("symbol", token.Symbol), + zap.Int("index", index+1), zap.Int("count", len(tokens))) + r, err := cg.GetSymbolDailyPrice(token.CoingeckoID) if err != nil { fmt.Println(err) @@ -37,4 +50,6 @@ func RunPrices(output string) { } + logger.Info("finished wormhole-explorer-analytics") + } diff --git a/analytic/cmd/main.go b/analytics/cmd/service/run.go similarity index 90% rename from analytic/cmd/main.go rename to analytics/cmd/service/run.go index 7deae099..66f3c30f 100644 --- a/analytic/cmd/main.go +++ b/analytics/cmd/service/run.go @@ -1,4 +1,4 @@ -package main +package service import ( "context" @@ -13,11 +13,11 @@ import ( "github.com/aws/aws-sdk-go-v2/credentials" "github.com/go-redis/redis/v8" influxdb2 "github.com/influxdata/influxdb-client-go/v2" - "github.com/wormhole-foundation/wormhole-explorer/analytic/config" - "github.com/wormhole-foundation/wormhole-explorer/analytic/consumer" - "github.com/wormhole-foundation/wormhole-explorer/analytic/http/infrastructure" - "github.com/wormhole-foundation/wormhole-explorer/analytic/metric" - "github.com/wormhole-foundation/wormhole-explorer/analytic/queue" + "github.com/wormhole-foundation/wormhole-explorer/analytics/config" + "github.com/wormhole-foundation/wormhole-explorer/analytics/consumer" + "github.com/wormhole-foundation/wormhole-explorer/analytics/http/infrastructure" + "github.com/wormhole-foundation/wormhole-explorer/analytics/metric" + "github.com/wormhole-foundation/wormhole-explorer/analytics/queue" wormscanNotionalCache "github.com/wormhole-foundation/wormhole-explorer/common/client/cache/notional" sqs_client "github.com/wormhole-foundation/wormhole-explorer/common/client/sqs" health "github.com/wormhole-foundation/wormhole-explorer/common/health" @@ -36,7 +36,7 @@ func handleExit() { } } -func main() { +func Run() { defer handleExit() rootCtx, rootCtxCancel := context.WithCancel(context.Background()) @@ -47,9 +47,9 @@ func main() { } // build logger - logger := logger.New("wormhole-explorer-analytic", logger.WithLevel(config.LogLevel)) + logger := logger.New("wormhole-explorer-analytics", logger.WithLevel(config.LogLevel)) - logger.Info("Starting wormhole-explorer-analytic ...") + logger.Info("Starting wormhole-explorer-analytics ...") // create influxdb client. influxCli := newInfluxClient(config.InfluxUrl, config.InfluxToken) @@ -101,7 +101,7 @@ func main() { metric.Close() logger.Info("Closing Http server ...") server.Stop() - logger.Info("Finished wormhole-explorer-analytic") + logger.Info("Finished wormhole-explorer-analytics") } // Creates a callbacks depending on whether the execution is local (memory queue) or not (SQS queue) diff --git a/influx-backfiller/cmd/token_fetcher/main.go b/analytics/cmd/token_fetcher/main.go similarity index 96% rename from influx-backfiller/cmd/token_fetcher/main.go rename to analytics/cmd/token_fetcher/main.go index ce0b6c04..13384ad0 100644 --- a/influx-backfiller/cmd/token_fetcher/main.go +++ b/analytics/cmd/token_fetcher/main.go @@ -11,7 +11,7 @@ import ( "time" "github.com/mr-tron/base58" - "github.com/xlabs/influx-backfiller/coingecko" + "github.com/wormhole-foundation/wormhole-explorer/analytics/coingecko" ) func main() { diff --git a/influx-backfiller/coingecko/api.go b/analytics/coingecko/api.go similarity index 100% rename from influx-backfiller/coingecko/api.go rename to analytics/coingecko/api.go diff --git a/influx-backfiller/coingecko/assets.go b/analytics/coingecko/assets.go similarity index 100% rename from influx-backfiller/coingecko/assets.go rename to analytics/coingecko/assets.go diff --git a/influx-backfiller/coingecko/types.go b/analytics/coingecko/types.go similarity index 100% rename from influx-backfiller/coingecko/types.go rename to analytics/coingecko/types.go diff --git a/analytic/config/config.go b/analytics/config/config.go similarity index 100% rename from analytic/config/config.go rename to analytics/config/config.go diff --git a/analytic/consumer/consumer.go b/analytics/consumer/consumer.go similarity index 90% rename from analytic/consumer/consumer.go rename to analytics/consumer/consumer.go index 673f72a5..c6d840d9 100644 --- a/analytic/consumer/consumer.go +++ b/analytics/consumer/consumer.go @@ -3,8 +3,8 @@ package consumer import ( "context" - "github.com/wormhole-foundation/wormhole-explorer/analytic/metric" - "github.com/wormhole-foundation/wormhole-explorer/analytic/queue" + "github.com/wormhole-foundation/wormhole-explorer/analytics/metric" + "github.com/wormhole-foundation/wormhole-explorer/analytics/queue" sdk "github.com/wormhole-foundation/wormhole/sdk/vaa" "go.uber.org/zap" ) diff --git a/analytic/go.mod b/analytics/go.mod similarity index 95% rename from analytic/go.mod rename to analytics/go.mod index abccbdd6..6023af77 100644 --- a/analytic/go.mod +++ b/analytics/go.mod @@ -1,4 +1,4 @@ -module github.com/wormhole-foundation/wormhole-explorer/analytic +module github.com/wormhole-foundation/wormhole-explorer/analytics go 1.19 @@ -11,6 +11,8 @@ require ( github.com/influxdata/influxdb-client-go/v2 v2.12.2 github.com/joho/godotenv v1.5.1 github.com/sethvargo/go-envconfig v0.9.0 + github.com/shopspring/decimal v1.3.1 + github.com/spf13/cobra v1.7.0 github.com/wormhole-foundation/wormhole-explorer/common v0.0.0-00010101000000-000000000000 github.com/wormhole-foundation/wormhole/sdk v0.0.0-20230426150516-e695fad0bed8 go.uber.org/zap v1.24.0 @@ -37,6 +39,7 @@ require ( github.com/golang/snappy v0.0.4 // indirect github.com/google/uuid v1.3.0 // indirect github.com/holiman/uint256 v1.2.1 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 // indirect github.com/klauspost/compress v1.15.9 // indirect github.com/mattn/go-colorable v0.1.13 // indirect @@ -48,6 +51,7 @@ require ( github.com/rivo/uniseg v0.2.0 // indirect github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94 // indirect github.com/savsgio/gotils v0.0.0-20220530130905-52f3993e8d6d // indirect + github.com/spf13/pflag v1.0.5 // indirect github.com/tinylib/msgp v1.1.6 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasthttp v1.44.0 // indirect diff --git a/analytic/go.sum b/analytics/go.sum similarity index 96% rename from analytic/go.sum rename to analytics/go.sum index d74adc18..959d4cf7 100644 --- a/analytic/go.sum +++ b/analytics/go.sum @@ -34,6 +34,7 @@ github.com/btcsuite/btcd/btcec/v2 v2.2.0/go.mod h1:U7MHm051Al6XmscBQ0BoNydpOTsFA github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -73,6 +74,8 @@ github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/holiman/uint256 v1.2.1 h1:XRtyuda/zw2l+Bq/38n5XUoEF72aSOu/77Thd9pPp2o= github.com/holiman/uint256 v1.2.1/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/influxdata/influxdb-client-go/v2 v2.12.2 h1:uYABKdrEKlYm+++qfKdbgaHKBPmoWR5wpbmj6MBB/2g= github.com/influxdata/influxdb-client-go/v2 v2.12.2/go.mod h1:YteV91FiQxRdccyJ2cHvj2f/5sq4y4Njqu1fQzsQCOU= github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 h1:vilfsDSy7TDxedi9gyBkMvAirat/oRcL0lFdJBf6tdM= @@ -121,12 +124,19 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94 h1:rmMl4fXJhKMNWl+K+r/fq4FbbKI+Ia2m9hYBLm2h4G4= github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94/go.mod h1:90zrgN3D/WJsDd1iXHT96alCoN2KJo6/4x1DZC3wZs8= github.com/savsgio/gotils v0.0.0-20220530130905-52f3993e8d6d h1:Q+gqLBOPkFGHyCJxXMRqtUgUbTjI8/Ze8vu8GGyNFwo= github.com/savsgio/gotils v0.0.0-20220530130905-52f3993e8d6d/go.mod h1:Gy+0tqhJvgGlqnTF8CVGP0AaGRjwBtXs/a5PA0Y3+A4= github.com/sethvargo/go-envconfig v0.9.0 h1:Q6FQ6hVEeTECULvkJZakq3dZMeBQ3JUpcKMfPQbKMDE= github.com/sethvargo/go-envconfig v0.9.0/go.mod h1:Iz1Gy1Sf3T64TQlJSvee81qDhf7YIlt8GMUX6yyNFs0= +github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= +github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= diff --git a/analytic/http/infrastructure/controller.go b/analytics/http/infrastructure/controller.go similarity index 100% rename from analytic/http/infrastructure/controller.go rename to analytics/http/infrastructure/controller.go diff --git a/analytic/http/infrastructure/server.go b/analytics/http/infrastructure/server.go similarity index 100% rename from analytic/http/infrastructure/server.go rename to analytics/http/infrastructure/server.go diff --git a/analytic/metric/metric.go b/analytics/metric/metric.go similarity index 100% rename from analytic/metric/metric.go rename to analytics/metric/metric.go diff --git a/analytic/metric/types.go b/analytics/metric/types.go similarity index 100% rename from analytic/metric/types.go rename to analytics/metric/types.go diff --git a/influx-backfiller/parser/vaa_csv.go b/analytics/parser/vaa_csv.go similarity index 100% rename from influx-backfiller/parser/vaa_csv.go rename to analytics/parser/vaa_csv.go diff --git a/influx-backfiller/prices/prices.go b/analytics/prices/prices.go similarity index 100% rename from influx-backfiller/prices/prices.go rename to analytics/prices/prices.go diff --git a/analytic/queue/queue.go b/analytics/queue/queue.go similarity index 100% rename from analytic/queue/queue.go rename to analytics/queue/queue.go diff --git a/analytic/queue/vaa_sqs.go b/analytics/queue/vaa_sqs.go similarity index 100% rename from analytic/queue/vaa_sqs.go rename to analytics/queue/vaa_sqs.go diff --git a/analytic/scripts/all_messages_5m.flux b/analytics/scripts/all_messages_5m.flux similarity index 100% rename from analytic/scripts/all_messages_5m.flux rename to analytics/scripts/all_messages_5m.flux diff --git a/analytic/scripts/asset_volumes_24h.flux b/analytics/scripts/asset_volumes_24h.flux similarity index 100% rename from analytic/scripts/asset_volumes_24h.flux rename to analytics/scripts/asset_volumes_24h.flux diff --git a/analytic/scripts/chain_pair_transfers_24h.flux b/analytics/scripts/chain_pair_transfers_24h.flux similarity index 100% rename from analytic/scripts/chain_pair_transfers_24h.flux rename to analytics/scripts/chain_pair_transfers_24h.flux diff --git a/analytic/scripts/total_tx_count.flux b/analytics/scripts/total_tx_count.flux similarity index 100% rename from analytic/scripts/total_tx_count.flux rename to analytics/scripts/total_tx_count.flux diff --git a/analytic/scripts/total_tx_volume.flux b/analytics/scripts/total_tx_volume.flux similarity index 100% rename from analytic/scripts/total_tx_volume.flux rename to analytics/scripts/total_tx_volume.flux diff --git a/analytic/scripts/vaa_count_1h.flux b/analytics/scripts/vaa_count_1h.flux similarity index 100% rename from analytic/scripts/vaa_count_1h.flux rename to analytics/scripts/vaa_count_1h.flux diff --git a/influx-backfiller/common.go b/analytics/support/common.go similarity index 100% rename from influx-backfiller/common.go rename to analytics/support/common.go diff --git a/deploy/analytic/env/staging.env b/deploy/analytic/env/staging.env deleted file mode 100644 index 30c28d0c..00000000 --- a/deploy/analytic/env/staging.env +++ /dev/null @@ -1,15 +0,0 @@ -ENVIRONMENT=staging -NAMESPACE=wormscan -NAME=wormscan-analytic -REPLICAS=2 -IMAGE_NAME= -RESOURCES_LIMITS_MEMORY=30Mi -RESOURCES_LIMITS_CPU=20m -RESOURCES_REQUESTS_MEMORY=15Mi -RESOURCES_REQUESTS_CPU=10m -SQS_URL= -SQS_AWS_REGION= -P2P_NETWORK=mainnet -PPROF_ENABLED=true -AWS_IAM_ROLE= -CACHE_CHANNEL=WORMSCAN:NOTIONAL \ No newline at end of file diff --git a/deploy/analytic/analytic-service.yaml b/deploy/analytics/analytics-service.yaml similarity index 100% rename from deploy/analytic/analytic-service.yaml rename to deploy/analytics/analytics-service.yaml diff --git a/deploy/analytic/env/production.env b/deploy/analytics/env/production.env similarity index 82% rename from deploy/analytic/env/production.env rename to deploy/analytics/env/production.env index c0dcc176..6e2bc3ee 100644 --- a/deploy/analytic/env/production.env +++ b/deploy/analytics/env/production.env @@ -1,6 +1,6 @@ ENVIRONMENT=production NAMESPACE=wormscan -NAME=wormscan-analytic +NAME=wormscan-analytics REPLICAS=2 IMAGE_NAME= RESOURCES_LIMITS_MEMORY=64Mi diff --git a/deploy/analytics/env/staging.env b/deploy/analytics/env/staging.env new file mode 100644 index 00000000..aac13d57 --- /dev/null +++ b/deploy/analytics/env/staging.env @@ -0,0 +1,15 @@ +ENVIRONMENT=staging +NAMESPACE=wormscan +NAME=wormscan-analytics +REPLICAS=2 +IMAGE_NAME= +RESOURCES_LIMITS_MEMORY=64Mi +RESOURCES_LIMITS_CPU=200m +RESOURCES_REQUESTS_MEMORY=32Mi +RESOURCES_REQUESTS_CPU=100m +SQS_URL= +SQS_AWS_REGION= +P2P_NETWORK=mainnet +PPROF_ENABLED=true +AWS_IAM_ROLE= +CACHE_CHANNEL=WORMSCAN:NOTIONAL \ No newline at end of file diff --git a/deploy/analytic/env/test.env b/deploy/analytics/env/test.env similarity index 82% rename from deploy/analytic/env/test.env rename to deploy/analytics/env/test.env index 9feacc90..5a5c57de 100644 --- a/deploy/analytic/env/test.env +++ b/deploy/analytics/env/test.env @@ -1,6 +1,6 @@ ENVIRONMENT=test NAMESPACE=wormscan-testnet -NAME=wormscan-analytic +NAME=wormscan-analytics REPLICAS=1 IMAGE_NAME= RESOURCES_LIMITS_MEMORY=128Mi diff --git a/deploy/analytic/sa.yaml b/deploy/analytics/sa.yaml similarity index 100% rename from deploy/analytic/sa.yaml rename to deploy/analytics/sa.yaml diff --git a/go.work b/go.work index 7f7367db..32bed2be 100644 --- a/go.work +++ b/go.work @@ -1,12 +1,11 @@ go 1.19 use ( - ./analytic + ./analytics ./api ./common ./contract-watcher ./fly - ./influx-backfiller ./parser ./pipeline ./spy diff --git a/influx-backfiller/.gitignore b/influx-backfiller/.gitignore deleted file mode 100644 index a265f4da..00000000 --- a/influx-backfiller/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -.env -.vscode -bin/influx-backfiller diff --git a/influx-backfiller/Makefile b/influx-backfiller/Makefile deleted file mode 100644 index 1bf769d7..00000000 --- a/influx-backfiller/Makefile +++ /dev/null @@ -1,11 +0,0 @@ -SHELL := /bin/bash - - -build: - CGO_ENABLED=0 GOOS=linux go build -o ./bin/influx-backfiller - -test: - go test -v -cover ./... - - -.PHONY: build test diff --git a/influx-backfiller/README.md b/influx-backfiller/README.md deleted file mode 100755 index fe87fcec..00000000 --- a/influx-backfiller/README.md +++ /dev/null @@ -1,32 +0,0 @@ -# influx backfiller - -Takes CSV file with VAAs as input, and generates a [line protocol](https://docs.influxdata.com/influxdb/cloud/reference/syntax/line-protocol/) file for bulk loading into InfluxDB. - - - -## Usage - -Run the program to generate InfluxDB dump files: - -```bash -./influx-backfiller metrics vaa-count --input vaas-signed.csv --output vaa-count.csv -./influx-backfiller metrics vaa-volume --input vaas-signed.csv --output vaa-volume.csv -``` - -Then load the files into InfluxDB: - -```bash -influx write --bucket wormhole-explorer --file vaa-count.csv -influx write --bucket wormhole-explorer --file vaa-volume.csv -``` - -## Historic Prices - -The prices file is generated with `cmd/symbol_historic` and uses the coingecko api to fetch daily prices for -all supported symbols. - -There is a compressed version of the file already generated called [`prices.csv.gz`](https://github.com/XLabs/influx-backfiller/blob/master/prices.csv.gz). - - - - diff --git a/influx-backfiller/bin/.gitkeep b/influx-backfiller/bin/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/influx-backfiller/go.mod b/influx-backfiller/go.mod deleted file mode 100755 index a63e7dfc..00000000 --- a/influx-backfiller/go.mod +++ /dev/null @@ -1,27 +0,0 @@ -module github.com/xlabs/influx-backfiller - -go 1.19 - -require ( - github.com/mr-tron/base58 v1.2.0 - github.com/shopspring/decimal v1.3.1 - github.com/spf13/cobra v1.7.0 - github.com/wormhole-foundation/wormhole/sdk v0.0.0-20230501192138-3eefb74bfb4e -) - -require ( - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect -) - -require ( - github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect - github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect - github.com/ethereum/go-ethereum v1.10.21 // indirect - github.com/holiman/uint256 v1.2.1 // indirect - github.com/test-go/testify v1.1.4 - golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect - golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect -) diff --git a/influx-backfiller/go.sum b/influx-backfiller/go.sum deleted file mode 100755 index a13bbd12..00000000 --- a/influx-backfiller/go.sum +++ /dev/null @@ -1,39 +0,0 @@ -github.com/btcsuite/btcd/btcec/v2 v2.2.0 h1:fzn1qaOt32TuLjFlkzYSsBC35Q3KUjT1SwPxiMSCF5k= -github.com/btcsuite/btcd/btcec/v2 v2.2.0/go.mod h1:U7MHm051Al6XmscBQ0BoNydpOTsFAn707034b5nY8zU= -github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= -github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= -github.com/ethereum/go-ethereum v1.10.21 h1:5lqsEx92ZaZzRyOqBEXux4/UR06m296RGzN3ol3teJY= -github.com/ethereum/go-ethereum v1.10.21/go.mod h1:EYFyF19u3ezGLD4RqOkLq+ZCXzYbLoNDdZlMt7kyKFg= -github.com/holiman/uint256 v1.2.1 h1:XRtyuda/zw2l+Bq/38n5XUoEF72aSOu/77Thd9pPp2o= -github.com/holiman/uint256 v1.2.1/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= -github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= -github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= -github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= -github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= -github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= -github.com/test-go/testify v1.1.4 h1:Tf9lntrKUMHiXQ07qBScBTSA0dhYQlu83hswqelv1iE= -github.com/test-go/testify v1.1.4/go.mod h1:rH7cfJo/47vWGdi4GPj16x3/t1xGOj2YxzmNQzk2ghU= -github.com/wormhole-foundation/wormhole/sdk v0.0.0-20230501192138-3eefb74bfb4e h1:Yf0Mhp9MA8xU5MmyuBO3uzyUEm70c34ddM5vf3vwxb0= -github.com/wormhole-foundation/wormhole/sdk v0.0.0-20230501192138-3eefb74bfb4e/go.mod h1:dE12DOucCq23gjGGGhtbyx41FBxuHxjpPvG+ArO+8t0= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/influx-backfiller/main.go b/influx-backfiller/main.go deleted file mode 100644 index 23203ff3..00000000 --- a/influx-backfiller/main.go +++ /dev/null @@ -1,9 +0,0 @@ -package main - -import ( - "github.com/xlabs/influx-backfiller/cmd" -) - -func main() { - cmd.Execute() -}