Fix and refactor of analytics (#376)

Fix vaa-volume metric backfiller
Rename analytic to analytics
Move influx-backfiller as part of analytics

Makefile

@@ -8,9 +8,9 @@ help:
 	@sed -n 's/^##//p' ${MAKEFILE_LIST} | column -t -s ':' |  sed -e 's/^/ /'
 
 build:
+	make -C analytics/ build
 	make -C api/ build
 	make -C fly/ build
-	make -C influx-backfiller/ build
 	make -C spy/ build
 	make -C parser/ build
 	make -C tx-tracker/ build
@@ -20,9 +20,9 @@ doc:
 	swag init -pd
 
 test:
+	cd analytics && go test -v -cover ./...
 	cd api && go test -v -cover ./...
 	cd fly && go test -v -cover ./...
-	cd influx-backfiller && go test -v -cover ./...
 	cd spy && go test -v -cover ./...
 	cd parser && go test -v -cover ./...
 	cd tx-tracker && go test -v -cover ./...

analytics/.gitignore

@@ -1,3 +1,3 @@
 __debug_bin
 .env
-analytic
+bin/*

analytics/Dockerfile

@@ -3,11 +3,11 @@ FROM --platform=linux/amd64 docker.io/golang:1.19.2@sha256:0467d7d12d170ed8d998a
 
 WORKDIR /app
 
-COPY analytic analytic
+COPY analytics analytics
 COPY common common
 
 # Build the Go app
-RUN cd analytic && CGO_ENABLED=0 GOOS=linux go build -o "./analytic-pipeline" cmd/main.go
+RUN cd analytics && CGO_ENABLED=0 GOOS=linux go build -o "./analytics-pipeline" cmd/main.go
 
 ############################
 # STEP 2 build a small image
@@ -16,6 +16,6 @@ FROM alpine
 #Copy certificates
 COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 # Copy our static executable.
-COPY --from=build "/app/analytic/analytic-pipeline" "/analytic-pipeline"
+COPY --from=build "/app/analytics/analytics-pipeline" "/analytics-pipeline"
 # Run the binary.
-ENTRYPOINT ["/analytic-pipeline"]
+ENTRYPOINT ["/analytics-pipeline"]

analytics/Makefile

@@ -8,7 +8,7 @@ help:
 	@sed -n 's/^##//p' ${MAKEFILE_LIST} | column -t -s ':' |  sed -e 's/^/ /'
 
 build:
-	go build -o analytic cmd/main.go
+	go build -o bin/analytics cmd/main.go
 	
 doc:
 	swag init -pd

analytics/cmd/main.go

@@ -1,26 +1,44 @@
-package cmd
+package main
 
 import (
-	"os"
-
 	"github.com/spf13/cobra"
-	"github.com/xlabs/influx-backfiller/cmd/metrics"
-	"github.com/xlabs/influx-backfiller/cmd/prices"
+	"github.com/wormhole-foundation/wormhole-explorer/analytics/cmd/metrics"
+	"github.com/wormhole-foundation/wormhole-explorer/analytics/cmd/prices"
+	"github.com/wormhole-foundation/wormhole-explorer/analytics/cmd/service"
 )
 
-// Execute executes the root command.
-func Execute() error {
+func main() {
+	execute()
+}
 
+func execute() error {
 	root := &cobra.Command{
-		Use: "backfiller",
+		Use: "analytics",
 		Run: func(cmd *cobra.Command, args []string) {
 			if len(args) == 0 {
-				_ = cmd.Help()
-				os.Exit(0)
+				service.Run()
 			}
 		},
 	}
 
+	addServiceCommand(root)
+	addBackfiller(root)
+
+	return root.Execute()
+}
+
+func addServiceCommand(root *cobra.Command) {
+	serviceCommand := &cobra.Command{
+		Use:   "service",
+		Short: "Run analytics as service",
+		Run: func(_ *cobra.Command, _ []string) {
+			service.Run()
+		},
+	}
+	root.AddCommand(serviceCommand)
+}
+
+func addBackfiller(root *cobra.Command) {
 	metrics := &cobra.Command{
 		Use: "metrics",
 	}
@@ -33,8 +51,6 @@ func Execute() error {
 	}
 	addPricesCommand(prices)
 	root.AddCommand(prices)
-
-	return root.Execute()
 }
 
 func addVaaCountCommand(parent *cobra.Command) {
@@ -46,30 +62,33 @@ func addVaaCountCommand(parent *cobra.Command) {
 			metrics.RunVaaCount(input, output)
 		},
 	}
-	//input flag
+	// input flag
 	vaaCountCmd.Flags().StringVar(&input, "input", "", "path to input vaa file")
 	vaaCountCmd.MarkFlagRequired("input")
-	//output flag
+	// output flag
 	vaaCountCmd.Flags().StringVar(&output, "output", "", "path to output file")
 	vaaCountCmd.MarkFlagRequired("output")
 	parent.AddCommand(vaaCountCmd)
 }
 
 func addVaaVolumeCommand(parent *cobra.Command) {
-	var input, output string
+	var input, output, prices string
 	vaaVolumeCmd := &cobra.Command{
 		Use:   "vaa-volume",
 		Short: "Generate volume metrics from a VAA csv file",
 		Run: func(_ *cobra.Command, _ []string) {
-			metrics.RunVaaVolume(input, output)
+			metrics.RunVaaVolume(input, output, prices)
 		},
 	}
-	//input flag
+	// input flag
 	vaaVolumeCmd.Flags().StringVar(&input, "input", "", "path to input vaa file")
 	vaaVolumeCmd.MarkFlagRequired("input")
-	//output flag
+	// output flag
 	vaaVolumeCmd.Flags().StringVar(&output, "output", "", "path to output file")
 	vaaVolumeCmd.MarkFlagRequired("output")
+	// prices flag
+	vaaVolumeCmd.Flags().StringVar(&prices, "prices", "prices.csv", "path to prices file")
+
 	parent.AddCommand(vaaVolumeCmd)
 }
 
@@ -82,7 +101,7 @@ func addPricesCommand(root *cobra.Command) {
 			prices.RunPrices(output)
 		},
 	}
-	//output flag
+	// output flag
 	vaaCountCmd.Flags().StringVar(&output, "output", "", "path to output file")
 	vaaCountCmd.MarkFlagRequired("output")
 	root.AddCommand(vaaCountCmd)

analytics/cmd/metrics/common.go

@@ -24,7 +24,14 @@ func convertPointToLineProtocol(point *write.Point) string {
 	}
 	var tmp []string
 	for _, f := range point.FieldList() {
-		tmp = append(tmp, fmt.Sprintf("%s=%v", f.Key, f.Value))
+		switch f.Value.(type) {
+		case string:
+			tmp = append(tmp, fmt.Sprintf("%s=\"%v\"", f.Key, f.Value))
+		case uint64, uint32, uint16, uint8:
+			tmp = append(tmp, fmt.Sprintf("%s=%vu", f.Key, f.Value))
+		default:
+			tmp = append(tmp, fmt.Sprintf("%s=%v", f.Key, f.Value))
+		}
 	}
 	fields := strings.Join(tmp, ",")
 

analytics/cmd/metrics/vaa_count.go

@@ -4,9 +4,9 @@ import (
 	"context"
 	"os"
 
-	"github.com/wormhole-foundation/wormhole-explorer/analytic/metric"
+	"github.com/wormhole-foundation/wormhole-explorer/analytics/metric"
+	"github.com/wormhole-foundation/wormhole-explorer/analytics/parser"
 	sdk "github.com/wormhole-foundation/wormhole/sdk/vaa"
-	"github.com/xlabs/influx-backfiller/parser"
 )
 
 func RunVaaCount(inputFile, outputFile string) {

analytics/cmd/metrics/volume.go

@@ -6,17 +6,18 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"log"
 	"os"
 	"strings"
 	"time"
 
 	"github.com/influxdata/influxdb-client-go/v2/api/write"
 	"github.com/shopspring/decimal"
-	"github.com/wormhole-foundation/wormhole-explorer/analytic/metric"
+	"github.com/wormhole-foundation/wormhole-explorer/analytics/metric"
+	"github.com/wormhole-foundation/wormhole-explorer/analytics/prices"
 	"github.com/wormhole-foundation/wormhole-explorer/common/domain"
+	"github.com/wormhole-foundation/wormhole-explorer/common/logger"
 	sdk "github.com/wormhole-foundation/wormhole/sdk/vaa"
-	"github.com/xlabs/influx-backfiller/prices"
+	"go.uber.org/zap"
 )
 
 type LineParser struct {
@@ -27,12 +28,17 @@ type LineParser struct {
 
 // read a csv file with VAAs and convert into a decoded csv file
 // ready to upload to the database
-func RunVaaVolume(inputFile, outputFile string) {
+func RunVaaVolume(inputFile, outputFile, pricesFile string) {
+
+	// build logger
+	logger := logger.New("wormhole-explorer-analytics")
+
+	logger.Info("starting wormhole-explorer-analytics ...")
 
 	// open input file
 	f, err := os.Open(inputFile)
 	if err != nil {
-		panic(err)
+		logger.Fatal("opening input file", zap.Error(err))
 	}
 	defer f.Close()
 
@@ -40,22 +46,22 @@ func RunVaaVolume(inputFile, outputFile string) {
 	missingTokensFile := "missing_tokens.csv"
 	fmissingTokens, err := os.Create(missingTokensFile)
 	if err != nil {
-		panic(err)
+		logger.Fatal("creating missing tokens file", zap.Error(err))
 	}
 	defer fmissingTokens.Close()
 
 	//open output file for writing
 	fout, err := os.Create(outputFile)
 	if err != nil {
-		panic(err)
+		logger.Fatal("creating output file", zap.Error(err))
 	}
 	defer fout.Close()
 
 	// init price cache!
-	fmt.Println("loading historical prices...")
-	lp := NewLineParser()
+	logger.Info("loading historical prices...")
+	lp := NewLineParser(pricesFile)
 	lp.PriceCache.InitCache()
-	fmt.Println("done!")
+	logger.Info("loaded historical prices")
 
 	r := bufio.NewReader(f)
 
@@ -68,7 +74,7 @@ func RunVaaVolume(inputFile, outputFile string) {
 			if err == io.EOF {
 				break
 			}
-			log.Fatalf("a real error happened here: %v\n", err)
+			logger.Fatal("a real error happened here", zap.Error(err))
 		}
 		nl, err := lp.ParseLine(line)
 		if err != nil {
@@ -91,17 +97,18 @@ func RunVaaVolume(inputFile, outputFile string) {
 
 	}
 
-	for k, v := range lp.MissingTokensCounter {
-		fmt.Printf("missing token %s %s %d\n", k.String(), lp.MissingTokens[k], v)
+	for k := range lp.MissingTokensCounter {
 		fmissingTokens.WriteString(fmt.Sprintf("%s,%s,%d\n", k.String(), lp.MissingTokens[k], lp.MissingTokensCounter[k]))
 	}
 
-	fmt.Println("done!")
+	logger.Info("missing tokens", zap.Int("count", len(lp.MissingTokens)))
+
+	logger.Info("finished wormhole-explorer-analytics")
 
 }
 
-func NewLineParser() *LineParser {
-	priceCache := prices.NewCoinPricesCache("prices.csv")
+func NewLineParser(filename string) *LineParser {
+	priceCache := prices.NewCoinPricesCache(filename)
 	return &LineParser{
 		MissingTokens:        make(map[sdk.Address]sdk.ChainID),
 		MissingTokensCounter: make(map[sdk.Address]int),

analytics/cmd/prices/run.go

@@ -5,8 +5,10 @@ import (
 	"os"
 	"time"
 
+	"github.com/wormhole-foundation/wormhole-explorer/analytics/coingecko"
 	"github.com/wormhole-foundation/wormhole-explorer/common/domain"
-	"github.com/xlabs/influx-backfiller/coingecko"
+	"github.com/wormhole-foundation/wormhole-explorer/common/logger"
+	"go.uber.org/zap"
 )
 
 // go througth the symbol list provided by wormhole
@@ -14,16 +16,27 @@ import (
 // and save it to a file
 func RunPrices(output string) {
 
+	// build logger
+	logger := logger.New("wormhole-explorer-analytics")
+
+	logger.Info("starting wormhole-explorer-analytics ...")
+
 	cg := coingecko.NewCoinGeckoAPI("")
 
 	pricesOutput, err := os.Create(output)
 	if err != nil {
-		panic(err)
+		logger.Fatal("creating file", zap.Error(err))
 	}
 	defer pricesOutput.Close()
 
-	for _, token := range domain.GetAllTokens() {
-		fmt.Printf("%s [%s]\n", token.CoingeckoID, token.Symbol)
+	tokens := domain.GetAllTokens()
+	logger.Info("found tokens", zap.Int("count", len(tokens)))
+	for index, token := range tokens {
+		logger.Info("processing token",
+			zap.String("coingeckoID", token.CoingeckoID),
+			zap.Stringer("symbol", token.Symbol),
+			zap.Int("index", index+1), zap.Int("count", len(tokens)))
+
 		r, err := cg.GetSymbolDailyPrice(token.CoingeckoID)
 		if err != nil {
 			fmt.Println(err)
@@ -37,4 +50,6 @@ func RunPrices(output string) {
 
 	}
 
+	logger.Info("finished wormhole-explorer-analytics")
+
 }

analytics/cmd/service/run.go

@@ -1,4 +1,4 @@
-package main
+package service
 
 import (
 	"context"
@@ -13,11 +13,11 @@ import (
 	"github.com/aws/aws-sdk-go-v2/credentials"
 	"github.com/go-redis/redis/v8"
 	influxdb2 "github.com/influxdata/influxdb-client-go/v2"
-	"github.com/wormhole-foundation/wormhole-explorer/analytic/config"
-	"github.com/wormhole-foundation/wormhole-explorer/analytic/consumer"
-	"github.com/wormhole-foundation/wormhole-explorer/analytic/http/infrastructure"
-	"github.com/wormhole-foundation/wormhole-explorer/analytic/metric"
-	"github.com/wormhole-foundation/wormhole-explorer/analytic/queue"
+	"github.com/wormhole-foundation/wormhole-explorer/analytics/config"
+	"github.com/wormhole-foundation/wormhole-explorer/analytics/consumer"
+	"github.com/wormhole-foundation/wormhole-explorer/analytics/http/infrastructure"
+	"github.com/wormhole-foundation/wormhole-explorer/analytics/metric"
+	"github.com/wormhole-foundation/wormhole-explorer/analytics/queue"
 	wormscanNotionalCache "github.com/wormhole-foundation/wormhole-explorer/common/client/cache/notional"
 	sqs_client "github.com/wormhole-foundation/wormhole-explorer/common/client/sqs"
 	health "github.com/wormhole-foundation/wormhole-explorer/common/health"
@@ -36,7 +36,7 @@ func handleExit() {
 	}
 }
 
-func main() {
+func Run() {
 	defer handleExit()
 	rootCtx, rootCtxCancel := context.WithCancel(context.Background())
 
@@ -47,9 +47,9 @@ func main() {
 	}
 
 	// build logger
-	logger := logger.New("wormhole-explorer-analytic", logger.WithLevel(config.LogLevel))
+	logger := logger.New("wormhole-explorer-analytics", logger.WithLevel(config.LogLevel))
 
-	logger.Info("Starting wormhole-explorer-analytic ...")
+	logger.Info("Starting wormhole-explorer-analytics ...")
 
 	// create influxdb client.
 	influxCli := newInfluxClient(config.InfluxUrl, config.InfluxToken)
@@ -101,7 +101,7 @@ func main() {
 	metric.Close()
 	logger.Info("Closing Http server ...")
 	server.Stop()
-	logger.Info("Finished wormhole-explorer-analytic")
+	logger.Info("Finished wormhole-explorer-analytics")
 }
 
 // Creates a callbacks depending on whether the execution is local (memory queue) or not (SQS queue)

analytics/cmd/token_fetcher/main.go

@@ -11,7 +11,7 @@ import (
 	"time"
 
 	"github.com/mr-tron/base58"
-	"github.com/xlabs/influx-backfiller/coingecko"
+	"github.com/wormhole-foundation/wormhole-explorer/analytics/coingecko"
 )
 
 func main() {

analytics/consumer/consumer.go

@@ -3,8 +3,8 @@ package consumer
 import (
 	"context"
 
-	"github.com/wormhole-foundation/wormhole-explorer/analytic/metric"
-	"github.com/wormhole-foundation/wormhole-explorer/analytic/queue"
+	"github.com/wormhole-foundation/wormhole-explorer/analytics/metric"
+	"github.com/wormhole-foundation/wormhole-explorer/analytics/queue"
 	sdk "github.com/wormhole-foundation/wormhole/sdk/vaa"
 	"go.uber.org/zap"
 )

analytics/go.mod

@@ -1,4 +1,4 @@
-module github.com/wormhole-foundation/wormhole-explorer/analytic
+module github.com/wormhole-foundation/wormhole-explorer/analytics
 
 go 1.19
 
@@ -11,6 +11,8 @@ require (
 	github.com/influxdata/influxdb-client-go/v2 v2.12.2
 	github.com/joho/godotenv v1.5.1
 	github.com/sethvargo/go-envconfig v0.9.0
+	github.com/shopspring/decimal v1.3.1
+	github.com/spf13/cobra v1.7.0
 	github.com/wormhole-foundation/wormhole-explorer/common v0.0.0-00010101000000-000000000000
 	github.com/wormhole-foundation/wormhole/sdk v0.0.0-20230426150516-e695fad0bed8
 	go.uber.org/zap v1.24.0
@@ -37,6 +39,7 @@ require (
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/holiman/uint256 v1.2.1 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 // indirect
 	github.com/klauspost/compress v1.15.9 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
@@ -48,6 +51,7 @@ require (
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94 // indirect
 	github.com/savsgio/gotils v0.0.0-20220530130905-52f3993e8d6d // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/tinylib/msgp v1.1.6 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasthttp v1.44.0 // indirect

analytics/go.sum

@@ -34,6 +34,7 @@ github.com/btcsuite/btcd/btcec/v2 v2.2.0/go.mod h1:U7MHm051Al6XmscBQ0BoNydpOTsFA
 github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U=
 github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
 github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -73,6 +74,8 @@ github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/holiman/uint256 v1.2.1 h1:XRtyuda/zw2l+Bq/38n5XUoEF72aSOu/77Thd9pPp2o=
 github.com/holiman/uint256 v1.2.1/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/influxdata/influxdb-client-go/v2 v2.12.2 h1:uYABKdrEKlYm+++qfKdbgaHKBPmoWR5wpbmj6MBB/2g=
 github.com/influxdata/influxdb-client-go/v2 v2.12.2/go.mod h1:YteV91FiQxRdccyJ2cHvj2f/5sq4y4Njqu1fQzsQCOU=
 github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 h1:vilfsDSy7TDxedi9gyBkMvAirat/oRcL0lFdJBf6tdM=
@@ -121,12 +124,19 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94 h1:rmMl4fXJhKMNWl+K+r/fq4FbbKI+Ia2m9hYBLm2h4G4=
 github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94/go.mod h1:90zrgN3D/WJsDd1iXHT96alCoN2KJo6/4x1DZC3wZs8=
 github.com/savsgio/gotils v0.0.0-20220530130905-52f3993e8d6d h1:Q+gqLBOPkFGHyCJxXMRqtUgUbTjI8/Ze8vu8GGyNFwo=
 github.com/savsgio/gotils v0.0.0-20220530130905-52f3993e8d6d/go.mod h1:Gy+0tqhJvgGlqnTF8CVGP0AaGRjwBtXs/a5PA0Y3+A4=
 github.com/sethvargo/go-envconfig v0.9.0 h1:Q6FQ6hVEeTECULvkJZakq3dZMeBQ3JUpcKMfPQbKMDE=
 github.com/sethvargo/go-envconfig v0.9.0/go.mod h1:Iz1Gy1Sf3T64TQlJSvee81qDhf7YIlt8GMUX6yyNFs0=
+github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
+github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
+github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
+github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=

deploy/analytic/env/staging.env

@@ -1,15 +0,0 @@
-ENVIRONMENT=staging
-NAMESPACE=wormscan
-NAME=wormscan-analytic
-REPLICAS=2
-IMAGE_NAME=
-RESOURCES_LIMITS_MEMORY=30Mi
-RESOURCES_LIMITS_CPU=20m
-RESOURCES_REQUESTS_MEMORY=15Mi
-RESOURCES_REQUESTS_CPU=10m
-SQS_URL=
-SQS_AWS_REGION=
-P2P_NETWORK=mainnet
-PPROF_ENABLED=true
-AWS_IAM_ROLE=
-CACHE_CHANNEL=WORMSCAN:NOTIONAL

deploy/analytics/env/production.env

@@ -1,6 +1,6 @@
 ENVIRONMENT=production
 NAMESPACE=wormscan
-NAME=wormscan-analytic
+NAME=wormscan-analytics
 REPLICAS=2
 IMAGE_NAME=
 RESOURCES_LIMITS_MEMORY=64Mi

deploy/analytics/env/staging.env

@@ -0,0 +1,15 @@
+ENVIRONMENT=staging
+NAMESPACE=wormscan
+NAME=wormscan-analytics
+REPLICAS=2
+IMAGE_NAME=
+RESOURCES_LIMITS_MEMORY=64Mi
+RESOURCES_LIMITS_CPU=200m
+RESOURCES_REQUESTS_MEMORY=32Mi
+RESOURCES_REQUESTS_CPU=100m
+SQS_URL=
+SQS_AWS_REGION=
+P2P_NETWORK=mainnet
+PPROF_ENABLED=true
+AWS_IAM_ROLE=
+CACHE_CHANNEL=WORMSCAN:NOTIONAL

deploy/analytics/env/test.env

@@ -1,6 +1,6 @@
 ENVIRONMENT=test
 NAMESPACE=wormscan-testnet
-NAME=wormscan-analytic
+NAME=wormscan-analytics
 REPLICAS=1
 IMAGE_NAME=
 RESOURCES_LIMITS_MEMORY=128Mi

go.work

@@ -1,12 +1,11 @@
 go 1.19
 
 use (
-	./analytic
+	./analytics
 	./api
 	./common
 	./contract-watcher
 	./fly
-	./influx-backfiller
 	./parser
 	./pipeline
 	./spy

influx-backfiller/.gitignore

@@ -1,3 +0,0 @@
-.env
-.vscode
-bin/influx-backfiller

influx-backfiller/Makefile

@@ -1,11 +0,0 @@
-SHELL := /bin/bash
-
-
-build:
-	CGO_ENABLED=0 GOOS=linux go build -o ./bin/influx-backfiller
-
-test:
-	go test -v -cover ./...
-
-
-.PHONY: build test

influx-backfiller/README.md

@@ -1,32 +0,0 @@
-# influx backfiller
-
-Takes CSV file with VAAs as input, and generates a [line protocol](https://docs.influxdata.com/influxdb/cloud/reference/syntax/line-protocol/) file for bulk loading into InfluxDB.
-
-
-
-## Usage
-
-Run the program to generate InfluxDB dump files:
-
-```bash
-./influx-backfiller metrics vaa-count --input vaas-signed.csv --output vaa-count.csv
-./influx-backfiller metrics vaa-volume --input vaas-signed.csv --output vaa-volume.csv
-```
-
-Then load the files into InfluxDB:
-
-```bash
-influx write --bucket wormhole-explorer --file vaa-count.csv
-influx write --bucket wormhole-explorer --file vaa-volume.csv
-```
-
-## Historic Prices
-
-The prices file is generated with `cmd/symbol_historic` and uses the coingecko api to fetch daily prices for
-all supported symbols. 
-
-There is a compressed version of the file already generated called [`prices.csv.gz`](https://github.com/XLabs/influx-backfiller/blob/master/prices.csv.gz).
-
-
-
-

influx-backfiller/go.mod

@@ -1,27 +0,0 @@
-module github.com/xlabs/influx-backfiller
-
-go 1.19
-
-require (
-	github.com/mr-tron/base58 v1.2.0
-	github.com/shopspring/decimal v1.3.1
-	github.com/spf13/cobra v1.7.0
-	github.com/wormhole-foundation/wormhole/sdk v0.0.0-20230501192138-3eefb74bfb4e
-)
-
-require (
-	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/inconshreveable/mousetrap v1.1.0 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/spf13/pflag v1.0.5 // indirect
-)
-
-require (
-	github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
-	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
-	github.com/ethereum/go-ethereum v1.10.21 // indirect
-	github.com/holiman/uint256 v1.2.1 // indirect
-	github.com/test-go/testify v1.1.4
-	golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect
-	golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
-)

influx-backfiller/go.sum

@@ -1,39 +0,0 @@
-github.com/btcsuite/btcd/btcec/v2 v2.2.0 h1:fzn1qaOt32TuLjFlkzYSsBC35Q3KUjT1SwPxiMSCF5k=
-github.com/btcsuite/btcd/btcec/v2 v2.2.0/go.mod h1:U7MHm051Al6XmscBQ0BoNydpOTsFAn707034b5nY8zU=
-github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U=
-github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0=
-github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc=
-github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc=
-github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs=
-github.com/ethereum/go-ethereum v1.10.21 h1:5lqsEx92ZaZzRyOqBEXux4/UR06m296RGzN3ol3teJY=
-github.com/ethereum/go-ethereum v1.10.21/go.mod h1:EYFyF19u3ezGLD4RqOkLq+ZCXzYbLoNDdZlMt7kyKFg=
-github.com/holiman/uint256 v1.2.1 h1:XRtyuda/zw2l+Bq/38n5XUoEF72aSOu/77Thd9pPp2o=
-github.com/holiman/uint256 v1.2.1/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw=
-github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
-github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
-github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o=
-github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
-github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
-github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
-github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
-github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
-github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
-github.com/test-go/testify v1.1.4 h1:Tf9lntrKUMHiXQ07qBScBTSA0dhYQlu83hswqelv1iE=
-github.com/test-go/testify v1.1.4/go.mod h1:rH7cfJo/47vWGdi4GPj16x3/t1xGOj2YxzmNQzk2ghU=
-github.com/wormhole-foundation/wormhole/sdk v0.0.0-20230501192138-3eefb74bfb4e h1:Yf0Mhp9MA8xU5MmyuBO3uzyUEm70c34ddM5vf3vwxb0=
-github.com/wormhole-foundation/wormhole/sdk v0.0.0-20230501192138-3eefb74bfb4e/go.mod h1:dE12DOucCq23gjGGGhtbyx41FBxuHxjpPvG+ArO+8t0=
-golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY=
-golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k=
-golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
-gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

influx-backfiller/main.go

@@ -1,9 +0,0 @@
-package main
-
-import (
-	"github.com/xlabs/influx-backfiller/cmd"
-)
-
-func main() {
-	cmd.Execute()
-}