From 4fd5d73b65556cabd7a2d5e29f33be6bd5689976 Mon Sep 17 00:00:00 2001 From: Mariano <9205080+marianososto@users.noreply.github.com> Date: Wed, 3 Apr 2024 17:03:57 -0300 Subject: [PATCH 01/26] add 2 new tasks for collecting chain activity every day and hour --- analytics/scripts/chain_activity_1d.flux | 39 ++++++++++++++++++++++++ analytics/scripts/chain_activity_1h.flux | 37 ++++++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 analytics/scripts/chain_activity_1d.flux create mode 100644 analytics/scripts/chain_activity_1h.flux diff --git a/analytics/scripts/chain_activity_1d.flux b/analytics/scripts/chain_activity_1d.flux new file mode 100644 index 00000000..315e091b --- /dev/null +++ b/analytics/scripts/chain_activity_1d.flux @@ -0,0 +1,39 @@ +import "date" + +//stop = date.truncate(t: now(), unit: 1h) +//start = date.truncate(t: -1h,unit: 1h) + + +runTask = (start,stop,srcBucket,destBucket,destMeasurement) => { + data = from(bucket: srcBucket) + |> range(start: start,stop: stop) + |> filter(fn: (r) => r._measurement == "vaa_volume_v2" and r._field == "volume") + |> set(key: "_measurement", value: destMeasurement) + |> group(columns: ["emitter_chain", "destination_chain", "app_id"]) + +notional = data + |> sum(column: "_value") + |> rename(columns: {_value: "notional"}) + +txs = data + |> count(column: "_value") + |> rename(columns: {_value: "txs"}) + + +return join(tables: {t1: notional, t2: txs}, on: ["emitter_chain","destination_chain","app_id"]) + |> set(key: "_time", value: string(v:start)) + |> to(bucket: destBucket) +} + + +bucketInfinite = "wormscan-mainnet-staging" +destMeasurement = "chain_activity_1d" +stop = date.truncate(t: now(),unit: 24h) +start = date.sub(d: 1d, from: stop) + +option task = { + name: "calculate chain activity every day", + every: 1d, +} + +runTask(start:start, stop: stop, srcBucket: bucketInfinite, destBucket: bucketInfinite, destMeasurement: destMeasurement) \ No newline at end of file diff --git a/analytics/scripts/chain_activity_1h.flux b/analytics/scripts/chain_activity_1h.flux new file mode 100644 index 00000000..83f34d61 --- /dev/null +++ b/analytics/scripts/chain_activity_1h.flux @@ -0,0 +1,37 @@ +import "date" + + +runTask = (start,stop,srcBucket,destBucket,destMeasurement) => { + data = from(bucket: srcBucket) + |> range(start: start,stop: stop) + |> filter(fn: (r) => r._measurement == "vaa_volume_v2" and r._field == "volume") + |> set(key: "_measurement", value: destMeasurement) + |> group(columns: ["emitter_chain", "destination_chain", "app_id"]) + +notional = data + |> sum(column: "_value") + |> rename(columns: {_value: "notional"}) + +txs = data + |> count(column: "_value") + |> rename(columns: {_value: "txs"}) + + +return join(tables: {t1: notional, t2: txs}, on: ["emitter_chain","destination_chain","app_id"]) + |> set(key: "_time", value: string(v:start)) + |> to(bucket: destBucket) +} + + +bucketInfinite = "wormscan-mainnet-staging" +destMeasurement = "chain_activity_1h" + +stop = date.truncate(t: now(),unit: 1h) +start = date.sub(d: 1h, from: stop) + +option task = { + name: "calculate chain activity every hour", + every: 1h, +} + +runTask(start:start, stop: stop, srcBucket: bucketInfinite, destBucket: bucketInfinite, destMeasurement: destMeasurement) \ No newline at end of file From cfd934c764e49270ce678a28ad132acba97e57b9 Mon Sep 17 00:00:00 2001 From: Mariano <9205080+marianososto@users.noreply.github.com> Date: Mon, 8 Apr 2024 15:55:27 -0300 Subject: [PATCH 02/26] making progress --- api/handlers/transactions/model.go | 30 +++++ api/handlers/transactions/repository.go | 120 ++++++++++++++++++ api/handlers/transactions/service.go | 37 +++++- api/routes/wormscan/routes.go | 1 + .../wormscan/transactions/controller.go | 33 +++++ 5 files changed, 219 insertions(+), 2 deletions(-) diff --git a/api/handlers/transactions/model.go b/api/handlers/transactions/model.go index 2a8a90c6..a7270e88 100644 --- a/api/handlers/transactions/model.go +++ b/api/handlers/transactions/model.go @@ -143,6 +143,15 @@ type ChainActivityResult struct { Volume uint64 `mapstructure:"_value" json:"volume"` } +type ChainActivityTopResult struct { + Time time.Time `json:"from" mapstructure:"_time"` + To time.Time `json:"to" mapstructure:"to"` + ChainSourceID string `mapstructure:"emitter_chain" json:"emitter_chain"` + ChainDestinationID string `mapstructure:"destination_chain" json:"destination_chain"` + Volume uint64 `mapstructure:"notional" json:"volume"` + Txs uint64 `mapstructure:"count" json:"count"` +} + type ChainActivityTimeSpan string const ( @@ -202,3 +211,24 @@ type TransactionDto struct { Payload map[string]interface{} `bson:"payload"` StandardizedProperties map[string]interface{} `bson:"standardizedProperties"` } + +type ChainActivityTopsQuery struct { + SourceChain sdk.ChainID `json:"source_chain"` + TargetChain sdk.ChainID `json:"target_chain"` + From time.Time `json:"from"` + To time.Time `json:"to"` + TimeInterval TimeInterval `json:"time_interval"` +} + +type TimeInterval string + +const ( + Hour TimeInterval = "1h" + Day TimeInterval = "1d" + Week TimeInterval = "1w" + Month TimeInterval = "1m0" +) + +func (t TimeInterval) IsValid() bool { + return t == Hour || t == Day || t == Week || t == Month +} diff --git a/api/handlers/transactions/repository.go b/api/handlers/transactions/repository.go index 37a189bf..b250ceaa 100644 --- a/api/handlers/transactions/repository.go +++ b/api/handlers/transactions/repository.go @@ -3,6 +3,7 @@ package transactions import ( "context" "fmt" + "github.com/valyala/fasthttp" "strconv" "strings" "sync" @@ -1048,3 +1049,122 @@ func (r *Repository) ListTransactionsByAddress( return documents, nil } + +func (r *Repository) FindChainActivityTops(ctx *fasthttp.RequestCtx, q *ChainActivityTopsQuery) ([]ChainActivityTopResult, error) { + query := r.buildChainActivityQueryTops(q) + result, err := r.queryAPI.Query(ctx, query) + if err != nil { + return nil, err + } + if result.Err() != nil { + return nil, result.Err() + } + var response []ChainActivityTopResult + for result.Next() { + var row ChainActivityTopResult + if err := mapstructure.Decode(result.Record().Values(), &row); err != nil { + return nil, err + } + response = append(response, row) + } + + return response, nil +} + +func (r *Repository) buildChainActivityQueryTops(q *ChainActivityTopsQuery) string { + if q.TimeInterval == Day { + query := ` + import "date" + + data = from(bucket: "%s") + |> range(start: %s,stop: %s) + |> filter(fn: (r) => r._measurement == "vaa_volume_v2" and r.version == "v2" ) + %s + %s + |> filter(fn: (r) => r._field == "volume") + |> drop(columns:["token_chain","token_address","version","app_id"]) + |> window(every: 1d) + + notional = data + |> sum() + |> duplicate(column: "_start", as: "_time") + |> drop(columns:["_measurement","_field"]) + |> rename(columns: {_stop: "to"}) + |> window(every: inf) + |> rename(columns: {_value: "notional"}) + |> drop(columns:["_start","_stop"]) + + txs = data + |> count() + |> duplicate(column: "_start", as: "_time") + |> drop(columns:["_measurement","_field"]) + |> rename(columns: {_stop: "to"}) + |> window(every: inf) + |> rename(columns: {_value: "count"}) + |> drop(columns:["_start","_stop"]) + + join(tables: {t1: notional, t2: txs}, on: ["_time","to","emitter_chain","destination_chain"]) + ` + filterSourceChain := "" + if q.SourceChain != sdk.ChainIDUnset { + filterSourceChain = "|> filter(fn: (r) => r.emitter_chain == \"" + strconv.Itoa(int(q.SourceChain)) + "\")" + } + filterDestinationChain := "" + if q.TargetChain != sdk.ChainIDUnset { + filterDestinationChain = "|> filter(fn: (r) => r.destination_chain == \"" + strconv.Itoa(int(q.TargetChain)) + "\")" + } + + start := q.From.Truncate(24 * time.Hour).UTC().Format(time.RFC3339) + stop := q.To.Truncate(24 * time.Hour).UTC().Format(time.RFC3339) + return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, filterSourceChain, filterDestinationChain) + } + + if q.TimeInterval == Month { + query := ` + import "date" + + data = from(bucket: "%s") + |> range(start: %s,stop: %s) + |> filter(fn: (r) => r._measurement == "vaa_volume_v2" and r.version == "v2" ) + %s + %s + |> filter(fn: (r) => r._field == "volume") + |> drop(columns:["token_chain","token_address","version","app_id"]) + |> window(every: 1mo) + + notional = data + |> sum() + |> duplicate(column: "_start", as: "_time") + |> drop(columns:["_measurement","_field"]) + |> rename(columns: {_stop: "to"}) + |> window(every: inf) + |> rename(columns: {_value: "notional"}) + |> drop(columns:["_start","_stop"]) + + txs = data + |> count() + |> duplicate(column: "_start", as: "_time") + |> drop(columns:["_measurement","_field"]) + |> rename(columns: {_stop: "to"}) + |> window(every: inf) + |> rename(columns: {_value: "count"}) + |> drop(columns:["_start","_stop"]) + + join(tables: {t1: notional, t2: txs}, on: ["_time","to","emitter_chain","destination_chain"]) + ` + filterSourceChain := "" + if q.SourceChain != sdk.ChainIDUnset { + filterSourceChain = "|> filter(fn: (r) => r.emitter_chain == \"" + strconv.Itoa(int(q.SourceChain)) + "\")" + } + filterDestinationChain := "" + if q.TargetChain != sdk.ChainIDUnset { + filterDestinationChain = "|> filter(fn: (r) => r.destination_chain == \"" + strconv.Itoa(int(q.TargetChain)) + "\")" + } + + start := q.From.Truncate(24 * time.Hour).UTC().Format(time.RFC3339) + stop := q.To.Truncate(24 * time.Hour).UTC().Format(time.RFC3339) + return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, filterSourceChain, filterDestinationChain) + } + + return "" // TODO: implement +} diff --git a/api/handlers/transactions/service.go b/api/handlers/transactions/service.go index b08d5c0d..8f929280 100644 --- a/api/handlers/transactions/service.go +++ b/api/handlers/transactions/service.go @@ -2,12 +2,13 @@ package transactions import ( "context" + errors "errors" "fmt" + "github.com/valyala/fasthttp" "strings" "time" "github.com/wormhole-foundation/wormhole-explorer/api/cacheable" - "github.com/wormhole-foundation/wormhole-explorer/api/internal/errors" errs "github.com/wormhole-foundation/wormhole-explorer/api/internal/errors" "github.com/wormhole-foundation/wormhole-explorer/api/internal/metrics" "github.com/wormhole-foundation/wormhole-explorer/api/internal/pagination" @@ -34,6 +35,7 @@ const ( topAssetsByVolumeKey = "wormscan:top-assets-by-volume" topChainPairsByNumTransfersKey = "wormscan:top-chain-pairs-by-num-transfers" chainActivityKey = "wormscan:chain-activity" + chainActivityTopsKey = "wormscan:chain-activity-tops" ) // NewService create a new Service. @@ -157,7 +159,7 @@ func (s *Service) GetTransactionByID( return nil, err } if len(output) == 0 { - return nil, errors.ErrNotFound + return nil, errs.ErrNotFound } // Return matching document @@ -167,3 +169,34 @@ func (s *Service) GetTransactionByID( func (s *Service) GetTokenProvider() *domain.TokenProvider { return s.tokenProvider } + +func (s *Service) GetChainActivityTops(ctx *fasthttp.RequestCtx, q *ChainActivityTopsQuery) (interface{}, error) { + + timeDuration := q.To.Sub(q.From) + + if q.TimeInterval == Hour && timeDuration > 15*24*time.Hour { + return nil, errors.New("time range is too large for hourly data. Max time range allowed: 15 days") + } + + if q.TimeInterval == Day { + if timeDuration < 24*time.Hour { + return nil, errors.New("time range is too small for daily data. Min time range allowed: 2 day") + } + + if timeDuration > 365*24*time.Hour { + return nil, errors.New("time range is too large for daily data. Max time range allowed: 1 year") + } + } + + if q.TimeInterval == Month { + if timeDuration < 30*24*time.Hour { + return nil, errors.New("time range is too small for monthly data. Min time range allowed: 60 days") + } + + if timeDuration > 10*365*24*time.Hour { + return nil, errors.New("time range is too large for monthly data. Max time range allowed: 1 year") + } + } + + return s.repo.FindChainActivityTops(ctx, q) +} diff --git a/api/routes/wormscan/routes.go b/api/routes/wormscan/routes.go index 1d10b94a..a9aaf6fc 100644 --- a/api/routes/wormscan/routes.go +++ b/api/routes/wormscan/routes.go @@ -85,6 +85,7 @@ func RegisterRoutes( api.Get("/last-txs", transactionCtrl.GetLastTransactions) api.Get("/scorecards", transactionCtrl.GetScorecards) api.Get("/x-chain-activity", transactionCtrl.GetChainActivity) + api.Post("/x-chain-activity/search", transactionCtrl.GetChainActivityTops) api.Get("/top-assets-by-volume", transactionCtrl.GetTopAssets) api.Get("/top-chain-pairs-by-num-transfers", transactionCtrl.GetTopChainPairs) api.Get("token/:chain/:token_address", transactionCtrl.GetTokenByChainAndAddress) diff --git a/api/routes/wormscan/transactions/controller.go b/api/routes/wormscan/transactions/controller.go index 06390eba..1251a130 100644 --- a/api/routes/wormscan/transactions/controller.go +++ b/api/routes/wormscan/transactions/controller.go @@ -1,7 +1,9 @@ package transactions import ( + "encoding/json" "strconv" + "time" "github.com/gofiber/fiber/v2" "github.com/shopspring/decimal" @@ -182,6 +184,37 @@ func (c *Controller) GetTopAssets(ctx *fiber.Ctx) error { return ctx.JSON(response) } +func (c *Controller) GetChainActivityTops(ctx *fiber.Ctx) error { + + payload := &transactions.ChainActivityTopsQuery{} + err := json.Unmarshal(ctx.Request().Body(), payload) + if err != nil { + return response.NewInvalidParamError(ctx, "invalid request body", err) + } + + if !payload.TimeInterval.IsValid() { + return response.NewInvalidParamError(ctx, "invalid time interval", nil) + } + + nowUTC := time.Now().UTC() + if nowUTC.Before(payload.To.UTC()) { + payload.To = nowUTC + } + + if payload.To.Sub(payload.From) <= 0 { + return response.NewInvalidParamError(ctx, "invalid time range", nil) + } + + // Get the chain activity. + activity, err := c.srv.GetChainActivityTops(ctx.Context(), payload) + if err != nil { + c.logger.Error("Error getting chain activity", zap.Error(err)) + return err + } + + return ctx.JSON(activity) +} + // GetChainActivity godoc // @Description Returns a list of chain pairs by origin chain and destination chain. // @Description The list could be rendered by notional or transaction count. From 50aa4f9ed7bd6c479bc09b66b27a58d0355d4659 Mon Sep 17 00:00:00 2001 From: Mariano <9205080+marianososto@users.noreply.github.com> Date: Wed, 10 Apr 2024 14:21:27 -0300 Subject: [PATCH 03/26] change query 2 --- api/handlers/transactions/model.go | 2 +- api/handlers/transactions/repository.go | 101 +++++++----------------- 2 files changed, 30 insertions(+), 73 deletions(-) diff --git a/api/handlers/transactions/model.go b/api/handlers/transactions/model.go index a7270e88..aafbaeb9 100644 --- a/api/handlers/transactions/model.go +++ b/api/handlers/transactions/model.go @@ -148,7 +148,7 @@ type ChainActivityTopResult struct { To time.Time `json:"to" mapstructure:"to"` ChainSourceID string `mapstructure:"emitter_chain" json:"emitter_chain"` ChainDestinationID string `mapstructure:"destination_chain" json:"destination_chain"` - Volume uint64 `mapstructure:"notional" json:"volume"` + Volume uint64 `mapstructure:"volume" json:"volume"` Txs uint64 `mapstructure:"count" json:"count"` } diff --git a/api/handlers/transactions/repository.go b/api/handlers/transactions/repository.go index b250ceaa..d15f33f8 100644 --- a/api/handlers/transactions/repository.go +++ b/api/handlers/transactions/repository.go @@ -1072,85 +1072,41 @@ func (r *Repository) FindChainActivityTops(ctx *fasthttp.RequestCtx, q *ChainAct } func (r *Repository) buildChainActivityQueryTops(q *ChainActivityTopsQuery) string { + if q.TimeInterval == Hour { + query := ` + import "date" + + from(bucket: "%s") + |> range(start: %s,stop: %s) + |> filter(fn: (r) => r._measurement == "chain_activity_1h_v4") + %s + %s + |> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value") + ` + filterSourceChain := "" + if q.SourceChain != sdk.ChainIDUnset { + filterSourceChain = "|> filter(fn: (r) => r.emitter_chain == \"" + strconv.Itoa(int(q.SourceChain)) + "\")" + } + filterDestinationChain := "" + if q.TargetChain != sdk.ChainIDUnset { + filterDestinationChain = "|> filter(fn: (r) => r.destination_chain == \"" + strconv.Itoa(int(q.TargetChain)) + "\")" + } + + start := q.From.UTC().Format(time.RFC3339) + stop := q.To.UTC().Format(time.RFC3339) + return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, filterSourceChain, filterDestinationChain) + } + if q.TimeInterval == Day { query := ` import "date" - data = from(bucket: "%s") + from(bucket: "%s") |> range(start: %s,stop: %s) - |> filter(fn: (r) => r._measurement == "vaa_volume_v2" and r.version == "v2" ) + |> filter(fn: (r) => r._measurement == "chain_activity_1d") %s %s - |> filter(fn: (r) => r._field == "volume") - |> drop(columns:["token_chain","token_address","version","app_id"]) - |> window(every: 1d) - - notional = data - |> sum() - |> duplicate(column: "_start", as: "_time") - |> drop(columns:["_measurement","_field"]) - |> rename(columns: {_stop: "to"}) - |> window(every: inf) - |> rename(columns: {_value: "notional"}) - |> drop(columns:["_start","_stop"]) - - txs = data - |> count() - |> duplicate(column: "_start", as: "_time") - |> drop(columns:["_measurement","_field"]) - |> rename(columns: {_stop: "to"}) - |> window(every: inf) - |> rename(columns: {_value: "count"}) - |> drop(columns:["_start","_stop"]) - - join(tables: {t1: notional, t2: txs}, on: ["_time","to","emitter_chain","destination_chain"]) - ` - filterSourceChain := "" - if q.SourceChain != sdk.ChainIDUnset { - filterSourceChain = "|> filter(fn: (r) => r.emitter_chain == \"" + strconv.Itoa(int(q.SourceChain)) + "\")" - } - filterDestinationChain := "" - if q.TargetChain != sdk.ChainIDUnset { - filterDestinationChain = "|> filter(fn: (r) => r.destination_chain == \"" + strconv.Itoa(int(q.TargetChain)) + "\")" - } - - start := q.From.Truncate(24 * time.Hour).UTC().Format(time.RFC3339) - stop := q.To.Truncate(24 * time.Hour).UTC().Format(time.RFC3339) - return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, filterSourceChain, filterDestinationChain) - } - - if q.TimeInterval == Month { - query := ` - import "date" - - data = from(bucket: "%s") - |> range(start: %s,stop: %s) - |> filter(fn: (r) => r._measurement == "vaa_volume_v2" and r.version == "v2" ) - %s - %s - |> filter(fn: (r) => r._field == "volume") - |> drop(columns:["token_chain","token_address","version","app_id"]) - |> window(every: 1mo) - - notional = data - |> sum() - |> duplicate(column: "_start", as: "_time") - |> drop(columns:["_measurement","_field"]) - |> rename(columns: {_stop: "to"}) - |> window(every: inf) - |> rename(columns: {_value: "notional"}) - |> drop(columns:["_start","_stop"]) - - txs = data - |> count() - |> duplicate(column: "_start", as: "_time") - |> drop(columns:["_measurement","_field"]) - |> rename(columns: {_stop: "to"}) - |> window(every: inf) - |> rename(columns: {_value: "count"}) - |> drop(columns:["_start","_stop"]) - - join(tables: {t1: notional, t2: txs}, on: ["_time","to","emitter_chain","destination_chain"]) + |> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value") ` filterSourceChain := "" if q.SourceChain != sdk.ChainIDUnset { @@ -1165,6 +1121,7 @@ func (r *Repository) buildChainActivityQueryTops(q *ChainActivityTopsQuery) stri stop := q.To.Truncate(24 * time.Hour).UTC().Format(time.RFC3339) return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, filterSourceChain, filterDestinationChain) } + //TODO: implement month and yearly return "" // TODO: implement } From 3c24e5e5409e8434b6e3f1d684ad4b09d895981d Mon Sep 17 00:00:00 2001 From: Mariano <9205080+marianososto@users.noreply.github.com> Date: Wed, 10 Apr 2024 16:46:57 -0300 Subject: [PATCH 04/26] add query by month and year --- api/handlers/transactions/model.go | 3 +- api/handlers/transactions/repository.go | 64 +++++++++++++++++-------- api/handlers/transactions/service.go | 10 ++++ 3 files changed, 56 insertions(+), 21 deletions(-) diff --git a/api/handlers/transactions/model.go b/api/handlers/transactions/model.go index aafbaeb9..19d93648 100644 --- a/api/handlers/transactions/model.go +++ b/api/handlers/transactions/model.go @@ -227,8 +227,9 @@ const ( Day TimeInterval = "1d" Week TimeInterval = "1w" Month TimeInterval = "1m0" + Year TimeInterval = "1y" ) func (t TimeInterval) IsValid() bool { - return t == Hour || t == Day || t == Week || t == Month + return t == Hour || t == Day || t == Week || t == Month || t == Year } diff --git a/api/handlers/transactions/repository.go b/api/handlers/transactions/repository.go index d15f33f8..c4149ce0 100644 --- a/api/handlers/transactions/repository.go +++ b/api/handlers/transactions/repository.go @@ -1072,6 +1072,16 @@ func (r *Repository) FindChainActivityTops(ctx *fasthttp.RequestCtx, q *ChainAct } func (r *Repository) buildChainActivityQueryTops(q *ChainActivityTopsQuery) string { + + filterSourceChain := "" + if q.SourceChain != sdk.ChainIDUnset { + filterSourceChain = "|> filter(fn: (r) => r.emitter_chain == \"" + strconv.Itoa(int(q.SourceChain)) + "\")" + } + filterDestinationChain := "" + if q.TargetChain != sdk.ChainIDUnset { + filterDestinationChain = "|> filter(fn: (r) => r.destination_chain == \"" + strconv.Itoa(int(q.TargetChain)) + "\")" + } + if q.TimeInterval == Hour { query := ` import "date" @@ -1082,16 +1092,8 @@ func (r *Repository) buildChainActivityQueryTops(q *ChainActivityTopsQuery) stri %s %s |> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value") + |> map(fn: (r) => ({r with to: date.add(d: 1h,to: r._time)})) ` - filterSourceChain := "" - if q.SourceChain != sdk.ChainIDUnset { - filterSourceChain = "|> filter(fn: (r) => r.emitter_chain == \"" + strconv.Itoa(int(q.SourceChain)) + "\")" - } - filterDestinationChain := "" - if q.TargetChain != sdk.ChainIDUnset { - filterDestinationChain = "|> filter(fn: (r) => r.destination_chain == \"" + strconv.Itoa(int(q.TargetChain)) + "\")" - } - start := q.From.UTC().Format(time.RFC3339) stop := q.To.UTC().Format(time.RFC3339) return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, filterSourceChain, filterDestinationChain) @@ -1107,21 +1109,43 @@ func (r *Repository) buildChainActivityQueryTops(q *ChainActivityTopsQuery) stri %s %s |> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value") + |> map(fn: (r) => ({r with to: date.add(d: 1d,to: r._time)})) ` - filterSourceChain := "" - if q.SourceChain != sdk.ChainIDUnset { - filterSourceChain = "|> filter(fn: (r) => r.emitter_chain == \"" + strconv.Itoa(int(q.SourceChain)) + "\")" - } - filterDestinationChain := "" - if q.TargetChain != sdk.ChainIDUnset { - filterDestinationChain = "|> filter(fn: (r) => r.destination_chain == \"" + strconv.Itoa(int(q.TargetChain)) + "\")" - } - start := q.From.Truncate(24 * time.Hour).UTC().Format(time.RFC3339) stop := q.To.Truncate(24 * time.Hour).UTC().Format(time.RFC3339) return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, filterSourceChain, filterDestinationChain) } - //TODO: implement month and yearly - return "" // TODO: implement + if q.TimeInterval == Month { + query := ` + data = from(bucket: "%s") + |> range(start: %s,stop: %s) + |> filter(fn: (r) => r._measurement == "chain_activity_1d") + %s + %s + |> window(every: 1mo) + |> sum() + |> map(fn: (r) => ({r with to: date.add(d: 1mo,to: r._time)})) + ` + + start := time.Date(q.From.Year(), q.From.Month(), 1, 0, 0, 0, 0, q.From.Location()).UTC().Format(time.RFC3339) + stop := time.Date(q.To.Year(), q.To.Month(), 1, 0, 0, 0, 0, q.To.Location()).UTC().Format(time.RFC3339) + return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, filterSourceChain, filterDestinationChain) + } + + query := ` + data = from(bucket: "%s") + |> range(start: %s,stop: %s) + |> filter(fn: (r) => r._measurement == "chain_activity_1d") + %s + %s + |> window(every: 1y) + |> sum() + |> map(fn: (r) => ({r with to: date.add(d: 1y,to: r._time)})) + ` + + start := time.Date(q.From.Year(), 1, 1, 0, 0, 0, 0, q.From.Location()).UTC().Format(time.RFC3339) + stop := time.Date(q.To.Year(), 1, 1, 0, 0, 0, 0, q.To.Location()).UTC().Format(time.RFC3339) + return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, filterSourceChain, filterDestinationChain) + } diff --git a/api/handlers/transactions/service.go b/api/handlers/transactions/service.go index 8f929280..4dd2f074 100644 --- a/api/handlers/transactions/service.go +++ b/api/handlers/transactions/service.go @@ -198,5 +198,15 @@ func (s *Service) GetChainActivityTops(ctx *fasthttp.RequestCtx, q *ChainActivit } } + if q.TimeInterval == Year { + if timeDuration < 365*24*time.Hour { + return nil, errors.New("time range is too small for yearly data. Min time range allowed: 1 year") + } + + if timeDuration > 10*365*24*time.Hour { + return nil, errors.New("time range is too large for yearly data. Max time range allowed: 10 year") + } + } + return s.repo.FindChainActivityTops(ctx, q) } From 0bfdb8aeedd25184877b1904b0d9cf3bf658e1dd Mon Sep 17 00:00:00 2001 From: Mariano <9205080+marianososto@users.noreply.github.com> Date: Thu, 11 Apr 2024 09:10:59 -0300 Subject: [PATCH 05/26] changes on task --- analytics/scripts/chain_activity_1d.flux | 11 +++-------- analytics/scripts/chain_activity_1h.flux | 7 +++---- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/analytics/scripts/chain_activity_1d.flux b/analytics/scripts/chain_activity_1d.flux index 315e091b..db41bc33 100644 --- a/analytics/scripts/chain_activity_1d.flux +++ b/analytics/scripts/chain_activity_1d.flux @@ -1,9 +1,5 @@ import "date" -//stop = date.truncate(t: now(), unit: 1h) -//start = date.truncate(t: -1h,unit: 1h) - - runTask = (start,stop,srcBucket,destBucket,destMeasurement) => { data = from(bucket: srcBucket) |> range(start: start,stop: stop) @@ -13,13 +9,12 @@ runTask = (start,stop,srcBucket,destBucket,destMeasurement) => { notional = data |> sum(column: "_value") - |> rename(columns: {_value: "notional"}) + |> rename(columns: {_field: "notional"}) txs = data |> count(column: "_value") - |> rename(columns: {_value: "txs"}) - - + |> rename(columns: {_field: "count"}) + return join(tables: {t1: notional, t2: txs}, on: ["emitter_chain","destination_chain","app_id"]) |> set(key: "_time", value: string(v:start)) |> to(bucket: destBucket) diff --git a/analytics/scripts/chain_activity_1h.flux b/analytics/scripts/chain_activity_1h.flux index 83f34d61..7fc43595 100644 --- a/analytics/scripts/chain_activity_1h.flux +++ b/analytics/scripts/chain_activity_1h.flux @@ -10,13 +10,12 @@ runTask = (start,stop,srcBucket,destBucket,destMeasurement) => { notional = data |> sum(column: "_value") - |> rename(columns: {_value: "notional"}) + |> rename(columns: {_field: "notional"}) txs = data |> count(column: "_value") - |> rename(columns: {_value: "txs"}) - - + |> rename(columns: {_field: "count"}) + return join(tables: {t1: notional, t2: txs}, on: ["emitter_chain","destination_chain","app_id"]) |> set(key: "_time", value: string(v:start)) |> to(bucket: destBucket) From 217b8570e44d8256a1af0ba4528df67bb03d3171 Mon Sep 17 00:00:00 2001 From: Mariano <9205080+marianososto@users.noreply.github.com> Date: Thu, 11 Apr 2024 17:20:54 -0300 Subject: [PATCH 06/26] more changes --- analytics/scripts/chain_activity_1d.flux | 37 ++++++++++++----------- analytics/scripts/chain_activity_1h.flux | 38 +++++++++++++----------- api/handlers/transactions/model.go | 2 +- api/handlers/transactions/repository.go | 10 +++++-- 4 files changed, 49 insertions(+), 38 deletions(-) diff --git a/analytics/scripts/chain_activity_1d.flux b/analytics/scripts/chain_activity_1d.flux index db41bc33..0c0c2d4d 100644 --- a/analytics/scripts/chain_activity_1d.flux +++ b/analytics/scripts/chain_activity_1d.flux @@ -1,27 +1,30 @@ import "date" runTask = (start,stop,srcBucket,destBucket,destMeasurement) => { - data = from(bucket: srcBucket) - |> range(start: start,stop: stop) - |> filter(fn: (r) => r._measurement == "vaa_volume_v2" and r._field == "volume") - |> set(key: "_measurement", value: destMeasurement) - |> group(columns: ["emitter_chain", "destination_chain", "app_id"]) - -notional = data - |> sum(column: "_value") - |> rename(columns: {_field: "notional"}) - -txs = data - |> count(column: "_value") - |> rename(columns: {_field: "count"}) + data = from(bucket: srcBucket) + |> range(start: start,stop: stop) + |> filter(fn: (r) => r._measurement == "vaa_volume_v2" and r._field == "volume") + |> group(columns: ["emitter_chain", "destination_chain", "app_id"]) -return join(tables: {t1: notional, t2: txs}, on: ["emitter_chain","destination_chain","app_id"]) - |> set(key: "_time", value: string(v:start)) - |> to(bucket: destBucket) + data + |> sum(column: "_value") + |> set(key: "_field", value: "volume") + |> map(fn: (r) => ({ r with _time: start })) + |> set(key: "to", value: string(v:date.add(d: 1h, to: start))) + |> set(key: "_measurement", value: destMeasurement) + |> to(bucket: destBucket) + + return data + |> count(column: "_value") + |> set(key: "_field", value: "count") + |> map(fn: (r) => ({ r with _time: start })) + |> set(key: "to", value: string(v:date.add(d: 1h, to: start))) + |> set(key: "_measurement", value: destMeasurement) + |> to(bucket: destBucket) } -bucketInfinite = "wormscan-mainnet-staging" +bucketInfinite = "wormscan" destMeasurement = "chain_activity_1d" stop = date.truncate(t: now(),unit: 24h) start = date.sub(d: 1d, from: stop) diff --git a/analytics/scripts/chain_activity_1h.flux b/analytics/scripts/chain_activity_1h.flux index 7fc43595..744b6f35 100644 --- a/analytics/scripts/chain_activity_1h.flux +++ b/analytics/scripts/chain_activity_1h.flux @@ -2,27 +2,31 @@ import "date" runTask = (start,stop,srcBucket,destBucket,destMeasurement) => { - data = from(bucket: srcBucket) - |> range(start: start,stop: stop) - |> filter(fn: (r) => r._measurement == "vaa_volume_v2" and r._field == "volume") - |> set(key: "_measurement", value: destMeasurement) - |> group(columns: ["emitter_chain", "destination_chain", "app_id"]) - -notional = data - |> sum(column: "_value") - |> rename(columns: {_field: "notional"}) - -txs = data - |> count(column: "_value") - |> rename(columns: {_field: "count"}) -return join(tables: {t1: notional, t2: txs}, on: ["emitter_chain","destination_chain","app_id"]) - |> set(key: "_time", value: string(v:start)) - |> to(bucket: destBucket) + data = from(bucket: srcBucket) + |> range(start: start,stop: stop) + |> filter(fn: (r) => r._measurement == "vaa_volume_v2" and r._field == "volume") + |> group(columns: ["emitter_chain", "destination_chain", "app_id"]) + + data + |> sum(column: "_value") + |> set(key: "_field", value: "volume") + |> map(fn: (r) => ({ r with _time: start })) + |> set(key: "to", value: string(v:date.add(d: 1h, to: start))) + |> set(key: "_measurement", value: destMeasurement) + |> to(bucket: destBucket) + + return data + |> count(column: "_value") + |> set(key: "_field", value: "count") + |> map(fn: (r) => ({ r with _time: start })) + |> set(key: "to", value: string(v:date.add(d: 1h, to: start))) + |> set(key: "_measurement", value: destMeasurement) + |> to(bucket: destBucket) } -bucketInfinite = "wormscan-mainnet-staging" +bucketInfinite = "wormscan" destMeasurement = "chain_activity_1h" stop = date.truncate(t: now(),unit: 1h) diff --git a/api/handlers/transactions/model.go b/api/handlers/transactions/model.go index 19d93648..31697684 100644 --- a/api/handlers/transactions/model.go +++ b/api/handlers/transactions/model.go @@ -145,7 +145,7 @@ type ChainActivityResult struct { type ChainActivityTopResult struct { Time time.Time `json:"from" mapstructure:"_time"` - To time.Time `json:"to" mapstructure:"to"` + To string `json:"to" mapstructure:"to"` ChainSourceID string `mapstructure:"emitter_chain" json:"emitter_chain"` ChainDestinationID string `mapstructure:"destination_chain" json:"destination_chain"` Volume uint64 `mapstructure:"volume" json:"volume"` diff --git a/api/handlers/transactions/repository.go b/api/handlers/transactions/repository.go index c4149ce0..9b93a96f 100644 --- a/api/handlers/transactions/repository.go +++ b/api/handlers/transactions/repository.go @@ -1065,6 +1065,10 @@ func (r *Repository) FindChainActivityTops(ctx *fasthttp.RequestCtx, q *ChainAct if err := mapstructure.Decode(result.Record().Values(), &row); err != nil { return nil, err } + parsedTime, errTime := time.Parse(time.RFC3339Nano, row.To) + if errTime == nil { + row.To = parsedTime.Format(time.RFC3339) + } response = append(response, row) } @@ -1088,15 +1092,15 @@ func (r *Repository) buildChainActivityQueryTops(q *ChainActivityTopsQuery) stri from(bucket: "%s") |> range(start: %s,stop: %s) - |> filter(fn: (r) => r._measurement == "chain_activity_1h_v4") + |> filter(fn: (r) => r._measurement == "chain_activity_1h_test") %s %s |> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value") - |> map(fn: (r) => ({r with to: date.add(d: 1h,to: r._time)})) + //|> map(fn: (r) => ({r with to: date.add(d: 1h,to: r._time)})) ` start := q.From.UTC().Format(time.RFC3339) stop := q.To.UTC().Format(time.RFC3339) - return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, filterSourceChain, filterDestinationChain) + return fmt.Sprintf(query, "wormscan-24hours-mainnet-staging", start, stop, filterSourceChain, filterDestinationChain) } if q.TimeInterval == Day { From e18e14ae468a0928d947e716257e5124970fe4bc Mon Sep 17 00:00:00 2001 From: Mariano <9205080+marianososto@users.noreply.github.com> Date: Thu, 11 Apr 2024 17:39:41 -0300 Subject: [PATCH 07/26] change to 1d --- analytics/scripts/chain_activity_1d.flux | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/analytics/scripts/chain_activity_1d.flux b/analytics/scripts/chain_activity_1d.flux index 0c0c2d4d..fffb2199 100644 --- a/analytics/scripts/chain_activity_1d.flux +++ b/analytics/scripts/chain_activity_1d.flux @@ -18,7 +18,7 @@ runTask = (start,stop,srcBucket,destBucket,destMeasurement) => { |> count(column: "_value") |> set(key: "_field", value: "count") |> map(fn: (r) => ({ r with _time: start })) - |> set(key: "to", value: string(v:date.add(d: 1h, to: start))) + |> set(key: "to", value: string(v:date.add(d: 1d, to: start))) |> set(key: "_measurement", value: destMeasurement) |> to(bucket: destBucket) } From 6e4d5a64b63e9fc4f4ef46da380cd720978a485c Mon Sep 17 00:00:00 2001 From: Mariano <9205080+marianososto@users.noreply.github.com> Date: Thu, 11 Apr 2024 17:41:10 -0300 Subject: [PATCH 08/26] add 1d --- analytics/scripts/chain_activity_1d.flux | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/analytics/scripts/chain_activity_1d.flux b/analytics/scripts/chain_activity_1d.flux index fffb2199..3e21e1cd 100644 --- a/analytics/scripts/chain_activity_1d.flux +++ b/analytics/scripts/chain_activity_1d.flux @@ -10,7 +10,7 @@ runTask = (start,stop,srcBucket,destBucket,destMeasurement) => { |> sum(column: "_value") |> set(key: "_field", value: "volume") |> map(fn: (r) => ({ r with _time: start })) - |> set(key: "to", value: string(v:date.add(d: 1h, to: start))) + |> set(key: "to", value: string(v:date.add(d: 1d, to: start))) |> set(key: "_measurement", value: destMeasurement) |> to(bucket: destBucket) From f9ac8671925d84a75704547b2b8ba83da8e6a51f Mon Sep 17 00:00:00 2001 From: Mariano <9205080+marianososto@users.noreply.github.com> Date: Fri, 12 Apr 2024 09:10:59 -0300 Subject: [PATCH 09/26] fix query --- api/handlers/transactions/repository.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/api/handlers/transactions/repository.go b/api/handlers/transactions/repository.go index 9b93a96f..b8950701 100644 --- a/api/handlers/transactions/repository.go +++ b/api/handlers/transactions/repository.go @@ -1096,7 +1096,6 @@ func (r *Repository) buildChainActivityQueryTops(q *ChainActivityTopsQuery) stri %s %s |> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value") - //|> map(fn: (r) => ({r with to: date.add(d: 1h,to: r._time)})) ` start := q.From.UTC().Format(time.RFC3339) stop := q.To.UTC().Format(time.RFC3339) @@ -1113,7 +1112,6 @@ func (r *Repository) buildChainActivityQueryTops(q *ChainActivityTopsQuery) stri %s %s |> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value") - |> map(fn: (r) => ({r with to: date.add(d: 1d,to: r._time)})) ` start := q.From.Truncate(24 * time.Hour).UTC().Format(time.RFC3339) stop := q.To.Truncate(24 * time.Hour).UTC().Format(time.RFC3339) From dbc08b8181e55d1bebe40fbd78cebb93b8756149 Mon Sep 17 00:00:00 2001 From: Mariano <9205080+marianososto@users.noreply.github.com> Date: Mon, 15 Apr 2024 13:22:25 -0300 Subject: [PATCH 10/26] adjust queryies --- api/handlers/transactions/repository.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/api/handlers/transactions/repository.go b/api/handlers/transactions/repository.go index b8950701..5245baea 100644 --- a/api/handlers/transactions/repository.go +++ b/api/handlers/transactions/repository.go @@ -1092,14 +1092,16 @@ func (r *Repository) buildChainActivityQueryTops(q *ChainActivityTopsQuery) stri from(bucket: "%s") |> range(start: %s,stop: %s) - |> filter(fn: (r) => r._measurement == "chain_activity_1h_test") + |> filter(fn: (r) => r._measurement == "chain_activity_1h") %s %s |> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value") + |> group() + |> sort(columns:["_time"],desc:false) ` start := q.From.UTC().Format(time.RFC3339) stop := q.To.UTC().Format(time.RFC3339) - return fmt.Sprintf(query, "wormscan-24hours-mainnet-staging", start, stop, filterSourceChain, filterDestinationChain) + return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, filterSourceChain, filterDestinationChain) } if q.TimeInterval == Day { @@ -1112,6 +1114,8 @@ func (r *Repository) buildChainActivityQueryTops(q *ChainActivityTopsQuery) stri %s %s |> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value") + |> group() + |> sort(columns:["_time"],desc:false) ` start := q.From.Truncate(24 * time.Hour).UTC().Format(time.RFC3339) stop := q.To.Truncate(24 * time.Hour).UTC().Format(time.RFC3339) @@ -1128,6 +1132,8 @@ func (r *Repository) buildChainActivityQueryTops(q *ChainActivityTopsQuery) stri |> window(every: 1mo) |> sum() |> map(fn: (r) => ({r with to: date.add(d: 1mo,to: r._time)})) + |> group() + |> sort(columns:["_time"],desc:false) ` start := time.Date(q.From.Year(), q.From.Month(), 1, 0, 0, 0, 0, q.From.Location()).UTC().Format(time.RFC3339) @@ -1144,6 +1150,8 @@ func (r *Repository) buildChainActivityQueryTops(q *ChainActivityTopsQuery) stri |> window(every: 1y) |> sum() |> map(fn: (r) => ({r with to: date.add(d: 1y,to: r._time)})) + |> group() + |> sort(columns:["_time"],desc:false) ` start := time.Date(q.From.Year(), 1, 1, 0, 0, 0, 0, q.From.Location()).UTC().Format(time.RFC3339) From 713270f8e89f0a708941af3d04ca71674e07e269 Mon Sep 17 00:00:00 2001 From: Mariano <9205080+marianososto@users.noreply.github.com> Date: Mon, 15 Apr 2024 22:19:51 -0300 Subject: [PATCH 11/26] change the way the query is executed --- api/handlers/transactions/repository.go | 70 +++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/api/handlers/transactions/repository.go b/api/handlers/transactions/repository.go index 5245baea..d90fb28e 100644 --- a/api/handlers/transactions/repository.go +++ b/api/handlers/transactions/repository.go @@ -1087,6 +1087,38 @@ func (r *Repository) buildChainActivityQueryTops(q *ChainActivityTopsQuery) stri } if q.TimeInterval == Hour { + /*query := ` + import "date" + import "join" + + data = from(bucket: "%s") + |> range(start: %s,stop: %s) + |> filter(fn: (r) => r._measurement == "chain_activity_1h") + %s + %s + + vols = data + |> filter(fn: (r) => r._field == "volume" and r._value > 0) + |> group() + |> drop(columns:["_field"]) + |> rename(columns: {_value: "volume"}) + + counts = data + |> filter(fn: (r) => r._field == "count") + |> drop(columns:["_field"]) + |> rename(columns: {_value: "count"}) + |> group() + + join.left( + left: vols, + right: counts, + on: (l, r) => l._time == r._time and l.to == r.to and l.emitter_chain == r.emitter_chain and l.app_id == r.app_id and l.destination_chain == r.destination_chain, + as: (l, r) => ({l with count: r.count}), + ) + |> sort(columns:["_time"],desc:false) + ` + */ + query := ` import "date" @@ -1095,16 +1127,51 @@ func (r *Repository) buildChainActivityQueryTops(q *ChainActivityTopsQuery) stri |> filter(fn: (r) => r._measurement == "chain_activity_1h") %s %s + |> filter(fn: (r) => (r._field == "volume" and r._value > 0) or (r._field == "count")) |> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value") + |> filter(fn : (r) => exists r.volume) |> group() |> sort(columns:["_time"],desc:false) ` + start := q.From.UTC().Format(time.RFC3339) stop := q.To.UTC().Format(time.RFC3339) return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, filterSourceChain, filterDestinationChain) } if q.TimeInterval == Day { + /*query := ` + import "date" + import "join" + + data = from(bucket: "%s") + |> range(start: %s,stop: %s) + |> filter(fn: (r) => r._measurement == "chain_activity_1d") + %s + %s + + vols = data + |> filter(fn: (r) => r._field == "volume" and r._value > 0) + |> group() + |> drop(columns:["_field"]) + |> rename(columns: {_value: "volume"}) + + counts = data + |> filter(fn: (r) => r._field == "count") + |> drop(columns:["_field"]) + |> rename(columns: {_value: "count"}) + |> group() + + join.left( + left: vols, + right: counts, + on: (l, r) => l._time == r._time and l.to == r.to and l.emitter_chain == r.emitter_chain and l.app_id == r.app_id and l.destination_chain == r.destination_chain, + as: (l, r) => ({l with count: r.count}), + ) + |> sort(columns:["_time"],desc:false) + ` + */ + query := ` import "date" @@ -1113,10 +1180,13 @@ func (r *Repository) buildChainActivityQueryTops(q *ChainActivityTopsQuery) stri |> filter(fn: (r) => r._measurement == "chain_activity_1d") %s %s + |> filter(fn: (r) => (r._field == "volume" and r._value > 0) or (r._field == "count")) |> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value") + |> filter(fn : (r) => exists r.volume) |> group() |> sort(columns:["_time"],desc:false) ` + start := q.From.Truncate(24 * time.Hour).UTC().Format(time.RFC3339) stop := q.To.Truncate(24 * time.Hour).UTC().Format(time.RFC3339) return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, filterSourceChain, filterDestinationChain) From c3354f79f9aac07393d5d69e580c531e003e50fb Mon Sep 17 00:00:00 2001 From: Mariano <9205080+marianososto@users.noreply.github.com> Date: Tue, 16 Apr 2024 16:29:51 -0300 Subject: [PATCH 12/26] changes on query --- api/handlers/transactions/model.go | 2 +- api/handlers/transactions/repository.go | 194 +++++++++++++----------- 2 files changed, 106 insertions(+), 90 deletions(-) diff --git a/api/handlers/transactions/model.go b/api/handlers/transactions/model.go index 31697684..a925ffc9 100644 --- a/api/handlers/transactions/model.go +++ b/api/handlers/transactions/model.go @@ -147,7 +147,7 @@ type ChainActivityTopResult struct { Time time.Time `json:"from" mapstructure:"_time"` To string `json:"to" mapstructure:"to"` ChainSourceID string `mapstructure:"emitter_chain" json:"emitter_chain"` - ChainDestinationID string `mapstructure:"destination_chain" json:"destination_chain"` + ChainDestinationID string `mapstructure:"destination_chain" json:"destination_chain,omitempty"` Volume uint64 `mapstructure:"volume" json:"volume"` Txs uint64 `mapstructure:"count" json:"count"` } diff --git a/api/handlers/transactions/repository.go b/api/handlers/transactions/repository.go index d90fb28e..4fea9692 100644 --- a/api/handlers/transactions/repository.go +++ b/api/handlers/transactions/repository.go @@ -1077,17 +1077,22 @@ func (r *Repository) FindChainActivityTops(ctx *fasthttp.RequestCtx, q *ChainAct func (r *Repository) buildChainActivityQueryTops(q *ChainActivityTopsQuery) string { + filterTargetChain := "" + if q.TargetChain != sdk.ChainIDUnset { + filterTargetChain = "|> filter(fn: (r) => r.destination_chain == \"" + strconv.Itoa(int(q.TargetChain)) + "\")" + } + filterSourceChain := "" if q.SourceChain != sdk.ChainIDUnset { filterSourceChain = "|> filter(fn: (r) => r.emitter_chain == \"" + strconv.Itoa(int(q.SourceChain)) + "\")" } - filterDestinationChain := "" - if q.TargetChain != sdk.ChainIDUnset { - filterDestinationChain = "|> filter(fn: (r) => r.destination_chain == \"" + strconv.Itoa(int(q.TargetChain)) + "\")" - } if q.TimeInterval == Hour { - /*query := ` + + start := q.From.UTC().Format(time.RFC3339) + stop := q.To.UTC().Format(time.RFC3339) + + query := ` import "date" import "join" @@ -1096,51 +1101,38 @@ func (r *Repository) buildChainActivityQueryTops(q *ChainActivityTopsQuery) stri |> filter(fn: (r) => r._measurement == "chain_activity_1h") %s %s + |> drop(columns:["destination_chain"]) - vols = data - |> filter(fn: (r) => r._field == "volume" and r._value > 0) - |> group() - |> drop(columns:["_field"]) - |> rename(columns: {_value: "volume"}) + vols = data + |> filter(fn: (r) => (r._field == "volume" and r._value > 0)) + |> group(columns:["_time","to","emitter_chain"]) + |> sum() + |> rename(columns: {_value: "volume"}) counts = data - |> filter(fn: (r) => r._field == "count") - |> drop(columns:["_field"]) - |> rename(columns: {_value: "count"}) - |> group() + |> filter(fn: (r) => (r._field == "count")) + |> group(columns:["_time","to","emitter_chain"]) + |> sum() + |> rename(columns: {_value: "count"}) - join.left( + join.inner( left: vols, right: counts, - on: (l, r) => l._time == r._time and l.to == r.to and l.emitter_chain == r.emitter_chain and l.app_id == r.app_id and l.destination_chain == r.destination_chain, + on: (l, r) => l._time == r._time and l.to == r.to and l.emitter_chain == r.emitter_chain, as: (l, r) => ({l with count: r.count}), ) - |> sort(columns:["_time"],desc:false) + |> group() + |> sort(columns:["emitter_chain","_time"],desc:false) ` - */ - - query := ` - import "date" - - from(bucket: "%s") - |> range(start: %s,stop: %s) - |> filter(fn: (r) => r._measurement == "chain_activity_1h") - %s - %s - |> filter(fn: (r) => (r._field == "volume" and r._value > 0) or (r._field == "count")) - |> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value") - |> filter(fn : (r) => exists r.volume) - |> group() - |> sort(columns:["_time"],desc:false) - ` - - start := q.From.UTC().Format(time.RFC3339) - stop := q.To.UTC().Format(time.RFC3339) - return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, filterSourceChain, filterDestinationChain) + return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, filterSourceChain, filterTargetChain) } if q.TimeInterval == Day { - /*query := ` + + start := q.From.Truncate(24 * time.Hour).UTC().Format(time.RFC3339) + stop := q.To.Truncate(24 * time.Hour).UTC().Format(time.RFC3339) + + query := ` import "date" import "join" @@ -1149,83 +1141,107 @@ func (r *Repository) buildChainActivityQueryTops(q *ChainActivityTopsQuery) stri |> filter(fn: (r) => r._measurement == "chain_activity_1d") %s %s + |> drop(columns:["destination_chain"]) - vols = data - |> filter(fn: (r) => r._field == "volume" and r._value > 0) - |> group() - |> drop(columns:["_field"]) - |> rename(columns: {_value: "volume"}) + vols = data + |> filter(fn: (r) => (r._field == "volume" and r._value > 0)) + |> group(columns:["_time","to","emitter_chain"]) + |> sum() + |> rename(columns: {_value: "volume"}) counts = data - |> filter(fn: (r) => r._field == "count") - |> drop(columns:["_field"]) - |> rename(columns: {_value: "count"}) - |> group() + |> filter(fn: (r) => (r._field == "count")) + |> group(columns:["_time","to","emitter_chain"]) + |> sum() + |> rename(columns: {_value: "count"}) - join.left( + join.inner( left: vols, right: counts, - on: (l, r) => l._time == r._time and l.to == r.to and l.emitter_chain == r.emitter_chain and l.app_id == r.app_id and l.destination_chain == r.destination_chain, + on: (l, r) => l._time == r._time and l.to == r.to and l.emitter_chain == r.emitter_chain, as: (l, r) => ({l with count: r.count}), ) - |> sort(columns:["_time"],desc:false) + |> group() + |> sort(columns:["emitter_chain","_time"],desc:false) ` - */ - - query := ` - import "date" - - from(bucket: "%s") - |> range(start: %s,stop: %s) - |> filter(fn: (r) => r._measurement == "chain_activity_1d") - %s - %s - |> filter(fn: (r) => (r._field == "volume" and r._value > 0) or (r._field == "count")) - |> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value") - |> filter(fn : (r) => exists r.volume) - |> group() - |> sort(columns:["_time"],desc:false) - ` - - start := q.From.Truncate(24 * time.Hour).UTC().Format(time.RFC3339) - stop := q.To.Truncate(24 * time.Hour).UTC().Format(time.RFC3339) - return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, filterSourceChain, filterDestinationChain) + return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, filterSourceChain, filterTargetChain) } if q.TimeInterval == Month { query := ` - data = from(bucket: "%s") - |> range(start: %s,stop: %s) - |> filter(fn: (r) => r._measurement == "chain_activity_1d") + import "date" + import "join" + + data = from(bucket: "%s") + |> range(start: %s,stop: %s) + |> filter(fn: (r) => r._measurement == "chain_activity_1d") %s %s + |> drop(columns:["destination_chain"]) |> window(every: 1mo) - |> sum() - |> map(fn: (r) => ({r with to: date.add(d: 1mo,to: r._time)})) + + vols = data + |> filter(fn: (r) => (r._field == "volume" and r._value > 0)) + |> group(columns:["_time","to","emitter_chain"]) + |> sum() + |> rename(columns: {_value: "volume"}) + + counts = data + |> filter(fn: (r) => (r._field == "count")) + |> group(columns:["_time","to","emitter_chain"]) + |> sum() + |> rename(columns: {_value: "count"}) + + join.inner( + left: vols, + right: counts, + on: (l, r) => l._time == r._time and l.to == r.to and l.emitter_chain == r.emitter_chain, + as: (l, r) => ({l with count: r.count}), + ) |> group() - |> sort(columns:["_time"],desc:false) + |> sort(columns:["emitter_chain","_time"],desc:false) ` start := time.Date(q.From.Year(), q.From.Month(), 1, 0, 0, 0, 0, q.From.Location()).UTC().Format(time.RFC3339) stop := time.Date(q.To.Year(), q.To.Month(), 1, 0, 0, 0, 0, q.To.Location()).UTC().Format(time.RFC3339) - return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, filterSourceChain, filterDestinationChain) + return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, filterSourceChain, filterTargetChain) } query := ` - data = from(bucket: "%s") - |> range(start: %s,stop: %s) - |> filter(fn: (r) => r._measurement == "chain_activity_1d") - %s - %s - |> window(every: 1y) - |> sum() - |> map(fn: (r) => ({r with to: date.add(d: 1y,to: r._time)})) - |> group() - |> sort(columns:["_time"],desc:false) - ` + import "date" + import "join" + data = from(bucket: "%s") + |> range(start: %s,stop: %s) + |> filter(fn: (r) => r._measurement == "chain_activity_1d") + %s + %s + |> drop(columns:["destination_chain"]) + |> window(every: 1y) + + vols = data + |> filter(fn: (r) => (r._field == "volume" and r._value > 0)) + |> group(columns:["_time","to","emitter_chain"]) + |> sum() + |> rename(columns: {_value: "volume"}) + + counts = data + |> filter(fn: (r) => (r._field == "count")) + |> group(columns:["_time","to","emitter_chain"]) + |> sum() + |> rename(columns: {_value: "count"}) + + join.inner( + left: vols, + right: counts, + on: (l, r) => l._time == r._time and l.to == r.to and l.emitter_chain == r.emitter_chain, + as: (l, r) => ({l with count: r.count}), + ) + |> group() + |> sort(columns:["emitter_chain","_time"],desc:false) + ` start := time.Date(q.From.Year(), 1, 1, 0, 0, 0, 0, q.From.Location()).UTC().Format(time.RFC3339) stop := time.Date(q.To.Year(), 1, 1, 0, 0, 0, 0, q.To.Location()).UTC().Format(time.RFC3339) - return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, filterSourceChain, filterDestinationChain) + return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, filterSourceChain, filterTargetChain) } From fd5142e699a062040c093fac6ea15ddbada8cc45 Mon Sep 17 00:00:00 2001 From: Mariano <9205080+marianososto@users.noreply.github.com> Date: Tue, 16 Apr 2024 18:24:38 -0300 Subject: [PATCH 13/26] making more progress --- api/handlers/transactions/model.go | 2 +- api/handlers/transactions/repository.go | 35 ++++++++++++++----------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/api/handlers/transactions/model.go b/api/handlers/transactions/model.go index a925ffc9..8b1fb08c 100644 --- a/api/handlers/transactions/model.go +++ b/api/handlers/transactions/model.go @@ -226,7 +226,7 @@ const ( Hour TimeInterval = "1h" Day TimeInterval = "1d" Week TimeInterval = "1w" - Month TimeInterval = "1m0" + Month TimeInterval = "1mo" Year TimeInterval = "1y" ) diff --git a/api/handlers/transactions/repository.go b/api/handlers/transactions/repository.go index 4fea9692..010848c2 100644 --- a/api/handlers/transactions/repository.go +++ b/api/handlers/transactions/repository.go @@ -1169,37 +1169,40 @@ func (r *Repository) buildChainActivityQueryTops(q *ChainActivityTopsQuery) stri if q.TimeInterval == Month { query := ` - import "date" - import "join" + import "date" + import "join" - data = from(bucket: "%s") - |> range(start: %s,stop: %s) - |> filter(fn: (r) => r._measurement == "chain_activity_1d") - %s - %s - |> drop(columns:["destination_chain"]) - |> window(every: 1mo) + data = from(bucket: "%s") + |> range(start: %s,stop: %s) + |> filter(fn: (r) => r._measurement == "chain_activity_1d") + %s + %s + |> drop(columns:["destination_chain","to","app_id"]) + |> window(every: 1mo, period:1mo) + |> drop(columns:["_time"]) + |> rename(columns: {_start: "_time"}) + |> map(fn: (r) => ({r with to: string(v: r._stop)})) - vols = data + vols = data |> filter(fn: (r) => (r._field == "volume" and r._value > 0)) |> group(columns:["_time","to","emitter_chain"]) |> sum() |> rename(columns: {_value: "volume"}) - counts = data + counts = data |> filter(fn: (r) => (r._field == "count")) |> group(columns:["_time","to","emitter_chain"]) |> sum() |> rename(columns: {_value: "count"}) - join.inner( + join.inner( left: vols, right: counts, - on: (l, r) => l._time == r._time and l.to == r.to and l.emitter_chain == r.emitter_chain, + on: (l, r) => l._time == r._time and l.emitter_chain == r.emitter_chain, as: (l, r) => ({l with count: r.count}), - ) - |> group() - |> sort(columns:["emitter_chain","_time"],desc:false) + ) + |> group() + |> sort(columns:["emitter_chain","_time"],desc:false) ` start := time.Date(q.From.Year(), q.From.Month(), 1, 0, 0, 0, 0, q.From.Location()).UTC().Format(time.RFC3339) From 7c4819bab047df0f0299a4a6d5fcf619fb2b29cb Mon Sep 17 00:00:00 2001 From: Mariano <9205080+marianososto@users.noreply.github.com> Date: Wed, 17 Apr 2024 10:38:20 -0300 Subject: [PATCH 14/26] fix per year query --- api/handlers/transactions/repository.go | 35 ++++++++++++++----------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/api/handlers/transactions/repository.go b/api/handlers/transactions/repository.go index 010848c2..f2132397 100644 --- a/api/handlers/transactions/repository.go +++ b/api/handlers/transactions/repository.go @@ -1211,37 +1211,40 @@ func (r *Repository) buildChainActivityQueryTops(q *ChainActivityTopsQuery) stri } query := ` - import "date" - import "join" + import "date" + import "join" - data = from(bucket: "%s") - |> range(start: %s,stop: %s) - |> filter(fn: (r) => r._measurement == "chain_activity_1d") - %s - %s - |> drop(columns:["destination_chain"]) - |> window(every: 1y) + data = from(bucket: "%s") + |> range(start: %s,stop: %s) + |> filter(fn: (r) => r._measurement == "chain_activity_1d") + %s + %s + |> drop(columns:["destination_chain","to","app_id"]) + |> window(every: 1y, period:1y) + |> drop(columns:["_time"]) + |> rename(columns: {_start: "_time"}) + |> map(fn: (r) => ({r with to: string(v: r._stop)})) - vols = data + vols = data |> filter(fn: (r) => (r._field == "volume" and r._value > 0)) |> group(columns:["_time","to","emitter_chain"]) |> sum() |> rename(columns: {_value: "volume"}) - counts = data + counts = data |> filter(fn: (r) => (r._field == "count")) |> group(columns:["_time","to","emitter_chain"]) |> sum() |> rename(columns: {_value: "count"}) - join.inner( + join.inner( left: vols, right: counts, - on: (l, r) => l._time == r._time and l.to == r.to and l.emitter_chain == r.emitter_chain, + on: (l, r) => l._time == r._time and l.emitter_chain == r.emitter_chain, as: (l, r) => ({l with count: r.count}), - ) - |> group() - |> sort(columns:["emitter_chain","_time"],desc:false) + ) + |> group() + |> sort(columns:["emitter_chain","_time"],desc:false) ` start := time.Date(q.From.Year(), 1, 1, 0, 0, 0, 0, q.From.Location()).UTC().Format(time.RFC3339) stop := time.Date(q.To.Year(), 1, 1, 0, 0, 0, 0, q.To.Location()).UTC().Format(time.RFC3339) From 3f5675b957a62430362138ed9879349140574068 Mon Sep 17 00:00:00 2001 From: Mariano <9205080+marianososto@users.noreply.github.com> Date: Thu, 18 Apr 2024 10:18:36 -0300 Subject: [PATCH 15/26] add a second group of tasks for downsampling --- .../scripts/chain_activity_emitter_1d.flux | 41 +++++++++++++++++++ .../scripts/chain_activity_emitter_1h.flux | 41 +++++++++++++++++++ ...ux => chain_activity_emitter_dest_1d.flux} | 0 ...ux => chain_activity_emitter_dest_1h.flux} | 0 4 files changed, 82 insertions(+) create mode 100644 analytics/scripts/chain_activity_emitter_1d.flux create mode 100644 analytics/scripts/chain_activity_emitter_1h.flux rename analytics/scripts/{chain_activity_1d.flux => chain_activity_emitter_dest_1d.flux} (100%) rename analytics/scripts/{chain_activity_1h.flux => chain_activity_emitter_dest_1h.flux} (100%) diff --git a/analytics/scripts/chain_activity_emitter_1d.flux b/analytics/scripts/chain_activity_emitter_1d.flux new file mode 100644 index 00000000..8d88f8dc --- /dev/null +++ b/analytics/scripts/chain_activity_emitter_1d.flux @@ -0,0 +1,41 @@ +import "date" + + +runTask = (start,stop,srcBucket,destBucket,destMeasurement) => { + + data = from(bucket: "wormscan-mainnet-staging") + |> range(start: start,stop: stop) + |> filter(fn: (r) => r._measurement == "vaa_volume_v2" and r.version == "v2") + |> filter(fn: (r) => r._field == "volume" and r._value > 0) + |> drop(columns:["destination_chain","app_id","token_chain","token_address","version","_measurement","_time"]) + |> group(columns: ["emitter_chain"]) + |> rename(columns: {_start: "_time"}) + + data + |> sum(column: "_value") + |> set(key: "_field", value: "volume") + |> set(key: "to", value: string(v:stop)) + |> set(key: "_measurement", value: destMeasurement) + |> to(bucket: destBucket) + + return data + |> count(column: "_value") + |> set(key: "_field", value: "count") + |> set(key: "to", value: string(v:stop)) + |> set(key: "_measurement", value: destMeasurement) + |> to(bucket: destBucket) +} + + +bucketInfinite = "wormscan" +destMeasurement = "emitter_chain_activity_1d" + +stop = date.truncate(t: now(),unit: 1d) +start = date.sub(d: 1d, from: stop) + +option task = { + name: "calculate chain activity every day", + every: 1d, +} + +runTask(start:start, stop: stop, srcBucket: bucketInfinite, destBucket: bucketInfinite, destMeasurement: destMeasurement) \ No newline at end of file diff --git a/analytics/scripts/chain_activity_emitter_1h.flux b/analytics/scripts/chain_activity_emitter_1h.flux new file mode 100644 index 00000000..5e3d2b94 --- /dev/null +++ b/analytics/scripts/chain_activity_emitter_1h.flux @@ -0,0 +1,41 @@ +import "date" + + +runTask = (start,stop,srcBucket,destBucket,destMeasurement) => { + + data = from(bucket: "wormscan-mainnet-staging") + |> range(start: start,stop: stop) + |> filter(fn: (r) => r._measurement == "vaa_volume_v2" and r.version == "v2") + |> filter(fn: (r) => r._field == "volume" and r._value > 0) + |> drop(columns:["destination_chain","app_id","token_chain","token_address","version","_measurement","_time"]) + |> group(columns: ["emitter_chain"]) + |> rename(columns: {_start: "_time"}) + + data + |> sum(column: "_value") + |> set(key: "_field", value: "volume") + |> set(key: "to", value: string(v:stop)) + |> set(key: "_measurement", value: destMeasurement) + |> to(bucket: destBucket) + + return data + |> count(column: "_value") + |> set(key: "_field", value: "count") + |> set(key: "to", value: string(v:stop)) + |> set(key: "_measurement", value: destMeasurement) + |> to(bucket: destBucket) +} + + +bucketInfinite = "wormscan" +destMeasurement = "emitter_chain_activity_1h" + +stop = date.truncate(t: now(),unit: 1h) +start = date.sub(d: 1h, from: stop) + +option task = { + name: "calculate chain activity every hour", + every: 1h, +} + +runTask(start:start, stop: stop, srcBucket: bucketInfinite, destBucket: bucketInfinite, destMeasurement: destMeasurement) \ No newline at end of file diff --git a/analytics/scripts/chain_activity_1d.flux b/analytics/scripts/chain_activity_emitter_dest_1d.flux similarity index 100% rename from analytics/scripts/chain_activity_1d.flux rename to analytics/scripts/chain_activity_emitter_dest_1d.flux diff --git a/analytics/scripts/chain_activity_1h.flux b/analytics/scripts/chain_activity_emitter_dest_1h.flux similarity index 100% rename from analytics/scripts/chain_activity_1h.flux rename to analytics/scripts/chain_activity_emitter_dest_1h.flux From 7c49ca0f9dacefe2d404bb18b95e07ed23d80532 Mon Sep 17 00:00:00 2001 From: Mariano <9205080+marianososto@users.noreply.github.com> Date: Thu, 18 Apr 2024 12:02:37 -0300 Subject: [PATCH 16/26] add app_id --- api/handlers/transactions/model.go | 1 + api/handlers/transactions/repository.go | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/api/handlers/transactions/model.go b/api/handlers/transactions/model.go index 8b1fb08c..d2924efb 100644 --- a/api/handlers/transactions/model.go +++ b/api/handlers/transactions/model.go @@ -215,6 +215,7 @@ type TransactionDto struct { type ChainActivityTopsQuery struct { SourceChain sdk.ChainID `json:"source_chain"` TargetChain sdk.ChainID `json:"target_chain"` + AppId string `json:"app_id"` From time.Time `json:"from"` To time.Time `json:"to"` TimeInterval TimeInterval `json:"time_interval"` diff --git a/api/handlers/transactions/repository.go b/api/handlers/transactions/repository.go index f2132397..53baa0cf 100644 --- a/api/handlers/transactions/repository.go +++ b/api/handlers/transactions/repository.go @@ -1087,6 +1087,11 @@ func (r *Repository) buildChainActivityQueryTops(q *ChainActivityTopsQuery) stri filterSourceChain = "|> filter(fn: (r) => r.emitter_chain == \"" + strconv.Itoa(int(q.SourceChain)) + "\")" } + filterAppId := "" + if q.AppId != "" { + filterAppId = "|> filter(fn: (r) => r.app_id == \"" + q.AppId + "\")" + } + if q.TimeInterval == Hour { start := q.From.UTC().Format(time.RFC3339) @@ -1101,6 +1106,7 @@ func (r *Repository) buildChainActivityQueryTops(q *ChainActivityTopsQuery) stri |> filter(fn: (r) => r._measurement == "chain_activity_1h") %s %s + %s |> drop(columns:["destination_chain"]) vols = data @@ -1124,7 +1130,7 @@ func (r *Repository) buildChainActivityQueryTops(q *ChainActivityTopsQuery) stri |> group() |> sort(columns:["emitter_chain","_time"],desc:false) ` - return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, filterSourceChain, filterTargetChain) + return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, filterSourceChain, filterTargetChain, filterAppId) } if q.TimeInterval == Day { @@ -1141,6 +1147,7 @@ func (r *Repository) buildChainActivityQueryTops(q *ChainActivityTopsQuery) stri |> filter(fn: (r) => r._measurement == "chain_activity_1d") %s %s + %s |> drop(columns:["destination_chain"]) vols = data @@ -1164,7 +1171,7 @@ func (r *Repository) buildChainActivityQueryTops(q *ChainActivityTopsQuery) stri |> group() |> sort(columns:["emitter_chain","_time"],desc:false) ` - return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, filterSourceChain, filterTargetChain) + return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, filterSourceChain, filterTargetChain, filterAppId) } if q.TimeInterval == Month { @@ -1177,6 +1184,7 @@ func (r *Repository) buildChainActivityQueryTops(q *ChainActivityTopsQuery) stri |> filter(fn: (r) => r._measurement == "chain_activity_1d") %s %s + %s |> drop(columns:["destination_chain","to","app_id"]) |> window(every: 1mo, period:1mo) |> drop(columns:["_time"]) @@ -1207,7 +1215,7 @@ func (r *Repository) buildChainActivityQueryTops(q *ChainActivityTopsQuery) stri start := time.Date(q.From.Year(), q.From.Month(), 1, 0, 0, 0, 0, q.From.Location()).UTC().Format(time.RFC3339) stop := time.Date(q.To.Year(), q.To.Month(), 1, 0, 0, 0, 0, q.To.Location()).UTC().Format(time.RFC3339) - return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, filterSourceChain, filterTargetChain) + return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, filterSourceChain, filterTargetChain, filterAppId) } query := ` @@ -1219,6 +1227,7 @@ func (r *Repository) buildChainActivityQueryTops(q *ChainActivityTopsQuery) stri |> filter(fn: (r) => r._measurement == "chain_activity_1d") %s %s + %s |> drop(columns:["destination_chain","to","app_id"]) |> window(every: 1y, period:1y) |> drop(columns:["_time"]) @@ -1248,6 +1257,6 @@ func (r *Repository) buildChainActivityQueryTops(q *ChainActivityTopsQuery) stri ` start := time.Date(q.From.Year(), 1, 1, 0, 0, 0, 0, q.From.Location()).UTC().Format(time.RFC3339) stop := time.Date(q.To.Year(), 1, 1, 0, 0, 0, 0, q.To.Location()).UTC().Format(time.RFC3339) - return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, filterSourceChain, filterTargetChain) + return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, filterSourceChain, filterTargetChain, filterAppId) } From 567e2415b719fe15695ef0752bc53e52c1716340 Mon Sep 17 00:00:00 2001 From: Mariano <9205080+marianososto@users.noreply.github.com> Date: Thu, 18 Apr 2024 14:49:10 -0300 Subject: [PATCH 17/26] update swagger docs --- api/docs/docs.go | 63 +++++++++++++++++++ api/docs/swagger.json | 63 +++++++++++++++++++ api/docs/swagger.yaml | 44 +++++++++++++ api/handlers/transactions/model.go | 2 + api/handlers/transactions/service.go | 2 +- .../wormscan/transactions/controller.go | 10 +++ 6 files changed, 183 insertions(+), 1 deletion(-) diff --git a/api/docs/docs.go b/api/docs/docs.go index 45c1646a..1636d1e5 100644 --- a/api/docs/docs.go +++ b/api/docs/docs.go @@ -3325,6 +3325,52 @@ const docTemplate = `{ } } }, + "transactions.ChainActivityTopResult": { + "type": "object", + "properties": { + "count": { + "type": "integer" + }, + "destination_chain": { + "type": "string" + }, + "emitter_chain": { + "type": "string" + }, + "from": { + "type": "string" + }, + "to": { + "type": "string" + }, + "volume": { + "type": "integer" + } + } + }, + "transactions.ChainActivityTopsQuery": { + "type": "object", + "properties": { + "app_id": { + "type": "string" + }, + "from": { + "type": "string" + }, + "source_chain": { + "$ref": "#/definitions/vaa.ChainID" + }, + "target_chain": { + "$ref": "#/definitions/vaa.ChainID" + }, + "time_interval": { + "$ref": "#/definitions/transactions.TimeInterval" + }, + "to": { + "type": "string" + } + } + }, "transactions.ChainPair": { "type": "object", "properties": { @@ -3459,6 +3505,23 @@ const docTemplate = `{ } } }, + "transactions.TimeInterval": { + "type": "string", + "enum": [ + "1h", + "1d", + "1w", + "1mo", + "1y" + ], + "x-enum-varnames": [ + "Hour", + "Day", + "Week", + "Month", + "Year" + ] + }, "transactions.Token": { "type": "object", "properties": { diff --git a/api/docs/swagger.json b/api/docs/swagger.json index cf92f224..8ad5863e 100644 --- a/api/docs/swagger.json +++ b/api/docs/swagger.json @@ -3318,6 +3318,52 @@ } } }, + "transactions.ChainActivityTopResult": { + "type": "object", + "properties": { + "count": { + "type": "integer" + }, + "destination_chain": { + "type": "string" + }, + "emitter_chain": { + "type": "string" + }, + "from": { + "type": "string" + }, + "to": { + "type": "string" + }, + "volume": { + "type": "integer" + } + } + }, + "transactions.ChainActivityTopsQuery": { + "type": "object", + "properties": { + "app_id": { + "type": "string" + }, + "from": { + "type": "string" + }, + "source_chain": { + "$ref": "#/definitions/vaa.ChainID" + }, + "target_chain": { + "$ref": "#/definitions/vaa.ChainID" + }, + "time_interval": { + "$ref": "#/definitions/transactions.TimeInterval" + }, + "to": { + "type": "string" + } + } + }, "transactions.ChainPair": { "type": "object", "properties": { @@ -3452,6 +3498,23 @@ } } }, + "transactions.TimeInterval": { + "type": "string", + "enum": [ + "1h", + "1d", + "1w", + "1mo", + "1y" + ], + "x-enum-varnames": [ + "Hour", + "Day", + "Week", + "Month", + "Year" + ] + }, "transactions.Token": { "type": "object", "properties": { diff --git a/api/docs/swagger.yaml b/api/docs/swagger.yaml index 02ef4289..e9537d5e 100644 --- a/api/docs/swagger.yaml +++ b/api/docs/swagger.yaml @@ -824,6 +824,36 @@ definitions: $ref: '#/definitions/transactions.Tx' type: array type: object + transactions.ChainActivityTopResult: + properties: + count: + type: integer + destination_chain: + type: string + emitter_chain: + type: string + from: + type: string + to: + type: string + volume: + type: integer + type: object + transactions.ChainActivityTopsQuery: + properties: + app_id: + type: string + from: + type: string + source_chain: + $ref: '#/definitions/vaa.ChainID' + target_chain: + $ref: '#/definitions/vaa.ChainID' + time_interval: + $ref: '#/definitions/transactions.TimeInterval' + to: + type: string + type: object transactions.ChainPair: properties: destinationChain: @@ -917,6 +947,20 @@ definitions: description: Total value locked in USD. type: string type: object + transactions.TimeInterval: + enum: + - 1h + - 1d + - 1w + - 1mo + - 1y + type: string + x-enum-varnames: + - Hour + - Day + - Week + - Month + - Year transactions.Token: properties: coingeckoId: diff --git a/api/handlers/transactions/model.go b/api/handlers/transactions/model.go index d2924efb..d510e01e 100644 --- a/api/handlers/transactions/model.go +++ b/api/handlers/transactions/model.go @@ -152,6 +152,8 @@ type ChainActivityTopResult struct { Txs uint64 `mapstructure:"count" json:"count"` } +type ChainActivityTopResults []ChainActivityTopResult + type ChainActivityTimeSpan string const ( diff --git a/api/handlers/transactions/service.go b/api/handlers/transactions/service.go index 4dd2f074..0317f2b6 100644 --- a/api/handlers/transactions/service.go +++ b/api/handlers/transactions/service.go @@ -170,7 +170,7 @@ func (s *Service) GetTokenProvider() *domain.TokenProvider { return s.tokenProvider } -func (s *Service) GetChainActivityTops(ctx *fasthttp.RequestCtx, q *ChainActivityTopsQuery) (interface{}, error) { +func (s *Service) GetChainActivityTops(ctx *fasthttp.RequestCtx, q *ChainActivityTopsQuery) (ChainActivityTopResults, error) { timeDuration := q.To.Sub(q.From) diff --git a/api/routes/wormscan/transactions/controller.go b/api/routes/wormscan/transactions/controller.go index 1251a130..a347e152 100644 --- a/api/routes/wormscan/transactions/controller.go +++ b/api/routes/wormscan/transactions/controller.go @@ -184,6 +184,16 @@ func (c *Controller) GetTopAssets(ctx *fiber.Ctx) error { return ctx.JSON(response) } +// GetChainActivityTops godoc +// @Description Search, for a specific period of time, the number of transactions and the volume. +// @Tags wormholescan +// @ID x-chain-activity-tops +// @Method Post +// @Param {object} body transactions.ChainActivityTopsQuery true "Specify the query filters" +// @Success 200 {object} transactions.ChainActivityTopResults +// @Failure 400 +// @Failure 500 +// @Router /api/v1/x-chain-activity [get] func (c *Controller) GetChainActivityTops(ctx *fiber.Ctx) error { payload := &transactions.ChainActivityTopsQuery{} From 54d39ad6e721f6224ba1f4baf9a2160f4810cc13 Mon Sep 17 00:00:00 2001 From: Mariano <9205080+marianososto@users.noreply.github.com> Date: Thu, 18 Apr 2024 18:25:58 -0300 Subject: [PATCH 18/26] optimize new tasks --- .../scripts/chain_activity_emitter_1d.flux | 41 +++++++++---------- .../scripts/chain_activity_emitter_1h.flux | 41 +++++++++---------- 2 files changed, 40 insertions(+), 42 deletions(-) diff --git a/analytics/scripts/chain_activity_emitter_1d.flux b/analytics/scripts/chain_activity_emitter_1d.flux index 8d88f8dc..cc4afd5f 100644 --- a/analytics/scripts/chain_activity_emitter_1d.flux +++ b/analytics/scripts/chain_activity_emitter_1d.flux @@ -3,30 +3,29 @@ import "date" runTask = (start,stop,srcBucket,destBucket,destMeasurement) => { - data = from(bucket: "wormscan-mainnet-staging") - |> range(start: start,stop: stop) - |> filter(fn: (r) => r._measurement == "vaa_volume_v2" and r.version == "v2") - |> filter(fn: (r) => r._field == "volume" and r._value > 0) - |> drop(columns:["destination_chain","app_id","token_chain","token_address","version","_measurement","_time"]) - |> group(columns: ["emitter_chain"]) - |> rename(columns: {_start: "_time"}) + data = from(bucket: srcBucket) + |> range(start: start,stop: stop) + |> filter(fn: (r) => r._measurement == "vaa_volume_v2" and r.version == "v2") + |> filter(fn: (r) => r._field == "volume" and r._value > 0) + |> drop(columns:["destination_chain","app_id","token_chain","token_address","version","_measurement","_time"]) + |> rename(columns: {_start: "_time"}) + |> group(columns: ["emitter_chain","_time"]) - data - |> sum(column: "_value") - |> set(key: "_field", value: "volume") - |> set(key: "to", value: string(v:stop)) - |> set(key: "_measurement", value: destMeasurement) - |> to(bucket: destBucket) + vols = data + |> sum(column: "_value") + |> set(key: "_field", value: "volume") + |> set(key: "to", value: string(v:stop)) + |> set(key: "_measurement", value: destMeasurement) + |> to(bucket: destBucket) - return data - |> count(column: "_value") - |> set(key: "_field", value: "count") - |> set(key: "to", value: string(v:stop)) - |> set(key: "_measurement", value: destMeasurement) - |> to(bucket: destBucket) + counts = data + |> count(column: "_value") + |> set(key: "_field", value: "count") + |> set(key: "to", value: string(v:stop)) + |> set(key: "_measurement", value: destMeasurement)] + |> to(bucket: destBucket) } - bucketInfinite = "wormscan" destMeasurement = "emitter_chain_activity_1d" @@ -34,7 +33,7 @@ stop = date.truncate(t: now(),unit: 1d) start = date.sub(d: 1d, from: stop) option task = { - name: "calculate chain activity every day", + name: "calculate chain activity per emitter every day", every: 1d, } diff --git a/analytics/scripts/chain_activity_emitter_1h.flux b/analytics/scripts/chain_activity_emitter_1h.flux index 5e3d2b94..72121ec3 100644 --- a/analytics/scripts/chain_activity_emitter_1h.flux +++ b/analytics/scripts/chain_activity_emitter_1h.flux @@ -3,30 +3,29 @@ import "date" runTask = (start,stop,srcBucket,destBucket,destMeasurement) => { - data = from(bucket: "wormscan-mainnet-staging") - |> range(start: start,stop: stop) - |> filter(fn: (r) => r._measurement == "vaa_volume_v2" and r.version == "v2") - |> filter(fn: (r) => r._field == "volume" and r._value > 0) - |> drop(columns:["destination_chain","app_id","token_chain","token_address","version","_measurement","_time"]) - |> group(columns: ["emitter_chain"]) - |> rename(columns: {_start: "_time"}) + data = from(bucket: srcBucket) + |> range(start: start,stop: stop) + |> filter(fn: (r) => r._measurement == "vaa_volume_v2" and r.version == "v2") + |> filter(fn: (r) => r._field == "volume" and r._value > 0) + |> drop(columns:["destination_chain","app_id","token_chain","token_address","version","_measurement","_time"]) + |> rename(columns: {_start: "_time"}) + |> group(columns: ["emitter_chain","_time"]) - data - |> sum(column: "_value") - |> set(key: "_field", value: "volume") - |> set(key: "to", value: string(v:stop)) - |> set(key: "_measurement", value: destMeasurement) - |> to(bucket: destBucket) + vols = data + |> sum(column: "_value") + |> set(key: "_field", value: "volume") + |> set(key: "to", value: string(v:stop)) + |> set(key: "_measurement", value: destMeasurement) + |> to(bucket: destBucket) - return data - |> count(column: "_value") - |> set(key: "_field", value: "count") - |> set(key: "to", value: string(v:stop)) - |> set(key: "_measurement", value: destMeasurement) - |> to(bucket: destBucket) + counts = data + |> count(column: "_value") + |> set(key: "_field", value: "count") + |> set(key: "to", value: string(v:stop)) + |> set(key: "_measurement", value: destMeasurement)] + |> to(bucket: destBucket) } - bucketInfinite = "wormscan" destMeasurement = "emitter_chain_activity_1h" @@ -34,7 +33,7 @@ stop = date.truncate(t: now(),unit: 1h) start = date.sub(d: 1h, from: stop) option task = { - name: "calculate chain activity every hour", + name: "calculate chain activity per emitter every hour", every: 1h, } From 073b20c87e41b855799094887ee3332e8ca879a5 Mon Sep 17 00:00:00 2001 From: Mariano <9205080+marianososto@users.noreply.github.com> Date: Fri, 19 Apr 2024 15:21:59 -0300 Subject: [PATCH 19/26] fix W --- analytics/scripts/chain_activity_emitter_1h.flux | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/analytics/scripts/chain_activity_emitter_1h.flux b/analytics/scripts/chain_activity_emitter_1h.flux index 72121ec3..eccc9039 100644 --- a/analytics/scripts/chain_activity_emitter_1h.flux +++ b/analytics/scripts/chain_activity_emitter_1h.flux @@ -18,11 +18,11 @@ runTask = (start,stop,srcBucket,destBucket,destMeasurement) => { |> set(key: "_measurement", value: destMeasurement) |> to(bucket: destBucket) - counts = data + return data |> count(column: "_value") |> set(key: "_field", value: "count") |> set(key: "to", value: string(v:stop)) - |> set(key: "_measurement", value: destMeasurement)] + |> set(key: "_measurement", value: destMeasurement) |> to(bucket: destBucket) } From caf47e4fb2f8906e5dc13b7c0b8bf2ee261ecb83 Mon Sep 17 00:00:00 2001 From: Mariano <9205080+marianososto@users.noreply.github.com> Date: Fri, 19 Apr 2024 15:22:19 -0300 Subject: [PATCH 20/26] fix W --- analytics/scripts/chain_activity_emitter_1d.flux | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/analytics/scripts/chain_activity_emitter_1d.flux b/analytics/scripts/chain_activity_emitter_1d.flux index cc4afd5f..5ec356a6 100644 --- a/analytics/scripts/chain_activity_emitter_1d.flux +++ b/analytics/scripts/chain_activity_emitter_1d.flux @@ -18,11 +18,11 @@ runTask = (start,stop,srcBucket,destBucket,destMeasurement) => { |> set(key: "_measurement", value: destMeasurement) |> to(bucket: destBucket) - counts = data + return data |> count(column: "_value") |> set(key: "_field", value: "count") |> set(key: "to", value: string(v:stop)) - |> set(key: "_measurement", value: destMeasurement)] + |> set(key: "_measurement", value: destMeasurement) |> to(bucket: destBucket) } From 976c8496af59f536600ed4509664ed4dbba84f55 Mon Sep 17 00:00:00 2001 From: Mariano <9205080+marianososto@users.noreply.github.com> Date: Fri, 19 Apr 2024 17:39:50 -0300 Subject: [PATCH 21/26] start using the new measurement --- api/handlers/transactions/model.go | 3 +- api/handlers/transactions/repository.go | 128 +++++++++++++++++++++--- 2 files changed, 117 insertions(+), 14 deletions(-) diff --git a/api/handlers/transactions/model.go b/api/handlers/transactions/model.go index d510e01e..a8f60864 100644 --- a/api/handlers/transactions/model.go +++ b/api/handlers/transactions/model.go @@ -228,11 +228,10 @@ type TimeInterval string const ( Hour TimeInterval = "1h" Day TimeInterval = "1d" - Week TimeInterval = "1w" Month TimeInterval = "1mo" Year TimeInterval = "1y" ) func (t TimeInterval) IsValid() bool { - return t == Hour || t == Day || t == Week || t == Month || t == Year + return t == Hour || t == Day || t == Month || t == Year } diff --git a/api/handlers/transactions/repository.go b/api/handlers/transactions/repository.go index 53baa0cf..33d5d9e6 100644 --- a/api/handlers/transactions/repository.go +++ b/api/handlers/transactions/repository.go @@ -1077,6 +1077,21 @@ func (r *Repository) FindChainActivityTops(ctx *fasthttp.RequestCtx, q *ChainAct func (r *Repository) buildChainActivityQueryTops(q *ChainActivityTopsQuery) string { + var start, stop string + if q.TimeInterval == Hour { + start = q.From.Truncate(1 * time.Hour).UTC().Format(time.RFC3339) + stop = q.To.Truncate(1 * time.Hour).UTC().Format(time.RFC3339) + } else if q.TimeInterval == Day { + start = q.From.Truncate(24 * time.Hour).UTC().Format(time.RFC3339) + stop = q.To.Truncate(24 * time.Hour).UTC().Format(time.RFC3339) + } else if q.TimeInterval == Month { + start = time.Date(q.From.Year(), q.From.Month(), 1, 0, 0, 0, 0, q.From.Location()).UTC().Format(time.RFC3339) + stop = time.Date(q.To.Year(), q.To.Month(), 1, 0, 0, 0, 0, q.To.Location()).UTC().Format(time.RFC3339) + } else { + start = time.Date(q.From.Year(), 1, 1, 0, 0, 0, 0, q.From.Location()).UTC().Format(time.RFC3339) + stop = time.Date(q.To.Year(), 1, 1, 0, 0, 0, 0, q.To.Location()).UTC().Format(time.RFC3339) + } + filterTargetChain := "" if q.TargetChain != sdk.ChainIDUnset { filterTargetChain = "|> filter(fn: (r) => r.destination_chain == \"" + strconv.Itoa(int(q.TargetChain)) + "\")" @@ -1092,11 +1107,108 @@ func (r *Repository) buildChainActivityQueryTops(q *ChainActivityTopsQuery) stri filterAppId = "|> filter(fn: (r) => r.app_id == \"" + q.AppId + "\")" } + if q.TargetChain == sdk.ChainIDUnset && q.AppId == "" { + + measurement := "" + switch q.TimeInterval { + case Hour: + measurement = "emitter_chain_activity_1h" + default: + measurement = "emitter_chain_activity_1d" + } + + if q.TimeInterval == Hour || q.TimeInterval == Day { + query := ` + import "date" + + from(bucket: "%s") + |> range(start: %s,stop: %s) + |> filter(fn: (r) => r._measurement == "%s") + %s + |> pivot(rowKey:["_time","emitter_chain"], columnKey: ["_field"], valueColumn: "_value") + |> sort(columns:["emitter_chain","_time"],desc:false) + ` + return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, measurement, filterSourceChain) + } + + if q.TimeInterval == Month { + query := ` + import "date" + import "join" + + data = from(bucket: "%s") + |> range(start: %s,stop: %s) + |> filter(fn: (r) => r._measurement == "%s") + %s + |> drop(columns:["to"]) + |> window(every: 1mo, period:1mo) + |> drop(columns:["_time"]) + |> rename(columns: {_start: "_time"}) + |> map(fn: (r) => ({r with to: string(v: r._stop)})) + + vols = data + |> filter(fn: (r) => (r._field == "volume" and r._value > 0)) + |> group(columns:["_time","to","emitter_chain"]) + |> sum() + |> rename(columns: {_value: "volume"}) + + counts = data + |> filter(fn: (r) => (r._field == "count")) + |> group(columns:["_time","to","emitter_chain"]) + |> sum() + |> rename(columns: {_value: "count"}) + + join.inner( + left: vols, + right: counts, + on: (l, r) => l._time == r._time and l.emitter_chain == r.emitter_chain, + as: (l, r) => ({l with count: r.count}), + ) + |> group() + |> sort(columns:["emitter_chain","_time"],desc:false) + ` + return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, measurement, filterSourceChain) + } + + query := ` + import "date" + import "join" + + data = from(bucket: "%s") + |> range(start: %s,stop: %s) + |> filter(fn: (r) => r._measurement == "%s") + %s + |> drop(columns:["to"]) + |> window(every: 1y, period:1y) + |> drop(columns:["_time"]) + |> rename(columns: {_start: "_time"}) + |> map(fn: (r) => ({r with to: string(v: r._stop)})) + + vols = data + |> group(columns:["_time","to","emitter_chain"]) + |> sum() + |> rename(columns: {_value: "volume"}) + + counts = data + |> filter(fn: (r) => (r._field == "count")) + |> group(columns:["_time","to","emitter_chain"]) + |> sum() + |> rename(columns: {_value: "count"}) + + join.inner( + left: vols, + right: counts, + on: (l, r) => l._time == r._time and l.emitter_chain == r.emitter_chain, + as: (l, r) => ({l with count: r.count}), + ) + |> group() + |> sort(columns:["emitter_chain","_time"],desc:false) + ` + return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, measurement, filterSourceChain) + + } + if q.TimeInterval == Hour { - - start := q.From.UTC().Format(time.RFC3339) - stop := q.To.UTC().Format(time.RFC3339) - query := ` import "date" import "join" @@ -1135,9 +1247,6 @@ func (r *Repository) buildChainActivityQueryTops(q *ChainActivityTopsQuery) stri if q.TimeInterval == Day { - start := q.From.Truncate(24 * time.Hour).UTC().Format(time.RFC3339) - stop := q.To.Truncate(24 * time.Hour).UTC().Format(time.RFC3339) - query := ` import "date" import "join" @@ -1212,9 +1321,6 @@ func (r *Repository) buildChainActivityQueryTops(q *ChainActivityTopsQuery) stri |> group() |> sort(columns:["emitter_chain","_time"],desc:false) ` - - start := time.Date(q.From.Year(), q.From.Month(), 1, 0, 0, 0, 0, q.From.Location()).UTC().Format(time.RFC3339) - stop := time.Date(q.To.Year(), q.To.Month(), 1, 0, 0, 0, 0, q.To.Location()).UTC().Format(time.RFC3339) return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, filterSourceChain, filterTargetChain, filterAppId) } @@ -1255,8 +1361,6 @@ func (r *Repository) buildChainActivityQueryTops(q *ChainActivityTopsQuery) stri |> group() |> sort(columns:["emitter_chain","_time"],desc:false) ` - start := time.Date(q.From.Year(), 1, 1, 0, 0, 0, 0, q.From.Location()).UTC().Format(time.RFC3339) - stop := time.Date(q.To.Year(), 1, 1, 0, 0, 0, 0, q.To.Location()).UTC().Format(time.RFC3339) return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, filterSourceChain, filterTargetChain, filterAppId) } From 1fb256f40409588184f6ab20d345a8529f30d645 Mon Sep 17 00:00:00 2001 From: Mariano <9205080+marianososto@users.noreply.github.com> Date: Mon, 22 Apr 2024 12:25:55 -0300 Subject: [PATCH 22/26] change endpoint signature --- api/docs/docs.go | 107 +++++++++++------- api/docs/swagger.json | 107 +++++++++++------- api/docs/swagger.yaml | 75 +++++++----- api/handlers/transactions/model.go | 24 ++-- api/handlers/transactions/repository.go | 34 +++--- api/handlers/transactions/service.go | 10 +- api/middleware/extract_parameters.go | 12 +- api/routes/wormscan/routes.go | 2 +- .../wormscan/transactions/controller.go | 45 ++++++-- 9 files changed, 253 insertions(+), 163 deletions(-) diff --git a/api/docs/docs.go b/api/docs/docs.go index 1636d1e5..a49382fa 100644 --- a/api/docs/docs.go +++ b/api/docs/docs.go @@ -1760,6 +1760,73 @@ const docTemplate = `{ } } }, + "/api/v1/x-chain-activity/search": { + "get": { + "description": "Search, for a specific period of time, the number of transactions and the volume.", + "tags": [ + "wormholescan" + ], + "operationId": "x-chain-activity-tops", + "parameters": [ + { + "type": "string", + "description": "Time span, supported values: 1d, 1mo and 1y", + "name": "timespan", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "From date, supported format 2006-01-02T15:04:05Z07:00", + "name": "from", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "To date, supported format 2006-01-02T15:04:05Z07:00", + "name": "to", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "Search by appId", + "name": "appId", + "in": "query" + }, + { + "type": "string", + "description": "Search by sourceChain", + "name": "sourceChain", + "in": "query" + }, + { + "type": "string", + "description": "Search by targetChain", + "name": "targetChain", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/transactions.ChainActivityTopResult" + } + } + }, + "400": { + "description": "Bad Request" + }, + "500": { + "description": "Internal Server Error" + } + } + } + }, "/swagger.json": { "get": { "description": "Returns the swagger specification for this API.", @@ -3348,29 +3415,6 @@ const docTemplate = `{ } } }, - "transactions.ChainActivityTopsQuery": { - "type": "object", - "properties": { - "app_id": { - "type": "string" - }, - "from": { - "type": "string" - }, - "source_chain": { - "$ref": "#/definitions/vaa.ChainID" - }, - "target_chain": { - "$ref": "#/definitions/vaa.ChainID" - }, - "time_interval": { - "$ref": "#/definitions/transactions.TimeInterval" - }, - "to": { - "type": "string" - } - } - }, "transactions.ChainPair": { "type": "object", "properties": { @@ -3505,23 +3549,6 @@ const docTemplate = `{ } } }, - "transactions.TimeInterval": { - "type": "string", - "enum": [ - "1h", - "1d", - "1w", - "1mo", - "1y" - ], - "x-enum-varnames": [ - "Hour", - "Day", - "Week", - "Month", - "Year" - ] - }, "transactions.Token": { "type": "object", "properties": { diff --git a/api/docs/swagger.json b/api/docs/swagger.json index 8ad5863e..44697db7 100644 --- a/api/docs/swagger.json +++ b/api/docs/swagger.json @@ -1753,6 +1753,73 @@ } } }, + "/api/v1/x-chain-activity/search": { + "get": { + "description": "Search, for a specific period of time, the number of transactions and the volume.", + "tags": [ + "wormholescan" + ], + "operationId": "x-chain-activity-tops", + "parameters": [ + { + "type": "string", + "description": "Time span, supported values: 1d, 1mo and 1y", + "name": "timespan", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "From date, supported format 2006-01-02T15:04:05Z07:00", + "name": "from", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "To date, supported format 2006-01-02T15:04:05Z07:00", + "name": "to", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "Search by appId", + "name": "appId", + "in": "query" + }, + { + "type": "string", + "description": "Search by sourceChain", + "name": "sourceChain", + "in": "query" + }, + { + "type": "string", + "description": "Search by targetChain", + "name": "targetChain", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/transactions.ChainActivityTopResult" + } + } + }, + "400": { + "description": "Bad Request" + }, + "500": { + "description": "Internal Server Error" + } + } + } + }, "/swagger.json": { "get": { "description": "Returns the swagger specification for this API.", @@ -3341,29 +3408,6 @@ } } }, - "transactions.ChainActivityTopsQuery": { - "type": "object", - "properties": { - "app_id": { - "type": "string" - }, - "from": { - "type": "string" - }, - "source_chain": { - "$ref": "#/definitions/vaa.ChainID" - }, - "target_chain": { - "$ref": "#/definitions/vaa.ChainID" - }, - "time_interval": { - "$ref": "#/definitions/transactions.TimeInterval" - }, - "to": { - "type": "string" - } - } - }, "transactions.ChainPair": { "type": "object", "properties": { @@ -3498,23 +3542,6 @@ } } }, - "transactions.TimeInterval": { - "type": "string", - "enum": [ - "1h", - "1d", - "1w", - "1mo", - "1y" - ], - "x-enum-varnames": [ - "Hour", - "Day", - "Week", - "Month", - "Year" - ] - }, "transactions.Token": { "type": "object", "properties": { diff --git a/api/docs/swagger.yaml b/api/docs/swagger.yaml index e9537d5e..92db5b78 100644 --- a/api/docs/swagger.yaml +++ b/api/docs/swagger.yaml @@ -839,21 +839,6 @@ definitions: volume: type: integer type: object - transactions.ChainActivityTopsQuery: - properties: - app_id: - type: string - from: - type: string - source_chain: - $ref: '#/definitions/vaa.ChainID' - target_chain: - $ref: '#/definitions/vaa.ChainID' - time_interval: - $ref: '#/definitions/transactions.TimeInterval' - to: - type: string - type: object transactions.ChainPair: properties: destinationChain: @@ -947,20 +932,6 @@ definitions: description: Total value locked in USD. type: string type: object - transactions.TimeInterval: - enum: - - 1h - - 1d - - 1w - - 1mo - - 1y - type: string - x-enum-varnames: - - Hour - - Day - - Week - - Month - - Year transactions.Token: properties: coingeckoId: @@ -2370,6 +2341,52 @@ paths: description: Internal Server Error tags: - wormholescan + /api/v1/x-chain-activity/search: + get: + description: Search, for a specific period of time, the number of transactions + and the volume. + operationId: x-chain-activity-tops + parameters: + - description: 'Time span, supported values: 1d, 1mo and 1y' + in: query + name: timespan + required: true + type: string + - description: From date, supported format 2006-01-02T15:04:05Z07:00 + in: query + name: from + required: true + type: string + - description: To date, supported format 2006-01-02T15:04:05Z07:00 + in: query + name: to + required: true + type: string + - description: Search by appId + in: query + name: appId + type: string + - description: Search by sourceChain + in: query + name: sourceChain + type: string + - description: Search by targetChain + in: query + name: targetChain + type: string + responses: + "200": + description: OK + schema: + items: + $ref: '#/definitions/transactions.ChainActivityTopResult' + type: array + "400": + description: Bad Request + "500": + description: Internal Server Error + tags: + - wormholescan /swagger.json: get: description: Returns the swagger specification for this API. diff --git a/api/handlers/transactions/model.go b/api/handlers/transactions/model.go index a8f60864..adc3beee 100644 --- a/api/handlers/transactions/model.go +++ b/api/handlers/transactions/model.go @@ -215,23 +215,23 @@ type TransactionDto struct { } type ChainActivityTopsQuery struct { - SourceChain sdk.ChainID `json:"source_chain"` - TargetChain sdk.ChainID `json:"target_chain"` - AppId string `json:"app_id"` - From time.Time `json:"from"` - To time.Time `json:"to"` - TimeInterval TimeInterval `json:"time_interval"` + SourceChain *sdk.ChainID `json:"source_chain"` + TargetChain *sdk.ChainID `json:"target_chain"` + AppId string `json:"app_id"` + From time.Time `json:"from"` + To time.Time `json:"to"` + Timespan Timespan `json:"timespan"` } -type TimeInterval string +type Timespan string const ( - Hour TimeInterval = "1h" - Day TimeInterval = "1d" - Month TimeInterval = "1mo" - Year TimeInterval = "1y" + Hour Timespan = "1h" + Day Timespan = "1d" + Month Timespan = "1mo" + Year Timespan = "1y" ) -func (t TimeInterval) IsValid() bool { +func (t Timespan) IsValid() bool { return t == Hour || t == Day || t == Month || t == Year } diff --git a/api/handlers/transactions/repository.go b/api/handlers/transactions/repository.go index 33d5d9e6..e72ca847 100644 --- a/api/handlers/transactions/repository.go +++ b/api/handlers/transactions/repository.go @@ -1050,7 +1050,7 @@ func (r *Repository) ListTransactionsByAddress( return documents, nil } -func (r *Repository) FindChainActivityTops(ctx *fasthttp.RequestCtx, q *ChainActivityTopsQuery) ([]ChainActivityTopResult, error) { +func (r *Repository) FindChainActivityTops(ctx *fasthttp.RequestCtx, q ChainActivityTopsQuery) ([]ChainActivityTopResult, error) { query := r.buildChainActivityQueryTops(q) result, err := r.queryAPI.Query(ctx, query) if err != nil { @@ -1062,7 +1062,7 @@ func (r *Repository) FindChainActivityTops(ctx *fasthttp.RequestCtx, q *ChainAct var response []ChainActivityTopResult for result.Next() { var row ChainActivityTopResult - if err := mapstructure.Decode(result.Record().Values(), &row); err != nil { + if err = mapstructure.Decode(result.Record().Values(), &row); err != nil { return nil, err } parsedTime, errTime := time.Parse(time.RFC3339Nano, row.To) @@ -1075,16 +1075,16 @@ func (r *Repository) FindChainActivityTops(ctx *fasthttp.RequestCtx, q *ChainAct return response, nil } -func (r *Repository) buildChainActivityQueryTops(q *ChainActivityTopsQuery) string { +func (r *Repository) buildChainActivityQueryTops(q ChainActivityTopsQuery) string { var start, stop string - if q.TimeInterval == Hour { + if q.Timespan == Hour { start = q.From.Truncate(1 * time.Hour).UTC().Format(time.RFC3339) stop = q.To.Truncate(1 * time.Hour).UTC().Format(time.RFC3339) - } else if q.TimeInterval == Day { + } else if q.Timespan == Day { start = q.From.Truncate(24 * time.Hour).UTC().Format(time.RFC3339) stop = q.To.Truncate(24 * time.Hour).UTC().Format(time.RFC3339) - } else if q.TimeInterval == Month { + } else if q.Timespan == Month { start = time.Date(q.From.Year(), q.From.Month(), 1, 0, 0, 0, 0, q.From.Location()).UTC().Format(time.RFC3339) stop = time.Date(q.To.Year(), q.To.Month(), 1, 0, 0, 0, 0, q.To.Location()).UTC().Format(time.RFC3339) } else { @@ -1093,13 +1093,13 @@ func (r *Repository) buildChainActivityQueryTops(q *ChainActivityTopsQuery) stri } filterTargetChain := "" - if q.TargetChain != sdk.ChainIDUnset { - filterTargetChain = "|> filter(fn: (r) => r.destination_chain == \"" + strconv.Itoa(int(q.TargetChain)) + "\")" + if q.TargetChain != nil { + filterTargetChain = "|> filter(fn: (r) => r.destination_chain == \"" + strconv.Itoa(int(*q.TargetChain)) + "\")" } filterSourceChain := "" - if q.SourceChain != sdk.ChainIDUnset { - filterSourceChain = "|> filter(fn: (r) => r.emitter_chain == \"" + strconv.Itoa(int(q.SourceChain)) + "\")" + if q.SourceChain != nil { + filterSourceChain = "|> filter(fn: (r) => r.emitter_chain == \"" + strconv.Itoa(int(*q.SourceChain)) + "\")" } filterAppId := "" @@ -1107,17 +1107,17 @@ func (r *Repository) buildChainActivityQueryTops(q *ChainActivityTopsQuery) stri filterAppId = "|> filter(fn: (r) => r.app_id == \"" + q.AppId + "\")" } - if q.TargetChain == sdk.ChainIDUnset && q.AppId == "" { + if q.TargetChain == nil && q.AppId == "" { measurement := "" - switch q.TimeInterval { + switch q.Timespan { case Hour: measurement = "emitter_chain_activity_1h" default: measurement = "emitter_chain_activity_1d" } - if q.TimeInterval == Hour || q.TimeInterval == Day { + if q.Timespan == Hour || q.Timespan == Day { query := ` import "date" @@ -1131,7 +1131,7 @@ func (r *Repository) buildChainActivityQueryTops(q *ChainActivityTopsQuery) stri return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, measurement, filterSourceChain) } - if q.TimeInterval == Month { + if q.Timespan == Month { query := ` import "date" import "join" @@ -1208,7 +1208,7 @@ func (r *Repository) buildChainActivityQueryTops(q *ChainActivityTopsQuery) stri } - if q.TimeInterval == Hour { + if q.Timespan == Hour { query := ` import "date" import "join" @@ -1245,7 +1245,7 @@ func (r *Repository) buildChainActivityQueryTops(q *ChainActivityTopsQuery) stri return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, filterSourceChain, filterTargetChain, filterAppId) } - if q.TimeInterval == Day { + if q.Timespan == Day { query := ` import "date" @@ -1283,7 +1283,7 @@ func (r *Repository) buildChainActivityQueryTops(q *ChainActivityTopsQuery) stri return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, filterSourceChain, filterTargetChain, filterAppId) } - if q.TimeInterval == Month { + if q.Timespan == Month { query := ` import "date" import "join" diff --git a/api/handlers/transactions/service.go b/api/handlers/transactions/service.go index 0317f2b6..6e6f4ca9 100644 --- a/api/handlers/transactions/service.go +++ b/api/handlers/transactions/service.go @@ -170,15 +170,15 @@ func (s *Service) GetTokenProvider() *domain.TokenProvider { return s.tokenProvider } -func (s *Service) GetChainActivityTops(ctx *fasthttp.RequestCtx, q *ChainActivityTopsQuery) (ChainActivityTopResults, error) { +func (s *Service) GetChainActivityTops(ctx *fasthttp.RequestCtx, q ChainActivityTopsQuery) (ChainActivityTopResults, error) { timeDuration := q.To.Sub(q.From) - if q.TimeInterval == Hour && timeDuration > 15*24*time.Hour { + if q.Timespan == Hour && timeDuration > 15*24*time.Hour { return nil, errors.New("time range is too large for hourly data. Max time range allowed: 15 days") } - if q.TimeInterval == Day { + if q.Timespan == Day { if timeDuration < 24*time.Hour { return nil, errors.New("time range is too small for daily data. Min time range allowed: 2 day") } @@ -188,7 +188,7 @@ func (s *Service) GetChainActivityTops(ctx *fasthttp.RequestCtx, q *ChainActivit } } - if q.TimeInterval == Month { + if q.Timespan == Month { if timeDuration < 30*24*time.Hour { return nil, errors.New("time range is too small for monthly data. Min time range allowed: 60 days") } @@ -198,7 +198,7 @@ func (s *Service) GetChainActivityTops(ctx *fasthttp.RequestCtx, q *ChainActivit } } - if q.TimeInterval == Year { + if q.Timespan == Year { if timeDuration < 365*24*time.Hour { return nil, errors.New("time range is too small for yearly data. Min time range allowed: 1 year") } diff --git a/api/middleware/extract_parameters.go b/api/middleware/extract_parameters.go index 1e5b5b50..0ba910a1 100644 --- a/api/middleware/extract_parameters.go +++ b/api/middleware/extract_parameters.go @@ -61,10 +61,6 @@ func ExtractToChain(c *fiber.Ctx, l *zap.Logger) (*sdk.ChainID, error) { return &result, nil } -func ExtractChain(c *fiber.Ctx, l *zap.Logger) (*sdk.ChainID, error) { - return extractChainQueryParam(c, l, "chain") -} - func ExtractSourceChain(c *fiber.Ctx, l *zap.Logger) (*sdk.ChainID, error) { return extractChainQueryParam(c, l, "sourceChain") } @@ -74,12 +70,10 @@ func ExtractTargetChain(c *fiber.Ctx, l *zap.Logger) (*sdk.ChainID, error) { } func extractChainQueryParam(c *fiber.Ctx, l *zap.Logger, queryParam string) (*sdk.ChainID, error) { - param := c.Query(queryParam) if param == "" { return nil, nil } - chain, err := strconv.ParseInt(param, 10, 16) if err != nil { requestID := fmt.Sprintf("%v", c.Locals("requestid")) @@ -90,7 +84,6 @@ func extractChainQueryParam(c *fiber.Ctx, l *zap.Logger, queryParam string) (*sd return nil, response.NewInvalidParamError(c, "INVALID CHAIN VALUE", errors.WithStack(err)) } - result := sdk.ChainID(chain) return &result, nil } @@ -358,14 +351,13 @@ func ExtractTimeSpanAndSampleRate(c *fiber.Ctx, l *zap.Logger) (string, string, return timeSpan, sampleRate, nil } -func ExtractTime(c *fiber.Ctx, queryParam string) (*time.Time, error) { +func ExtractTime(c *fiber.Ctx, timeLayout, queryParam string) (*time.Time, error) { // get the start_time from query params date := c.Query(queryParam, "") if date == "" { return nil, nil } - - t, err := time.Parse("20060102T150405Z", date) + t, err := time.Parse(timeLayout, date) if err != nil { return nil, response.NewInvalidQueryParamError(c, fmt.Sprintf("INVALID <%s> QUERY PARAMETER", queryParam), nil) } diff --git a/api/routes/wormscan/routes.go b/api/routes/wormscan/routes.go index a9aaf6fc..382ff098 100644 --- a/api/routes/wormscan/routes.go +++ b/api/routes/wormscan/routes.go @@ -85,7 +85,7 @@ func RegisterRoutes( api.Get("/last-txs", transactionCtrl.GetLastTransactions) api.Get("/scorecards", transactionCtrl.GetScorecards) api.Get("/x-chain-activity", transactionCtrl.GetChainActivity) - api.Post("/x-chain-activity/search", transactionCtrl.GetChainActivityTops) + api.Get("/x-chain-activity/search", transactionCtrl.GetChainActivityTops) api.Get("/top-assets-by-volume", transactionCtrl.GetTopAssets) api.Get("/top-chain-pairs-by-num-transfers", transactionCtrl.GetTopChainPairs) api.Get("token/:chain/:token_address", transactionCtrl.GetTokenByChainAndAddress) diff --git a/api/routes/wormscan/transactions/controller.go b/api/routes/wormscan/transactions/controller.go index a347e152..3180eeb4 100644 --- a/api/routes/wormscan/transactions/controller.go +++ b/api/routes/wormscan/transactions/controller.go @@ -1,7 +1,6 @@ package transactions import ( - "encoding/json" "strconv" "time" @@ -188,22 +187,50 @@ func (c *Controller) GetTopAssets(ctx *fiber.Ctx) error { // @Description Search, for a specific period of time, the number of transactions and the volume. // @Tags wormholescan // @ID x-chain-activity-tops -// @Method Post -// @Param {object} body transactions.ChainActivityTopsQuery true "Specify the query filters" +// @Method Get +// @Param timespan query string true "Time span, supported values: 1d, 1mo and 1y" +// @Param from query string true "From date, supported format 2006-01-02T15:04:05Z07:00" +// @Param to query string true "To date, supported format 2006-01-02T15:04:05Z07:00" +// @Param appId query string false "Search by appId" +// @Param sourceChain query string false "Search by sourceChain" +// @Param targetChain query string false "Search by targetChain" // @Success 200 {object} transactions.ChainActivityTopResults // @Failure 400 // @Failure 500 -// @Router /api/v1/x-chain-activity [get] +// @Router /api/v1/x-chain-activity/search [get] func (c *Controller) GetChainActivityTops(ctx *fiber.Ctx) error { - payload := &transactions.ChainActivityTopsQuery{} - err := json.Unmarshal(ctx.Request().Body(), payload) + sourceChain, err := middleware.ExtractSourceChain(ctx, c.logger) if err != nil { - return response.NewInvalidParamError(ctx, "invalid request body", err) + return err + } + targetChain, err := middleware.ExtractTargetChain(ctx, c.logger) + if err != nil { + return err + } + from, err := middleware.ExtractTime(ctx, time.RFC3339, "from") + if err != nil { + return err + } + to, err := middleware.ExtractTime(ctx, time.RFC3339, "to") + if err != nil { + return err + } + if from == nil || to == nil { + return response.NewInvalidParamError(ctx, "missing from/to query params ", nil) } - if !payload.TimeInterval.IsValid() { - return response.NewInvalidParamError(ctx, "invalid time interval", nil) + payload := transactions.ChainActivityTopsQuery{ + SourceChain: sourceChain, + TargetChain: targetChain, + From: *from, + To: *to, + AppId: middleware.ExtractAppId(ctx, c.logger), + Timespan: transactions.Timespan(ctx.Query("timespan")), + } + + if !payload.Timespan.IsValid() { + return response.NewInvalidParamError(ctx, "invalid timespan", nil) } nowUTC := time.Now().UTC() From 12d4130a6ae9700a93a6fdd5c0c0f877ec1d7dff Mon Sep 17 00:00:00 2001 From: Mariano <9205080+marianososto@users.noreply.github.com> Date: Thu, 25 Apr 2024 09:49:55 -0300 Subject: [PATCH 23/26] update endpoint name --- api/docs/docs.go | 30 +++++++++++++++---- api/docs/swagger.json | 30 +++++++++++++++---- api/docs/swagger.yaml | 26 ++++++++++++++-- api/routes/wormscan/routes.go | 2 +- .../wormscan/transactions/controller.go | 4 +-- 5 files changed, 76 insertions(+), 16 deletions(-) diff --git a/api/docs/docs.go b/api/docs/docs.go index a49382fa..d8c0ff52 100644 --- a/api/docs/docs.go +++ b/api/docs/docs.go @@ -1760,7 +1760,7 @@ const docTemplate = `{ } } }, - "/api/v1/x-chain-activity/search": { + "/api/v1/x-chain-activity/tops": { "get": { "description": "Search, for a specific period of time, the number of transactions and the volume.", "tags": [ @@ -3681,7 +3681,6 @@ const docTemplate = `{ 14, 15, 16, - 17, 18, 19, 20, @@ -3689,24 +3688,35 @@ const docTemplate = `{ 22, 23, 24, + 25, 26, 28, 29, 30, 32, + 33, 34, 35, + 36, + 37, + 38, + 39, 3104, 4000, 4001, 4002, 4003, 4004, + 4005, + 4006, + 4007, + 4008, 10002, 10003, 10004, 10005, - 10006 + 10006, + 10007 ], "x-enum-varnames": [ "ChainIDUnset", @@ -3726,7 +3736,6 @@ const docTemplate = `{ "ChainIDCelo", "ChainIDNear", "ChainIDMoonbeam", - "ChainIDNeon", "ChainIDTerra2", "ChainIDInjective", "ChainIDOsmosis", @@ -3734,24 +3743,35 @@ const docTemplate = `{ "ChainIDAptos", "ChainIDArbitrum", "ChainIDOptimism", + "ChainIDGnosis", "ChainIDPythNet", "ChainIDXpla", "ChainIDBtc", "ChainIDBase", "ChainIDSei", + "ChainIDRootstock", "ChainIDScroll", "ChainIDMantle", + "ChainIDBlast", + "ChainIDXLayer", + "ChainIDLinea", + "ChainIDBerachain", "ChainIDWormchain", "ChainIDCosmoshub", "ChainIDEvmos", "ChainIDKujira", "ChainIDNeutron", "ChainIDCelestia", + "ChainIDStargaze", + "ChainIDSeda", + "ChainIDDymension", + "ChainIDProvenance", "ChainIDSepolia", "ChainIDArbitrumSepolia", "ChainIDBaseSepolia", "ChainIDOptimismSepolia", - "ChainIDHolesky" + "ChainIDHolesky", + "ChainIDPolygonSepolia" ] }, "vaa.VaaDoc": { diff --git a/api/docs/swagger.json b/api/docs/swagger.json index 44697db7..e6882765 100644 --- a/api/docs/swagger.json +++ b/api/docs/swagger.json @@ -1753,7 +1753,7 @@ } } }, - "/api/v1/x-chain-activity/search": { + "/api/v1/x-chain-activity/tops": { "get": { "description": "Search, for a specific period of time, the number of transactions and the volume.", "tags": [ @@ -3674,7 +3674,6 @@ 14, 15, 16, - 17, 18, 19, 20, @@ -3682,24 +3681,35 @@ 22, 23, 24, + 25, 26, 28, 29, 30, 32, + 33, 34, 35, + 36, + 37, + 38, + 39, 3104, 4000, 4001, 4002, 4003, 4004, + 4005, + 4006, + 4007, + 4008, 10002, 10003, 10004, 10005, - 10006 + 10006, + 10007 ], "x-enum-varnames": [ "ChainIDUnset", @@ -3719,7 +3729,6 @@ "ChainIDCelo", "ChainIDNear", "ChainIDMoonbeam", - "ChainIDNeon", "ChainIDTerra2", "ChainIDInjective", "ChainIDOsmosis", @@ -3727,24 +3736,35 @@ "ChainIDAptos", "ChainIDArbitrum", "ChainIDOptimism", + "ChainIDGnosis", "ChainIDPythNet", "ChainIDXpla", "ChainIDBtc", "ChainIDBase", "ChainIDSei", + "ChainIDRootstock", "ChainIDScroll", "ChainIDMantle", + "ChainIDBlast", + "ChainIDXLayer", + "ChainIDLinea", + "ChainIDBerachain", "ChainIDWormchain", "ChainIDCosmoshub", "ChainIDEvmos", "ChainIDKujira", "ChainIDNeutron", "ChainIDCelestia", + "ChainIDStargaze", + "ChainIDSeda", + "ChainIDDymension", + "ChainIDProvenance", "ChainIDSepolia", "ChainIDArbitrumSepolia", "ChainIDBaseSepolia", "ChainIDOptimismSepolia", - "ChainIDHolesky" + "ChainIDHolesky", + "ChainIDPolygonSepolia" ] }, "vaa.VaaDoc": { diff --git a/api/docs/swagger.yaml b/api/docs/swagger.yaml index 92db5b78..ae8cece6 100644 --- a/api/docs/swagger.yaml +++ b/api/docs/swagger.yaml @@ -1027,7 +1027,6 @@ definitions: - 14 - 15 - 16 - - 17 - 18 - 19 - 20 @@ -1035,24 +1034,35 @@ definitions: - 22 - 23 - 24 + - 25 - 26 - 28 - 29 - 30 - 32 + - 33 - 34 - 35 + - 36 + - 37 + - 38 + - 39 - 3104 - 4000 - 4001 - 4002 - 4003 - 4004 + - 4005 + - 4006 + - 4007 + - 4008 - 10002 - 10003 - 10004 - 10005 - 10006 + - 10007 type: integer x-enum-varnames: - ChainIDUnset @@ -1072,7 +1082,6 @@ definitions: - ChainIDCelo - ChainIDNear - ChainIDMoonbeam - - ChainIDNeon - ChainIDTerra2 - ChainIDInjective - ChainIDOsmosis @@ -1080,24 +1089,35 @@ definitions: - ChainIDAptos - ChainIDArbitrum - ChainIDOptimism + - ChainIDGnosis - ChainIDPythNet - ChainIDXpla - ChainIDBtc - ChainIDBase - ChainIDSei + - ChainIDRootstock - ChainIDScroll - ChainIDMantle + - ChainIDBlast + - ChainIDXLayer + - ChainIDLinea + - ChainIDBerachain - ChainIDWormchain - ChainIDCosmoshub - ChainIDEvmos - ChainIDKujira - ChainIDNeutron - ChainIDCelestia + - ChainIDStargaze + - ChainIDSeda + - ChainIDDymension + - ChainIDProvenance - ChainIDSepolia - ChainIDArbitrumSepolia - ChainIDBaseSepolia - ChainIDOptimismSepolia - ChainIDHolesky + - ChainIDPolygonSepolia vaa.VaaDoc: properties: appId: @@ -2341,7 +2361,7 @@ paths: description: Internal Server Error tags: - wormholescan - /api/v1/x-chain-activity/search: + /api/v1/x-chain-activity/tops: get: description: Search, for a specific period of time, the number of transactions and the volume. diff --git a/api/routes/wormscan/routes.go b/api/routes/wormscan/routes.go index 382ff098..00c5a77c 100644 --- a/api/routes/wormscan/routes.go +++ b/api/routes/wormscan/routes.go @@ -85,7 +85,7 @@ func RegisterRoutes( api.Get("/last-txs", transactionCtrl.GetLastTransactions) api.Get("/scorecards", transactionCtrl.GetScorecards) api.Get("/x-chain-activity", transactionCtrl.GetChainActivity) - api.Get("/x-chain-activity/search", transactionCtrl.GetChainActivityTops) + api.Get("/x-chain-activity/tops", transactionCtrl.GetChainActivityTops) api.Get("/top-assets-by-volume", transactionCtrl.GetTopAssets) api.Get("/top-chain-pairs-by-num-transfers", transactionCtrl.GetTopChainPairs) api.Get("token/:chain/:token_address", transactionCtrl.GetTokenByChainAndAddress) diff --git a/api/routes/wormscan/transactions/controller.go b/api/routes/wormscan/transactions/controller.go index 3180eeb4..22b7a51f 100644 --- a/api/routes/wormscan/transactions/controller.go +++ b/api/routes/wormscan/transactions/controller.go @@ -184,7 +184,7 @@ func (c *Controller) GetTopAssets(ctx *fiber.Ctx) error { } // GetChainActivityTops godoc -// @Description Search, for a specific period of time, the number of transactions and the volume. +// @Description Search for a specific period of time the number of transactions and the volume. // @Tags wormholescan // @ID x-chain-activity-tops // @Method Get @@ -197,7 +197,7 @@ func (c *Controller) GetTopAssets(ctx *fiber.Ctx) error { // @Success 200 {object} transactions.ChainActivityTopResults // @Failure 400 // @Failure 500 -// @Router /api/v1/x-chain-activity/search [get] +// @Router /api/v1/x-chain-activity/tops [get] func (c *Controller) GetChainActivityTops(ctx *fiber.Ctx) error { sourceChain, err := middleware.ExtractSourceChain(ctx, c.logger) From 1680e983bb111760a2cb50080d0c0ac47448f8f0 Mon Sep 17 00:00:00 2001 From: Mariano <9205080+marianososto@users.noreply.github.com> Date: Thu, 25 Apr 2024 10:26:48 -0300 Subject: [PATCH 24/26] fix indents --- analytics/scripts/chain_activity_emitter_1d.flux | 4 ++-- analytics/scripts/chain_activity_emitter_1h.flux | 4 ++-- analytics/scripts/chain_activity_emitter_dest_1d.flux | 4 ++-- analytics/scripts/chain_activity_emitter_dest_1h.flux | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/analytics/scripts/chain_activity_emitter_1d.flux b/analytics/scripts/chain_activity_emitter_1d.flux index 5ec356a6..630290f9 100644 --- a/analytics/scripts/chain_activity_emitter_1d.flux +++ b/analytics/scripts/chain_activity_emitter_1d.flux @@ -6,9 +6,9 @@ runTask = (start,stop,srcBucket,destBucket,destMeasurement) => { data = from(bucket: srcBucket) |> range(start: start,stop: stop) |> filter(fn: (r) => r._measurement == "vaa_volume_v2" and r.version == "v2") - |> filter(fn: (r) => r._field == "volume" and r._value > 0) + |> filter(fn: (r) => r._field == "volume" and r._value > 0) |> drop(columns:["destination_chain","app_id","token_chain","token_address","version","_measurement","_time"]) - |> rename(columns: {_start: "_time"}) + |> rename(columns: {_start: "_time"}) |> group(columns: ["emitter_chain","_time"]) vols = data diff --git a/analytics/scripts/chain_activity_emitter_1h.flux b/analytics/scripts/chain_activity_emitter_1h.flux index eccc9039..b61971a7 100644 --- a/analytics/scripts/chain_activity_emitter_1h.flux +++ b/analytics/scripts/chain_activity_emitter_1h.flux @@ -6,9 +6,9 @@ runTask = (start,stop,srcBucket,destBucket,destMeasurement) => { data = from(bucket: srcBucket) |> range(start: start,stop: stop) |> filter(fn: (r) => r._measurement == "vaa_volume_v2" and r.version == "v2") - |> filter(fn: (r) => r._field == "volume" and r._value > 0) + |> filter(fn: (r) => r._field == "volume" and r._value > 0) |> drop(columns:["destination_chain","app_id","token_chain","token_address","version","_measurement","_time"]) - |> rename(columns: {_start: "_time"}) + |> rename(columns: {_start: "_time"}) |> group(columns: ["emitter_chain","_time"]) vols = data diff --git a/analytics/scripts/chain_activity_emitter_dest_1d.flux b/analytics/scripts/chain_activity_emitter_dest_1d.flux index 3e21e1cd..3a153cb8 100644 --- a/analytics/scripts/chain_activity_emitter_dest_1d.flux +++ b/analytics/scripts/chain_activity_emitter_dest_1d.flux @@ -3,8 +3,8 @@ import "date" runTask = (start,stop,srcBucket,destBucket,destMeasurement) => { data = from(bucket: srcBucket) |> range(start: start,stop: stop) - |> filter(fn: (r) => r._measurement == "vaa_volume_v2" and r._field == "volume") - |> group(columns: ["emitter_chain", "destination_chain", "app_id"]) + |> filter(fn: (r) => r._measurement == "vaa_volume_v2" and r._field == "volume") + |> group(columns: ["emitter_chain", "destination_chain", "app_id"]) data |> sum(column: "_value") diff --git a/analytics/scripts/chain_activity_emitter_dest_1h.flux b/analytics/scripts/chain_activity_emitter_dest_1h.flux index 744b6f35..256402b4 100644 --- a/analytics/scripts/chain_activity_emitter_dest_1h.flux +++ b/analytics/scripts/chain_activity_emitter_dest_1h.flux @@ -5,8 +5,8 @@ runTask = (start,stop,srcBucket,destBucket,destMeasurement) => { data = from(bucket: srcBucket) |> range(start: start,stop: stop) - |> filter(fn: (r) => r._measurement == "vaa_volume_v2" and r._field == "volume") - |> group(columns: ["emitter_chain", "destination_chain", "app_id"]) + |> filter(fn: (r) => r._measurement == "vaa_volume_v2" and r._field == "volume") + |> group(columns: ["emitter_chain", "destination_chain", "app_id"]) data |> sum(column: "_value") From 5e1a8a847ce8153a0f3a00db4cf393941cb988c8 Mon Sep 17 00:00:00 2001 From: Mariano <9205080+marianososto@users.noreply.github.com> Date: Fri, 26 Apr 2024 11:19:11 -0300 Subject: [PATCH 25/26] code review changes --- api/handlers/transactions/repository.go | 93 +++++++++++++++---------- 1 file changed, 58 insertions(+), 35 deletions(-) diff --git a/api/handlers/transactions/repository.go b/api/handlers/transactions/repository.go index e72ca847..07dc31a2 100644 --- a/api/handlers/transactions/repository.go +++ b/api/handlers/transactions/repository.go @@ -1078,16 +1078,21 @@ func (r *Repository) FindChainActivityTops(ctx *fasthttp.RequestCtx, q ChainActi func (r *Repository) buildChainActivityQueryTops(q ChainActivityTopsQuery) string { var start, stop string - if q.Timespan == Hour { + + switch q.Timespan { + case Hour: start = q.From.Truncate(1 * time.Hour).UTC().Format(time.RFC3339) stop = q.To.Truncate(1 * time.Hour).UTC().Format(time.RFC3339) - } else if q.Timespan == Day { + break + case Day: start = q.From.Truncate(24 * time.Hour).UTC().Format(time.RFC3339) stop = q.To.Truncate(24 * time.Hour).UTC().Format(time.RFC3339) - } else if q.Timespan == Month { + break + case Month: start = time.Date(q.From.Year(), q.From.Month(), 1, 0, 0, 0, 0, q.From.Location()).UTC().Format(time.RFC3339) stop = time.Date(q.To.Year(), q.To.Month(), 1, 0, 0, 0, 0, q.To.Location()).UTC().Format(time.RFC3339) - } else { + break + default: start = time.Date(q.From.Year(), 1, 1, 0, 0, 0, 0, q.From.Location()).UTC().Format(time.RFC3339) stop = time.Date(q.To.Year(), 1, 1, 0, 0, 0, 0, q.To.Location()).UTC().Format(time.RFC3339) } @@ -1108,17 +1113,35 @@ func (r *Repository) buildChainActivityQueryTops(q ChainActivityTopsQuery) strin } if q.TargetChain == nil && q.AppId == "" { + return r.buildQueryChainActivityTopsByEmitter(q, start, stop, filterSourceChain) + } - measurement := "" - switch q.Timespan { - case Hour: - measurement = "emitter_chain_activity_1h" - default: - measurement = "emitter_chain_activity_1d" - } + var query string + switch q.Timespan { + case Hour: + query = r.buildQueryChainActivityHourly(start, stop, filterSourceChain, filterTargetChain, filterAppId) + case Day: + query = r.buildQueryChainActivityDaily(start, stop, filterSourceChain, filterTargetChain, filterAppId) + case Month: + query = r.buildQueryChainActivityMonthly(start, stop, filterSourceChain, filterTargetChain, filterAppId) + default: + query = r.buildQueryChainActivityYearly(start, stop, filterSourceChain, filterTargetChain, filterAppId) + } + return query +} - if q.Timespan == Hour || q.Timespan == Day { - query := ` +func (r *Repository) buildQueryChainActivityTopsByEmitter(q ChainActivityTopsQuery, start, stop, filterSourceChain string) string { + + measurement := "" + switch q.Timespan { + case Hour: + measurement = "emitter_chain_activity_1h" + default: + measurement = "emitter_chain_activity_1d" + } + + if q.Timespan == Hour || q.Timespan == Day { + query := ` import "date" from(bucket: "%s") @@ -1128,11 +1151,11 @@ func (r *Repository) buildChainActivityQueryTops(q ChainActivityTopsQuery) strin |> pivot(rowKey:["_time","emitter_chain"], columnKey: ["_field"], valueColumn: "_value") |> sort(columns:["emitter_chain","_time"],desc:false) ` - return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, measurement, filterSourceChain) - } + return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, measurement, filterSourceChain) + } - if q.Timespan == Month { - query := ` + if q.Timespan == Month { + query := ` import "date" import "join" @@ -1167,10 +1190,10 @@ func (r *Repository) buildChainActivityQueryTops(q ChainActivityTopsQuery) strin |> group() |> sort(columns:["emitter_chain","_time"],desc:false) ` - return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, measurement, filterSourceChain) - } + return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, measurement, filterSourceChain) + } - query := ` + query := ` import "date" import "join" @@ -1204,12 +1227,12 @@ func (r *Repository) buildChainActivityQueryTops(q ChainActivityTopsQuery) strin |> group() |> sort(columns:["emitter_chain","_time"],desc:false) ` - return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, measurement, filterSourceChain) + return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, measurement, filterSourceChain) - } +} - if q.Timespan == Hour { - query := ` +func (r *Repository) buildQueryChainActivityHourly(start, stop, filterSourceChain, filterTargetChain, filterAppId string) string { + query := ` import "date" import "join" @@ -1242,12 +1265,12 @@ func (r *Repository) buildChainActivityQueryTops(q ChainActivityTopsQuery) strin |> group() |> sort(columns:["emitter_chain","_time"],desc:false) ` - return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, filterSourceChain, filterTargetChain, filterAppId) - } + return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, filterSourceChain, filterTargetChain, filterAppId) +} - if q.Timespan == Day { +func (r *Repository) buildQueryChainActivityDaily(start, stop, filterSourceChain, filterTargetChain, filterAppId string) string { - query := ` + query := ` import "date" import "join" @@ -1280,11 +1303,11 @@ func (r *Repository) buildChainActivityQueryTops(q ChainActivityTopsQuery) strin |> group() |> sort(columns:["emitter_chain","_time"],desc:false) ` - return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, filterSourceChain, filterTargetChain, filterAppId) - } + return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, filterSourceChain, filterTargetChain, filterAppId) +} - if q.Timespan == Month { - query := ` +func (r *Repository) buildQueryChainActivityMonthly(start, stop, filterSourceChain, filterTargetChain, filterAppId string) string { + query := ` import "date" import "join" @@ -1321,9 +1344,10 @@ func (r *Repository) buildChainActivityQueryTops(q ChainActivityTopsQuery) strin |> group() |> sort(columns:["emitter_chain","_time"],desc:false) ` - return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, filterSourceChain, filterTargetChain, filterAppId) - } + return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, filterSourceChain, filterTargetChain, filterAppId) +} +func (r *Repository) buildQueryChainActivityYearly(start, stop, filterSourceChain, filterTargetChain, filterAppId string) string { query := ` import "date" import "join" @@ -1362,5 +1386,4 @@ func (r *Repository) buildChainActivityQueryTops(q ChainActivityTopsQuery) strin |> sort(columns:["emitter_chain","_time"],desc:false) ` return fmt.Sprintf(query, r.bucketInfiniteRetention, start, stop, filterSourceChain, filterTargetChain, filterAppId) - } From 5d27910bca26a8bd60fff53352736a423c5bebf4 Mon Sep 17 00:00:00 2001 From: Mariano <9205080+marianososto@users.noreply.github.com> Date: Fri, 26 Apr 2024 11:25:53 -0300 Subject: [PATCH 26/26] remove unnecessary break --- api/handlers/transactions/repository.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/api/handlers/transactions/repository.go b/api/handlers/transactions/repository.go index 07dc31a2..fffb7201 100644 --- a/api/handlers/transactions/repository.go +++ b/api/handlers/transactions/repository.go @@ -1083,15 +1083,12 @@ func (r *Repository) buildChainActivityQueryTops(q ChainActivityTopsQuery) strin case Hour: start = q.From.Truncate(1 * time.Hour).UTC().Format(time.RFC3339) stop = q.To.Truncate(1 * time.Hour).UTC().Format(time.RFC3339) - break case Day: start = q.From.Truncate(24 * time.Hour).UTC().Format(time.RFC3339) stop = q.To.Truncate(24 * time.Hour).UTC().Format(time.RFC3339) - break case Month: start = time.Date(q.From.Year(), q.From.Month(), 1, 0, 0, 0, 0, q.From.Location()).UTC().Format(time.RFC3339) stop = time.Date(q.To.Year(), q.To.Month(), 1, 0, 0, 0, 0, q.To.Location()).UTC().Format(time.RFC3339) - break default: start = time.Date(q.From.Year(), 1, 1, 0, 0, 0, 0, q.From.Location()).UTC().Format(time.RFC3339) stop = time.Date(q.To.Year(), 1, 1, 0, 0, 0, 0, q.To.Location()).UTC().Format(time.RFC3339)