BigTable: query optimizations and return payloads
Change-Id: If3a3c9a207518a26fbb8d924b5b9a9053c62c3a7 commit-id:00c2b83a
This commit is contained in:
parent
ba5d55c1b9
commit
c4bced0e52
|
@ -45,6 +45,9 @@ var TokenTransferEmitters = map[string]string{
|
|||
"000000000000000000000000784999135aaa8a3ca5914468852fdddbddd8789d": "terra10pyejy66429refv3g35g2t7am0was7ya7kz2a4", // terra
|
||||
}
|
||||
|
||||
// this address is an emitter for BSC and Polygon.
|
||||
var sharedEmitterAddress = "0000000000000000000000005a58505a96d1dbf8df91cb21b54419fc36e93fde"
|
||||
|
||||
type (
|
||||
TokenTransfer struct {
|
||||
PayloadId uint8
|
||||
|
@ -204,6 +207,15 @@ func writePayloadToBigTable(ctx context.Context, rowKey string, colFam string, m
|
|||
}
|
||||
return nil
|
||||
}
|
||||
func TrimUnicodeFromByteArray(b []byte) []byte {
|
||||
// Escaped Unicode that has been observed in payload's token names and symbol:
|
||||
null := "\u0000"
|
||||
start := "\u0002"
|
||||
ack := "\u0006"
|
||||
tab := "\u0009"
|
||||
control := "\u0012"
|
||||
return bytes.Trim(b, null+start+ack+tab+control)
|
||||
}
|
||||
|
||||
// ProcessVAA is triggered by a PubSub message, emitted after row is saved to BigTable by guardiand
|
||||
func ProcessVAA(ctx context.Context, m PubSubMessage) error {
|
||||
|
@ -223,7 +235,11 @@ func ProcessVAA(ctx context.Context, m PubSubMessage) error {
|
|||
emitterHex := signedVaa.EmitterAddress.String()
|
||||
payloadId := int(signedVaa.Payload[0])
|
||||
|
||||
if _, ok := TokenTransferEmitters[emitterHex]; ok {
|
||||
// BSC and Polygon have the same contract address: "0x5a58505a96d1dbf8df91cb21b54419fc36e93fde".
|
||||
// The BSC contract is the NFT emitter address.
|
||||
// The Polygon contract is the token transfer emitter address.
|
||||
// Due to that, ensure that the block below only runs for token transfers by checking for chain == 4 and emitter addaress.
|
||||
if _, ok := TokenTransferEmitters[emitterHex]; ok && !(signedVaa.EmitterChain == 4 && signedVaa.EmitterAddress.String() == sharedEmitterAddress) {
|
||||
// figure out if it's a transfer or asset metadata
|
||||
|
||||
if payloadId == 1 {
|
||||
|
@ -239,7 +255,12 @@ func ProcessVAA(ctx context.Context, m PubSubMessage) error {
|
|||
ts := bigtable.Now()
|
||||
mutation.Set(colFam, "PayloadId", ts, []byte(fmt.Sprint(payload.PayloadId)))
|
||||
// TODO: find a better way of representing amount as a string
|
||||
mutation.Set(colFam, "Amount", ts, []byte(fmt.Sprint(payload.Amount[3])))
|
||||
amount := []byte(fmt.Sprint(payload.Amount[3]))
|
||||
if payload.Amount[2] != 0 {
|
||||
log.Printf("payload.Amount is larger than uint64 for row %v", rowKey)
|
||||
amount = payload.Amount.Bytes()
|
||||
}
|
||||
mutation.Set(colFam, "Amount", ts, amount)
|
||||
mutation.Set(colFam, "OriginAddress", ts, []byte(hex.EncodeToString(payload.OriginAddress[:])))
|
||||
mutation.Set(colFam, "OriginChain", ts, []byte(fmt.Sprint(payload.OriginChain)))
|
||||
mutation.Set(colFam, "TargetAddress", ts, []byte(hex.EncodeToString(payload.TargetAddress[:])))
|
||||
|
@ -270,8 +291,8 @@ func ProcessVAA(ctx context.Context, m PubSubMessage) error {
|
|||
mutation.Set(colFam, "TokenAddress", ts, []byte(hex.EncodeToString(payload.TokenAddress[:])))
|
||||
mutation.Set(colFam, "TokenChain", ts, []byte(fmt.Sprint(payload.TokenChain)))
|
||||
mutation.Set(colFam, "Decimals", ts, []byte(fmt.Sprint(payload.Decimals)))
|
||||
mutation.Set(colFam, "Name", ts, []byte(payload.Name[:]))
|
||||
mutation.Set(colFam, "Symbol", ts, []byte(payload.Symbol[:]))
|
||||
mutation.Set(colFam, "Name", ts, TrimUnicodeFromByteArray(payload.Name[:]))
|
||||
mutation.Set(colFam, "Symbol", ts, TrimUnicodeFromByteArray(payload.Symbol[:]))
|
||||
writeErr := writePayloadToBigTable(ctx, rowKey, colFam, mutation)
|
||||
if writeErr != nil {
|
||||
log.Println("wrote TokenTransferPayload to bigtable!", rowKey)
|
||||
|
@ -298,11 +319,10 @@ func ProcessVAA(ctx context.Context, m PubSubMessage) error {
|
|||
mutation.Set(colFam, "PayloadId", ts, []byte(fmt.Sprint(payload.PayloadId)))
|
||||
mutation.Set(colFam, "OriginAddress", ts, []byte(hex.EncodeToString(payload.OriginAddress[:])))
|
||||
mutation.Set(colFam, "OriginChain", ts, []byte(fmt.Sprint(payload.OriginChain)))
|
||||
mutation.Set(colFam, "Symbol", ts, []byte(payload.Symbol[:]))
|
||||
mutation.Set(colFam, "Name", ts, []byte(payload.Name[:]))
|
||||
// TODO: find a better way of representing tokenId as a string
|
||||
mutation.Set(colFam, "TokenId", ts, []byte(fmt.Sprint(payload.TokenId[3])))
|
||||
mutation.Set(colFam, "URI", ts, []byte(payload.URI))
|
||||
mutation.Set(colFam, "Symbol", ts, TrimUnicodeFromByteArray(payload.Symbol[:]))
|
||||
mutation.Set(colFam, "Name", ts, TrimUnicodeFromByteArray(payload.Name[:]))
|
||||
mutation.Set(colFam, "TokenId", ts, payload.TokenId.Bytes())
|
||||
mutation.Set(colFam, "URI", ts, TrimUnicodeFromByteArray(payload.URI))
|
||||
mutation.Set(colFam, "TargetAddress", ts, []byte(hex.EncodeToString(payload.TargetAddress[:])))
|
||||
mutation.Set(colFam, "TargetChain", ts, []byte(fmt.Sprint(payload.TargetChain)))
|
||||
|
||||
|
|
|
@ -9,6 +9,8 @@ import (
|
|||
"log"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"cloud.google.com/go/bigtable"
|
||||
)
|
||||
|
||||
// fetch a single row by the row key
|
||||
|
@ -105,7 +107,7 @@ func ReadRow(w http.ResponseWriter, r *http.Request) {
|
|||
}
|
||||
rowKey = emitterChain + ":" + emitterAddress + ":" + sequence
|
||||
|
||||
row, err := tbl.ReadRow(r.Context(), rowKey)
|
||||
row, err := tbl.ReadRow(r.Context(), rowKey, bigtable.RowFilter(bigtable.LatestNFilter(1)))
|
||||
if err != nil {
|
||||
http.Error(w, "Error reading rows", http.StatusInternalServerError)
|
||||
log.Printf("tbl.ReadRows(): %v", err)
|
||||
|
|
|
@ -73,8 +73,16 @@ var columnFamilies = []string{
|
|||
"TokenTransferDetails",
|
||||
"ChainDetails",
|
||||
}
|
||||
var messagePubFam = columnFamilies[0]
|
||||
var quorumStateFam = columnFamilies[1]
|
||||
var transferPayloadFam = columnFamilies[2]
|
||||
var metaPayloadFam = columnFamilies[3]
|
||||
var nftPayloadFam = columnFamilies[4]
|
||||
var transferDetailsFam = columnFamilies[5]
|
||||
var chainDetailsFam = columnFamilies[6]
|
||||
|
||||
type (
|
||||
// Summary is MessagePublication data & QuorumState data
|
||||
Summary struct {
|
||||
EmitterChain string
|
||||
EmitterAddress string
|
||||
|
@ -84,16 +92,57 @@ type (
|
|||
SignedVAABytes []byte
|
||||
QuorumTime string
|
||||
}
|
||||
// Details is a Summary, with the VAA decoded as SignedVAA
|
||||
// Details is a Summary extended with all the post-processing ColumnFamilies
|
||||
Details struct {
|
||||
SignedVAA *vaa.VAA
|
||||
EmitterChain string
|
||||
EmitterAddress string
|
||||
Sequence string
|
||||
InitiatingTxID string
|
||||
Payload []byte
|
||||
SignedVAABytes []byte
|
||||
QuorumTime string
|
||||
Summary
|
||||
SignedVAA *vaa.VAA
|
||||
TokenTransferPayload *TokenTransferPayload
|
||||
AssetMetaPayload *AssetMetaPayload
|
||||
NFTTransferPayload *NFTTransferPayload
|
||||
TransferDetails *TransferDetails
|
||||
ChainDetails *ChainDetails
|
||||
}
|
||||
// The following structs match the ColumnFamiles they are named after
|
||||
TokenTransferPayload struct {
|
||||
Amount string
|
||||
OriginAddress string
|
||||
OriginChain string
|
||||
TargetAddress string
|
||||
TargetChain string
|
||||
}
|
||||
AssetMetaPayload struct {
|
||||
TokenAddress string
|
||||
TokenChain string
|
||||
Decimals string
|
||||
Symbol string
|
||||
Name string
|
||||
}
|
||||
NFTTransferPayload struct {
|
||||
OriginAddress string
|
||||
OriginChain string
|
||||
Symbol string
|
||||
Name string
|
||||
TokenId string
|
||||
URI string
|
||||
TargetAddress string
|
||||
TargetChain string
|
||||
}
|
||||
TransferDetails struct {
|
||||
Amount string
|
||||
Decimals string
|
||||
NotionalUSDStr string
|
||||
TokenPriceUSDStr string
|
||||
TransferTimestamp string
|
||||
OriginSymbol string
|
||||
OriginName string
|
||||
OriginTokenAddress string
|
||||
// fields below exist on the row, but no need to return them currently.
|
||||
// NotionalUSD uint64
|
||||
// TokenPriceUSD uint64
|
||||
}
|
||||
ChainDetails struct {
|
||||
SenderAddress string
|
||||
ReceiverAddress string
|
||||
}
|
||||
)
|
||||
|
||||
|
@ -115,9 +164,9 @@ func chainIdStringToType(chainId string) vaa.ChainID {
|
|||
|
||||
func makeSummary(row bigtable.Row) *Summary {
|
||||
summary := &Summary{}
|
||||
if _, ok := row[columnFamilies[0]]; ok {
|
||||
if _, ok := row[messagePubFam]; ok {
|
||||
|
||||
for _, item := range row[columnFamilies[0]] {
|
||||
for _, item := range row[messagePubFam] {
|
||||
switch item.Column {
|
||||
case "MessagePublication:InitiatingTxID":
|
||||
summary.InitiatingTxID = string(item.Value)
|
||||
|
@ -144,8 +193,8 @@ func makeSummary(row bigtable.Row) *Summary {
|
|||
}
|
||||
summary.Sequence = seq
|
||||
}
|
||||
if _, ok := row[columnFamilies[1]]; ok {
|
||||
item := row[columnFamilies[1]][0]
|
||||
if _, ok := row[quorumStateFam]; ok {
|
||||
item := row[quorumStateFam][0]
|
||||
summary.SignedVAABytes = item.Value
|
||||
summary.QuorumTime = item.Timestamp.Time().String()
|
||||
}
|
||||
|
@ -153,8 +202,9 @@ func makeSummary(row bigtable.Row) *Summary {
|
|||
}
|
||||
|
||||
func makeDetails(row bigtable.Row) *Details {
|
||||
deets := &Details{}
|
||||
sum := makeSummary(row)
|
||||
deets := &Details{
|
||||
deets.Summary = Summary{
|
||||
EmitterChain: sum.EmitterChain,
|
||||
EmitterAddress: sum.EmitterAddress,
|
||||
Sequence: sum.Sequence,
|
||||
|
@ -163,10 +213,127 @@ func makeDetails(row bigtable.Row) *Details {
|
|||
SignedVAABytes: sum.SignedVAABytes,
|
||||
QuorumTime: sum.QuorumTime,
|
||||
}
|
||||
if _, ok := row[columnFamilies[1]]; ok {
|
||||
item := row[columnFamilies[1]][0]
|
||||
|
||||
if _, ok := row[quorumStateFam]; ok {
|
||||
item := row[quorumStateFam][0]
|
||||
deets.SignedVAA, _ = vaa.Unmarshal(item.Value)
|
||||
}
|
||||
if _, ok := row[transferPayloadFam]; ok {
|
||||
tokenTransferPayload := &TokenTransferPayload{}
|
||||
for _, item := range row[transferPayloadFam] {
|
||||
switch item.Column {
|
||||
case "TokenTransferPayload:Amount":
|
||||
tokenTransferPayload.Amount = string(item.Value)
|
||||
case "TokenTransferPayload:OriginAddress":
|
||||
tokenTransferPayload.OriginAddress = string(item.Value)
|
||||
case "TokenTransferPayload:OriginChain":
|
||||
tokenTransferPayload.OriginChain = string(item.Value)
|
||||
case "TokenTransferPayload:TargetAddress":
|
||||
tokenTransferPayload.TargetAddress = string(item.Value)
|
||||
case "TokenTransferPayload:TargetChain":
|
||||
tokenTransferPayload.TargetChain = string(item.Value)
|
||||
}
|
||||
}
|
||||
deets.TokenTransferPayload = tokenTransferPayload
|
||||
}
|
||||
if _, ok := row[metaPayloadFam]; ok {
|
||||
assetMetaPayload := &AssetMetaPayload{}
|
||||
for _, item := range row[metaPayloadFam] {
|
||||
switch item.Column {
|
||||
case "AssetMetaPayload:TokenAddress":
|
||||
assetMetaPayload.TokenAddress = string(item.Value)
|
||||
case "AssetMetaPayload:TokenChain":
|
||||
assetMetaPayload.TokenChain = string(item.Value)
|
||||
case "AssetMetaPayload:Decimals":
|
||||
assetMetaPayload.Decimals = string(item.Value)
|
||||
case "AssetMetaPayload:Symbol":
|
||||
assetMetaPayload.Symbol = string(item.Value)
|
||||
case "AssetMetaPayload:Name":
|
||||
assetMetaPayload.Name = string(item.Value)
|
||||
}
|
||||
}
|
||||
deets.AssetMetaPayload = assetMetaPayload
|
||||
}
|
||||
if _, ok := row[nftPayloadFam]; ok {
|
||||
nftTransferPayload := &NFTTransferPayload{}
|
||||
for _, item := range row[nftPayloadFam] {
|
||||
switch item.Column {
|
||||
case "NFTTransferPayload:OriginAddress":
|
||||
nftTransferPayload.OriginAddress = string(item.Value)
|
||||
case "NFTTransferPayload:OriginChain":
|
||||
nftTransferPayload.OriginChain = string(item.Value)
|
||||
case "NFTTransferPayload:Symbol":
|
||||
nftTransferPayload.Symbol = string(item.Value)
|
||||
case "NFTTransferPayload:Name":
|
||||
nftTransferPayload.Name = string(item.Value)
|
||||
case "NFTTransferPayload:TokenId":
|
||||
nftTransferPayload.TokenId = string(item.Value)
|
||||
case "NFTTransferPayload:URI":
|
||||
nftTransferPayload.URI = string(TrimUnicodeFromByteArray(item.Value))
|
||||
case "NFTTransferPayload:TargetAddress":
|
||||
nftTransferPayload.TargetAddress = string(item.Value)
|
||||
case "NFTTransferPayload:TargetChain":
|
||||
nftTransferPayload.TargetChain = string(item.Value)
|
||||
}
|
||||
}
|
||||
deets.NFTTransferPayload = nftTransferPayload
|
||||
}
|
||||
if _, ok := row[transferDetailsFam]; ok {
|
||||
transferDetails := &TransferDetails{}
|
||||
for _, item := range row[transferDetailsFam] {
|
||||
switch item.Column {
|
||||
case "TokenTransferDetails:Amount":
|
||||
transferDetails.Amount = string(item.Value)
|
||||
case "TokenTransferDetails:Decimals":
|
||||
transferDetails.Decimals = string(item.Value)
|
||||
case "TokenTransferDetails:NotionalUSDStr":
|
||||
transferDetails.NotionalUSDStr = string(item.Value)
|
||||
case "TokenTransferDetails:TokenPriceUSDStr":
|
||||
transferDetails.TokenPriceUSDStr = string(item.Value)
|
||||
case "TokenTransferDetails:TransferTimestamp":
|
||||
transferDetails.TransferTimestamp = string(item.Value)
|
||||
case "TokenTransferDetails:OriginSymbol":
|
||||
transferDetails.OriginSymbol = string(item.Value)
|
||||
case "TokenTransferDetails:OriginName":
|
||||
transferDetails.OriginName = string(item.Value)
|
||||
case "TokenTransferDetails:OriginTokenAddress":
|
||||
transferDetails.OriginTokenAddress = string(item.Value)
|
||||
|
||||
// NotionalUSD and TokenPriceUSD are more percise than the string versions returned,
|
||||
// however the precision is not required, so leaving this commented out for now.
|
||||
|
||||
// case "TokenTransferDetails:NotionalUSD":
|
||||
// reader := bytes.NewReader(item.Value)
|
||||
// var notionalUSD uint64
|
||||
// if err := binary.Read(reader, binary.BigEndian, ¬ionalUSD); err != nil {
|
||||
// log.Fatalf("failed to read NotionalUSD of row: %v. err %v ", row.Key(), err)
|
||||
// }
|
||||
// transferDetails.NotionalUSD = notionalUSD
|
||||
|
||||
// case "TokenTransferDetails:TokenPriceUSD":
|
||||
// reader := bytes.NewReader(item.Value)
|
||||
// var tokenPriceUSD uint64
|
||||
// if err := binary.Read(reader, binary.BigEndian, &tokenPriceUSD); err != nil {
|
||||
// log.Fatalf("failed to read TokenPriceUSD of row: %v. err %v", row.Key(), err)
|
||||
// }
|
||||
// transferDetails.NotionalUSD = tokenPriceUSD
|
||||
|
||||
}
|
||||
}
|
||||
deets.TransferDetails = transferDetails
|
||||
}
|
||||
if _, ok := row[chainDetailsFam]; ok {
|
||||
chainDetails := &ChainDetails{}
|
||||
for _, item := range row[chainDetailsFam] {
|
||||
switch item.Column {
|
||||
case "ChainDetails:SenderAddress":
|
||||
chainDetails.SenderAddress = string(item.Value)
|
||||
case "ChainDetails:ReceiverAddress":
|
||||
chainDetails.ReceiverAddress = string(item.Value)
|
||||
}
|
||||
}
|
||||
deets.ChainDetails = chainDetails
|
||||
}
|
||||
return deets
|
||||
}
|
||||
|
||||
|
|
|
@ -54,6 +54,7 @@ func fetchRowsInInterval(tbl *bigtable.Table, ctx context.Context, prefix string
|
|||
}, bigtable.RowFilter(
|
||||
bigtable.ChainFilters(
|
||||
// combine filters to get only what we need:
|
||||
bigtable.FamilyFilter(columnFamilies[1]),
|
||||
bigtable.CellsPerRowLimitFilter(1), // only the first cell in each column (helps for devnet where sequence resets)
|
||||
bigtable.TimestampRangeFilter(start, end), // within time range
|
||||
bigtable.StripValueFilter(), // no columns/values, just the row.Key()
|
||||
|
|
|
@ -92,7 +92,7 @@ func Transaction(w http.ResponseWriter, r *http.Request) {
|
|||
}
|
||||
|
||||
key := result.Key()
|
||||
row, err := tbl.ReadRow(r.Context(), key)
|
||||
row, err := tbl.ReadRow(r.Context(), key, bigtable.RowFilter(bigtable.LatestNFilter(1)))
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
w.Write([]byte(err.Error()))
|
||||
|
|
Loading…
Reference in New Issue