From f23afd88c0215e921cb2b0f76b9c2b1c4ed834fe Mon Sep 17 00:00:00 2001 From: justinschuldt Date: Tue, 9 Nov 2021 14:58:13 -0600 Subject: [PATCH] BigTable: column families for secondary data Change-Id: I3d9514b21b7cd95ed16c65a2bd3c2e951ae36802 commit-id:5c9a7e93 --- devnet/bigtable.yaml | 2 +- event_database/cloud_functions/shared.go | 18 +++++--- event_database/database.go | 10 ++++- explorer_data/bigtable_event_schema.md | 53 ++++++++++++++++++++++++ 4 files changed, 76 insertions(+), 7 deletions(-) diff --git a/devnet/bigtable.yaml b/devnet/bigtable.yaml index b911b877..36ab15a6 100644 --- a/devnet/bigtable.yaml +++ b/devnet/bigtable.yaml @@ -45,7 +45,7 @@ spec: gcloud --quiet components install beta cbt bigtable gcloud --quiet beta emulators bigtable start --host-port=0.0.0.0:8086 & sleep 3 - cbt createtable v2Events "families=MessagePublication,Signatures,VAAState,QuorumState" + cbt createtable v2Events "families=MessagePublication,QuorumState,TokenTransferPayload,AssetMetaPayload,NFTTransferPayload,TokenTransferDetails,ChainDetails" nc -lkp 2000 0.0.0.0 readinessProbe: diff --git a/event_database/cloud_functions/shared.go b/event_database/cloud_functions/shared.go index eab9eeba..bb270370 100644 --- a/event_database/cloud_functions/shared.go +++ b/event_database/cloud_functions/shared.go @@ -40,7 +40,15 @@ func init() { tbl = client.Open("v2Events") } -var columnFamilies = []string{"MessagePublication", "Signatures", "VAAState", "QuorumState"} +var columnFamilies = []string{ + "MessagePublication", + "QuorumState", + "TokenTransferPayload", + "AssetMetaPayload", + "NFTTransferPayload", + "TokenTransferDetails", + "ChainDetails", +} type ( Summary struct { @@ -112,8 +120,8 @@ func makeSummary(row bigtable.Row) *Summary { } summary.Sequence = seq } - if _, ok := row[columnFamilies[3]]; ok { - item := row[columnFamilies[3]][0] + if _, ok := row[columnFamilies[1]]; ok { + item := row[columnFamilies[1]][0] summary.SignedVAABytes = item.Value summary.QuorumTime = item.Timestamp.Time().String() } @@ -131,8 +139,8 @@ func makeDetails(row bigtable.Row) *Details { SignedVAABytes: sum.SignedVAABytes, QuorumTime: sum.QuorumTime, } - if _, ok := row[columnFamilies[3]]; ok { - item := row[columnFamilies[3]][0] + if _, ok := row[columnFamilies[1]]; ok { + item := row[columnFamilies[1]][0] deets.SignedVAA, _ = vaa.Unmarshal(item.Value) } return deets diff --git a/event_database/database.go b/event_database/database.go index 6f45b738..7fd99696 100644 --- a/event_database/database.go +++ b/event_database/database.go @@ -11,7 +11,15 @@ import ( const tableName = "v2Events" // These column family names match the guardian code that does the inserting. -var columnFamilies = []string{"MessagePublication", "QuorumState"} +var columnFamilies = []string{ + "MessagePublication", + "QuorumState", + "TokenTransferPayload", + "AssetMetaPayload", + "NFTTransferPayload", + "TokenTransferDetails", + "ChainDetails" +} func main() { project := flag.String("project", "", "The Google Cloud Platform project ID. Required.") diff --git a/explorer_data/bigtable_event_schema.md b/explorer_data/bigtable_event_schema.md index 90bef5ff..99797bee 100644 --- a/explorer_data/bigtable_event_schema.md +++ b/explorer_data/bigtable_event_schema.md @@ -4,6 +4,10 @@ Row keys contain the MessageID, delimited by colons, like so: `EmitterChain:EmitterAddress:Sequence`. +- `EmitterAddress` left padded with `0`s to 32 bytes, then hex encoded. + +- `Sequence` left padded with `0`s to 16 characters, so rows are ordered in the sequence they occured. BigTable Rows are sorted lexicographically by row key. + BigTable can only be queried for data in the row key. Only row key data is indexed. You cannot query based on the value of a column; however you may filter based on column value. ### Column Families @@ -16,10 +20,21 @@ The column families listed below represent data unique to a phase of the attesta - `QuorumState` stores the signed VAA once quorum is reached. +- `TokenTransferPayload` stores the decoded payload of transfer messages. + +- `AssetMetaPayload` stores the decoded payload of asset metadata messages. + +- `NFTTransferPayload` stores the decoded payload of NFT transfer messages. + +- `TokenTransferDetails` stores information about the transfer. + +- `ChainDetails` stores chain-native data supplimented from external source(s). + ### Column Qualifiers Each column qualifier below is prefixed with its column family. +#### MessagePublication - `MessagePublication:Version` Version of the VAA schema. - `MessagePublication:GuardianSetIndex` The index of the active Guardian set. - `MessagePublication:Timestamp` Timestamp when the VAA was created by the Guardian. @@ -30,4 +45,42 @@ Each column qualifier below is prefixed with its column family. - `MessagePublication:InitiatingTxID` The transaction identifier of the user's interaction with the contract. - `MessagePublication:Payload` The payload of the user's message. +#### QuorumState - `QuorumState:SignedVAA` the VAA with the signatures that contributed to quorum. + +#### TokenTransferPayload +- `TokenTransferPayload:PayloadId` the payload identifier of the payload. +- `TokenTransferPayload:Amount` the amount of the transfer. +- `TokenTransferPayload:OriginAddress` the address the transfer originates from. +- `TokenTransferPayload:OriginChain` the chain identifier of the chain the transfer originates from. +- `TokenTransferPayload:TargetAdress` the destination address of the transfer. +- `TokenTransferPayload:TargetChain` the destination chain identifier of the transfer. + +#### AssetMetaPayload +- `AssetMetaPayload:PayloadId` the payload identifier of the payload. +- `AssetMetaPayload:TokenAddress` the address of the token. left padded with `0`s to 32 bytes. +- `AssetMetaPayload:TokenChain` the chain identifier of the chain the transfer originates from. +- `AssetMetaPayload:Decimals` the number of decimals of the token. +- `AssetMetaPayload:Symbol` the ticker symbol of the token. +- `AssetMetaPayload:Name` the name of the token. + +#### NFTTransferPayload +- `NFTTransferPayload:PayloadId` the payload identifier of the payload. +- `NFTTransferPayload:OriginAddress` the address the transfer originates from. +- `NFTTransferPayload:OriginChain` the chain identifier of the chain the transfer originates from. +- `NFTTransferPayload:Symbol` the symbol of the nft. +- `NFTTransferPayload:Name` the name of the nft. +- `NFTTransferPayload:TokenId` the token identifier of the nft. +- `NFTTransferPayload:URI` the URI of the nft. +- `NFTTransferPayload:TargetAddress` the destination address of the transfer. +- `NFTTransferPayload:TargetChain` the destination chain identifier of the transfer. + +#### TokenTransferDetails +- `TokenTransferDetails:Amount` the amount transfered. +- `TokenTransferDetails:OriginSymbol` the symbol of the token sent to wormhole. +- `TokenTransferDetails:OriginName` the name of the token sent to wormhole. +- `TokenTransferDetails:TargetSymbol` the symbol of the token disbursed by wormhole. +- `TokenTransferDetails:TargetName` the name of the token disbursed by wormhole. + +#### ChainDetails +- `ChainDetails:SenderAddress` the native address that sent the message.