diff --git a/cmd/radiance/blockstore/yaml/shreds.go b/cmd/radiance/blockstore/yaml/shreds.go new file mode 100644 index 0000000..39e3670 --- /dev/null +++ b/cmd/radiance/blockstore/yaml/shreds.go @@ -0,0 +1,64 @@ +package yaml + +import ( + "encoding/base64" + "encoding/json" + + "github.com/certusone/radiance/pkg/blockstore" + "github.com/certusone/radiance/pkg/shred" +) + +// entryBatch is a YAML-friendly version of blockstore.Entries. +type entryBatch struct { + Shreds []uint32 `yaml:"shreds,flow"` + EncodedSize int `yaml:"encoded_size,omitempty"` + Entries []entry `yaml:"entries"` +} + +func makeEntryBatch(b *blockstore.Entries, withTxs bool) entryBatch { + es := make([]entry, len(b.Entries)) + for i, e := range b.Entries { + es[i] = makeEntry(&e, withTxs) + } + shreds := make([]uint32, len(b.Shreds)) + for i, s := range b.Shreds { + shreds[i] = s.CommonHeader().Index + } + return entryBatch{ + Entries: es, + Shreds: shreds, + EncodedSize: len(b.Raw), + } +} + +// entry is a YAML-friendly version of shred.Entry. +type entry struct { + NumHashes uint64 `yaml:"num_hashes"` + Hash string `yaml:"hash"` + NumTxns int `yaml:"num_txns"` + Txns []any `yaml:"txns,omitempty"` +} + +func makeEntry(e *shred.Entry, withTxs bool) entry { + var txJSONs []any + if withTxs { + // Hacky and slow serializer to make txn YAML output tolerable + // + // The main problem is that the YAML serializer formats byte slices as arrays, + // whereas JSON serializer outputs base64 strings, which is what we want. + // + // This indirection effectively creates a dynamic data structure out of a strongly typed Txn, + // replacing all byte slices in with strings. + txJSONs = make([]any, len(e.Txns)) + for i, txn := range e.Txns { + txJSON, _ := json.Marshal(txn) + _ = json.Unmarshal(txJSON, &txJSONs[i]) + } + } + return entry{ + NumHashes: e.NumHashes, + Hash: base64.StdEncoding.EncodeToString(e.Hash[:]), + NumTxns: len(e.Txns), + Txns: txJSONs, + } +} diff --git a/cmd/radiance/blockstore/yaml/yaml.go b/cmd/radiance/blockstore/yaml/yaml.go index 6c9f8ac..c39a70a 100644 --- a/cmd/radiance/blockstore/yaml/yaml.go +++ b/cmd/radiance/blockstore/yaml/yaml.go @@ -26,6 +26,7 @@ var ( flagSlots = flags.String("slots", "", "Slots to dump") flagEntries = flags.Bool("entries", false, "Also dump slot entries") flagShreds = flags.Bool("shreds", false, "Also dump shreds") + flagTxns = flags.Bool("txs", false, "Also dump transactions") ) func init() { @@ -136,11 +137,16 @@ func dumpDataEntries(db *blockstore.DB, meta *blockstore.SlotMeta) { return } - fmt.Println(" entries:") + yamlEntries := make([]entryBatch, len(entries)) + for i, x := range entries { + yamlEntries[i] = makeEntryBatch(&x, *flagTxns) + } + + fmt.Println(" entry_batches:") enc := newYAMLPrinter(3) defer enc.Close() - if err := enc.Encode(entries); err != nil { + if err := enc.Encode(yamlEntries); err != nil { panic(err.Error()) } } diff --git a/pkg/blockstore/shreds.go b/pkg/blockstore/shreds.go index fd8ea13..449d5d8 100644 --- a/pkg/blockstore/shreds.go +++ b/pkg/blockstore/shreds.go @@ -57,7 +57,13 @@ func sliceSortedByRange[T constraints.Ordered](list []T, start T, stop T) []T { return list } -func (d *DB) GetEntries(meta *SlotMeta) ([]shred.Entry, error) { +type Entries struct { + Entries []shred.Entry + Raw []byte + Shreds []shred.Shred +} + +func (d *DB) GetEntries(meta *SlotMeta) ([]Entries, error) { shreds, err := d.GetDataShreds(meta.Slot, 0, uint32(meta.Received)) if err != nil { return nil, err @@ -66,8 +72,7 @@ func (d *DB) GetEntries(meta *SlotMeta) ([]shred.Entry, error) { } // DataShredsToEntries reassembles shreds to entries containing transactions. -func DataShredsToEntries(meta *SlotMeta, shreds []shred.Shred) ([]shred.Entry, error) { - var entries []shred.Entry +func DataShredsToEntries(meta *SlotMeta, shreds []shred.Shred) (entries []Entries, err error) { ranges := meta.entryRanges() for _, r := range ranges { parts := shreds[r.startIdx : r.endIdx+1] @@ -84,7 +89,11 @@ func DataShredsToEntries(meta *SlotMeta, shreds []shred.Shred) ([]shred.Entry, e return nil, fmt.Errorf("cannot decode entry at %d:[%d-%d]: %w", meta.Slot, r.startIdx, r.endIdx, err) } - entries = append(entries, subEntries.Entries...) + entries = append(entries, Entries{ + Entries: subEntries.Entries, + Raw: entryBytes[:dec.Position()], + Shreds: parts, + }) } return entries, nil } diff --git a/pkg/shred/shred.go b/pkg/shred/shred.go index c7cfca0..d9537c4 100644 --- a/pkg/shred/shred.go +++ b/pkg/shred/shred.go @@ -1,10 +1,6 @@ package shred -import ( - "encoding/json" - - "github.com/gagliardetto/solana-go" -) +import "github.com/gagliardetto/solana-go" type Shred interface { CommonHeader() *CommonHeader @@ -74,24 +70,3 @@ type Entry struct { NumTxns uint64 `bin:"sizeof=Txns"` Txns []solana.Transaction } - -func (e Entry) MarshalYAML() (any, error) { - // Hacky and slow serializer to make YAML output tolerable - txJSONs := make([]any, len(e.Txns)) - for i, txn := range e.Txns { - txJSON, err := json.Marshal(txn) - if err != nil { - return nil, err - } - _ = json.Unmarshal(txJSON, &txJSONs[i]) - } - return struct { - NumHashes uint64 `yaml:"num_hashes"` - Hash string `yaml:"hash"` - Txns []any `yaml:"txns"` - }{ - NumHashes: e.NumHashes, - Hash: e.Hash.String(), - Txns: txJSONs, - }, nil -}