blockstore: accurately encode entry batches as YAML
Updates blockstore dump YAML schema to reflect that slices of shreds map to slices of entries. Adds a new "entry_batches" wrapper list that annotates each slice of entries with shred range and encoded size.
This commit is contained in:
parent
b902a03a05
commit
d7cd878bc9
|
@ -0,0 +1,64 @@
|
||||||
|
package yaml
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json"
|
||||||
|
|
||||||
|
"github.com/certusone/radiance/pkg/blockstore"
|
||||||
|
"github.com/certusone/radiance/pkg/shred"
|
||||||
|
)
|
||||||
|
|
||||||
|
// entryBatch is a YAML-friendly version of blockstore.Entries.
|
||||||
|
type entryBatch struct {
|
||||||
|
Shreds []uint32 `yaml:"shreds,flow"`
|
||||||
|
EncodedSize int `yaml:"encoded_size,omitempty"`
|
||||||
|
Entries []entry `yaml:"entries"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func makeEntryBatch(b *blockstore.Entries, withTxs bool) entryBatch {
|
||||||
|
es := make([]entry, len(b.Entries))
|
||||||
|
for i, e := range b.Entries {
|
||||||
|
es[i] = makeEntry(&e, withTxs)
|
||||||
|
}
|
||||||
|
shreds := make([]uint32, len(b.Shreds))
|
||||||
|
for i, s := range b.Shreds {
|
||||||
|
shreds[i] = s.CommonHeader().Index
|
||||||
|
}
|
||||||
|
return entryBatch{
|
||||||
|
Entries: es,
|
||||||
|
Shreds: shreds,
|
||||||
|
EncodedSize: len(b.Raw),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// entry is a YAML-friendly version of shred.Entry.
|
||||||
|
type entry struct {
|
||||||
|
NumHashes uint64 `yaml:"num_hashes"`
|
||||||
|
Hash string `yaml:"hash"`
|
||||||
|
NumTxns int `yaml:"num_txns"`
|
||||||
|
Txns []any `yaml:"txns,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func makeEntry(e *shred.Entry, withTxs bool) entry {
|
||||||
|
var txJSONs []any
|
||||||
|
if withTxs {
|
||||||
|
// Hacky and slow serializer to make txn YAML output tolerable
|
||||||
|
//
|
||||||
|
// The main problem is that the YAML serializer formats byte slices as arrays,
|
||||||
|
// whereas JSON serializer outputs base64 strings, which is what we want.
|
||||||
|
//
|
||||||
|
// This indirection effectively creates a dynamic data structure out of a strongly typed Txn,
|
||||||
|
// replacing all byte slices in with strings.
|
||||||
|
txJSONs = make([]any, len(e.Txns))
|
||||||
|
for i, txn := range e.Txns {
|
||||||
|
txJSON, _ := json.Marshal(txn)
|
||||||
|
_ = json.Unmarshal(txJSON, &txJSONs[i])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return entry{
|
||||||
|
NumHashes: e.NumHashes,
|
||||||
|
Hash: base64.StdEncoding.EncodeToString(e.Hash[:]),
|
||||||
|
NumTxns: len(e.Txns),
|
||||||
|
Txns: txJSONs,
|
||||||
|
}
|
||||||
|
}
|
|
@ -26,6 +26,7 @@ var (
|
||||||
flagSlots = flags.String("slots", "", "Slots to dump")
|
flagSlots = flags.String("slots", "", "Slots to dump")
|
||||||
flagEntries = flags.Bool("entries", false, "Also dump slot entries")
|
flagEntries = flags.Bool("entries", false, "Also dump slot entries")
|
||||||
flagShreds = flags.Bool("shreds", false, "Also dump shreds")
|
flagShreds = flags.Bool("shreds", false, "Also dump shreds")
|
||||||
|
flagTxns = flags.Bool("txs", false, "Also dump transactions")
|
||||||
)
|
)
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
|
@ -136,11 +137,16 @@ func dumpDataEntries(db *blockstore.DB, meta *blockstore.SlotMeta) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Println(" entries:")
|
yamlEntries := make([]entryBatch, len(entries))
|
||||||
|
for i, x := range entries {
|
||||||
|
yamlEntries[i] = makeEntryBatch(&x, *flagTxns)
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Println(" entry_batches:")
|
||||||
|
|
||||||
enc := newYAMLPrinter(3)
|
enc := newYAMLPrinter(3)
|
||||||
defer enc.Close()
|
defer enc.Close()
|
||||||
if err := enc.Encode(entries); err != nil {
|
if err := enc.Encode(yamlEntries); err != nil {
|
||||||
panic(err.Error())
|
panic(err.Error())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -57,7 +57,13 @@ func sliceSortedByRange[T constraints.Ordered](list []T, start T, stop T) []T {
|
||||||
return list
|
return list
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d *DB) GetEntries(meta *SlotMeta) ([]shred.Entry, error) {
|
type Entries struct {
|
||||||
|
Entries []shred.Entry
|
||||||
|
Raw []byte
|
||||||
|
Shreds []shred.Shred
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *DB) GetEntries(meta *SlotMeta) ([]Entries, error) {
|
||||||
shreds, err := d.GetDataShreds(meta.Slot, 0, uint32(meta.Received))
|
shreds, err := d.GetDataShreds(meta.Slot, 0, uint32(meta.Received))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
@ -66,8 +72,7 @@ func (d *DB) GetEntries(meta *SlotMeta) ([]shred.Entry, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// DataShredsToEntries reassembles shreds to entries containing transactions.
|
// DataShredsToEntries reassembles shreds to entries containing transactions.
|
||||||
func DataShredsToEntries(meta *SlotMeta, shreds []shred.Shred) ([]shred.Entry, error) {
|
func DataShredsToEntries(meta *SlotMeta, shreds []shred.Shred) (entries []Entries, err error) {
|
||||||
var entries []shred.Entry
|
|
||||||
ranges := meta.entryRanges()
|
ranges := meta.entryRanges()
|
||||||
for _, r := range ranges {
|
for _, r := range ranges {
|
||||||
parts := shreds[r.startIdx : r.endIdx+1]
|
parts := shreds[r.startIdx : r.endIdx+1]
|
||||||
|
@ -84,7 +89,11 @@ func DataShredsToEntries(meta *SlotMeta, shreds []shred.Shred) ([]shred.Entry, e
|
||||||
return nil, fmt.Errorf("cannot decode entry at %d:[%d-%d]: %w",
|
return nil, fmt.Errorf("cannot decode entry at %d:[%d-%d]: %w",
|
||||||
meta.Slot, r.startIdx, r.endIdx, err)
|
meta.Slot, r.startIdx, r.endIdx, err)
|
||||||
}
|
}
|
||||||
entries = append(entries, subEntries.Entries...)
|
entries = append(entries, Entries{
|
||||||
|
Entries: subEntries.Entries,
|
||||||
|
Raw: entryBytes[:dec.Position()],
|
||||||
|
Shreds: parts,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
return entries, nil
|
return entries, nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,10 +1,6 @@
|
||||||
package shred
|
package shred
|
||||||
|
|
||||||
import (
|
import "github.com/gagliardetto/solana-go"
|
||||||
"encoding/json"
|
|
||||||
|
|
||||||
"github.com/gagliardetto/solana-go"
|
|
||||||
)
|
|
||||||
|
|
||||||
type Shred interface {
|
type Shred interface {
|
||||||
CommonHeader() *CommonHeader
|
CommonHeader() *CommonHeader
|
||||||
|
@ -74,24 +70,3 @@ type Entry struct {
|
||||||
NumTxns uint64 `bin:"sizeof=Txns"`
|
NumTxns uint64 `bin:"sizeof=Txns"`
|
||||||
Txns []solana.Transaction
|
Txns []solana.Transaction
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e Entry) MarshalYAML() (any, error) {
|
|
||||||
// Hacky and slow serializer to make YAML output tolerable
|
|
||||||
txJSONs := make([]any, len(e.Txns))
|
|
||||||
for i, txn := range e.Txns {
|
|
||||||
txJSON, err := json.Marshal(txn)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
_ = json.Unmarshal(txJSON, &txJSONs[i])
|
|
||||||
}
|
|
||||||
return struct {
|
|
||||||
NumHashes uint64 `yaml:"num_hashes"`
|
|
||||||
Hash string `yaml:"hash"`
|
|
||||||
Txns []any `yaml:"txns"`
|
|
||||||
}{
|
|
||||||
NumHashes: e.NumHashes,
|
|
||||||
Hash: e.Hash.String(),
|
|
||||||
Txns: txJSONs,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
Loading…
Reference in New Issue