certusone
/
radiance
mirror of https://github.com/certusone/radiance.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

remove ipld, car packages 

The Solana IPLD project has moved to
https://github.com/rpcpool/yellowstone-faithful
This commit is contained in:
Richard Patel 2023-06-10 15:50:49 +00:00
parent 058eba3bde
commit ddf48545d2
18 changed files with 4 additions and 7732 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
cmd/radiance/car/car.go
View File

@ -1,22 +0,0 @@
//go:build !lite
package car
import (
"github.com/spf13/cobra"
"go.firedancer.io/radiance/cmd/radiance/car/create"
"go.firedancer.io/radiance/cmd/radiance/car/dump"
)
var Cmd = cobra.Command{
Use: "car",
Short: "Manage IPLD Content-addressable ARchives",
Long: "https://ipld.io/specs/transport/car/",
}
func init() {
Cmd.AddCommand(
&create.Cmd,
&dump.Cmd,
)
}

9
cmd/radiance/car/car_norocks.go
View File

@ -1,9 +0,0 @@
//go:build lite
package car
import (
"github.com/spf13/cobra"
)
var Cmd cobra.Command

81
cmd/radiance/car/create/create.go
View File

@ -1,81 +0,0 @@
//go:build !lite
package create
import (
"path/filepath"
"strconv"
"time"
"github.com/spf13/cobra"
"go.firedancer.io/radiance/pkg/ipld/cargen"
"k8s.io/klog/v2"
"go.firedancer.io/radiance/pkg/blockstore"
)
var Cmd = cobra.Command{
Use: "create <epoch>",
Short: "Create CAR files from blockstore",
Long: "Extracts Solana ledger data from blockstore (RocksDB) databases,\n" +
"and outputs IPLD CARs (content-addressable archives).\n" +
"\n" +
"Produces at least one CAR per epoch.\n" +
"CAR archive contents are deterministic.",
Args: cobra.ExactArgs(1),
}
var flags = Cmd.Flags()
var (
flagOut = flags.StringP("out", "o", "", "Output directory")
flagDBs = flags.StringArray("db", nil, "Path to RocksDB (can be specified multiple times)")
)
func init() {
Cmd.Run = run
}
func run(c *cobra.Command, args []string) {
start := time.Now()
outPath := filepath.Clean(*flagOut)
epochStr := args[0]
epoch, err := strconv.ParseUint(epochStr, 10, 32)
if err != nil {
klog.Exitf("Invalid epoch arg: %s", epochStr)
}
// Open blockstores
dbPaths := *flagDBs
handles := make([]blockstore.WalkHandle, len(*flagDBs))
for i := range handles {
var err error
handles[i].DB, err = blockstore.OpenReadOnly(dbPaths[i])
if err != nil {
klog.Exitf("Failed to open blockstore at %s: %s", dbPaths[i], err)
}
}
// Create new walker object
walker, err := blockstore.NewBlockWalk(handles, 2 /*TODO*/)
if err != nil {
klog.Exitf("Failed to create multi-DB iterator: %s", err)
}
defer walker.Close()
// Create new cargen worker.
w, err := cargen.NewWorker(outPath, epoch, walker)
if err != nil {
klog.Exitf("Failed to init cargen: %s", err)
}
ctx := c.Context()
if err = w.Run(ctx); err != nil {
klog.Exitf("FATAL: %s", err)
}
klog.Info("DONE")
timeTaken := time.Since(start)
klog.Infof("Time taken: %s", timeTaken)
}

57
cmd/radiance/car/dump/dump.go
View File

@ -1,57 +0,0 @@
package dump
import (
"errors"
"fmt"
"io"
"os"
bin "github.com/gagliardetto/binary"
"github.com/gagliardetto/solana-go"
"github.com/ipld/go-car"
"github.com/multiformats/go-multicodec"
"github.com/spf13/cobra"
"k8s.io/klog/v2"
)
var Cmd = cobra.Command{
Use: "dump <car>",
Short: "Dump the contents of a CAR file",
Args: cobra.ExactArgs(1),
}
func init() {
Cmd.Run = run
}
func run(_ *cobra.Command, args []string) {
f, err := os.Open(args[0])
if err != nil {
klog.Exit(err.Error())
}
defer f.Close()
rd, err := car.NewCarReader(f)
if err != nil {
klog.Exitf("Failed to open CAR: %s", err)
}
for {
block, err := rd.Next()
if errors.Is(err, io.EOF) {
break
} else if err != nil {
klog.Exitf("Failed to read block: %s", err)
}
if multicodec.Code(block.Cid().Type()) == multicodec.Raw {
var tx solana.Transaction
if err := bin.UnmarshalBin(&tx, block.RawData()); err != nil {
klog.Errorf("Invalid CID %s: %s", block.Cid(), err)
continue
} else if len(tx.Signatures) == 0 {
klog.Errorf("Invalid CID %s: tx has zero signatures", block.Cid())
continue
}
fmt.Println(tx.Signatures[0].String())
}
}
}

9
cmd/radiance/main.go
View File

@ -3,15 +3,15 @@ package main
import (
"context"
"flag"
"go.firedancer.io/radiance/cmd/radiance/bigtable"
"go.firedancer.io/radiance/cmd/radiance/tpu_quic"
"go.firedancer.io/radiance/cmd/radiance/tpu_udp"
"os"
"os/signal"
"go.firedancer.io/radiance/cmd/radiance/bigtable"
"go.firedancer.io/radiance/cmd/radiance/tpu_quic"
"go.firedancer.io/radiance/cmd/radiance/tpu_udp"
"github.com/spf13/cobra"
"go.firedancer.io/radiance/cmd/radiance/blockstore"
"go.firedancer.io/radiance/cmd/radiance/car"
"go.firedancer.io/radiance/cmd/radiance/gossip"
"go.firedancer.io/radiance/cmd/radiance/replay"
"k8s.io/klog/v2"
@ -34,7 +34,6 @@ func init() {
cmd.AddCommand(
&bigtable.Cmd,
&blockstore.Cmd,
&car.Cmd,
&gossip.Cmd,
&replay.Cmd,
&tpu_udp.Cmd,

79
pkg/ipld/car/car.go
View File

@ -1,79 +0,0 @@
// Package car implements the CARv1 file format.
package car
import (
"bytes"
"github.com/ipfs/go-cid"
"github.com/ipld/go-ipld-prime/codec/dagcbor"
"github.com/ipld/go-ipld-prime/datamodel"
"github.com/multiformats/go-multicodec"
)
// IdentityCID is the "zero-length "identity" multihash with "raw" codec".
//
// This is the best-practices placeholder value to refer to a non-existent or unknown object.
var IdentityCID cid.Cid
func init() {
id, err := cid.Cast([]byte{0x01, 0x55, 0x00, 0x00})
if err != nil {
panic("failed to create zero-length identity multihash with raw codec lmfao")
}
IdentityCID = id
}
// Block is a length-cid-data tuple.
// These make up most of CARv1.
//
// See https://ipld.io/specs/transport/car/carv1/#data
type Block struct {
Length int
Cid cid.Cid
Data []byte
}
// NewBlockFromRaw creates a new CIDv1 with the given multicodec contentType on the fly.
func NewBlockFromRaw(data []byte, contentType uint64) Block {
cidBuilder := cid.V1Builder{
Codec: contentType,
MhType: uint64(multicodec.Sha2_256),
}
id, err := cidBuilder.Sum(data)
if err != nil {
// Something is wrong with go-cid if this fails.
panic("failed to construct CID: " + err.Error())
}
return Block{
Length: id.ByteLen() + len(data),
Cid: id,
Data: data,
}
}
func NewBlockFromCBOR(node datamodel.Node, contentType uint64) (Block, error) {
// TODO: This could be rewritten as zero-copy
var buf bytes.Buffer
if err := dagcbor.Encode(node, &buf); err != nil {
return Block{}, err
}
return NewBlockFromRaw(buf.Bytes(), contentType), nil
}
// TotalLen returns the total length of the block, including the length prefix.
func (b Block) TotalLen() int {
return leb128Len(uint64(b.Length)) + b.Length
}
// leb128Len is like len(leb128.FromUInt64(x)).
// But without an allocation, therefore should be preferred.
func leb128Len(x uint64) (n int) {
n = 1
for {
x >>= 7
if x == 0 {
return
}
n++
}
}

59
pkg/ipld/car/car_test.go
View File

@ -1,59 +0,0 @@
package car
import (
"encoding/binary"
"fmt"
"testing"
"github.com/filecoin-project/go-leb128"
"github.com/stretchr/testify/assert"
)
func TestIdentityCIDStr(t *testing.T) {
assert.Equal(t, "bafkqaaa", IdentityCID.String())
}
func TestLeb128Len(t *testing.T) {
cases := []struct {
n uint64
len int
}{
{0, 1},
{1, 1},
{2, 1},
{3, 1},
{4, 1},
{5, 1},
{63, 1},
{64, 1},
{65, 1},
{100, 1},
{127, 1},
{128, 2},
{129, 2},
{2141192192, 5},
{^uint64(0), 10},
}
for _, tc := range cases {
t.Run(fmt.Sprintf("N_%d", tc.n), func(t *testing.T) {
assert.Equal(t, tc.len, leb128Len(tc.n))
})
}
}
// Sanity-check asserting that Go Uvarint and LEB128 are compatible.
func TestUvarintIsLeb128(t *testing.T) {
cases := []uint64{
0, 1, 2, 3, 4, 5, 63, 64,
65, 100, 127, 128, 129,
2141192192, ^uint64(0),
}
for _, tc := range cases {
t.Run(fmt.Sprintf("N_%d", tc), func(t *testing.T) {
var v0Buf [binary.MaxVarintLen64]byte
v0 := v0Buf[:binary.PutUvarint(v0Buf[:], tc)]
v1 := leb128.FromUInt64(tc)
assert.Equal(t, v0, v1)
})
}
}

91
pkg/ipld/car/writer.go
View File

@ -1,91 +0,0 @@
package car
import (
"io"
"github.com/filecoin-project/go-leb128"
"github.com/ipfs/go-cid"
"github.com/ipld/go-car"
)
type OutStream interface {
WriteBlock(Block) error
}
// Writer produces CARv1 files with size tracking.
//
// The implementation is kinda memory-efficient.
// Needs up to IPLD block size plus peanuts of memory.
//
// # Rationale
//
// github.com/ipld/go-car/v2 is not helpful because it wants to traverse a complete IPLD link system.
// However we create an IPLD link system (Merkle-DAG) on the fly in a single pass as we read the chain.
// CARv1 is simple enough that we can roll a custom block writer, so no big deal.
type Writer struct {
out countingWriter
}
// NewWriter creates a new CARv1 Writer and writes the header.
func NewWriter(out io.Writer) (*Writer, error) {
w := &Writer{out: newCountingWriter(out)}
// Deliberately using the go-car v0 library here.
// go-car v2 doesn't seem to expose the CARv1 header format.
hdr := car.CarHeader{
Roots: []cid.Cid{IdentityCID}, // placeholder
Version: 1,
}
if err := car.WriteHeader(&hdr, w.out); err != nil {
return nil, err
}
return w, nil
}
// Write raw bytes to CAR.
func (w *Writer) Write(data []byte) (n int, err error) {
return w.out.Write(data)
}
// WriteBlock writes out a length-CID-value tuple.
func (w *Writer) WriteBlock(b Block) (err error) {
if _, err = w.out.Write(leb128.FromUInt64(uint64(b.Length))); err != nil {
return err
}
if _, err = w.out.Write(b.Cid.Bytes()); err != nil {
return err
}
_, err = w.out.Write(b.Data)
return
}
// Written returns the number of bytes written so far.
func (w *Writer) Written() int64 {
return w.out.written()
}
// countingWriter wraps io.Writer, but counts number of written bytes.
// Not thread safe.
type countingWriter struct {
io.Writer
n *int64
}
func newCountingWriter(w io.Writer) countingWriter {
return countingWriter{
Writer: w,
n: new(int64),
}
}
func (c countingWriter) Write(data []byte) (n int, err error) {
n, err = c.Writer.Write(data)
*c.n += int64(n)
return
}
// written returns number of bytes written so far.
func (c countingWriter) written() int64 {
return *c.n
}

134
pkg/ipld/car/writer_test.go
View File

@ -1,134 +0,0 @@
package car
import (
"bytes"
"context"
"io"
"testing"
"github.com/ipfs/go-cid"
ds "github.com/ipfs/go-datastore"
blockstore "github.com/ipfs/go-ipfs-blockstore"
cbornode "github.com/ipfs/go-ipld-cbor"
"github.com/ipld/go-car"
"github.com/multiformats/go-multicodec"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestNewWriter(t *testing.T) {
var buf bytes.Buffer
w, err := NewWriter(&buf)
require.NoError(t, err)
require.NotNil(t, w)
// Ensure that CAR header has been written.
assert.Equal(t, []byte{
0x19, // length 0x19 follows
0xa2, // map with two items
0x65, 0x72, 0x6f, 0x6f, 0x74, 0x73, // map key: "roots"
0x81, // map value: array of one root
0xd8, 0x2a, 0x45, 0x00, 0x01, 0x55, 0x00, 0x00, // CID 00 01 55 00 00 (identity)
0x67, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, // map key: "version"
0x01, // integer 0x01
}, buf.Bytes())
// Ensure CBOR can be deserialized.
var x struct {
Roots []cid.Cid `refmt:"roots"`
Version uint64 `refmt:"version"`
}
cbornode.RegisterCborType(x)
err = cbornode.DecodeInto(buf.Bytes()[1:], &x)
require.NoError(t, err)
assert.Equal(t, []cid.Cid{IdentityCID}, x.Roots)
assert.Equal(t, uint64(1), x.Version)
}
func TestNewWriter_Error(t *testing.T) {
var mock mockWriter
mock.err = io.ErrClosedPipe
w, err := NewWriter(&mock)
assert.Nil(t, w)
assert.Same(t, mock.err, err)
}
func TestWriter(t *testing.T) {
var buf bytes.Buffer
w, err := NewWriter(&buf)
require.NoError(t, err)
require.NotNil(t, w)
// Write a bunch of data
require.NoError(t, w.WriteBlock(NewBlockFromRaw([]byte{}, uint64(multicodec.Raw))))
require.NoError(t, w.WriteBlock(NewBlockFromRaw([]byte("hello world"), uint64(multicodec.Raw))))
assert.Equal(t, int64(buf.Len()), w.Written())
// Load using ipld/go-car library
ctx := context.Background()
store := blockstore.NewBlockstore(ds.NewMapDatastore())
ch, err := car.LoadCar(ctx, store, &buf)
require.NoError(t, err)
assert.Equal(t, &car.CarHeader{
Roots: []cid.Cid{IdentityCID},
Version: 1,
}, ch)
allKeys, err := store.AllKeysChan(ctx)
require.NoError(t, err)
var keys []cid.Cid
for key := range allKeys {
keys = append(keys, key)
t.Log(key.String())
}
assert.Len(t, keys, 2)
b, err := store.Get(ctx, cid.MustParse("bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku"))
require.NoError(t, err)
assert.Equal(t, b.RawData(), []byte{})
b, err = store.Get(ctx, cid.MustParse("bafkreifzjut3te2nhyekklss27nh3k72ysco7y32koao5eei66wof36n5e"))
require.NoError(t, err)
assert.Equal(t, b.RawData(), []byte("hello world"))
}
func TestCountingWriter(t *testing.T) {
var mock mockWriter
w := newCountingWriter(&mock)
assert.Equal(t, int64(0), w.written())
// successful write
mock.n, mock.err = 5, nil
n, err := w.Write([]byte("hello"))
assert.Equal(t, mock.n, n)
assert.Equal(t, mock.err, err)
assert.Equal(t, int64(5), w.written())
// partial write
mock.n, mock.err = 3, io.ErrClosedPipe
n, err = w.Write([]byte("hello"))
assert.Equal(t, mock.n, n)
assert.Equal(t, mock.err, err)
assert.Equal(t, int64(8), w.written())
// failed write
mock.n, mock.err = 0, io.ErrClosedPipe
n, err = w.Write([]byte("hello"))
assert.Equal(t, mock.n, n)
assert.Equal(t, mock.err, err)
assert.Equal(t, int64(8), w.written())
}
type mockWriter struct {
n int
err error
}
func (m *mockWriter) Write(_ []byte) (int, error) {
return m.n, m.err
}

294
pkg/ipld/cargen/cargen.go
View File

@ -1,294 +0,0 @@
// Package cargen transforms blockstores into CAR files.
package cargen
import (
"bufio"
"context"
"errors"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"go.firedancer.io/radiance/pkg/blockstore"
"go.firedancer.io/radiance/pkg/ipld/car"
"go.firedancer.io/radiance/pkg/ipld/ipldgen"
"go.firedancer.io/radiance/pkg/shred"
"k8s.io/klog/v2"
)
// MaxCARSize is the maximum size of a CARv1 file.
//
// Dictated by Filecoin's preferred sector size (currently 32 GiB).
// cargen will attempt to pack CARs as large as possible but never exceed.
//
// Filecoin miners may append CARv2 indexes, which would exceed the total CAR size.
const MaxCARSize = 1 << 35
// Worker extracts Solana blocks from blockstore and produces CAR files.
//
// # Data Source
//
// Solana validator RocksDB archives serve as input.
// Any gaps in history will be detected and thrown as an error.
//
// The worker uses a blockstore.BlockWalk to seamlessly iterate over multiple RocksDBs if needed.
// This is useful if an epoch is split across multiple archives.
//
// # Epochs
//
// Each worker processes a single epoch of Solana history (432000 slots) in rooted order.
//
// Transforming a single epoch, which takes about a day on mainnet, should take a few hours to transform into CAR.
//
// Because of epoch alignment, it is safe to run multiple workers in parallel on distinct epochs.
// Therefore, the CAR generation process can be trivially parallelized by launching multiple instances.
//
// The CAR output will be byte-by-byte deterministic with regard to Solana's authenticated ledger content.
// In other words, regardless of which node operator runs this tool, they should always get the same CAR file.
//
// # Blocks
//
// Each block will be parsed and turned into an IPLD graph.
//
// The procedure respects Worker.CARSize and splits data across multiple CARs if needed.
// This allows us to assign a slot range to each CAR for the reader's convenience, at negligible alignment cost.
//
// # Output
//
// Each run produces one or more car files in the target directory,
// named `ledger-e{epoch}-s{slot}.car`, where slot is the first slot number in the epoch.
//
// The interlinked IPLD blocks are internally encoded with DAG-CBOR.
// Except for the ipldgen.SolanaTx "leaf" nodes, which are encoded using bincode (native).
type Worker struct {
dir string
walk blockstore.BlockWalkI
epoch uint64
stop uint64 // exclusive
handle carHandle
CARSize uint
}
// NewWorker creates a new worker to transform an epoch from blockstore.BlockWalk into CAR files in the given dir.
//
// Creates the directory if it doesn't exist yet.
func NewWorker(dir string, epoch uint64, walk blockstore.BlockWalkI) (*Worker, error) {
if err := os.Mkdir(dir, 0777); err != nil && !errors.Is(err, fs.ErrExist) {
return nil, err
}
// Seek to epoch start and make sure we have all data
const epochLen = 432000
start := epoch * epochLen
stop := start + epochLen
if !walk.Seek(start) {
return nil, fmt.Errorf("slot %d not available in any DB", start)
}
// TODO: This is not robust; if the DB starts in the middle of the epoch, the first slots are going to be skipped.
klog.Infof("Starting at slot %d", start)
slotsAvailable := walk.SlotsAvailable()
if slotsAvailable < epochLen {
return nil, fmt.Errorf("need slots [%d:%d] (epoch %d) but only have up to %d",
start, stop, epoch, start+slotsAvailable)
}
w := &Worker{
dir: dir,
walk: walk,
epoch: epoch,
stop: stop,
CARSize: MaxCARSize,
}
return w, nil
}
func (w *Worker) Run(ctx context.Context) error {
for ctx.Err() == nil {
next, err := w.step()
if err != nil {
return err
}
if !next {
break
}
}
if w.handle.ok() {
if err := w.handle.close(); err != nil {
return err
}
}
return ctx.Err()
}
// step iterates one block forward.
func (w *Worker) step() (next bool, err error) {
meta, ok := w.walk.Next()
if !ok {
return false, nil
}
if meta.Slot > w.stop {
return false, nil
}
entries, err := w.walk.Entries(meta)
if err != nil {
return false, fmt.Errorf("failed to get entry at slot %d: %w", meta.Slot, err)
}
if err := w.ensureHandle(meta.Slot); err != nil {
return false, err
}
if err := w.writeSlot(meta, entries); err != nil {
return false, err
}
if err := w.splitHandle(meta.Slot); err != nil {
return false, err
}
return true, nil
}
// ensureHandle makes sure we have a CAR handle that we can write to.
func (w *Worker) ensureHandle(slot uint64) error {
if w.handle.ok() {
w.handle.lastOffset = w.handle.writer.Written()
return nil
}
return w.handle.open(w.dir, w.epoch, slot)
}
// splitHandle creates a new CAR file if the current one is oversized.
//
// Internally moves blocks that exceed max CAR size from old to new file.
func (w *Worker) splitHandle(slot uint64) (err error) {
size := w.handle.writer.Written()
if size <= int64(w.CARSize) {
return nil
}
// CAR is too large and needs to be split.
klog.Infof("CAR file %s too large, splitting...", w.handle.file.Name())
// Create new target CAR.
var newCAR carHandle
if err = newCAR.open(w.dir, w.epoch, slot); err != nil {
return err
}
// Seek old CAR back to before block.
w.handle.writer = nil
if err = w.handle.cache.Flush(); err != nil {
return fmt.Errorf("failed to flush CAR cache: %w", err)
}
klog.Infof("Seeking to offset %d and copying rest", w.handle.lastOffset)
if _, err = w.handle.file.Seek(w.handle.lastOffset, io.SeekStart); err != nil {
return fmt.Errorf("failed to rewind CAR: %w", err)
}
// Move block from old to new.
if _, err = io.Copy(newCAR.writer, w.handle.file); err != nil {
return fmt.Errorf("failed to move block between CARs: %w", err)
}
// Truncate old handle to make it fit max size.
if err = w.handle.file.Truncate(w.handle.lastOffset); err != nil {
return fmt.Errorf("failed to truncate old CAR (%s) to %d bytes: %w",
w.handle.file.Name(), w.handle.lastOffset, err)
}
// Swap handles.
err = w.handle.close()
w.handle = newCAR
if written := w.handle.writer.Written(); written > int64(w.CARSize) {
klog.Errorf("Slot %d exceeds size of a single CAR (%d > %d)", slot, written, w.CARSize)
}
return err
}
// writeSlot writes a filled Solana slot to the CAR.
// Creates multiple IPLD blocks internally.
func (w *Worker) writeSlot(meta *blockstore.SlotMeta, entries [][]shred.Entry) error {
slot := meta.Slot
asm := ipldgen.NewBlockAssembler(w.handle.writer, slot)
entryNum := 0
klog.V(3).Infof("Slot %d", slot)
for i, batch := range entries {
klog.V(6).Infof("Slot %d batch %d", slot, i)
for j, entry := range batch {
pos := ipldgen.EntryPos{
Slot: slot,
EntryIndex: entryNum,
Batch: i,
BatchIndex: j,
LastShred: -1,
}
if j == len(batch)-1 {
// We map "last shred of batch" to each "last entry of batch"
// so we can reconstruct the shred/entry-batch assignments.
if i >= len(meta.EntryEndIndexes) {
return fmt.Errorf("out-of-bounds batch index %d (have %d batches in slot %d)",
i, len(meta.EntryEndIndexes), slot)
}
pos.LastShred = int(meta.EntryEndIndexes[i])
}
if err := asm.WriteEntry(entry, pos); err != nil {
return fmt.Errorf("failed to write slot %d shred %d (batch %d index %d): %s",
slot, entryNum, i, j, err)
}
entryNum++
}
}
// TODO roll up into ledger entries
if _, err := asm.Finish(); err != nil {
klog.Exitf("Failed to write block: %s", err)
}
return nil
}
type carHandle struct {
file *os.File
cache *bufio.Writer
writer *car.Writer
lastOffset int64
}
const writeBufSize = 16384
func (c *carHandle) open(dir string, epoch uint64, slot uint64) error {
if c.ok() {
return fmt.Errorf("handle not closed")
}
p := filepath.Join(dir, fmt.Sprintf("ledger-e%d-s%d.car", epoch, slot))
f, err := os.OpenFile(p, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666)
if err != nil {
return fmt.Errorf("failed to create CAR: %w", err)
}
cache := bufio.NewWriterSize(f, writeBufSize)
writer, err := car.NewWriter(cache)
if err != nil {
return fmt.Errorf("failed to start CAR at %s: %w", p, err)
}
*c = carHandle{
file: f,
cache: cache,
writer: writer,
lastOffset: 0,
}
klog.Infof("Created new CAR file %s", f.Name())
return nil
}
func (c *carHandle) ok() bool {
return c.writer != nil
}
func (c *carHandle) close() (err error) {
if err = c.cache.Flush(); err != nil {
return err
}
err = c.file.Close()
*c = carHandle{}
return
}

270
pkg/ipld/cargen/cargen_test.go
View File

@ -1,270 +0,0 @@
package cargen
import (
"bytes"
"context"
"errors"
"io"
"os"
"path/filepath"
"sort"
"strings"
"testing"
bin "github.com/gagliardetto/binary"
"github.com/gagliardetto/solana-go"
"github.com/go-logr/logr/testr"
"github.com/ipfs/go-cid"
"github.com/ipld/go-car"
"github.com/ipld/go-ipld-prime/codec/dagcbor"
"github.com/ipld/go-ipld-prime/node/basicnode"
"github.com/multiformats/go-multicodec"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.firedancer.io/radiance/pkg/blockstore"
"go.firedancer.io/radiance/pkg/shred"
"k8s.io/klog/v2"
)
// mockBlockWalk is a mock implementation of blockstore.BlockWalkI.
type mockBlockWalk struct {
queue []*blockstore.SlotMeta
entries map[*blockstore.SlotMeta][][]shred.Entry
staged [][]shred.Entry
slot uint64
left int64
}
func newMockBlockWalk() *mockBlockWalk {
return &mockBlockWalk{
queue: nil,
entries: make(map[*blockstore.SlotMeta][][]shred.Entry),
staged: nil,
}
}
func (m *mockBlockWalk) append(meta *blockstore.SlotMeta, entries [][]shred.Entry) {
m.queue = append(m.queue, meta)
m.entries[meta] = entries
}
func (m *mockBlockWalk) Seek(slot uint64) (ok bool) {
for len(m.queue) > 0 && m.queue[len(m.queue)-1].Slot < slot {
delete(m.entries, m.queue[0])
m.queue = m.queue[1:]
}
return len(m.queue) > 0
}
func (m *mockBlockWalk) SlotsAvailable() uint64 {
if m.left <= 0 {
return 0
}
return uint64(m.left)
}
func (m *mockBlockWalk) Next() (meta *blockstore.SlotMeta, ok bool) {
if len(m.queue) == 0 {
m.left = 0
return nil, false
}
meta = m.queue[0]
m.queue = m.queue[1:]
m.left -= int64(meta.Slot) - int64(m.slot)
m.staged = m.entries[meta]
delete(m.entries, meta)
m.slot = meta.Slot
ok = true
return
}
func (m *mockBlockWalk) Entries(*blockstore.SlotMeta) ([][]shred.Entry, error) {
return m.staged, nil
}
func (m *mockBlockWalk) Close() {
m.queue = nil
m.entries = nil
m.staged = nil
}
func TestGen_Empty(t *testing.T) {
walk := newMockBlockWalk()
dir := t.TempDir()
worker, err := NewWorker(dir, 0, walk)
assert.EqualError(t, err, "slot 0 not available in any DB")
assert.Nil(t, worker)
}
func TestGen_Split(t *testing.T) {
klog.SetLogger(testr.New(t))
t.Cleanup(klog.ClearLogger)
walk := newMockBlockWalk()
walk.left = 432000
walk.append(
&blockstore.SlotMeta{Slot: 432000, EntryEndIndexes: []uint32{0}},
[][]shred.Entry{
{
{
Txns: []solana.Transaction{
{
Signatures: make([]solana.Signature, 1),
Message: solana.Message{
AccountKeys: make([]solana.PublicKey, 3),
Header: solana.MessageHeader{},
RecentBlockhash: solana.Hash{},
Instructions: []solana.CompiledInstruction{
{
Accounts: make([]uint16, 2),
Data: make(solana.Base58, 64),
},
},
},
},
},
},
},
})
walk.append(
&blockstore.SlotMeta{Slot: 432002, EntryEndIndexes: []uint32{0, 1}},
[][]shred.Entry{
{
{
Txns: []solana.Transaction{
{
Signatures: make([]solana.Signature, 1),
Message: solana.Message{
AccountKeys: make([]solana.PublicKey, 1),
Header: solana.MessageHeader{},
RecentBlockhash: solana.Hash{},
Instructions: []solana.CompiledInstruction{
{
Accounts: make([]uint16, 1),
Data: make(solana.Base58, 20),
},
},
},
},
},
},
},
{
{
Txns: []solana.Transaction{
{
Signatures: make([]solana.Signature, 1),
Message: solana.Message{
AccountKeys: make([]solana.PublicKey, 1),
Header: solana.MessageHeader{},
RecentBlockhash: solana.Hash{},
Instructions: []solana.CompiledInstruction{
{
Accounts: make([]uint16, 1),
Data: make(solana.Base58, 20),
},
},
},
},
},
},
},
})
dir := t.TempDir()
worker, err := NewWorker(dir, 1, walk)
require.NoError(t, err)
require.NotNil(t, worker)
worker.CARSize = 1024
require.NoError(t, worker.Run(context.Background()))
entries, err := os.ReadDir(dir)
require.NoError(t, err)
sort.Slice(entries, func(i, j int) bool {
return strings.Compare(entries[i].Name(), entries[j].Name()) < 0
})
assert.Len(t, entries, 2)
cases := map[string]struct {
size int64
cids []string
}{
"ledger-e1-s432000.car": {
size: 534,
cids: []string{
"bafkreicloicbw5yk5bpkthpwgnxrjy4iwqiitbvxs52tbkxm3fqcmgvl7a", // KindTx
"bafyreiawzny3vxq6bir23qjai6fcujyiciqz7iftnibrfzbuvtoy7fvqtq", // KindEntry
"bafyreiaihhxk2rfo5lgilus3wp3mqglv2do3zgtfqpp5zfygrpzdbez3am", // KindBlock
},
},
"ledger-e1-s432002.car": {
size: 782,
cids: []string{
"bafkreihgemfmup2imylvtyp2wj3fymakbx2bfyi2rlzxxq4xv3ycgmc3da", // KindTx
"bafyreih5wmt3ly25phouneamyxg5fs4uu3ma6kdvgwfregnmjsibkuipeq", // KindEntry
"bafkreihgemfmup2imylvtyp2wj3fymakbx2bfyi2rlzxxq4xv3ycgmc3da", // KindTx
"bafyreih5wmt3ly25phouneamyxg5fs4uu3ma6kdvgwfregnmjsibkuipeq", // KindEntry
"bafyreibqnqnmnhunzwcopnc7srlvq5p5bbgex2xspb6ayzf265rcwtaiie", // KindBlock
},
},
}
for name, tc := range cases {
filePath := filepath.Join(dir, name)
t.Run("Parse_"+name, func(t *testing.T) {
// match file size
info, err := os.Stat(filePath)
require.NoError(t, err)
assert.Equal(t, tc.size, info.Size())
// ensure CARs decode
f, err := os.Open(filepath.Join(dir, name))
require.NoError(t, err)
defer f.Close()
rd, err := car.NewCarReader(f)
require.NoError(t, err)
var cids []cid.Cid
for {
block, err := rd.Next()
if errors.Is(err, io.EOF) {
break
}
require.NoError(t, err)
cid := block.Cid()
cidType := cid.Type()
t.Logf("CID=%s Multicodec=%#x", cid, cidType)
switch multicodec.Code(cidType) {
case multicodec.Raw:
var tx solana.Transaction
require.NoError(t, bin.UnmarshalBin(&tx, block.RawData()))
t.Logf(" Txn: %s", &tx)
case multicodec.DagCbor:
decodeOpt := dagcbor.DecodeOptions{
AllowLinks: true,
}
builder := basicnode.Prototype.Any.NewBuilder()
require.NoError(t, decodeOpt.Decode(builder, bytes.NewReader(block.RawData())))
node := builder.Build()
t.Logf(" Entry: %s", node.Kind())
default:
panic("Unexpected entry")
}
cids = append(cids, cid)
}
// match CIDs
cidStrs := make([]string, len(cids))
for i, c := range cids {
cidStrs[i] = c.String()
}
assert.Equal(t, tc.cids, cidStrs)
})
}
}

316
pkg/ipld/ipldgen/ipldgen.go
View File

@ -1,316 +0,0 @@
// Package ipldgen transforms Solana ledger data into IPLD DAGs.
package ipldgen
import (
bin "github.com/gagliardetto/binary"
"github.com/gagliardetto/solana-go"
"github.com/ipld/go-ipld-prime/datamodel"
cidlink "github.com/ipld/go-ipld-prime/linking/cid"
"github.com/multiformats/go-multicodec"
"go.firedancer.io/radiance/pkg/ipld/car"
"go.firedancer.io/radiance/pkg/ipld/ipldsch"
"go.firedancer.io/radiance/pkg/shred"
)
// CIDLen is the serialized size in bytes of a Raw/DagCbor CIDv1
const CIDLen = 36
// TargetBlockSize is the target size of variable-length IPLD blocks (e.g. link lists).
// Don't set this to the IPFS max block size, as we might overrun by a few kB.
const TargetBlockSize = 1 << 19
// lengthPrefixSize is the max practical size of an array length prefix.
const lengthPrefixSize = 4
// IPLD node identifier
const (
KindTx = iota
KindEntry
KindBlock
)
type BlockAssembler struct {
writer car.OutStream
slot uint64
entries []cidlink.Link
shredding []shredding
}
type shredding struct {
entryEndIdx uint
shredEndIdx uint
}
func NewBlockAssembler(writer car.OutStream, slot uint64) *BlockAssembler {
return &BlockAssembler{
writer: writer,
slot: slot,
}
}
type EntryPos struct {
Slot uint64
EntryIndex int
Batch int
BatchIndex int
LastShred int
}
// WriteEntry appends a ledger entry to the CAR.
func (b *BlockAssembler) WriteEntry(entry shred.Entry, pos EntryPos) error {
txList, err := NewTxListAssembler(b.writer).Assemble(entry.Txns)
if err != nil {
return err
}
builder := ipldsch.Type.Entry.NewBuilder()
entryMap, err := builder.BeginMap(9)
if err != nil {
return err
}
if pos.LastShred > 0 {
b.shredding = append(b.shredding, shredding{
entryEndIdx: uint(pos.EntryIndex),
shredEndIdx: uint(pos.LastShred),
})
}
var nodeAsm datamodel.NodeAssembler
nodeAsm, err = entryMap.AssembleEntry("kind")
if err != nil {
return err
}
if err = nodeAsm.AssignInt(int64(KindEntry)); err != nil {
return err
}
nodeAsm, err = entryMap.AssembleEntry("numHashes")
if err != nil {
return err
}
if err = nodeAsm.AssignInt(int64(entry.NumHashes)); err != nil {
return err
}
nodeAsm, err = entryMap.AssembleEntry("hash")
if err != nil {
return err
}
if err = nodeAsm.AssignBytes(entry.Hash[:]); err != nil {
return err
}
nodeAsm, err = entryMap.AssembleEntry("txs")
if err != nil {
return err
}
if err = nodeAsm.AssignNode(txList); err != nil {
return err
}
if err = entryMap.Finish(); err != nil {
return err
}
node := builder.Build().(ipldsch.Entry).Representation()
block, err := car.NewBlockFromCBOR(node, uint64(multicodec.DagCbor))
if err != nil {
return err
}
b.entries = append(b.entries, cidlink.Link{Cid: block.Cid})
return b.writer.WriteBlock(block)
}
// Finish appends block metadata to the CAR and returns the root CID.
func (b *BlockAssembler) Finish() (link cidlink.Link, err error) {
builder := ipldsch.Type.Block.NewBuilder()
entryMap, err := builder.BeginMap(4)
if err != nil {
return link, err
}
var nodeAsm datamodel.NodeAssembler
nodeAsm, err = entryMap.AssembleEntry("kind")
if err != nil {
return link, err
}
if err = nodeAsm.AssignInt(int64(KindBlock)); err != nil {
return link, err
}
nodeAsm, err = entryMap.AssembleEntry("slot")
if err != nil {
return link, err
}
if err = nodeAsm.AssignInt(int64(b.slot)); err != nil {
return link, err
}
nodeAsm, err = entryMap.AssembleEntry("shredding")
if err != nil {
return link, err
}
list, err := nodeAsm.BeginList(int64(len(b.shredding)))
if err != nil {
return link, err
}
for _, s := range b.shredding {
tuple, err := list.AssembleValue().BeginMap(2)
if err != nil {
return link, err
}
entry, err := tuple.AssembleEntry("entryEndIdx")
if err != nil {
return link, err
}
if err = entry.AssignInt(int64(s.entryEndIdx)); err != nil {
return link, err
}
entry, err = tuple.AssembleEntry("shredEndIdx")
if err != nil {
return link, err
}
if err = entry.AssignInt(int64(s.shredEndIdx)); err != nil {
return link, err
}
if err = tuple.Finish(); err != nil {
return link, err
}
}
if err = list.Finish(); err != nil {
return link, err
}
nodeAsm, err = entryMap.AssembleEntry("entries")
if err != nil {
return link, err
}
list, err = nodeAsm.BeginList(int64(len(b.entries)))
if err != nil {
return link, err
}
for _, entry := range b.entries {
if err = list.AssembleValue().AssignLink(entry); err != nil {
return link, err
}
}
if err = list.Finish(); err != nil {
return link, err
}
if err = entryMap.Finish(); err != nil {
return link, err
}
node := builder.Build().(ipldsch.Block).Representation()
block, err := car.NewBlockFromCBOR(node, uint64(multicodec.DagCbor))
if err != nil {
return link, err
}
if err = b.writer.WriteBlock(block); err != nil {
return link, err
}
return cidlink.Link{Cid: block.Cid}, nil
}
// TxListAssembler produces a Merkle tree of transactions with wide branches.
type TxListAssembler struct {
writer car.OutStream
links []cidlink.Link
}
func NewTxListAssembler(writer car.OutStream) TxListAssembler {
return TxListAssembler{writer: writer}
}
// Assemble produces a transaction list DAG and returns the root node.
func (t TxListAssembler) Assemble(txs []solana.Transaction) (datamodel.Node, error) {
for _, tx := range txs {
if err := t.writeTx(tx); err != nil {
return nil, err
}
}
return t.finish()
}
// writeTx writes out SolanaTx to the CAR and appends CID to memory.
func (t *TxListAssembler) writeTx(tx solana.Transaction) error {
buf, err := bin.MarshalBin(tx)
if err != nil {
panic("failed to marshal tx: " + err.Error())
}
leaf := car.NewBlockFromRaw(buf, uint64(multicodec.Raw))
if err := t.writer.WriteBlock(leaf); err != nil {
return err
}
t.links = append(t.links, cidlink.Link{Cid: leaf.Cid})
return nil
}
// finish recursively writes out RadianceTx into a tree structure until the root fits.
func (t *TxListAssembler) finish() (datamodel.Node, error) {
node, err := t.pack()
if err != nil {
return nil, err
}
// Terminator: Reached root, stop merklerizing.
if len(t.links) == 0 {
return node, nil
}
// Create left link.
block, err := car.NewBlockFromCBOR(node, uint64(multicodec.DagCbor))
if err != nil {
return nil, err
}
var links []cidlink.Link
links = append(links, cidlink.Link{Cid: block.Cid})
if err := t.writer.WriteBlock(block); err != nil {
return nil, err
}
// Create right links.
for len(t.links) > 0 {
node, err = t.pack()
if err != nil {
return nil, err
}
block, err = car.NewBlockFromCBOR(node, uint64(multicodec.DagCbor))
if err != nil {
return nil, err
}
if err = t.writer.WriteBlock(block); err != nil {
return nil, err
}
links = append(links, cidlink.Link{Cid: block.Cid})
}
// Move up layer.
t.links = links
return t.finish()
}
// pack moves as many CIDs as possible into a node.
func (t *TxListAssembler) pack() (node datamodel.Node, err error) {
builder := ipldsch.Type.TransactionList.NewBuilder()
list, err := builder.BeginList(0)
if err != nil {
return nil, err
}
// Pack nodes until we fill TargetBlockSize.
left := TargetBlockSize - lengthPrefixSize
for ; len(t.links) > 0 && left >= CIDLen; left -= CIDLen {
link := t.links[0]
t.links = t.links[1:]
if err := list.AssembleValue().AssignLink(link); err != nil {
return nil, err
}
}
if err := list.Finish(); err != nil {
return nil, err
}
node = builder.Build()
return node, nil
}

46
pkg/ipld/ipldgen/ipldgen_test.go
View File

@ -1,46 +0,0 @@
package ipldgen
import (
"fmt"
"testing"
"github.com/ipfs/go-cid"
"github.com/multiformats/go-multicodec"
"github.com/multiformats/go-multihash"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.firedancer.io/radiance/pkg/ipld/car"
)
func TestCIDLen(t *testing.T) {
// Check whether codecs actually result in a CID sized CIDLen.
// This is important for our allocation strategies during merklerization.
codecs := []uint64{
uint64(multicodec.Raw),
uint64(multicodec.DagCbor),
}
for _, codec := range codecs {
t.Run(fmt.Sprintf("Codec_%#x", codec), func(t *testing.T) {
builder := cid.V1Builder{
Codec: codec,
MhType: multihash.SHA2_256,
}
id, err := builder.Sum(nil)
require.NoError(t, err)
assert.Equal(t, id.ByteLen(), CIDLen)
})
}
}
type nullCARWriter struct{}
func (nullCARWriter) WriteBlock(car.Block) error {
return nil
}
func TestBlockAssembler_Empty(t *testing.T) {
asm := NewBlockAssembler(nullCARWriter{}, 42)
link, err := asm.Finish()
require.NoError(t, err)
t.Log(link)
}

2
pkg/ipld/ipldsch/ipldsch.go
View File

@ -1,2 +0,0 @@
// Package ipldsch provides Solana ledger IPLD schemas.
package ipldsch

51
pkg/ipld/ipldsch/ipldsch_minima.go
View File

@ -1,51 +0,0 @@
package ipldsch
// Code generated by go-ipld-prime gengo. DO NOT EDIT.
import (
"fmt"
"github.com/ipld/go-ipld-prime/datamodel"
"github.com/ipld/go-ipld-prime/schema"
)
const (
midvalue = schema.Maybe(4)
allowNull = schema.Maybe(5)
)
type maState uint8
const (
maState_initial maState = iota
maState_midKey
maState_expectValue
maState_midValue
maState_finished
)
type laState uint8
const (
laState_initial laState = iota
laState_midValue
laState_finished
)
type _ErrorThunkAssembler struct {
e error
}
func (ea _ErrorThunkAssembler) BeginMap(_ int64) (datamodel.MapAssembler, error) { return nil, ea.e }
func (ea _ErrorThunkAssembler) BeginList(_ int64) (datamodel.ListAssembler, error) { return nil, ea.e }
func (ea _ErrorThunkAssembler) AssignNull() error { return ea.e }
func (ea _ErrorThunkAssembler) AssignBool(bool) error { return ea.e }
func (ea _ErrorThunkAssembler) AssignInt(int64) error { return ea.e }
func (ea _ErrorThunkAssembler) AssignFloat(float64) error { return ea.e }
func (ea _ErrorThunkAssembler) AssignString(string) error { return ea.e }
func (ea _ErrorThunkAssembler) AssignBytes([]byte) error { return ea.e }
func (ea _ErrorThunkAssembler) AssignLink(datamodel.Link) error { return ea.e }
func (ea _ErrorThunkAssembler) AssignNode(datamodel.Node) error { return ea.e }
func (ea _ErrorThunkAssembler) Prototype() datamodel.NodePrototype {
panic(fmt.Errorf("cannot get prototype from error-carrying assembler: already derailed with error: %w", ea.e))
}

6065
pkg/ipld/ipldsch/ipldsch_satisfaction.go
View File

File diff suppressed because it is too large Load Diff

127
pkg/ipld/ipldsch/ipldsch_types.go
View File

@ -1,127 +0,0 @@
package ipldsch
// Code generated by go-ipld-prime gengo. DO NOT EDIT.
import (
"github.com/ipld/go-ipld-prime/datamodel"
)
var _ datamodel.Node = nil // suppress errors when this dependency is not referenced
// Type is a struct embeding a NodePrototype/Type for every Node implementation in this package.
// One of its major uses is to start the construction of a value.
// You can use it like this:
//
// ipldsch.Type.YourTypeName.NewBuilder().BeginMap() //...
//
// and:
//
// ipldsch.Type.OtherTypeName.NewBuilder().AssignString("x") // ...
var Type typeSlab
type typeSlab struct {
Block _Block__Prototype
Block__Repr _Block__ReprPrototype
Bool _Bool__Prototype
Bool__Repr _Bool__ReprPrototype
Bytes _Bytes__Prototype
Bytes__Repr _Bytes__ReprPrototype
Entry _Entry__Prototype
Entry__Repr _Entry__ReprPrototype
Float _Float__Prototype
Float__Repr _Float__ReprPrototype
Hash _Hash__Prototype
Hash__Repr _Hash__ReprPrototype
Int _Int__Prototype
Int__Repr _Int__ReprPrototype
Link _Link__Prototype
Link__Repr _Link__ReprPrototype
List__Link _List__Link__Prototype
List__Link__Repr _List__Link__ReprPrototype
List__Shredding _List__Shredding__Prototype
List__Shredding__Repr _List__Shredding__ReprPrototype
Shredding _Shredding__Prototype
Shredding__Repr _Shredding__ReprPrototype
String _String__Prototype
String__Repr _String__ReprPrototype
Transaction _Transaction__Prototype
Transaction__Repr _Transaction__ReprPrototype
TransactionList _TransactionList__Prototype
TransactionList__Repr _TransactionList__ReprPrototype
}
// --- type definitions follow ---
// Block matches the IPLD Schema type "Block". It has struct type-kind, and may be interrogated like map kind.
type Block = *_Block
type _Block struct {
kind _Int
slot _Int
entries _List__Link
shredding _List__Shredding
}
// Bool matches the IPLD Schema type "Bool". It has bool kind.
type Bool = *_Bool
type _Bool struct{ x bool }
// Bytes matches the IPLD Schema type "Bytes". It has bytes kind.
type Bytes = *_Bytes
type _Bytes struct{ x []byte }
// Entry matches the IPLD Schema type "Entry". It has struct type-kind, and may be interrogated like map kind.
type Entry = *_Entry
type _Entry struct {
kind _Int
numHashes _Int
hash _Hash
txs _TransactionList
}
// Float matches the IPLD Schema type "Float". It has float kind.
type Float = *_Float
type _Float struct{ x float64 }
// Hash matches the IPLD Schema type "Hash". It has bytes kind.
type Hash = *_Hash
type _Hash struct{ x []byte }
// Int matches the IPLD Schema type "Int". It has int kind.
type Int = *_Int
type _Int struct{ x int64 }
// Link matches the IPLD Schema type "Link". It has link kind.
type Link = *_Link
type _Link struct{ x datamodel.Link }
// List__Link matches the IPLD Schema type "List__Link". It has list kind.
type List__Link = *_List__Link
type _List__Link struct {
x []_Link
}
// List__Shredding matches the IPLD Schema type "List__Shredding". It has list kind.
type List__Shredding = *_List__Shredding
type _List__Shredding struct {
x []_Shredding
}
// Shredding matches the IPLD Schema type "Shredding". It has struct type-kind, and may be interrogated like map kind.
type Shredding = *_Shredding
type _Shredding struct {
entryEndIdx _Int
shredEndIdx _Int
}
// String matches the IPLD Schema type "String". It has string kind.
type String = *_String
type _String struct{ x string }
// Transaction matches the IPLD Schema type "Transaction". It has bytes kind.
type Transaction = *_Transaction
type _Transaction struct{ x []byte }
// TransactionList matches the IPLD Schema type "TransactionList". It has list kind.
type TransactionList = *_TransactionList
type _TransactionList struct {
x []_Link
}

24
pkg/ipld/ipldsch/ledger.ipld
View File

@ -1,24 +0,0 @@
type Block struct {
kind Int
slot Int
entries [ Link ]
shredding [ Shredding ]
} representation tuple
type Shredding struct {
entryEndIdx Int
shredEndIdx Int
} representation tuple
type Entry struct {
kind Int
numHashes Int
hash Hash
txs TransactionList
} representation tuple
type Hash bytes
type TransactionList [ Link ]
type Transaction bytes