tpuproxy/pkg/ipld/ipldgen/ipldgen.go

// Package ipldgen transforms Solana ledger data into IPLD DAGs.
package ipldgen

import (
	bin "github.com/gagliardetto/binary"
	"github.com/gagliardetto/solana-go"
	"github.com/ipld/go-ipld-prime/datamodel"
	cidlink "github.com/ipld/go-ipld-prime/linking/cid"
	"github.com/multiformats/go-multicodec"
	"go.firedancer.io/radiance/pkg/ipld/car"
	"go.firedancer.io/radiance/pkg/ipld/ipldsch"
	"go.firedancer.io/radiance/pkg/shred"
)

// CIDLen is the serialized size in bytes of a Raw/DagCbor CIDv1
const CIDLen = 36

// TargetBlockSize is the target size of variable-length IPLD blocks (e.g. link lists).
// Don't set this to the IPFS max block size, as we might overrun by a few kB.
const TargetBlockSize = 1 << 19

// lengthPrefixSize is the max practical size of an array length prefix.
const lengthPrefixSize = 4

// IPLD node identifier
const (
	KindTx = iota
	KindEntry
	KindBlock
)

type BlockAssembler struct {
	writer    car.OutStream
	slot      uint64
	entries   []cidlink.Link
	shredding []shredding
}

type shredding struct {
	entryEndIdx uint
	shredEndIdx uint
}

func NewBlockAssembler(writer car.OutStream, slot uint64) *BlockAssembler {
	return &BlockAssembler{
		writer: writer,
		slot:   slot,
	}
}

type EntryPos struct {
	Slot       uint64
	EntryIndex int
	Batch      int
	BatchIndex int
	LastShred  int
}

// WriteEntry appends a ledger entry to the CAR.
func (b *BlockAssembler) WriteEntry(entry shred.Entry, pos EntryPos) error {
	txList, err := NewTxListAssembler(b.writer).Assemble(entry.Txns)
	if err != nil {
		return err
	}
	builder := ipldsch.Type.Entry.NewBuilder()
	entryMap, err := builder.BeginMap(9)
	if err != nil {
		return err
	}

	if pos.LastShred > 0 {
		b.shredding = append(b.shredding, shredding{
			entryEndIdx: uint(pos.EntryIndex),
			shredEndIdx: uint(pos.LastShred),
		})
	}

	var nodeAsm datamodel.NodeAssembler

	nodeAsm, err = entryMap.AssembleEntry("kind")
	if err != nil {
		return err
	}
	if err = nodeAsm.AssignInt(int64(KindEntry)); err != nil {
		return err
	}

	nodeAsm, err = entryMap.AssembleEntry("numHashes")
	if err != nil {
		return err
	}
	if err = nodeAsm.AssignInt(int64(entry.NumHashes)); err != nil {
		return err
	}

	nodeAsm, err = entryMap.AssembleEntry("hash")
	if err != nil {
		return err
	}
	if err = nodeAsm.AssignBytes(entry.Hash[:]); err != nil {
		return err
	}

	nodeAsm, err = entryMap.AssembleEntry("txs")
	if err != nil {
		return err
	}
	if err = nodeAsm.AssignNode(txList); err != nil {
		return err
	}

	if err = entryMap.Finish(); err != nil {
		return err
	}
	node := builder.Build().(ipldsch.Entry).Representation()
	block, err := car.NewBlockFromCBOR(node, uint64(multicodec.DagCbor))
	if err != nil {
		return err
	}
	b.entries = append(b.entries, cidlink.Link{Cid: block.Cid})
	return b.writer.WriteBlock(block)
}

// Finish appends block metadata to the CAR and returns the root CID.
func (b *BlockAssembler) Finish() (link cidlink.Link, err error) {
	builder := ipldsch.Type.Block.NewBuilder()
	entryMap, err := builder.BeginMap(4)
	if err != nil {
		return link, err
	}

	var nodeAsm datamodel.NodeAssembler

	nodeAsm, err = entryMap.AssembleEntry("kind")
	if err != nil {
		return link, err
	}
	if err = nodeAsm.AssignInt(int64(KindBlock)); err != nil {
		return link, err
	}

	nodeAsm, err = entryMap.AssembleEntry("slot")
	if err != nil {
		return link, err
	}
	if err = nodeAsm.AssignInt(int64(b.slot)); err != nil {
		return link, err
	}

	nodeAsm, err = entryMap.AssembleEntry("shredding")
	if err != nil {
		return link, err
	}
	list, err := nodeAsm.BeginList(int64(len(b.shredding)))
	if err != nil {
		return link, err
	}
	for _, s := range b.shredding {
		tuple, err := list.AssembleValue().BeginMap(2)
		if err != nil {
			return link, err
		}
		entry, err := tuple.AssembleEntry("entryEndIdx")
		if err != nil {
			return link, err
		}
		if err = entry.AssignInt(int64(s.entryEndIdx)); err != nil {
			return link, err
		}
		entry, err = tuple.AssembleEntry("shredEndIdx")
		if err != nil {
			return link, err
		}
		if err = entry.AssignInt(int64(s.shredEndIdx)); err != nil {
			return link, err
		}
		if err = tuple.Finish(); err != nil {
			return link, err
		}
	}
	if err = list.Finish(); err != nil {
		return link, err
	}

	nodeAsm, err = entryMap.AssembleEntry("entries")
	if err != nil {
		return link, err
	}
	list, err = nodeAsm.BeginList(int64(len(b.entries)))
	if err != nil {
		return link, err
	}
	for _, entry := range b.entries {
		if err = list.AssembleValue().AssignLink(entry); err != nil {
			return link, err
		}
	}
	if err = list.Finish(); err != nil {
		return link, err
	}

	if err = entryMap.Finish(); err != nil {
		return link, err
	}
	node := builder.Build().(ipldsch.Block).Representation()
	block, err := car.NewBlockFromCBOR(node, uint64(multicodec.DagCbor))
	if err != nil {
		return link, err
	}
	if err = b.writer.WriteBlock(block); err != nil {
		return link, err
	}

	return cidlink.Link{Cid: block.Cid}, nil
}

// TxListAssembler produces a Merkle tree of transactions with wide branches.
type TxListAssembler struct {
	writer car.OutStream
	links  []cidlink.Link
}

func NewTxListAssembler(writer car.OutStream) TxListAssembler {
	return TxListAssembler{writer: writer}
}

// Assemble produces a transaction list DAG and returns the root node.
func (t TxListAssembler) Assemble(txs []solana.Transaction) (datamodel.Node, error) {
	for _, tx := range txs {
		if err := t.writeTx(tx); err != nil {
			return nil, err
		}
	}
	return t.finish()
}

// writeTx writes out SolanaTx to the CAR and appends CID to memory.
func (t *TxListAssembler) writeTx(tx solana.Transaction) error {
	buf, err := bin.MarshalBin(tx)
	if err != nil {
		panic("failed to marshal tx: " + err.Error())
	}
	leaf := car.NewBlockFromRaw(buf, uint64(multicodec.Raw))
	if err := t.writer.WriteBlock(leaf); err != nil {
		return err
	}
	t.links = append(t.links, cidlink.Link{Cid: leaf.Cid})
	return nil
}

// finish recursively writes out RadianceTx into a tree structure until the root fits.
func (t *TxListAssembler) finish() (datamodel.Node, error) {
	node, err := t.pack()
	if err != nil {
		return nil, err
	}
	// Terminator: Reached root, stop merklerizing.
	if len(t.links) == 0 {
		return node, nil
	}

	// Create left link.
	block, err := car.NewBlockFromCBOR(node, uint64(multicodec.DagCbor))
	if err != nil {
		return nil, err
	}
	var links []cidlink.Link
	links = append(links, cidlink.Link{Cid: block.Cid})
	if err := t.writer.WriteBlock(block); err != nil {
		return nil, err
	}
	// Create right links.
	for len(t.links) > 0 {
		node, err = t.pack()
		if err != nil {
			return nil, err
		}
		block, err = car.NewBlockFromCBOR(node, uint64(multicodec.DagCbor))
		if err != nil {
			return nil, err
		}
		if err = t.writer.WriteBlock(block); err != nil {
			return nil, err
		}
		links = append(links, cidlink.Link{Cid: block.Cid})
	}

	// Move up layer.
	t.links = links
	return t.finish()
}

// pack moves as many CIDs as possible into a node.
func (t *TxListAssembler) pack() (node datamodel.Node, err error) {
	builder := ipldsch.Type.TransactionList.NewBuilder()
	list, err := builder.BeginList(0)
	if err != nil {
		return nil, err
	}

	// Pack nodes until we fill TargetBlockSize.
	left := TargetBlockSize - lengthPrefixSize
	for ; len(t.links) > 0 && left >= CIDLen; left -= CIDLen {
		link := t.links[0]
		t.links = t.links[1:]
		if err := list.AssembleValue().AssignLink(link); err != nil {
			return nil, err
		}
	}

	if err := list.Finish(); err != nil {
		return nil, err
	}
	node = builder.Build()
	return node, nil
}