shred: improve DataShredsToEntries performance
* Don't use reflection for shred decoding * Add shred decoding benchmark
This commit is contained in:
gagliardetto
GitHub
parent
869f92e798
commit
f6a9e60f31
|
|
@ -188,7 +188,7 @@ func (w *worker) readSlot() (shouldContinue bool) {
|
|||
var numTxns uint64
|
||||
for _, outer := range entries {
|
||||
for _, e := range outer.Entries {
|
||||
numTxns += e.NumTxns
|
||||
numTxns += uint64(len(e.Txns))
|
||||
if *flagDumpSigs {
|
||||
for _, tx := range e.Txns {
|
||||
if len(tx.Signatures) > 0 {
|
||||
|
|
|
|||
4
go.mod
4
go.mod
|
|
@ -7,8 +7,8 @@ require (
|
|||
github.com/LiamHaworth/go-tproxy v0.0.0-20190726054950-ef7efd7f24ed
|
||||
github.com/VividCortex/ewma v1.2.0
|
||||
github.com/coreos/go-systemd/v22 v22.4.0
|
||||
github.com/gagliardetto/binary v0.7.1
|
||||
github.com/gagliardetto/solana-go v1.6.0
|
||||
github.com/gagliardetto/binary v0.7.2-0.20220919103307-4270bf2d6707
|
||||
github.com/gagliardetto/solana-go v1.6.1-0.20220919093524-f869c94c374c
|
||||
github.com/golang/protobuf v1.5.2
|
||||
github.com/google/gopacket v1.1.19
|
||||
github.com/google/nftables v0.0.0-20220611213346-a346d51f53b3
|
||||
|
|
|
|||
10
go.sum
10
go.sum
|
|
@ -189,11 +189,13 @@ github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiD
|
|||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
|
||||
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
|
||||
github.com/gagliardetto/binary v0.7.1 h1:6ggDQ26vR+4xEvl/S13NcdLK3MUCi4oSy73pS9aI1cI=
|
||||
github.com/gagliardetto/binary v0.7.1/go.mod h1:aOfYkc20U0deHaHn/LVZXiqlkDbFAX0FpTlDhsXa0S0=
|
||||
github.com/gagliardetto/binary v0.7.2-0.20220918220700-6a22eaf7d87e h1:zjzIVQW0cSNeX+kWcFZDplhORfidR5eJk+43xi9d23k=
|
||||
github.com/gagliardetto/binary v0.7.2-0.20220918220700-6a22eaf7d87e/go.mod h1:aOfYkc20U0deHaHn/LVZXiqlkDbFAX0FpTlDhsXa0S0=
|
||||
github.com/gagliardetto/binary v0.7.2-0.20220919103307-4270bf2d6707 h1:+hXU6rzuhxH2BldiRebMkAchNy46c2aYHrLsyS314dA=
|
||||
github.com/gagliardetto/binary v0.7.2-0.20220919103307-4270bf2d6707/go.mod h1:aOfYkc20U0deHaHn/LVZXiqlkDbFAX0FpTlDhsXa0S0=
|
||||
github.com/gagliardetto/gofuzz v1.2.2/go.mod h1:bkH/3hYLZrMLbfYWA0pWzXmi5TTRZnu4pMGZBkqMKvY=
|
||||
github.com/gagliardetto/solana-go v1.6.0 h1:NDpsNr46IgPaL0BTSwWR1bftCiWrBEnycWNBBlJkWkA=
|
||||
github.com/gagliardetto/solana-go v1.6.0/go.mod h1:053e9WcQ/SN1po8VYDC1890jU3bDBEsH4skoZq7U5Is=
|
||||
github.com/gagliardetto/solana-go v1.6.1-0.20220919093524-f869c94c374c h1:9IV9f0FYQctGzzrZjeyhFhNC7aApbP3icBAH30TTKxI=
|
||||
github.com/gagliardetto/solana-go v1.6.1-0.20220919093524-f869c94c374c/go.mod h1:a8cIq86a+lf5Q3iyU7GyUL+orUN7xKG8j71aIYy96cs=
|
||||
github.com/gagliardetto/treeout v0.1.4 h1:ozeYerrLCmCubo1TcIjFiOWTTGteOOHND1twdFpgwaw=
|
||||
github.com/gagliardetto/treeout v0.1.4/go.mod h1:loUefvXTrlRG5rYmJmExNryyBRh8f89VZhmMOyCyqok=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
|
|
|
|||
|
|
@ -83,12 +83,11 @@ func DataShredsToEntries(meta *SlotMeta, shreds []shred.Shred) (entries []Entrie
|
|||
if len(entryBytes) == 0 {
|
||||
continue
|
||||
}
|
||||
dec := bin.NewBinDecoder(entryBytes)
|
||||
var subEntries struct {
|
||||
NumEntries uint64 `bin:"sizeof=Entries"`
|
||||
Entries []shred.Entry
|
||||
}
|
||||
if err := dec.Decode(&subEntries); err != nil {
|
||||
var dec bin.Decoder
|
||||
dec.SetEncoding(bin.EncodingBin)
|
||||
dec.Reset(entryBytes)
|
||||
var subEntries SubEntries
|
||||
if err := subEntries.UnmarshalWithDecoder(&dec); err != nil {
|
||||
return nil, fmt.Errorf("cannot decode entry at %d:[%d-%d]: %w",
|
||||
meta.Slot, r.startIdx, r.endIdx, err)
|
||||
}
|
||||
|
|
@ -101,6 +100,29 @@ func DataShredsToEntries(meta *SlotMeta, shreds []shred.Shred) (entries []Entrie
|
|||
return entries, nil
|
||||
}
|
||||
|
||||
type SubEntries struct {
|
||||
Entries []shred.Entry
|
||||
}
|
||||
|
||||
func (se *SubEntries) UnmarshalWithDecoder(decoder *bin.Decoder) (err error) {
|
||||
// read the number of entries:
|
||||
numEntries, err := decoder.ReadUint64(bin.LE)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read number of entries: %w", err)
|
||||
}
|
||||
if numEntries > uint64(decoder.Remaining()) {
|
||||
return fmt.Errorf("not enough bytes to read %d entries", numEntries)
|
||||
}
|
||||
// read the entries:
|
||||
se.Entries = make([]shred.Entry, numEntries)
|
||||
for i := uint64(0); i < numEntries; i++ {
|
||||
if err = se.Entries[i].UnmarshalWithDecoder(decoder); err != nil {
|
||||
return fmt.Errorf("failed to read entry %d: %w", i, err)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (d *DB) GetAllDataShreds(slot uint64) ([]shred.Shred, error) {
|
||||
return d.getAllShreds(d.CfDataShred, slot)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,27 @@
|
|||
package blockstore
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/certusone/radiance/fixtures"
|
||||
)
|
||||
|
||||
func BenchmarkDataShredsToEntries_mainnet102815960(b *testing.B) {
|
||||
rawShreds := fixtures.DataShreds(nil, "mainnet", 102815960)
|
||||
shreds := parseShreds(nil, rawShreds, 2)
|
||||
meta := &SlotMeta{
|
||||
Consumed: 1427,
|
||||
Received: 1427,
|
||||
LastIndex: 1426,
|
||||
NumEntryEndIndexes: 574,
|
||||
EntryEndIndexes: mainnet_102815960_EntryEndIndexes,
|
||||
}
|
||||
b.ResetTimer()
|
||||
b.ReportAllocs()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, err := DataShredsToEntries(meta, shreds)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -29,390 +29,358 @@ func TestDataShredsToEntries_Mainnet_Genesis(t *testing.T) {
|
|||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("BEgmsE3RjuKda9PpnF8r5Ch4HJMHttEhTt38jsyuKuaV"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("7GSbH4g5eTo5jehuAcQmKuoFnTNaYdFRWYJraMCYJ4pY"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("cXQPc1rqMHwxtDfLEJ9CdFWguD9LMnuKaAadwgnoAU9"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("GsdubVEKzBBxdwpRrYPsCoF6TxkwLkJC7vaoAcWvQ6zj"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("GVPeccPmHPAD7sN5za67jvuvRbyJDrZMdFPJgc5k4ApF"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("HCKdcqbc9xQjg4dAqRtJSte4teNRNGuRdQaXbcFqwmsi"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("8JC6mNJbS8JFEm7bhEgkRzk9n6aueX49wXm1hKbMRXtD"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("2weUJw23EwjYkDdH5TWP4iBwgXPW88YDgtLqFUnq5iW2"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("HDUsEkxvxps4KZHHcg1g1Kvkx6xHHQEjb6diEWEbykLs"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("EW44ssuR2hgVYr22WYvRhyWpSwvqpj9FqR268EmyWrp1"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("EZ8KLZWb4fJTqty1PYjmiZjBa7LG1vZiVP385Tf1wGSv"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("4rFoMK6rVXTwKcyeG7GKg7CGTTJpUaLV6nfk4aApHncA"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("2XfkNnbeLgiWFJG88cDvie63Ay4DhF9yVhpaLwVmZvYM"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("Dv8SnjbyB3FHhoct34FNUmsxzgkVTG82nxDaMKLXnHZv"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("8eMT19ctrhoh7buqug6FL5yaDybEhpdGspEfW4Vmr8B2"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("9N8vjyY7mV6Ln77LaG4MZJWdGDApi6gw2Mf3EZ7h74AM"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("AVMHML49UUPxhcVY2QEpULbNXHNxfEHKj1J2cAbEs9Vp"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("DhfgF92wssf9sussfcf2A9gZ9rTJs9k4CuDKf5K9aoMm"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("Hzg7Ptw7fH8gqfaN5FjADcB4kw1XyfZgeCTJhcz9Dt9Y"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("7UDgQWeCGsdjgKDkLDgQ8TA5uuWhUnk6Yf3UwjKRAvS4"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("2ZDGg77CJd9PrGakiApavtY8nTwM6a35ENbuLe1sLXqJ"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("GUDUf6Dj2PZ9qsySXE85qWRj2Ax7peVnTh8AQa4wEHq2"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("FBWpsKo4XZ5jxHh2L3yr17jpe9JdZd7wTDzskA6zYUQW"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("8o6zDK3dC1rJ6UvaZazcYRACTX1zjAvUsGjua2YMWJtX"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("93krGb8CjfAzFWXbcXMUgaiAkLuWQ9wPiwpgG4L153Ji"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("14ngqzcDAwQSkGyYXbV8KHtMtQSup7FFSa7qHS74qS65"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("EdokMAJe1FEkXJeoL4CB1FvKMan63PJ3AgjZzLT5ZHF4"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("2gJtJPAmHY4VCQkF7EmpsBGNRi3122uFebcPJZAVvozf"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("ALffShUMdNB2548HAKU5dwdd1K8JNiApytn8RBKFA5MY"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("3YEHfjSheJX87bME3wNvXN6kEfuHHwsxn7EjaMTRFE9Z"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("2t1F4YsuJSYHNwougVtqHgJHZ7yBswcKpxJnUUYigE1Q"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("8vXxmsQaMQqE3zMmS1FF3UHDho9rVY6sotp5BSNEbS9X"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("CZuyDW5rZFcDHChQRS6FAAcTQHjt5Xh1CL6sKQoMPFND"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("5gws32wDVTpuFrNhCqZwxxu5B8CKvSW9yf4VrqbNWjH7"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("5PJC6VVWQnvqaioJ2wevRFhmoQNCBX9s5h1Kbj6JE5no"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("6WAAcuMZHv1rd1b6v92fqny4dsGSY7Hy6EJpGMmAAznh"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("5joYwpeat27k6XXs3NzJfV7hkw4juX7CCUZvb98dTWJt"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("CNqKzPgR3HUEdf7j6bqj63z86ov5oPTjEitJX8vMucsP"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("HTCaWhSSNviX6YDAyMteKpcgQkAY4GzaMvmKXdkqsvx5"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("4yEgDCGrWidGYjWXL5B3Hcfg7GhwLUdpBYbausUxT2Md"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("HmVzH2rs2oMw2jka3faNT4StoUQhKEVNRsucSbFJgUzQ"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("8UiBLqWrzUhEEtEbf4UhtikFD4qWqzE8aGoimbubT33r"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("uw3ierG673dDLNHTLBAYk6vGp2XzP6MnabjzasqkRLg"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("HkVC9Dm1xm2A4tiFtqodjRBawX38Y4WhPHn5dxDLHowT"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("FMfLjob5dfTmXPuY97TuuWBRZk4qEzkYao6TdZQXSNU9"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("6brZ9hfpcpqq9AUNSa1WYrDWwUazR7uUYDutNn7AxrYq"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("GddzudgJRbP7pZYw7s62VwBzeyjQTcKC6mkN8Ke7Vc6p"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("Gwk1E6CeD1a4Az9Y6Dw9gm1cXWnYFAGioj3UrmttPFGs"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("497LZxPFN8UMparCsDRM2Xwctz9Yr6ggRRxrRWYiYUnr"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("GgV4yq8mU1H8DPqegPqEoyU66jc8dvrrvy5P1kURpUmB"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("mgHymgwdvc5fNukape9sMCAd5HTKoEQQTwxNNtb8Rat"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("2LPLYYGtvfzVBXS7nYYYhBezo36PpvpPJR3TrNfAj3tB"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("FsHZMmXjqNd11HHuCFUTifCC1PzDa5Qa73Et1Pq5CgEN"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("8uq7LJ8GNCoNcsTBgqWLNnxAmR9G2954fdJLw7d22ze8"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("3LaPqPMbS57jESrHL6j8i4CDSCFqioHYM4nw8pMHkyH7"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("5H8mKSpdwXfa7B7NcS7NSRA5qFvmYsnSmfFj1URwF9VV"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("C1dX3PxkzPUeRX4WhsLPgULnisGHWtx1iEw7gTUqbEQe"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("CPosYdr3P8mndtQXF4zkr11KcVuXucMjqrvL7sxJjz4L"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("GrLSqSMZPyeQDySx9RGmFwhgLa67jrhEXsVpZuSwQ51c"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("CBmrfRRb2zghqbHyeiPdfXWGrC5naMT3USR35aeLVYRH"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("4EzStQd1KbsBrmHhfR5r8a6NhaYFAN5bi7t5tVmEMh5U"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("ExLNu3PUdfsyq4DCWYb7VaafoBguxj4bRgyxbfuACQaq"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("Ap9zjPARQp8PRFGFirio2WP7zghbALResHdS9u4FwXZr"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("4sGjMW1sUnHzSxGspuhpqLDx6wiyjNtZAMdL4VZHirAn"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
}, entries[0].Entries)
|
||||
}
|
||||
|
||||
var mainnet_102815960_EntryEndIndexes = []uint32{
|
||||
0, 1, 2, 3, 5, 16, 17, 18, 20, 21, 22, 23, 24, 26, 28, 30, 31, 34, 35, 37, 38, 39, 41, 42, 43, 44, 45, 46,
|
||||
47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 59, 60, 62, 63, 69, 74, 75, 77, 81, 82, 83, 86, 87, 88, 89, 90, 91,
|
||||
92, 93, 94, 95, 101, 102, 103, 104, 105, 107, 108, 114, 115, 116, 117, 123, 124, 127, 133, 136, 137, 138,
|
||||
143, 144, 145, 146, 147, 148, 152, 154, 155, 156, 159, 169, 170, 171, 177, 178, 180, 182, 183, 184, 185,
|
||||
186, 187, 188, 189, 190, 191, 196, 200, 201, 202, 203, 204, 205, 206, 207, 210, 211, 212, 213, 214, 215,
|
||||
216, 217, 218, 219, 221, 223, 232, 235, 237, 239, 242, 243, 245, 246, 249, 251, 253, 254, 255, 260, 265,
|
||||
285, 286, 287, 292, 293, 301, 302, 303, 304, 305, 313, 314, 315, 327, 334, 335, 343, 344, 345, 346, 355,
|
||||
364, 370, 371, 372, 373, 379, 380, 381, 382, 389, 398, 405, 406, 417, 418, 425, 426, 427, 428, 429, 430,
|
||||
439, 440, 441, 450, 452, 453, 454, 455, 463, 464, 465, 466, 467, 468, 478, 479, 480, 481, 482, 483, 484,
|
||||
495, 496, 497, 498, 499, 500, 501, 514, 516, 517, 532, 533, 534, 535, 536, 537, 547, 549, 558, 559, 560,
|
||||
572, 573, 574, 575, 586, 587, 588, 589, 590, 591, 602, 603, 613, 616, 625, 626, 627, 628, 629, 630, 636,
|
||||
637, 638, 644, 645, 646, 653, 654, 655, 656, 663, 664, 665, 666, 675, 677, 684, 685, 686, 687, 688, 689,
|
||||
690, 695, 700, 702, 708, 716, 722, 725, 726, 727, 731, 733, 735, 741, 742, 743, 745, 746, 747, 748, 751,
|
||||
752, 754, 759, 766, 770, 776, 777, 778, 779, 780, 781, 786, 791, 799, 802, 810, 813, 820, 821, 822, 823,
|
||||
824, 825, 826, 827, 828, 829, 830, 838, 846, 848, 849, 850, 851, 852, 861, 862, 863, 873, 874, 875, 876,
|
||||
877, 878, 879, 880, 881, 882, 892, 893, 894, 895, 896, 907, 909, 910, 911, 912, 913, 914, 915, 916, 917,
|
||||
918, 919, 920, 921, 933, 934, 935, 936, 937, 938, 939, 940, 954, 955, 956, 957, 958, 959, 975, 976, 977,
|
||||
978, 994, 997, 998, 999, 1013, 1014, 1015, 1016, 1017, 1018, 1019, 1020, 1021, 1022, 1039, 1040, 1041, 1056,
|
||||
1059, 1060, 1061, 1077, 1078, 1079, 1080, 1081, 1082, 1083, 1084, 1085, 1086, 1087, 1090, 1091, 1092, 1093,
|
||||
1094, 1095, 1096, 1097, 1098, 1099, 1100, 1101, 1115, 1117, 1118, 1119, 1122, 1123, 1124, 1125, 1126, 1138,
|
||||
1139, 1140, 1141, 1142, 1143, 1144, 1145, 1146, 1147, 1148, 1149, 1150, 1151, 1164, 1165, 1166, 1167, 1168,
|
||||
1182, 1184, 1189, 1190, 1191, 1192, 1193, 1194, 1195, 1196, 1197, 1198, 1199, 1213, 1214, 1215, 1220, 1222,
|
||||
1223, 1224, 1226, 1227, 1228, 1229, 1230, 1231, 1232, 1233, 1234, 1235, 1236, 1237, 1238, 1239, 1240, 1241,
|
||||
1242, 1243, 1244, 1245, 1246, 1247, 1262, 1265, 1266, 1267, 1272, 1273, 1274, 1275, 1276, 1277, 1278, 1279,
|
||||
1289, 1292, 1293, 1294, 1295, 1296, 1297, 1298, 1299, 1300, 1301, 1302, 1303, 1304, 1305, 1306, 1307, 1308,
|
||||
1309, 1310, 1311, 1322, 1325, 1326, 1327, 1328, 1329, 1330, 1331, 1332, 1333, 1334, 1335, 1336, 1337, 1338,
|
||||
1347, 1350, 1351, 1352, 1353, 1354, 1355, 1356, 1357, 1358, 1359, 1360, 1367, 1368, 1369, 1370, 1371, 1372,
|
||||
1373, 1374, 1379, 1384, 1385, 1386, 1388, 1389, 1394, 1395, 1396, 1397, 1398, 1403, 1404, 1407, 1416, 1417,
|
||||
1422, 1423, 1425, 1426,
|
||||
}
|
||||
|
||||
func TestDataShredsToEntries_Mainnet_Recent(t *testing.T) {
|
||||
rawShreds := fixtures.DataShreds(t, "mainnet", 102815960)
|
||||
shreds := parseShreds(t, rawShreds, 2)
|
||||
|
|
@ -421,41 +389,59 @@ func TestDataShredsToEntries_Mainnet_Recent(t *testing.T) {
|
|||
Received: 1427,
|
||||
LastIndex: 1426,
|
||||
NumEntryEndIndexes: 574,
|
||||
EntryEndIndexes: []uint32{
|
||||
0, 1, 2, 3, 5, 16, 17, 18, 20, 21, 22, 23, 24, 26, 28, 30, 31, 34, 35, 37, 38, 39, 41, 42, 43, 44, 45, 46,
|
||||
47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 59, 60, 62, 63, 69, 74, 75, 77, 81, 82, 83, 86, 87, 88, 89, 90, 91,
|
||||
92, 93, 94, 95, 101, 102, 103, 104, 105, 107, 108, 114, 115, 116, 117, 123, 124, 127, 133, 136, 137, 138,
|
||||
143, 144, 145, 146, 147, 148, 152, 154, 155, 156, 159, 169, 170, 171, 177, 178, 180, 182, 183, 184, 185,
|
||||
186, 187, 188, 189, 190, 191, 196, 200, 201, 202, 203, 204, 205, 206, 207, 210, 211, 212, 213, 214, 215,
|
||||
216, 217, 218, 219, 221, 223, 232, 235, 237, 239, 242, 243, 245, 246, 249, 251, 253, 254, 255, 260, 265,
|
||||
285, 286, 287, 292, 293, 301, 302, 303, 304, 305, 313, 314, 315, 327, 334, 335, 343, 344, 345, 346, 355,
|
||||
364, 370, 371, 372, 373, 379, 380, 381, 382, 389, 398, 405, 406, 417, 418, 425, 426, 427, 428, 429, 430,
|
||||
439, 440, 441, 450, 452, 453, 454, 455, 463, 464, 465, 466, 467, 468, 478, 479, 480, 481, 482, 483, 484,
|
||||
495, 496, 497, 498, 499, 500, 501, 514, 516, 517, 532, 533, 534, 535, 536, 537, 547, 549, 558, 559, 560,
|
||||
572, 573, 574, 575, 586, 587, 588, 589, 590, 591, 602, 603, 613, 616, 625, 626, 627, 628, 629, 630, 636,
|
||||
637, 638, 644, 645, 646, 653, 654, 655, 656, 663, 664, 665, 666, 675, 677, 684, 685, 686, 687, 688, 689,
|
||||
690, 695, 700, 702, 708, 716, 722, 725, 726, 727, 731, 733, 735, 741, 742, 743, 745, 746, 747, 748, 751,
|
||||
752, 754, 759, 766, 770, 776, 777, 778, 779, 780, 781, 786, 791, 799, 802, 810, 813, 820, 821, 822, 823,
|
||||
824, 825, 826, 827, 828, 829, 830, 838, 846, 848, 849, 850, 851, 852, 861, 862, 863, 873, 874, 875, 876,
|
||||
877, 878, 879, 880, 881, 882, 892, 893, 894, 895, 896, 907, 909, 910, 911, 912, 913, 914, 915, 916, 917,
|
||||
918, 919, 920, 921, 933, 934, 935, 936, 937, 938, 939, 940, 954, 955, 956, 957, 958, 959, 975, 976, 977,
|
||||
978, 994, 997, 998, 999, 1013, 1014, 1015, 1016, 1017, 1018, 1019, 1020, 1021, 1022, 1039, 1040, 1041, 1056,
|
||||
1059, 1060, 1061, 1077, 1078, 1079, 1080, 1081, 1082, 1083, 1084, 1085, 1086, 1087, 1090, 1091, 1092, 1093,
|
||||
1094, 1095, 1096, 1097, 1098, 1099, 1100, 1101, 1115, 1117, 1118, 1119, 1122, 1123, 1124, 1125, 1126, 1138,
|
||||
1139, 1140, 1141, 1142, 1143, 1144, 1145, 1146, 1147, 1148, 1149, 1150, 1151, 1164, 1165, 1166, 1167, 1168,
|
||||
1182, 1184, 1189, 1190, 1191, 1192, 1193, 1194, 1195, 1196, 1197, 1198, 1199, 1213, 1214, 1215, 1220, 1222,
|
||||
1223, 1224, 1226, 1227, 1228, 1229, 1230, 1231, 1232, 1233, 1234, 1235, 1236, 1237, 1238, 1239, 1240, 1241,
|
||||
1242, 1243, 1244, 1245, 1246, 1247, 1262, 1265, 1266, 1267, 1272, 1273, 1274, 1275, 1276, 1277, 1278, 1279,
|
||||
1289, 1292, 1293, 1294, 1295, 1296, 1297, 1298, 1299, 1300, 1301, 1302, 1303, 1304, 1305, 1306, 1307, 1308,
|
||||
1309, 1310, 1311, 1322, 1325, 1326, 1327, 1328, 1329, 1330, 1331, 1332, 1333, 1334, 1335, 1336, 1337, 1338,
|
||||
1347, 1350, 1351, 1352, 1353, 1354, 1355, 1356, 1357, 1358, 1359, 1360, 1367, 1368, 1369, 1370, 1371, 1372,
|
||||
1373, 1374, 1379, 1384, 1385, 1386, 1388, 1389, 1394, 1395, 1396, 1397, 1398, 1403, 1404, 1407, 1416, 1417,
|
||||
1422, 1423, 1425, 1426,
|
||||
},
|
||||
EntryEndIndexes: mainnet_102815960_EntryEndIndexes,
|
||||
}
|
||||
entries, err := DataShredsToEntries(meta, shreds)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 574, len(entries))
|
||||
{
|
||||
transactions := findTransactions(entries, 10)
|
||||
assert.Equal(t, 10, len(transactions))
|
||||
assert.Equal(t,
|
||||
solana.MustSignatureFromBase58("5ruUBRn3CNNcFneJtEaJBaUjhtHtEb6Q9Hu2ygTSA9TjLr6FtE2WGGBkt7nUAHc1fnBaL8DGMS6maeCYHpRzuY6K"),
|
||||
transactions[0].Signatures[0],
|
||||
)
|
||||
assert.Equal(t,
|
||||
solana.MustSignatureFromBase58("REnUxGXrE22u5KsxynCoMQDuVY6eyAZFPyxg8HccHJQtr3GXQ6eA4T2ZhZyfC1NgbR1wt9EiRjYaWNgtDCSTwim"),
|
||||
transactions[1].Signatures[0],
|
||||
)
|
||||
assert.Equal(t,
|
||||
solana.MustSignatureFromBase58("5YfaAQEZYrU84ftEGSW8i97ZfQ5aezZrvazTu6VCHFJ9xWUxYo6WwXV86diMNt1LRSNho3Ev6kvzHN9eN8z488NW"),
|
||||
transactions[2].Signatures[0],
|
||||
)
|
||||
assert.Equal(t,
|
||||
solana.MustSignatureFromBase58("662prxNx8sgR7s3vT7Xkzj4uYG72xjA9frBfm4QFZNqbGQEhFMNjCti7zBDBnWDTjMZKV1bo53rxDrrwP9D1kGHR"),
|
||||
transactions[3].Signatures[0],
|
||||
)
|
||||
assert.Equal(t,
|
||||
solana.MustSignatureFromBase58("43yAFJnaZm5caXfUvzXDKaeYhgYzfBYvHpwzTiEhs1uRHDWibVGSTS2eh6eZiYaZgQbjkYCR7dZ5TsKwTYFNXHi1"),
|
||||
transactions[4].Signatures[0],
|
||||
)
|
||||
assert.Equal(t,
|
||||
solana.MustSignatureFromBase58("2wbiisDTsTQnWedDogjTU2qV2frLPt7GeS1wbn7TzziYxTU7MHCAP7po8s8yRbfU6Y59uY9hdFUKwxZK7ehZYUAW"),
|
||||
transactions[5].Signatures[0],
|
||||
)
|
||||
assert.Equal(t,
|
||||
solana.MustSignatureFromBase58("3JbSLxaNTVTdhSp2HLjxbqNiWxdPz6dcd3Xw5sFnzyG5yFQehLQVi6UjPZ68msJALxSgDJWX9xm8YjYC9CUJMVj"),
|
||||
transactions[6].Signatures[0],
|
||||
)
|
||||
assert.Equal(t,
|
||||
solana.MustSignatureFromBase58("2oZxvCcSqKG719R8MNguXvnRTCcUHAbyn14gnYEskB9SE1bhvzKBhrWFDAx6VyE3ac2JnzrBUcZvJLmWBK7EJ4XJ"),
|
||||
transactions[7].Signatures[0],
|
||||
)
|
||||
assert.Equal(t,
|
||||
solana.MustSignatureFromBase58("4GFhtws8R3jD68t45fAKmoUUKtEHoicKyNK53mpgTJ778NSepZnGdZminXTmX2GzMNpqsft23VrZVn3qrHseEo7c"),
|
||||
transactions[8].Signatures[0],
|
||||
)
|
||||
assert.Equal(t,
|
||||
solana.MustSignatureFromBase58("2p5jNjuLeMufeDH4euwNXqb9fi3gFH7R6cprPVVzGozM6R7qhXTuahN8gm2J5mjHk1NJdb7WTG9a6GturYpWUPhq"),
|
||||
transactions[9].Signatures[0],
|
||||
)
|
||||
}
|
||||
{
|
||||
transactions := findTransactions(entries, 4000)
|
||||
assert.Equal(t, 3177, len(transactions))
|
||||
}
|
||||
}
|
||||
|
||||
func parseShreds(t testing.TB, raw [][]byte, version int) (shreds []shred.Shred) {
|
||||
|
|
@ -468,3 +454,19 @@ func parseShreds(t testing.TB, raw [][]byte, version int) (shreds []shred.Shred)
|
|||
}
|
||||
return shreds
|
||||
}
|
||||
|
||||
func findTransactions(entries []Entries, limit int) []solana.Transaction {
|
||||
transactions := make([]solana.Transaction, 0, limit)
|
||||
for _, entry := range entries {
|
||||
for _, e := range entry.Entries {
|
||||
for txIndex := range e.Txns {
|
||||
tx := e.Txns[txIndex]
|
||||
transactions = append(transactions, tx)
|
||||
if len(transactions) >= limit {
|
||||
return transactions
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return transactions
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,11 @@
|
|||
package shred
|
||||
|
||||
import "github.com/gagliardetto/solana-go"
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
bin "github.com/gagliardetto/binary"
|
||||
"github.com/gagliardetto/solana-go"
|
||||
)
|
||||
|
||||
type Shred interface {
|
||||
CommonHeader() *CommonHeader
|
||||
|
|
@ -79,6 +84,33 @@ func (d *DataV2Header) LastInSlot() bool {
|
|||
type Entry struct {
|
||||
NumHashes uint64
|
||||
Hash solana.Hash
|
||||
NumTxns uint64 `bin:"sizeof=Txns"`
|
||||
Txns []solana.Transaction
|
||||
}
|
||||
|
||||
func (en *Entry) UnmarshalWithDecoder(decoder *bin.Decoder) (err error) {
|
||||
// read the number of hashes:
|
||||
if en.NumHashes, err = decoder.ReadUint64(bin.LE); err != nil {
|
||||
return fmt.Errorf("failed to read number of hashes: %w", err)
|
||||
}
|
||||
// read the hash:
|
||||
_, err = decoder.Read(en.Hash[:])
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read hash: %w", err)
|
||||
}
|
||||
// read the number of transactions:
|
||||
numTxns, err := decoder.ReadUint64(bin.LE)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read number of transactions: %w", err)
|
||||
}
|
||||
if numTxns > uint64(decoder.Remaining()) {
|
||||
return fmt.Errorf("not enough bytes to read %d transactions", numTxns)
|
||||
}
|
||||
// read the transactions:
|
||||
en.Txns = make([]solana.Transaction, numTxns)
|
||||
for i := uint64(0); i < numTxns; i++ {
|
||||
if err = en.Txns[i].UnmarshalWithDecoder(decoder); err != nil {
|
||||
return fmt.Errorf("failed to read transaction %d: %w", i, err)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue