shred: improve DataShredsToEntries performance
* Don't use reflection for shred decoding * Add shred decoding benchmark
This commit is contained in:
parent
869f92e798
commit
f6a9e60f31
|
@ -188,7 +188,7 @@ func (w *worker) readSlot() (shouldContinue bool) {
|
|||
var numTxns uint64
|
||||
for _, outer := range entries {
|
||||
for _, e := range outer.Entries {
|
||||
numTxns += e.NumTxns
|
||||
numTxns += uint64(len(e.Txns))
|
||||
if *flagDumpSigs {
|
||||
for _, tx := range e.Txns {
|
||||
if len(tx.Signatures) > 0 {
|
||||
|
|
4
go.mod
4
go.mod
|
@ -7,8 +7,8 @@ require (
|
|||
github.com/LiamHaworth/go-tproxy v0.0.0-20190726054950-ef7efd7f24ed
|
||||
github.com/VividCortex/ewma v1.2.0
|
||||
github.com/coreos/go-systemd/v22 v22.4.0
|
||||
github.com/gagliardetto/binary v0.7.1
|
||||
github.com/gagliardetto/solana-go v1.6.0
|
||||
github.com/gagliardetto/binary v0.7.2-0.20220919103307-4270bf2d6707
|
||||
github.com/gagliardetto/solana-go v1.6.1-0.20220919093524-f869c94c374c
|
||||
github.com/golang/protobuf v1.5.2
|
||||
github.com/google/gopacket v1.1.19
|
||||
github.com/google/nftables v0.0.0-20220611213346-a346d51f53b3
|
||||
|
|
10
go.sum
10
go.sum
|
@ -189,11 +189,13 @@ github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiD
|
|||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
|
||||
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
|
||||
github.com/gagliardetto/binary v0.7.1 h1:6ggDQ26vR+4xEvl/S13NcdLK3MUCi4oSy73pS9aI1cI=
|
||||
github.com/gagliardetto/binary v0.7.1/go.mod h1:aOfYkc20U0deHaHn/LVZXiqlkDbFAX0FpTlDhsXa0S0=
|
||||
github.com/gagliardetto/binary v0.7.2-0.20220918220700-6a22eaf7d87e h1:zjzIVQW0cSNeX+kWcFZDplhORfidR5eJk+43xi9d23k=
|
||||
github.com/gagliardetto/binary v0.7.2-0.20220918220700-6a22eaf7d87e/go.mod h1:aOfYkc20U0deHaHn/LVZXiqlkDbFAX0FpTlDhsXa0S0=
|
||||
github.com/gagliardetto/binary v0.7.2-0.20220919103307-4270bf2d6707 h1:+hXU6rzuhxH2BldiRebMkAchNy46c2aYHrLsyS314dA=
|
||||
github.com/gagliardetto/binary v0.7.2-0.20220919103307-4270bf2d6707/go.mod h1:aOfYkc20U0deHaHn/LVZXiqlkDbFAX0FpTlDhsXa0S0=
|
||||
github.com/gagliardetto/gofuzz v1.2.2/go.mod h1:bkH/3hYLZrMLbfYWA0pWzXmi5TTRZnu4pMGZBkqMKvY=
|
||||
github.com/gagliardetto/solana-go v1.6.0 h1:NDpsNr46IgPaL0BTSwWR1bftCiWrBEnycWNBBlJkWkA=
|
||||
github.com/gagliardetto/solana-go v1.6.0/go.mod h1:053e9WcQ/SN1po8VYDC1890jU3bDBEsH4skoZq7U5Is=
|
||||
github.com/gagliardetto/solana-go v1.6.1-0.20220919093524-f869c94c374c h1:9IV9f0FYQctGzzrZjeyhFhNC7aApbP3icBAH30TTKxI=
|
||||
github.com/gagliardetto/solana-go v1.6.1-0.20220919093524-f869c94c374c/go.mod h1:a8cIq86a+lf5Q3iyU7GyUL+orUN7xKG8j71aIYy96cs=
|
||||
github.com/gagliardetto/treeout v0.1.4 h1:ozeYerrLCmCubo1TcIjFiOWTTGteOOHND1twdFpgwaw=
|
||||
github.com/gagliardetto/treeout v0.1.4/go.mod h1:loUefvXTrlRG5rYmJmExNryyBRh8f89VZhmMOyCyqok=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
|
|
|
@ -83,12 +83,11 @@ func DataShredsToEntries(meta *SlotMeta, shreds []shred.Shred) (entries []Entrie
|
|||
if len(entryBytes) == 0 {
|
||||
continue
|
||||
}
|
||||
dec := bin.NewBinDecoder(entryBytes)
|
||||
var subEntries struct {
|
||||
NumEntries uint64 `bin:"sizeof=Entries"`
|
||||
Entries []shred.Entry
|
||||
}
|
||||
if err := dec.Decode(&subEntries); err != nil {
|
||||
var dec bin.Decoder
|
||||
dec.SetEncoding(bin.EncodingBin)
|
||||
dec.Reset(entryBytes)
|
||||
var subEntries SubEntries
|
||||
if err := subEntries.UnmarshalWithDecoder(&dec); err != nil {
|
||||
return nil, fmt.Errorf("cannot decode entry at %d:[%d-%d]: %w",
|
||||
meta.Slot, r.startIdx, r.endIdx, err)
|
||||
}
|
||||
|
@ -101,6 +100,29 @@ func DataShredsToEntries(meta *SlotMeta, shreds []shred.Shred) (entries []Entrie
|
|||
return entries, nil
|
||||
}
|
||||
|
||||
type SubEntries struct {
|
||||
Entries []shred.Entry
|
||||
}
|
||||
|
||||
func (se *SubEntries) UnmarshalWithDecoder(decoder *bin.Decoder) (err error) {
|
||||
// read the number of entries:
|
||||
numEntries, err := decoder.ReadUint64(bin.LE)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read number of entries: %w", err)
|
||||
}
|
||||
if numEntries > uint64(decoder.Remaining()) {
|
||||
return fmt.Errorf("not enough bytes to read %d entries", numEntries)
|
||||
}
|
||||
// read the entries:
|
||||
se.Entries = make([]shred.Entry, numEntries)
|
||||
for i := uint64(0); i < numEntries; i++ {
|
||||
if err = se.Entries[i].UnmarshalWithDecoder(decoder); err != nil {
|
||||
return fmt.Errorf("failed to read entry %d: %w", i, err)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (d *DB) GetAllDataShreds(slot uint64) ([]shred.Shred, error) {
|
||||
return d.getAllShreds(d.CfDataShred, slot)
|
||||
}
|
||||
|
|
|
@ -0,0 +1,27 @@
|
|||
package blockstore
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/certusone/radiance/fixtures"
|
||||
)
|
||||
|
||||
func BenchmarkDataShredsToEntries_mainnet102815960(b *testing.B) {
|
||||
rawShreds := fixtures.DataShreds(nil, "mainnet", 102815960)
|
||||
shreds := parseShreds(nil, rawShreds, 2)
|
||||
meta := &SlotMeta{
|
||||
Consumed: 1427,
|
||||
Received: 1427,
|
||||
LastIndex: 1426,
|
||||
NumEntryEndIndexes: 574,
|
||||
EntryEndIndexes: mainnet_102815960_EntryEndIndexes,
|
||||
}
|
||||
b.ResetTimer()
|
||||
b.ReportAllocs()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, err := DataShredsToEntries(meta, shreds)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
}
|
|
@ -29,399 +29,327 @@ func TestDataShredsToEntries_Mainnet_Genesis(t *testing.T) {
|
|||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("BEgmsE3RjuKda9PpnF8r5Ch4HJMHttEhTt38jsyuKuaV"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("7GSbH4g5eTo5jehuAcQmKuoFnTNaYdFRWYJraMCYJ4pY"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("cXQPc1rqMHwxtDfLEJ9CdFWguD9LMnuKaAadwgnoAU9"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("GsdubVEKzBBxdwpRrYPsCoF6TxkwLkJC7vaoAcWvQ6zj"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("GVPeccPmHPAD7sN5za67jvuvRbyJDrZMdFPJgc5k4ApF"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("HCKdcqbc9xQjg4dAqRtJSte4teNRNGuRdQaXbcFqwmsi"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("8JC6mNJbS8JFEm7bhEgkRzk9n6aueX49wXm1hKbMRXtD"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("2weUJw23EwjYkDdH5TWP4iBwgXPW88YDgtLqFUnq5iW2"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("HDUsEkxvxps4KZHHcg1g1Kvkx6xHHQEjb6diEWEbykLs"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("EW44ssuR2hgVYr22WYvRhyWpSwvqpj9FqR268EmyWrp1"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("EZ8KLZWb4fJTqty1PYjmiZjBa7LG1vZiVP385Tf1wGSv"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("4rFoMK6rVXTwKcyeG7GKg7CGTTJpUaLV6nfk4aApHncA"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("2XfkNnbeLgiWFJG88cDvie63Ay4DhF9yVhpaLwVmZvYM"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("Dv8SnjbyB3FHhoct34FNUmsxzgkVTG82nxDaMKLXnHZv"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("8eMT19ctrhoh7buqug6FL5yaDybEhpdGspEfW4Vmr8B2"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("9N8vjyY7mV6Ln77LaG4MZJWdGDApi6gw2Mf3EZ7h74AM"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("AVMHML49UUPxhcVY2QEpULbNXHNxfEHKj1J2cAbEs9Vp"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("DhfgF92wssf9sussfcf2A9gZ9rTJs9k4CuDKf5K9aoMm"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("Hzg7Ptw7fH8gqfaN5FjADcB4kw1XyfZgeCTJhcz9Dt9Y"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("7UDgQWeCGsdjgKDkLDgQ8TA5uuWhUnk6Yf3UwjKRAvS4"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("2ZDGg77CJd9PrGakiApavtY8nTwM6a35ENbuLe1sLXqJ"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("GUDUf6Dj2PZ9qsySXE85qWRj2Ax7peVnTh8AQa4wEHq2"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("FBWpsKo4XZ5jxHh2L3yr17jpe9JdZd7wTDzskA6zYUQW"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("8o6zDK3dC1rJ6UvaZazcYRACTX1zjAvUsGjua2YMWJtX"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("93krGb8CjfAzFWXbcXMUgaiAkLuWQ9wPiwpgG4L153Ji"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("14ngqzcDAwQSkGyYXbV8KHtMtQSup7FFSa7qHS74qS65"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("EdokMAJe1FEkXJeoL4CB1FvKMan63PJ3AgjZzLT5ZHF4"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("2gJtJPAmHY4VCQkF7EmpsBGNRi3122uFebcPJZAVvozf"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("ALffShUMdNB2548HAKU5dwdd1K8JNiApytn8RBKFA5MY"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("3YEHfjSheJX87bME3wNvXN6kEfuHHwsxn7EjaMTRFE9Z"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("2t1F4YsuJSYHNwougVtqHgJHZ7yBswcKpxJnUUYigE1Q"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("8vXxmsQaMQqE3zMmS1FF3UHDho9rVY6sotp5BSNEbS9X"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("CZuyDW5rZFcDHChQRS6FAAcTQHjt5Xh1CL6sKQoMPFND"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("5gws32wDVTpuFrNhCqZwxxu5B8CKvSW9yf4VrqbNWjH7"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("5PJC6VVWQnvqaioJ2wevRFhmoQNCBX9s5h1Kbj6JE5no"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("6WAAcuMZHv1rd1b6v92fqny4dsGSY7Hy6EJpGMmAAznh"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("5joYwpeat27k6XXs3NzJfV7hkw4juX7CCUZvb98dTWJt"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("CNqKzPgR3HUEdf7j6bqj63z86ov5oPTjEitJX8vMucsP"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("HTCaWhSSNviX6YDAyMteKpcgQkAY4GzaMvmKXdkqsvx5"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("4yEgDCGrWidGYjWXL5B3Hcfg7GhwLUdpBYbausUxT2Md"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("HmVzH2rs2oMw2jka3faNT4StoUQhKEVNRsucSbFJgUzQ"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("8UiBLqWrzUhEEtEbf4UhtikFD4qWqzE8aGoimbubT33r"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("uw3ierG673dDLNHTLBAYk6vGp2XzP6MnabjzasqkRLg"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("HkVC9Dm1xm2A4tiFtqodjRBawX38Y4WhPHn5dxDLHowT"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("FMfLjob5dfTmXPuY97TuuWBRZk4qEzkYao6TdZQXSNU9"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("6brZ9hfpcpqq9AUNSa1WYrDWwUazR7uUYDutNn7AxrYq"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("GddzudgJRbP7pZYw7s62VwBzeyjQTcKC6mkN8Ke7Vc6p"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("Gwk1E6CeD1a4Az9Y6Dw9gm1cXWnYFAGioj3UrmttPFGs"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("497LZxPFN8UMparCsDRM2Xwctz9Yr6ggRRxrRWYiYUnr"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("GgV4yq8mU1H8DPqegPqEoyU66jc8dvrrvy5P1kURpUmB"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("mgHymgwdvc5fNukape9sMCAd5HTKoEQQTwxNNtb8Rat"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("2LPLYYGtvfzVBXS7nYYYhBezo36PpvpPJR3TrNfAj3tB"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("FsHZMmXjqNd11HHuCFUTifCC1PzDa5Qa73Et1Pq5CgEN"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("8uq7LJ8GNCoNcsTBgqWLNnxAmR9G2954fdJLw7d22ze8"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("3LaPqPMbS57jESrHL6j8i4CDSCFqioHYM4nw8pMHkyH7"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("5H8mKSpdwXfa7B7NcS7NSRA5qFvmYsnSmfFj1URwF9VV"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("C1dX3PxkzPUeRX4WhsLPgULnisGHWtx1iEw7gTUqbEQe"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("CPosYdr3P8mndtQXF4zkr11KcVuXucMjqrvL7sxJjz4L"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("GrLSqSMZPyeQDySx9RGmFwhgLa67jrhEXsVpZuSwQ51c"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("CBmrfRRb2zghqbHyeiPdfXWGrC5naMT3USR35aeLVYRH"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("4EzStQd1KbsBrmHhfR5r8a6NhaYFAN5bi7t5tVmEMh5U"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("ExLNu3PUdfsyq4DCWYb7VaafoBguxj4bRgyxbfuACQaq"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("Ap9zjPARQp8PRFGFirio2WP7zghbALResHdS9u4FwXZr"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
{
|
||||
NumHashes: 12500,
|
||||
Hash: solana.MustHashFromBase58("4sGjMW1sUnHzSxGspuhpqLDx6wiyjNtZAMdL4VZHirAn"),
|
||||
NumTxns: 0,
|
||||
Txns: []solana.Transaction{},
|
||||
},
|
||||
}, entries[0].Entries)
|
||||
}
|
||||
|
||||
func TestDataShredsToEntries_Mainnet_Recent(t *testing.T) {
|
||||
rawShreds := fixtures.DataShreds(t, "mainnet", 102815960)
|
||||
shreds := parseShreds(t, rawShreds, 2)
|
||||
meta := &SlotMeta{
|
||||
Consumed: 1427,
|
||||
Received: 1427,
|
||||
LastIndex: 1426,
|
||||
NumEntryEndIndexes: 574,
|
||||
EntryEndIndexes: []uint32{
|
||||
var mainnet_102815960_EntryEndIndexes = []uint32{
|
||||
0, 1, 2, 3, 5, 16, 17, 18, 20, 21, 22, 23, 24, 26, 28, 30, 31, 34, 35, 37, 38, 39, 41, 42, 43, 44, 45, 46,
|
||||
47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 59, 60, 62, 63, 69, 74, 75, 77, 81, 82, 83, 86, 87, 88, 89, 90, 91,
|
||||
92, 93, 94, 95, 101, 102, 103, 104, 105, 107, 108, 114, 115, 116, 117, 123, 124, 127, 133, 136, 137, 138,
|
||||
|
@ -451,11 +379,69 @@ func TestDataShredsToEntries_Mainnet_Recent(t *testing.T) {
|
|||
1347, 1350, 1351, 1352, 1353, 1354, 1355, 1356, 1357, 1358, 1359, 1360, 1367, 1368, 1369, 1370, 1371, 1372,
|
||||
1373, 1374, 1379, 1384, 1385, 1386, 1388, 1389, 1394, 1395, 1396, 1397, 1398, 1403, 1404, 1407, 1416, 1417,
|
||||
1422, 1423, 1425, 1426,
|
||||
},
|
||||
}
|
||||
|
||||
func TestDataShredsToEntries_Mainnet_Recent(t *testing.T) {
|
||||
rawShreds := fixtures.DataShreds(t, "mainnet", 102815960)
|
||||
shreds := parseShreds(t, rawShreds, 2)
|
||||
meta := &SlotMeta{
|
||||
Consumed: 1427,
|
||||
Received: 1427,
|
||||
LastIndex: 1426,
|
||||
NumEntryEndIndexes: 574,
|
||||
EntryEndIndexes: mainnet_102815960_EntryEndIndexes,
|
||||
}
|
||||
entries, err := DataShredsToEntries(meta, shreds)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 574, len(entries))
|
||||
{
|
||||
transactions := findTransactions(entries, 10)
|
||||
assert.Equal(t, 10, len(transactions))
|
||||
assert.Equal(t,
|
||||
solana.MustSignatureFromBase58("5ruUBRn3CNNcFneJtEaJBaUjhtHtEb6Q9Hu2ygTSA9TjLr6FtE2WGGBkt7nUAHc1fnBaL8DGMS6maeCYHpRzuY6K"),
|
||||
transactions[0].Signatures[0],
|
||||
)
|
||||
assert.Equal(t,
|
||||
solana.MustSignatureFromBase58("REnUxGXrE22u5KsxynCoMQDuVY6eyAZFPyxg8HccHJQtr3GXQ6eA4T2ZhZyfC1NgbR1wt9EiRjYaWNgtDCSTwim"),
|
||||
transactions[1].Signatures[0],
|
||||
)
|
||||
assert.Equal(t,
|
||||
solana.MustSignatureFromBase58("5YfaAQEZYrU84ftEGSW8i97ZfQ5aezZrvazTu6VCHFJ9xWUxYo6WwXV86diMNt1LRSNho3Ev6kvzHN9eN8z488NW"),
|
||||
transactions[2].Signatures[0],
|
||||
)
|
||||
assert.Equal(t,
|
||||
solana.MustSignatureFromBase58("662prxNx8sgR7s3vT7Xkzj4uYG72xjA9frBfm4QFZNqbGQEhFMNjCti7zBDBnWDTjMZKV1bo53rxDrrwP9D1kGHR"),
|
||||
transactions[3].Signatures[0],
|
||||
)
|
||||
assert.Equal(t,
|
||||
solana.MustSignatureFromBase58("43yAFJnaZm5caXfUvzXDKaeYhgYzfBYvHpwzTiEhs1uRHDWibVGSTS2eh6eZiYaZgQbjkYCR7dZ5TsKwTYFNXHi1"),
|
||||
transactions[4].Signatures[0],
|
||||
)
|
||||
assert.Equal(t,
|
||||
solana.MustSignatureFromBase58("2wbiisDTsTQnWedDogjTU2qV2frLPt7GeS1wbn7TzziYxTU7MHCAP7po8s8yRbfU6Y59uY9hdFUKwxZK7ehZYUAW"),
|
||||
transactions[5].Signatures[0],
|
||||
)
|
||||
assert.Equal(t,
|
||||
solana.MustSignatureFromBase58("3JbSLxaNTVTdhSp2HLjxbqNiWxdPz6dcd3Xw5sFnzyG5yFQehLQVi6UjPZ68msJALxSgDJWX9xm8YjYC9CUJMVj"),
|
||||
transactions[6].Signatures[0],
|
||||
)
|
||||
assert.Equal(t,
|
||||
solana.MustSignatureFromBase58("2oZxvCcSqKG719R8MNguXvnRTCcUHAbyn14gnYEskB9SE1bhvzKBhrWFDAx6VyE3ac2JnzrBUcZvJLmWBK7EJ4XJ"),
|
||||
transactions[7].Signatures[0],
|
||||
)
|
||||
assert.Equal(t,
|
||||
solana.MustSignatureFromBase58("4GFhtws8R3jD68t45fAKmoUUKtEHoicKyNK53mpgTJ778NSepZnGdZminXTmX2GzMNpqsft23VrZVn3qrHseEo7c"),
|
||||
transactions[8].Signatures[0],
|
||||
)
|
||||
assert.Equal(t,
|
||||
solana.MustSignatureFromBase58("2p5jNjuLeMufeDH4euwNXqb9fi3gFH7R6cprPVVzGozM6R7qhXTuahN8gm2J5mjHk1NJdb7WTG9a6GturYpWUPhq"),
|
||||
transactions[9].Signatures[0],
|
||||
)
|
||||
}
|
||||
{
|
||||
transactions := findTransactions(entries, 4000)
|
||||
assert.Equal(t, 3177, len(transactions))
|
||||
}
|
||||
}
|
||||
|
||||
func parseShreds(t testing.TB, raw [][]byte, version int) (shreds []shred.Shred) {
|
||||
|
@ -468,3 +454,19 @@ func parseShreds(t testing.TB, raw [][]byte, version int) (shreds []shred.Shred)
|
|||
}
|
||||
return shreds
|
||||
}
|
||||
|
||||
func findTransactions(entries []Entries, limit int) []solana.Transaction {
|
||||
transactions := make([]solana.Transaction, 0, limit)
|
||||
for _, entry := range entries {
|
||||
for _, e := range entry.Entries {
|
||||
for txIndex := range e.Txns {
|
||||
tx := e.Txns[txIndex]
|
||||
transactions = append(transactions, tx)
|
||||
if len(transactions) >= limit {
|
||||
return transactions
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return transactions
|
||||
}
|
||||
|
|
|
@ -1,6 +1,11 @@
|
|||
package shred
|
||||
|
||||
import "github.com/gagliardetto/solana-go"
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
bin "github.com/gagliardetto/binary"
|
||||
"github.com/gagliardetto/solana-go"
|
||||
)
|
||||
|
||||
type Shred interface {
|
||||
CommonHeader() *CommonHeader
|
||||
|
@ -79,6 +84,33 @@ func (d *DataV2Header) LastInSlot() bool {
|
|||
type Entry struct {
|
||||
NumHashes uint64
|
||||
Hash solana.Hash
|
||||
NumTxns uint64 `bin:"sizeof=Txns"`
|
||||
Txns []solana.Transaction
|
||||
}
|
||||
|
||||
func (en *Entry) UnmarshalWithDecoder(decoder *bin.Decoder) (err error) {
|
||||
// read the number of hashes:
|
||||
if en.NumHashes, err = decoder.ReadUint64(bin.LE); err != nil {
|
||||
return fmt.Errorf("failed to read number of hashes: %w", err)
|
||||
}
|
||||
// read the hash:
|
||||
_, err = decoder.Read(en.Hash[:])
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read hash: %w", err)
|
||||
}
|
||||
// read the number of transactions:
|
||||
numTxns, err := decoder.ReadUint64(bin.LE)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read number of transactions: %w", err)
|
||||
}
|
||||
if numTxns > uint64(decoder.Remaining()) {
|
||||
return fmt.Errorf("not enough bytes to read %d transactions", numTxns)
|
||||
}
|
||||
// read the transactions:
|
||||
en.Txns = make([]solana.Transaction, numTxns)
|
||||
for i := uint64(0); i < numTxns; i++ {
|
||||
if err = en.Txns[i].UnmarshalWithDecoder(decoder); err != nil {
|
||||
return fmt.Errorf("failed to read transaction %d: %w", i, err)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue