diff --git a/pkg/gossip/bloom.go b/pkg/gossip/bloom.go index 66938af..2e295ef 100644 --- a/pkg/gossip/bloom.go +++ b/pkg/gossip/bloom.go @@ -55,7 +55,15 @@ func BloomNumKeys(m, n float64) float64 { return math.Max(1, math.Round((m/n)*math.Log(2))) } -func (b *Bloom) Pos(key *[32]byte, k uint64) uint64 { +func BloomMaxItems(m, p, k float64) float64 { + return math.Ceil(m / (-k / math.Log(1-math.Exp(math.Log(p)/k)))) +} + +func BloomMaskBits(numItems, maxItems float64) uint32 { + return uint32(math.Max(math.Ceil(math.Log2(numItems/maxItems)), 0)) +} + +func (b *Bloom) Pos(key *Hash, k uint64) uint64 { return FNV1a(key[:], k) % b.Bits.Len } @@ -67,7 +75,7 @@ func (b *Bloom) Clear() { b.NumBitsSet = 0 } -func (b *Bloom) Add(key *[32]byte) { +func (b *Bloom) Add(key *Hash) { for _, k := range b.Keys { pos := b.Pos(key, k) if !b.Bits.Get(pos) { @@ -77,7 +85,7 @@ func (b *Bloom) Add(key *[32]byte) { } } -func (b *Bloom) Contains(key *[32]byte) bool { +func (b *Bloom) Contains(key *Hash) bool { for _, k := range b.Keys { if !b.Bits.Get(b.Pos(key, k)) { return false diff --git a/pkg/gossip/bloom_test.go b/pkg/gossip/bloom_test.go index eda8ed1..ea21b99 100644 --- a/pkg/gossip/bloom_test.go +++ b/pkg/gossip/bloom_test.go @@ -61,7 +61,7 @@ func TestBloom_AddContains(t *testing.T) { // known keys to avoid false positives in the test bloom.Keys = []uint64{0, 1, 2, 3} - var key [32]byte + var key Hash key = sha256.Sum256([]byte("hello")) assert.False(t, bloom.Contains(&key)) @@ -85,3 +85,7 @@ func TestBloom_Randomness(t *testing.T) { assert.NotEqual(t, b1.Keys, b2.Keys) } + +func TestBloom_MaxItems(t *testing.T) { + assert.Equal(t, float64(9), BloomMaxItems(80, 0.01, 8)) +} diff --git a/pkg/gossip/crds.go b/pkg/gossip/crds.go new file mode 100644 index 0000000..00d2322 --- /dev/null +++ b/pkg/gossip/crds.go @@ -0,0 +1,74 @@ +package gossip + +import ( + "crypto/ed25519" + "encoding/binary" + "math" +) + +// CrdsBloomP is bloom filter 'p' parameter (probability) +const CrdsBloomP = 0.1 + +func (f *CrdsFilter) Contains(item *Hash) bool { + if !f.TestMask(item) { + return false + } + return f.Filter.Contains(item) +} + +func (f *CrdsFilter) TestMask(item *Hash) bool { + ones := uint64(math.MaxUint64) >> f.MaskBits + bits := binary.LittleEndian.Uint64(item[:8]) | ones + return bits == f.Mask +} + +type CrdsFilterSet []CrdsFilter + +func NewCrdsFilterSet(numItems, maxBytes uint64) CrdsFilterSet { + maxBits := maxBytes * 8 + maxItems := BloomMaxItems(float64(maxBits), CrdsBloomP, 8) + maskBits := BloomMaskBits(float64(numItems), maxItems) + filters := make([]CrdsFilter, 1<> maskBits), + MaskBits: maskBits, + } + } + return filters +} + +func (c CrdsFilterSet) Add(h Hash) { + index := binary.LittleEndian.Uint64(h[:8]) + index >>= 64 - c[0].MaskBits + c[index].Filter.Add(&h) +} + +func (c *CrdsValue) Sign(identity ed25519.PrivateKey) error { + // Write pubkey into data field + pubkey := ed25519.PublicKey(c.Data.Pubkey()[:]) + copy( + pubkey[:ed25519.PublicKeySize], + identity.Public().(ed25519.PublicKey)[:ed25519.PublicKeySize], + ) + + msg, err := c.Data.BincodeSerialize() + if err != nil { + return err + } + sig := ed25519.Sign(identity, msg) + copy(c.Signature[:ed25519.SignatureSize], sig[:ed25519.SignatureSize]) + return nil +} + +func (c *CrdsValue) VerifySignature() bool { + msg, err := c.Data.BincodeSerialize() + if err != nil { + return false + } + pubkey := ed25519.PublicKey(c.Data.Pubkey()[:]) + return ed25519.Verify(pubkey, msg, c.Signature[:]) +} diff --git a/pkg/gossip/crds_test.go b/pkg/gossip/crds_test.go new file mode 100644 index 0000000..33672a3 --- /dev/null +++ b/pkg/gossip/crds_test.go @@ -0,0 +1,46 @@ +package gossip + +import ( + "math" + "math/rand" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewCrdsFilterSet(t *testing.T) { + filters := NewCrdsFilterSet(55345017, 4098) + assert.Equal(t, 16384, len(filters)) + maskBits := filters[0].MaskBits + rightShift := 64 - maskBits + ones := uint64(math.MaxUint64) >> maskBits + for i, filter := range filters { + assert.Equal(t, maskBits, filter.MaskBits) + assert.Equal(t, uint64(i), filter.Mask>>rightShift) + assert.Equal(t, ones, ones&filter.Mask) + } +} + +func TestCrdsFilterSet_Add(t *testing.T) { + filters := NewCrdsFilterSet(9672788, 8196) + hashValues := make([]Hash, 1024) + for i := range hashValues { + rand.Read(hashValues[i][:]) + filters.Add(hashValues[i]) + } + for _, hashValue := range hashValues { + numHits := uint(0) + falsePositives := uint(0) + for _, filter := range filters { + if filter.TestMask(&hashValue) { + numHits++ + assert.True(t, filter.Contains(&hashValue)) + assert.True(t, filter.Filter.Contains(&hashValue)) + } else if filter.Filter.Contains(&hashValue) { + falsePositives++ + } + } + assert.Equal(t, numHits, uint(1)) + assert.Less(t, falsePositives, uint(5)) + } +}