gossip: add CrdsFilter

pkg/gossip/bloom.go

@@ -55,7 +55,15 @@ func BloomNumKeys(m, n float64) float64 {
 	return math.Max(1, math.Round((m/n)*math.Log(2)))
 }
 
-func (b *Bloom) Pos(key *[32]byte, k uint64) uint64 {
+func BloomMaxItems(m, p, k float64) float64 {
+	return math.Ceil(m / (-k / math.Log(1-math.Exp(math.Log(p)/k))))
+}
+
+func BloomMaskBits(numItems, maxItems float64) uint32 {
+	return uint32(math.Max(math.Ceil(math.Log2(numItems/maxItems)), 0))
+}
+
+func (b *Bloom) Pos(key *Hash, k uint64) uint64 {
 	return FNV1a(key[:], k) % b.Bits.Len
 }
 
@@ -67,7 +75,7 @@ func (b *Bloom) Clear() {
 	b.NumBitsSet = 0
 }
 
-func (b *Bloom) Add(key *[32]byte) {
+func (b *Bloom) Add(key *Hash) {
 	for _, k := range b.Keys {
 		pos := b.Pos(key, k)
 		if !b.Bits.Get(pos) {
@@ -77,7 +85,7 @@ func (b *Bloom) Add(key *[32]byte) {
 	}
 }
 
-func (b *Bloom) Contains(key *[32]byte) bool {
+func (b *Bloom) Contains(key *Hash) bool {
 	for _, k := range b.Keys {
 		if !b.Bits.Get(b.Pos(key, k)) {
 			return false

pkg/gossip/bloom_test.go

@@ -61,7 +61,7 @@ func TestBloom_AddContains(t *testing.T) {
 	// known keys to avoid false positives in the test
 	bloom.Keys = []uint64{0, 1, 2, 3}
 
-	var key [32]byte
+	var key Hash
 
 	key = sha256.Sum256([]byte("hello"))
 	assert.False(t, bloom.Contains(&key))
@@ -85,3 +85,7 @@ func TestBloom_Randomness(t *testing.T) {
 
 	assert.NotEqual(t, b1.Keys, b2.Keys)
 }
+
+func TestBloom_MaxItems(t *testing.T) {
+	assert.Equal(t, float64(9), BloomMaxItems(80, 0.01, 8))
+}

pkg/gossip/crds.go

@@ -0,0 +1,74 @@
+package gossip
+
+import (
+	"crypto/ed25519"
+	"encoding/binary"
+	"math"
+)
+
+// CrdsBloomP is bloom filter 'p' parameter (probability)
+const CrdsBloomP = 0.1
+
+func (f *CrdsFilter) Contains(item *Hash) bool {
+	if !f.TestMask(item) {
+		return false
+	}
+	return f.Filter.Contains(item)
+}
+
+func (f *CrdsFilter) TestMask(item *Hash) bool {
+	ones := uint64(math.MaxUint64) >> f.MaskBits
+	bits := binary.LittleEndian.Uint64(item[:8]) | ones
+	return bits == f.Mask
+}
+
+type CrdsFilterSet []CrdsFilter
+
+func NewCrdsFilterSet(numItems, maxBytes uint64) CrdsFilterSet {
+	maxBits := maxBytes * 8
+	maxItems := BloomMaxItems(float64(maxBits), CrdsBloomP, 8)
+	maskBits := BloomMaskBits(float64(numItems), maxItems)
+	filters := make([]CrdsFilter, 1<<maskBits)
+	for i := uint64(0); i < uint64(len(filters)); i++ {
+		bloom := NewBloomRandom(uint64(maxItems), CrdsBloomP, maxBits)
+		seed := i << (64 - maskBits)
+		filters[i] = CrdsFilter{
+			Filter:   *bloom,
+			Mask:     seed | (math.MaxUint64 >> maskBits),
+			MaskBits: maskBits,
+		}
+	}
+	return filters
+}
+
+func (c CrdsFilterSet) Add(h Hash) {
+	index := binary.LittleEndian.Uint64(h[:8])
+	index >>= 64 - c[0].MaskBits
+	c[index].Filter.Add(&h)
+}
+
+func (c *CrdsValue) Sign(identity ed25519.PrivateKey) error {
+	// Write pubkey into data field
+	pubkey := ed25519.PublicKey(c.Data.Pubkey()[:])
+	copy(
+		pubkey[:ed25519.PublicKeySize],
+		identity.Public().(ed25519.PublicKey)[:ed25519.PublicKeySize],
+	)
+
+	msg, err := c.Data.BincodeSerialize()
+	if err != nil {
+		return err
+	}
+	sig := ed25519.Sign(identity, msg)
+	copy(c.Signature[:ed25519.SignatureSize], sig[:ed25519.SignatureSize])
+	return nil
+}
+
+func (c *CrdsValue) VerifySignature() bool {
+	msg, err := c.Data.BincodeSerialize()
+	if err != nil {
+		return false
+	}
+	pubkey := ed25519.PublicKey(c.Data.Pubkey()[:])
+	return ed25519.Verify(pubkey, msg, c.Signature[:])
+}

pkg/gossip/crds_test.go

@@ -0,0 +1,46 @@
+package gossip
+
+import (
+	"math"
+	"math/rand"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNewCrdsFilterSet(t *testing.T) {
+	filters := NewCrdsFilterSet(55345017, 4098)
+	assert.Equal(t, 16384, len(filters))
+	maskBits := filters[0].MaskBits
+	rightShift := 64 - maskBits
+	ones := uint64(math.MaxUint64) >> maskBits
+	for i, filter := range filters {
+		assert.Equal(t, maskBits, filter.MaskBits)
+		assert.Equal(t, uint64(i), filter.Mask>>rightShift)
+		assert.Equal(t, ones, ones&filter.Mask)
+	}
+}
+
+func TestCrdsFilterSet_Add(t *testing.T) {
+	filters := NewCrdsFilterSet(9672788, 8196)
+	hashValues := make([]Hash, 1024)
+	for i := range hashValues {
+		rand.Read(hashValues[i][:])
+		filters.Add(hashValues[i])
+	}
+	for _, hashValue := range hashValues {
+		numHits := uint(0)
+		falsePositives := uint(0)
+		for _, filter := range filters {
+			if filter.TestMask(&hashValue) {
+				numHits++
+				assert.True(t, filter.Contains(&hashValue))
+				assert.True(t, filter.Filter.Contains(&hashValue))
+			} else if filter.Filter.Contains(&hashValue) {
+				falsePositives++
+			}
+		}
+		assert.Equal(t, numHits, uint(1))
+		assert.Less(t, falsePositives, uint(5))
+	}
+}