From 275d51581c726bb77b3bada8e9dbe38dae10a489 Mon Sep 17 00:00:00 2001 From: Richard Patel Date: Thu, 8 Sep 2022 10:30:54 +0200 Subject: [PATCH] gossip: port more bloom tests --- pkg/gossip/bloom.go | 11 ++++--- pkg/gossip/bloom_test.go | 68 ++++++++++++++++++++++++++++++++++++++++ pkg/gossip/schema.go | 40 +++++++++++++++++++++-- pkg/gossip/schema.yaml | 2 +- 4 files changed, 112 insertions(+), 9 deletions(-) diff --git a/pkg/gossip/bloom.go b/pkg/gossip/bloom.go index 051c8b4..01578a5 100644 --- a/pkg/gossip/bloom.go +++ b/pkg/gossip/bloom.go @@ -15,7 +15,7 @@ import ( ) func NewBloom(numBits uint64, keys []uint64) *Bloom { - bits := make([]byte, (numBits+7)/8) + bits := make([]uint64, (numBits+63)/64) ret := &Bloom{ Keys: keys, Bits: BitVecU64{ @@ -33,7 +33,7 @@ func NewBloomRandom(numItems uint64, falseRate float64, maxBits uint64) *Bloom { if maxBits < numBits { numBits = maxBits } - if maxBits == 0 { + if numBits == 0 { numBits = 1 } numKeys := uint64(BloomNumKeys(float64(numBits), float64(numItems))) @@ -80,7 +80,7 @@ func (b *Bloom) Add(key *[32]byte) { func (b *Bloom) Contains(key *[32]byte) bool { for _, k := range b.Keys { if !b.Bits.Get(b.Pos(key, k)) { - return true + return false } } return true @@ -123,9 +123,10 @@ func (bv *BitVecU64) Set(pos uint64, b bool) { if pos >= bv.Len { panic("get bit out of bounds") } + bits := *bv.Bits.Value if b { - (*bv.Bits.Value)[pos/64] |= 1 << (pos % 64) + bits[pos/64] |= 1 << (pos % 64) } else { - (*bv.Bits.Value)[pos/64] &= ^uint8(1 << (pos % 64)) + bits[pos/64] &= ^uint64(1 << (pos % 64)) } } diff --git a/pkg/gossip/bloom_test.go b/pkg/gossip/bloom_test.go index 5dc565e..eda8ed1 100644 --- a/pkg/gossip/bloom_test.go +++ b/pkg/gossip/bloom_test.go @@ -1,11 +1,51 @@ package gossip import ( + "crypto/sha256" + "sort" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +func TestNewBloomRandom(t *testing.T) { + cases := []struct { + name string + numItems uint64 + falseRate float64 + maxBits uint64 + + wantKeys int + wantBits uint64 + }{ + { + name: "Empty", + numItems: 0, falseRate: 0.1, maxBits: 100, + wantKeys: 0, wantBits: 1, + }, + { + name: "Random", + numItems: 10, falseRate: 0.1, maxBits: 100, + wantKeys: 3, wantBits: 48, + }, + { + name: "Random", + numItems: 100, falseRate: 0.1, maxBits: 100, + wantKeys: 1, wantBits: 100, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + bloom := NewBloomRandom(tc.numItems, tc.falseRate, tc.maxBits) + require.NotNil(t, bloom) + assert.Equal(t, tc.wantKeys, len(bloom.Keys)) + assert.Equal(t, tc.wantBits, bloom.Bits.Len) + }) + } +} + func TestBloom_FilterMath(t *testing.T) { assert.Equal(t, uint64(480), uint64(BloomNumBits(100, 0.1))) assert.Equal(t, uint64(959), uint64(BloomNumBits(100, 0.01))) @@ -16,4 +56,32 @@ func TestBloom_FilterMath(t *testing.T) { } func TestBloom_AddContains(t *testing.T) { + bloom := NewBloomRandom(100, 0.1, 100) + require.NotNil(t, bloom) + // known keys to avoid false positives in the test + bloom.Keys = []uint64{0, 1, 2, 3} + + var key [32]byte + + key = sha256.Sum256([]byte("hello")) + assert.False(t, bloom.Contains(&key)) + bloom.Add(&key) + assert.True(t, bloom.Contains(&key)) + + key = sha256.Sum256([]byte("world")) + assert.False(t, bloom.Contains(&key)) + bloom.Add(&key) + assert.True(t, bloom.Contains(&key)) +} + +func TestBloom_Randomness(t *testing.T) { + b1 := NewBloomRandom(10, 0.1, 100) + b2 := NewBloomRandom(10, 0.1, 100) + require.NotNil(t, b1) + require.NotNil(t, b2) + + sort.Slice(b1.Keys, func(i, j int) bool { return b1.Keys[i] < b1.Keys[j] }) + sort.Slice(b2.Keys, func(i, j int) bool { return b2.Keys[i] < b2.Keys[j] }) + + assert.NotEqual(t, b1.Keys, b2.Keys) } diff --git a/pkg/gossip/schema.go b/pkg/gossip/schema.go index 5440ccf..ce5efe5 100644 --- a/pkg/gossip/schema.go +++ b/pkg/gossip/schema.go @@ -71,14 +71,14 @@ func BincodeDeserializeBitVecU64(input []byte) (BitVecU64, error) { } type BitVecU64Inner struct { - Value *[]uint8 + Value *[]uint64 } func (obj *BitVecU64Inner) Serialize(serializer serde.Serializer) error { if err := serializer.IncreaseContainerDepth(); err != nil { return err } - if err := serialize_option_vector_u8(obj.Value, serializer); err != nil { + if err := serialize_option_vector_u64(obj.Value, serializer); err != nil { return err } serializer.DecreaseContainerDepth() @@ -101,7 +101,7 @@ func DeserializeBitVecU64Inner(deserializer serde.Deserializer) (BitVecU64Inner, if err := deserializer.IncreaseContainerDepth(); err != nil { return obj, err } - if val, err := deserialize_option_vector_u8(deserializer); err == nil { + if val, err := deserialize_option_vector_u64(deserializer); err == nil { obj.Value = val } else { return obj, err @@ -3314,6 +3314,40 @@ func deserialize_option_u64(deserializer serde.Deserializer) (*uint64, error) { } } +func serialize_option_vector_u64(value *[]uint64, serializer serde.Serializer) error { + if value != nil { + if err := serializer.SerializeOptionTag(true); err != nil { + return err + } + if err := serialize_vector_u64((*value), serializer); err != nil { + return err + } + } else { + if err := serializer.SerializeOptionTag(false); err != nil { + return err + } + } + return nil +} + +func deserialize_option_vector_u64(deserializer serde.Deserializer) (*[]uint64, error) { + tag, err := deserializer.DeserializeOptionTag() + if err != nil { + return nil, err + } + if tag { + value := new([]uint64) + if val, err := deserialize_vector_u64(deserializer); err == nil { + *value = val + } else { + return nil, err + } + return value, nil + } else { + return nil, nil + } +} + func serialize_option_vector_u8(value *[]uint8, serializer serde.Serializer) error { if value != nil { if err := serializer.SerializeOptionTag(true); err != nil { diff --git a/pkg/gossip/schema.yaml b/pkg/gossip/schema.yaml index 5949f92..fc549e9 100644 --- a/pkg/gossip/schema.yaml +++ b/pkg/gossip/schema.yaml @@ -142,7 +142,7 @@ BitVecU64: BitVecU64Inner: NEWTYPESTRUCT: OPTION: - SEQ: U8 + SEQ: U64 # ------------------------ # CRDTs