From 3a4ffb48505cce59b9a0e5465bca1526c5fd7766 Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Mon, 8 Jun 2020 20:30:03 -0400 Subject: [PATCH 1/7] lower log level for gossiped put messages --- network/network.go | 6 +++++- snow/engine/avalanche/transitive.go | 6 +++++- snow/engine/snowman/transitive.go | 6 +++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/network/network.go b/network/network.go index 9780c07..7bfc4f7 100644 --- a/network/network.go +++ b/network/network.go @@ -39,6 +39,10 @@ const ( defaultGetVersionTimeout = 2 * time.Second defaultAllowPrivateIPs = true defaultGossipSize = 50 + + // Request ID used when sending a Put message to gossip an accepted container + // (ie not sent in response to a Get) + GossipMsgRequestID = math.MaxUint32 ) // Network defines the functionality of the networking library. @@ -620,7 +624,7 @@ func (n *network) Track(ip utils.IPDesc) { // assumes the stateLock is not held. func (n *network) gossipContainer(chainID, containerID ids.ID, container []byte) error { - msg, err := n.b.Put(chainID, math.MaxUint32, containerID, container) + msg, err := n.b.Put(chainID, GossipMsgRequestID, containerID, container) if err != nil { return fmt.Errorf("attempted to pack too large of a Put message.\nContainer length: %d", len(container)) } diff --git a/snow/engine/avalanche/transitive.go b/snow/engine/avalanche/transitive.go index e48b167..37f0c9a 100644 --- a/snow/engine/avalanche/transitive.go +++ b/snow/engine/avalanche/transitive.go @@ -169,7 +169,11 @@ func (t *Transitive) Put(vdr ids.ShortID, requestID uint32, vtxID ids.ID, vtxByt t.Config.Context.Log.Verbo("Put(%s, %d, %s) called", vdr, requestID, vtxID) if !t.bootstrapped { // Bootstrapping unfinished --> didn't call Get --> this message is invalid - t.Config.Context.Log.Debug("dropping Put(%s, %d, %s) due to bootstrapping", vdr, requestID, vtxID) + if requestID == network.GossipMsgRequestID { + t.Config.Context.Log.Verbo("dropping gossip Put(%s, %d, %s) due to bootstrapping", vdr, requestID, vtxID) + } else { + t.Config.Context.Log.Debug("dropping Put(%s, %d, %s) due to bootstrapping", vdr, requestID, vtxID) + } return nil } diff --git a/snow/engine/snowman/transitive.go b/snow/engine/snowman/transitive.go index f155f5e..5461c44 100644 --- a/snow/engine/snowman/transitive.go +++ b/snow/engine/snowman/transitive.go @@ -185,7 +185,11 @@ func (t *Transitive) GetAncestors(vdr ids.ShortID, requestID uint32, blkID ids.I func (t *Transitive) Put(vdr ids.ShortID, requestID uint32, blkID ids.ID, blkBytes []byte) error { // bootstrapping isn't done --> we didn't send any gets --> this put is invalid if !t.bootstrapped { - t.Config.Context.Log.Debug("dropping Put(%s, %d, %s) due to bootstrapping", vdr, requestID, blkID) + if requestID == network.GossipMsgRequestID { + t.Config.Context.Log.Verbo("dropping gossip Put(%s, %d, %s) due to bootstrapping", vdr, requestID, blkID) + } else { + t.Config.Context.Log.Debug("dropping Put(%s, %d, %s) due to bootstrapping", vdr, requestID, blkID) + } return nil } From 8fdeef5eb64f4459ec5f3baf65ccc0e4c90a4aa8 Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Tue, 16 Jun 2020 15:17:13 -0400 Subject: [PATCH 2/7] pre-allocate slices for List in set, bag, shortSet --- ids/bag.go | 17 ++++++++++--- ids/bag_benchmark_test.go | 53 +++++++++++++++++++++++++++++++++++++++ ids/bag_test.go | 4 +-- ids/set.go | 14 +++++++++-- ids/set_benchmark_test.go | 53 +++++++++++++++++++++++++++++++++++++++ ids/short_set.go | 13 ++++++++-- 6 files changed, 144 insertions(+), 10 deletions(-) create mode 100644 ids/bag_benchmark_test.go create mode 100644 ids/set_benchmark_test.go diff --git a/ids/bag.go b/ids/bag.go index de0af46..1d16c64 100644 --- a/ids/bag.go +++ b/ids/bag.go @@ -8,6 +8,10 @@ import ( "strings" ) +const ( + minBagSize = 16 +) + // Bag is a multiset of IDs. // // A bag has the ability to split and filter on it's bits for ease of use for @@ -25,7 +29,7 @@ type Bag struct { func (b *Bag) init() { if b.counts == nil { - b.counts = make(map[[32]byte]int) + b.counts = make(map[[32]byte]int, minBagSize) } } @@ -72,16 +76,21 @@ func (b *Bag) AddCount(id ID, count int) { } // Count returns the number of times the id has been added. -func (b *Bag) Count(id ID) int { return b.counts[*id.ID] } +func (b *Bag) Count(id ID) int { + b.init() + return b.counts[*id.ID] +} // Len returns the number of times an id has been added. func (b *Bag) Len() int { return b.size } // List returns a list of all ids that have been added. func (b *Bag) List() []ID { - idList := []ID(nil) + idList := make([]ID, len(b.counts), len(b.counts)) + i := 0 for id := range b.counts { - idList = append(idList, NewID(id)) + idList[i] = NewID(id) + i++ } return idList } diff --git a/ids/bag_benchmark_test.go b/ids/bag_benchmark_test.go new file mode 100644 index 0000000..e856505 --- /dev/null +++ b/ids/bag_benchmark_test.go @@ -0,0 +1,53 @@ +package ids + +import ( + "crypto/rand" + "testing" +) + +// +func BenchmarkBagListSmall(b *testing.B) { + smallLen := 5 + bag := Bag{} + for i := 0; i < smallLen; i++ { + var idBytes [32]byte + rand.Read(idBytes[:]) + NewID(idBytes) + bag.Add(NewID(idBytes)) + } + b.ResetTimer() + for n := 0; n < b.N; n++ { + bag.List() + } +} + +func BenchmarkBagListMedium(b *testing.B) { + mediumLen := 25 + bag := Bag{} + for i := 0; i < mediumLen; i++ { + var idBytes [32]byte + rand.Read(idBytes[:]) + NewID(idBytes) + bag.Add(NewID(idBytes)) + } + b.ResetTimer() + + for n := 0; n < b.N; n++ { + bag.List() + } +} + +func BenchmarkBagListLarsge(b *testing.B) { + largeLen := 100000 + bag := Bag{} + for i := 0; i < largeLen; i++ { + var idBytes [32]byte + rand.Read(idBytes[:]) + NewID(idBytes) + bag.Add(NewID(idBytes)) + } + b.ResetTimer() + for n := 0; n < b.N; n++ { + bag.List() + } +} diff --git a/ids/bag_test.go b/ids/bag_test.go index ed35233..af0965b 100644 --- a/ids/bag_test.go +++ b/ids/bag_test.go @@ -18,8 +18,8 @@ func TestBagAdd(t *testing.T) { } else if count := bag.Count(id1); count != 0 { t.Fatalf("Bag.Count returned %d expected %d", count, 0) } else if size := bag.Len(); size != 0 { - t.Fatalf("Bag.Len returned %d expected %d", count, 0) - } else if list := bag.List(); list != nil { + t.Fatalf("Bag.Len returned %d elements expected %d", count, 0) + } else if list := bag.List(); len(list) != 0 { t.Fatalf("Bag.List returned %v expected %v", list, nil) } else if mode, freq := bag.Mode(); !mode.IsZero() { t.Fatalf("Bag.Mode[0] returned %s expected %s", mode, ID{}) diff --git a/ids/set.go b/ids/set.go index 9d0b1ec..c3aa024 100644 --- a/ids/set.go +++ b/ids/set.go @@ -7,11 +7,19 @@ import ( "strings" ) +const ( + // The minimum capacity of a set + minSetSize = 16 +) + // Set is a set of IDs type Set map[[32]byte]bool func (ids *Set) init(size int) { if *ids == nil { + if minSetSize > size { + size = minSetSize + } *ids = make(map[[32]byte]bool, size) } } @@ -70,9 +78,11 @@ func (ids *Set) Clear() { *ids = nil } // List converts this set into a list func (ids Set) List() []ID { - idList := []ID(nil) + idList := make([]ID, ids.Len(), ids.Len()) + i := 0 for id := range ids { - idList = append(idList, NewID(id)) + idList[i] = NewID(id) + i++ } return idList } diff --git a/ids/set_benchmark_test.go b/ids/set_benchmark_test.go new file mode 100644 index 0000000..c88b4f9 --- /dev/null +++ b/ids/set_benchmark_test.go @@ -0,0 +1,53 @@ +package ids + +import ( + "crypto/rand" + "testing" +) + +// +func BenchmarkSetListSmall(b *testing.B) { + smallLen := 5 + set := Set{} + for i := 0; i < smallLen; i++ { + var idBytes [32]byte + rand.Read(idBytes[:]) + NewID(idBytes) + set.Add(NewID(idBytes)) + } + b.ResetTimer() + for n := 0; n < b.N; n++ { + set.List() + } +} + +func BenchmarkSetListMedium(b *testing.B) { + mediumLen := 25 + set := Set{} + for i := 0; i < mediumLen; i++ { + var idBytes [32]byte + rand.Read(idBytes[:]) + NewID(idBytes) + set.Add(NewID(idBytes)) + } + b.ResetTimer() + + for n := 0; n < b.N; n++ { + set.List() + } +} + +func BenchmarkSetListLarsge(b *testing.B) { + largeLen := 100000 + set := Set{} + for i := 0; i < largeLen; i++ { + var idBytes [32]byte + rand.Read(idBytes[:]) + NewID(idBytes) + set.Add(NewID(idBytes)) + } + b.ResetTimer() + for n := 0; n < b.N; n++ { + set.List() + } +} diff --git a/ids/short_set.go b/ids/short_set.go index 690cc3a..6977863 100644 --- a/ids/short_set.go +++ b/ids/short_set.go @@ -5,11 +5,18 @@ package ids import "strings" +const ( + minShortSetSize = 16 +) + // ShortSet is a set of ShortIDs type ShortSet map[[20]byte]bool func (ids *ShortSet) init(size int) { if *ids == nil { + if minShortSetSize > size { + size = minShortSetSize + } *ids = make(map[[20]byte]bool, size) } } @@ -65,9 +72,11 @@ func (ids ShortSet) CappedList(size int) []ShortID { // List converts this set into a list func (ids ShortSet) List() []ShortID { - idList := make([]ShortID, len(ids))[:0] + idList := make([]ShortID, len(ids), len(ids)) + i := 0 for id := range ids { - idList = append(idList, NewShortID(id)) + idList[i] = NewShortID(id) + i++ } return idList } From 191cd485935bc1b90d1bd445c248fea03af3becf Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Tue, 16 Jun 2020 15:34:34 -0400 Subject: [PATCH 3/7] add minimum map size to Blocker --- snow/events/blocker.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/snow/events/blocker.go b/snow/events/blocker.go index 6bfdd7b..3491b24 100644 --- a/snow/events/blocker.go +++ b/snow/events/blocker.go @@ -10,12 +10,16 @@ import ( "github.com/ava-labs/gecko/ids" ) +const ( + minBlockerSize = 16 +) + // Blocker tracks objects that are blocked type Blocker map[[32]byte][]Blockable func (b *Blocker) init() { if *b == nil { - *b = make(map[[32]byte][]Blockable) + *b = make(map[[32]byte][]Blockable, minBlockerSize) } } From 4ecd92efba77399ce0c5637ccbe56b3a845fa5d2 Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Tue, 16 Jun 2020 15:43:30 -0400 Subject: [PATCH 4/7] add minimum size to uniqueBag and Requests --- ids/unique_bag.go | 6 +++++- snow/engine/common/requests.go | 8 ++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/ids/unique_bag.go b/ids/unique_bag.go index d5d3e36..461a6bc 100644 --- a/ids/unique_bag.go +++ b/ids/unique_bag.go @@ -8,12 +8,16 @@ import ( "strings" ) +const ( + minUniqueBagSize = 16 +) + // UniqueBag ... type UniqueBag map[[32]byte]BitSet func (b *UniqueBag) init() { if *b == nil { - *b = make(map[[32]byte]BitSet) + *b = make(map[[32]byte]BitSet, minUniqueBagSize) } } diff --git a/snow/engine/common/requests.go b/snow/engine/common/requests.go index 22d5759..51f6cb3 100644 --- a/snow/engine/common/requests.go +++ b/snow/engine/common/requests.go @@ -7,6 +7,10 @@ import ( "github.com/ava-labs/gecko/ids" ) +const ( + minRequestsSize = 32 +) + type req struct { vdr ids.ShortID id uint32 @@ -22,7 +26,7 @@ type Requests struct { // are only in one request at a time. func (r *Requests) Add(vdr ids.ShortID, requestID uint32, containerID ids.ID) { if r.reqsToID == nil { - r.reqsToID = make(map[[20]byte]map[uint32]ids.ID) + r.reqsToID = make(map[[20]byte]map[uint32]ids.ID, minRequestsSize) } vdrKey := vdr.Key() vdrReqs, ok := r.reqsToID[vdrKey] @@ -33,7 +37,7 @@ func (r *Requests) Add(vdr ids.ShortID, requestID uint32, containerID ids.ID) { vdrReqs[requestID] = containerID if r.idToReq == nil { - r.idToReq = make(map[[32]byte]req) + r.idToReq = make(map[[32]byte]req, minRequestsSize) } r.idToReq[containerID.Key()] = req{ vdr: vdr, From 77d24022fefab4a7d8966cd69004e4b2dd1f6c85 Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Tue, 16 Jun 2020 16:11:21 -0400 Subject: [PATCH 5/7] add minimumCacheSize --- cache/lru_cache.go | 8 +++-- cache/lru_cache_benchmark_test.go | 53 +++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 2 deletions(-) create mode 100644 cache/lru_cache_benchmark_test.go diff --git a/cache/lru_cache.go b/cache/lru_cache.go index 629d6bd..ae04138 100644 --- a/cache/lru_cache.go +++ b/cache/lru_cache.go @@ -10,6 +10,10 @@ import ( "github.com/ava-labs/gecko/ids" ) +const ( + minCacheSize = 32 +) + type entry struct { Key ids.ID Value interface{} @@ -59,7 +63,7 @@ func (c *LRU) Flush() { func (c *LRU) init() { if c.entryMap == nil { - c.entryMap = make(map[[32]byte]*list.Element) + c.entryMap = make(map[[32]byte]*list.Element, minCacheSize) } if c.entryList == nil { c.entryList = list.New() @@ -134,6 +138,6 @@ func (c *LRU) evict(key ids.ID) { func (c *LRU) flush() { c.init() - c.entryMap = make(map[[32]byte]*list.Element) + c.entryMap = make(map[[32]byte]*list.Element, minCacheSize) c.entryList = list.New() } diff --git a/cache/lru_cache_benchmark_test.go b/cache/lru_cache_benchmark_test.go new file mode 100644 index 0000000..6bdbaf8 --- /dev/null +++ b/cache/lru_cache_benchmark_test.go @@ -0,0 +1,53 @@ +package cache + +import ( + "crypto/rand" + "testing" + + "github.com/ava-labs/gecko/ids" +) + +func BenchmarkLRUCachePutSmall(b *testing.B) { + smallLen := 5 + cache := &LRU{Size: smallLen} + for n := 0; n < b.N; n++ { + for i := 0; i < smallLen; i++ { + var idBytes [32]byte + rand.Read(idBytes[:]) + cache.Put(ids.NewID(idBytes), n) + } + b.StopTimer() + cache.Flush() + b.StartTimer() + } +} + +func BenchmarkLRUCachePutMedium(b *testing.B) { + mediumLen := 250 + cache := &LRU{Size: mediumLen} + for n := 0; n < b.N; n++ { + for i := 0; i < mediumLen; i++ { + var idBytes [32]byte + rand.Read(idBytes[:]) + cache.Put(ids.NewID(idBytes), n) + } + b.StopTimer() + cache.Flush() + b.StartTimer() + } +} + +func BenchmarkLRUCachePutLarge(b *testing.B) { + largeLen := 10000 + cache := &LRU{Size: largeLen} + for n := 0; n < b.N; n++ { + for i := 0; i < largeLen; i++ { + var idBytes [32]byte + rand.Read(idBytes[:]) + cache.Put(ids.NewID(idBytes), n) + } + b.StopTimer() + cache.Flush() + b.StartTimer() + } +} From aca163714d1dd076fce3fee0f19f350b7487cbed Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Tue, 16 Jun 2020 17:21:23 -0400 Subject: [PATCH 6/7] fixed typo --- ids/bag_benchmark_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ids/bag_benchmark_test.go b/ids/bag_benchmark_test.go index e856505..7007ddb 100644 --- a/ids/bag_benchmark_test.go +++ b/ids/bag_benchmark_test.go @@ -37,7 +37,7 @@ func BenchmarkBagListMedium(b *testing.B) { } } -func BenchmarkBagListLarsge(b *testing.B) { +func BenchmarkBagListLarge(b *testing.B) { largeLen := 100000 bag := Bag{} for i := 0; i < largeLen; i++ { From e0d00e25c720119b46fc9f3001819a19bdef4c47 Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Tue, 16 Jun 2020 17:23:48 -0400 Subject: [PATCH 7/7] fix typo --- ids/set_benchmark_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ids/set_benchmark_test.go b/ids/set_benchmark_test.go index c88b4f9..17c1c7a 100644 --- a/ids/set_benchmark_test.go +++ b/ids/set_benchmark_test.go @@ -37,7 +37,7 @@ func BenchmarkSetListMedium(b *testing.B) { } } -func BenchmarkSetListLarsge(b *testing.B) { +func BenchmarkSetListLarge(b *testing.B) { largeLen := 100000 set := Set{} for i := 0; i < largeLen; i++ {