From 296d2235455dc62627d9c3feaf31306497fd4aae Mon Sep 17 00:00:00 2001 From: Alexis Sellier Date: Thu, 28 Sep 2017 17:26:24 +0200 Subject: [PATCH 1/4] Add Release & Error methods to Iterator We need this to properly support LevelDB backend, which needs to be released if we don't want to leak memory. --- db/db.go | 5 ++++- db/mem_db.go | 10 ++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/db/db.go b/db/db.go index aa8ff48a..51283ac2 100644 --- a/db/db.go +++ b/db/db.go @@ -10,10 +10,10 @@ type DB interface { DeleteSync([]byte) Close() NewBatch() Batch + Iterator() Iterator // For debugging Print() - Iterator() Iterator Stats() map[string]string } @@ -28,6 +28,9 @@ type Iterator interface { Key() []byte Value() []byte + + Release() + Error() error } //----------------------------------------------------------------------------- diff --git a/db/mem_db.go b/db/mem_db.go index db40227e..04d48cac 100644 --- a/db/mem_db.go +++ b/db/mem_db.go @@ -97,6 +97,16 @@ func (it *memDBIterator) Value() []byte { return it.db.Get(it.Key()) } +func (it *memDBIterator) Release() { + it.db = nil + it.keys = nil + return +} + +func (it *memDBIterator) Error() error { + return nil +} + func (db *MemDB) Iterator() Iterator { it := newMemDBIterator() it.db = db From 219d4e8427b2ddb0e9146ca7319bce2e1f22a6ce Mon Sep 17 00:00:00 2001 From: Alexis Sellier Date: Thu, 28 Sep 2017 17:35:01 +0200 Subject: [PATCH 2/4] Add IteratorPrefix method to Iterator --- db/db.go | 1 + db/go_level_db.go | 5 +++++ db/mem_db.go | 9 ++++++++- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/db/db.go b/db/db.go index 51283ac2..8156c1e9 100644 --- a/db/db.go +++ b/db/db.go @@ -11,6 +11,7 @@ type DB interface { Close() NewBatch() Batch Iterator() Iterator + IteratorPrefix([]byte) Iterator // For debugging Print() diff --git a/db/go_level_db.go b/db/go_level_db.go index 54ae1149..31c85ce8 100644 --- a/db/go_level_db.go +++ b/db/go_level_db.go @@ -7,6 +7,7 @@ import ( "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/errors" "github.com/syndtr/goleveldb/leveldb/opt" + "github.com/syndtr/goleveldb/leveldb/util" . "github.com/tendermint/tmlibs/common" ) @@ -119,6 +120,10 @@ func (db *GoLevelDB) Iterator() Iterator { return db.db.NewIterator(nil, nil) } +func (db *GoLevelDB) IteratorPrefix(prefix []byte) Iterator { + return db.db.NewIterator(util.BytesPrefix(prefix), nil) +} + func (db *GoLevelDB) NewBatch() Batch { batch := new(leveldb.Batch) return &goLevelDBBatch{db, batch} diff --git a/db/mem_db.go b/db/mem_db.go index 04d48cac..56163329 100644 --- a/db/mem_db.go +++ b/db/mem_db.go @@ -2,6 +2,7 @@ package db import ( "fmt" + "strings" "sync" ) @@ -108,6 +109,10 @@ func (it *memDBIterator) Error() error { } func (db *MemDB) Iterator() Iterator { + return db.IteratorPrefix([]byte{}) +} + +func (db *MemDB) IteratorPrefix(prefix []byte) Iterator { it := newMemDBIterator() it.db = db it.last = -1 @@ -117,7 +122,9 @@ func (db *MemDB) Iterator() Iterator { // unfortunately we need a copy of all of the keys for key, _ := range db.db { - it.keys = append(it.keys, key) + if strings.HasPrefix(key, string(prefix)) { + it.keys = append(it.keys, key) + } } return it } From e9e6ec3a2c22b3139c5787dd3e6c8a61960ec522 Mon Sep 17 00:00:00 2001 From: Alexis Sellier Date: Thu, 28 Sep 2017 17:43:47 +0200 Subject: [PATCH 3/4] Implement Key/Value on Iterator for GoLevelDB This is needed because leveldb reuses the keys on each iteration. In our wrapper, we copy the key/value so that it is safe to store. --- db/go_level_db.go | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/db/go_level_db.go b/db/go_level_db.go index 31c85ce8..2d2ba44e 100644 --- a/db/go_level_db.go +++ b/db/go_level_db.go @@ -6,6 +6,7 @@ import ( "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/errors" + "github.com/syndtr/goleveldb/leveldb/iterator" "github.com/syndtr/goleveldb/leveldb/opt" "github.com/syndtr/goleveldb/leveldb/util" @@ -116,12 +117,32 @@ func (db *GoLevelDB) Stats() map[string]string { return stats } +type goLevelDBIterator struct { + iterator.Iterator +} + +func (it *goLevelDBIterator) Key() []byte { + key := it.Key() + k := make([]byte, len(key)) + copy(k, key) + + return k +} + +func (it *goLevelDBIterator) Value() []byte { + val := it.Value() + v := make([]byte, len(val)) + copy(v, val) + + return v +} + func (db *GoLevelDB) Iterator() Iterator { - return db.db.NewIterator(nil, nil) + return &goLevelDBIterator{db.db.NewIterator(nil, nil)} } func (db *GoLevelDB) IteratorPrefix(prefix []byte) Iterator { - return db.db.NewIterator(util.BytesPrefix(prefix), nil) + return &goLevelDBIterator{db.db.NewIterator(util.BytesPrefix(prefix), nil)} } func (db *GoLevelDB) NewBatch() Batch { From 6b1e3bcee3cf69fbac004cef8f8f3788f12cd58e Mon Sep 17 00:00:00 2001 From: Alexis Sellier Date: Fri, 13 Oct 2017 13:03:43 +0200 Subject: [PATCH 4/4] Add comments about copying --- db/go_level_db.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/db/go_level_db.go b/db/go_level_db.go index 2d2ba44e..f3ed79e3 100644 --- a/db/go_level_db.go +++ b/db/go_level_db.go @@ -121,6 +121,7 @@ type goLevelDBIterator struct { iterator.Iterator } +// Key returns a copy of the current key. func (it *goLevelDBIterator) Key() []byte { key := it.Key() k := make([]byte, len(key)) @@ -129,6 +130,7 @@ func (it *goLevelDBIterator) Key() []byte { return k } +// Value returns a copy of the current value. func (it *goLevelDBIterator) Value() []byte { val := it.Value() v := make([]byte, len(val))