certusone
/
cosmos-sdk
mirror of https://github.com/certusone/cosmos-sdk.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge PR #2143: Speedup IAVL iterator by removing defers when unneeded.

This commit is contained in:
Christopher Goes 2018-08-28 13:51:47 +02:00 committed by GitHub
parent 5ed1775681 2c3a4fc025
commit d3021d4c2f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 73 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
Gopkg.lock generated
View File

@ -34,11 +34,11 @@
[[projects]] [[projects]]
branch = "master" branch = "master"
digest = "1:2c00f064ba355903866cbfbf3f7f4c0fe64af6638cc7d1b8bdcf3181bc67f1d8" digest = "1:6aabc1566d6351115d561d038da82a4c19b46c3b6e17f4a0a2fa60260663dc79"
name = "github.com/btcsuite/btcd" name = "github.com/btcsuite/btcd"
packages = ["btcec"] packages = ["btcec"]
pruneopts = "UT" pruneopts = "UT"
revision = "f899737d7f2764dc13e4d01ff00108ec58f766a9" revision = "d81d8877b8f327112e94e814937143a71d1692a7"
[[projects]] [[projects]]
digest = "1:386de157f7d19259a7f9c81f26ce011223ce0f090353c1152ffdf730d7d10ac2" digest = "1:386de157f7d19259a7f9c81f26ce011223ce0f090353c1152ffdf730d7d10ac2"
@ -71,7 +71,7 @@
version = "v1.4.7" version = "v1.4.7"
[[projects]] [[projects]]
digest = "1:fdf5169073fb0ad6dc12a70c249145e30f4058647bea25f0abd48b6d9f228a11" digest = "1:fa30c0652956e159cdb97dcb2ef8b8db63ed668c02a5c3a40961c8f0641252fe"
name = "github.com/go-kit/kit" name = "github.com/go-kit/kit"
packages = [ packages = [
"log", "log",
@ -103,7 +103,7 @@
version = "v1.7.0" version = "v1.7.0"
[[projects]] [[projects]]
digest = "1:35621fe20f140f05a0c4ef662c26c0ab4ee50bca78aa30fe87d33120bd28165e" digest = "1:212285efb97b9ec2e20550d81f0446cb7897e57cbdfd7301b1363ab113d8be45"
name = "github.com/gogo/protobuf" name = "github.com/gogo/protobuf"
packages = [ packages = [
"gogoproto", "gogoproto",
@ -118,7 +118,7 @@
version = "v1.1.1" version = "v1.1.1"
[[projects]] [[projects]]
digest = "1:17fe264ee908afc795734e8c4e63db2accabaf57326dbf21763a7d6b86096260" digest = "1:cb22af0ed7c72d495d8be1106233ee553898950f15fd3f5404406d44c2e86888"
name = "github.com/golang/protobuf" name = "github.com/golang/protobuf"
packages = [ packages = [
"proto", "proto",
@ -165,12 +165,13 @@
[[projects]] [[projects]]
branch = "master" branch = "master"
digest = "1:12247a2e99a060cc692f6680e5272c8adf0b8f572e6bce0d7095e624c958a240" digest = "1:ac64f01acc5eeea9dde40e326de6b6471e501392ec06524c3b51033aa50789bc"
name = "github.com/hashicorp/hcl" name = "github.com/hashicorp/hcl"
packages = [ packages = [
".", ".",
"hcl/ast", "hcl/ast",
"hcl/parser", "hcl/parser",
"hcl/printer",
"hcl/scanner", "hcl/scanner",
"hcl/strconv", "hcl/strconv",
"hcl/token", "hcl/token",
@ -230,12 +231,12 @@
version = "v1.0.1" version = "v1.0.1"
[[projects]] [[projects]]
branch = "master" digest = "1:645110e089152bd0f4a011a2648fbb0e4df5977be73ca605781157ac297f50c4"
digest = "1:5ab79470a1d0fb19b041a624415612f8236b3c06070161a910562f2b2d064355"
name = "github.com/mitchellh/mapstructure" name = "github.com/mitchellh/mapstructure"
packages = ["."] packages = ["."]
pruneopts = "UT" pruneopts = "UT"
revision = "f15292f7a699fcc1a38a80977f80a046874ba8ac" revision = "fa473d140ef3c6adf42d6b391fe76707f1f243c8"
version = "v1.0.0"
[[projects]] [[projects]]
digest = "1:95741de3af260a92cc5c7f3f3061e85273f5a81b5db20d4bd68da74bd521675e" digest = "1:95741de3af260a92cc5c7f3f3061e85273f5a81b5db20d4bd68da74bd521675e"
@ -262,7 +263,7 @@
version = "v1.0.0" version = "v1.0.0"
[[projects]] [[projects]]
digest = "1:c1a04665f9613e082e1209cf288bf64f4068dcd6c87a64bf1c4ff006ad422ba0" digest = "1:98225904b7abff96c052b669b25788f18225a36673fba022fb93514bb9a2a64e"
name = "github.com/prometheus/client_golang" name = "github.com/prometheus/client_golang"
packages = [ packages = [
"prometheus", "prometheus",
@ -273,7 +274,7 @@
[[projects]] [[projects]]
branch = "master" branch = "master"
digest = "1:2d5cd61daa5565187e1d96bae64dbbc6080dacf741448e9629c64fd93203b0d4" digest = "1:0f37e09b3e92aaeda5991581311f8dbf38944b36a3edec61cc2d1991f527554a"
name = "github.com/prometheus/client_model" name = "github.com/prometheus/client_model"
packages = ["go"] packages = ["go"]
pruneopts = "UT" pruneopts = "UT"
@ -281,7 +282,7 @@
[[projects]] [[projects]]
branch = "master" branch = "master"
digest = "1:63b68062b8968092eb86bedc4e68894bd096ea6b24920faca8b9dcf451f54bb5" digest = "1:dad2e5a2153ee7a6c9ab8fc13673a16ee4fb64434a7da980965a3741b0c981a3"
name = "github.com/prometheus/common" name = "github.com/prometheus/common"
packages = [ packages = [
"expfmt", "expfmt",
@ -293,7 +294,7 @@
[[projects]] [[projects]]
branch = "master" branch = "master"
digest = "1:8c49953a1414305f2ff5465147ee576dd705487c35b15918fcd4efdc0cb7a290" digest = "1:a37c98f4b7a66bb5c539c0539f0915a74ef1c8e0b3b6f45735289d94cae92bfd"
name = "github.com/prometheus/procfs" name = "github.com/prometheus/procfs"
packages = [ packages = [
".", ".",
@ -312,7 +313,7 @@
revision = "e2704e165165ec55d062f5919b4b29494e9fa790" revision = "e2704e165165ec55d062f5919b4b29494e9fa790"
[[projects]] [[projects]]
digest = "1:bd1ae00087d17c5a748660b8e89e1043e1e5479d0fea743352cda2f8dd8c4f84" digest = "1:37ace7f35375adec11634126944bdc45a673415e2fcc07382d03b75ec76ea94c"
name = "github.com/spf13/afero" name = "github.com/spf13/afero"
packages = [ packages = [
".", ".",
@ -331,7 +332,7 @@
version = "v1.2.0" version = "v1.2.0"
[[projects]] [[projects]]
digest = "1:7ffc0983035bc7e297da3688d9fe19d60a420e9c38bef23f845c53788ed6a05e" digest = "1:627ab2f549a6a55c44f46fa24a4307f4d0da81bfc7934ed0473bf38b24051d26"
name = "github.com/spf13/cobra" name = "github.com/spf13/cobra"
packages = ["."] packages = ["."]
pruneopts = "UT" pruneopts = "UT"
@ -363,7 +364,7 @@
version = "v1.0.0" version = "v1.0.0"
[[projects]] [[projects]]
digest = "1:7e8d267900c7fa7f35129a2a37596e38ed0f11ca746d6d9ba727980ee138f9f6" digest = "1:73697231b93fb74a73ebd8384b68b9a60c57ea6b13c56d2425414566a72c8e6d"
name = "github.com/stretchr/testify" name = "github.com/stretchr/testify"
packages = [ packages = [
"assert", "assert",
@ -375,7 +376,7 @@
[[projects]] [[projects]]
branch = "master" branch = "master"
digest = "1:f2ffd421680b0a3f7887501b3c6974bcf19217ecd301d0e2c9b681940ec363d5" digest = "1:442d2ffa75ffae302ce8800bf4144696b92bef02917923ea132ce2d39efe7d65"
name = "github.com/syndtr/goleveldb" name = "github.com/syndtr/goleveldb"
packages = [ packages = [
"leveldb", "leveldb",
@ -396,7 +397,7 @@
[[projects]] [[projects]]
branch = "master" branch = "master"
digest = "1:087aaa7920e5d0bf79586feb57ce01c35c830396ab4392798112e8aae8c47722" digest = "1:203b409c21115233a576f99e8f13d8e07ad82b25500491f7e1cca12588fb3232"
name = "github.com/tendermint/ed25519" name = "github.com/tendermint/ed25519"
packages = [ packages = [
".", ".",
@ -423,7 +424,7 @@
version = "v0.9.2" version = "v0.9.2"
[[projects]] [[projects]]
digest = "1:4f15e95fe3888cc75dd34f407d6394cbc7fd3ff24920851b92b295f6a8b556e6" digest = "1:963f6c04345ce36f900c1d6367200eebc3cc2db6ee632ff865ea8dcf64b748a0"
name = "github.com/tendermint/tendermint" name = "github.com/tendermint/tendermint"
packages = [ packages = [
"abci/client", "abci/client",
@ -490,7 +491,7 @@
version = "v0.23.1-rc0" version = "v0.23.1-rc0"
[[projects]] [[projects]]
digest = "1:bf6d9a827ea3cad964c2f863302e4f6823170d0b5ed16f72cf1184a7c615067e" digest = "1:ad879bb8c71020a3f92f0c61f414d93eae1d5dc2f37023b6abaa3cc84b00165e"
name = "github.com/tendermint/tmlibs" name = "github.com/tendermint/tmlibs"
packages = ["cli"] packages = ["cli"]
pruneopts = "UT" pruneopts = "UT"
@ -506,7 +507,7 @@
[[projects]] [[projects]]
branch = "master" branch = "master"
digest = "1:27507554c6d4f060d8d700c31c624a43d3a92baa634e178ddc044bdf7d13b44a" digest = "1:2a3ce1f08dcae8bac666deb6e4c88b5d7170c510da38fd746231144cac351704"
name = "golang.org/x/crypto" name = "golang.org/x/crypto"
packages = [ packages = [
"blowfish", "blowfish",
@ -528,7 +529,7 @@
revision = "614d502a4dac94afa3a6ce146bd1736da82514c6" revision = "614d502a4dac94afa3a6ce146bd1736da82514c6"
[[projects]] [[projects]]
digest = "1:d36f55a999540d29b6ea3c2ea29d71c76b1d9853fdcd3e5c5cb4836f2ba118f1" digest = "1:04dda8391c3e2397daf254ac68003f30141c069b228d06baec8324a5f81dc1e9"
name = "golang.org/x/net" name = "golang.org/x/net"
packages = [ packages = [
"context", "context",
@ -545,17 +546,17 @@
[[projects]] [[projects]]
branch = "master" branch = "master"
digest = "1:a0e12bc26f317c0e2d497baf767285e1790e526e8dd46553c5a67fcbc8692157" digest = "1:c8baf78f0ac6eb27c645e264fe5e8a74d5a50db188ab41a7ff3b275c112e0735"
name = "golang.org/x/sys" name = "golang.org/x/sys"
packages = [ packages = [
"cpu", "cpu",
"unix", "unix",
] ]
pruneopts = "UT" pruneopts = "UT"
revision = "3b58ed4ad3395d483fc92d5d14123ce2c3581fec" revision = "11551d06cbcc94edc80a0facaccbda56473c19c1"
[[projects]] [[projects]]
digest = "1:a2ab62866c75542dd18d2b069fec854577a20211d7c0ea6ae746072a1dccdd18" digest = "1:7509ba4347d1f8de6ae9be8818b0cd1abc3deeffe28aeaf4be6d4b6b5178d9ca"
name = "golang.org/x/text" name = "golang.org/x/text"
packages = [ packages = [
"collate", "collate",
@ -586,7 +587,7 @@
revision = "c66870c02cf823ceb633bcd05be3c7cda29976f4" revision = "c66870c02cf823ceb633bcd05be3c7cda29976f4"
[[projects]] [[projects]]
digest = "1:2dab32a43451e320e49608ff4542fdfc653c95dcc35d0065ec9c6c3dd540ed74" digest = "1:4515e3030c440845b046354fd5d57671238428b820deebce2e9dabb5cd3c51ac"
name = "google.golang.org/grpc" name = "google.golang.org/grpc"
packages = [ packages = [
".", ".",

1
PENDING.md
View File

@ -63,6 +63,7 @@ IMPROVEMENTS
* SDK * SDK
* [tools] Make get_vendor_deps deletes `.vendor-new` directories, in case scratch files are present. * [tools] Make get_vendor_deps deletes `.vendor-new` directories, in case scratch files are present.
* [cli] \#1632 Add integration tests to ensure `basecoind init && basecoind` start sequences run successfully for both `democoin` and `basecoin` examples. * [cli] \#1632 Add integration tests to ensure `basecoind init && basecoind` start sequences run successfully for both `democoin` and `basecoin` examples.
* [store] Speedup IAVL iteration, and consequently everything that requires IAVL iteration. [#2143](https://github.com/cosmos/cosmos-sdk/issues/2143)
* [simulation] Make timestamps randomized [#2153](https://github.com/cosmos/cosmos-sdk/pull/2153) * [simulation] Make timestamps randomized [#2153](https://github.com/cosmos/cosmos-sdk/pull/2153)
* Tendermint * Tendermint

35
store/iavlstore.go
View File

@ -332,39 +332,42 @@ func (iter *iavlIterator) Domain() (start, end []byte) {
func (iter *iavlIterator) Valid() bool { func (iter *iavlIterator) Valid() bool {
iter.waitInit() iter.waitInit()
iter.mtx.Lock() iter.mtx.Lock()
defer iter.mtx.Unlock()
return !iter.invalid validity := !iter.invalid
iter.mtx.Unlock()
return validity
} }
// Implements Iterator. // Implements Iterator.
func (iter *iavlIterator) Next() { func (iter *iavlIterator) Next() {
iter.waitInit() iter.waitInit()
iter.mtx.Lock() iter.mtx.Lock()
defer iter.mtx.Unlock() iter.assertIsValid(true)
iter.assertIsValid()
iter.receiveNext() iter.receiveNext()
iter.mtx.Unlock()
} }
// Implements Iterator. // Implements Iterator.
func (iter *iavlIterator) Key() []byte { func (iter *iavlIterator) Key() []byte {
iter.waitInit() iter.waitInit()
iter.mtx.Lock() iter.mtx.Lock()
defer iter.mtx.Unlock() iter.assertIsValid(true)
iter.assertIsValid()
return iter.key key := iter.key
iter.mtx.Unlock()
return key
} }
// Implements Iterator. // Implements Iterator.
func (iter *iavlIterator) Value() []byte { func (iter *iavlIterator) Value() []byte {
iter.waitInit() iter.waitInit()
iter.mtx.Lock() iter.mtx.Lock()
defer iter.mtx.Unlock() iter.assertIsValid(true)
iter.assertIsValid()
return iter.value val := iter.value
iter.mtx.Unlock()
return val
} }
// Implements Iterator. // Implements Iterator.
@ -375,14 +378,14 @@ func (iter *iavlIterator) Close() {
//---------------------------------------- //----------------------------------------
func (iter *iavlIterator) setNext(key, value []byte) { func (iter *iavlIterator) setNext(key, value []byte) {
iter.assertIsValid() iter.assertIsValid(false)
iter.key = key iter.key = key
iter.value = value iter.value = value
} }
func (iter *iavlIterator) setInvalid() { func (iter *iavlIterator) setInvalid() {
iter.assertIsValid() iter.assertIsValid(false)
iter.invalid = true iter.invalid = true
} }
@ -400,8 +403,14 @@ func (iter *iavlIterator) receiveNext() {
} }
} }
func (iter *iavlIterator) assertIsValid() { // assertIsValid panics if the iterator is invalid. If unlockMutex is true,
// it also unlocks the mutex before panicing, to prevent deadlocks in code that
// recovers from panics
func (iter *iavlIterator) assertIsValid(unlockMutex bool) {
if iter.invalid { if iter.invalid {
if unlockMutex {
iter.mtx.Unlock()
}
panic("invalid iterator") panic("invalid iterator")
} }
} }

23
store/iavlstore_test.go
View File

@ -464,3 +464,26 @@ func TestIAVLStoreQuery(t *testing.T) {
require.Equal(t, uint32(sdk.CodeOK), qres.Code) require.Equal(t, uint32(sdk.CodeOK), qres.Code)
require.Equal(t, v1, qres.Value) require.Equal(t, v1, qres.Value)
} }
func BenchmarkIAVLIteratorNext(b *testing.B) {
db := dbm.NewMemDB()
treeSize := 1000
tree := iavl.NewVersionedTree(db, cacheSize)
for i := 0; i < treeSize; i++ {
key := cmn.RandBytes(4)
value := cmn.RandBytes(50)
tree.Set(key, value)
}
iavlStore := newIAVLStore(tree, numRecent, storeEvery)
iterators := make([]Iterator, b.N/treeSize)
for i := 0; i < len(iterators); i++ {
iterators[i] = iavlStore.Iterator([]byte{0}, []byte{255, 255, 255, 255, 255})
}
b.ResetTimer()
for i := 0; i < len(iterators); i++ {
iter := iterators[i]
for j := 0; j < treeSize; j++ {
iter.Next()
}
}
}