From f4484ab3b5a1b708588d0ac0e6a2472f6584bda0 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Sat, 9 Dec 2017 12:35:51 -0800 Subject: [PATCH] Implementing iavlStore... --- glide.lock | 4 +- store/iavlstore.go | 202 +++++++++++++--------------------------- store/rootmultistore.go | 41 ++++---- store/types.go | 2 +- 4 files changed, 88 insertions(+), 161 deletions(-) diff --git a/glide.lock b/glide.lock index 3c6422226..b3a182bb3 100644 --- a/glide.lock +++ b/glide.lock @@ -131,12 +131,12 @@ imports: - keys/wordlist - nano - name: github.com/tendermint/go-wire - version: 99d2169a1e39c65983eacaa1da867d6f3218e1c9 + version: 078a95fc9117752ced2590cb4632efdf48e41c84 subpackages: - data - data/base58 - name: github.com/tendermint/iavl - version: 7198bc18ac0cf4ac0f57802d8022695640b79ec7 + version: 03e6c011329bce607eed69d60737c899519f1f70 - name: github.com/tendermint/light-client version: 76313d625e662ed7b284d066d68ff71edd7a9fac subpackages: diff --git a/store/iavlstore.go b/store/iavlstore.go index 6dd9aa617..bb7c6d03f 100644 --- a/store/iavlstore.go +++ b/store/iavlstore.go @@ -1,5 +1,3 @@ -// XXX Need to s/Committer/CommitStore/g - package store import ( @@ -13,152 +11,119 @@ import ( dbm "github.com/tendermint/tmlibs/db" ) -// NewIAVLLoader returns a CommitterLoader that returns -// an IAVLCommitter -func NewIAVLLoader(dbName string, cacheSize int, nHistoricalVersions uint64) CommitterLoader { - l := iavlLoader{ - dbName: dbName, - cacheSize: cacheSize, - nHistoricalVersions: nHistoricalVersions, +// NewIAVLStoreLoader returns a CommitStoreLoader that returns an iavlStore +func NewIAVLStoreLoader(dbName string, cacheSize int, numHistory int64) CommitStoreLoader { + l := iavlStoreLoader{ + dbName: dbName, + cacheSize: cacheSize, + numHistory: numHistory, } - return CommitterLoader(l.Load) + return l.Load } -var _ CacheIterKVStore = (*IAVLCommitter)(nil) -var _ Committer = (*IAVLCommitter)(nil) +var _ IterKVStore = (*iavlStore)(nil) +var _ CommitStore = (*iavlStore)(nil) -// IAVLCommitter Implements IterKVStore and Committer -type IAVLCommitter struct { - // we must store the last height here, as it is needed - // for saving the versioned tree - lastHeight uint64 +// iavlStore Implements IterKVStore and CommitStore. +type iavlStore struct { - // nHistoricalVersions is how many old versions we hold onto, - // uses a naive "hold last X versions" algorithm - nHistoricalVersions uint64 - - // this is all historical data and connection to - // the db + // The underlying tree. tree *iavl.VersionedTree - // this is the current working state to be saved - // on the next commit - IAVLStore + // How many old versions we hold onto. + numHistory int64 } -// NewIAVLCommitter properly initializes a committer -// that is ready to use as a IterKVStore -func NewIAVLCommitter(tree *iavl.VersionedTree, - lastHeight uint64, nHistoricalVersions uint64) *IAVLCommitter { - ic := &IAVLCommitter{ - tree: tree, - lastHeight: lastHeight, - nHistoricalVersions: nHistoricalVersions, +// CONTRACT: tree should be fully loaded. +func newIAVLStore(tree *iavl.VersionedTree, numHistory int64) *iavlStore { + st := &iavlStore{ + tree: tree, + numHistory: numHistory, } - ic.updateStore() - return ic + return st } -// Commit syncs the working state and -// saves another version to the db -func (i *IAVLCommitter) Commit() CommitID { - // TODO: sync working state?? - // I think this is done already just by writing to tree.Tree() +// Commit persists the store. +func (st *iavlStore) Commit() CommitID { - // save a new version - ic.lastHeight++ - hash, err := ic.tree.SaveVersion(ic.lastHeight) + // Save a new version. + hash, version, err := st.tree.SaveVersion() if err != nil { - // TODO: do we want to extend Commit to - // allow returning errors? + // TODO: Do we want to extend Commit to allow returning errors? panic(err) } - // now point working state to the new status - ic.updateStore() - - // release an old version of history - if ic.nHistoricalVersions <= ic.lastHeight { - release := ic.lastHeight - ic.nHistoricalVersions - ic.tree.DeleteVersion(release) + // Release an old version of history + if st.numHistory < st.tree.Version() { + toRelease := version - st.numHistory + st.tree.DeleteVersion(toRelease) } return CommitID{ - Version: ic.lastHeight, + Version: version, Hash: hash, } } -// store returns a wrapper around the current writable state -func (ic *IAVLCommitter) updateStore() { - ic.IAVLStore = IAVLStore{ic.tree.Tree()} -} - -// IAVLStore is the writable state (not history) and -// implements the IterKVStore interface. -type IAVLStore struct { - tree *iavl.Tree -} - // CacheWrap implements IterKVStore. -func (is IAVLStore) CacheWrap() CacheWriter { - return is.CacheIterKVStore() +func (st *iavlStore) CacheWrap() CacheWriter { + return st.CacheIterKVStore() } // CacheIterKVStore implements IterKVStore. -func (is IAVLStore) CacheIterKVStore() CacheIterKVStore { - // TODO: Add CacheWrap to IAVLTree. - return i +func (st *iavlStore) CacheIterKVStore() CacheIterKVStore { + // XXX Create generic IterKVStore wrapper. + return nil } // Set implements IterKVStore. -func (is IAVLStore) Set(key, value []byte) (prev []byte) { - _, prev = is.tree.Get(key) - is.tree.Set(key, value) +func (st *iavlStore) Set(key, value []byte) (prev []byte) { + _, prev = st.tree.Get(key) + st.tree.Set(key, value) return prev } // Get implements IterKVStore. -func (is IAVLStore) Get(key []byte) (value []byte, exists bool) { - _, v := is.tree.Get(key) +func (st *iavlStore) Get(key []byte) (value []byte, exists bool) { + _, v := st.tree.Get(key) return v, (v != nil) } // Has implements IterKVStore. -func (is IAVLStore) Has(key []byte) (exists bool) { - return is.tree.Has(key) +func (st *iavlStore) Has(key []byte) (exists bool) { + return st.tree.Has(key) } // Remove implements IterKVStore. -func (is IAVLStore) Remove(key []byte) (prev []byte, removed bool) { - return is.tree.Remove(key) +func (st *iavlStore) Remove(key []byte) (prev []byte, removed bool) { + return st.tree.Remove(key) } // Iterator implements IterKVStore. -func (is IAVLStore) Iterator(start, end []byte) Iterator { - // TODO: this needs changes to IAVL tree +func (st *iavlStore) Iterator(start, end []byte) Iterator { + // XXX Create iavlIterator (without modifying tendermint/iavl) return nil } // ReverseIterator implements IterKVStore. -func (is IAVLStore) ReverseIterator(start, end []byte) Iterator { - // TODO +func (st *iavlStore) ReverseIterator(start, end []byte) Iterator { + // XXX Create iavlIterator (without modifying tendermint/iavl) return nil } // First implements IterKVStore. func (is IAVLStore) First(start, end []byte) (kv KVPair, ok bool) { - // TODO + // XXX return KVPair{}, false } // Last implements IterKVStore. func (is IAVLStore) Last(start, end []byte) (kv KVPair, ok bool) { - // TODO + // XXX return KVPair{}, false } -var _ IterKVStore = IAVLStore{} +//---------------------------------------- type iavlIterator struct { // TODO @@ -167,94 +132,55 @@ type iavlIterator struct { var _ Iterator = (*iavlIterator)(nil) // Domain implements Iterator -// -// The start & end (exclusive) limits to iterate over. -// If end < start, then the Iterator goes in reverse order. func (ii *iavlIterator) Domain() (start, end []byte) { // TODO return nil, nil } // Valid implements Iterator -// -// Returns if the current position is valid. func (ii *iavlIterator) Valid() bool { // TODO return false } // Next implements Iterator -// -// Next moves the iterator to the next key/value pair. func (ii *iavlIterator) Next() { // TODO } // Key implements Iterator -// -// Key returns the key of the current key/value pair, or nil if done. -// The caller should not modify the contents of the returned slice, and -// its contents may change after calling Next(). func (ii *iavlIterator) Key() []byte { // TODO return nil } // Value implements Iterator -// -// Value returns the key of the current key/value pair, or nil if done. -// The caller should not modify the contents of the returned slice, and -// its contents may change after calling Next(). func (ii *iavlIterator) Value() []byte { // TODO return nil } // Release implements Iterator -// -// Releases any resources and iteration-locks func (ii *iavlIterator) Release() { // TODO } -// iavlLoader contains info on what store we want to load from -type iavlLoader struct { - dbName string - cacheSize int - nHistoricalVersion uint64 +//---------------------------------------- + +// iavlStoreLoader contains info on what store we want to load from +type iavlStoreLoader struct { + db dbm.DB + cacheSize int + numHistory int64 } -// Load implements CommitLoader type -func (il iavlLoader) Load(id CommitID) (Committer, error) { - // memory backed case, just for testing - if il.dbName == "" { - tree := iavl.NewVersionedTree(0, dbm.NewMemDB()) - store := NewIAVLCommitter(tree, 0, il.nHistoricalVersions) - return store, nil - } - - // Expand the path fully - dbPath, err := filepath.Abs(il.dbName) +// Load implements CommitLoader. +func (isl iavlLoader) Load(id CommitID) (CommitStore, error) { + tree := iavl.NewVersionedTree(isl.db, isl.cacheSize) + err := tree.Load() if err != nil { - return nil, errors.New("Invalid Database Name") + return nil, err } - - // Some external calls accidently add a ".db", which is now removed - dbPath = strings.TrimSuffix(dbPath, path.Ext(dbPath)) - - // Split the database name into it's components (dir, name) - dir := filepath.Dir(dbPath) - name := filepath.Base(dbPath) - - // Open database called "dir/name.db", if it doesn't exist it will be created - db := dbm.NewDB(name, dbm.LevelDBBackendStr, dir) - tree := iavl.NewVersionedTree(il.cacheSize, db) - if err = tree.Load(); err != nil { - return nil, errors.New("Loading tree: " + err.Error()) - } - - // TODO: load the version stored in id - store := NewIAVLCommitter(tree, tree.LatestVersion(), - il.nHistoricalVersions) + store := newIAVLStore(tree, isl.numHistory) return store, nil } diff --git a/store/rootmultistore.go b/store/rootmultistore.go index c45406ddd..b1fa41b2c 100644 --- a/store/rootmultistore.go +++ b/store/rootmultistore.go @@ -45,7 +45,7 @@ func (rs *rootMultiStore) SetCommitStoreLoader(name string, loader CommitStoreLo // Call once after all calls to SetCommitStoreLoader are complete. func (rs *rootMultiStore) LoadLatestVersion() error { ver := getLatestVersion(rs.db) - rs.LoadVersion(ver) + return rs.LoadVersion(ver) } // NOTE: Returns 0 unless LoadVersion() or LoadLatestVersion() is called. @@ -71,7 +71,10 @@ func (rs *rootMultiStore) LoadVersion(ver int64) error { // Otherwise, version is 1 or greater // Load commitState - var state commitState = loadCommitState(rs.db, ver) + state, err := loadCommitState(rs.db, ver) + if err != nil { + return err + } // Load each Substore var newSubstores = make(map[string]CommitStore) @@ -112,12 +115,10 @@ func (rs *rootMultiStore) doCommit() commitState { commitID := store.Commit() // Record CommitID - substores = append(substores, - substore{ - Name: name, - CommitID: commitID, - }, - ) + substore := substore{} + substore.Name = name + substore.CommitID = commitID + substores = append(substores, substore) } // Incr curVersion @@ -134,7 +135,7 @@ func (rs *rootMultiStore) doCommit() commitState { // Implements CommitStore func (rs *rootMultiStore) Commit() CommitID { - version := rs.version + version := rs.curVersion // Needs to be transactional batch := rs.db.NewBatch() @@ -145,15 +146,15 @@ func (rs *rootMultiStore) Commit() CommitID { if err != nil { panic(err) } - commitStateKey := fmt.Sprintf(commitStateKeyFmt, rs.version) - batch.Set(commitStateKey, stateBytes) + commitStateKey := fmt.Sprintf(commitStateKeyFmt, rs.curVersion) + batch.Set([]byte(commitStateKey), stateBytes) // Save the latest version - latestBytes, _ := wire.Marshal(rs.version) // Does not error - batch.Set(latestVersionKey, latestBytes) + latestBytes, _ := wire.Marshal(rs.curVersion) // Does not error + batch.Set([]byte(latestVersionKey), latestBytes) batch.Write() - rs.version += 1 + rs.curVersion += 1 commitID := CommitID{ Version: version, Hash: state.Hash(), @@ -179,17 +180,17 @@ func (rs *rootMultiStore) CacheMultiStore() CacheMultiStore { // Implements MultiStore func (rs *rootMultiStore) GetCommitStore(name string) CommitStore { - return rs.store[name] + return rs.substores[name] } // Implements MultiStore func (rs *rootMultiStore) GetKVStore(name string) KVStore { - return rs.store[name].(KVStore) + return rs.substores[name].(KVStore) } // Implements MultiStore func (rs *rootMultiStore) GetIterKVStore(name string) IterKVStore { - return rs.store[name].(IterKVStore) + return rs.substores[name].(IterKVStore) } //---------------------------------------- @@ -210,8 +211,8 @@ func loadCommitState(db dbm.DB, ver int64) (commitState, error) { // Load from DB. commitStateKey := fmt.Sprintf(commitStateKeyFmt, ver) - stateBytes := db.Get(commitStateKey, ver) - if bz == nil { + stateBytes := db.Get([]byte(commitStateKey)) + if stateBytes == nil { return commitState{}, fmt.Errorf("Failed to load rootMultiStore: no data") } @@ -264,7 +265,7 @@ func (sc substoreCore) Hash() []byte { func getLatestVersion(db dbm.DB) int64 { var latest int64 - latestBytes := db.Get(latestVersionKey) + latestBytes := db.Get([]byte(latestVersionKey)) if latestBytes == nil { return 0 } diff --git a/store/types.go b/store/types.go index 724577091..22e42ee7f 100644 --- a/store/types.go +++ b/store/types.go @@ -88,7 +88,7 @@ type IterKVStore interface { CacheIterKVStore() CacheIterKVStore // CacheWrap() returns a CacheIterKVStore. - CacheWrap() CacheWrap + // CacheWrap() defined in KVStore } type CacheIterKVStore interface {