From c3296f2e01000b45f033fa355d5e477bece4b599 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Wed, 20 Jun 2018 01:42:37 -0700 Subject: [PATCH] Garbage collect DBProvider (unoptimized); Certifier creation takes a client --- cmd/tendermint/commands/lite.go | 2 +- lite/dbprovider.go | 117 +++++++++++++++++++++++++------- lite/proxy/certifier.go | 24 ++++--- 3 files changed, 105 insertions(+), 38 deletions(-) diff --git a/cmd/tendermint/commands/lite.go b/cmd/tendermint/commands/lite.go index 6987b7f1..53b3ec18 100644 --- a/cmd/tendermint/commands/lite.go +++ b/cmd/tendermint/commands/lite.go @@ -68,7 +68,7 @@ func runProxy(cmd *cobra.Command, args []string) error { // First, connect a client node := rpcclient.NewHTTP(nodeAddr, "/websocket") - cert, err := proxy.GetCertifier(chainID, home, nodeAddr) + cert, err := proxy.GetCertifier(chainID, home, node) if err != nil { return err } diff --git a/lite/dbprovider.go b/lite/dbprovider.go index 149a0ed3..81710c9e 100644 --- a/lite/dbprovider.go +++ b/lite/dbprovider.go @@ -12,36 +12,11 @@ import ( dbm "github.com/tendermint/tmlibs/db" ) -func signedHeaderKey(chainID string, height int64) []byte { - return []byte(fmt.Sprintf("%s/%010d/sh", chainID, height)) -} - -var signedHeaderKeyPattern = regexp.MustCompile(`([^/]+)/([0-9]*)/sh`) - -func parseSignedHeaderKey(key []byte) (chainID string, height int64, ok bool) { - submatch := signedHeaderKeyPattern.FindSubmatch(key) - if submatch == nil { - return "", 0, false - } - chainID = string(submatch[1]) - heightStr := string(submatch[2]) - heightInt, err := strconv.Atoi(heightStr) - if err != nil { - return "", 0, false - } - height = int64(heightInt) - ok = true // good! - return -} - -func validatorSetKey(chainID string, height int64) []byte { - return []byte(fmt.Sprintf("%s/%010d/vs", chainID, height)) -} - type DBProvider struct { chainID string db dbm.DB cdc *amino.Codec + limit int } func NewDBProvider(db dbm.DB) *DBProvider { @@ -52,6 +27,11 @@ func NewDBProvider(db dbm.DB) *DBProvider { return dbp } +func (dbp *DBProvider) SetLimit(limit int) *DBProvider { + dbp.limit = limit + return dbp +} + // Implements PersistentProvider. func (dbp *DBProvider) SaveFullCommit(fc FullCommit) error { @@ -85,6 +65,13 @@ func (dbp *DBProvider) SaveFullCommit(fc FullCommit) error { // And write sync. batch.WriteSync() + + // Garbage collect. + // TODO: optimize later. + if dbp.limit > 0 { + dbp.deleteAfterN(fc.ChainID(), dbp.limit) + } + return nil } @@ -166,3 +153,81 @@ func (dbp *DBProvider) fillFullCommit(sh types.SignedHeader) (FullCommit, error) NextValidators: nextValset, }, nil } + +func (dbp *DBProvider) deleteAfterN(chainID string, after int) error { + itr := dbp.db.ReverseIterator( + signedHeaderKey(chainID, 1<<63-1), + signedHeaderKey(chainID, 0), + ) + defer itr.Close() + + var lastHeight int64 = 1<<63 - 1 + var numSeen = 0 + + for itr.Valid() { + key := itr.Key() + _, height, ok := parseChainKeyPrefix(key) + if !ok { + return fmt.Errorf("unexpected key %v", key) + } else { + if height < lastHeight { + lastHeight = height + numSeen += 1 + } + if numSeen > after { + dbp.db.Delete(key) + } + } + } + return nil +} + +//---------------------------------------- + +func signedHeaderKey(chainID string, height int64) []byte { + return []byte(fmt.Sprintf("%s/%010d/sh", chainID, height)) +} + +var signedHeaderKeyPattern = regexp.MustCompile(`([^/]+)/([0-9]*)/sh`) + +func parseSignedHeaderKey(key []byte) (chainID string, height int64, ok bool) { + submatch := signedHeaderKeyPattern.FindSubmatch(key) + if submatch == nil { + return "", 0, false + } + chainID = string(submatch[1]) + heightStr := string(submatch[2]) + heightInt, err := strconv.Atoi(heightStr) + if err != nil { + return "", 0, false + } + height = int64(heightInt) + ok = true // good! + return +} + +func validatorSetKey(chainID string, height int64) []byte { + return []byte(fmt.Sprintf("%s/%010d/vs", chainID, height)) +} + +func chainKeyPrefix(chainID string, height int64) []byte { + return []byte(fmt.Sprintf("%s/%010d/", chainID, height)) +} + +var chainKeyPrefixPattern = regexp.MustCompile(`([^/]+)/([0-9]*)/`) + +func parseChainKeyPrefix(key []byte) (chainID string, height int64, ok bool) { + submatch := chainKeyPrefixPattern.FindSubmatch(key) + if submatch == nil { + return "", 0, false + } + chainID = string(submatch[1]) + heightStr := string(submatch[2]) + heightInt, err := strconv.Atoi(heightStr) + if err != nil { + return "", 0, false + } + height = int64(heightInt) + ok = true // good! + return +} diff --git a/lite/proxy/certifier.go b/lite/proxy/certifier.go index a6765402..772af58f 100644 --- a/lite/proxy/certifier.go +++ b/lite/proxy/certifier.go @@ -6,22 +6,24 @@ import ( dbm "github.com/tendermint/tmlibs/db" ) -func GetCertifier(chainID, rootDir, nodeAddr string) (*lite.InquiringCertifier, error) { +func GetCertifier(chainID, rootDir string, client lclient.SignStatusClient) (*lite.InquiringCertifier, error) { trust := lite.NewMultiProvider( - lite.NewDBProvider(dbm.NewMemDB()), + lite.NewDBProvider(dbm.NewMemDB()).SetLimit(10), lite.NewDBProvider(dbm.NewDB("trust-base", dbm.LevelDBBackend, rootDir)), ) + source := lclient.NewProvider(chainID, client) - source := lclient.NewHTTPProvider(chainID, nodeAddr) - - // XXX: total insecure hack to avoid `init` - fc, err := source.LatestFullCommit(chainID, 1, 1) + // TODO: Make this more secure, e.g. make it interactive in the console? + _, err := trust.LatestFullCommit(chainID, 1, 1<<63-1) if err != nil { - return nil, err - } - err = trust.SaveFullCommit(fc) - if err != nil { - return nil, err + fc, err := source.LatestFullCommit(chainID, 1, 1) + if err != nil { + return nil, err + } + err = trust.SaveFullCommit(fc) + if err != nil { + return nil, err + } } cert, err := lite.NewInquiringCertifier(chainID, trust, source)