From ba1b9319865b583476b8ef0d6862fee83f2c31aa Mon Sep 17 00:00:00 2001 From: Larry Ruane Date: Fri, 1 Apr 2022 12:31:36 -0600 Subject: [PATCH] add --sync-from-height command-line option This causes lightwalletd to discard cached blocks at the given height and beyond. This in turn causes it to re-fetch those blocks from zcashd. It's similar to --redownload, except that option discards all blocks (equivalent to --sync-from-height 0, but the existing --redownload is retained for compatibility). This is mostly intended for testing. It's sometimes useful to force the node to (re)sync some recent blocks, but redownloading all of them takes around an hour. --- cmd/root.go | 10 +++++++++- common/cache.go | 21 ++++++++++++--------- common/cache_test.go | 4 ++-- common/common.go | 1 + common/common_test.go | 8 ++++---- frontend/frontend_test.go | 2 +- 6 files changed, 29 insertions(+), 17 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 461a8c7..b88342e 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -55,6 +55,7 @@ var rootCmd = &cobra.Command{ GenCertVeryInsecure: viper.GetBool("gen-cert-very-insecure"), DataDir: viper.GetString("data-dir"), Redownload: viper.GetBool("redownload"), + SyncFromHeight: viper.GetInt("sync-from-height"), PingEnable: viper.GetBool("ping-very-insecure"), Darkside: viper.GetBool("darkside-very-insecure"), DarksideTimeout: viper.GetUint64("darkside-timeout"), @@ -239,7 +240,11 @@ func startServer(opts *common.Options) error { os.Stderr.WriteString(fmt.Sprintf("\n ** Can't create db directory: %s\n\n", dbPath)) os.Exit(1) } - cache := common.NewBlockCache(dbPath, chainName, saplingHeight, opts.Redownload) + syncFromHeight := opts.SyncFromHeight + if opts.Redownload { + syncFromHeight = 0 + } + cache := common.NewBlockCache(dbPath, chainName, saplingHeight, syncFromHeight) if !opts.Darkside { go common.BlockIngestor(cache, 0 /*loop forever*/) } else { @@ -325,6 +330,7 @@ func init() { rootCmd.Flags().Bool("no-tls-very-insecure", false, "run without the required TLS certificate, only for debugging, DO NOT use in production") rootCmd.Flags().Bool("gen-cert-very-insecure", false, "run with self-signed TLS certificate, only for debugging, DO NOT use in production") rootCmd.Flags().Bool("redownload", false, "re-fetch all blocks from zcashd; reinitialize local cache files") + rootCmd.Flags().Int("sync-from-height", -1, "re-fetch blocks from zcashd start at this height") rootCmd.Flags().String("data-dir", "/var/lib/lightwalletd", "data directory (such as db)") rootCmd.Flags().Bool("ping-very-insecure", false, "allow Ping GRPC for testing") rootCmd.Flags().Bool("darkside-very-insecure", false, "run with GRPC-controllable mock zcashd for integration testing (shuts down after 30 minutes)") @@ -356,6 +362,8 @@ func init() { viper.SetDefault("gen-cert-very-insecure", false) viper.BindPFlag("redownload", rootCmd.Flags().Lookup("redownload")) viper.SetDefault("redownload", false) + viper.BindPFlag("sync-from-height", rootCmd.Flags().Lookup("sync-from-height")) + viper.SetDefault("sync-from-height", -1) viper.BindPFlag("data-dir", rootCmd.Flags().Lookup("data-dir")) viper.SetDefault("data-dir", "/var/lib/lightwalletd") viper.BindPFlag("ping-very-insecure", rootCmd.Flags().Lookup("ping-very-insecure")) diff --git a/common/cache.go b/common/cache.go index 8b5ffb8..d4c68d1 100644 --- a/common/cache.go +++ b/common/cache.go @@ -191,7 +191,8 @@ func (c *BlockCache) Reset(startHeight int) { // NewBlockCache returns an instance of a block cache object. // (No locking here, we assume this is single-threaded.) -func NewBlockCache(dbPath string, chainName string, startHeight int, redownload bool) *BlockCache { +// syncFromHeight < 0 means latest (tip) height. +func NewBlockCache(dbPath string, chainName string, startHeight int, syncFromHeight int) *BlockCache { c := &BlockCache{} c.firstBlock = startHeight c.nextBlock = startHeight @@ -208,18 +209,20 @@ func NewBlockCache(dbPath string, chainName string, startHeight int, redownload if err != nil { Log.Fatal("open ", c.lengthsName, " failed: ", err) } - if redownload { - if err := c.lengthsFile.Truncate(0); err != nil { - Log.Fatal("truncate lengths file failed: ", err) - } - if err := c.blocksFile.Truncate(0); err != nil { - Log.Fatal("truncate blocks file failed: ", err) - } - } lengths, err := ioutil.ReadFile(c.lengthsName) if err != nil { Log.Fatal("read ", c.lengthsName, " failed: ", err) } + // 4 bytes per lengths[] value (block length) + if syncFromHeight >= 0 { + if syncFromHeight < startHeight { + syncFromHeight = startHeight + } + if (syncFromHeight-startHeight)*4 < len(lengths) { + // discard the entries at and beyond (newer than) the specified height + lengths = lengths[:(syncFromHeight-startHeight)*4] + } + } // The last entry in starts[] is where to write the next block. var offset int64 diff --git a/common/cache_test.go b/common/cache_test.go index 2493163..11a803e 100644 --- a/common/cache_test.go +++ b/common/cache_test.go @@ -58,7 +58,7 @@ func TestCache(t *testing.T) { // Pretend Sapling starts at 289460. os.RemoveAll(unitTestPath) - cache = NewBlockCache(unitTestPath, unitTestChain, 289460, true) + cache = NewBlockCache(unitTestPath, unitTestChain, 289460, 0) // Initially cache is empty. if cache.GetLatestHeight() != -1 { @@ -75,7 +75,7 @@ func TestCache(t *testing.T) { fillCache(t) // Simulate a restart to ensure the db files are read correctly. - cache = NewBlockCache(unitTestPath, unitTestChain, 289460, false) + cache = NewBlockCache(unitTestPath, unitTestChain, 289460, -1) // Should still be 6 blocks. if cache.nextBlock != 289466 { diff --git a/common/common.go b/common/common.go index 56ad5f6..c8a1a92 100644 --- a/common/common.go +++ b/common/common.go @@ -42,6 +42,7 @@ type Options struct { NoTLSVeryInsecure bool `json:"no_tls_very_insecure,omitempty"` GenCertVeryInsecure bool `json:"gen_cert_very_insecure,omitempty"` Redownload bool `json:"redownload"` + SyncFromHeight int `json:"sync_from_height"` DataDir string `json:"data_dir"` PingEnable bool `json:"ping_enable"` Darkside bool `json:"darkside"` diff --git a/common/common_test.go b/common/common_test.go index af2b3aa..1ff26d3 100644 --- a/common/common_test.go +++ b/common/common_test.go @@ -62,7 +62,7 @@ func TestMain(m *testing.M) { blockJSON, _ := json.Marshal(scan.Text()) blocks = append(blocks, blockJSON) } - testcache = NewBlockCache(unitTestPath, unitTestChain, 380640, true) + testcache = NewBlockCache(unitTestPath, unitTestChain, 380640, 0) // Setup is done; run all tests. exitcode := m.Run() @@ -355,7 +355,7 @@ func TestBlockIngestor(t *testing.T) { Time.Sleep = sleepStub Time.Now = nowStub os.RemoveAll(unitTestPath) - testcache = NewBlockCache(unitTestPath, unitTestChain, 380640, false) + testcache = NewBlockCache(unitTestPath, unitTestChain, 380640, -1) BlockIngestor(testcache, 11) if step != 19 { t.Error("unexpected final step", step) @@ -488,7 +488,7 @@ func TestGetBlockRange(t *testing.T) { testT = t RawRequest = getblockStub os.RemoveAll(unitTestPath) - testcache = NewBlockCache(unitTestPath, unitTestChain, 380640, true) + testcache = NewBlockCache(unitTestPath, unitTestChain, 380640, 0) blockChan := make(chan *walletrpc.CompactBlock) errChan := make(chan error) go GetBlockRange(testcache, blockChan, errChan, 380640, 380642) @@ -567,7 +567,7 @@ func TestGetBlockRangeReverse(t *testing.T) { testT = t RawRequest = getblockStubReverse os.RemoveAll(unitTestPath) - testcache = NewBlockCache(unitTestPath, unitTestChain, 380640, true) + testcache = NewBlockCache(unitTestPath, unitTestChain, 380640, 0) blockChan := make(chan *walletrpc.CompactBlock) errChan := make(chan error) diff --git a/frontend/frontend_test.go b/frontend/frontend_test.go index 65a2c26..38199df 100644 --- a/frontend/frontend_test.go +++ b/frontend/frontend_test.go @@ -36,7 +36,7 @@ const ( func testsetup() (walletrpc.CompactTxStreamerServer, *common.BlockCache) { os.RemoveAll(unitTestPath) - cache := common.NewBlockCache(unitTestPath, unitTestChain, 380640, true) + cache := common.NewBlockCache(unitTestPath, unitTestChain, 380640, 0) lwd, err := NewLwdStreamer(cache, "main", false /* enablePing */) if err != nil { os.Stderr.WriteString(fmt.Sprint("NewLwdStreamer failed:", err))