add --sync-from-height command-line option

This causes lightwalletd to discard cached blocks at the given height
and beyond. This in turn causes it to re-fetch those blocks from zcashd.
It's similar to --redownload, except that option discards all blocks
(equivalent to --sync-from-height 0, but the existing --redownload is
retained for compatibility).

This is mostly intended for testing. It's sometimes useful to force the
node to (re)sync some recent blocks, but redownloading all of them takes
around an hour.
This commit is contained in:
Larry Ruane 2022-04-01 12:31:36 -06:00 committed by Larry Ruane
parent dfac02093d
commit ba1b931986
6 changed files with 29 additions and 17 deletions

View File

@ -55,6 +55,7 @@ var rootCmd = &cobra.Command{
GenCertVeryInsecure: viper.GetBool("gen-cert-very-insecure"), GenCertVeryInsecure: viper.GetBool("gen-cert-very-insecure"),
DataDir: viper.GetString("data-dir"), DataDir: viper.GetString("data-dir"),
Redownload: viper.GetBool("redownload"), Redownload: viper.GetBool("redownload"),
SyncFromHeight: viper.GetInt("sync-from-height"),
PingEnable: viper.GetBool("ping-very-insecure"), PingEnable: viper.GetBool("ping-very-insecure"),
Darkside: viper.GetBool("darkside-very-insecure"), Darkside: viper.GetBool("darkside-very-insecure"),
DarksideTimeout: viper.GetUint64("darkside-timeout"), DarksideTimeout: viper.GetUint64("darkside-timeout"),
@ -239,7 +240,11 @@ func startServer(opts *common.Options) error {
os.Stderr.WriteString(fmt.Sprintf("\n ** Can't create db directory: %s\n\n", dbPath)) os.Stderr.WriteString(fmt.Sprintf("\n ** Can't create db directory: %s\n\n", dbPath))
os.Exit(1) os.Exit(1)
} }
cache := common.NewBlockCache(dbPath, chainName, saplingHeight, opts.Redownload) syncFromHeight := opts.SyncFromHeight
if opts.Redownload {
syncFromHeight = 0
}
cache := common.NewBlockCache(dbPath, chainName, saplingHeight, syncFromHeight)
if !opts.Darkside { if !opts.Darkside {
go common.BlockIngestor(cache, 0 /*loop forever*/) go common.BlockIngestor(cache, 0 /*loop forever*/)
} else { } else {
@ -325,6 +330,7 @@ func init() {
rootCmd.Flags().Bool("no-tls-very-insecure", false, "run without the required TLS certificate, only for debugging, DO NOT use in production") rootCmd.Flags().Bool("no-tls-very-insecure", false, "run without the required TLS certificate, only for debugging, DO NOT use in production")
rootCmd.Flags().Bool("gen-cert-very-insecure", false, "run with self-signed TLS certificate, only for debugging, DO NOT use in production") rootCmd.Flags().Bool("gen-cert-very-insecure", false, "run with self-signed TLS certificate, only for debugging, DO NOT use in production")
rootCmd.Flags().Bool("redownload", false, "re-fetch all blocks from zcashd; reinitialize local cache files") rootCmd.Flags().Bool("redownload", false, "re-fetch all blocks from zcashd; reinitialize local cache files")
rootCmd.Flags().Int("sync-from-height", -1, "re-fetch blocks from zcashd start at this height")
rootCmd.Flags().String("data-dir", "/var/lib/lightwalletd", "data directory (such as db)") rootCmd.Flags().String("data-dir", "/var/lib/lightwalletd", "data directory (such as db)")
rootCmd.Flags().Bool("ping-very-insecure", false, "allow Ping GRPC for testing") rootCmd.Flags().Bool("ping-very-insecure", false, "allow Ping GRPC for testing")
rootCmd.Flags().Bool("darkside-very-insecure", false, "run with GRPC-controllable mock zcashd for integration testing (shuts down after 30 minutes)") rootCmd.Flags().Bool("darkside-very-insecure", false, "run with GRPC-controllable mock zcashd for integration testing (shuts down after 30 minutes)")
@ -356,6 +362,8 @@ func init() {
viper.SetDefault("gen-cert-very-insecure", false) viper.SetDefault("gen-cert-very-insecure", false)
viper.BindPFlag("redownload", rootCmd.Flags().Lookup("redownload")) viper.BindPFlag("redownload", rootCmd.Flags().Lookup("redownload"))
viper.SetDefault("redownload", false) viper.SetDefault("redownload", false)
viper.BindPFlag("sync-from-height", rootCmd.Flags().Lookup("sync-from-height"))
viper.SetDefault("sync-from-height", -1)
viper.BindPFlag("data-dir", rootCmd.Flags().Lookup("data-dir")) viper.BindPFlag("data-dir", rootCmd.Flags().Lookup("data-dir"))
viper.SetDefault("data-dir", "/var/lib/lightwalletd") viper.SetDefault("data-dir", "/var/lib/lightwalletd")
viper.BindPFlag("ping-very-insecure", rootCmd.Flags().Lookup("ping-very-insecure")) viper.BindPFlag("ping-very-insecure", rootCmd.Flags().Lookup("ping-very-insecure"))

View File

@ -191,7 +191,8 @@ func (c *BlockCache) Reset(startHeight int) {
// NewBlockCache returns an instance of a block cache object. // NewBlockCache returns an instance of a block cache object.
// (No locking here, we assume this is single-threaded.) // (No locking here, we assume this is single-threaded.)
func NewBlockCache(dbPath string, chainName string, startHeight int, redownload bool) *BlockCache { // syncFromHeight < 0 means latest (tip) height.
func NewBlockCache(dbPath string, chainName string, startHeight int, syncFromHeight int) *BlockCache {
c := &BlockCache{} c := &BlockCache{}
c.firstBlock = startHeight c.firstBlock = startHeight
c.nextBlock = startHeight c.nextBlock = startHeight
@ -208,18 +209,20 @@ func NewBlockCache(dbPath string, chainName string, startHeight int, redownload
if err != nil { if err != nil {
Log.Fatal("open ", c.lengthsName, " failed: ", err) Log.Fatal("open ", c.lengthsName, " failed: ", err)
} }
if redownload {
if err := c.lengthsFile.Truncate(0); err != nil {
Log.Fatal("truncate lengths file failed: ", err)
}
if err := c.blocksFile.Truncate(0); err != nil {
Log.Fatal("truncate blocks file failed: ", err)
}
}
lengths, err := ioutil.ReadFile(c.lengthsName) lengths, err := ioutil.ReadFile(c.lengthsName)
if err != nil { if err != nil {
Log.Fatal("read ", c.lengthsName, " failed: ", err) Log.Fatal("read ", c.lengthsName, " failed: ", err)
} }
// 4 bytes per lengths[] value (block length)
if syncFromHeight >= 0 {
if syncFromHeight < startHeight {
syncFromHeight = startHeight
}
if (syncFromHeight-startHeight)*4 < len(lengths) {
// discard the entries at and beyond (newer than) the specified height
lengths = lengths[:(syncFromHeight-startHeight)*4]
}
}
// The last entry in starts[] is where to write the next block. // The last entry in starts[] is where to write the next block.
var offset int64 var offset int64

View File

@ -58,7 +58,7 @@ func TestCache(t *testing.T) {
// Pretend Sapling starts at 289460. // Pretend Sapling starts at 289460.
os.RemoveAll(unitTestPath) os.RemoveAll(unitTestPath)
cache = NewBlockCache(unitTestPath, unitTestChain, 289460, true) cache = NewBlockCache(unitTestPath, unitTestChain, 289460, 0)
// Initially cache is empty. // Initially cache is empty.
if cache.GetLatestHeight() != -1 { if cache.GetLatestHeight() != -1 {
@ -75,7 +75,7 @@ func TestCache(t *testing.T) {
fillCache(t) fillCache(t)
// Simulate a restart to ensure the db files are read correctly. // Simulate a restart to ensure the db files are read correctly.
cache = NewBlockCache(unitTestPath, unitTestChain, 289460, false) cache = NewBlockCache(unitTestPath, unitTestChain, 289460, -1)
// Should still be 6 blocks. // Should still be 6 blocks.
if cache.nextBlock != 289466 { if cache.nextBlock != 289466 {

View File

@ -42,6 +42,7 @@ type Options struct {
NoTLSVeryInsecure bool `json:"no_tls_very_insecure,omitempty"` NoTLSVeryInsecure bool `json:"no_tls_very_insecure,omitempty"`
GenCertVeryInsecure bool `json:"gen_cert_very_insecure,omitempty"` GenCertVeryInsecure bool `json:"gen_cert_very_insecure,omitempty"`
Redownload bool `json:"redownload"` Redownload bool `json:"redownload"`
SyncFromHeight int `json:"sync_from_height"`
DataDir string `json:"data_dir"` DataDir string `json:"data_dir"`
PingEnable bool `json:"ping_enable"` PingEnable bool `json:"ping_enable"`
Darkside bool `json:"darkside"` Darkside bool `json:"darkside"`

View File

@ -62,7 +62,7 @@ func TestMain(m *testing.M) {
blockJSON, _ := json.Marshal(scan.Text()) blockJSON, _ := json.Marshal(scan.Text())
blocks = append(blocks, blockJSON) blocks = append(blocks, blockJSON)
} }
testcache = NewBlockCache(unitTestPath, unitTestChain, 380640, true) testcache = NewBlockCache(unitTestPath, unitTestChain, 380640, 0)
// Setup is done; run all tests. // Setup is done; run all tests.
exitcode := m.Run() exitcode := m.Run()
@ -355,7 +355,7 @@ func TestBlockIngestor(t *testing.T) {
Time.Sleep = sleepStub Time.Sleep = sleepStub
Time.Now = nowStub Time.Now = nowStub
os.RemoveAll(unitTestPath) os.RemoveAll(unitTestPath)
testcache = NewBlockCache(unitTestPath, unitTestChain, 380640, false) testcache = NewBlockCache(unitTestPath, unitTestChain, 380640, -1)
BlockIngestor(testcache, 11) BlockIngestor(testcache, 11)
if step != 19 { if step != 19 {
t.Error("unexpected final step", step) t.Error("unexpected final step", step)
@ -488,7 +488,7 @@ func TestGetBlockRange(t *testing.T) {
testT = t testT = t
RawRequest = getblockStub RawRequest = getblockStub
os.RemoveAll(unitTestPath) os.RemoveAll(unitTestPath)
testcache = NewBlockCache(unitTestPath, unitTestChain, 380640, true) testcache = NewBlockCache(unitTestPath, unitTestChain, 380640, 0)
blockChan := make(chan *walletrpc.CompactBlock) blockChan := make(chan *walletrpc.CompactBlock)
errChan := make(chan error) errChan := make(chan error)
go GetBlockRange(testcache, blockChan, errChan, 380640, 380642) go GetBlockRange(testcache, blockChan, errChan, 380640, 380642)
@ -567,7 +567,7 @@ func TestGetBlockRangeReverse(t *testing.T) {
testT = t testT = t
RawRequest = getblockStubReverse RawRequest = getblockStubReverse
os.RemoveAll(unitTestPath) os.RemoveAll(unitTestPath)
testcache = NewBlockCache(unitTestPath, unitTestChain, 380640, true) testcache = NewBlockCache(unitTestPath, unitTestChain, 380640, 0)
blockChan := make(chan *walletrpc.CompactBlock) blockChan := make(chan *walletrpc.CompactBlock)
errChan := make(chan error) errChan := make(chan error)

View File

@ -36,7 +36,7 @@ const (
func testsetup() (walletrpc.CompactTxStreamerServer, *common.BlockCache) { func testsetup() (walletrpc.CompactTxStreamerServer, *common.BlockCache) {
os.RemoveAll(unitTestPath) os.RemoveAll(unitTestPath)
cache := common.NewBlockCache(unitTestPath, unitTestChain, 380640, true) cache := common.NewBlockCache(unitTestPath, unitTestChain, 380640, 0)
lwd, err := NewLwdStreamer(cache, "main", false /* enablePing */) lwd, err := NewLwdStreamer(cache, "main", false /* enablePing */)
if err != nil { if err != nil {
os.Stderr.WriteString(fmt.Sprint("NewLwdStreamer failed:", err)) os.Stderr.WriteString(fmt.Sprint("NewLwdStreamer failed:", err))