diff --git a/Makefile b/Makefile index dab58ea..f27fde0 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,7 @@ GENERATED_FILES := docs/rtd/index.html walletrpc/compact_formats.pb.go walletrpc PWD := $(shell pwd) -.PHONY: all dep build clean test coverage lint doc simpledoc +.PHONY: all dep build clean test coverage lint doc simpledoc proto all: first-make-timestamp build $(GENERATED_FILES) @@ -85,6 +85,8 @@ doc: docs/rtd/index.html docs/rtd/index.html: walletrpc/compact_formats.proto walletrpc/service.proto walletrpc/darkside.proto docker run --rm -v $(PWD)/docs/rtd:/out -v $(PWD)/walletrpc:/protos pseudomuto/protoc-gen-doc +proto: walletrpc/service.pb.go walletrpc/darkside.pb.go + walletrpc/service.pb.go: walletrpc/service.proto cd walletrpc && protoc service.proto --go_out=plugins=grpc:. diff --git a/cmd/root.go b/cmd/root.go index a2c328c..5302a94 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -49,6 +49,7 @@ var rootCmd = &cobra.Command{ DataDir: viper.GetString("data-dir"), Redownload: viper.GetBool("redownload"), Darkside: viper.GetBool("darkside-very-insecure"), + DarksideTimeout: viper.GetUint64("darkside-timeout"), } common.Log.Debugf("Options: %#v\n", opts) @@ -162,8 +163,12 @@ func startServer(opts *common.Options) error { // sending transactions, but in the future it could back a different type // of block streamer. + var saplingHeight int + var blockHeight int + var chainName string + var branchID string if opts.Darkside { - common.RawRequest = common.DarkSideRawRequest + chainName = "darkside" } else { rpcClient, err := frontend.NewZRPCFromConf(opts.ZcashConfPath) if err != nil { @@ -173,13 +178,15 @@ func startServer(opts *common.Options) error { } // Indirect function for test mocking (so unit tests can talk to stub functions). common.RawRequest = rpcClient.RawRequest + // Get the sapling activation height from the RPC + // (this first RPC also verifies that we can communicate with zcashd) + saplingHeight, blockHeight, chainName, branchID = common.GetSaplingInfo() + common.Log.Info("Got sapling height ", saplingHeight, + " block height ", blockHeight, + " chain ", chainName, + " branchID ", branchID) } - // Get the sapling activation height from the RPC - // (this first RPC also verifies that we can communicate with zcashd) - saplingHeight, blockHeight, chainName, branchID := common.GetSaplingInfo() - common.Log.Info("Got sapling height ", saplingHeight, " block height ", blockHeight, " chain ", chainName, " branchID ", branchID) - dbPath := filepath.Join(opts.DataDir, "db") if opts.Darkside { os.RemoveAll(filepath.Join(dbPath, chainName)) @@ -194,7 +201,12 @@ func startServer(opts *common.Options) error { os.Exit(1) } cache := common.NewBlockCache(dbPath, chainName, saplingHeight, opts.Redownload) - go common.BlockIngestor(cache, 0 /*loop forever*/) + if !opts.Darkside { + go common.BlockIngestor(cache, 0 /*loop forever*/) + } else { + // Darkside wants to control starting the block ingestor. + common.DarksideInit(cache, int(opts.DarksideTimeout)) + } // Compact transaction service initialization { @@ -270,6 +282,7 @@ func init() { rootCmd.Flags().Bool("redownload", false, "re-fetch all blocks from zcashd; reinitialize local cache files") rootCmd.Flags().String("data-dir", "/var/lib/lightwalletd", "data directory (such as db)") rootCmd.Flags().Bool("darkside-very-insecure", false, "run with GRPC-controllable mock zcashd for integration testing (shuts down after 30 minutes)") + rootCmd.Flags().Int("darkside-timeout", 30, "override 30 minute default darkside timeout") viper.BindPFlag("grpc-bind-addr", rootCmd.Flags().Lookup("grpc-bind-addr")) viper.SetDefault("grpc-bind-addr", "127.0.0.1:9067") @@ -293,6 +306,8 @@ func init() { viper.SetDefault("data-dir", "/var/lib/lightwalletd") viper.BindPFlag("darkside-very-insecure", rootCmd.Flags().Lookup("darkside-very-insecure")) viper.SetDefault("darkside-very-insecure", false) + viper.BindPFlag("darkside-timeout", rootCmd.Flags().Lookup("darkside-timeout")) + viper.SetDefault("darkside-timeout", 30) logger.SetFormatter(&logrus.TextFormatter{ //DisableColors: true, diff --git a/common/cache.go b/common/cache.go index 111c402..708d1f4 100644 --- a/common/cache.go +++ b/common/cache.go @@ -17,10 +17,6 @@ import ( "github.com/zcash/lightwalletd/walletrpc" ) -type blockCacheEntry struct { - data []byte -} - // BlockCache contains a consecutive set of recent compact blocks in marshalled form. type BlockCache struct { lengthsName, blocksName string // pathnames @@ -44,7 +40,7 @@ func (c *BlockCache) GetLatestHash() []byte { return c.latestHash } -// HashMismatch indicates if the given prev-hash doesn't match the most recent block's hash +// HashMismatch indicates if the given prev-hash doesn't match the most recent block's hash // so reorgs can be detected. func (c *BlockCache) HashMismatch(prevhash []byte) bool { c.mutex.RLock() @@ -175,6 +171,12 @@ func (c *BlockCache) setLatestHash() { } } +func (c *BlockCache) Reset(startHeight int) { + c.setDbFiles(c.firstBlock) // empty the cache + c.firstBlock = startHeight + c.nextBlock = startHeight +} + // NewBlockCache returns an instance of a block cache object. // (No locking here, we assume this is single-threaded.) func NewBlockCache(dbPath string, chainName string, startHeight int, redownload bool) *BlockCache { @@ -267,7 +269,7 @@ func (c *BlockCache) Add(height int, block *walletrpc.CompactBlock) error { } bheight := int(block.Height) - // TODO COINBASE-HEIGHT: restore this check after coinbase height is fixed + // XXX check? TODO COINBASE-HEIGHT: restore this check after coinbase height is fixed if false && bheight != height { // This could only happen if zcashd returned the wrong // block (not the height we requested). @@ -370,7 +372,7 @@ func (c *BlockCache) Sync() { c.blocksFile.Sync() } -// Currently used only for testing. +// Close is Currently used only for testing. func (c *BlockCache) Close() { // Some operating system require you to close files before you can remove them. if c.lengthsFile != nil { diff --git a/common/common.go b/common/common.go index d5982d4..d00e4e7 100644 --- a/common/common.go +++ b/common/common.go @@ -1,6 +1,7 @@ // Copyright (c) 2019-2020 The Zcash developers // Distributed under the MIT software license, see the accompanying // file COPYING or https://www.opensource.org/licenses/mit-license.php . + package common import ( @@ -34,6 +35,7 @@ type Options struct { Redownload bool `json:"redownload"` DataDir string `json:"data-dir"` Darkside bool `json:"darkside"` + DarksideTimeout uint64 `json:"darkside-timeout"` } // RawRequest points to the function to send a an RPC request to zcashd; @@ -49,10 +51,27 @@ var Sleep func(d time.Duration) // Log as a global variable simplifies logging var Log *logrus.Entry +type ( + Upgradeinfo struct { + // there are other fields that aren't needed here, omit them + ActivationHeight int + } + ConsensusInfo struct { + Nextblock string + Chaintip string + } + Blockchaininfo struct { + Chain string + Upgrades map[string]Upgradeinfo + Headers int + Consensus ConsensusInfo + } +) + // GetSaplingInfo returns the result of the getblockchaininfo RPC to zcashd func GetSaplingInfo() (int, int, string, string) { // This request must succeed or we can't go on; give zcashd time to start up - var f interface{} + var blockchaininfo Blockchaininfo retryCount := 0 for { result, rpcErr := RawRequest("getblockchaininfo", []json.RawMessage{}) @@ -60,7 +79,7 @@ func GetSaplingInfo() (int, int, string, string) { if retryCount > 0 { Log.Warn("getblockchaininfo RPC successful") } - err := json.Unmarshal(result, &f) + err := json.Unmarshal(result, &blockchaininfo) if err != nil { Log.Fatalf("error parsing JSON getblockchaininfo response: %v", err) } @@ -79,23 +98,14 @@ func GetSaplingInfo() (int, int, string, string) { Sleep(time.Duration(10+retryCount*5) * time.Second) // backoff } - chainName := f.(map[string]interface{})["chain"].(string) - - upgradeJSON := f.(map[string]interface{})["upgrades"] - // If the sapling consensus branch doesn't exist, it must be regtest - saplingHeight := float64(0) - if saplingJSON, ok := upgradeJSON.(map[string]interface{})["76b809bb"]; ok { // Sapling ID - saplingHeight = saplingJSON.(map[string]interface{})["activationheight"].(float64) + var saplingHeight int + if saplingJSON, ok := blockchaininfo.Upgrades["76b809bb"]; ok { // Sapling ID + saplingHeight = saplingJSON.ActivationHeight } - blockHeight := f.(map[string]interface{})["headers"].(float64) - - consensus := f.(map[string]interface{})["consensus"] - - branchID := consensus.(map[string]interface{})["nextblock"].(string) - - return int(saplingHeight), int(blockHeight), chainName, branchID + return saplingHeight, blockchaininfo.Headers, blockchaininfo.Chain, + blockchaininfo.Consensus.Nextblock } func getBlockFromRPC(height int) (*walletrpc.CompactBlock, error) { @@ -133,14 +143,31 @@ func getBlockFromRPC(height int) (*walletrpc.CompactBlock, error) { return nil, errors.New("received overlong message") } - // TODO COINBASE-HEIGHT: restore this check after coinbase height is fixed - if false && block.GetHeight() != height { + if block.GetHeight() != height { return nil, errors.New("received unexpected height block") } return block.ToCompact(), nil } +var ( + ingestorRunning bool + stopIngestorChan = make(chan struct{}) +) + +func startIngestor(c *BlockCache) { + if !ingestorRunning { + ingestorRunning = true + go BlockIngestor(c, 0) + } +} +func stopIngestor() { + if ingestorRunning { + ingestorRunning = false + stopIngestorChan <- struct{}{} + } +} + // BlockIngestor runs as a goroutine and polls zcashd for new blocks, adding them // to the cache. The repetition count, rep, is nonzero only for unit-testing. func BlockIngestor(c *BlockCache, rep int) { @@ -152,6 +179,13 @@ func BlockIngestor(c *BlockCache, rep int) { // Start listening for new blocks for i := 0; rep == 0 || i < rep; i++ { + // stop if requested + select { + case <-stopIngestorChan: + return + default: + } + height := c.GetNextHeight() block, err := getBlockFromRPC(height) if err != nil { @@ -178,9 +212,10 @@ func BlockIngestor(c *BlockCache, rep int) { // Wait a bit then retry the same height. c.Sync() if lastHeightLogged+1 != height { - Log.Info("Ingestor waiting for block: ", height) + Log.Info("Ingestor: waiting for block: ", height) + lastHeightLogged = height - 1 } - Sleep(10 * time.Second) + Sleep(2 * time.Second) wait = false continue } @@ -190,7 +225,7 @@ func BlockIngestor(c *BlockCache, rep int) { // and there's no new block yet, but we want to back up // so we detect a reorg in which the new chain is the // same length or shorter. - reorgCount += 1 + reorgCount++ if reorgCount > 100 { Log.Fatal("Reorg exceeded max of 100 blocks! Help!") } @@ -212,7 +247,6 @@ func BlockIngestor(c *BlockCache, rep int) { } // Try backing up c.Reorg(height - 1) - Sleep(1 * time.Second) continue } // We have a valid block to add. @@ -253,7 +287,7 @@ func GetBlock(cache *BlockCache, height int) (*walletrpc.CompactBlock, error) { } // GetBlockRange returns a sequence of consecutive blocks in the given range. -func GetBlockRange(cache *BlockCache, blockOut chan<- walletrpc.CompactBlock, errOut chan<- error, start, end int) { +func GetBlockRange(cache *BlockCache, blockOut chan<- *walletrpc.CompactBlock, errOut chan<- error, start, end int) { // Go over [start, end] inclusive for i := start; i <= end; i++ { block, err := GetBlock(cache, i) @@ -261,7 +295,7 @@ func GetBlockRange(cache *BlockCache, blockOut chan<- walletrpc.CompactBlock, er errOut <- err return } - blockOut <- *block + blockOut <- block } errOut <- nil } diff --git a/common/common_test.go b/common/common_test.go index 09f3b63..db1f981 100644 --- a/common/common_test.go +++ b/common/common_test.go @@ -179,7 +179,7 @@ func getblockStub(method string, params []json.RawMessage) (json.RawMessage, err // this should cause one 10s sleep (then retry). return nil, errors.New("-8: Block height out of range") case 4: - if sleepCount != 1 || sleepDuration != 10*time.Second { + if sleepCount != 1 || sleepDuration != 2*time.Second { testT.Error("unexpected sleeps", sleepCount, sleepDuration) } if height != "380642" { @@ -189,7 +189,7 @@ func getblockStub(method string, params []json.RawMessage) (json.RawMessage, err // wait then a check for reorg to shorter chain (back up one). return nil, errors.New("-8: Block height out of range") case 5: - if sleepCount != 2 || sleepDuration != 11*time.Second { + if sleepCount != 1 || sleepDuration != 2*time.Second { testT.Error("unexpected sleeps", sleepCount, sleepDuration) } // Back up to 41. @@ -200,7 +200,7 @@ func getblockStub(method string, params []json.RawMessage) (json.RawMessage, err // ingestor will immediately re-request the next block (42). return blocks[1], nil case 6: - if sleepCount != 2 || sleepDuration != 11*time.Second { + if sleepCount != 1 || sleepDuration != 2*time.Second { testT.Error("unexpected sleeps", sleepCount, sleepDuration) } if height != "380642" { @@ -209,7 +209,7 @@ func getblockStub(method string, params []json.RawMessage) (json.RawMessage, err // Block 42 has now finally appeared, it will immediately ask for 43. return blocks[2], nil case 7: - if sleepCount != 2 || sleepDuration != 11*time.Second { + if sleepCount != 1 || sleepDuration != 2*time.Second { testT.Error("unexpected sleeps", sleepCount, sleepDuration) } if height != "380643" { @@ -221,7 +221,7 @@ func getblockStub(method string, params []json.RawMessage) (json.RawMessage, err return blocks[3], nil case 8: blocks[3][9]-- // repair first byte of the prevhash - if sleepCount != 3 || sleepDuration != 12*time.Second { + if sleepCount != 1 || sleepDuration != 2*time.Second { testT.Error("unexpected sleeps", sleepCount, sleepDuration) } if height != "380642" { @@ -229,7 +229,7 @@ func getblockStub(method string, params []json.RawMessage) (json.RawMessage, err } return blocks[2], nil case 9: - if sleepCount != 3 || sleepDuration != 12*time.Second { + if sleepCount != 1 || sleepDuration != 2*time.Second { testT.Error("unexpected sleeps", sleepCount, sleepDuration) } if height != "380643" { @@ -239,7 +239,7 @@ func getblockStub(method string, params []json.RawMessage) (json.RawMessage, err // failure, should cause 10s sleep, retry return nil, nil case 10: - if sleepCount != 4 || sleepDuration != 22*time.Second { + if sleepCount != 2 || sleepDuration != 12*time.Second { testT.Error("unexpected sleeps", sleepCount, sleepDuration) } if height != "380643" { @@ -248,7 +248,7 @@ func getblockStub(method string, params []json.RawMessage) (json.RawMessage, err // Back to sunny-day return blocks[3], nil case 11: - if sleepCount != 4 || sleepDuration != 22*time.Second { + if sleepCount != 2 || sleepDuration != 12*time.Second { testT.Error("unexpected sleeps", sleepCount, sleepDuration) } if height != "380644" { @@ -282,7 +282,7 @@ func TestGetBlockRange(t *testing.T) { RawRequest = getblockStub os.RemoveAll(unitTestPath) testcache := NewBlockCache(unitTestPath, unitTestChain, 380640, true) - blockChan := make(chan walletrpc.CompactBlock) + blockChan := make(chan *walletrpc.CompactBlock) errChan := make(chan error) go GetBlockRange(testcache, blockChan, errChan, 380640, 380642) diff --git a/common/darkside.go b/common/darkside.go index 33a8331..3c82233 100644 --- a/common/darkside.go +++ b/common/darkside.go @@ -2,167 +2,474 @@ package common import ( "bufio" + "bytes" + "crypto/sha256" "encoding/hex" "encoding/json" "errors" - "os" + "fmt" + "net/http" "strconv" + "strings" + "sync" "time" + + "github.com/zcash/lightwalletd/parser" ) -type DarksideZcashdState struct { - start_height int - sapling_activation int - branch_id string - chain_name string - // Should always be nonempty. Index 0 is the block at height start_height. - blocks []string - incoming_transactions [][]byte - server_start time.Time +type darksideState struct { + resetted bool + startHeight int // activeBlocks[0] corresponds to this height + branchID string + chainName string + cache *BlockCache + mutex sync.RWMutex + + // This is the highest (latest) block height currently being presented + // by the mock zcashd. + latestHeight int + + // These blocks (up to and including tip) are presented by mock zcashd. + // activeBlocks[0] is the block at height startHeight. + activeBlocks [][]byte // full blocks, binary, as from zcashd getblock rpc + + // Staged blocks are waiting to be applied (by ApplyStaged()) to activeBlocks. + // They are in order of arrival (not necessarily sorted by height), and are + // applied in arrival order. + stagedBlocks [][]byte // full blocks, binary + + // These are full transactions as received from the wallet by SendTransaction(). + // They are conceptually in the mempool. They are not yet available to be fetched + // by GetTransaction(). They can be fetched by darkside GetIncomingTransaction(). + incomingTransactions [][]byte + + // These transactions come from StageTransactions(); they will be merged into + // activeBlocks by ApplyStaged() (and this list then cleared). + stagedTransactions []stagedTx } -var state *DarksideZcashdState = nil +var state darksideState -func DarkSideRawRequest(method string, params []json.RawMessage) (json.RawMessage, error) { +type stagedTx struct { + height int + bytes []byte +} - if state == nil { - state = &DarksideZcashdState{ - start_height: 1000, - sapling_activation: 1000, - branch_id: "2bb40e60", // Blossom - chain_name: "darkside", - blocks: make([]string, 0), - incoming_transactions: make([][]byte, 0), - server_start: time.Now(), - } +// DarksideEnabled is true if --darkside-very-insecure was given on +// the command line. +var DarksideEnabled bool - testBlocks, err := os.Open("./testdata/default-darkside-blocks") - if err != nil { - Log.Fatal("Error loading default darksidewalletd blocks") - } - scan := bufio.NewScanner(testBlocks) - for scan.Scan() { // each line (block) - block := scan.Bytes() - state.blocks = append(state.blocks, string(block)) - } - } - - if time.Now().Sub(state.server_start).Minutes() >= 30 { +// DarksideInit should be called once at startup in darksidewalletd mode. +func DarksideInit(c *BlockCache, timeout int) { + Log.Info("Darkside mode running") + DarksideEnabled = true + state.cache = c + RawRequest = darksideRawRequest + go func() { + time.Sleep(time.Duration(timeout) * time.Minute) Log.Fatal("Shutting down darksidewalletd to prevent accidental deployment in production.") + }() +} + +// DarksideReset allows the wallet test code to specify values +// that are returned by GetLightdInfo(). +func DarksideReset(sa int, bi, cn string) error { + Log.Info("Reset(saplingActivation=", sa, ")") + stopIngestor() + state = darksideState{ + resetted: true, + startHeight: sa, + latestHeight: -1, + branchID: bi, + chainName: cn, + cache: state.cache, + activeBlocks: make([][]byte, 0), + stagedBlocks: make([][]byte, 0), + incomingTransactions: make([][]byte, 0), + stagedTransactions: make([]stagedTx, 0), + } + state.cache.Reset(sa) + return nil +} + +// DarksideAddBlock adds a single block to the active blocks list. +func addBlockActive(blockBytes []byte) error { + block := parser.NewBlock() + rest, err := block.ParseFromSlice(blockBytes) + if err != nil { + return err + } + if len(rest) != 0 { + return errors.New("block serialization is too long") + } + blockHeight := block.GetHeight() + // first block, add to existing blocks slice if possible + if blockHeight > state.startHeight+len(state.activeBlocks) { + return errors.New(fmt.Sprint("adding block at height ", blockHeight, + " would create a gap in the blockchain")) + } + if blockHeight < state.startHeight { + return errors.New(fmt.Sprint("adding block at height ", blockHeight, + " is lower than Sapling activation height ", state.startHeight)) + } + // Drop the block that will be overwritten, and its children, then add block. + state.activeBlocks = state.activeBlocks[:blockHeight-state.startHeight] + state.activeBlocks = append(state.activeBlocks, blockBytes) + return nil +} + +// Set missing prev hashes of the blocks in the active chain +func setPrevhash() { + var prevhash []byte + for _, blockBytes := range state.activeBlocks { + // Set this block's prevhash. + block := parser.NewBlock() + rest, err := block.ParseFromSlice(blockBytes) + if err != nil { + Log.Fatal(err) + } + if len(rest) != 0 { + Log.Fatal(errors.New("block is too long")) + } + if prevhash != nil { + copy(blockBytes[4:4+32], prevhash) + } + prevhash = block.GetEncodableHash() + Log.Info("height ", block.GetHeight(), " hash ", + hex.EncodeToString(block.GetDisplayHash())) + } +} + +// DarksideApplyStaged moves the staging area to the active block list. +// If this returns an error, the state could be weird; perhaps it may +// be better to simply crash. +func DarksideApplyStaged(height int) error { + state.mutex.Lock() + defer state.mutex.Unlock() + if !state.resetted { + return errors.New("please call Reset first") + } + Log.Info("ApplyStaged(height=", height, ")") + // Move the staged blocks into active list + stagedBlocks := state.stagedBlocks + state.stagedBlocks = nil + for _, blockBytes := range stagedBlocks { + if err := addBlockActive(blockBytes); err != nil { + return err + } + } + if height > state.startHeight+len(state.activeBlocks)-1 { + // this is hard to recover from + Log.Fatal("ApplyStaged height ", height, + " is greater than the highest height ", + state.startHeight+len(state.activeBlocks)-1) } + // Add staged transactions into blocks. Note we're not trying to + // recover to the initial state; maybe it's better to just crash + // on errors. + stagedTransactions := state.stagedTransactions + state.stagedTransactions = nil + for _, tx := range stagedTransactions { + if tx.height < state.startHeight { + return errors.New("transaction height too low") + } + if tx.height >= state.startHeight+len(state.activeBlocks) { + return errors.New("transaction height too high") + } + block := state.activeBlocks[tx.height-state.startHeight] + if block[1487] == 253 { + return errors.New("too many transactions in a block (max 253)") + } + block[1487]++ // one more transaction + block[68]++ // hack HashFinalSaplingRoot to mod the block hash + block = append(block, tx.bytes...) + state.activeBlocks[tx.height-state.startHeight] = block + } + setPrevhash() + state.latestHeight = height + Log.Info("active blocks from ", state.startHeight, + " to ", state.startHeight+len(state.activeBlocks)-1, + ", latest presented height ", state.latestHeight) + + // The block ingestor can only run if there are blocks + if len(state.activeBlocks) > 0 { + startIngestor(state.cache) + } else { + stopIngestor() + } + return nil +} + +// DarksideGetIncomingTransactions returns all transactions we're +// received via SendTransaction(). +func DarksideGetIncomingTransactions() [][]byte { + return state.incomingTransactions +} + +// DarksideStageBlocks opens and reads blocks from the given URL and +// adds them to the staging area. +func DarksideStageBlocks(url string) error { + if !state.resetted { + return errors.New("please call Reset first") + } + Log.Info("StageBlocks(url=", url, ")") + resp, err := http.Get(url) + if err != nil { + return err + } + defer resp.Body.Close() + // some blocks are too large, especially when encoded in hex, for the + // default buffer size, so set up a larger one; 8mb should be enough. + scan := bufio.NewScanner(resp.Body) + var scanbuf []byte + scan.Buffer(scanbuf, 8*1000*1000) + for scan.Scan() { // each line (block) + blockHex := scan.Text() + if blockHex == "404: Not Found" { + // special case error (http resource not found, bad pathname) + return errors.New(blockHex) + } + blockBytes, err := hex.DecodeString(blockHex) + if err != nil { + return err + } + state.stagedBlocks = append(state.stagedBlocks, blockBytes) + } + return scan.Err() +} + +// DarksideStageBlockStream adds the block to the staging area +func DarksideStageBlockStream(blockHex string) error { + if !state.resetted { + return errors.New("please call Reset first") + } + Log.Info("StageBlocks()") + blockBytes, err := hex.DecodeString(blockHex) + if err != nil { + return err + } + state.stagedBlocks = append(state.stagedBlocks, blockBytes) + return nil +} + +// DarksideStageBlocksCreate creates empty blocks and adds them to the staging area. +func DarksideStageBlocksCreate(height int32, nonce int32, count int32) error { + if !state.resetted { + return errors.New("please call Reset first") + } + Log.Info("StageBlocksCreate(height=", height, ", nonce=", nonce, "count=", count, ")") + for i := 0; i < int(count); i++ { + + fakeCoinbase := "0400008085202f890100000000000000000000000000000000000000000000000000" + + "00000000000000ffffffff2a03d12c0c00043855975e464b8896790758f824ceac97836" + + "22c17ed38f1669b8a45ce1da857dbbe7950e2ffffffff02a0ebce1d000000001976a914" + + "7ed15946ec14ae0cd8fa8991eb6084452eb3f77c88ac405973070000000017a914e445cf" + + "a944b6f2bdacefbda904a81d5fdd26d77f8700000000000000000000000000000000000000" + + // This coinbase transaction was pulled from block 797905, whose + // little-endian encoding is 0xD12C0C00. Replace it with the block + // number we want. + fakeCoinbase = strings.Replace(fakeCoinbase, "d12c0c00", + fmt.Sprintf("%02x", height&0xFF)+ + fmt.Sprintf("%02x", (height>>8)&0xFF)+ + fmt.Sprintf("%02x", (height>>16)&0xFF)+ + fmt.Sprintf("%02x", (height>>24)&0xFF), 1) + fakeCoinbaseBytes, err := hex.DecodeString(fakeCoinbase) + if err != nil { + Log.Fatal(err) + } + + hashOfTxnsAndHeight := sha256.Sum256([]byte(string(nonce) + "#" + string(height))) + blockHeader := &parser.BlockHeader{ + RawBlockHeader: &parser.RawBlockHeader{ + Version: 4, // start: 0 + HashPrevBlock: make([]byte, 32), // start: 4 + HashMerkleRoot: hashOfTxnsAndHeight[:], // start: 36 + HashFinalSaplingRoot: make([]byte, 32), // start: 68 + Time: 1, // start: 100 + NBitsBytes: make([]byte, 4), // start: 104 + Nonce: make([]byte, 32), // start: 108 + Solution: make([]byte, 1344), // starts: 140, 143 + }, // length: 1487 + } + + headerBytes, err := blockHeader.MarshalBinary() + if err != nil { + Log.Fatal(err) + } + blockBytes := make([]byte, 0) + blockBytes = append(blockBytes, headerBytes...) + blockBytes = append(blockBytes, byte(1)) + blockBytes = append(blockBytes, fakeCoinbaseBytes...) + state.stagedBlocks = append(state.stagedBlocks, blockBytes) + height++ + } + return nil +} + +// DarksideClearIncomingTransactions empties the incoming transaction list. +func DarksideClearIncomingTransactions() { + state.incomingTransactions = make([][]byte, 0) +} + +func darksideRawRequest(method string, params []json.RawMessage) (json.RawMessage, error) { switch method { case "getblockchaininfo": - type upgradeinfo struct { - // there are other fields that aren't needed here, omit them - ActivationHeight int `json:"activationheight"` - } - type consensus struct { - Nextblock string `json:"nextblock"` - Chaintip string `json:"chaintip"` - } - blockchaininfo := struct { - Chain string `json:"chain"` - Upgrades map[string]upgradeinfo `json:"upgrades"` - Headers int `json:"headers"` - Consensus consensus `json:"consensus"` - }{ - Chain: state.chain_name, - Upgrades: map[string]upgradeinfo{ - "76b809bb": upgradeinfo{ActivationHeight: state.sapling_activation}, + blockchaininfo := Blockchaininfo{ + Chain: state.chainName, + Upgrades: map[string]Upgradeinfo{ + "76b809bb": {ActivationHeight: state.startHeight}, }, - Headers: state.start_height + len(state.blocks) - 1, - Consensus: consensus{state.branch_id, state.branch_id}, + Headers: state.latestHeight, + Consensus: ConsensusInfo{state.branchID, state.branchID}, } return json.Marshal(blockchaininfo) case "getblock": - var height string - err := json.Unmarshal(params[0], &height) + var heightStr string + err := json.Unmarshal(params[0], &heightStr) if err != nil { - return nil, errors.New("Failed to parse getblock request.") + return nil, errors.New("failed to parse getblock request") } - height_i, err := strconv.Atoi(height) + height, err := strconv.Atoi(heightStr) if err != nil { - return nil, errors.New("Error parsing height as integer.") + return nil, errors.New("error parsing height as integer") } - index := height_i - state.start_height - - if index == len(state.blocks) { - // The current ingestor keeps going until it sees this error, - // meaning it's up to the latest height. - return nil, errors.New("-8:") + state.mutex.RLock() + defer state.mutex.RUnlock() + const notFoundErr = "-8:" + if len(state.activeBlocks) == 0 { + return nil, errors.New(notFoundErr) } - - if index < 0 || index > len(state.blocks) { - // If an integration test can reach this, it could be a bug, so generate an error. - Log.Errorf("getblock request made for out-of-range height %d (have %d to %d)", height_i, state.start_height, state.start_height+len(state.blocks)-1) - return nil, errors.New("-8:") + if height > state.latestHeight { + return nil, errors.New(notFoundErr) } - - return []byte("\"" + state.blocks[index] + "\""), nil + if height < state.startHeight { + return nil, errors.New(fmt.Sprint("getblock: requesting height ", height, + " is less than sapling activation height")) + } + index := height - state.startHeight + if index >= len(state.activeBlocks) { + return nil, errors.New(notFoundErr) + } + return []byte("\"" + hex.EncodeToString(state.activeBlocks[index]) + "\""), nil case "getaddresstxids": // Not required for minimal reorg testing. - return nil, errors.New("Not implemented yet.") + return nil, errors.New("not implemented yet") case "getrawtransaction": - // Not required for minimal reorg testing. - return nil, errors.New("Not implemented yet.") + return darksideGetRawTransaction(params) case "sendrawtransaction": var rawtx string err := json.Unmarshal(params[0], &rawtx) if err != nil { - return nil, errors.New("Failed to parse sendrawtransaction JSON.") + return nil, errors.New("failed to parse sendrawtransaction JSON") } - txbytes, err := hex.DecodeString(rawtx) + txBytes, err := hex.DecodeString(rawtx) if err != nil { - return nil, errors.New("Failed to parse sendrawtransaction value as a hex string.") + return nil, errors.New("failed to parse sendrawtransaction value as a hex string") } - state.incoming_transactions = append(state.incoming_transactions, txbytes) - return nil, nil - - case "x_setstate": - var new_state map[string]interface{} - - err := json.Unmarshal(params[0], &new_state) + // Parse the transaction to get its hash (txid). + tx := parser.NewTransaction() + rest, err := tx.ParseFromSlice(txBytes) if err != nil { - Log.Fatal("Could not unmarshal the provided state.") + return nil, err } - - block_strings := make([]string, 0) - for _, block_str := range new_state["blocks"].([]interface{}) { - block_strings = append(block_strings, block_str.(string)) + if len(rest) != 0 { + return nil, errors.New("transaction serialization is too long") } - - state = &DarksideZcashdState{ - start_height: int(new_state["start_height"].(float64)), - sapling_activation: int(new_state["sapling_activation"].(float64)), - branch_id: new_state["branch_id"].(string), - chain_name: new_state["chain_name"].(string), - blocks: block_strings, - incoming_transactions: state.incoming_transactions, - server_start: state.server_start, - } - - return nil, nil - - case "x_getincomingtransactions": - txlist := "[" - for i, tx := range state.incoming_transactions { - txlist += "\"" + hex.EncodeToString(tx) + "\"" - // add commas after all but the last - if i < len(state.incoming_transactions)-1 { - txlist += ", " - } - } - txlist += "]" - - return []byte(txlist), nil - + state.incomingTransactions = append(state.incomingTransactions, txBytes) + return tx.GetDisplayHash(), nil default: - return nil, errors.New("There was an attempt to call an unsupported RPC.") + return nil, errors.New("there was an attempt to call an unsupported RPC") } } + +func darksideGetRawTransaction(params []json.RawMessage) (json.RawMessage, error) { + if !state.resetted { + return nil, errors.New("please call Reset first") + } + // remove the double-quotes from the beginning and end of the hex txid string + txbytes, err := hex.DecodeString(string(params[0][1 : 1+64])) + if err != nil { + return nil, errors.New("-9: " + err.Error()) + } + // Linear search for the tx, somewhat inefficient but this is test code + // and there aren't many blocks. If this becomes a performance problem, + // we can maintain a map of transactions indexed by txid. + for _, b := range state.activeBlocks { + block := parser.NewBlock() + rest, err := block.ParseFromSlice(b) + if err != nil { + // this would be strange; we've already parsed this block + return nil, errors.New("-9: " + err.Error()) + } + if len(rest) != 0 { + return nil, errors.New("-9: block serialization is too long") + } + for _, tx := range block.Transactions() { + if bytes.Equal(tx.GetDisplayHash(), txbytes) { + reply := struct { + Hex string `json:"hex"` + Height int `json:"height"` + }{hex.EncodeToString(tx.Bytes()), block.GetHeight()} + return json.Marshal(reply) + } + } + } + return nil, errors.New("-5: No information available about transaction") +} + +// DarksideStageTransaction adds the given transaction to the staging area. +func DarksideStageTransaction(height int, txBytes []byte) error { + if !state.resetted { + return errors.New("please call Reset first") + } + Log.Info("StageTransactions(height=", height, ")") + state.stagedTransactions = append(state.stagedTransactions, + stagedTx{ + height: height, + bytes: txBytes, + }) + return nil +} + +// DarksideStageTransactionsURL reads a list of transactions (hex-encoded, one +// per line) from the given URL, and associates them with the given height. +func DarksideStageTransactionsURL(height int, url string) error { + if !state.resetted { + return errors.New("please call Reset first") + } + Log.Info("StageTransactionsURL(height=", height, "url=", url, ")") + resp, err := http.Get(url) + if err != nil { + return err + } + defer resp.Body.Close() + // some blocks are too large, especially when encoded in hex, for the + // default buffer size, so set up a larger one; 8mb should be enough. + scan := bufio.NewScanner(resp.Body) + var scanbuf []byte + scan.Buffer(scanbuf, 8*1000*1000) + for scan.Scan() { // each line (transaction) + transactionHex := scan.Text() + if transactionHex == "404: Not Found" { + // special case error (http resource not found, bad pathname) + return errors.New(transactionHex) + } + transactionBytes, err := hex.DecodeString(transactionHex) + if err != nil { + return err + } + state.stagedTransactions = append(state.stagedTransactions, stagedTx{ + height: height, + bytes: transactionBytes, + }) + } + return scan.Err() + +} diff --git a/docs/darksidewalletd.md b/docs/darksidewalletd.md index e84c3bd..7ff44a0 100644 --- a/docs/darksidewalletd.md +++ b/docs/darksidewalletd.md @@ -13,6 +13,16 @@ same darksidewalletd at the same time. Darksidewalletd should only be used for testing, and therefore is hard-coded to shut down after 30 minutes of operation to prevent accidental deployment as a server. +## Security warning + +Leaving darksidewalletd running puts your machine at greater risk because (a) it +may be possible to use file: paths with `DarksideSetBlocksURL` to read arbitrary +files on your system, and (b) also using `DarksideSetBlocksURL`, someone can +force your system to make a web request to an arbitrary URL (which could have +your system download questionable material, perform attacks on other systems, +etc.). The maximum 30-minute run time limit built into darksidewalletd mitigates +these risks, but users should still be cautious. + ## Dependencies Lightwalletd and most dependencies of lightwalletd, including Go version 1.11 or @@ -34,7 +44,7 @@ after 30 minutes. ### Default set of blocks -There’s a file in the repo called ./testdata/default-darkside-blocks. This +There’s a file in the repo called ./testdata/darkside/init-blocks. This contains the blocks darksidewalletd loads by default. The format of the file is one hex-encoded block per line. @@ -74,7 +84,7 @@ echo “some hex-encoded transaction you want to put in block 1003” > blocksA/ ``` This will output the blocks, one hex-encoded block per line (the same format as -./testdata/default-darkside-blocks). +./testdata/darkside/init-blocks). Tip: Because nothing is checking the full validity of transactions, you can get any hex-encoded transaction you want from a block explorer and put those in the @@ -93,20 +103,28 @@ submit the blocks, which internally uses grpcurl, e.g.: ``` ./genblocks --blocks-dir blocksA > blocksA.txt -./utils/submitblocks.sh 1000 1000 blocksA.txt +./utils/submitblocks.sh 1000 blocksA.txt ``` -In the submitblocks.sh command, the first “1000” is the height to serve the -first block at (so that if blocksA.txt contains 6 blocks they will be served as -heights 1000, 1001, 1002, 1003, 1004, and 1005. If the genblocks tool was used -to create the blocksA file, then this argument must match what was given to -genblocks, otherwise the heights in the coinbase transactions will not match up -with the height lightwalletd is serving the blocks as. The second “1000” sets -the value that lightwalletd will report the sapling activation height to be. +In the submitblocks.sh command, the “1000” sets the value that lightwalletd will +report the sapling activation height to be. -Tip: The DarksideSetState expects a complete set of blocks for the mock zcashd -to serve, if you want to just add one block, for example, you need to re-submit -all of the blocks including the new one. +Tip: You may submit blocks incrementally, that is, submit 1000-1005 followed +by 1006-1008, the result is 1000-1008. You can't create a gap in the range (say, +1000-1005 then 1007-1009). + +If you submit overlapping ranges, the expected things happen. For example, first +submit 1000-1005, then 1003-1007, the result is 1000-1007 (the original 1000-1002 +followed by the new 1003-1007). This is how you can create a reorg starting at 1003. +You can get the same effect slightly less efficiently by submitting 1000-1007 (that +is, resubmitting the original 1000-1002 followed by the new 1003-1007). + +If you first submit 1000-1005, then 1001-1002, the result will be 1000-1002 +(1003-1005 are dropped; it's not possible to "insert" blocks into a range). +Likewise, first submit 1005-1008, then 1000-1006, the result is only 1000-1006. An +easy way to state it is that all earlier blocks beyond the end of the extent of +the range being submitted now are dropped. But blocks before the start of the range +being submitted now are preserved if doing so doesn't create a gap. ## Tutorial ### Triggering a Reorg @@ -143,7 +161,7 @@ Now you can start darksidewalletd and it’ll load the blocksA blocks: That will have loaded and be serving the blocksA blocks. We can push up the blocksB blocks using ./utils/submitblocks.sh: -`./utils/submitblocks.sh 1000 1000 testdata/darkside-blocks-reorg` +`./utils/submitblocks.sh 1000 testdata/darkside-blocks-reorg` We should now see a reorg in server.log: @@ -152,6 +170,28 @@ We should now see a reorg in server.log: {"app":"frontend-grpc","hash":"a244942179988ea6e56a3a55509fcf22673df26200c67bebd93504385a1a7c4f","height":1004,"level":"warning","msg":"REORG","phash":"06e7c72646e3d51417de25bd83896c682b72bdf5be680908d621cba86d222798","reorg":1,"time":"2020-03-23T13:59:44-06:00"} ``` +### Precomputed block ranges + +The ECC has already created some block ranges to simulate reorgs in +the repository https://github.com/zcash-hackworks/darksidewalletd-test-data. +This may relieve you of the task of generating test blocks. There's a `gRPC` method +called `SetBlocksURL` that takes a resource location (anything that can be +given to `curl`; indeed, the lightwalletd uses `curl`). Here's an example: + +`grpcurl -plaintext -d '{"url":"https://raw.githubusercontent.com/zcash-hackworks/darksidewalletd-test-data/master/blocks-663242-663251"}' localhost:9067 cash.z.wallet.sdk.rpc.DarksideStreamer/SetBlocksURL` + +When lightwalletd starts up in darksidewalletd mode, it automatically does the +equivalent of: + +`grpcurl -plaintext -d '{"url":"file:testdata/darkside/init-blocks"}' localhost:9067 cash.z.wallet.sdk.rpc.DarksideStreamer/SetBlocksURL` + +which is also equivalent to (the `-d @` tells `grpcurl` to read from standard input): +``` +cat testdata/darkside/init-blocks | +sed 's/^/{"block":"/;s/$/"}/' | +grpcurl -plaintext -d @ localhost:9067 cash.z.wallet.sdk.rpc.DarksideStreamer/SetBlocks +``` + ## Use cases Check out some of the potential security test cases here: [wallet <-> @@ -160,9 +200,6 @@ tests](https://github.com/zcash/lightwalletd/blob/master/docs/integration-tests. ## Source Code * cmd/genblocks -- tool for generating fake block sets. -* testdata/default-darkside-blocks -- the set of blocks loaded by default +* testdata/darkside/init-blocks -- the set of blocks loaded by default * common/darkside.go -- implementation of darksidewalletd * frontend/service.go -- entrypoints for darksidewalletd GRPC APIs - - - diff --git a/docs/rtd/index.html b/docs/rtd/index.html index 11c4d45..dae36b3 100644 --- a/docs/rtd/index.html +++ b/docs/rtd/index.html @@ -206,7 +206,27 @@