diff --git a/cmd/root.go b/cmd/root.go index a2c328c..b302007 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -163,7 +163,7 @@ func startServer(opts *common.Options) error { // of block streamer. if opts.Darkside { - common.RawRequest = common.DarkSideRawRequest + common.DarksideInit() } else { rpcClient, err := frontend.NewZRPCFromConf(opts.ZcashConfPath) if err != nil { diff --git a/common/cache.go b/common/cache.go index 111c402..f2e4f75 100644 --- a/common/cache.go +++ b/common/cache.go @@ -370,7 +370,7 @@ func (c *BlockCache) Sync() { c.blocksFile.Sync() } -// Currently used only for testing. +// Close is Currently used only for testing. func (c *BlockCache) Close() { // Some operating system require you to close files before you can remove them. if c.lengthsFile != nil { diff --git a/common/common.go b/common/common.go index d5982d4..e7a3d04 100644 --- a/common/common.go +++ b/common/common.go @@ -1,6 +1,7 @@ // Copyright (c) 2019-2020 The Zcash developers // Distributed under the MIT software license, see the accompanying // file COPYING or https://www.opensource.org/licenses/mit-license.php . + package common import ( @@ -133,8 +134,7 @@ func getBlockFromRPC(height int) (*walletrpc.CompactBlock, error) { return nil, errors.New("received overlong message") } - // TODO COINBASE-HEIGHT: restore this check after coinbase height is fixed - if false && block.GetHeight() != height { + if block.GetHeight() != height { return nil, errors.New("received unexpected height block") } @@ -190,7 +190,7 @@ func BlockIngestor(c *BlockCache, rep int) { // and there's no new block yet, but we want to back up // so we detect a reorg in which the new chain is the // same length or shorter. - reorgCount += 1 + reorgCount++ if reorgCount > 100 { Log.Fatal("Reorg exceeded max of 100 blocks! Help!") } diff --git a/common/darkside.go b/common/darkside.go index 33a8331..57c89ce 100644 --- a/common/darkside.go +++ b/common/darkside.go @@ -5,49 +5,176 @@ import ( "encoding/hex" "encoding/json" "errors" + "io" + "io/ioutil" "os" + "os/exec" "strconv" + "strings" "time" + + "github.com/zcash/lightwalletd/parser" ) -type DarksideZcashdState struct { - start_height int - sapling_activation int - branch_id string - chain_name string +type darksideState struct { + startHeight int + saplingActivation int + branchID string + chainName string // Should always be nonempty. Index 0 is the block at height start_height. - blocks []string - incoming_transactions [][]byte - server_start time.Time + blocks [][]byte // full blocks, binary, as from zcashd getblock rpc + incomingTransactions [][]byte // full transactions, binary, zcashd getrawtransaction txid + serverStart time.Time } -var state *DarksideZcashdState = nil +var state darksideState -func DarkSideRawRequest(method string, params []json.RawMessage) (json.RawMessage, error) { +func DarksideIsEnabled() bool { + return state.chainName != "" +} - if state == nil { - state = &DarksideZcashdState{ - start_height: 1000, - sapling_activation: 1000, - branch_id: "2bb40e60", // Blossom - chain_name: "darkside", - blocks: make([]string, 0), - incoming_transactions: make([][]byte, 0), - server_start: time.Now(), - } +func DarksideInit() { + state = darksideState{ + startHeight: -1, + saplingActivation: -1, + branchID: "2bb40e60", // Blossom + chainName: "darkside", + blocks: make([][]byte, 0), + incomingTransactions: make([][]byte, 0), + serverStart: time.Now(), + } + RawRequest = darksideRawRequest + f := "testdata/darkside/init-blocks" + testBlocks, err := os.Open(f) + if err != nil { + Log.Warn("Error opening default darksidewalletd blocks file", f) + return + } + if err = readBlocks(testBlocks); err != nil { + Log.Warn("Error loading default darksidewalletd blocks") + } +} - testBlocks, err := os.Open("./testdata/default-darkside-blocks") +// DarksideAddBlock adds a single block to the blocks list. +func DarksideAddBlock(blockHex string) error { + if blockHex == "404: Not Found" { + // special case error (http resource not found, bad pathname) + return errors.New(blockHex) + } + blockData, err := hex.DecodeString(blockHex) + if err != nil { + return err + } + block := parser.NewBlock() + rest, err := block.ParseFromSlice(blockData) + if err != nil { + return err + } + if len(rest) != 0 { + return errors.New("block serialization is too long") + } + blockHeight := block.GetHeight() + // first block, add to existing blocks slice if possible + if blockHeight > state.startHeight+len(state.blocks) { + // The new block can't contiguously extend the existing + // range, so we have to drop the existing range. + state.blocks = state.blocks[:0] + } else if blockHeight < state.startHeight { + // This block will replace the entire existing range. + state.blocks = state.blocks[:0] + } else { + // Drop the block that will be overwritten, and its children. + state.blocks = state.blocks[:blockHeight-state.startHeight] + } + if len(state.blocks) == 0 { + state.startHeight = blockHeight + } else { + // Set this block's prevhash. + prevblock := parser.NewBlock() + rest, err := prevblock.ParseFromSlice(state.blocks[len(state.blocks)-1]) if err != nil { - Log.Fatal("Error loading default darksidewalletd blocks") + return err } - scan := bufio.NewScanner(testBlocks) - for scan.Scan() { // each line (block) - block := scan.Bytes() - state.blocks = append(state.blocks, string(block)) + if len(rest) != 0 { + return errors.New("block is too long") + } + copy(blockData[4:4+32], prevblock.GetEncodableHash()) + } + if state.saplingActivation < 0 { + state.saplingActivation = blockHeight + } + state.blocks = append(state.blocks, blockData) + return nil +} + +func readBlocks(src io.Reader) error { + // some blocks are too large, especially when encoded in hex, for the + // default buffer size, so set up a larger one; 8mb should be enough. + scan := bufio.NewScanner(src) + var scanbuf []byte + scan.Buffer(scanbuf, 8*1000*1000) + for scan.Scan() { // each line (block) + if err := DarksideAddBlock(scan.Text()); err != nil { + return err } } + if scan.Err() != nil { + return scan.Err() + } + return nil +} - if time.Now().Sub(state.server_start).Minutes() >= 30 { +func DarksideSetMetaState(sa int32, bi, cn string) error { + state.saplingActivation = int(sa) + state.branchID = bi + state.chainName = cn + return nil +} + +func DarksideGetIncomingTransactions() [][]byte { + return state.incomingTransactions +} + +func DarksideSetBlocksURL(url string) error { + if strings.HasPrefix(url, "file:") && len(url) >= 6 && url[5] != '/' { + dir, err := os.Getwd() + if err != nil { + return err + } + url = "file:" + dir + string(os.PathSeparator) + url[5:] + } + cmd := exec.Command("curl", "--silent", "--show-error", url) + stdout, err := cmd.StdoutPipe() + if err != nil { + return err + } + stderr, err := cmd.StderrPipe() + if err != nil { + return err + } + if err := cmd.Start(); err != nil { + return err + } + defer cmd.Wait() + err = readBlocks(stdout) + if err != nil { + return err + } + stderroutstr, err := ioutil.ReadAll(stderr) + if err != nil { + return err + } + if len(stderroutstr) > 0 { + return errors.New(string(stderroutstr)) + } + return nil +} + +func DarksideSendTransaction(txbytes []byte) { + state.incomingTransactions = append(state.incomingTransactions, txbytes) +} +func darksideRawRequest(method string, params []json.RawMessage) (json.RawMessage, error) { + if time.Now().Sub(state.serverStart).Minutes() >= 30 { Log.Fatal("Shutting down darksidewalletd to prevent accidental deployment in production.") } @@ -67,102 +194,65 @@ func DarkSideRawRequest(method string, params []json.RawMessage) (json.RawMessag Headers int `json:"headers"` Consensus consensus `json:"consensus"` }{ - Chain: state.chain_name, + Chain: state.chainName, Upgrades: map[string]upgradeinfo{ - "76b809bb": upgradeinfo{ActivationHeight: state.sapling_activation}, + "76b809bb": {ActivationHeight: state.saplingActivation}, }, - Headers: state.start_height + len(state.blocks) - 1, - Consensus: consensus{state.branch_id, state.branch_id}, + Headers: state.startHeight + len(state.blocks) - 1, + Consensus: consensus{state.branchID, state.branchID}, } return json.Marshal(blockchaininfo) case "getblock": - var height string - err := json.Unmarshal(params[0], &height) + var heightStr string + err := json.Unmarshal(params[0], &heightStr) if err != nil { - return nil, errors.New("Failed to parse getblock request.") + return nil, errors.New("failed to parse getblock request") } - height_i, err := strconv.Atoi(height) + height, err := strconv.Atoi(heightStr) if err != nil { - return nil, errors.New("Error parsing height as integer.") + return nil, errors.New("error parsing height as integer") } - index := height_i - state.start_height + index := height - state.startHeight - if index == len(state.blocks) { + const notFoundErr = "-8:" + if state.saplingActivation < 0 || index == len(state.blocks) { // The current ingestor keeps going until it sees this error, // meaning it's up to the latest height. - return nil, errors.New("-8:") + return nil, errors.New(notFoundErr) } if index < 0 || index > len(state.blocks) { // If an integration test can reach this, it could be a bug, so generate an error. - Log.Errorf("getblock request made for out-of-range height %d (have %d to %d)", height_i, state.start_height, state.start_height+len(state.blocks)-1) - return nil, errors.New("-8:") + Log.Errorf("getblock request made for out-of-range height %d (have %d to %d)", + height, state.startHeight, state.startHeight+len(state.blocks)-1) + return nil, errors.New(notFoundErr) } - - return []byte("\"" + state.blocks[index] + "\""), nil + return []byte("\"" + hex.EncodeToString(state.blocks[index]) + "\""), nil case "getaddresstxids": // Not required for minimal reorg testing. - return nil, errors.New("Not implemented yet.") + return nil, errors.New("not implemented yet") case "getrawtransaction": // Not required for minimal reorg testing. - return nil, errors.New("Not implemented yet.") + return nil, errors.New("not implemented yet") case "sendrawtransaction": var rawtx string err := json.Unmarshal(params[0], &rawtx) if err != nil { - return nil, errors.New("Failed to parse sendrawtransaction JSON.") + return nil, errors.New("failed to parse sendrawtransaction JSON") } txbytes, err := hex.DecodeString(rawtx) if err != nil { - return nil, errors.New("Failed to parse sendrawtransaction value as a hex string.") + return nil, errors.New("failed to parse sendrawtransaction value as a hex string") } - state.incoming_transactions = append(state.incoming_transactions, txbytes) + state.incomingTransactions = append(state.incomingTransactions, txbytes) return nil, nil - case "x_setstate": - var new_state map[string]interface{} - - err := json.Unmarshal(params[0], &new_state) - if err != nil { - Log.Fatal("Could not unmarshal the provided state.") - } - - block_strings := make([]string, 0) - for _, block_str := range new_state["blocks"].([]interface{}) { - block_strings = append(block_strings, block_str.(string)) - } - - state = &DarksideZcashdState{ - start_height: int(new_state["start_height"].(float64)), - sapling_activation: int(new_state["sapling_activation"].(float64)), - branch_id: new_state["branch_id"].(string), - chain_name: new_state["chain_name"].(string), - blocks: block_strings, - incoming_transactions: state.incoming_transactions, - server_start: state.server_start, - } - - return nil, nil - - case "x_getincomingtransactions": - txlist := "[" - for i, tx := range state.incoming_transactions { - txlist += "\"" + hex.EncodeToString(tx) + "\"" - // add commas after all but the last - if i < len(state.incoming_transactions)-1 { - txlist += ", " - } - } - txlist += "]" - - return []byte(txlist), nil - default: - return nil, errors.New("There was an attempt to call an unsupported RPC.") + return nil, errors.New("there was an attempt to call an unsupported RPC") } } diff --git a/docs/darksidewalletd.md b/docs/darksidewalletd.md index e84c3bd..ed57894 100644 --- a/docs/darksidewalletd.md +++ b/docs/darksidewalletd.md @@ -34,7 +34,7 @@ after 30 minutes. ### Default set of blocks -There’s a file in the repo called ./testdata/default-darkside-blocks. This +There’s a file in the repo called ./testdata/darkside/init-blocks. This contains the blocks darksidewalletd loads by default. The format of the file is one hex-encoded block per line. @@ -74,7 +74,7 @@ echo “some hex-encoded transaction you want to put in block 1003” > blocksA/ ``` This will output the blocks, one hex-encoded block per line (the same format as -./testdata/default-darkside-blocks). +./testdata/darkside/init-blocks). Tip: Because nothing is checking the full validity of transactions, you can get any hex-encoded transaction you want from a block explorer and put those in the @@ -93,20 +93,29 @@ submit the blocks, which internally uses grpcurl, e.g.: ``` ./genblocks --blocks-dir blocksA > blocksA.txt -./utils/submitblocks.sh 1000 1000 blocksA.txt +./utils/submitblocks.sh 1000 blocksA.txt ``` -In the submitblocks.sh command, the first “1000” is the height to serve the -first block at (so that if blocksA.txt contains 6 blocks they will be served as -heights 1000, 1001, 1002, 1003, 1004, and 1005. If the genblocks tool was used -to create the blocksA file, then this argument must match what was given to -genblocks, otherwise the heights in the coinbase transactions will not match up -with the height lightwalletd is serving the blocks as. The second “1000” sets -the value that lightwalletd will report the sapling activation height to be. +In the submitblocks.sh command, the “1000” sets the value that lightwalletd will +report the sapling activation height to be. -Tip: The DarksideSetState expects a complete set of blocks for the mock zcashd -to serve, if you want to just add one block, for example, you need to re-submit -all of the blocks including the new one. +Tip: You may submit blocks incrementally, that is, submit 1000-1005 followed +by 1006-1008, the result is 1000-1008. If you create a gap in the range (say, +1000-1005 then 1007-1009), then the earlier range is forgotten; the only range +is 1007-1009 because it doesn't make sense to have a gap. + +If you submit overlapping ranges, the expected things happen. For example, first +submit 1000-1005, then 1003-1007, the result is 1000-1007 (the original 1000-1002 +followed by the new 1003-1007). This is how you can create a reorg starting at 1003. +You can get the same effect slightly less efficiently by submitting 1000-1007 (that +is, resubmitting the original 1000-1002 followed by the new 1003-1007). + +If you first submit 1000-1005, then 1001-1002, the result will be 1000-1002 +(1003-1005 are dropped; it's not possible to "insert" blocks into an range). +Likewise, first submit 1005-1008, then 1000-1006, the result is only 1000-1006. As +easy way to state it is that all earlier blocks beyond the end of the extent of +the range being submitted now are dropped. But blocks before the start of the range +being submitted now are preserved if doing so doesn't create a gap. ## Tutorial ### Triggering a Reorg @@ -143,7 +152,7 @@ Now you can start darksidewalletd and it’ll load the blocksA blocks: That will have loaded and be serving the blocksA blocks. We can push up the blocksB blocks using ./utils/submitblocks.sh: -`./utils/submitblocks.sh 1000 1000 testdata/darkside-blocks-reorg` +`./utils/submitblocks.sh 1000 testdata/darkside-blocks-reorg` We should now see a reorg in server.log: @@ -152,6 +161,28 @@ We should now see a reorg in server.log: {"app":"frontend-grpc","hash":"a244942179988ea6e56a3a55509fcf22673df26200c67bebd93504385a1a7c4f","height":1004,"level":"warning","msg":"REORG","phash":"06e7c72646e3d51417de25bd83896c682b72bdf5be680908d621cba86d222798","reorg":1,"time":"2020-03-23T13:59:44-06:00"} ``` +### Precomputed block ranges + +The ECC has already created some block ranges to simulate reorgs in +the repository https://github.com/zcash-hackworks/darksidewalletd-test-data. +This may relieve you of the task of generating test blocks. There's a `gRPC` method +called `SetBlocksURL` that takes a resource location (anything that can be +given to `curl`; indeed, the lightwalletd uses `curl`). Here's an example: + +`grpcurl -plaintext -d '{"url":"https://raw.githubusercontent.com/zcash-hackworks/darksidewalletd-test-data/master/blocks-663242-663251"}' localhost:9067 cash.z.wallet.sdk.rpc.DarksideStreamer/SetBlocksURL` + +When lightwalletd starts up in darksidewalletd mode, it automatically does the +equivalent of: + +`grpcurl -plaintext -d '{"url":"file:testdata/darkside/init-blocks"}' localhost:9067 cash.z.wallet.sdk.rpc.DarksideStreamer/SetBlocksURL` + +which is also equivalent to (the `-d @` tells `grpcurl` to read from standard input): +``` +cat testdata/darkside/init-blocks | +sed 's/^/{"block":"/;s/$/"}/' | +grpcurl -plaintext -d @ localhost:9067 cash.z.wallet.sdk.rpc.DarksideStreamer/SetBlocks +``` + ## Use cases Check out some of the potential security test cases here: [wallet <-> @@ -160,9 +191,6 @@ tests](https://github.com/zcash/lightwalletd/blob/master/docs/integration-tests. ## Source Code * cmd/genblocks -- tool for generating fake block sets. -* testdata/default-darkside-blocks -- the set of blocks loaded by default +* testdata/darkside/init-blocks -- the set of blocks loaded by default * common/darkside.go -- implementation of darksidewalletd * frontend/service.go -- entrypoints for darksidewalletd GRPC APIs - - - diff --git a/docs/rtd/index.html b/docs/rtd/index.html index 11c4d45..e8ed0ad 100644 --- a/docs/rtd/index.html +++ b/docs/rtd/index.html @@ -206,7 +206,19 @@