From a4f968823fdd66420df2ffafa083bc19054aeca2 Mon Sep 17 00:00:00 2001 From: Larry Ruane Date: Fri, 31 Jan 2020 15:17:03 -0700 Subject: [PATCH] test improvements, and minor cleanups --- CONTRIBUTING.md | 10 +- Dockerfile | 6 +- Makefile | 20 +- README.md | 4 +- cmd/server/main.go | 91 +++---- cmd/server/server_test.go | 66 ++++- common/cache.go | 29 ++- common/cache_test.go | 64 +++-- common/common.go | 90 ++++--- common/common_test.go | 290 +++++++++++++++++++++ common/generatecerts.go | 18 +- docker/gen_cert.sh | 2 +- docker/zcash.conf | 2 +- docs/architecture.md | 10 +- docs/docker-compose-setup.md | 2 +- frontend/frontend_test.go | 472 ++++++++++++++++++++++++++++++++++- frontend/rpc_client.go | 20 +- frontend/service.go | 111 +++----- go.mod | 2 +- parser/block.go | 8 +- parser/block_header.go | 19 +- parser/block_test.go | 4 + parser/transaction.go | 4 +- parser/transaction_test.go | 30 ++- testdata/getsaplinginfo | 1 + 25 files changed, 1111 insertions(+), 264 deletions(-) create mode 100644 common/common_test.go create mode 100644 testdata/getsaplinginfo diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 26e0de3..92d69e0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -19,7 +19,7 @@ Note: Please replace `your_username`, with your actual GitHub username git clone git@github.com:your_username/lightwalletd.git cd lightwalletd git remote set-url origin git@github.com:your_username/lightwalletd.git -git remote add upstream git@github.com:zcash-hackworks/lightwalletd.git +git remote add upstream git@github.com:zcash/lightwalletd.git git remote set-url --push upstream DISABLED git fetch upstream git branch -u upstream/master master @@ -39,7 +39,7 @@ After issuing the above commands, your `.git/config` file should look similar to remote = upstream merge = refs/heads/master [remote "upstream"] - url = git@github.com:zcash-hackworks/lightalletd.git + url = git@github.com:zcash/lightalletd.git fetch = +refs/heads/*:refs/remotes/upstream/* pushurl = DISABLED ``` @@ -64,7 +64,7 @@ To checkout an existing branch (assuming you are in `lightwalletd` directory): ```bash git checkout [existing_branch_name] ``` -If you are fixing a bug or implementing a new feature, you likely will want to create a new branch. If you are reviewing code or working on existing branches, you likely will checkout an existing branch. To view the list of current Lightwalletd GitHub issues, click [here](https://github.com/zcash-hackworks/lightwalletd/issues). +If you are fixing a bug or implementing a new feature, you likely will want to create a new branch. If you are reviewing code or working on existing branches, you likely will checkout an existing branch. To view the list of current Lightwalletd GitHub issues, click [here](https://github.com/zcash/lightwalletd/issues). ## Make & Commit Changes If you have created a new branch or checked out an existing one, it is time to make changes to your local source code. Below are some formalities for commits: @@ -83,7 +83,7 @@ git remote -v ```bash origin git@github.com:your_username/lightwalletd.git (fetch) origin git@github.com:your_username/lightwalletd.git (push) -upstream git@github.com:zcash-hackworks/lightwalletd.git (fetch) +upstream git@github.com:zcash/lightwalletd.git (fetch) upstream DISABLED (push) ``` This output should be consistent with your `.git/config`: @@ -96,7 +96,7 @@ This output should be consistent with your `.git/config`: url = git@github.com:your_username/lightwalletd.git fetch = +refs/heads/*:refs/remotes/origin/* [remote "upstream"] - url = git@github.com:zcash-hackworks/lightwalletd.git + url = git@github.com:zcash/lightwalletd.git fetch = +refs/heads/*:refs/remotes/upstream/* pushurl = DISABLED ``` diff --git a/Dockerfile b/Dockerfile index b8f4ea4..6ec6ab9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,8 +43,8 @@ # Create layer in case you want to modify local lightwalletd code FROM golang:1.13 AS lightwalletd_base -ADD . /go/src/github.com/zcash-hackworks/lightwalletd -WORKDIR /go/src/github.com/zcash-hackworks/lightwalletd +ADD . /go/src/github.com/zcash/lightwalletd +WORKDIR /go/src/github.com/zcash/lightwalletd RUN make \ && /usr/bin/install -c ./server /usr/bin/ @@ -60,4 +60,4 @@ USER $LWD_USER WORKDIR /srv/$LWD_USER ENTRYPOINT ["server"] -CMD ["--help"] \ No newline at end of file +CMD ["--help"] diff --git a/Makefile b/Makefile index b152674..79ff1bf 100644 --- a/Makefile +++ b/Makefile @@ -20,14 +20,14 @@ all: build # Lint golang files lint: - @golint -set_exit_status + golint -set_exit_status show_tests: @echo ${GO_TEST_FILES} # Run unittests test: - @go test -v -coverprofile=coverage.txt -covermode=atomic ./... + go test -v ./... # Run data race detector race: @@ -35,19 +35,21 @@ race: # Run memory sanitizer (need to ensure proper build flag is set) msan: - @go test -v -msan -short ${GO_TEST_FILES} + go test -v -msan -short ${GO_TEST_FILES} + +# Generate global code coverage report, ignore generated *.pb.go files -# Generate global code coverage report coverage: - @go test -coverprofile=coverage.out -covermode=atomic ./... + go test -coverprofile=coverage.out ./... + sed -i '/\.pb\.go/d' coverage.out # Generate code coverage report -coverage_report: - @go tool cover -func=coverage.out +coverage_report: coverage + go tool cover -func=coverage.out # Generate code coverage report in HTML -coverage_html: - @go tool cover -html=coverage.out -o coverage.html +coverage_html: coverage + go tool cover -html=coverage.out # Generate documents docs: diff --git a/README.md b/README.md index 8e4ddf7..7b00551 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ The Lightwalletd Server is experimental and a work in progress. Use it at your o # Overview -[lightwalletd](https://github.com/zcash-hackworks/lightwalletd) is a backend service that provides a bandwidth-efficient interface to the Zcash blockchain. Currently, lightwalletd supports the Sapling protocol version as its primary concern. The intended purpose of lightwalletd is to support the development of mobile-friendly shielded light wallets. +[lightwalletd](https://github.com/zcash/lightwalletd) is a backend service that provides a bandwidth-efficient interface to the Zcash blockchain. Currently, lightwalletd supports the Sapling protocol version as its primary concern. The intended purpose of lightwalletd is to support the development of mobile-friendly shielded light wallets. lightwalletd is a backend service that provides a bandwidth-efficient interface to the Zcash blockchain for mobile and other wallets, such as [Zecwallet](https://github.com/adityapk00/zecwallet-lite-lib). @@ -25,7 +25,7 @@ Lightwalletd has not yet undergone audits or been subject to rigorous testing. I To view status of [CI pipeline](https://gitlab.com/mdr0id/lightwalletd/pipelines) -To view detailed [Codecov](https://codecov.io/gh/zcash-hackworks/lightwalletd) report +To view detailed [Codecov](https://codecov.io/gh/zcash/lightwalletd) report # Local/Developer docker-compose Usage diff --git a/cmd/server/main.go b/cmd/server/main.go index 823a32b..bd3eb5e 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -16,12 +16,11 @@ import ( "google.golang.org/grpc/peer" "google.golang.org/grpc/reflection" - "github.com/zcash-hackworks/lightwalletd/common" - "github.com/zcash-hackworks/lightwalletd/frontend" - "github.com/zcash-hackworks/lightwalletd/walletrpc" + "github.com/zcash/lightwalletd/common" + "github.com/zcash/lightwalletd/frontend" + "github.com/zcash/lightwalletd/walletrpc" ) -var log *logrus.Entry var logger = logrus.New() func init() { @@ -32,10 +31,10 @@ func init() { }) onexit := func() { - fmt.Printf("Lightwalletd died with a Fatal error. Check logfile for details.\n") + fmt.Println("Lightwalletd died with a Fatal error. Check logfile for details.") } - log = logger.WithFields(logrus.Fields{ + common.Log = logger.WithFields(logrus.Fields{ "app": "frontend-grpc", }) @@ -48,12 +47,7 @@ func LoggingInterceptor() grpc.ServerOption { return grpc.UnaryInterceptor(logInterceptor) } -func logInterceptor( - ctx context.Context, - req interface{}, - info *grpc.UnaryServerInfo, - handler grpc.UnaryHandler, -) (interface{}, error) { +func logInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { reqLog := loggerFromContext(ctx) start := time.Now() @@ -77,9 +71,9 @@ func logInterceptor( func loggerFromContext(ctx context.Context) *logrus.Entry { // TODO: anonymize the addresses. cryptopan? if peerInfo, ok := peer.FromContext(ctx); ok { - return log.WithFields(logrus.Fields{"peer_addr": peerInfo.Addr}) + return common.Log.WithFields(logrus.Fields{"peer_addr": peerInfo.Addr}) } - return log.WithFields(logrus.Fields{"peer_addr": "unknown"}) + return common.Log.WithFields(logrus.Fields{"peer_addr": "unknown"}) } type Options struct { @@ -89,7 +83,6 @@ type Options struct { logLevel uint64 `json:"log_level,omitempty"` logPath string `json:"log_file,omitempty"` zcashConfPath string `json:"zcash_conf,omitempty"` - veryInsecure bool `json:"very_insecure,omitempty"` cacheSize int `json:"cache_size,omitempty"` wantVersion bool } @@ -122,6 +115,23 @@ func main() { return } + // production (unlike unit tests) use the real sleep function + common.Sleep = time.Sleep + + if opts.logPath != "" { + // instead write parsable logs for logstash/splunk/etc + output, err := os.OpenFile(opts.logPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + os.Stderr.WriteString(fmt.Sprintf("Cannot open log file %s: %v\n", + opts.logPath, err)) + os.Exit(1) + } + defer output.Close() + logger.SetOutput(output) + logger.SetFormatter(&logrus.JSONFormatter{}) + } + logger.SetLevel(logrus.Level(opts.logLevel)) + filesThatShouldExist := []string{ opts.zcashConfPath, } @@ -134,41 +144,27 @@ func main() { for _, filename := range filesThatShouldExist { if !fileExists(filename) { - os.Stderr.WriteString(fmt.Sprintf("\n ** File does not exist: %s\n\n", filename)) - flag.Usage() + common.Log.WithFields(logrus.Fields{ + "filename": filename, + }).Error("cannot open required file") + os.Stderr.WriteString("Cannot open required file: " + filename + "\n") os.Exit(1) } } - if opts.logPath != "" { - // instead write parsable logs for logstash/splunk/etc - output, err := os.OpenFile(opts.logPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - log.WithFields(logrus.Fields{ - "error": err, - "path": opts.logPath, - }).Fatal("couldn't open log file") - } - defer output.Close() - logger.SetOutput(output) - logger.SetFormatter(&logrus.JSONFormatter{}) - } - - logger.SetLevel(logrus.Level(opts.logLevel)) - // gRPC initialization var server *grpc.Server var transportCreds credentials.TransportCredentials var err error if (opts.tlsCertPath == "") && (opts.tlsKeyPath == "") { - log.Warning("Certificate and key not provided, generating self signed values") + common.Log.Warning("Certificate and key not provided, generating self signed values") tlsCert := common.GenerateCerts() transportCreds = credentials.NewServerTLSFromCert(tlsCert) } else { transportCreds, err = credentials.NewServerTLSFromFile(opts.tlsCertPath, opts.tlsKeyPath) if err != nil { - log.WithFields(logrus.Fields{ + common.Log.WithFields(logrus.Fields{ "cert_file": opts.tlsCertPath, "key_path": opts.tlsKeyPath, "error": err, @@ -188,15 +184,18 @@ func main() { rpcClient, err := frontend.NewZRPCFromConf(opts.zcashConfPath) if err != nil { - log.WithFields(logrus.Fields{ + common.Log.WithFields(logrus.Fields{ "error": err, }).Fatal("setting up RPC connection to zcashd") } + // indirect function for test mocking (so unit tests can talk to stub functions) + common.RawRequest = rpcClient.RawRequest + // Get the sapling activation height from the RPC // (this first RPC also verifies that we can communicate with zcashd) - saplingHeight, blockHeight, chainName, branchID := common.GetSaplingInfo(rpcClient, log) - log.Info("Got sapling height ", saplingHeight, " chain ", chainName, " branchID ", branchID) + saplingHeight, blockHeight, chainName, branchID := common.GetSaplingInfo() + common.Log.Info("Got sapling height ", saplingHeight, " chain ", chainName, " branchID ", branchID) // Initialize the cache cache := common.NewBlockCache(opts.cacheSize) @@ -207,12 +206,13 @@ func main() { cacheStart = saplingHeight } - go common.BlockIngestor(rpcClient, cache, log, cacheStart) + // The last argument, repetition count, is only nonzero for testing + go common.BlockIngestor(cache, cacheStart, 0) // Compact transaction service initialization - service, err := frontend.NewLwdStreamer(rpcClient, cache, log) + service, err := frontend.NewLwdStreamer(cache) if err != nil { - log.WithFields(logrus.Fields{ + common.Log.WithFields(logrus.Fields{ "error": err, }).Fatal("couldn't create backend") } @@ -223,7 +223,7 @@ func main() { // Start listening listener, err := net.Listen("tcp", opts.bindAddr) if err != nil { - log.WithFields(logrus.Fields{ + common.Log.WithFields(logrus.Fields{ "bind_addr": opts.bindAddr, "error": err, }).Fatal("couldn't create listener") @@ -234,17 +234,18 @@ func main() { signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM) go func() { s := <-signals - log.WithFields(logrus.Fields{ + common.Log.WithFields(logrus.Fields{ "signal": s.String(), }).Info("caught signal, stopping gRPC server") + os.Stderr.WriteString("Caught signal: " + s.String() + "\n") os.Exit(1) }() - log.Infof("Starting gRPC server on %s", opts.bindAddr) + common.Log.Infof("Starting gRPC server on %s", opts.bindAddr) err = server.Serve(listener) if err != nil { - log.WithFields(logrus.Fields{ + common.Log.WithFields(logrus.Fields{ "error": err, }).Fatal("gRPC server exited") } diff --git a/cmd/server/server_test.go b/cmd/server/server_test.go index c0ce629..9213007 100644 --- a/cmd/server/server_test.go +++ b/cmd/server/server_test.go @@ -1,8 +1,72 @@ package main import ( + "context" + "fmt" + "os" "testing" + + "errors" + "github.com/sirupsen/logrus" + "github.com/zcash/lightwalletd/common" + "google.golang.org/grpc" + "google.golang.org/grpc/peer" ) -func TestString_read(t *testing.T) { +var step int + +func testhandler(ctx context.Context, req interface{}) (interface{}, error) { + step++ + switch step { + case 1: + return nil, errors.New("test error") + case 2: + return nil, nil + } + return nil, nil +} + +func TestLogInterceptor(t *testing.T) { + output, err := os.OpenFile("test-log", os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) + if err != nil { + os.Stderr.WriteString(fmt.Sprint("Cannot open test-log:", err)) + os.Exit(1) + } + logger := logrus.New() + logger.SetOutput(output) + common.Log = logger.WithFields(logrus.Fields{ + "app": "test", + }) + var req interface{} + resp, err := logInterceptor(peer.NewContext(context.Background(), &peer.Peer{}), + &req, &grpc.UnaryServerInfo{}, testhandler) + if err == nil { + t.Fatal("unexpected success") + } + if resp != nil { + t.Fatal("unexpected response", resp) + } + resp, err = logInterceptor(context.Background(), &req, &grpc.UnaryServerInfo{}, testhandler) + if err != nil { + t.Fatal("unexpected error", err) + } + if resp != nil { + t.Fatal("unexpected response", resp) + } + os.Remove("test-log") + step = 0 +} + +func TestFileExists(t *testing.T) { + if fileExists("nonexistent-file") { + t.Fatal("fileExists unexpected success") + } + // If the path exists but is a directory, should return false + if fileExists(".") { + t.Fatal("fileExists unexpected success") + } + // The following file should exist, it's what's being tested + if !fileExists("main.go") { + t.Fatal("fileExists failed") + } } diff --git a/common/cache.go b/common/cache.go index 4861b90..f317735 100644 --- a/common/cache.go +++ b/common/cache.go @@ -5,33 +5,37 @@ import ( "sync" "github.com/golang/protobuf/proto" - "github.com/zcash-hackworks/lightwalletd/walletrpc" + "github.com/zcash/lightwalletd/walletrpc" ) -type BlockCacheEntry struct { +type blockCacheEntry struct { data []byte hash []byte } +// BlockCache contains a set of recent compact blocks in marshalled form. type BlockCache struct { MaxEntries int // m[firstBlock..nextBlock) are valid - m map[int]*BlockCacheEntry + m map[int]*blockCacheEntry firstBlock int nextBlock int mutex sync.RWMutex } +// NewBlockCache returns an instance of a block cache object. func NewBlockCache(maxEntries int) *BlockCache { return &BlockCache{ MaxEntries: maxEntries, - m: make(map[int]*BlockCacheEntry), + m: make(map[int]*blockCacheEntry), } } -func (c *BlockCache) Add(height int, block *walletrpc.CompactBlock) (error, bool) { +// Add adds the given block to the cache at the given height, returning true +// if a reorg was detected. +func (c *BlockCache) Add(height int, block *walletrpc.CompactBlock) (bool, error) { // Invariant: m[firstBlock..nextBlock) are valid. c.mutex.Lock() defer c.mutex.Unlock() @@ -63,16 +67,15 @@ func (c *BlockCache) Add(height int, block *walletrpc.CompactBlock) (error, bool // Detect reorg, ingestor needs to handle it if height > c.firstBlock && !bytes.Equal(block.PrevHash, c.m[height-1].hash) { - return nil, true + return true, nil } // Add the entry and update the counters data, err := proto.Marshal(block) if err != nil { - println("Error marshalling block!") - return err, false + return false, err } - c.m[height] = &BlockCacheEntry{ + c.m[height] = &blockCacheEntry{ data: data, hash: block.GetHash(), } @@ -87,9 +90,11 @@ func (c *BlockCache) Add(height int, block *walletrpc.CompactBlock) (error, bool } // Invariant: m[firstBlock..nextBlock) are valid. - return nil, false + return false, nil } +// Get returns the compact block at the requested height if it is +// in the cache, else nil. func (c *BlockCache) Get(height int) *walletrpc.CompactBlock { c.mutex.RLock() defer c.mutex.RUnlock() @@ -108,7 +113,9 @@ func (c *BlockCache) Get(height int) *walletrpc.CompactBlock { return serialized } -func (c *BlockCache) GetLatestBlock() int { +// GetLatestHeight returns the block with the greatest height, or nil +// if the cache is empty. +func (c *BlockCache) GetLatestHeight() int { c.mutex.RLock() defer c.mutex.RUnlock() if c.firstBlock == c.nextBlock { diff --git a/common/cache_test.go b/common/cache_test.go index f52b813..6306f4c 100644 --- a/common/cache_test.go +++ b/common/cache_test.go @@ -6,8 +6,8 @@ import ( "io/ioutil" "testing" - "github.com/zcash-hackworks/lightwalletd/parser" - "github.com/zcash-hackworks/lightwalletd/walletrpc" + "github.com/zcash/lightwalletd/parser" + "github.com/zcash/lightwalletd/walletrpc" ) func TestCache(t *testing.T) { @@ -36,7 +36,7 @@ func TestCache(t *testing.T) { for _, test := range compactTests { blockData, _ := hex.DecodeString(test.Full) block := parser.NewBlock() - blockData, err = block.ParseFromSlice(blockData) + _, err = block.ParseFromSlice(blockData) if err != nil { t.Fatal(err) } @@ -44,21 +44,30 @@ func TestCache(t *testing.T) { } // initially empty cache - if cache.GetLatestBlock() != -1 { - t.Fatal("unexpected GetLatestBlock") + if cache.GetLatestHeight() != -1 { + t.Fatal("unexpected GetLatestHeight") + } + + // Test handling an invalid block (nil will do) + reorg, err := cache.Add(21, nil) + if err == nil { + t.Error("expected error:", err) + } + if reorg { + t.Fatal("unexpected reorg") } // normal, sunny-day case, 6 blocks, add as blocks 10-15 for i, compact := range compacts { - err, reorg := cache.Add(10+i, compact) + reorg, err = cache.Add(10+i, compact) if err != nil { t.Fatal(err) } if reorg { t.Fatal("unexpected reorg") } - if cache.GetLatestBlock() != 10+i { - t.Fatal("unexpected GetLatestBlock") + if cache.GetLatestHeight() != 10+i { + t.Fatal("unexpected GetLatestHeight") } // The test blocks start at height 289460 if int(cache.Get(10+i).Height) != 289460+i { @@ -82,7 +91,7 @@ func TestCache(t *testing.T) { // We can re-add the last block (with the same data) and // that should just replace and not be considered a reorg - err, reorg := cache.Add(15, compacts[5]) + reorg, err = cache.Add(15, compacts[5]) if err != nil { t.Fatal(err) } @@ -101,7 +110,7 @@ func TestCache(t *testing.T) { // Simulate a reorg by resubmitting as the next block, 16, any block with // the wrote prev-hash (let's use the first, just because it's handy) - err, reorg = cache.Add(16, compacts[0]) + reorg, err = cache.Add(16, compacts[0]) if err != nil { t.Fatal(err) } @@ -112,8 +121,8 @@ func TestCache(t *testing.T) { if cache.Get(16) != nil { t.Fatal("unexpected block 16 exists") } - if cache.GetLatestBlock() != 15 { - t.Fatal("unexpected GetLatestBlock") + if cache.GetLatestHeight() != 15 { + t.Fatal("unexpected GetLatestHeight") } if int(cache.Get(15).Height) != 289460+5 { t.Fatal("unexpected Get") @@ -127,7 +136,7 @@ func TestCache(t *testing.T) { // Let's back up one block, to height 15, request it from zcashd, // but let's say this block is from the new branch, so we haven't // gone back far enough, so this will still be disallowed. - err, reorg = cache.Add(15, compacts[0]) + reorg, err = cache.Add(15, compacts[0]) if err != nil { t.Fatal(err) } @@ -138,8 +147,8 @@ func TestCache(t *testing.T) { if cache.Get(15) != nil { t.Fatal("unexpected block 15 exists") } - if cache.GetLatestBlock() != 14 { - t.Fatal("unexpected GetLatestBlock") + if cache.GetLatestHeight() != 14 { + t.Fatal("unexpected GetLatestHeight") } if int(cache.Get(14).Height) != 289460+4 { t.Fatal("unexpected Get") @@ -154,7 +163,7 @@ func TestCache(t *testing.T) { // (In this test, we're replacing 13 with the same block; in // real life, we'd be replacing it with a different version of // 13 that has the same prev-hash). - err, reorg = cache.Add(13, compacts[3]) + reorg, err = cache.Add(13, compacts[3]) if err != nil { t.Fatal(err) } @@ -166,8 +175,8 @@ func TestCache(t *testing.T) { if cache.Get(14) != nil { t.Fatal("unexpected block 14 exists") } - if cache.GetLatestBlock() != 13 { - t.Fatal("unexpected GetLatestBlock") + if cache.GetLatestHeight() != 13 { + t.Fatal("unexpected GetLatestHeight") } if int(cache.Get(13).Height) != 289460+3 { t.Fatal("unexpected Get") @@ -181,15 +190,15 @@ func TestCache(t *testing.T) { } // Now we can continue forward from here - err, reorg = cache.Add(14, compacts[4]) + reorg, err = cache.Add(14, compacts[4]) if err != nil { t.Fatal(err) } if reorg { t.Fatal("unexpected reorg") } - if cache.GetLatestBlock() != 14 { - t.Fatal("unexpected GetLatestBlock") + if cache.GetLatestHeight() != 14 { + t.Fatal("unexpected GetLatestHeight") } if int(cache.Get(14).Height) != 289460+4 { t.Fatal("unexpected Get") @@ -205,15 +214,15 @@ func TestCache(t *testing.T) { if cache.firstBlock != 12 { t.Fatal("unexpected firstBlock") } - err, reorg = cache.Add(10, compacts[0]) + reorg, err = cache.Add(10, compacts[0]) if err != nil { t.Fatal(err) } if reorg { t.Fatal("unexpected reorg") } - if cache.GetLatestBlock() != 10 { - t.Fatal("unexpected GetLatestBlock") + if cache.GetLatestHeight() != 10 { + t.Fatal("unexpected GetLatestHeight") } if int(cache.Get(10).Height) != 289460+0 { t.Fatal("unexpected Get") @@ -225,15 +234,15 @@ func TestCache(t *testing.T) { // Another weird case (not currently possible) is adding a block at // a height that is not one higher than the current latest block. // This should remove the entire cache before adding the new entry. - err, reorg = cache.Add(20, compacts[0]) + reorg, err = cache.Add(20, compacts[0]) if err != nil { t.Fatal(err) } if reorg { t.Fatal("unexpected reorg") } - if cache.GetLatestBlock() != 20 { - t.Fatal("unexpected GetLatestBlock") + if cache.GetLatestHeight() != 20 { + t.Fatal("unexpected GetLatestHeight") } if int(cache.Get(20).Height) != 289460 { t.Fatal("unexpected Get") @@ -241,4 +250,5 @@ func TestCache(t *testing.T) { if len(cache.m) != 1 { t.Fatal("unexpected number of cache entries") } + // the cache deleted block 15 (it's definitely wrong) } diff --git a/common/common.go b/common/common.go index 18eeefb..9c32f7e 100644 --- a/common/common.go +++ b/common/common.go @@ -7,41 +7,55 @@ import ( "strings" "time" - "github.com/btcsuite/btcd/rpcclient" "github.com/pkg/errors" "github.com/sirupsen/logrus" - "github.com/zcash-hackworks/lightwalletd/parser" - "github.com/zcash-hackworks/lightwalletd/walletrpc" + "github.com/zcash/lightwalletd/parser" + "github.com/zcash/lightwalletd/walletrpc" ) -func GetSaplingInfo(rpcClient *rpcclient.Client, log *logrus.Entry) (int, int, string, string) { +// RawRequest points to the function to send a an RPC request to zcashd; +// in production, it points to btcsuite/btcd/rpcclient/rawrequest.go:RawRequest(); +// in unit tests it points to a function to mock RPCs to zcashd. +var RawRequest func(method string, params []json.RawMessage) (json.RawMessage, error) + +// Sleep allows a request to time.Sleep() to be mocked for testing; +// in production, it points to the standard library time.Sleep(); +// in unit tests it points to a mock function. +var Sleep func(d time.Duration) + +// Log as a global variable simplifies logging +var Log *logrus.Entry + +// GetSaplingInfo returns the result of the getblockchaininfo RPC to zcashd +func GetSaplingInfo() (int, int, string, string) { // This request must succeed or we can't go on; give zcashd time to start up var f interface{} retryCount := 0 for { - result, rpcErr := rpcClient.RawRequest("getblockchaininfo", make([]json.RawMessage, 0)) + result, rpcErr := RawRequest("getblockchaininfo", []json.RawMessage{}) if rpcErr == nil { if retryCount > 0 { - log.Warn("getblockchaininfo RPC successful") + Log.Warn("getblockchaininfo RPC successful") } err := json.Unmarshal(result, &f) if err != nil { - log.Fatalf("error parsing JSON getblockchaininfo response: %v", err) + Log.Fatalf("error parsing JSON getblockchaininfo response: %v", err) } break } retryCount++ if retryCount > 10 { - log.WithFields(logrus.Fields{ + Log.WithFields(logrus.Fields{ "timeouts": retryCount, }).Fatal("unable to issue getblockchaininfo RPC call to zcashd node") } - log.WithFields(logrus.Fields{ + Log.WithFields(logrus.Fields{ "error": rpcErr.Error(), "retry": retryCount, }).Warn("error with getblockchaininfo rpc, retrying...") - time.Sleep(time.Duration(10+retryCount*5) * time.Second) // backoff + Sleep(time.Duration(10+retryCount*5) * time.Second) // backoff } + chainName := f.(map[string]interface{})["chain"].(string) upgradeJSON := f.(map[string]interface{})["upgrades"] @@ -51,23 +65,22 @@ func GetSaplingInfo(rpcClient *rpcclient.Client, log *logrus.Entry) (int, int, s blockHeight := f.(map[string]interface{})["headers"].(float64) consensus := f.(map[string]interface{})["consensus"] + branchID := consensus.(map[string]interface{})["nextblock"].(string) return int(saplingHeight), int(blockHeight), chainName, branchID } -func getBlockFromRPC(rpcClient *rpcclient.Client, height int) (*walletrpc.CompactBlock, error) { +func getBlockFromRPC(height int) (*walletrpc.CompactBlock, error) { params := make([]json.RawMessage, 2) params[0] = json.RawMessage("\"" + strconv.Itoa(height) + "\"") - params[1] = json.RawMessage("0") - result, rpcErr := rpcClient.RawRequest("getblock", params) + params[1] = json.RawMessage("0") // non-verbose (raw hex) + result, rpcErr := RawRequest("getblock", params) // For some reason, the error responses are not JSON if rpcErr != nil { - errParts := strings.SplitN(rpcErr.Error(), ":", 2) - errCode, err := strconv.ParseInt(errParts[0], 10, 32) // Check to see if we are requesting a height the zcashd doesn't have yet - if err == nil && errCode == -8 { + if (strings.Split(rpcErr.Error(), ":"))[0] == "-8" { return nil, nil } return nil, errors.Wrap(rpcErr, "error requesting block") @@ -96,42 +109,41 @@ func getBlockFromRPC(rpcClient *rpcclient.Client, height int) (*walletrpc.Compac return block.ToCompact(), nil } -func BlockIngestor(rpcClient *rpcclient.Client, cache *BlockCache, log *logrus.Entry, startHeight int) { +// BlockIngestor runs as a goroutine and polls zcashd for new blocks, adding them +// to the cache. The repetition count, rep, is nonzero only for unit-testing. +func BlockIngestor(cache *BlockCache, startHeight int, rep int) { reorgCount := 0 height := startHeight // Start listening for new blocks retryCount := 0 - for { - block, err := getBlockFromRPC(rpcClient, height) + for i := 0; rep == 0 || i < rep; i++ { + block, err := getBlockFromRPC(height) if block == nil || err != nil { if err != nil { - log.WithFields(logrus.Fields{ + Log.WithFields(logrus.Fields{ "height": height, "error": err, }).Warn("error with getblock rpc") retryCount++ if retryCount > 10 { - log.WithFields(logrus.Fields{ + Log.WithFields(logrus.Fields{ "timeouts": retryCount, }).Fatal("unable to issue RPC call to zcashd node") } } // We're up to date in our polling; wait for a new block - time.Sleep(10 * time.Second) + Sleep(10 * time.Second) continue } retryCount = 0 - log.Info("Ingestor adding block to cache: ", height) - err, reorg := cache.Add(height, block) - + if (height % 100) == 0 { + Log.Info("Ingestor adding block to cache: ", height) + } + reorg, err := cache.Add(height, block) if err != nil { - // It's unclear how this will recover, but we certainly - // don't want to loop full-speed - log.Error("Error adding block to cache: ", err) - time.Sleep(10 * time.Second) - continue + Log.Fatal("Cache add failed") } // Check for reorgs once we have inital block hash from startup @@ -141,9 +153,9 @@ func BlockIngestor(rpcClient *rpcclient.Client, cache *BlockCache, log *logrus.E height -= 2 reorgCount++ if reorgCount > 10 { - log.Fatal("Reorg exceeded max of 100 blocks! Help!") + Log.Fatal("Reorg exceeded max of 100 blocks! Help!") } - log.WithFields(logrus.Fields{ + Log.WithFields(logrus.Fields{ "height": height, "hash": displayHash(block.Hash), "phash": displayHash(block.PrevHash), @@ -156,7 +168,10 @@ func BlockIngestor(rpcClient *rpcclient.Client, cache *BlockCache, log *logrus.E } } -func GetBlock(rpcClient *rpcclient.Client, cache *BlockCache, height int) (*walletrpc.CompactBlock, error) { +// GetBlock returns the compact block at the requested height, first by querying +// the cache, then, if not found, will request the block from zcashd. It returns +// nil if no block exists at this height. +func GetBlock(cache *BlockCache, height int) (*walletrpc.CompactBlock, error) { // First, check the cache to see if we have the block block := cache.Get(height) if block != nil { @@ -164,7 +179,7 @@ func GetBlock(rpcClient *rpcclient.Client, cache *BlockCache, height int) (*wall } // Not in the cache, ask zcashd - block, err := getBlockFromRPC(rpcClient, height) + block, err := getBlockFromRPC(height) if err != nil { return nil, err } @@ -175,12 +190,11 @@ func GetBlock(rpcClient *rpcclient.Client, cache *BlockCache, height int) (*wall return block, nil } -func GetBlockRange(rpcClient *rpcclient.Client, cache *BlockCache, - blockOut chan<- walletrpc.CompactBlock, errOut chan<- error, start, end int) { - +// GetBlockRange returns a sequence of consecutive blocks in the given range. +func GetBlockRange(cache *BlockCache, blockOut chan<- walletrpc.CompactBlock, errOut chan<- error, start, end int) { // Go over [start, end] inclusive for i := start; i <= end; i++ { - block, err := GetBlock(rpcClient, cache, i) + block, err := GetBlock(cache, i) if err != nil { errOut <- err return diff --git a/common/common_test.go b/common/common_test.go new file mode 100644 index 0000000..d71c3e1 --- /dev/null +++ b/common/common_test.go @@ -0,0 +1,290 @@ +package common + +import ( + "bufio" + "encoding/hex" + "encoding/json" + "fmt" + "io/ioutil" + "os" + "strings" + "testing" + "time" + + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "github.com/zcash/lightwalletd/walletrpc" +) + +// ------------------------------------------ Setup +// +// This section does some setup things that may (even if not currently) +// be useful across multiple tests. + +var ( + testT *testing.T + + // The various stub callbacks need to sequence through states + step int + + getblockchaininfoReply []byte + logger = logrus.New() + + getsaplinginfo []byte + + blocks [][]byte // four test blocks +) + +// TestMain does common setup that's shared across multiple tests +func TestMain(m *testing.M) { + output, err := os.OpenFile("test-log", os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) + if err != nil { + os.Stderr.WriteString(fmt.Sprintf("Cannot open test-log: %v", err)) + os.Exit(1) + } + logger.SetOutput(output) + Log = logger.WithFields(logrus.Fields{ + "app": "test", + }) + + getsaplinginfo, err := ioutil.ReadFile("../testdata/getsaplinginfo") + if err != nil { + os.Stderr.WriteString(fmt.Sprintf("Cannot open testdata/getsaplinginfo: %v", err)) + os.Exit(1) + } + getblockchaininfoReply, _ = hex.DecodeString(string(getsaplinginfo)) + + // Several tests need test blocks; read all 4 into memory just once + // (for efficiency). + testBlocks, err := os.Open("../testdata/blocks") + if err != nil { + os.Stderr.WriteString(fmt.Sprintf("Cannot open testdata/blocks: %v", err)) + os.Exit(1) + } + scan := bufio.NewScanner(testBlocks) + for scan.Scan() { // each line (block) + block := scan.Bytes() + // Enclose the hex string in quotes (to make it json, to match what's + // returned by the RPC) + block = []byte("\"" + string(block) + "\"") + blocks = append(blocks, block) + } + + // Setup is done; run all tests. + exitcode := m.Run() + + // cleanup + os.Remove("test-log") + + os.Exit(exitcode) +} + +// Allow tests to verify that sleep has been called (for retries) +var sleepCount int +var sleepDuration time.Duration + +func sleepStub(d time.Duration) { + sleepCount++ + sleepDuration += d +} + +// ------------------------------------------ GetSaplingInfo() + +func getblockchaininfoStub(method string, params []json.RawMessage) (json.RawMessage, error) { + step++ + // Test retry logic (for the moment, it's very simple, just one retry). + switch step { + case 1: + return getblockchaininfoReply, errors.New("first failure") + } + if sleepCount != 1 || sleepDuration != 15*time.Second { + testT.Error("unexpected sleeps", sleepCount, sleepDuration) + } + return getblockchaininfoReply, nil +} + +func TestGetSaplingInfo(t *testing.T) { + testT = t + RawRequest = getblockchaininfoStub + Sleep = sleepStub + saplingHeight, blockHeight, chainName, branchID := GetSaplingInfo() + + // Ensure the retry happened as expected + logFile, err := ioutil.ReadFile("test-log") + if err != nil { + t.Fatal("Cannot read test-log", err) + } + logStr := string(logFile) + if !strings.Contains(logStr, "retrying") { + t.Fatal("Cannot find retrying in test-log") + } + if !strings.Contains(logStr, "retry=1") { + t.Fatal("Cannot find retry=1 in test-log") + } + + // Check the success case (second attempt) + if saplingHeight != 419200 { + t.Error("unexpected saplingHeight", saplingHeight) + } + if blockHeight != 677713 { + t.Error("unexpected blockHeight", blockHeight) + } + if chainName != "main" { + t.Error("unexpected chainName", chainName) + } + if branchID != "2bb40e60" { + t.Error("unexpected branchID", branchID) + } + + if sleepCount != 1 || sleepDuration != 15*time.Second { + t.Error("unexpected sleeps", sleepCount, sleepDuration) + } + step = 0 + sleepCount = 0 + sleepDuration = 0 +} + +// ------------------------------------------ BlockIngestor() + +// There are four test blocks, 0..3 +func getblockStub(method string, params []json.RawMessage) (json.RawMessage, error) { + var height string + err := json.Unmarshal(params[0], &height) + if err != nil { + testT.Fatal("could not unmarshal height") + } + + step++ + switch step { + case 1: + if height != "20" { + testT.Error("unexpected height") + } + // Sunny-day + return blocks[0], nil + case 2: + if height != "21" { + testT.Error("unexpected height") + } + // Sunny-day + return blocks[1], nil + case 3: + if height != "22" { + testT.Error("unexpected height") + } + // This should cause one sleep (then retry) + return nil, errors.New("-8: Block height out of range") + case 4: + if sleepCount != 1 || sleepDuration != 10*time.Second { + testT.Error("unexpected sleeps", sleepCount, sleepDuration) + } + // should re-request the same height + if height != "22" { + testT.Error("unexpected height") + } + // Back to sunny-day + return blocks[2], nil + case 5: + if height != "23" { + testT.Error("unexpected height") + } + // Simulate a reorg (it doesn't matter which block we return here, as + // long as its prevhash doesn't match the latest block's hash) + return blocks[2], nil + case 6: + // When a reorg occurs, the ingestor backs up 2 blocks + if height != "21" { // 23 - 2 + testT.Error("unexpected height") + } + return blocks[1], nil + case 7: + if height != "22" { + testT.Error("unexpected height") + } + // Should fail to Unmarshal the block, sleep, retry + return nil, nil + case 8: + if sleepCount != 2 || sleepDuration != 20*time.Second { + testT.Error("unexpected sleeps", sleepCount, sleepDuration) + } + if height != "22" { + testT.Error("unexpected height") + } + // Back to sunny-day + return blocks[2], nil + } + if height != "23" { + testT.Error("unexpected height") + } + testT.Error("getblockStub called too many times") + return nil, nil +} + +func TestBlockIngestor(t *testing.T) { + testT = t + RawRequest = getblockStub + Sleep = sleepStub + testcache := NewBlockCache(4) + BlockIngestor(testcache, 20, 7) + if step != 7 { + t.Error("unexpected final step", step) + } + step = 0 + sleepCount = 0 + sleepDuration = 0 +} + +func TestGetBlockRange(t *testing.T) { + testT = t + RawRequest = getblockStub + testcache := NewBlockCache(4) + blockChan := make(chan walletrpc.CompactBlock) + errChan := make(chan error) + go GetBlockRange(testcache, blockChan, errChan, 20, 22) + + // read in block 20 + select { + case err := <-errChan: + // this will also catch context.DeadlineExceeded from the timeout + t.Fatal("unexpected error:", err) + case cBlock := <-blockChan: + if cBlock.Height != 380640 { + t.Fatal("unexpected Height:", cBlock.Height) + } + } + + // read in block 21 + select { + case err := <-errChan: + // this will also catch context.DeadlineExceeded from the timeout + t.Fatal("unexpected error:", err) + case cBlock := <-blockChan: + if cBlock.Height != 380641 { + t.Fatal("unexpected Height:", cBlock.Height) + } + } + + // try to read in block 22, but this will fail (see case 3 above) + select { + case err := <-errChan: + // this will also catch context.DeadlineExceeded from the timeout + if err.Error() != "block requested is newer than latest block" { + t.Fatal("unexpected error:", err) + } + case _ = <-blockChan: + t.Fatal("reading height 22 should have failed") + } + + // check goroutine GetBlockRange() reaching the end of the range (and exiting) + go GetBlockRange(testcache, blockChan, errChan, 1, 0) + err := <-errChan + if err != nil { + t.Fatal("unexpected err return") + } +} + +func TestGenerateCerts(t *testing.T) { + if GenerateCerts() == nil { + t.Fatal("GenerateCerts returned nil") + } +} diff --git a/common/generatecerts.go b/common/generatecerts.go index 2b9cb79..73cea08 100644 --- a/common/generatecerts.go +++ b/common/generatecerts.go @@ -7,14 +7,13 @@ import ( "crypto/x509" "crypto/x509/pkix" "encoding/pem" - "fmt" - "log" "math/big" "time" ) // GenerateCerts create self signed certificate for local development use -func GenerateCerts() (cert *tls.Certificate) { +// (and, if using grpcurl, specify the -insecure argument option) +func GenerateCerts() *tls.Certificate { privKey, err := rsa.GenerateKey(rand.Reader, 2048) publicKey := &privKey.PublicKey @@ -22,7 +21,7 @@ func GenerateCerts() (cert *tls.Certificate) { serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) if err != nil { - log.Fatalf("Failed to generate serial number: %s", err) + Log.Fatal("Failed to generate serial number:", err) } template := x509.Certificate{ @@ -43,18 +42,17 @@ func GenerateCerts() (cert *tls.Certificate) { certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, publicKey, privKey) if err != nil { - log.Fatalf("Failed to create certificate: %s", err) + Log.Fatal("Failed to create certificate:", err) } // PEM encode the certificate (this is a standard TLS encoding) b := pem.Block{Type: "CERTIFICATE", Bytes: certDER} certPEM := pem.EncodeToMemory(&b) - fmt.Printf("%s\n", certPEM) // PEM encode the private key privBytes, err := x509.MarshalPKCS8PrivateKey(privKey) if err != nil { - log.Fatalf("Unable to marshal private key: %v", err) + Log.Fatal("Unable to marshal private key:", err) } keyPEM := pem.EncodeToMemory(&pem.Block{ Type: "RSA PRIVATE KEY", Bytes: privBytes, @@ -63,10 +61,8 @@ func GenerateCerts() (cert *tls.Certificate) { // Create a TLS cert using the private key and certificate tlsCert, err := tls.X509KeyPair(certPEM, keyPEM) if err != nil { - log.Fatalf("invalid key pair: %v", err) + Log.Fatal("invalid key pair:", err) } - cert = &tlsCert - return cert - + return &tlsCert } diff --git a/docker/gen_cert.sh b/docker/gen_cert.sh index e8a0a83..3079f9d 100644 --- a/docker/gen_cert.sh +++ b/docker/gen_cert.sh @@ -3,4 +3,4 @@ ## In debian # apt-get update && apt-get install -y openssl -openssl req -x509 -nodes -newkey rsa:2048 -keyout ./cert.key -out ./cert.pem -days 3650 -subj "/C=US/ST=CA/O=MyOrg, Inc./CN=lightwalletd.testnet.local" -reqexts SAN -config <(cat /etc/ssl/openssl.cnf <(printf "\n[SAN]\nsubjectAltName=DNS:lightwalletd.testnet.local,DNS:127.0.0.1,DNS:localhost")) \ No newline at end of file +openssl req -x509 -nodes -newkey rsa:2048 -keyout ./cert.key -out ./cert.pem -days 3650 -subj "/C=US/ST=CA/O=MyOrg, Inc./CN=lightwalletd.testnet.local" -reqexts SAN -config <(cat /etc/ssl/openssl.cnf <(printf "\n[SAN]\nsubjectAltName=DNS:lightwalletd.testnet.local,DNS:127.0.0.1,DNS:localhost")) diff --git a/docker/zcash.conf b/docker/zcash.conf index cc00b10..bed2d2f 100644 --- a/docker/zcash.conf +++ b/docker/zcash.conf @@ -1,4 +1,4 @@ rpcuser=zcashrpc rpcpassword=notsecure rpcbind=zcashd -rpcport=3434 \ No newline at end of file +rpcport=3434 diff --git a/docs/architecture.md b/docs/architecture.md index f0e62e0..847c937 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -42,7 +42,7 @@ First, install [Go >= 1.11](https://golang.org/dl/#stable). Older versions of Go Now clone this repo and start the ingester. The first run will start slow as Go builds the sqlite C interface: ``` -$ git clone https://github.com/zcash-hackworks/lightwalletd +$ git clone https://github.com/zcash/lightwalletd $ cd lightwalletd $ go run cmd/ingest/main.go --conf-file --db-path ``` @@ -55,7 +55,7 @@ The frontend is the component that talks to clients. It exposes an API that allows a client to query for current blockheight, request ranges of compact block data, request specific transaction details, and send new Zcash transactions. -The API is specified in [Protocol Buffers](https://developers.google.com/protocol-buffers/) and implemented using [gRPC](https://grpc.io). You can find the exact details in [these files](https://github.com/zcash-hackworks/lightwalletd/tree/master/walletrpc). +The API is specified in [Protocol Buffers](https://developers.google.com/protocol-buffers/) and implemented using [gRPC](https://grpc.io). You can find the exact details in [these files](https://github.com/zcash/lightwalletd/tree/master/walletrpc). **How do I run it?** @@ -66,7 +66,7 @@ First, install [Go >= 1.11](https://golang.org/dl/#stable). Older versions of Go Now clone this repo and start the frontend. The first run will start slow as Go builds the sqlite C interface: ``` -$ git clone https://github.com/zcash-hackworks/lightwalletd +$ git clone https://github.com/zcash/lightwalletd $ cd lightwalletd $ go run cmd/server/main.go --db-path --bind-addr 0.0.0.0:9067 ``` @@ -79,13 +79,13 @@ x509 Certificates! This software relies on the confidentiality and integrity of Otherwise, not much! This is a very simple piece of software. Make sure you point it at the same storage as the ingester. See the "Production" section for some caveats. -Support for users sending transactions will require the ability to make JSON-RPC calls to a zcashd instance. By default the frontend tries to pull RPC credentials from your zcashd.conf file, but you can specify other credentials via command line flag. In the future, it should be possible to do this with environment variables [(#2)](https://github.com/zcash-hackworks/lightwalletd/issues/2). +Support for users sending transactions will require the ability to make JSON-RPC calls to a zcashd instance. By default the frontend tries to pull RPC credentials from your zcashd.conf file, but you can specify other credentials via command line flag. In the future, it should be possible to do this with environment variables [(#2)](https://github.com/zcash/lightwalletd/issues/2). ## Storage The storage provider is the component that caches compact blocks and their metadata for the frontend to retrieve and serve to clients. -It currently assumes a SQL database. The schema can be found [here](https://github.com/zcash-hackworks/lightwalletd/blob/d53507cc39e8da52e14d08d9c63fee96d3bd16c3/storage/sqlite3.go#L15), but they're extremely provisional. We expect that anyone deploying lightwalletd at scale will adapt it to their own existing data infrastructure. +It currently assumes a SQL database. The schema can be found [here](https://github.com/zcash/lightwalletd/blob/d53507cc39e8da52e14d08d9c63fee96d3bd16c3/storage/sqlite3.go#L15), but they're extremely provisional. We expect that anyone deploying lightwalletd at scale will adapt it to their own existing data infrastructure. **How do I run it?** diff --git a/docs/docker-compose-setup.md b/docs/docker-compose-setup.md index 5f8c6dd..a58c2a0 100644 --- a/docs/docker-compose-setup.md +++ b/docs/docker-compose-setup.md @@ -97,4 +97,4 @@ Loki as a rich query syntax to help with log in many ways, for example combine 2 ![grafana-explore4](./images/grafana-explore-4.png) -See more here: https://github.com/grafana/loki/blob/master/docs/logql.md \ No newline at end of file +See more here: https://github.com/grafana/loki/blob/master/docs/logql.md diff --git a/frontend/frontend_test.go b/frontend/frontend_test.go index adaee6c..bad0c30 100644 --- a/frontend/frontend_test.go +++ b/frontend/frontend_test.go @@ -1,8 +1,478 @@ package frontend import ( + "bufio" + "bytes" + "context" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "os" + "strings" "testing" + + "github.com/sirupsen/logrus" + "github.com/zcash/lightwalletd/common" + "github.com/zcash/lightwalletd/walletrpc" ) -func TestString_read(t *testing.T) { +var ( + testT *testing.T + logger = logrus.New() + step int + + cache *common.BlockCache + lwd walletrpc.CompactTxStreamerServer + blocks [][]byte // four test blocks + rawTxData [][]byte +) + +func TestMain(m *testing.M) { + output, err := os.OpenFile("test-log", os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) + if err != nil { + os.Stderr.WriteString(fmt.Sprint("Cannot open test-log:", err)) + os.Exit(1) + } + logger.SetOutput(output) + common.Log = logger.WithFields(logrus.Fields{ + "app": "test", + }) + + cache = common.NewBlockCache(4) + lwd, err = NewLwdStreamer(cache) + if err != nil { + os.Stderr.WriteString(fmt.Sprint("NewLwdStreamer failed:", err)) + os.Exit(1) + } + + // Several tests need test blocks; read all 4 into memory just once + // (for efficiency). + testBlocks, err := os.Open("../testdata/blocks") + if err != nil { + os.Stderr.WriteString(fmt.Sprint("Error:", err)) + os.Exit(1) + } + scan := bufio.NewScanner(testBlocks) + for scan.Scan() { // each line (block) + block := scan.Bytes() + // Enclose the hex string in quotes (to make it json, to match what's + // returned by the RPC) + block = []byte("\"" + string(block) + "\"") + blocks = append(blocks, block) + } + + testData, err := os.Open("../testdata/zip243_raw_tx") + if err != nil { + os.Stderr.WriteString(fmt.Sprint("Error:", err)) + os.Exit(1) + } + defer testData.Close() + + // Parse the raw transactions file + rawTxData = [][]byte{} + scan = bufio.NewScanner(testData) + for scan.Scan() { + dataLine := scan.Text() + // Skip the comments + if strings.HasPrefix(dataLine, "#") { + continue + } + + txData, err := hex.DecodeString(dataLine) + if err != nil { + os.Stderr.WriteString(fmt.Sprint("Error:", err)) + os.Exit(1) + } + + rawTxData = append(rawTxData, txData) + } + + // Setup is done; run all tests. + exitcode := m.Run() + + // cleanup + os.Remove("test-log") + + os.Exit(exitcode) +} + +func TestGetTransaction(t *testing.T) { + // GetTransaction() will mostly be tested below via TestGetAddressTxids + rawtx, err := lwd.GetTransaction(context.Background(), + &walletrpc.TxFilter{}) + if err == nil { + testT.Fatal("GetTransaction unexpectedly succeeded") + } + if err.Error() != "Please call GetTransaction with txid" { + testT.Fatal("GetTransaction unexpected error message") + } + if rawtx != nil { + testT.Fatal("GetTransaction non-nil rawtx returned") + } + + rawtx, err = lwd.GetTransaction(context.Background(), + &walletrpc.TxFilter{Block: &walletrpc.BlockID{Hash: []byte{}}}) + if err == nil { + testT.Fatal("GetTransaction unexpectedly succeeded") + } + if err.Error() != "Can't GetTransaction with a blockhash+num. Please call GetTransaction with txid" { + testT.Fatal("GetTransaction unexpected error message") + } + if rawtx != nil { + testT.Fatal("GetTransaction non-nil rawtx returned") + } +} + +func getblockStub(method string, params []json.RawMessage) (json.RawMessage, error) { + step++ + var height string + err := json.Unmarshal(params[0], &height) + if err != nil { + testT.Fatal("could not unmarshal height") + } + if height != "1234" { + testT.Fatal("unexpected getblock height", height) + } + + // Test retry logic (for the moment, it's very simple, just one retry). + switch step { + case 1: + return blocks[0], nil + case 2: + return nil, errors.New("getblock test error") + } + testT.Fatal("unexpected call to getblockStub") + return nil, nil +} + +func TestGetLatestBlock(t *testing.T) { + testT = t + common.RawRequest = getblockStub + // This argument is not used (it may be in the future) + req := &walletrpc.ChainSpec{} + + blockID, err := lwd.GetLatestBlock(context.Background(), req) + if err == nil { + t.Fatal("GetLatestBlock should have failed, empty cache") + } + if err.Error() != "Cache is empty. Server is probably not yet ready" { + t.Fatal("GetLatestBlock incorrect error", err) + } + if blockID != nil { + t.Fatal("unexpected blockID", blockID) + } + + // This does zcashd rpc "getblock", calls getblockStub() above + block, err := common.GetBlock(cache, 1234) + if err != nil { + t.Fatal("getBlockFromRPC failed", err) + } + reorg, err := cache.Add(40, block) + if reorg { + t.Fatal("unexpected reorg") + } + if err != nil { + t.Fatal("cache.Add failed:", err) + } + blockID, err = lwd.GetLatestBlock(context.Background(), req) + if err != nil { + t.Fatal("lwd.GetLatestBlock failed", err) + } + if blockID.Height != 40 { + t.Fatal("unexpected blockID.height") + } + step = 0 +} + +// A valid address starts with "t", followed by 34 alpha characters; +// these should all be detected as invalid. +var addressTests = []string{ + "", // too short + "a", // too short + "t123456789012345678901234567890123", // one byte too short + "t12345678901234567890123456789012345", // one byte too long + "t123456789012345678901234567890123*", // invalid "*" + "s1234567890123456789012345678901234", // doesn't start with "t" + " t1234567890123456789012345678901234", // extra stuff before + "t1234567890123456789012345678901234 ", // extra stuff after + "\nt1234567890123456789012345678901234", // newline before + "t1234567890123456789012345678901234\n", // newline after +} + +func zcashdrpcStub(method string, params []json.RawMessage) (json.RawMessage, error) { + step++ + switch method { + case "getaddresstxids": + var filter struct { + Addresses []string `json: addresses` + Start float64 `json: start` + End float64 `json: end` + } + err := json.Unmarshal(params[0], &filter) + if err != nil { + testT.Fatal("could not unmarshal block filter") + } + if len(filter.Addresses) != 1 { + testT.Fatal("wrong number of addresses") + } + if filter.Addresses[0] != "t1234567890123456789012345678901234" { + testT.Fatal("wrong address") + } + if filter.Start != 20 { + testT.Fatal("wrong start") + } + if filter.End != 30 { + testT.Fatal("wrong end") + } + return []byte("[\"6732cf8d67aac5b82a2a0f0217a7d4aa245b2adb0b97fd2d923dfc674415e221\"]"), nil + case "getrawtransaction": + switch step { + case 2: + txstr := hex.EncodeToString(rawTxData[0]) + return []byte("{\"hex\": \"" + txstr + "\", \"height\": 1234567}"), nil + case 4: + // empty return value, should be okay + return []byte(""), errors.New("-5: test getrawtransaction error") + } + } + testT.Fatal("unexpected call to zcashdrpcStub") + return nil, nil +} + +type testgettx struct { + walletrpc.CompactTxStreamer_GetAddressTxidsServer +} + +func (tg *testgettx) Context() context.Context { + return context.Background() +} + +func (tg *testgettx) Send(tx *walletrpc.RawTransaction) error { + if !bytes.Equal(tx.Data, []byte(hex.EncodeToString(rawTxData[0]))) { + testT.Fatal("mismatch transaction data") + } + if tx.Height != 1234567 { + testT.Fatal("unexpected transaction height", tx.Height) + } + return nil +} + +func TestGetAddressTxids(t *testing.T) { + testT = t + common.RawRequest = zcashdrpcStub + addressBlockFilter := &walletrpc.TransparentAddressBlockFilter{ + Range: &walletrpc.BlockRange{ + Start: &walletrpc.BlockID{Height: 20}, + End: &walletrpc.BlockID{Height: 30}, + }, + } + + // Ensure that a bad address is detected + for i, addressTest := range addressTests { + addressBlockFilter.Address = addressTest + err := lwd.GetAddressTxids(addressBlockFilter, &testgettx{}) + if err == nil { + t.Fatal("GetAddressTxids should have failed on bad address, case", i) + } + if err.Error() != "Invalid address" { + t.Fatal("GetAddressTxids incorrect error on bad address, case", i) + } + } + + // valid address + addressBlockFilter.Address = "t1234567890123456789012345678901234" + err := lwd.GetAddressTxids(addressBlockFilter, &testgettx{}) + if err != nil { + t.Fatal("GetAddressTxids failed", err) + } + + // this time GetTransaction() will return an error + err = lwd.GetAddressTxids(addressBlockFilter, &testgettx{}) + if err == nil { + t.Fatal("GetAddressTxids succeeded") + } + step = 0 +} + +func TestGetBlock(t *testing.T) { + testT = t + common.RawRequest = getblockStub + _, err := lwd.GetBlock(context.Background(), &walletrpc.BlockID{}) + if err == nil { + t.Fatal("GetBlock should have failed") + } + _, err = lwd.GetBlock(context.Background(), &walletrpc.BlockID{Height: 0}) + if err == nil { + t.Fatal("GetBlock should have failed") + } + _, err = lwd.GetBlock(context.Background(), &walletrpc.BlockID{Hash: []byte{0}}) + if err == nil { + t.Fatal("GetBlock should have failed") + } + if err.Error() != "GetBlock by Hash is not yet implemented" { + t.Fatal("GetBlock hash unimplemented error message failed") + } + block, err := lwd.GetBlock(context.Background(), &walletrpc.BlockID{Height: 1234}) + if err != nil { + t.Fatal("GetBlock failed:", err) + } + if block.Height != 380640 { + t.Fatal("GetBlock returned unexpected block:", err) + } + // getblockStub() case 2: return error + block, err = lwd.GetBlock(context.Background(), &walletrpc.BlockID{Height: 1234}) + if err == nil { + t.Fatal("GetBlock should have failed") + } + if block != nil { + t.Fatal("GetBlock returned unexpected non-nil block") + } + step = 0 +} + +type testgetbrange struct { + walletrpc.CompactTxStreamer_GetAddressTxidsServer +} + +func (tg *testgetbrange) Context() context.Context { + return context.Background() +} + +func (tg *testgetbrange) Send(cb *walletrpc.CompactBlock) error { + return nil +} + +func TestGetBlockRange(t *testing.T) { + testT = t + common.RawRequest = getblockStub + blockrange := &walletrpc.BlockRange{ + Start: &walletrpc.BlockID{Height: 1234}, + End: &walletrpc.BlockID{Height: 1234}, + } + // getblockStub() case 1 (success) + err := lwd.GetBlockRange(blockrange, &testgetbrange{}) + if err != nil { + t.Fatal("GetBlockRange failed", err) + } + // getblockStub() case 2 (failure) + err = lwd.GetBlockRange(blockrange, &testgetbrange{}) + if err == nil { + t.Fatal("GetBlockRange should have failed") + } + step = 0 +} + +func getblockchaininfoStub(method string, params []json.RawMessage) (json.RawMessage, error) { + getsaplinginfo, _ := ioutil.ReadFile("../testdata/getsaplinginfo") + getblockchaininfoReply, _ := hex.DecodeString(string(getsaplinginfo)) + return getblockchaininfoReply, nil +} + +func TestGetLightdInfo(t *testing.T) { + testT = t + common.RawRequest = getblockchaininfoStub + ldinfo, err := lwd.GetLightdInfo(context.Background(), &walletrpc.Empty{}) + if err != nil { + t.Fatal("GetLightdInfo failed", err) + } + if ldinfo.Vendor != "ECC LightWalletD" { + t.Fatal("GetLightdInfo: unexpected vendor", ldinfo) + } + step = 0 +} + +func sendrawtransactionStub(method string, params []json.RawMessage) (json.RawMessage, error) { + step++ + if method != "sendrawtransaction" { + testT.Fatal("unexpected method") + } + if string(params[0]) != "\"07\"" { + testT.Fatal("unexpected tx data") + } + switch step { + case 1: + return []byte("sendtxresult"), nil + case 2: + return nil, errors.New("-17: some error") + } + testT.Fatal("unexpected call to sendrawtransactionStub") + return nil, nil +} + +func TestSendTransaction(t *testing.T) { + testT = t + common.RawRequest = sendrawtransactionStub + rawtx := walletrpc.RawTransaction{Data: []byte{7}} + sendresult, err := lwd.SendTransaction(context.Background(), &rawtx) + if err != nil { + t.Fatal("SendTransaction failed", err) + } + if sendresult.ErrorCode != 0 { + t.Fatal("SendTransaction unexpected ErrorCode return") + } + if sendresult.ErrorMessage != "sendtxresult" { + t.Fatal("SendTransaction unexpected ErrorMessage return") + } + + // sendrawtransactionStub case 2 (error) + // but note that the error is send within the response + sendresult, err = lwd.SendTransaction(context.Background(), &rawtx) + if err != nil { + t.Fatal("SendTransaction failed:", err) + } + if sendresult.ErrorCode != -17 { + t.Fatal("SendTransaction unexpected ErrorCode return") + } + if sendresult.ErrorMessage != "some error" { + t.Fatal("SendTransaction unexpected ErrorMessage return") + } + step = 0 +} + +var sampleconf = ` +testnet = 1 +rpcport = 18232 +rpcbind = 127.0.0.1 +rpcuser = testlightwduser +rpcpassword = testlightwdpassword +` + +func TestNewZRPCFromConf(t *testing.T) { + connCfg, err := connFromConf([]byte(sampleconf)) + if err != nil { + t.Fatal("connFromConf failed") + } + if connCfg.Host != "127.0.0.1:18232" { + t.Fatal("connFromConf returned unexpected Host") + } + if connCfg.User != "testlightwduser" { + t.Fatal("connFromConf returned unexpected User") + } + if connCfg.Pass != "testlightwdpassword" { + t.Fatal("connFromConf returned unexpected User") + } + if !connCfg.HTTPPostMode { + t.Fatal("connFromConf returned unexpected HTTPPostMode") + } + if !connCfg.DisableTLS { + t.Fatal("connFromConf returned unexpected DisableTLS") + } + + // can't pass an integer + connCfg, err = connFromConf(10) + if err == nil { + t.Fatal("connFromConf unexpected success") + } + + // Can't verify returned values, but at least run it + _, err = NewZRPCFromConf([]byte(sampleconf)) + if err != nil { + t.Fatal("NewZRPCFromClient failed") + } + _, err = NewZRPCFromConf(10) + if err == nil { + t.Fatal("NewZRPCFromClient unexpected success") + } } diff --git a/frontend/rpc_client.go b/frontend/rpc_client.go index 7508570..a519cb9 100644 --- a/frontend/rpc_client.go +++ b/frontend/rpc_client.go @@ -8,14 +8,30 @@ import ( ini "gopkg.in/ini.v1" ) -func NewZRPCFromConf(confPath string) (*rpcclient.Client, error) { +func NewZRPCFromConf(confPath interface{}) (*rpcclient.Client, error) { + connCfg, err := connFromConf(confPath) + if err != nil { + return nil, err + } + return rpcclient.New(connCfg, nil) +} + +// If passed a string, interpret as a path, open and read; if passed +// a byte slice, interpret as the config file content (used in testing). +func connFromConf(confPath interface{}) (*rpcclient.ConnConfig, error) { cfg, err := ini.Load(confPath) if err != nil { return nil, errors.Wrap(err, "failed to read config file") } rpcaddr := cfg.Section("").Key("rpcbind").String() + if rpcaddr == "" { + rpcaddr = "127.0.0.1" + } rpcport := cfg.Section("").Key("rpcport").String() + if rpcport == "" { + rpcport = "8232" // mainnet + } username := cfg.Section("").Key("rpcuser").String() password := cfg.Section("").Key("rpcpassword").String() @@ -29,5 +45,5 @@ func NewZRPCFromConf(confPath string) (*rpcclient.Client, error) { } // Notice the notification parameter is nil since notifications are // not supported in HTTP POST mode. - return rpcclient.New(connCfg, nil) + return connCfg, nil } diff --git a/frontend/service.go b/frontend/service.go index 50240ad..52d671e 100644 --- a/frontend/service.go +++ b/frontend/service.go @@ -10,11 +10,8 @@ import ( "strings" "time" - "github.com/btcsuite/btcd/rpcclient" - - "github.com/sirupsen/logrus" - "github.com/zcash-hackworks/lightwalletd/common" - "github.com/zcash-hackworks/lightwalletd/walletrpc" + "github.com/zcash/lightwalletd/common" + "github.com/zcash/lightwalletd/walletrpc" ) var ( @@ -23,36 +20,30 @@ var ( // the service type type LwdStreamer struct { - cache *common.BlockCache - client *rpcclient.Client - log *logrus.Entry + cache *common.BlockCache } -func NewLwdStreamer(client *rpcclient.Client, cache *common.BlockCache, log *logrus.Entry) (walletrpc.CompactTxStreamerServer, error) { - return &LwdStreamer{cache, client, log}, nil -} - -func (s *LwdStreamer) GetCache() *common.BlockCache { - return s.cache +func NewLwdStreamer(cache *common.BlockCache) (walletrpc.CompactTxStreamerServer, error) { + return &LwdStreamer{cache}, nil } func (s *LwdStreamer) GetLatestBlock(ctx context.Context, placeholder *walletrpc.ChainSpec) (*walletrpc.BlockID, error) { - latestBlock := s.cache.GetLatestBlock() + latestBlock := s.cache.GetLatestHeight() if latestBlock == -1 { - return nil, errors.New("Cache is empty. Server is probably not yet ready.") + return nil, errors.New("Cache is empty. Server is probably not yet ready") } // TODO: also return block hashes here return &walletrpc.BlockID{Height: uint64(latestBlock)}, nil } -func (s *LwdStreamer) GetAddressTxids(addressBlockFilter *walletrpc.TransparentAddressBlockFilter, resp walletrpc.CompactTxStreamer_GetAddressTxidsServer) error { +func (s *LwdStreamer) GetAddressTxids( addressBlockFilter *walletrpc.TransparentAddressBlockFilter, resp walletrpc.CompactTxStreamer_GetAddressTxidsServer) error { // Test to make sure Address is a single t address match, err := regexp.Match("\\At[a-zA-Z0-9]{34}\\z", []byte(addressBlockFilter.Address)) if err != nil || !match { - s.log.Errorf("Unrecognized address: %s", addressBlockFilter.Address) - return nil + common.Log.Error("Invalid address:", addressBlockFilter.Address) + return errors.New("Invalid address") } params := make([]json.RawMessage, 1) @@ -62,19 +53,19 @@ func (s *LwdStreamer) GetAddressTxids(addressBlockFilter *walletrpc.TransparentA params[0] = json.RawMessage(st) - result, rpcErr := s.client.RawRequest("getaddresstxids", params) + result, rpcErr := common.RawRequest("getaddresstxids", params) // For some reason, the error responses are not JSON if rpcErr != nil { - s.log.Errorf("Got error: %s", rpcErr.Error()) - return nil + common.Log.Errorf("GetAddressTxids error: %s", rpcErr.Error()) + return err } var txids []string err = json.Unmarshal(result, &txids) if err != nil { - s.log.Errorf("Got error: %s", err.Error()) - return nil + common.Log.Errorf("GetAddressTxids error: %s", err.Error()) + return err } timeout, cancel := context.WithTimeout(resp.Context(), 30*time.Second) @@ -89,10 +80,12 @@ func (s *LwdStreamer) GetAddressTxids(addressBlockFilter *walletrpc.TransparentA } tx, err := s.GetTransaction(timeout, &walletrpc.TxFilter{Hash: txid}) if err != nil { - s.log.Errorf("Got error: %s", err.Error()) - return nil + common.Log.Errorf("GetTransaction error: %s", err.Error()) + return err + } + if err = resp.Send(tx); err != nil { + return err } - resp.Send(tx) } return nil } @@ -107,7 +100,7 @@ func (s *LwdStreamer) GetBlock(ctx context.Context, id *walletrpc.BlockID) (*wal // TODO: Get block by hash return nil, errors.New("GetBlock by Hash is not yet implemented") } - cBlock, err := common.GetBlock(s.client, s.cache, int(id.Height)) + cBlock, err := common.GetBlock(s.cache, int(id.Height)) if err != nil { return nil, err @@ -120,7 +113,7 @@ func (s *LwdStreamer) GetBlockRange(span *walletrpc.BlockRange, resp walletrpc.C blockChan := make(chan walletrpc.CompactBlock) errChan := make(chan error) - go common.GetBlockRange(s.client, s.cache, blockChan, errChan, int(span.Start.Height), int(span.End.Height)) + go common.GetBlockRange(s.cache, blockChan, errChan, int(span.Start.Height), int(span.End.Height)) for { select { @@ -134,14 +127,9 @@ func (s *LwdStreamer) GetBlockRange(span *walletrpc.BlockRange, resp walletrpc.C } } } - - return nil } func (s *LwdStreamer) GetTransaction(ctx context.Context, txf *walletrpc.TxFilter) (*walletrpc.RawTransaction, error) { - var txBytes []byte - var txHeight float64 - if txf.Hash != nil { txid := txf.Hash for left, right := 0, len(txid)-1; left < right; left, right = left+1, right-1 { @@ -149,67 +137,40 @@ func (s *LwdStreamer) GetTransaction(ctx context.Context, txf *walletrpc.TxFilte } leHashString := hex.EncodeToString(txid) - // First call to get the raw transaction bytes - params := make([]json.RawMessage, 1) - params[0] = json.RawMessage("\"" + leHashString + "\"") - - result, rpcErr := s.client.RawRequest("getrawtransaction", params) - - var err error - // For some reason, the error responses are not JSON - if rpcErr != nil { - s.log.Errorf("Got error: %s", rpcErr.Error()) - errParts := strings.SplitN(rpcErr.Error(), ":", 2) - _, err = strconv.ParseInt(errParts[0], 10, 32) - return nil, err - } - var txhex string - err = json.Unmarshal(result, &txhex) - if err != nil { - return nil, err + params := []json.RawMessage{ + json.RawMessage("\"" + leHashString + "\""), + json.RawMessage("1"), } - txBytes, err = hex.DecodeString(txhex) - if err != nil { - return nil, err - } - // Second call to get height - params = make([]json.RawMessage, 2) - params[0] = json.RawMessage("\"" + leHashString + "\"") - params[1] = json.RawMessage("1") - - result, rpcErr = s.client.RawRequest("getrawtransaction", params) + result, rpcErr := common.RawRequest("getrawtransaction", params) // For some reason, the error responses are not JSON if rpcErr != nil { - s.log.Errorf("Got error: %s", rpcErr.Error()) - errParts := strings.SplitN(rpcErr.Error(), ":", 2) - _, err = strconv.ParseInt(errParts[0], 10, 32) - return nil, err + common.Log.Errorf("GetTransaction error: %s", rpcErr.Error()) + return nil, errors.New((strings.Split(rpcErr.Error(), ":"))[0]) } var txinfo interface{} - err = json.Unmarshal(result, &txinfo) + err := json.Unmarshal(result, &txinfo) if err != nil { return nil, err } - txHeight = txinfo.(map[string]interface{})["height"].(float64) - return &walletrpc.RawTransaction{Data: txBytes, Height: uint64(txHeight)}, nil + txBytes := txinfo.(map[string]interface{})["hex"].(string) + txHeight := txinfo.(map[string]interface{})["height"].(float64) + return &walletrpc.RawTransaction{Data: []byte(txBytes), Height: uint64(txHeight)}, nil } if txf.Block != nil && txf.Block.Hash != nil { - s.log.Error("Can't GetTransaction with a blockhash+num. Please call GetTransaction with txid") + common.Log.Error("Can't GetTransaction with a blockhash+num. Please call GetTransaction with txid") return nil, errors.New("Can't GetTransaction with a blockhash+num. Please call GetTransaction with txid") } - s.log.Error("Please call GetTransaction with txid") + common.Log.Error("Please call GetTransaction with txid") return nil, errors.New("Please call GetTransaction with txid") } // GetLightdInfo gets the LightWalletD (this server) info func (s *LwdStreamer) GetLightdInfo(ctx context.Context, in *walletrpc.Empty) (*walletrpc.LightdInfo, error) { - saplingHeight, blockHeight, chainName, consensusBranchId := common.GetSaplingInfo(s.client, s.log) + saplingHeight, blockHeight, chainName, consensusBranchId := common.GetSaplingInfo() - // TODO these are called Error but they aren't at the moment. - // A success will return code 0 and message txhash. return &walletrpc.LightdInfo{ Version: "0.2.1", Vendor: "ECC LightWalletD", @@ -240,7 +201,7 @@ func (s *LwdStreamer) SendTransaction(ctx context.Context, rawtx *walletrpc.RawT params := make([]json.RawMessage, 1) txHexString := hex.EncodeToString(rawtx.Data) params[0] = json.RawMessage("\"" + txHexString + "\"") - result, rpcErr := s.client.RawRequest("sendrawtransaction", params) + result, rpcErr := common.RawRequest("sendrawtransaction", params) var err error var errCode int64 diff --git a/go.mod b/go.mod index 077002e..a9c81e4 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/zcash-hackworks/lightwalletd +module github.com/zcash/lightwalletd go 1.12 diff --git a/parser/block.go b/parser/block.go index e47575f..888fed0 100644 --- a/parser/block.go +++ b/parser/block.go @@ -4,12 +4,12 @@ import ( "fmt" "github.com/pkg/errors" - "github.com/zcash-hackworks/lightwalletd/parser/internal/bytestring" - "github.com/zcash-hackworks/lightwalletd/walletrpc" + "github.com/zcash/lightwalletd/parser/internal/bytestring" + "github.com/zcash/lightwalletd/walletrpc" ) type Block struct { - hdr *blockHeader + hdr *BlockHeader vtx []*Transaction height int } @@ -56,7 +56,7 @@ func (b *Block) HasSaplingTransactions() bool { return false } -// see https://github.com/zcash-hackworks/lightwalletd/issues/17#issuecomment-467110828 +// see https://github.com/zcash/lightwalletd/issues/17#issuecomment-467110828 const genesisTargetDifficulty = 520617983 // GetHeight() extracts the block height from the coinbase transaction. See diff --git a/parser/block_header.go b/parser/block_header.go index ba55cea..82ff152 100644 --- a/parser/block_header.go +++ b/parser/block_header.go @@ -8,7 +8,7 @@ import ( "math/big" "github.com/pkg/errors" - "github.com/zcash-hackworks/lightwalletd/parser/internal/bytestring" + "github.com/zcash/lightwalletd/parser/internal/bytestring" ) const ( @@ -56,12 +56,14 @@ type rawBlockHeader struct { Solution []byte } -type blockHeader struct { +type BlockHeader struct { *rawBlockHeader cachedHash []byte targetThreshold *big.Int } +// CompactLengthPrefixedLen calculates the total number of bytes needed to +// encode 'length' bytes. func CompactLengthPrefixedLen(length int) int { if length < 253 { return 1 + length @@ -74,6 +76,7 @@ func CompactLengthPrefixedLen(length int) int { } } +// WriteCompactLengthPrefixedLen writes the given length to the stream. func WriteCompactLengthPrefixedLen(buf *bytes.Buffer, length int) { if length < 253 { binary.Write(buf, binary.LittleEndian, uint8(length)) @@ -113,8 +116,8 @@ func (hdr *rawBlockHeader) MarshalBinary() ([]byte, error) { return backing[:headerSize], nil } -func NewBlockHeader() *blockHeader { - return &blockHeader{ +func NewBlockHeader() *BlockHeader { + return &BlockHeader{ rawBlockHeader: new(rawBlockHeader), } } @@ -122,7 +125,7 @@ func NewBlockHeader() *blockHeader { // ParseFromSlice parses the block header struct from the provided byte slice, // advancing over the bytes read. If successful it returns the rest of the // slice, otherwise it returns the input slice unaltered along with an error. -func (hdr *blockHeader) ParseFromSlice(in []byte) (rest []byte, err error) { +func (hdr *BlockHeader) ParseFromSlice(in []byte) (rest []byte, err error) { s := bytestring.String(in) // Primary parsing layer: sort the bytes into things @@ -185,7 +188,7 @@ func parseNBits(b []byte) *big.Int { } // GetDisplayHash returns the bytes of a block hash in big-endian order. -func (hdr *blockHeader) GetDisplayHash() []byte { +func (hdr *BlockHeader) GetDisplayHash() []byte { if hdr.cachedHash != nil { return hdr.cachedHash } @@ -211,7 +214,7 @@ func (hdr *blockHeader) GetDisplayHash() []byte { } // GetEncodableHash returns the bytes of a block hash in little-endian wire order. -func (hdr *blockHeader) GetEncodableHash() []byte { +func (hdr *BlockHeader) GetEncodableHash() []byte { serializedHeader, err := hdr.MarshalBinary() if err != nil { @@ -226,7 +229,7 @@ func (hdr *blockHeader) GetEncodableHash() []byte { return digest[:] } -func (hdr *blockHeader) GetDisplayPrevHash() []byte { +func (hdr *BlockHeader) GetDisplayPrevHash() []byte { rhash := make([]byte, len(hdr.HashPrevBlock)) copy(rhash, hdr.HashPrevBlock) // Reverse byte order diff --git a/parser/block_test.go b/parser/block_test.go index b105010..30fe40d 100644 --- a/parser/block_test.go +++ b/parser/block_test.go @@ -2,6 +2,7 @@ package parser import ( "bufio" + "bytes" "encoding/hex" "encoding/json" "fmt" @@ -191,6 +192,9 @@ func TestCompactBlocks(t *testing.T) { t.Errorf("incorrect block prevhash in testnet block %x", test.BlockHash) continue } + if !bytes.Equal(block.GetPrevHash(), block.hdr.HashPrevBlock) { + t.Error("block and block header prevhash don't match") + } compact := block.ToCompact() marshaled, err := protobuf.Marshal(compact) diff --git a/parser/transaction.go b/parser/transaction.go index ce19d2b..0a9eb26 100644 --- a/parser/transaction.go +++ b/parser/transaction.go @@ -4,8 +4,8 @@ import ( "crypto/sha256" "github.com/pkg/errors" - "github.com/zcash-hackworks/lightwalletd/parser/internal/bytestring" - "github.com/zcash-hackworks/lightwalletd/walletrpc" + "github.com/zcash/lightwalletd/parser/internal/bytestring" + "github.com/zcash/lightwalletd/walletrpc" ) type rawTransaction struct { diff --git a/parser/transaction_test.go b/parser/transaction_test.go index 794083c..202590f 100644 --- a/parser/transaction_test.go +++ b/parser/transaction_test.go @@ -9,7 +9,7 @@ import ( "strings" "testing" - "github.com/zcash-hackworks/lightwalletd/parser/internal/bytestring" + "github.com/zcash/lightwalletd/parser/internal/bytestring" ) // "Human-readable" version of joinSplit struct defined in transaction.go. @@ -161,9 +161,8 @@ func TestSproutTransactionParser(t *testing.T) { defer testData.Close() // Parse the raw transactions file - rawTxData := make([][]byte, len(zip143tests)) + rawTxData := [][]byte{} scan := bufio.NewScanner(testData) - count := 0 for scan.Scan() { dataLine := scan.Text() // Skip the comments @@ -175,9 +174,7 @@ func TestSproutTransactionParser(t *testing.T) { if err != nil { t.Fatal(err) } - - rawTxData[count] = txData - count++ + rawTxData = append(rawTxData, txData) } for i, tt := range zip143tests { @@ -672,9 +669,8 @@ func TestSaplingTransactionParser(t *testing.T) { defer testData.Close() // Parse the raw transactions file - rawTxData := make([][]byte, len(zip243tests)) + rawTxData := [][]byte{} scan := bufio.NewScanner(testData) - count := 0 for scan.Scan() { dataLine := scan.Text() // Skip the comments @@ -686,9 +682,7 @@ func TestSaplingTransactionParser(t *testing.T) { if err != nil { t.Fatal(err) } - - rawTxData[count] = txData - count++ + rawTxData = append(rawTxData, txData) } for i, tt := range zip243tests { @@ -705,6 +699,20 @@ func TestSaplingTransactionParser(t *testing.T) { continue } + // If the transaction is shorter than it should be, parsing + // should fail gracefully + for j := 0; j < len(rawTxData[i]); j++ { + _, err := tx.ParseFromSlice(rawTxData[i][0:j]) + if err == nil { + t.Errorf("Test %d: Parsing transaction unexpected succeeded", i) + break + } + if len(rest) > 0 { + t.Errorf("Test %d: Parsing transaction unexpected rest", i) + break + } + } + // Transaction metadata if !subTestCommonBlockMeta(&tt, tx, t, i) { continue diff --git a/testdata/getsaplinginfo b/testdata/getsaplinginfo new file mode 100644 index 0000000..ac6bc0d --- /dev/null +++ b/testdata/getsaplinginfo @@ -0,0 +1 @@ +7b22636861696e223a226d61696e222c22626c6f636b73223a3637373731332c2268656164657273223a3637373731332c2262657374626c6f636b68617368223a2230303030303030303030306465376137323833343731626238636264363661393530663261333261373666323235306465303364313237366437643036386538222c22646966666963756c7479223a35313436383233372e38303135313331342c22766572696669636174696f6e70726f6772657373223a312c22636861696e776f726b223a2230303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030323432326237363262663437363637222c227072756e6564223a66616c73652c2273697a655f6f6e5f6469736b223a32333938313131363337302c22636f6d6d69746d656e7473223a313434303139322c2276616c7565506f6f6c73223a5b7b226964223a227370726f7574222c226d6f6e69746f726564223a747275652c22636861696e56616c7565223a3134323433342e35393136333732342c22636861696e56616c75655a6174223a31343234333435393136333732347d2c7b226964223a227361706c696e67222c226d6f6e69746f726564223a747275652c22636861696e56616c7565223a3235383330362e32393830353736362c22636861696e56616c75655a6174223a32353833303632393830353736367d5d2c22736f6674666f726b73223a5b7b226964223a226269703334222c2276657273696f6e223a322c22656e666f726365223a7b22737461747573223a747275652c22666f756e64223a343030302c227265717569726564223a3735302c2277696e646f77223a343030307d2c2272656a656374223a7b22737461747573223a747275652c22666f756e64223a343030302c227265717569726564223a3935302c2277696e646f77223a343030307d7d2c7b226964223a226269703636222c2276657273696f6e223a332c22656e666f726365223a7b22737461747573223a747275652c22666f756e64223a343030302c227265717569726564223a3735302c2277696e646f77223a343030307d2c2272656a656374223a7b22737461747573223a747275652c22666f756e64223a343030302c227265717569726564223a3935302c2277696e646f77223a343030307d7d2c7b226964223a226269703635222c2276657273696f6e223a342c22656e666f726365223a7b22737461747573223a747275652c22666f756e64223a343030302c227265717569726564223a3735302c2277696e646f77223a343030307d2c2272656a656374223a7b22737461747573223a747275652c22666f756e64223a343030302c227265717569726564223a3935302c2277696e646f77223a343030307d7d5d2c227570677261646573223a7b223562613831623139223a7b226e616d65223a224f76657277696e746572222c2261637469766174696f6e686569676874223a3334373530302c22737461747573223a22616374697665222c22696e666f223a225365652068747470733a2f2f7a2e636173682f757067726164652f6f76657277696e7465722e68746d6c20666f722064657461696c732e227d2c223736623830396262223a7b226e616d65223a225361706c696e67222c2261637469766174696f6e686569676874223a3431393230302c22737461747573223a22616374697665222c22696e666f223a225365652068747470733a2f2f7a2e636173682f757067726164652f7361706c696e672e68746d6c20666f722064657461696c732e227d2c223262623430653630223a7b226e616d65223a22426c6f73736f6d222c2261637469766174696f6e686569676874223a3635333630302c22737461747573223a22616374697665222c22696e666f223a225365652068747470733a2f2f7a2e636173682f757067726164652f626c6f73736f6d2e68746d6c20666f722064657461696c732e227d7d2c22636f6e73656e737573223a7b22636861696e746970223a223262623430653630222c226e657874626c6f636b223a223262623430653630227d7d