zecfoundation
/
lightwalletd
mirror of https://github.com/ZcashFoundation/lightwalletd.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Example usage of cobra and viper for configuration 

Remove extra configs
Move server startup to rootcmd
Fixed broken insecure flags and example config name
This commit is contained in:
Ben Wilson 2020-01-02 09:28:30 -05:00 committed by Larry Ruane
parent 3ffef5c18e
commit dffb18cf0f
8 changed files with 362 additions and 336 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

271
cmd/root.go Normal file
View File

@ -0,0 +1,271 @@
package cmd
import (
"fmt"
"net"
"os"
"os/signal"
"strings"
"syscall"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/reflection"
"github.com/zcash-hackworks/lightwalletd/common"
"github.com/zcash-hackworks/lightwalletd/common/logging"
"github.com/zcash-hackworks/lightwalletd/frontend"
"github.com/zcash-hackworks/lightwalletd/walletrpc"
)
var cfgFile string
var log *logrus.Entry
var logger = logrus.New()
// rootCmd represents the base command when called without any subcommands
var rootCmd = &cobra.Command{
Use: "lightwalletd",
Short: "Lightwalletd is a backend service to the Zcash blockchain",
Long: `Lightwalletd is a backend service that provides a
bandwidth-efficient interface to the Zcash blockchain`,
Run: func(cmd *cobra.Command, args []string) {
opts := &common.Options{
BindAddr: viper.GetString("bind-addr"),
TLSCertPath: viper.GetString("tls-cert"),
TLSKeyPath: viper.GetString("tls-key"),
LogLevel: viper.GetUint64("log-level"),
LogFile: viper.GetString("log-file"),
ZcashConfPath: viper.GetString("zcash-conf-path"),
NoTLSVeryInsecure: viper.GetBool("no-tls-very-insecure"),
CacheSize: viper.GetInt("cache-size"),
}
fmt.Printf("Options: %#v", opts)
filesThatShouldExist := []string{
opts.TLSCertPath,
opts.TLSKeyPath,
opts.LogFile,
opts.ZcashConfPath,
}
for _, filename := range filesThatShouldExist {
if !fileExists(opts.LogFile) {
os.OpenFile(opts.LogFile, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666)
}
if opts.NoTLSVeryInsecure && (filename == opts.TLSCertPath || filename == opts.TLSKeyPath) {
continue
}
if !fileExists(filename) {
os.Stderr.WriteString(fmt.Sprintf("\n ** File does not exist: %s\n\n", filename))
os.Exit(1)
}
}
// Start server and block, or exit
if err := startServer(opts); err != nil {
log.WithFields(logrus.Fields{
"error": err,
}).Fatal("couldn't create server")
}
},
}
func fileExists(filename string) bool {
info, err := os.Stat(filename)
if os.IsNotExist(err) {
return false
}
return !info.IsDir()
}
func startServer(opts *common.Options) error {
if opts.LogFile != "" {
// instead write parsable logs for logstash/splunk/etc
output, err := os.OpenFile(opts.LogFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
log.WithFields(logrus.Fields{
"error": err,
"path": opts.LogFile,
}).Fatal("couldn't open log file")
}
defer output.Close()
logger.SetOutput(output)
logger.SetFormatter(&logrus.JSONFormatter{})
}
logger.SetLevel(logrus.Level(opts.LogLevel))
// gRPC initialization
var server *grpc.Server
if opts.NoTLSVeryInsecure {
server = grpc.NewServer(logging.LoggingInterceptor())
} else {
transportCreds, err := credentials.NewServerTLSFromFile(opts.TLSCertPath, opts.TLSKeyPath)
if err != nil {
log.WithFields(logrus.Fields{
"cert_file": opts.TLSCertPath,
"key_path": opts.TLSKeyPath,
"error": err,
}).Fatal("couldn't load TLS credentials")
}
server = grpc.NewServer(grpc.Creds(transportCreds), logging.LoggingInterceptor())
}
// Enable reflection for debugging
if opts.LogLevel >= uint64(logrus.WarnLevel) {
reflection.Register(server)
}
// Initialize Zcash RPC client. Right now (Jan 2018) this is only for
// sending transactions, but in the future it could back a different type
// of block streamer.
rpcClient, err := frontend.NewZRPCFromConf(opts.ZcashConfPath)
if err != nil {
log.WithFields(logrus.Fields{
"error": err,
}).Fatal("setting up RPC connection to zcashd")
}
// Get the sapling activation height from the RPC
// (this first RPC also verifies that we can communicate with zcashd)
saplingHeight, blockHeight, chainName, branchID := common.GetSaplingInfo(rpcClient, log)
log.Info("Got sapling height ", saplingHeight, " chain ", chainName, " branchID ", branchID)
// Initialize the cache
cache := common.NewBlockCache(opts.CacheSize)
// Start the block cache importer at cacheSize blocks before current height
cacheStart := blockHeight - opts.CacheSize
if cacheStart < saplingHeight {
cacheStart = saplingHeight
}
go common.BlockIngestor(rpcClient, cache, log, cacheStart)
// Compact transaction service initialization
service, err := frontend.NewLwdStreamer(rpcClient, cache, log)
if err != nil {
log.WithFields(logrus.Fields{
"error": err,
}).Fatal("couldn't create backend")
}
// Register service
walletrpc.RegisterCompactTxStreamerServer(server, service)
// Start listening
listener, err := net.Listen("tcp", opts.BindAddr)
if err != nil {
log.WithFields(logrus.Fields{
"bind_addr": opts.BindAddr,
"error": err,
}).Fatal("couldn't create listener")
}
// Signal handler for graceful stops
signals := make(chan os.Signal, 1)
signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
go func() {
s := <-signals
log.WithFields(logrus.Fields{
"signal": s.String(),
}).Info("caught signal, stopping gRPC server")
os.Exit(1)
}()
log.Infof("Starting gRPC server on %s", opts.BindAddr)
err = server.Serve(listener)
if err != nil {
log.WithFields(logrus.Fields{
"error": err,
}).Fatal("gRPC server exited")
}
return nil
}
// Execute adds all child commands to the root command and sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
if err := rootCmd.Execute(); err != nil {
fmt.Println(err)
os.Exit(1)
}
}
func init() {
rootCmd.AddCommand(versionCmd)
cobra.OnInitialize(initConfig)
rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is current directory, lightwalletd.yaml)")
rootCmd.Flags().String("bind-addr", "127.0.0.1:9067", "the address to listen on")
rootCmd.Flags().String("tls-cert", "./cert.pem", "the path to a TLS certificate")
rootCmd.Flags().String("tls-key", "./cert.key", "the path to a TLS key file")
rootCmd.Flags().Int("log-level", int(logrus.InfoLevel), "log level (logrus 1-7)")
rootCmd.Flags().String("log-file", "./server.log", "log file to write to")
rootCmd.Flags().String("zcash-conf-path", "./zcash.conf", "conf file to pull RPC creds from")
rootCmd.Flags().Bool("no-tls-very-insecure", false, "run without the required TLS certificate, only for debugging, DO NOT use in production")
rootCmd.Flags().Int("cache-size", 80000, "number of blocks to hold in the cache")
viper.BindPFlag("bind-addr", rootCmd.Flags().Lookup("bind-addr"))
viper.SetDefault("bind-addr", "127.0.0.1:9067")
viper.BindPFlag("tls-cert", rootCmd.Flags().Lookup("tls-cert"))
viper.SetDefault("tls-cert", "./cert.pem")
viper.BindPFlag("tls-key", rootCmd.Flags().Lookup("tls-key"))
viper.SetDefault("tls-key", "./cert.key")
viper.BindPFlag("log-level", rootCmd.Flags().Lookup("log-level"))
viper.SetDefault("log-level", int(logrus.InfoLevel))
viper.BindPFlag("log-file", rootCmd.Flags().Lookup("log-file"))
viper.SetDefault("log-file", "./server.log")
viper.BindPFlag("zcash-conf-path", rootCmd.Flags().Lookup("zcash-conf-path"))
viper.SetDefault("zcash-conf-path", "./zcash.conf")
viper.BindPFlag("no-tls-very-insecure", rootCmd.Flags().Lookup("no-tls-very-insecure"))
viper.SetDefault("no-tls-very-insecure", false)
viper.BindPFlag("cache-size", rootCmd.Flags().Lookup("cache-size"))
viper.SetDefault("cache-size", 80000)
logger.SetFormatter(&logrus.TextFormatter{
//DisableColors: true,
FullTimestamp: true,
DisableLevelTruncation: true,
})
onexit := func() {
fmt.Printf("Lightwalletd died with a Fatal error. Check logfile for details.\n")
}
log = logger.WithFields(logrus.Fields{
"app": "lightwalletd",
})
logrus.RegisterExitHandler(onexit)
}
// initConfig reads in config file and ENV variables if set.
func initConfig() {
if cfgFile != "" {
// Use config file from the flag.
viper.SetConfigFile(cfgFile)
} else {
// Look in the current directory for a configuration file
viper.AddConfigPath(".")
// Viper auto appends extention to this config name
// For example, lightwalletd.yml
viper.SetConfigName("lightwalletd")
}
// Replace `-` in config options with `_` for ENV keys
replacer := strings.NewReplacer("-", "_")
viper.SetEnvKeyReplacer(replacer)
viper.AutomaticEnv() // read in environment variables that match
// If a config file is found, read it in.
var err error
if err = viper.ReadInConfig(); err == nil {
fmt.Println("Using config file:", viper.ConfigFileUsed())
}
}

261
cmd/server/main.go
View File

@ -1,261 +0,0 @@
// Copyright (c) 2019-2020 The Zcash developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or https://www.opensource.org/licenses/mit-license.php .
package main
import (
"context"
"flag"
"fmt"
"net"
"os"
"os/signal"
"syscall"
"time"
"github.com/sirupsen/logrus"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/peer"
"google.golang.org/grpc/reflection"
"github.com/zcash/lightwalletd/common"
"github.com/zcash/lightwalletd/frontend"
"github.com/zcash/lightwalletd/walletrpc"
)
var logger = logrus.New()
func init() {
logger.SetFormatter(&logrus.TextFormatter{
//DisableColors: true,
FullTimestamp: true,
DisableLevelTruncation: true,
})
onexit := func() {
fmt.Println("Lightwalletd died with a Fatal error. Check logfile for details.")
}
common.Log = logger.WithFields(logrus.Fields{
"app": "frontend-grpc",
})
logrus.RegisterExitHandler(onexit)
}
// TODO stream logging
func LoggingInterceptor() grpc.ServerOption {
return grpc.UnaryInterceptor(logInterceptor)
}
func logInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
reqLog := loggerFromContext(ctx)
start := time.Now()
resp, err := handler(ctx, req)
entry := reqLog.WithFields(logrus.Fields{
"method": info.FullMethod,
"duration": time.Since(start),
"error": err,
})
if err != nil {
entry.Error("call failed")
} else {
entry.Info("method called")
}
return resp, err
}
func loggerFromContext(ctx context.Context) *logrus.Entry {
// TODO: anonymize the addresses. cryptopan?
if peerInfo, ok := peer.FromContext(ctx); ok {
return common.Log.WithFields(logrus.Fields{"peer_addr": peerInfo.Addr})
}
return common.Log.WithFields(logrus.Fields{"peer_addr": "unknown"})
}
type Options struct {
bindAddr string `json:"bind_address,omitempty"`
tlsCertPath string `json:"tls_cert_path,omitempty"`
tlsKeyPath string `json:"tls_cert_key,omitempty"`
logLevel uint64 `json:"log_level,omitempty"`
logPath string `json:"log_file,omitempty"`
zcashConfPath string `json:"zcash_conf,omitempty"`
veryInsecure bool `json:"very_insecure,omitempty"`
cacheSize int `json:"cache_size,omitempty"`
wantVersion bool
}
func fileExists(filename string) bool {
info, err := os.Stat(filename)
if os.IsNotExist(err) {
return false
}
return !info.IsDir()
}
func main() {
opts := &Options{}
flag.StringVar(&opts.bindAddr, "bind-addr", "127.0.0.1:9067", "the address to listen on")
flag.StringVar(&opts.tlsCertPath, "tls-cert", "", "the path to a TLS certificate")
flag.StringVar(&opts.tlsKeyPath, "tls-key", "", "the path to a TLS key file")
flag.Uint64Var(&opts.logLevel, "log-level", uint64(logrus.InfoLevel), "log level (logrus 1-7)")
flag.StringVar(&opts.logPath, "log-file", "./server.log", "log file to write to")
flag.StringVar(&opts.zcashConfPath, "conf-file", "./zcash.conf", "conf file to pull RPC creds from")
flag.BoolVar(&opts.veryInsecure, "no-tls-very-insecure", false, "run without the required TLS certificate, only for debugging, DO NOT use in production")
flag.BoolVar(&opts.wantVersion, "version", false, "version (major.minor.patch)")
flag.IntVar(&opts.cacheSize, "cache-size", 80000, "number of blocks to hold in the cache")
// TODO prod metrics
// TODO support config from file and env vars
flag.Parse()
if opts.wantVersion {
fmt.Println("lightwalletd version v0.3.0")
return
}
// production (unlike unit tests) use the real sleep function
common.Sleep = time.Sleep
if opts.logPath != "" {
// instead write parsable logs for logstash/splunk/etc
output, err := os.OpenFile(opts.logPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
os.Stderr.WriteString(fmt.Sprintf("Cannot open log file %s: %v\n",
opts.logPath, err))
os.Exit(1)
}
defer output.Close()
logger.SetOutput(output)
logger.SetFormatter(&logrus.JSONFormatter{})
}
logger.SetLevel(logrus.Level(opts.logLevel))
filesThatShouldExist := []string{
opts.zcashConfPath,
}
if opts.tlsCertPath != "" {
filesThatShouldExist = append(filesThatShouldExist, opts.tlsCertPath)
}
if opts.tlsKeyPath != "" {
filesThatShouldExist = append(filesThatShouldExist, opts.tlsKeyPath)
}
for _, filename := range filesThatShouldExist {
if !fileExists(filename) {
common.Log.WithFields(logrus.Fields{
"filename": filename,
}).Error("cannot open required file")
os.Stderr.WriteString("Cannot open required file: " + filename + "\n")
os.Exit(1)
}
}
// gRPC initialization
var server *grpc.Server
var err error
if opts.veryInsecure {
server = grpc.NewServer(LoggingInterceptor())
} else {
var transportCreds credentials.TransportCredentials
if (opts.tlsCertPath == "") && (opts.tlsKeyPath == "") {
common.Log.Warning("Certificate and key not provided, generating self signed values")
tlsCert := common.GenerateCerts()
transportCreds = credentials.NewServerTLSFromCert(tlsCert)
} else {
transportCreds, err = credentials.NewServerTLSFromFile(opts.tlsCertPath, opts.tlsKeyPath)
if err != nil {
common.Log.WithFields(logrus.Fields{
"cert_file": opts.tlsCertPath,
"key_path": opts.tlsKeyPath,
"error": err,
}).Fatal("couldn't load TLS credentials")
}
}
server = grpc.NewServer(grpc.Creds(transportCreds), LoggingInterceptor())
}
// Enable reflection for debugging
if opts.logLevel >= uint64(logrus.WarnLevel) {
reflection.Register(server)
}
// Initialize Zcash RPC client. Right now (Jan 2018) this is only for
// sending transactions, but in the future it could back a different type
// of block streamer.
rpcClient, err := frontend.NewZRPCFromConf(opts.zcashConfPath)
if err != nil {
common.Log.WithFields(logrus.Fields{
"error": err,
}).Fatal("setting up RPC connection to zcashd")
}
// indirect function for test mocking (so unit tests can talk to stub functions)
common.RawRequest = rpcClient.RawRequest
// Get the sapling activation height from the RPC
// (this first RPC also verifies that we can communicate with zcashd)
saplingHeight, blockHeight, chainName, branchID := common.GetSaplingInfo()
common.Log.Info("Got sapling height ", saplingHeight, " chain ", chainName, " branchID ", branchID)
// Initialize the cache
cache := common.NewBlockCache(opts.cacheSize)
// Start the block cache importer at cacheSize blocks before current height
cacheStart := blockHeight - opts.cacheSize
if cacheStart < saplingHeight {
cacheStart = saplingHeight
}
// The last argument, repetition count, is only nonzero for testing
go common.BlockIngestor(cache, cacheStart, 0)
// Compact transaction service initialization
service, err := frontend.NewLwdStreamer(cache)
if err != nil {
common.Log.WithFields(logrus.Fields{
"error": err,
}).Fatal("couldn't create backend")
}
// Register service
walletrpc.RegisterCompactTxStreamerServer(server, service)
// Start listening
listener, err := net.Listen("tcp", opts.bindAddr)
if err != nil {
common.Log.WithFields(logrus.Fields{
"bind_addr": opts.bindAddr,
"error": err,
}).Fatal("couldn't create listener")
}
// Signal handler for graceful stops
signals := make(chan os.Signal, 1)
signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
go func() {
s := <-signals
common.Log.WithFields(logrus.Fields{
"signal": s.String(),
}).Info("caught signal, stopping gRPC server")
os.Stderr.WriteString("Caught signal: " + s.String() + "\n")
os.Exit(1)
}()
common.Log.Infof("Starting gRPC server on %s", opts.bindAddr)
err = server.Serve(listener)
if err != nil {
common.Log.WithFields(logrus.Fields{
"error": err,
}).Fatal("gRPC server exited")
}
}

75
cmd/server/server_test.go
View File

@ -1,75 +0,0 @@
// Copyright (c) 2019-2020 The Zcash developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or https://www.opensource.org/licenses/mit-license.php .
package main
import (
"context"
"fmt"
"os"
"testing"
"errors"
"github.com/sirupsen/logrus"
"github.com/zcash/lightwalletd/common"
"google.golang.org/grpc"
"google.golang.org/grpc/peer"
)
var step int
func testhandler(ctx context.Context, req interface{}) (interface{}, error) {
step++
switch step {
case 1:
return nil, errors.New("test error")
case 2:
return nil, nil
}
return nil, nil
}
func TestLogInterceptor(t *testing.T) {
output, err := os.OpenFile("test-log", os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
if err != nil {
os.Stderr.WriteString(fmt.Sprint("Cannot open test-log:", err))
os.Exit(1)
}
logger := logrus.New()
logger.SetOutput(output)
common.Log = logger.WithFields(logrus.Fields{
"app": "test",
})
var req interface{}
resp, err := logInterceptor(peer.NewContext(context.Background(), &peer.Peer{}),
&req, &grpc.UnaryServerInfo{}, testhandler)
if err == nil {
t.Fatal("unexpected success")
}
if resp != nil {
t.Fatal("unexpected response", resp)
}
resp, err = logInterceptor(context.Background(), &req, &grpc.UnaryServerInfo{}, testhandler)
if err != nil {
t.Fatal("unexpected error", err)
}
if resp != nil {
t.Fatal("unexpected response", resp)
}
os.Remove("test-log")
step = 0
}
func TestFileExists(t *testing.T) {
if fileExists("nonexistent-file") {
t.Fatal("fileExists unexpected success")
}
// If the path exists but is a directory, should return false
if fileExists(".") {
t.Fatal("fileExists unexpected success")
}
// The following file should exist, it's what's being tested
if !fileExists("main.go") {
t.Fatal("fileExists failed")
}
}

17
cmd/version.go Normal file
View File

@ -0,0 +1,17 @@
package cmd
import (
"fmt"
"github.com/spf13/cobra"
)
// versionCmd represents the version command
var versionCmd = &cobra.Command{
Use: "version",
Short: "Dispaly lightwalletd version",
Long: `Dispaly lightwalletd version.`,
Run: func(cmd *cobra.Command, args []string) {
fmt.Println("lightwalletd version v0.2.0")
},
}

11
common/common.go
View File

@ -16,6 +16,17 @@ import (
"github.com/zcash/lightwalletd/walletrpc"
)
type Options struct {
BindAddr string `json:"bind_address,omitempty"`
TLSCertPath string `json:"tls_cert_path,omitempty"`
TLSKeyPath string `json:"tls_cert_key,omitempty"`
LogLevel uint64 `json:"log_level,omitempty"`
LogFile string `json:"log_file,omitempty"`
ZcashConfPath string `json:"zcash_conf,omitempty"`
NoTLSVeryInsecure bool `json:"no_tls_very_insecure,omitempty"`
CacheSize int `json:"cache_size,omitempty"`
}
// RawRequest points to the function to send a an RPC request to zcashd;
// in production, it points to btcsuite/btcd/rpcclient/rawrequest.go:RawRequest();
// in unit tests it points to a function to mock RPCs to zcashd.

49
common/logging/logging.go Normal file
View File

@ -0,0 +1,49 @@
package logging
import (
"context"
"time"
"github.com/sirupsen/logrus"
log "github.com/sirupsen/logrus"
"google.golang.org/grpc"
"google.golang.org/grpc/peer"
)
func LoggingInterceptor() grpc.ServerOption {
return grpc.UnaryInterceptor(logInterceptor)
}
func loggerFromContext(ctx context.Context) *logrus.Entry {
// TODO: anonymize the addresses. cryptopan?
if peerInfo, ok := peer.FromContext(ctx); ok {
return log.WithFields(logrus.Fields{"peer_addr": peerInfo.Addr})
}
return log.WithFields(logrus.Fields{"peer_addr": "unknown"})
}
func logInterceptor(
ctx context.Context,
req interface{},
info *grpc.UnaryServerInfo,
handler grpc.UnaryHandler,
) (interface{}, error) {
reqLog := loggerFromContext(ctx)
start := time.Now()
resp, err := handler(ctx, req)
entry := reqLog.WithFields(logrus.Fields{
"method": info.FullMethod,
"duration": time.Since(start),
"error": err,
})
if err != nil {
entry.Error("call failed")
} else {
entry.Info("method called")
}
return resp, err
}

7
lightwalletd-example.yml Normal file
View File

@ -0,0 +1,7 @@
bind-addr: 0.0.0.0:9067
cache-size: 10
log-file: /dev/stdout
log-level: 10
tls-cert: /secrets/lightwallted/cert.pem
tls-key: /secrets/lightwallted/cert.key
zcash-conf-path: /srv/zcashd/zcash.conf

7
main.go Normal file
View File

@ -0,0 +1,7 @@
package main
import "github.com/zcash-hackworks/lightwalletd/cmd"
func main() {
cmd.Execute()
}