Merge pull request #211 from tendermint/feature/reimplement-merkleeyes

Reimplement merkleeyes

Makefile

@@ -32,6 +32,7 @@ test_cli: tests/cli/shunit2
 	./tests/cli/rpc.sh
 	./tests/cli/init.sh
 	./tests/cli/basictx.sh
+	./tests/cli/eyes.sh
 	./tests/cli/roles.sh
 	./tests/cli/counter.sh
 	./tests/cli/restart.sh

cmd/basecli/main.go

@@ -27,13 +27,12 @@ import (
 // BaseCli - main basecoin client command
 var BaseCli = &cobra.Command{
 	Use:   "basecli",
-	Short: "Light client for tendermint",
+	Short: "Light client for Tendermint",
-	Long: `Basecli is an version of tmcli including custom logic to
+	Long: `Basecli is a certifying light client for the basecoin abci app.
-present a nice (not raw hex) interface to the basecoin blockchain structure.
 
-This is a useful tool, but also serves to demonstrate how one can configure
+It leverages the power of the tendermint consensus algorithm get full
-tmcli to work for any custom abci app.
+cryptographic proof of all queries while only syncing a fraction of the
-`,
+block headers.`,
 }
 
 func main() {

cmd/eyes/init.go

@@ -0,0 +1,58 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+
+	tcmd "github.com/tendermint/tendermint/cmd/tendermint/commands"
+
+	"github.com/tendermint/basecoin/cmd/basecoin/commands"
+)
+
+// InitCmd - node initialization command
+var InitCmd = &cobra.Command{
+	Use:   "init",
+	Short: "Initialize eyes abci server",
+	RunE:  initCmd,
+}
+
+//nolint - flags
+var (
+	FlagChainID = "chain-id" //TODO group with other flags or remove? is this already a flag here?
+)
+
+func init() {
+	InitCmd.Flags().String(FlagChainID, "eyes_test_id", "Chain ID")
+}
+
+func initCmd(cmd *cobra.Command, args []string) error {
+	// this will ensure that config.toml is there if not yet created, and create dir
+	cfg, err := tcmd.ParseConfig()
+	if err != nil {
+		return err
+	}
+
+	genesis := getGenesisJSON(viper.GetString(commands.FlagChainID))
+	return commands.CreateGenesisValidatorFiles(cfg, genesis, cmd.Root().Name())
+}
+
+// TODO: better, auto-generate validator...
+func getGenesisJSON(chainID string) string {
+	return fmt.Sprintf(`{
+  "app_hash": "",
+  "chain_id": "%s",
+  "genesis_time": "0001-01-01T00:00:00.000Z",
+  "validators": [
+    {
+      "amount": 10,
+      "name": "",
+      "pub_key": {
+        "type": "ed25519",
+        "data": "7B90EA87E7DC0C7145C8C48C08992BE271C7234134343E8A8E8008E617DE7B30"
+      }
+    }
+  ]
+}`, chainID)
+}

cmd/eyes/main.go

@@ -0,0 +1,44 @@
+package main
+
+import (
+	"os"
+
+	"github.com/tendermint/tmlibs/cli"
+
+	"github.com/tendermint/basecoin"
+	"github.com/tendermint/basecoin/cmd/basecoin/commands"
+	"github.com/tendermint/basecoin/modules/base"
+	"github.com/tendermint/basecoin/modules/eyes"
+	"github.com/tendermint/basecoin/stack"
+)
+
+// BuildApp constructs the stack we want to use for this app
+func BuildApp() basecoin.Handler {
+	return stack.New(
+		base.Logger{},
+		stack.Recovery{},
+	).
+		// We do this to demo real usage, also embeds it under it's own namespace
+		Dispatch(
+			stack.WrapHandler(etc.NewHandler()),
+		)
+}
+
+func main() {
+	rt := commands.RootCmd
+	rt.Short = "eyes"
+	rt.Long = "A demo app to show key-value store with proofs over abci"
+
+	commands.Handler = BuildApp()
+
+	rt.AddCommand(
+		// out own init command to not require argument
+		InitCmd,
+		commands.StartCmd,
+		commands.UnsafeResetAllCmd,
+		commands.VersionCmd,
+	)
+
+	cmd := cli.PrepareMainCmd(rt, "EYE", os.ExpandEnv("$HOME/.eyes"))
+	cmd.Execute()
+}

cmd/eyescli/main.go

@@ -0,0 +1,67 @@
+package main
+
+import (
+	"os"
+
+	"github.com/spf13/cobra"
+
+	keycmd "github.com/tendermint/go-crypto/cmd"
+	"github.com/tendermint/tmlibs/cli"
+
+	"github.com/tendermint/basecoin/client/commands"
+	"github.com/tendermint/basecoin/client/commands/auto"
+	"github.com/tendermint/basecoin/client/commands/proxy"
+	"github.com/tendermint/basecoin/client/commands/query"
+	rpccmd "github.com/tendermint/basecoin/client/commands/rpc"
+	"github.com/tendermint/basecoin/client/commands/seeds"
+	txcmd "github.com/tendermint/basecoin/client/commands/txs"
+	etccmd "github.com/tendermint/basecoin/modules/eyes/commands"
+)
+
+// EyesCli - main basecoin client command
+var EyesCli = &cobra.Command{
+	Use:   "eyescli",
+	Short: "Light client for Tendermint",
+	Long:  `EyesCli is the light client for a merkle key-value store (eyes)`,
+}
+
+func main() {
+	commands.AddBasicFlags(EyesCli)
+
+	// Prepare queries
+	query.RootCmd.AddCommand(
+		// These are default parsers, but optional in your app (you can remove key)
+		query.TxQueryCmd,
+		query.KeyQueryCmd,
+		// this is our custom parser
+		etccmd.EtcQueryCmd,
+	)
+
+	// no middleware wrapers
+	txcmd.Middleware = txcmd.Wrappers{}
+	// txcmd.Middleware.Register(txcmd.RootCmd.PersistentFlags())
+
+	// just the etc commands
+	txcmd.RootCmd.AddCommand(
+		etccmd.SetTxCmd,
+		etccmd.RemoveTxCmd,
+	)
+
+	// Set up the various commands to use
+	EyesCli.AddCommand(
+		// we use out own init command to not require address arg
+		commands.InitCmd,
+		commands.ResetCmd,
+		keycmd.RootCmd,
+		seeds.RootCmd,
+		rpccmd.RootCmd,
+		query.RootCmd,
+		txcmd.RootCmd,
+		proxy.RootCmd,
+		commands.VersionCmd,
+		auto.AutoCompleteCmd,
+	)
+
+	cmd := cli.PrepareMainCmd(EyesCli, "EYE", os.ExpandEnv("$HOME/.eyescli"))
+	cmd.Execute()
+}

modules/eyes/commands/query.go

@@ -0,0 +1,44 @@
+package commands
+
+import (
+	"encoding/hex"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+
+	cmn "github.com/tendermint/tmlibs/common"
+
+	"github.com/tendermint/basecoin/client/commands"
+	"github.com/tendermint/basecoin/client/commands/query"
+	"github.com/tendermint/basecoin/modules/eyes"
+	"github.com/tendermint/basecoin/stack"
+)
+
+// EtcQueryCmd - command to query raw data
+var EtcQueryCmd = &cobra.Command{
+	Use:   "etc [key]",
+	Short: "Get data stored under key in etc",
+	RunE:  commands.RequireInit(etcQueryCmd),
+}
+
+func etcQueryCmd(cmd *cobra.Command, args []string) error {
+	var res etc.Data
+
+	arg, err := commands.GetOneArg(args, "key")
+	if err != nil {
+		return err
+	}
+	key, err := hex.DecodeString(cmn.StripHex(arg))
+	if err != nil {
+		return err
+	}
+
+	key = stack.PrefixedKey(etc.Name, key)
+	prove := !viper.GetBool(commands.FlagTrustNode)
+	height, err := query.GetParsed(key, &res, prove)
+	if err != nil {
+		return err
+	}
+
+	return query.OutputProof(res, height)
+}

modules/eyes/commands/tx.go

@@ -0,0 +1,63 @@
+package commands
+
+import (
+	"github.com/spf13/cobra"
+
+	"github.com/tendermint/basecoin/client/commands"
+	"github.com/tendermint/basecoin/client/commands/txs"
+	"github.com/tendermint/basecoin/modules/eyes"
+)
+
+// SetTxCmd is CLI command to set data
+var SetTxCmd = &cobra.Command{
+	Use:   "set",
+	Short: "Sets a key value pair",
+	RunE:  commands.RequireInit(setTxCmd),
+}
+
+// RemoveTxCmd is CLI command to remove data
+var RemoveTxCmd = &cobra.Command{
+	Use:   "remove",
+	Short: "Removes a key value pair",
+	RunE:  commands.RequireInit(removeTxCmd),
+}
+
+const (
+	// FlagKey is the cli flag to set the key
+	FlagKey = "key"
+	// FlagValue is the cli flag to set the value
+	FlagValue = "value"
+)
+
+func init() {
+	SetTxCmd.Flags().String(FlagKey, "", "Key to store data under (hex)")
+	SetTxCmd.Flags().String(FlagValue, "", "Data to store (hex)")
+
+	RemoveTxCmd.Flags().String(FlagKey, "", "Key under which to remove data (hex)")
+}
+
+// setTxCmd creates a SetTx, wraps, signs, and delivers it
+func setTxCmd(cmd *cobra.Command, args []string) error {
+	key, err := commands.ParseHexFlag(FlagKey)
+	if err != nil {
+		return err
+	}
+	value, err := commands.ParseHexFlag(FlagValue)
+	if err != nil {
+		return err
+	}
+
+	tx := etc.NewSetTx(key, value)
+	return txs.DoTx(tx)
+}
+
+// removeTxCmd creates a RemoveTx, wraps, signs, and delivers it
+func removeTxCmd(cmd *cobra.Command, args []string) error {
+	key, err := commands.ParseHexFlag(FlagKey)
+	if err != nil {
+		return err
+	}
+
+	tx := etc.NewRemoveTx(key)
+	return txs.DoTx(tx)
+}

modules/eyes/errors.go

@@ -0,0 +1,23 @@
+package etc
+
+import (
+	"fmt"
+
+	abci "github.com/tendermint/abci/types"
+
+	"github.com/tendermint/basecoin/errors"
+)
+
+var (
+	errMissingData = fmt.Errorf("All tx fields must be filled")
+
+	malformed = abci.CodeType_EncodingError
+)
+
+//nolint
+func ErrMissingData() errors.TMError {
+	return errors.WithCode(errMissingData, malformed)
+}
+func IsMissingDataErr(err error) bool {
+	return errors.IsSameError(errMissingData, err)
+}

modules/eyes/handler.go

@@ -0,0 +1,93 @@
+package etc
+
+import (
+	"github.com/tendermint/basecoin"
+	"github.com/tendermint/basecoin/errors"
+	"github.com/tendermint/basecoin/state"
+	wire "github.com/tendermint/go-wire"
+)
+
+const (
+	// Name of the module for registering it
+	Name = "etc"
+
+	// CostSet is the gas needed for the set operation
+	CostSet uint64 = 10
+	// CostRemove is the gas needed for the remove operation
+	CostRemove = 10
+)
+
+// Handler allows us to set and remove data
+type Handler struct {
+	basecoin.NopInitState
+	basecoin.NopInitValidate
+}
+
+var _ basecoin.Handler = Handler{}
+
+// NewHandler makes a role handler to modify data
+func NewHandler() Handler {
+	return Handler{}
+}
+
+// Name - return name space
+func (Handler) Name() string {
+	return Name
+}
+
+// CheckTx verifies if the transaction is properly formated
+func (h Handler) CheckTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx) (res basecoin.CheckResult, err error) {
+	err = tx.ValidateBasic()
+	if err != nil {
+		return
+	}
+
+	switch tx.Unwrap().(type) {
+	case SetTx:
+		res = basecoin.NewCheck(CostSet, "")
+	case RemoveTx:
+		res = basecoin.NewCheck(CostRemove, "")
+	default:
+		err = errors.ErrUnknownTxType(tx)
+	}
+	return
+}
+
+// DeliverTx tries to create a new role.
+//
+// Returns an error if the role already exists
+func (h Handler) DeliverTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx) (res basecoin.DeliverResult, err error) {
+	err = tx.ValidateBasic()
+	if err != nil {
+		return
+	}
+
+	switch t := tx.Unwrap().(type) {
+	case SetTx:
+		res, err = h.doSetTx(ctx, store, t)
+	case RemoveTx:
+		res, err = h.doRemoveTx(ctx, store, t)
+	default:
+		err = errors.ErrUnknownTxType(tx)
+	}
+	return
+}
+
+// doSetTx writes to the store, overwriting any previous value
+// note that an empty response in DeliverTx is OK with no log or data returned
+func (h Handler) doSetTx(ctx basecoin.Context, store state.SimpleDB, tx SetTx) (res basecoin.DeliverResult, err error) {
+	data := NewData(tx.Value, ctx.BlockHeight())
+	store.Set(tx.Key, wire.BinaryBytes(data))
+	return
+}
+
+// doRemoveTx deletes the value from the store and returns the last value
+// here we let res.Data to return the value over abci
+func (h Handler) doRemoveTx(ctx basecoin.Context, store state.SimpleDB, tx RemoveTx) (res basecoin.DeliverResult, err error) {
+	// we set res.Data so it gets returned to the client over the abci interface
+	res.Data = store.Get(tx.Key)
+	if len(res.Data) != 0 {
+		store.Remove(tx.Key)
+	}
+	return
+}

modules/eyes/handler_test.go

@@ -0,0 +1,61 @@
+package etc
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/tendermint/basecoin/stack"
+	"github.com/tendermint/basecoin/state"
+	wire "github.com/tendermint/go-wire"
+)
+
+func TestHandler(t *testing.T) {
+	assert, require := assert.New(t), require.New(t)
+
+	key := []byte("one")
+	val := []byte("foo")
+	var height uint64 = 123
+
+	h := NewHandler()
+	ctx := stack.MockContext("role-chain", height)
+	store := state.NewMemKVStore()
+
+	set := SetTx{Key: key, Value: val}.Wrap()
+	remove := RemoveTx{Key: key}.Wrap()
+	invalid := SetTx{}.Wrap()
+
+	// make sure pricing makes sense
+	cres, err := h.CheckTx(ctx, store, set)
+	require.Nil(err, "%+v", err)
+	require.True(cres.GasAllocated > 5, "%#v", cres)
+
+	// set the value, no error
+	dres, err := h.DeliverTx(ctx, store, set)
+	require.Nil(err, "%+v", err)
+
+	// get the data
+	var data Data
+	bs := store.Get(key)
+	require.NotEmpty(bs)
+	err = wire.ReadBinaryBytes(bs, &data)
+	require.Nil(err, "%+v", err)
+	assert.Equal(height, data.SetAt)
+	assert.EqualValues(val, data.Value)
+
+	// make sure pricing makes sense
+	cres, err = h.CheckTx(ctx, store, remove)
+	require.Nil(err, "%+v", err)
+	require.True(cres.GasAllocated > 5, "%#v", cres)
+
+	// remove the data returns the same as the above query
+	dres, err = h.DeliverTx(ctx, store, remove)
+	require.Nil(err, "%+v", err)
+	require.EqualValues(bs, dres.Data)
+
+	// make sure invalid fails both ways
+	_, err = h.CheckTx(ctx, store, invalid)
+	require.NotNil(err)
+	_, err = h.DeliverTx(ctx, store, invalid)
+	require.NotNil(err)
+}

modules/eyes/store.go

@@ -0,0 +1,20 @@
+package etc
+
+import "github.com/tendermint/go-wire/data"
+
+// Data is the struct we use to store in the merkle tree
+type Data struct {
+	// SetAt is the block height this was set at
+	SetAt uint64 `json:"set_at"`
+	// Value is the data that was stored.
+	// data.Bytes is like []byte but json encodes as hex not base64
+	Value data.Bytes `json:"value"`
+}
+
+// NewData creates a new Data item
+func NewData(value []byte, setAt uint64) Data {
+	return Data{
+		SetAt: setAt,
+		Value: value,
+	}
+}

modules/eyes/tx.go

@@ -0,0 +1,66 @@
+package etc
+
+import (
+	"github.com/tendermint/basecoin"
+	"github.com/tendermint/go-wire/data"
+)
+
+// nolint
+const (
+	TypeSet    = Name + "/set"
+	TypeRemove = Name + "/remove"
+
+	ByteSet    = 0xF4
+	ByteRemove = 0xF5
+)
+
+func init() {
+	basecoin.TxMapper.
+		RegisterImplementation(SetTx{}, TypeSet, ByteSet).
+		RegisterImplementation(RemoveTx{}, TypeRemove, ByteRemove)
+}
+
+// SetTx sets a key-value pair
+type SetTx struct {
+	Key   data.Bytes `json:"key"`
+	Value data.Bytes `json:"value"`
+}
+
+func NewSetTx(key, value []byte) basecoin.Tx {
+	return SetTx{Key: key, Value: value}.Wrap()
+}
+
+// Wrap - fulfills TxInner interface
+func (t SetTx) Wrap() basecoin.Tx {
+	return basecoin.Tx{t}
+}
+
+// ValidateBasic makes sure it is valid
+func (t SetTx) ValidateBasic() error {
+	if len(t.Key) == 0 || len(t.Value) == 0 {
+		return ErrMissingData()
+	}
+	return nil
+}
+
+// RemoveTx deletes the value at this key, returns old value
+type RemoveTx struct {
+	Key data.Bytes `json:"key"`
+}
+
+func NewRemoveTx(key []byte) basecoin.Tx {
+	return RemoveTx{Key: key}.Wrap()
+}
+
+// Wrap - fulfills TxInner interface
+func (t RemoveTx) Wrap() basecoin.Tx {
+	return basecoin.Tx{t}
+}
+
+// ValidateBasic makes sure it is valid
+func (t RemoveTx) ValidateBasic() error {
+	if len(t.Key) == 0 {
+		return ErrMissingData()
+	}
+	return nil
+}

tests/cli/eyes.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+
+# These global variables are required for common.sh
+SERVER_EXE=eyes
+CLIENT_EXE=eyescli
+
+oneTimeSetUp() {
+    # These are passed in as args
+    BASE_DIR=$HOME/.test_eyes
+    CHAIN_ID="eyes-cli-test"
+
+    rm -rf $BASE_DIR 2>/dev/null
+    mkdir -p $BASE_DIR
+
+    echo "Setting up genesis..."
+    SERVE_DIR=${BASE_DIR}/server
+    SERVER_LOG=${BASE_DIR}/${SERVER_EXE}.log
+
+    echo "Starting ${SERVER_EXE} server..."
+    export EYE_HOME=${SERVE_DIR}
+    ${SERVER_EXE} init --chain-id=$CHAIN_ID  >>$SERVER_LOG
+    startServer $SERVE_DIR $SERVER_LOG
+    [ $? = 0 ] || return 1
+
+    # Set up client - make sure you use the proper prefix if you set
+    #   a custom CLIENT_EXE
+    export EYE_HOME=${BASE_DIR}/client
+
+    initClient $CHAIN_ID
+    [ $? = 0 ] || return 1
+
+    printf "...Testing may begin!\n\n\n"
+}
+
+oneTimeTearDown() {
+    quickTearDown
+}
+
+test00SetGetRemove() {
+    KEY="CAFE6000"
+    VALUE="F00D4200"
+
+    assertFalse "line=${LINENO} data present" "${CLIENT_EXE} query etc ${KEY}"
+
+    # set data
+    TXRES=$(${CLIENT_EXE} tx set --key=${KEY} --value=${VALUE})
+    txSucceeded $? "$TXRES" "set cafe"
+    HASH=$(echo $TXRES | jq .hash | tr -d \")
+    TX_HEIGHT=$(echo $TXRES | jq .height)
+
+    # make sure it is set
+    DATA=$(${CLIENT_EXE} query etc ${KEY})
+    assertTrue "line=${LINENO} data not set" $?
+    assertEquals "line=${LINENO}" "\"${VALUE}\"" $(echo $DATA | jq .data.value)
+
+    # query the tx
+    TX=$(${CLIENT_EXE} query tx $HASH)
+    assertTrue "line=${LINENO}, found tx" $?
+    if [ -n "$DEBUG" ]; then echo $TX; echo; fi
+
+    assertEquals "line=${LINENO}, proper type" "\"etc/set\"" $(echo $TX | jq .data.type)
+    assertEquals "line=${LINENO}, proper key" "\"${KEY}\"" $(echo $TX | jq .data.data.key)
+    assertEquals "line=${LINENO}, proper value" "\"${VALUE}\"" $(echo $TX | jq .data.data.value)
+}
+
+
+# Load common then run these tests with shunit2!
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" #get this files directory
+. $DIR/common.sh
+. $DIR/shunit2
+