From cf0cb9558aaecbf3ddb071eb863df77e55d828ed Mon Sep 17 00:00:00 2001
From: Jae Kwon <jae@tendermint.com>
Date: Sat, 14 Jan 2017 21:03:12 -0800
Subject: [PATCH 001/132] Clean glide files

---
 glide.lock | 48 +++++++++++++++++++++---------------------------
 glide.yaml | 48 +++++++++++++++++++++++++++++-------------------
 2 files changed, 50 insertions(+), 46 deletions(-)

diff --git a/glide.lock b/glide.lock
index 179dc19a..0277c260 100644
--- a/glide.lock
+++ b/glide.lock
@@ -1,10 +1,12 @@
-hash: dcaf3fb1290b0d7942c86f0644a7431ac313247936eab9515b1ade9ffe579848
-updated: 2017-01-13T00:30:55.237750829-05:00
+hash: e283934fbbd221161d53a918db9e49db8c5be2b8929592b05ffe6b72c2ef0ab1
+updated: 2017-01-14T20:59:48.253671736-08:00
 imports:
 - name: github.com/btcsuite/btcd
-  version: 153dca5c1e4b5d1ea1523592495e5bedfa503391
+  version: afec1bd1245a4a19e6dfe1306974b733e7cbb9b8
   subpackages:
   - btcec
+- name: github.com/btcsuite/fastsha256
+  version: 637e656429416087660c84436a2a035d69d54e2e
 - name: github.com/BurntSushi/toml
   version: 99064174e013895bbd9b025c31100bd1d9b590ca
 - name: github.com/ebuchman/fail-test
@@ -16,23 +18,23 @@ imports:
   subpackages:
   - proto
 - name: github.com/golang/protobuf
-  version: 8ee79997227bf9b34611aee7946ae64735e6fd93
+  version: 1f49d83d9aa00e6ce4fc8258c71cc7786aec968a
   subpackages:
   - proto
 - name: github.com/golang/snappy
   version: d9eb7a3d35ec988b8585d4a0068e462c27d28380
 - name: github.com/gorilla/websocket
-  version: 17634340a83afe0cab595e40fbc63f6ffa1d8915
+  version: 3ab3a8b8831546bd18fd182c20687ca853b2bb13
 - name: github.com/jmhodges/levigo
   version: c42d9e0ca023e2198120196f842701bb4c55d7b9
 - name: github.com/mattn/go-colorable
-  version: d228849504861217f796da67fae4f6e347643f15
+  version: ed8eb9e318d7a84ce5915b495b7d35e0cfe7b5a8
 - name: github.com/mattn/go-isatty
-  version: 30a891c33c7cde7b02a981314b4228ec99380cca
+  version: 66b8e73f3f5cda9f96b69efd03dd3d7fc4a5cdb8
 - name: github.com/spf13/pflag
   version: 25f8b5b07aece3207895bf19f7ab517eb3b22a40
 - name: github.com/syndtr/goleveldb
-  version: 23851d93a2292dcc56e71a18ec9e0624d84a0f65
+  version: 6ae1797c0b42b9323fc27ff7dcf568df88f2f33d
   subpackages:
   - leveldb
   - leveldb/cache
@@ -47,7 +49,7 @@ imports:
   - leveldb/table
   - leveldb/util
 - name: github.com/tendermint/abci
-  version: 699d45bc678865b004b90213bf88a950f420973b
+  version: 6526ab2137fadd0f4d2e25002bbfc1784b4f3c27
   subpackages:
   - client
   - example/counter
@@ -65,7 +67,7 @@ imports:
 - name: github.com/tendermint/go-clist
   version: 3baa390bbaf7634251c42ad69a8682e7e3990552
 - name: github.com/tendermint/go-common
-  version: e289af53b6bf6af28da129d9ef64389a4cf7987f
+  version: 70e694ee76f09058ea38c9ba81b4aa621bd54df1
   subpackages:
   - test
 - name: github.com/tendermint/go-config
@@ -73,9 +75,9 @@ imports:
 - name: github.com/tendermint/go-crypto
   version: 4b11d62bdb324027ea01554e5767b71174680ba0
 - name: github.com/tendermint/go-db
-  version: 72f6dacd22a686cdf7fcd60286503e3aceda77ba
+  version: 2645626c33d8702739e52a61a55d705c2dfe4530
 - name: github.com/tendermint/go-events
-  version: fddee66d90305fccb6f6d84d16c34fa65ea5b7f6
+  version: 2337086736a6adeb2de6f66739b66ecd77535997
 - name: github.com/tendermint/go-flowrate
   version: a20c98e61957faa93b4014fbd902f20ab9317a6a
   subpackages:
@@ -83,13 +85,13 @@ imports:
 - name: github.com/tendermint/go-logger
   version: cefb3a45c0bf3c493a04e9bcd9b1540528be59f2
 - name: github.com/tendermint/go-merkle
-  version: 7a86b4486f2cd84ac885c5bbc609fdee2905f5d1
+  version: 2979c7eb8aa020fa1cf203654907dbb889703888
 - name: github.com/tendermint/go-p2p
-  version: 3d98f675f30dc4796546b8b890f895926152fa8d
+  version: 67c9086b7458eb45b1970483decd01cd744c477a
   subpackages:
   - upnp
 - name: github.com/tendermint/go-rpc
-  version: fcea0cda21f64889be00a0f4b6d13266b1a76ee7
+  version: 6177eb8398ebd4613fbecb71fd96d7c7d97303ec
   subpackages:
   - client
   - server
@@ -97,11 +99,11 @@ imports:
 - name: github.com/tendermint/go-wire
   version: 2f3b7aafe21c80b19b6ee3210ecb3e3d07c7a471
 - name: github.com/tendermint/log15
-  version: ae0f3d6450da9eac7074b439c8e1c3cabf0d5ce6
+  version: 9545b249b3aacafa97f79e0838b02b274adc6f5f
   subpackages:
   - term
 - name: golang.org/x/crypto
-  version: 7c6cc321c680f03b9ef0764448e780704f486b51
+  version: aa2481cbfe81d911eb62b642b7a6b5ec58bbea71
   subpackages:
   - curve25519
   - nacl/box
@@ -112,26 +114,18 @@ imports:
   - ripemd160
   - salsa20/salsa
 - name: golang.org/x/net
-  version: 60c41d1de8da134c05b7b40154a9a82bf5b7edb9
+  version: cfe3c2a7525b50c3d707256e371c90938cfef98a
   subpackages:
   - context
   - http2
   - http2/hpack
-  - idna
   - internal/timeseries
   - lex/httplex
   - trace
 - name: golang.org/x/sys
-  version: d75a52659825e75fff6158388dddc6a5b04f9ba5
+  version: 30de6d19a3bd89a5f38ae4028e23aaa5582648af
   subpackages:
   - unix
-- name: golang.org/x/text
-  version: 44f4f658a783b0cee41fe0a23b8fc91d9c120558
-  subpackages:
-  - secure/bidirule
-  - transform
-  - unicode/bidi
-  - unicode/norm
 - name: google.golang.org/grpc
   version: 50955793b0183f9de69bd78e2ec251cf20aab121
   subpackages:
diff --git a/glide.yaml b/glide.yaml
index bc8c5764..d51903d5 100644
--- a/glide.yaml
+++ b/glide.yaml
@@ -1,29 +1,39 @@
 package: github.com/tendermint/tendermint
 import:
+- package: github.com/tendermint/go-autofile
+  version: develop
+- package: github.com/tendermint/go-clist
+  version: develop
+- package: github.com/tendermint/go-common
+  version: develop
+- package: github.com/tendermint/go-config
+  version: develop
+- package: github.com/tendermint/go-crypto
+  version: develop
+- package: github.com/tendermint/go-db
+  version: develop
+- package: github.com/tendermint/go-events
+  version: develop
+- package: github.com/tendermint/go-logger
+  version: develop
+- package: github.com/tendermint/go-merkle
+  version: develop
+- package: github.com/tendermint/go-p2p
+  version: develop
+- package: github.com/tendermint/go-rpc
+  version: develop
+- package: github.com/tendermint/go-wire
+  version: develop
+- package: github.com/tendermint/abci
+  version: develop
+- package: github.com/tendermint/go-flowrate
+- package: github.com/tendermint/log15
+- package: github.com/tendermint/ed25519
 - package: github.com/gogo/protobuf
   subpackages:
   - proto
 - package: github.com/gorilla/websocket
 - package: github.com/spf13/pflag
-- package: github.com/tendermint/ed25519
-- package: github.com/tendermint/go-flowrate
-- package: github.com/tendermint/go-autofile
-- package: github.com/tendermint/go-clist
-- package: github.com/tendermint/go-common
-- package: github.com/tendermint/go-config
-- package: github.com/tendermint/go-crypto
-- package: github.com/tendermint/go-db
-- package: github.com/tendermint/go-events
-- package: github.com/tendermint/go-logger
-- package: github.com/tendermint/go-merkle
-- package: github.com/tendermint/go-p2p
-- package: github.com/tendermint/go-rpc
-- package: github.com/tendermint/go-wire
-- package: github.com/tendermint/log15
-- package: github.com/tendermint/abci
 - package: golang.org/x/crypto
   subpackages:
   - ripemd160
-- package: github.com/tendermint/go-flowrate
-  subpackages:
-  - flowrate 

From a073b1db9cc33fb90740ca3ef966bc791bab65df Mon Sep 17 00:00:00 2001
From: Jae Kwon <jae@tendermint.com>
Date: Sun, 15 Jan 2017 16:19:02 -0800
Subject: [PATCH 002/132] Refactor replay console -> replay_file in
 consensus/replay_file.go

---
 cmd/tendermint/main.go   |   9 +-
 consensus/replay.go      | 221 -------------------------------
 consensus/replay_file.go | 271 +++++++++++++++++++++++++++++++++++++++
 node/node.go             |  57 +-------
 4 files changed, 276 insertions(+), 282 deletions(-)
 create mode 100644 consensus/replay_file.go

diff --git a/cmd/tendermint/main.go b/cmd/tendermint/main.go
index 9f341051..e5abe5b2 100644
--- a/cmd/tendermint/main.go
+++ b/cmd/tendermint/main.go
@@ -8,6 +8,7 @@ import (
 	cfg "github.com/tendermint/go-config"
 	"github.com/tendermint/go-logger"
 	tmcfg "github.com/tendermint/tendermint/config/tendermint"
+	"github.com/tendermint/tendermint/consensus"
 	"github.com/tendermint/tendermint/node"
 	"github.com/tendermint/tendermint/version"
 )
@@ -41,11 +42,9 @@ Commands:
 	case "node":
 		node.RunNode(config)
 	case "replay":
-		if len(args) > 2 && args[1] == "console" {
-			node.RunReplayConsole(config, args[2])
-		} else {
-			node.RunReplay(config, args[1])
-		}
+		consensus.RunReplayFile(config, args[1], false)
+	case "replay_console":
+		consensus.RunReplayFile(config, args[1], true)
 	case "init":
 		init_files()
 	case "show_validator":
diff --git a/consensus/replay.go b/consensus/replay.go
index 8793c42f..d534827b 100644
--- a/consensus/replay.go
+++ b/consensus/replay.go
@@ -1,11 +1,9 @@
 package consensus
 
 import (
-	"bufio"
 	"errors"
 	"fmt"
 	"io"
-	"os"
 	"reflect"
 	"strconv"
 	"strings"
@@ -15,7 +13,6 @@ import (
 	. "github.com/tendermint/go-common"
 	"github.com/tendermint/go-wire"
 
-	sm "github.com/tendermint/tendermint/state"
 	"github.com/tendermint/tendermint/types"
 )
 
@@ -133,224 +130,6 @@ func (cs *ConsensusState) catchupReplay(csHeight int) error {
 	return nil
 }
 
-//--------------------------------------------------------
-// replay messages interactively or all at once
-
-// Interactive playback
-func (cs ConsensusState) ReplayConsole(file string) error {
-	return cs.replay(file, true)
-}
-
-// Full playback, with tests
-func (cs ConsensusState) ReplayMessages(file string) error {
-	return cs.replay(file, false)
-}
-
-// replay all msgs or start the console
-func (cs *ConsensusState) replay(file string, console bool) error {
-	if cs.IsRunning() {
-		return errors.New("cs is already running, cannot replay")
-	}
-	if cs.wal != nil {
-		return errors.New("cs wal is open, cannot replay")
-	}
-
-	cs.startForReplay()
-
-	// ensure all new step events are regenerated as expected
-	newStepCh := subscribeToEvent(cs.evsw, "replay-test", types.EventStringNewRoundStep(), 1)
-
-	// just open the file for reading, no need to use wal
-	fp, err := os.OpenFile(file, os.O_RDONLY, 0666)
-	if err != nil {
-		return err
-	}
-
-	pb := newPlayback(file, fp, cs, cs.state.Copy())
-	defer pb.fp.Close()
-
-	var nextN int // apply N msgs in a row
-	for pb.scanner.Scan() {
-		if nextN == 0 && console {
-			nextN = pb.replayConsoleLoop()
-		}
-
-		if err := pb.cs.readReplayMessage(pb.scanner.Bytes(), newStepCh); err != nil {
-			return err
-		}
-
-		if nextN > 0 {
-			nextN -= 1
-		}
-		pb.count += 1
-	}
-	return nil
-}
-
-//------------------------------------------------
-// playback manager
-
-type playback struct {
-	cs *ConsensusState
-
-	fp      *os.File
-	scanner *bufio.Scanner
-	count   int // how many lines/msgs into the file are we
-
-	// replays can be reset to beginning
-	fileName     string    // so we can close/reopen the file
-	genesisState *sm.State // so the replay session knows where to restart from
-}
-
-func newPlayback(fileName string, fp *os.File, cs *ConsensusState, genState *sm.State) *playback {
-	return &playback{
-		cs:           cs,
-		fp:           fp,
-		fileName:     fileName,
-		genesisState: genState,
-		scanner:      bufio.NewScanner(fp),
-	}
-}
-
-// go back count steps by resetting the state and running (pb.count - count) steps
-func (pb *playback) replayReset(count int, newStepCh chan interface{}) error {
-
-	pb.cs.Stop()
-	pb.cs.Wait()
-
-	newCS := NewConsensusState(pb.cs.config, pb.genesisState.Copy(), pb.cs.proxyAppConn, pb.cs.blockStore, pb.cs.mempool)
-	newCS.SetEventSwitch(pb.cs.evsw)
-	newCS.startForReplay()
-
-	pb.fp.Close()
-	fp, err := os.OpenFile(pb.fileName, os.O_RDONLY, 0666)
-	if err != nil {
-		return err
-	}
-	pb.fp = fp
-	pb.scanner = bufio.NewScanner(fp)
-	count = pb.count - count
-	log.Notice(Fmt("Reseting from %d to %d", pb.count, count))
-	pb.count = 0
-	pb.cs = newCS
-	for i := 0; pb.scanner.Scan() && i < count; i++ {
-		if err := pb.cs.readReplayMessage(pb.scanner.Bytes(), newStepCh); err != nil {
-			return err
-		}
-		pb.count += 1
-	}
-	return nil
-}
-
-func (cs *ConsensusState) startForReplay() {
-	// don't want to start full cs
-	cs.BaseService.OnStart()
-
-	log.Warn("Replay commands are disabled until someone updates them and writes tests")
-	/* TODO:!
-	// since we replay tocks we just ignore ticks
-		go func() {
-			for {
-				select {
-				case <-cs.tickChan:
-				case <-cs.Quit:
-					return
-				}
-			}
-		}()*/
-}
-
-// console function for parsing input and running commands
-func (pb *playback) replayConsoleLoop() int {
-	for {
-		fmt.Printf("> ")
-		bufReader := bufio.NewReader(os.Stdin)
-		line, more, err := bufReader.ReadLine()
-		if more {
-			Exit("input is too long")
-		} else if err != nil {
-			Exit(err.Error())
-		}
-
-		tokens := strings.Split(string(line), " ")
-		if len(tokens) == 0 {
-			continue
-		}
-
-		switch tokens[0] {
-		case "next":
-			// "next" -> replay next message
-			// "next N" -> replay next N messages
-
-			if len(tokens) == 1 {
-				return 0
-			} else {
-				i, err := strconv.Atoi(tokens[1])
-				if err != nil {
-					fmt.Println("next takes an integer argument")
-				} else {
-					return i
-				}
-			}
-
-		case "back":
-			// "back" -> go back one message
-			// "back N" -> go back N messages
-
-			// NOTE: "back" is not supported in the state machine design,
-			// so we restart and replay up to
-
-			// ensure all new step events are regenerated as expected
-			newStepCh := subscribeToEvent(pb.cs.evsw, "replay-test", types.EventStringNewRoundStep(), 1)
-			if len(tokens) == 1 {
-				pb.replayReset(1, newStepCh)
-			} else {
-				i, err := strconv.Atoi(tokens[1])
-				if err != nil {
-					fmt.Println("back takes an integer argument")
-				} else if i > pb.count {
-					fmt.Printf("argument to back must not be larger than the current count (%d)\n", pb.count)
-				} else {
-					pb.replayReset(i, newStepCh)
-				}
-			}
-
-		case "rs":
-			// "rs" -> print entire round state
-			// "rs short" -> print height/round/step
-			// "rs <field>" -> print another field of the round state
-
-			rs := pb.cs.RoundState
-			if len(tokens) == 1 {
-				fmt.Println(rs)
-			} else {
-				switch tokens[1] {
-				case "short":
-					fmt.Printf("%v/%v/%v\n", rs.Height, rs.Round, rs.Step)
-				case "validators":
-					fmt.Println(rs.Validators)
-				case "proposal":
-					fmt.Println(rs.Proposal)
-				case "proposal_block":
-					fmt.Printf("%v %v\n", rs.ProposalBlockParts.StringShort(), rs.ProposalBlock.StringShort())
-				case "locked_round":
-					fmt.Println(rs.LockedRound)
-				case "locked_block":
-					fmt.Printf("%v %v\n", rs.LockedBlockParts.StringShort(), rs.LockedBlock.StringShort())
-				case "votes":
-					fmt.Println(rs.Votes.StringIndented("    "))
-
-				default:
-					fmt.Println("Unknown option", tokens[1])
-				}
-			}
-		case "n":
-			fmt.Println(pb.count)
-		}
-	}
-	return 0
-}
-
 //--------------------------------------------------------------------------------
 
 // Parses marker lines of the form:
diff --git a/consensus/replay_file.go b/consensus/replay_file.go
new file mode 100644
index 00000000..1c2e9cb0
--- /dev/null
+++ b/consensus/replay_file.go
@@ -0,0 +1,271 @@
+package consensus
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+
+	. "github.com/tendermint/go-common"
+	cfg "github.com/tendermint/go-config"
+	dbm "github.com/tendermint/go-db"
+	bc "github.com/tendermint/tendermint/blockchain"
+	mempl "github.com/tendermint/tendermint/mempool"
+	"github.com/tendermint/tendermint/proxy"
+	sm "github.com/tendermint/tendermint/state"
+	"github.com/tendermint/tendermint/types"
+)
+
+//--------------------------------------------------------
+// replay messages interactively or all at once
+
+func RunReplayFile(config cfg.Config, walFile string, console bool) {
+	consensusState := newConsensusStateForReplay(config)
+
+	if err := consensusState.ReplayFile(walFile, console); err != nil {
+		Exit(Fmt("Error during consensus replay: %v", err))
+	}
+}
+
+// Replay msgs in file or start the console
+func (cs *ConsensusState) ReplayFile(file string, console bool) error {
+
+	if cs.IsRunning() {
+		return errors.New("cs is already running, cannot replay")
+	}
+	if cs.wal != nil {
+		return errors.New("cs wal is open, cannot replay")
+	}
+
+	cs.startForReplay()
+
+	// ensure all new step events are regenerated as expected
+	newStepCh := subscribeToEvent(cs.evsw, "replay-test", types.EventStringNewRoundStep(), 1)
+
+	// just open the file for reading, no need to use wal
+	fp, err := os.OpenFile(file, os.O_RDONLY, 0666)
+	if err != nil {
+		return err
+	}
+
+	pb := newPlayback(file, fp, cs, cs.state.Copy())
+	defer pb.fp.Close()
+
+	var nextN int // apply N msgs in a row
+	for pb.scanner.Scan() {
+		if nextN == 0 && console {
+			nextN = pb.replayConsoleLoop()
+		}
+
+		if err := pb.cs.readReplayMessage(pb.scanner.Bytes(), newStepCh); err != nil {
+			return err
+		}
+
+		if nextN > 0 {
+			nextN -= 1
+		}
+		pb.count += 1
+	}
+	return nil
+}
+
+//------------------------------------------------
+// playback manager
+
+type playback struct {
+	cs *ConsensusState
+
+	fp      *os.File
+	scanner *bufio.Scanner
+	count   int // how many lines/msgs into the file are we
+
+	// replays can be reset to beginning
+	fileName     string    // so we can close/reopen the file
+	genesisState *sm.State // so the replay session knows where to restart from
+}
+
+func newPlayback(fileName string, fp *os.File, cs *ConsensusState, genState *sm.State) *playback {
+	return &playback{
+		cs:           cs,
+		fp:           fp,
+		fileName:     fileName,
+		genesisState: genState,
+		scanner:      bufio.NewScanner(fp),
+	}
+}
+
+// go back count steps by resetting the state and running (pb.count - count) steps
+func (pb *playback) replayReset(count int, newStepCh chan interface{}) error {
+
+	pb.cs.Stop()
+	pb.cs.Wait()
+
+	newCS := NewConsensusState(pb.cs.config, pb.genesisState.Copy(), pb.cs.proxyAppConn, pb.cs.blockStore, pb.cs.mempool)
+	newCS.SetEventSwitch(pb.cs.evsw)
+	newCS.startForReplay()
+
+	pb.fp.Close()
+	fp, err := os.OpenFile(pb.fileName, os.O_RDONLY, 0666)
+	if err != nil {
+		return err
+	}
+	pb.fp = fp
+	pb.scanner = bufio.NewScanner(fp)
+	count = pb.count - count
+	log.Notice(Fmt("Reseting from %d to %d", pb.count, count))
+	pb.count = 0
+	pb.cs = newCS
+	for i := 0; pb.scanner.Scan() && i < count; i++ {
+		if err := pb.cs.readReplayMessage(pb.scanner.Bytes(), newStepCh); err != nil {
+			return err
+		}
+		pb.count += 1
+	}
+	return nil
+}
+
+func (cs *ConsensusState) startForReplay() {
+	// don't want to start full cs
+	cs.BaseService.OnStart()
+
+	log.Warn("Replay commands are disabled until someone updates them and writes tests")
+	/* TODO:!
+	// since we replay tocks we just ignore ticks
+		go func() {
+			for {
+				select {
+				case <-cs.tickChan:
+				case <-cs.Quit:
+					return
+				}
+			}
+		}()*/
+}
+
+// console function for parsing input and running commands
+func (pb *playback) replayConsoleLoop() int {
+	for {
+		fmt.Printf("> ")
+		bufReader := bufio.NewReader(os.Stdin)
+		line, more, err := bufReader.ReadLine()
+		if more {
+			Exit("input is too long")
+		} else if err != nil {
+			Exit(err.Error())
+		}
+
+		tokens := strings.Split(string(line), " ")
+		if len(tokens) == 0 {
+			continue
+		}
+
+		switch tokens[0] {
+		case "next":
+			// "next" -> replay next message
+			// "next N" -> replay next N messages
+
+			if len(tokens) == 1 {
+				return 0
+			} else {
+				i, err := strconv.Atoi(tokens[1])
+				if err != nil {
+					fmt.Println("next takes an integer argument")
+				} else {
+					return i
+				}
+			}
+
+		case "back":
+			// "back" -> go back one message
+			// "back N" -> go back N messages
+
+			// NOTE: "back" is not supported in the state machine design,
+			// so we restart and replay up to
+
+			// ensure all new step events are regenerated as expected
+			newStepCh := subscribeToEvent(pb.cs.evsw, "replay-test", types.EventStringNewRoundStep(), 1)
+			if len(tokens) == 1 {
+				pb.replayReset(1, newStepCh)
+			} else {
+				i, err := strconv.Atoi(tokens[1])
+				if err != nil {
+					fmt.Println("back takes an integer argument")
+				} else if i > pb.count {
+					fmt.Printf("argument to back must not be larger than the current count (%d)\n", pb.count)
+				} else {
+					pb.replayReset(i, newStepCh)
+				}
+			}
+
+		case "rs":
+			// "rs" -> print entire round state
+			// "rs short" -> print height/round/step
+			// "rs <field>" -> print another field of the round state
+
+			rs := pb.cs.RoundState
+			if len(tokens) == 1 {
+				fmt.Println(rs)
+			} else {
+				switch tokens[1] {
+				case "short":
+					fmt.Printf("%v/%v/%v\n", rs.Height, rs.Round, rs.Step)
+				case "validators":
+					fmt.Println(rs.Validators)
+				case "proposal":
+					fmt.Println(rs.Proposal)
+				case "proposal_block":
+					fmt.Printf("%v %v\n", rs.ProposalBlockParts.StringShort(), rs.ProposalBlock.StringShort())
+				case "locked_round":
+					fmt.Println(rs.LockedRound)
+				case "locked_block":
+					fmt.Printf("%v %v\n", rs.LockedBlockParts.StringShort(), rs.LockedBlock.StringShort())
+				case "votes":
+					fmt.Println(rs.Votes.StringIndented("    "))
+
+				default:
+					fmt.Println("Unknown option", tokens[1])
+				}
+			}
+		case "n":
+			fmt.Println(pb.count)
+		}
+	}
+	return 0
+}
+
+//--------------------------------------------------------------------------------
+
+// convenience for replay mode
+func newConsensusStateForReplay(config cfg.Config) *ConsensusState {
+	// Get BlockStore
+	blockStoreDB := dbm.NewDB("blockstore", config.GetString("db_backend"), config.GetString("db_dir"))
+	blockStore := bc.NewBlockStore(blockStoreDB)
+
+	// Get State
+	stateDB := dbm.NewDB("state", config.GetString("db_backend"), config.GetString("db_dir"))
+	state := sm.MakeGenesisStateFromFile(stateDB, config.GetString("genesis_file"))
+
+	// Create proxyAppConn connection (consensus, mempool, query)
+	proxyApp := proxy.NewAppConns(config, proxy.DefaultClientCreator(config), sm.NewHandshaker(config, state, blockStore))
+	_, err := proxyApp.Start()
+	if err != nil {
+		Exit(Fmt("Error starting proxy app conns: %v", err))
+	}
+
+	// add the chainid to the global config
+	config.Set("chain_id", state.ChainID)
+
+	// Make event switch
+	eventSwitch := types.NewEventSwitch()
+	if _, err := eventSwitch.Start(); err != nil {
+		Exit(Fmt("Failed to start event switch: %v", err))
+	}
+
+	mempool := mempl.NewMempool(config, proxyApp.Mempool())
+
+	consensusState := NewConsensusState(config, state.Copy(), proxyApp.Consensus(), blockStore, mempool)
+	consensusState.SetEventSwitch(eventSwitch)
+	return consensusState
+}
diff --git a/node/node.go b/node/node.go
index b6720a0e..43eba257 100644
--- a/node/node.go
+++ b/node/node.go
@@ -57,10 +57,8 @@ func NewNode(config cfg.Config, privValidator *types.PrivValidator, clientCreato
 	blockStoreDB := dbm.NewDB("blockstore", config.GetString("db_backend"), config.GetString("db_dir"))
 	blockStore := bc.NewBlockStore(blockStoreDB)
 
-	// Get State db
-	stateDB := dbm.NewDB("state", config.GetString("db_backend"), config.GetString("db_dir"))
-
 	// Get State
+	stateDB := dbm.NewDB("state", config.GetString("db_backend"), config.GetString("db_dir"))
 	state := sm.GetState(config, stateDB)
 
 	// Create the proxyApp, which manages connections (consensus, mempool, query)
@@ -387,59 +385,6 @@ func (n *Node) DialSeeds(seeds []string) {
 	n.sw.DialSeeds(seeds)
 }
 
-//------------------------------------------------------------------------------
-// replay
-
-// convenience for replay mode
-func newConsensusState(config cfg.Config) *consensus.ConsensusState {
-	// Get BlockStore
-	blockStoreDB := dbm.NewDB("blockstore", config.GetString("db_backend"), config.GetString("db_dir"))
-	blockStore := bc.NewBlockStore(blockStoreDB)
-
-	// Get State
-	stateDB := dbm.NewDB("state", config.GetString("db_backend"), config.GetString("db_dir"))
-	state := sm.MakeGenesisStateFromFile(stateDB, config.GetString("genesis_file"))
-
-	// Create proxyAppConn connection (consensus, mempool, query)
-	proxyApp := proxy.NewAppConns(config, proxy.DefaultClientCreator(config), sm.NewHandshaker(config, state, blockStore))
-	_, err := proxyApp.Start()
-	if err != nil {
-		Exit(Fmt("Error starting proxy app conns: %v", err))
-	}
-
-	// add the chainid to the global config
-	config.Set("chain_id", state.ChainID)
-
-	// Make event switch
-	eventSwitch := types.NewEventSwitch()
-	if _, err := eventSwitch.Start(); err != nil {
-		Exit(Fmt("Failed to start event switch: %v", err))
-	}
-
-	mempool := mempl.NewMempool(config, proxyApp.Mempool())
-
-	consensusState := consensus.NewConsensusState(config, state.Copy(), proxyApp.Consensus(), blockStore, mempool)
-	consensusState.SetEventSwitch(eventSwitch)
-	return consensusState
-}
-
-func RunReplayConsole(config cfg.Config, walFile string) {
-	consensusState := newConsensusState(config)
-
-	if err := consensusState.ReplayConsole(walFile); err != nil {
-		Exit(Fmt("Error during consensus replay: %v", err))
-	}
-}
-
-func RunReplay(config cfg.Config, walFile string) {
-	consensusState := newConsensusState(config)
-
-	if err := consensusState.ReplayMessages(walFile); err != nil {
-		Exit(Fmt("Error during consensus replay: %v", err))
-	}
-	log.Notice("Replay run successfully")
-}
-
 // Defaults to tcp
 func ProtocolAndAddress(listenAddr string) (string, string) {
 	protocol, address := "tcp", listenAddr

From 9a2dd8bc9279ed2a1a4d4f31cc151f8a621cceb3 Mon Sep 17 00:00:00 2001
From: Jae Kwon <jae@tendermint.com>
Date: Sun, 15 Jan 2017 16:59:10 -0800
Subject: [PATCH 003/132] Refactor Node; Node is a simple BaseService

---
 cmd/tendermint/main.go     |   3 +-
 cmd/tendermint/run_node.go |  56 +++++++++++++++
 node/node.go               | 142 +++++++++++++++----------------------
 node/node_test.go          |   8 +--
 rpc/test/helpers.go        |   6 --
 5 files changed, 118 insertions(+), 97 deletions(-)
 create mode 100644 cmd/tendermint/run_node.go

diff --git a/cmd/tendermint/main.go b/cmd/tendermint/main.go
index e5abe5b2..2338ad39 100644
--- a/cmd/tendermint/main.go
+++ b/cmd/tendermint/main.go
@@ -9,7 +9,6 @@ import (
 	"github.com/tendermint/go-logger"
 	tmcfg "github.com/tendermint/tendermint/config/tendermint"
 	"github.com/tendermint/tendermint/consensus"
-	"github.com/tendermint/tendermint/node"
 	"github.com/tendermint/tendermint/version"
 )
 
@@ -40,7 +39,7 @@ Commands:
 
 	switch args[0] {
 	case "node":
-		node.RunNode(config)
+		run_node(config)
 	case "replay":
 		consensus.RunReplayFile(config, args[1], false)
 	case "replay_console":
diff --git a/cmd/tendermint/run_node.go b/cmd/tendermint/run_node.go
new file mode 100644
index 00000000..9f13533a
--- /dev/null
+++ b/cmd/tendermint/run_node.go
@@ -0,0 +1,56 @@
+package main
+
+import (
+	"io/ioutil"
+	"time"
+
+	. "github.com/tendermint/go-common"
+	cfg "github.com/tendermint/go-config"
+	"github.com/tendermint/tendermint/node"
+	"github.com/tendermint/tendermint/types"
+)
+
+// Users wishing to:
+//	* Use an external signer for their validators
+//	* Supply an in-proc abci app
+// should import tendermint/tendermint and implement their own RunNode to
+// call NewNode with their custom priv validator and/or custom
+// proxy.ClientCreator interface
+func run_node(config cfg.Config) {
+
+	// Wait until the genesis doc becomes available
+	// This is for Mintnet compatibility.
+	// TODO: If Mintnet gets deprecated or genesis_file is
+	// always available, remove.
+	genDocFile := config.GetString("genesis_file")
+	if !FileExists(genDocFile) {
+		log.Notice(Fmt("Waiting for genesis file %v...", genDocFile))
+		for {
+			time.Sleep(time.Second)
+			if !FileExists(genDocFile) {
+				continue
+			}
+			jsonBlob, err := ioutil.ReadFile(genDocFile)
+			if err != nil {
+				Exit(Fmt("Couldn't read GenesisDoc file: %v", err))
+			}
+			genDoc := types.GenesisDocFromJSON(jsonBlob)
+			if genDoc.ChainID == "" {
+				PanicSanity(Fmt("Genesis doc %v must include non-empty chain_id", genDocFile))
+			}
+			config.Set("chain_id", genDoc.ChainID)
+		}
+	}
+
+	// Create & start node
+	n := node.NewNodeDefault(config)
+	if _, err := n.Start(); err != nil {
+		Exit(Fmt("Failed to start node: %v", err))
+	} else {
+		log.Notice("Started node", "nodeInfo", n.Switch().NodeInfo())
+	}
+
+	// Trap signal, run forever.
+	n.RunForever()
+
+}
diff --git a/node/node.go b/node/node.go
index 43eba257..91415ec7 100644
--- a/node/node.go
+++ b/node/node.go
@@ -2,13 +2,11 @@ package node
 
 import (
 	"bytes"
-	"io/ioutil"
 	"net"
 	"net/http"
 	"strings"
-	"time"
 
-	. "github.com/tendermint/go-common"
+	cmn "github.com/tendermint/go-common"
 	cfg "github.com/tendermint/go-config"
 	"github.com/tendermint/go-crypto"
 	dbm "github.com/tendermint/go-db"
@@ -30,6 +28,8 @@ import (
 import _ "net/http/pprof"
 
 type Node struct {
+	cmn.BaseService
+
 	config           cfg.Config
 	sw               *p2p.Switch
 	evsw             types.EventSwitch
@@ -64,7 +64,7 @@ func NewNode(config cfg.Config, privValidator *types.PrivValidator, clientCreato
 	// Create the proxyApp, which manages connections (consensus, mempool, query)
 	proxyApp := proxy.NewAppConns(config, clientCreator, sm.NewHandshaker(config, state, blockStore))
 	if _, err := proxyApp.Start(); err != nil {
-		Exit(Fmt("Error starting proxy app connections: %v", err))
+		cmn.Exit(cmn.Fmt("Error starting proxy app connections: %v", err))
 	}
 
 	// add the chainid and number of validators to the global config
@@ -78,7 +78,7 @@ func NewNode(config cfg.Config, privValidator *types.PrivValidator, clientCreato
 	eventSwitch := types.NewEventSwitch()
 	_, err := eventSwitch.Start()
 	if err != nil {
-		Exit(Fmt("Failed to start switch: %v", err))
+		cmn.Exit(cmn.Fmt("Failed to start switch: %v", err))
 	}
 
 	// Decide whether to fast-sync or not
@@ -126,14 +126,14 @@ func NewNode(config cfg.Config, privValidator *types.PrivValidator, clientCreato
 	if config.GetBool("filter_peers") {
 		// NOTE: addr is ip:port
 		sw.SetAddrFilter(func(addr net.Addr) error {
-			res := proxyApp.Query().QuerySync([]byte(Fmt("p2p/filter/addr/%s", addr.String())))
+			res := proxyApp.Query().QuerySync([]byte(cmn.Fmt("p2p/filter/addr/%s", addr.String())))
 			if res.IsOK() {
 				return nil
 			}
 			return res
 		})
 		sw.SetPubKeyFilter(func(pubkey crypto.PubKeyEd25519) error {
-			res := proxyApp.Query().QuerySync([]byte(Fmt("p2p/filter/pubkey/%X", pubkey.Bytes())))
+			res := proxyApp.Query().QuerySync([]byte(cmn.Fmt("p2p/filter/pubkey/%X", pubkey.Bytes())))
 			if res.IsOK() {
 				return nil
 			}
@@ -154,7 +154,7 @@ func NewNode(config cfg.Config, privValidator *types.PrivValidator, clientCreato
 		}()
 	}
 
-	return &Node{
+	node := &Node{
 		config:           config,
 		sw:               sw,
 		evsw:             eventSwitch,
@@ -168,17 +168,53 @@ func NewNode(config cfg.Config, privValidator *types.PrivValidator, clientCreato
 		privKey:          privKey,
 		proxyApp:         proxyApp,
 	}
+	node.BaseService = *cmn.NewBaseService(log, "Node", node)
+	return node
 }
 
-// Call Start() after adding the listeners.
-func (n *Node) Start() error {
+func (n *Node) OnStart() error {
+	n.BaseService.OnStart()
+
+	// Create & add listener
+	protocol, address := ProtocolAndAddress(n.config.GetString("node_laddr"))
+	l := p2p.NewDefaultListener(protocol, address, n.config.GetBool("skip_upnp"))
+	n.sw.AddListener(l)
+
+	// Start the switch
 	n.sw.SetNodeInfo(makeNodeInfo(n.config, n.sw, n.privKey))
 	n.sw.SetNodePrivKey(n.privKey)
 	_, err := n.sw.Start()
-	return err
+	if err != nil {
+		return err
+	}
+
+	// Dial out of seed nodes exist
+	if n.config.GetString("seeds") != "" {
+		seeds := strings.Split(n.config.GetString("seeds"), ",")
+		n.sw.DialSeeds(seeds)
+	}
+
+	// Run the RPC server
+	if n.config.GetString("rpc_laddr") != "" {
+		_, err := n.startRPC()
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
 }
 
-func (n *Node) Stop() {
+func (n *Node) RunForever() {
+	// Sleep forever and then...
+	cmn.TrapSignal(func() {
+		n.Stop()
+	})
+}
+
+func (n *Node) OnStop() {
+	n.BaseService.OnStop()
+
 	log.Notice("Stopping Node")
 	// TODO: gracefully disconnect from peers.
 	n.sw.Stop()
@@ -195,11 +231,10 @@ func SetEventSwitch(evsw types.EventSwitch, eventables ...types.Eventable) {
 // Add listeners before starting the Node.
 // The first listener is the primary listener (in NodeInfo)
 func (n *Node) AddListener(l p2p.Listener) {
-	log.Notice(Fmt("Added %v", l))
 	n.sw.AddListener(l)
 }
 
-func (n *Node) StartRPC() ([]net.Listener, error) {
+func (n *Node) startRPC() ([]net.Listener, error) {
 	rpccore.SetConfig(n.config)
 
 	rpccore.SetEventSwitch(n.evsw)
@@ -285,16 +320,16 @@ func makeNodeInfo(config cfg.Config, sw *p2p.Switch, privKey crypto.PrivKeyEd255
 		Network: config.GetString("chain_id"),
 		Version: version.Version,
 		Other: []string{
-			Fmt("wire_version=%v", wire.Version),
-			Fmt("p2p_version=%v", p2p.Version),
-			Fmt("consensus_version=%v", consensus.Version),
-			Fmt("rpc_version=%v/%v", rpc.Version, rpccore.Version),
+			cmn.Fmt("wire_version=%v", wire.Version),
+			cmn.Fmt("p2p_version=%v", p2p.Version),
+			cmn.Fmt("consensus_version=%v", consensus.Version),
+			cmn.Fmt("rpc_version=%v/%v", rpc.Version, rpccore.Version),
 		},
 	}
 
 	// include git hash in the nodeInfo if available
-	if rev, err := ReadFile(config.GetString("revision_file")); err == nil {
-		nodeInfo.Other = append(nodeInfo.Other, Fmt("revision=%v", string(rev)))
+	if rev, err := cmn.ReadFile(config.GetString("revision_file")); err == nil {
+		nodeInfo.Other = append(nodeInfo.Other, cmn.Fmt("revision=%v", string(rev)))
 	}
 
 	if !sw.IsListening() {
@@ -309,74 +344,11 @@ func makeNodeInfo(config cfg.Config, sw *p2p.Switch, privKey crypto.PrivKeyEd255
 	// We assume that the rpcListener has the same ExternalAddress.
 	// This is probably true because both P2P and RPC listeners use UPnP,
 	// except of course if the rpc is only bound to localhost
-	nodeInfo.ListenAddr = Fmt("%v:%v", p2pHost, p2pPort)
-	nodeInfo.Other = append(nodeInfo.Other, Fmt("rpc_addr=%v", rpcListenAddr))
+	nodeInfo.ListenAddr = cmn.Fmt("%v:%v", p2pHost, p2pPort)
+	nodeInfo.Other = append(nodeInfo.Other, cmn.Fmt("rpc_addr=%v", rpcListenAddr))
 	return nodeInfo
 }
 
-//------------------------------------------------------------------------------
-
-// Users wishing to:
-//	* use an external signer for their validators
-//	* supply an in-proc abci app
-// should fork tendermint/tendermint and implement RunNode to
-// call NewNode with their custom priv validator and/or custom
-// proxy.ClientCreator interface
-func RunNode(config cfg.Config) {
-	// Wait until the genesis doc becomes available
-	genDocFile := config.GetString("genesis_file")
-	if !FileExists(genDocFile) {
-		log.Notice(Fmt("Waiting for genesis file %v...", genDocFile))
-		for {
-			time.Sleep(time.Second)
-			if !FileExists(genDocFile) {
-				continue
-			}
-			jsonBlob, err := ioutil.ReadFile(genDocFile)
-			if err != nil {
-				Exit(Fmt("Couldn't read GenesisDoc file: %v", err))
-			}
-			genDoc := types.GenesisDocFromJSON(jsonBlob)
-			if genDoc.ChainID == "" {
-				PanicSanity(Fmt("Genesis doc %v must include non-empty chain_id", genDocFile))
-			}
-			config.Set("chain_id", genDoc.ChainID)
-		}
-	}
-
-	// Create & start node
-	n := NewNodeDefault(config)
-
-	protocol, address := ProtocolAndAddress(config.GetString("node_laddr"))
-	l := p2p.NewDefaultListener(protocol, address, config.GetBool("skip_upnp"))
-	n.AddListener(l)
-	err := n.Start()
-	if err != nil {
-		Exit(Fmt("Failed to start node: %v", err))
-	}
-
-	log.Notice("Started node", "nodeInfo", n.sw.NodeInfo())
-
-	// If seedNode is provided by config, dial out.
-	if config.GetString("seeds") != "" {
-		seeds := strings.Split(config.GetString("seeds"), ",")
-		n.sw.DialSeeds(seeds)
-	}
-
-	// Run the RPC server.
-	if config.GetString("rpc_laddr") != "" {
-		_, err := n.StartRPC()
-		if err != nil {
-			PanicCrisis(err)
-		}
-	}
-
-	// Sleep forever and then...
-	TrapSignal(func() {
-		n.Stop()
-	})
-}
-
 func (n *Node) NodeInfo() *p2p.NodeInfo {
 	return n.sw.NodeInfo()
 }
diff --git a/node/node_test.go b/node/node_test.go
index 880a2dab..2ab8e8dc 100644
--- a/node/node_test.go
+++ b/node/node_test.go
@@ -4,7 +4,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/tendermint/go-p2p"
 	"github.com/tendermint/tendermint/config/tendermint_test"
 )
 
@@ -13,12 +12,13 @@ func TestNodeStartStop(t *testing.T) {
 
 	// Create & start node
 	n := NewNodeDefault(config)
-	protocol, address := ProtocolAndAddress(config.GetString("node_laddr"))
-	l := p2p.NewDefaultListener(protocol, address, config.GetBool("skip_upnp"))
-	n.AddListener(l)
 	n.Start()
 	log.Notice("Started node", "nodeInfo", n.sw.NodeInfo())
+
+	// Wait a bit to initialize
+	// TODO remove time.Sleep(), make asynchronous.
 	time.Sleep(time.Second * 2)
+
 	ch := make(chan struct{}, 1)
 	go func() {
 		n.Stop()
diff --git a/rpc/test/helpers.go b/rpc/test/helpers.go
index da648248..76fcd4b3 100644
--- a/rpc/test/helpers.go
+++ b/rpc/test/helpers.go
@@ -6,7 +6,6 @@ import (
 
 	. "github.com/tendermint/go-common"
 	cfg "github.com/tendermint/go-config"
-	"github.com/tendermint/go-p2p"
 	"github.com/tendermint/go-wire"
 
 	client "github.com/tendermint/go-rpc/client"
@@ -57,13 +56,8 @@ func init() {
 func newNode(ready chan struct{}) {
 	// Create & start node
 	node = nm.NewNodeDefault(config)
-	protocol, address := nm.ProtocolAndAddress(config.GetString("node_laddr"))
-	l := p2p.NewDefaultListener(protocol, address, true)
-	node.AddListener(l)
 	node.Start()
 
-	// Run the RPC server.
-	node.StartRPC()
 	time.Sleep(time.Second)
 
 	ready <- struct{}{}

From e825d77736914aa100cbfadee0e2117f5709708e Mon Sep 17 00:00:00 2001
From: Anton Kaliaev <anton.kalyaev@gmail.com>
Date: Sun, 22 Jan 2017 18:00:08 +0400
Subject: [PATCH 004/132] add .editorconfig to maintain consistent coding style

---
 .editorconfig | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 .editorconfig

diff --git a/.editorconfig b/.editorconfig
new file mode 100644
index 00000000..d587999e
--- /dev/null
+++ b/.editorconfig
@@ -0,0 +1,15 @@
+# top-most EditorConfig file
+root = true
+
+# Unix-style newlines with a newline ending every file
+[*]
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[Makefile]
+indent_style = tab
+
+[*.sh]
+indent_style = tab

From c6375e414dde1b789589247306774602e87d4e5d Mon Sep 17 00:00:00 2001
From: Anton Kaliaev <anton.kalyaev@gmail.com>
Date: Sun, 22 Jan 2017 18:03:20 +0400
Subject: [PATCH 005/132] refactor Makefile

- mute most of the commands
- replace github.com/tendermint/tendermint with just "."
- introduce PACKAGES variable
- delete unused NEWLINE
---
 Makefile | 56 +++++++++++++++++++++++++++++++-------------------------
 1 file changed, 31 insertions(+), 25 deletions(-)

diff --git a/Makefile b/Makefile
index 8835722b..554b8a5b 100644
--- a/Makefile
+++ b/Makefile
@@ -1,59 +1,65 @@
-.PHONY: get_deps build all list_deps install
+GOTOOLS = \
+					github.com/mitchellh/gox \
+					github.com/Masterminds/glide
+PACKAGES=$(shell go list ./... | grep -v '/vendor/')
+BUILD_TAGS?=tendermint
+TMROOT = $${TMROOT:-$$HOME/.tendermint}
 
 all: get_deps install test
 
-TMROOT = $${TMROOT:-$$HOME/.tendermint}
-define NEWLINE
-
-
-endef
-NOVENDOR = go list github.com/tendermint/tendermint/... | grep -v /vendor/
-
 install: get_deps
-	go install github.com/tendermint/tendermint/cmd/tendermint
+	@go install ./cmd/tendermint
 
 build:
-	go build -o build/tendermint github.com/tendermint/tendermint/cmd/tendermint
+	go build -o build/tendermint ./cmd/tendermint
 
 build_race:
-	go build -race -o build/tendermint github.com/tendermint/tendermint/cmd/tendermint
+	go build -race -o build/tendermint ./cmd/tendermint
 
 test: build
-	go test `${NOVENDOR}`
+	@echo "--> Running go test"
+	@go test $(PACKAGES)
 
 test_race: build
-	go test -race `${NOVENDOR}`
+	@echo "--> Running go test --race"
+	@go test -race $(PACKAGES)
 
 test_integrations:
-	bash ./test/test.sh
+	@bash ./test/test.sh
 
 test100: build
-	for i in {1..100}; do make test; done
+	@for i in {1..100}; do make test; done
 
 draw_deps:
 	# requires brew install graphviz
 	go get github.com/hirokidaichi/goviz
-	goviz -i github.com/tendermint/tendermint/cmd/tendermint | dot -Tpng -o huge.png
+	goviz -i ./cmd/tendermint | dot -Tpng -o huge.png
 
 list_deps:
-	go list -f '{{join .Deps "\n"}}' github.com/tendermint/tendermint/... | \
+	@go list -f '{{join .Deps "\n"}}' ./... | \
 		grep -v /vendor/ | sort | uniq | \
-	  xargs go list -f '{{if not .Standard}}{{.ImportPath}}{{end}}'
+		xargs go list -f '{{if not .Standard}}{{.ImportPath}}{{end}}'
 
 get_deps:
-	go get -d `${NOVENDOR}`
-	go list -f '{{join .TestImports "\n"}}' github.com/tendermint/tendermint/... | \
+	@go get -d $(PACKAGES)
+	@go list -f '{{join .TestImports "\n"}}' ./... | \
 		grep -v /vendor/ | sort | uniq | \
 		xargs go get
 
-get_vendor_deps:
-	go get github.com/Masterminds/glide
-	rm -rf vendor/
-	glide install
+get_vendor_deps: tools
+	@rm -rf vendor/
+	@echo "--> Running glide install"
+	@glide install
 
 update_deps:
-	go get -d -u github.com/tendermint/tendermint/...
+	@echo "--> Updating dependencies"
+	@go get -d -u ./...
 
 revision:
 	-echo `git rev-parse --verify HEAD` > $(TMROOT)/revision
 	-echo `git rev-parse --verify HEAD` >> $(TMROOT)/revision_history
+
+tools:
+	go get -u -v $(GOTOOLS)
+
+.PHONY: install build build_race dist test test_race test_integrations test100 draw_deps list_deps get_deps get_vendor_deps update_deps revision tools

From cc2457f7d560caf96f083f74dcccd16ec20b2b71 Mon Sep 17 00:00:00 2001
From: Anton Kaliaev <anton.kalyaev@gmail.com>
Date: Mon, 23 Jan 2017 13:45:14 +0400
Subject: [PATCH 006/132] dist command to make binaries and package them for
 distribution

---
 .gitignore                            |  1 +
 Makefile                              |  4 ++
 scripts/dist.sh                       | 45 +++++++++++++++++++++
 scripts/dist_build.sh                 | 57 +++++++++++++++++++++++++++
 scripts/tendermint-builder/Dockerfile | 12 ++++++
 5 files changed, 119 insertions(+)
 create mode 100755 scripts/dist.sh
 create mode 100755 scripts/dist_build.sh
 create mode 100644 scripts/tendermint-builder/Dockerfile

diff --git a/.gitignore b/.gitignore
index acc957a9..4d8d3395 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,3 +12,4 @@ remote_dump
 vendor
 .vagrant
 test/p2p/data/
+.glide
diff --git a/Makefile b/Makefile
index 554b8a5b..424acccb 100644
--- a/Makefile
+++ b/Makefile
@@ -16,6 +16,10 @@ build:
 build_race:
 	go build -race -o build/tendermint ./cmd/tendermint
 
+# dist builds binaries for all platforms and packages them for distribution
+dist:
+	@BUILD_TAGS='$(BUILD_TAGS)' sh -c "'$(CURDIR)/scripts/dist.sh'"
+
 test: build
 	@echo "--> Running go test"
 	@go test $(PACKAGES)
diff --git a/scripts/dist.sh b/scripts/dist.sh
new file mode 100755
index 00000000..7d106f98
--- /dev/null
+++ b/scripts/dist.sh
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+set -e
+
+if [ -z "$VERSION" ]; then
+    echo "Please specify a version."
+    exit 1
+fi
+echo "==> Building version $VERSION..."
+
+# Get the parent directory of where this script is.
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done
+DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )"
+
+# Change into that dir because we expect that.
+cd "$DIR"
+
+# Generate the tag.
+if [ -z "$NOTAG" ]; then
+  echo "==> Tagging..."
+  git commit --allow-empty -a -m "Release v$VERSION"
+  git tag -a -m "Version $VERSION" "v${VERSION}" master
+fi
+
+# Do a hermetic build inside a Docker container.
+docker build -t tendermint/tendermint-builder scripts/tendermint-builder/
+docker run --rm -e "BUILD_TAGS=$BUILD_TAGS" -v "$(pwd)":/go/src/github.com/tendermint/tendermint tendermint/tendermint-builder ./scripts/dist_build.sh
+
+# Add "tendermint" and $VERSION prefix to package name.
+for FILENAME in $(find ./build/dist -mindepth 1 -maxdepth 1 -type f); do
+  FILENAME=$(basename "$FILENAME")
+  mv "./build/dist/${FILENAME}" "./build/dist/tendermint_${VERSION}_${FILENAME}"
+done
+
+# Make the checksums.
+pushd ./build/dist
+shasum -a256 ./* > "./tendermint_${VERSION}_SHA256SUMS"
+popd
+
+# Done
+echo
+echo "==> Results:"
+ls -hl ./build/dist
+
+exit 0
diff --git a/scripts/dist_build.sh b/scripts/dist_build.sh
new file mode 100755
index 00000000..947a3202
--- /dev/null
+++ b/scripts/dist_build.sh
@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+set -e
+
+# Get the parent directory of where this script is.
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done
+DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )"
+
+# Change into that dir because we expect that.
+cd "$DIR"
+
+# Get the git commit
+GIT_COMMIT="$(git rev-parse --short HEAD)"
+GIT_DESCRIBE="$(git describe --tags --always)"
+GIT_IMPORT="github.com/tendermint/tendermint/version"
+
+# Determine the arch/os combos we're building for
+XC_ARCH=${XC_ARCH:-"386 amd64 arm"}
+XC_OS=${XC_OS:-"solaris darwin freebsd linux windows"}
+
+# Delete the old dir
+echo "==> Removing old directory..."
+rm -rf build/dist/*
+mkdir -p build/dist
+
+# Make sure build tools are available.
+make tools
+
+# Get VENDORED dependencies
+make get_vendor_deps
+
+# Build!
+echo "==> Building..."
+"$(which gox)" \
+		-os="${XC_OS}" \
+		-arch="${XC_ARCH}" \
+		-osarch="!darwin/arm !solaris/amd64 !freebsd/amd64" \
+		-ldflags "-X ${GIT_IMPORT}.GitCommit='${GIT_COMMIT}' -X ${GIT_IMPORT}.GitDescribe='${GIT_DESCRIBE}'" \
+		-output "build/dist/{{.OS}}_{{.Arch}}/tendermint" \
+		-tags="${BUILD_TAGS}" \
+		github.com/tendermint/tendermint/cmd/tendermint
+
+# Zip all the files.
+echo "==> Packaging..."
+for PLATFORM in $(find ./build/dist -mindepth 1 -maxdepth 1 -type d); do
+		OSARCH=$(basename "${PLATFORM}")
+		echo "--> ${OSARCH}"
+
+		pushd "$PLATFORM" >/dev/null 2>&1
+		zip "../${OSARCH}.zip" ./*
+		popd >/dev/null 2>&1
+done
+
+# Remove build/dist/{{.OS}}_{{.Arch}} directories.
+rm -rf build/dist/*/
+
+exit 0
diff --git a/scripts/tendermint-builder/Dockerfile b/scripts/tendermint-builder/Dockerfile
new file mode 100644
index 00000000..84c198ee
--- /dev/null
+++ b/scripts/tendermint-builder/Dockerfile
@@ -0,0 +1,12 @@
+FROM golang:1.7.4
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+		zip \
+	&& rm -rf /var/lib/apt/lists/*
+
+# We want to ensure that release builds never have any cgo dependencies so we
+# switch that off at the highest level.
+ENV CGO_ENABLED 0
+
+RUN mkdir -p $GOPATH/src/github.com/tendermint/tendermint
+WORKDIR $GOPATH/src/github.com/tendermint/tendermint

From a388ff198dd35d829e0d8c6713fa7303a45f7ff6 Mon Sep 17 00:00:00 2001
From: Anton Kaliaev <anton.kalyaev@gmail.com>
Date: Mon, 23 Jan 2017 15:14:15 +0400
Subject: [PATCH 007/132] try to get version from version/version.go

---
 scripts/dist.sh    | 4 ++++
 version/version.go | 6 +++---
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/scripts/dist.sh b/scripts/dist.sh
index 7d106f98..8da29299 100755
--- a/scripts/dist.sh
+++ b/scripts/dist.sh
@@ -1,6 +1,10 @@
 #!/usr/bin/env bash
 set -e
 
+# Get the version from the environment, or try to figure it out.
+if [ -z $VERSION ]; then
+	VERSION=$(awk -F\" '/Version =/ { print $2; exit }' < version/version.go)
+fi
 if [ -z "$VERSION" ]; then
     echo "Please specify a version."
     exit 1
diff --git a/version/version.go b/version/version.go
index e0d7b426..395087d7 100644
--- a/version/version.go
+++ b/version/version.go
@@ -1,7 +1,7 @@
 package version
 
 const Maj = "0"
-const Min = "8" // validator set changes, tmsp->abci, app persistence/recovery, BFT-liveness fix
-const Fix = "0" //
+const Min = "8"
+const Fix = "0"
 
-const Version = Maj + "." + Min + "." + Fix
+const Version = "0.8.0"

From 17e822757b39fa6718a254d48e1d7121998e430a Mon Sep 17 00:00:00 2001
From: Anton Kaliaev <anton.kalyaev@gmail.com>
Date: Tue, 24 Jan 2017 21:19:45 +0400
Subject: [PATCH 008/132] output all commands

---
 cmd/tendermint/main.go | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/cmd/tendermint/main.go b/cmd/tendermint/main.go
index 2338ad39..cc57aa81 100644
--- a/cmd/tendermint/main.go
+++ b/cmd/tendermint/main.go
@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"os"
 
-	. "github.com/tendermint/go-common"
 	cfg "github.com/tendermint/go-config"
 	"github.com/tendermint/go-logger"
 	tmcfg "github.com/tendermint/tendermint/config/tendermint"
@@ -21,11 +20,16 @@ func main() {
 		fmt.Println(`Tendermint
 
 Commands:
-    node            Run the tendermint node
-    show_validator  Show this node's validator info
-    gen_validator   Generate new validator keypair
-    probe_upnp      Test UPnP functionality
-    version         Show version info
+    init                        Initialize tendermint
+    node                        Run the tendermint node
+    show_validator              Show this node's validator info
+    gen_validator               Generate new validator keypair
+    probe_upnp                  Test UPnP functionality
+    replay <walfile>            Replay messages from WAL
+    replay_console <walfile>    Replay messages from WAL in a console
+    unsafe_reset_all            (unsafe) Remove all the data and WAL, reset this node's validator
+    unsafe_reset_priv_validator (unsafe) Reset this node's validator
+    version                     Show version info
 `)
 		return
 	}
@@ -59,6 +63,7 @@ Commands:
 	case "version":
 		fmt.Println(version.Version)
 	default:
-		Exit(Fmt("Unknown command %v\n", args[0]))
+		fmt.Printf("Unknown command %v\n", args[0])
+		os.Exit(1)
 	}
 }

From 3b7a1d7149ad70de74354e04e853362bd6d31e72 Mon Sep 17 00:00:00 2001
From: Anton Kaliaev <anton.kalyaev@gmail.com>
Date: Tue, 24 Jan 2017 21:20:29 +0400
Subject: [PATCH 009/132] check that we have enough arguments

Otherwise:

```
panic: runtime error: index out of range

goroutine 1 [running]:
panic(0xbb8de0, 0xc82000e080)
        /usr/local/go/src/runtime/panic.go:464 +0x3e6
main.main()
        /go/src/github.com/tendermint/tendermint/cmd/tendermint/main.go:48 +0x811
```
---
 cmd/tendermint/main.go | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/cmd/tendermint/main.go b/cmd/tendermint/main.go
index cc57aa81..640f2c27 100644
--- a/cmd/tendermint/main.go
+++ b/cmd/tendermint/main.go
@@ -45,9 +45,19 @@ Commands:
 	case "node":
 		run_node(config)
 	case "replay":
-		consensus.RunReplayFile(config, args[1], false)
+		if len(args) > 1 {
+			consensus.RunReplayFile(config, args[1], false)
+		} else {
+			fmt.Println("replay requires an argument (walfile)")
+			os.Exit(1)
+		}
 	case "replay_console":
-		consensus.RunReplayFile(config, args[1], true)
+		if len(args) > 1 {
+			consensus.RunReplayFile(config, args[1], true)
+		} else {
+			fmt.Println("replay_console requires an argument (walfile)")
+			os.Exit(1)
+		}
 	case "init":
 		init_files()
 	case "show_validator":

From ce18332b524facf4d3294be04a573437862f2c71 Mon Sep 17 00:00:00 2001
From: Anton Kaliaev <anton.kalyaev@gmail.com>
Date: Fri, 27 Jan 2017 21:10:13 +0400
Subject: [PATCH 010/132] update Dockerfile

- update golang to 1.7.4
- version as env variable
- change DATA_ROOT from /tendermint/data to /tendermint (it's not just
  data that gets stored in DATA_ROOT; we create data folder on start; as
  a result we get /tendermint/data/data, which is confusing)
- remove noninteractive env vars (do we really need these?)
- remove nodejs dep (some apps may require nodejs, but core is not one
  of them; it was convenient before, but now I believe we ought to
  remove it because other people who are using java do not want a
  bloated container with nodejs)
- build tendermint inside a container (once again, it was convenient
  before, but now I am testing kubernetes and I don't want to wait every
  time TM compiles)
---
 DOCKER/Dockerfile | 72 ++++++++++++++++++++---------------------------
 DOCKER/run.sh     | 10 -------
 2 files changed, 31 insertions(+), 51 deletions(-)
 delete mode 100755 DOCKER/run.sh

diff --git a/DOCKER/Dockerfile b/DOCKER/Dockerfile
index 356fa8bf..e640d4c0 100644
--- a/DOCKER/Dockerfile
+++ b/DOCKER/Dockerfile
@@ -1,54 +1,44 @@
-# Pull base image.
-FROM golang:1.6
+FROM golang:1.7.4
 
-ENV USER tmuser
-ENV DATA_ROOT /data/tendermint
+# This is the release of tendermint to pull in.
+ENV TM_VERSION 0.8.0
+
+# Tendermint will be looking for genesis file in /tendermint (unless you change
+# `genesis_file` in config.toml). You can put your config.toml and private
+# validator file into /tendermint.
+#
+# The /tendermint/data dir is used by tendermint to store state.
+ENV DATA_ROOT /tendermint
+ENV TMROOT $DATA_ROOT
 
 # Set user right away for determinism
-RUN groupadd -r $USER \
-  && useradd -r -s /bin/false -g $USER $USER
-
-# Create home directory for USER
-# Needed for nodejs/nom
-RUN mkdir -p /home/$USER \
-  && chown -R $USER:$USER /home/$USER
+RUN groupadd -r tmuser && \
+    useradd -r -s /bin/false -g tmuser tmuser
 
 # Create directory for persistence and give our user ownership
-RUN mkdir -p $DATA_ROOT \
-  && chown -R $USER:$USER $DATA_ROOT
+RUN mkdir -p $DATA_ROOT && \
+    chown -R tmuser:tmuser $DATA_ROOT
 
-# Set the env variables to non-interactive
-ENV DEBIAN_FRONTEND noninteractive
-ENV DEBIAN_PRIORITY critical
-ENV DEBCONF_NOWARNINGS yes
-ENV TERM linux
-RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
-
-# Grab deps (git)
+# TODO replace with downloading a binary (this will allow us to replace golang
+# base container with alpine|jessie - 600MB vs 50MB)
 RUN apt-get update && \
-  apt-get install -y --no-install-recommends \
-    git && \
-  rm -rf /var/lib/apt/lists/*
+    apt-get install -y --no-install-recommends \
+      git && \
+    rm -rf /var/lib/apt/lists/*
+RUN mkdir -p $GOPATH/src/github.com/tendermint/tendermint && \
+    cd $GOPATH/src/github.com/tendermint/tendermint && \
+    git clone https://github.com/tendermint/tendermint.git . && \
+    git fetch && \
+    git reset --hard v$TM_VERSION && \
+    make install
 
-# Grab deps (node)
-RUN curl -sL https://deb.nodesource.com/setup_5.x | bash -
-RUN apt-get update && \
-  apt-get install -y --no-install-recommends \
-    nodejs && \
-  rm -rf /var/lib/apt/lists/*
-
-# Copy run.sh
-COPY ./run.sh $DATA_ROOT/run.sh
-RUN chmod +x $DATA_ROOT/run.sh
-
-# Persist data, set user
-WORKDIR $DATA_ROOT
+# Expose the data directory as a volume since there's mutable state in there
 VOLUME $DATA_ROOT
-USER $USER
-ENV TMROOT $DATA_ROOT
 
 EXPOSE 46656
 EXPOSE 46657
 
-# Run tendermint
-CMD ["./run.sh"]
+ENTRYPOINT ["tendermint"]
+
+# By default you'll get the dummy app
+CMD ["node", "--moniker=`hostname`", "--proxy_app=dummy"]
diff --git a/DOCKER/run.sh b/DOCKER/run.sh
deleted file mode 100755
index 903cf667..00000000
--- a/DOCKER/run.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-#! /bin/bash
-
-mkdir -p $GOPATH/src/$TMREPO
-cd $GOPATH/src/$TMREPO
-git clone https://$TMREPO.git .
-git fetch
-git reset --hard $TMHEAD
-go get -d $TMREPO/cmd/tendermint
-make
-tendermint node --seeds="$TMSEEDS" --moniker="$TMNAME"

From 449a29b8173fda8ccc7c8c165bea49b3d781b903 Mon Sep 17 00:00:00 2001
From: rigelrozanski <rigel.rozanski@gmail.com>
Date: Wed, 21 Dec 2016 17:21:48 -0500
Subject: [PATCH 011/132] fixed README link for contributing guidelines

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 424269c3..28730065 100644
--- a/README.md
+++ b/README.md
@@ -33,7 +33,7 @@ For more details (or if it fails), see the [install guide](https://tendermint.co
 
 ## Contributing
 
-Yay open source! Please see our [contributing guidelines](https://tendermint.com/guides/contributing).
+Yay open source! Please see our [contributing guidelines](https://tendermint.com/docs/guides/contributing).
 
 ## Resources
 

From 9257d648bf064515b257bb10ffb93ae000ed7495 Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Sat, 28 Jan 2017 00:12:39 -0500
Subject: [PATCH 012/132] test: update docker to 1.7.4

---
 test/docker/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/docker/Dockerfile b/test/docker/Dockerfile
index a663926e..e69f1ca6 100644
--- a/test/docker/Dockerfile
+++ b/test/docker/Dockerfile
@@ -1,5 +1,5 @@
 # Pull base image.
-FROM golang:1.6
+FROM golang:1.7.4
 
 # Grab deps (jq, hexdump, xxd, killall)
 RUN apt-get update && \

From 1af930441ce1e4a9563df17f4cce41b767ec150f Mon Sep 17 00:00:00 2001
From: Jae Kwon <jae@tendermint.com>
Date: Sat, 28 Jan 2017 08:27:13 -0800
Subject: [PATCH 013/132] Support new Query message for proofs

---
 consensus/mempool_test.go   |  7 ++++---
 glide.lock                  | 10 +++++-----
 node/node.go                | 30 +++++++++++++++++++-----------
 proxy/app_conn.go           |  8 ++++----
 rpc/core/abci.go            | 29 ++++++++++++++++-------------
 rpc/core/routes.go          |  6 +++---
 rpc/core/types/responses.go | 11 ++++-------
 rpc/test/client_test.go     | 22 ++++++++--------------
 8 files changed, 63 insertions(+), 60 deletions(-)

diff --git a/consensus/mempool_test.go b/consensus/mempool_test.go
index d298b5cb..4dd82a3f 100644
--- a/consensus/mempool_test.go
+++ b/consensus/mempool_test.go
@@ -5,9 +5,9 @@ import (
 	"testing"
 	"time"
 
+	abci "github.com/tendermint/abci/types"
 	"github.com/tendermint/tendermint/config/tendermint_test"
 	"github.com/tendermint/tendermint/types"
-	abci "github.com/tendermint/abci/types"
 
 	. "github.com/tendermint/go-common"
 )
@@ -161,6 +161,7 @@ func (app *CounterApplication) Commit() abci.Result {
 	}
 }
 
-func (app *CounterApplication) Query(query []byte) abci.Result {
-	return abci.NewResultOK(nil, Fmt("Query is not supported"))
+func (app *CounterApplication) Query(reqQuery abci.RequestQuery) (resQuery abci.ResponseQuery) {
+	resQuery.Log = "Query is not supported"
+	return
 }
diff --git a/glide.lock b/glide.lock
index 0277c260..a3cc726e 100644
--- a/glide.lock
+++ b/glide.lock
@@ -1,5 +1,5 @@
 hash: e283934fbbd221161d53a918db9e49db8c5be2b8929592b05ffe6b72c2ef0ab1
-updated: 2017-01-14T20:59:48.253671736-08:00
+updated: 2017-01-27T22:32:42.896956819-08:00
 imports:
 - name: github.com/btcsuite/btcd
   version: afec1bd1245a4a19e6dfe1306974b733e7cbb9b8
@@ -32,7 +32,7 @@ imports:
 - name: github.com/mattn/go-isatty
   version: 66b8e73f3f5cda9f96b69efd03dd3d7fc4a5cdb8
 - name: github.com/spf13/pflag
-  version: 25f8b5b07aece3207895bf19f7ab517eb3b22a40
+  version: 5ccb023bc27df288a957c5e994cd44fd19619465
 - name: github.com/syndtr/goleveldb
   version: 6ae1797c0b42b9323fc27ff7dcf568df88f2f33d
   subpackages:
@@ -49,7 +49,7 @@ imports:
   - leveldb/table
   - leveldb/util
 - name: github.com/tendermint/abci
-  version: 6526ab2137fadd0f4d2e25002bbfc1784b4f3c27
+  version: 8df0bc3a40ccad0d2be10e33c62c404e65c92502
   subpackages:
   - client
   - example/counter
@@ -67,7 +67,7 @@ imports:
 - name: github.com/tendermint/go-clist
   version: 3baa390bbaf7634251c42ad69a8682e7e3990552
 - name: github.com/tendermint/go-common
-  version: 70e694ee76f09058ea38c9ba81b4aa621bd54df1
+  version: 339e135776142939d82bc8e699db0bf391fd938d
   subpackages:
   - test
 - name: github.com/tendermint/go-config
@@ -85,7 +85,7 @@ imports:
 - name: github.com/tendermint/go-logger
   version: cefb3a45c0bf3c493a04e9bcd9b1540528be59f2
 - name: github.com/tendermint/go-merkle
-  version: 2979c7eb8aa020fa1cf203654907dbb889703888
+  version: 653cb1f631528351ddbc359b994eb0c96f0341cd
 - name: github.com/tendermint/go-p2p
   version: 67c9086b7458eb45b1970483decd01cd744c477a
   subpackages:
diff --git a/node/node.go b/node/node.go
index 91415ec7..d9eeb1f4 100644
--- a/node/node.go
+++ b/node/node.go
@@ -2,10 +2,12 @@ package node
 
 import (
 	"bytes"
+	"errors"
 	"net"
 	"net/http"
 	"strings"
 
+	abci "github.com/tendermint/abci/types"
 	cmn "github.com/tendermint/go-common"
 	cfg "github.com/tendermint/go-config"
 	"github.com/tendermint/go-crypto"
@@ -23,9 +25,9 @@ import (
 	sm "github.com/tendermint/tendermint/state"
 	"github.com/tendermint/tendermint/types"
 	"github.com/tendermint/tendermint/version"
-)
 
-import _ "net/http/pprof"
+	_ "net/http/pprof"
+)
 
 type Node struct {
 	cmn.BaseService
@@ -120,24 +122,30 @@ func NewNode(config cfg.Config, privValidator *types.PrivValidator, clientCreato
 		sw.AddReactor("PEX", pexReactor)
 	}
 
-	// filter peers by addr or pubkey with a abci query.
-	// if the query return code is OK, add peer
-	// XXX: query format subject to change
+	// Filter peers by addr or pubkey with an ABCI query.
+	// If the query return code is OK, add peer.
+	// XXX: Query format subject to change
 	if config.GetBool("filter_peers") {
 		// NOTE: addr is ip:port
 		sw.SetAddrFilter(func(addr net.Addr) error {
-			res := proxyApp.Query().QuerySync([]byte(cmn.Fmt("p2p/filter/addr/%s", addr.String())))
-			if res.IsOK() {
+			resQuery, err := proxyApp.Query().QuerySync(abci.RequestQuery{Path: cmn.Fmt("/p2p/filter/addr/%s", addr.String())})
+			if err != nil {
+				return err
+			}
+			if resQuery.Code.IsOK() {
 				return nil
 			}
-			return res
+			return errors.New(resQuery.Code.String())
 		})
 		sw.SetPubKeyFilter(func(pubkey crypto.PubKeyEd25519) error {
-			res := proxyApp.Query().QuerySync([]byte(cmn.Fmt("p2p/filter/pubkey/%X", pubkey.Bytes())))
-			if res.IsOK() {
+			resQuery, err := proxyApp.Query().QuerySync(abci.RequestQuery{Path: cmn.Fmt("/p2p/filter/pubkey/%X", pubkey.Bytes())})
+			if err != nil {
+				return err
+			}
+			if resQuery.Code.IsOK() {
 				return nil
 			}
-			return res
+			return errors.New(resQuery.Code.String())
 		})
 	}
 
diff --git a/proxy/app_conn.go b/proxy/app_conn.go
index 6abb8c7e..8b007737 100644
--- a/proxy/app_conn.go
+++ b/proxy/app_conn.go
@@ -34,8 +34,8 @@ type AppConnQuery interface {
 	Error() error
 
 	EchoSync(string) (res types.Result)
-	InfoSync() (types.ResponseInfo, error)
-	QuerySync(tx []byte) (res types.Result)
+	InfoSync() (resInfo types.ResponseInfo, err error)
+	QuerySync(reqQuery types.RequestQuery) (resQuery types.ResponseQuery, err error)
 
 	//	SetOptionSync(key string, value string) (res types.Result)
 }
@@ -139,6 +139,6 @@ func (app *appConnQuery) InfoSync() (types.ResponseInfo, error) {
 	return app.appConn.InfoSync()
 }
 
-func (app *appConnQuery) QuerySync(tx []byte) (res types.Result) {
-	return app.appConn.QuerySync(tx)
+func (app *appConnQuery) QuerySync(reqQuery types.RequestQuery) (types.ResponseQuery, error) {
+	return app.appConn.QuerySync(reqQuery)
 }
diff --git a/rpc/core/abci.go b/rpc/core/abci.go
index 03219343..cb748fe0 100644
--- a/rpc/core/abci.go
+++ b/rpc/core/abci.go
@@ -1,25 +1,28 @@
 package core
 
 import (
+	abci "github.com/tendermint/abci/types"
 	ctypes "github.com/tendermint/tendermint/rpc/core/types"
 )
 
 //-----------------------------------------------------------------------------
 
-func ABCIQuery(query []byte) (*ctypes.ResultABCIQuery, error) {
-	res := proxyAppQuery.QuerySync(query)
-	return &ctypes.ResultABCIQuery{res}, nil
-}
-
-func ABCIInfo() (*ctypes.ResultABCIInfo, error) {
-	res, err := proxyAppQuery.InfoSync()
+func ABCIQuery(path string, data []byte, prove bool) (*ctypes.ResultABCIQuery, error) {
+	resQuery, err := proxyAppQuery.QuerySync(abci.RequestQuery{
+		Path:  path,
+		Data:  data,
+		Prove: prove,
+	})
 	if err != nil {
 		return nil, err
 	}
-	return &ctypes.ResultABCIInfo{
-		Data:             res.Data,
-		Version:          res.Version,
-		LastBlockHeight:  res.LastBlockHeight,
-		LastBlockAppHash: res.LastBlockAppHash,
-	}, nil
+	return &ctypes.ResultABCIQuery{resQuery}, nil
+}
+
+func ABCIInfo() (*ctypes.ResultABCIInfo, error) {
+	resInfo, err := proxyAppQuery.InfoSync()
+	if err != nil {
+		return nil, err
+	}
+	return &ctypes.ResultABCIInfo{resInfo}, nil
 }
diff --git a/rpc/core/routes.go b/rpc/core/routes.go
index fa80bf2f..8396cd23 100644
--- a/rpc/core/routes.go
+++ b/rpc/core/routes.go
@@ -29,7 +29,7 @@ var Routes = map[string]*rpc.RPCFunc{
 	"broadcast_tx_async":  rpc.NewRPCFunc(BroadcastTxAsyncResult, "tx"),
 
 	// abci API
-	"abci_query": rpc.NewRPCFunc(ABCIQueryResult, "query"),
+	"abci_query": rpc.NewRPCFunc(ABCIQueryResult, "path,data,prove"),
 	"abci_info":  rpc.NewRPCFunc(ABCIInfoResult, ""),
 
 	// control API
@@ -163,8 +163,8 @@ func BroadcastTxAsyncResult(tx []byte) (ctypes.TMResult, error) {
 	}
 }
 
-func ABCIQueryResult(query []byte) (ctypes.TMResult, error) {
-	if r, err := ABCIQuery(query); err != nil {
+func ABCIQueryResult(path string, data []byte, prove bool) (ctypes.TMResult, error) {
+	if r, err := ABCIQuery(path, data, prove); err != nil {
 		return nil, err
 	} else {
 		return r, nil
diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go
index 5915c1db..1a5deaea 100644
--- a/rpc/core/types/responses.go
+++ b/rpc/core/types/responses.go
@@ -1,12 +1,12 @@
 package core_types
 
 import (
+	abci "github.com/tendermint/abci/types"
 	"github.com/tendermint/go-crypto"
 	"github.com/tendermint/go-p2p"
 	"github.com/tendermint/go-rpc/types"
 	"github.com/tendermint/go-wire"
 	"github.com/tendermint/tendermint/types"
-	abci "github.com/tendermint/abci/types"
 )
 
 type ResultBlockchainInfo struct {
@@ -64,7 +64,7 @@ type ResultBroadcastTx struct {
 }
 
 type ResultBroadcastTxCommit struct {
-	CheckTx  *abci.ResponseCheckTx  `json:"check_tx"`
+	CheckTx   *abci.ResponseCheckTx   `json:"check_tx"`
 	DeliverTx *abci.ResponseDeliverTx `json:"deliver_tx"`
 }
 
@@ -74,14 +74,11 @@ type ResultUnconfirmedTxs struct {
 }
 
 type ResultABCIInfo struct {
-	Data             string `json:"data"`
-	Version          string `json:"version"`
-	LastBlockHeight  uint64 `json:"last_block_height"`
-	LastBlockAppHash []byte `json:"last_block_app_hash"`
+	Response abci.ResponseInfo `json:"response"`
 }
 
 type ResultABCIQuery struct {
-	Result abci.Result `json:"result"`
+	Response abci.ResponseQuery `json:"response"`
 }
 
 type ResultUnsafeFlushMempool struct{}
diff --git a/rpc/test/client_test.go b/rpc/test/client_test.go
index da0b86f3..b17252df 100644
--- a/rpc/test/client_test.go
+++ b/rpc/test/client_test.go
@@ -8,12 +8,10 @@ import (
 	"testing"
 	"time"
 
+	abci "github.com/tendermint/abci/types"
 	. "github.com/tendermint/go-common"
-	"github.com/tendermint/go-wire"
 	ctypes "github.com/tendermint/tendermint/rpc/core/types"
 	"github.com/tendermint/tendermint/types"
-	"github.com/tendermint/abci/example/dummy"
-	abci "github.com/tendermint/abci/types"
 )
 
 //--------------------------------------------------------------------------------
@@ -134,7 +132,7 @@ func TestURIABCIQuery(t *testing.T) {
 	k, v := sendTx()
 	time.Sleep(time.Second)
 	tmResult := new(ctypes.TMResult)
-	_, err := clientURI.Call("abci_query", map[string]interface{}{"query": k}, tmResult)
+	_, err := clientURI.Call("abci_query", map[string]interface{}{"path": "", "data": k, "prove": false}, tmResult)
 	if err != nil {
 		panic(err)
 	}
@@ -144,7 +142,7 @@ func TestURIABCIQuery(t *testing.T) {
 func TestJSONABCIQuery(t *testing.T) {
 	k, v := sendTx()
 	tmResult := new(ctypes.TMResult)
-	_, err := clientJSON.Call("abci_query", []interface{}{k}, tmResult)
+	_, err := clientJSON.Call("abci_query", []interface{}{"", k, false}, tmResult)
 	if err != nil {
 		panic(err)
 	}
@@ -153,18 +151,14 @@ func TestJSONABCIQuery(t *testing.T) {
 
 func testABCIQuery(t *testing.T, statusI interface{}, value []byte) {
 	tmRes := statusI.(*ctypes.TMResult)
-	query := (*tmRes).(*ctypes.ResultABCIQuery)
-	if query.Result.IsErr() {
-		panic(Fmt("Query returned an err: %v", query))
+	resQuery := (*tmRes).(*ctypes.ResultABCIQuery)
+	if !resQuery.Response.Code.IsOK() {
+		panic(Fmt("Query returned an err: %v", resQuery))
 	}
 
-	qResult := new(dummy.QueryResult)
-	if err := wire.ReadJSONBytes(query.Result.Data, qResult); err != nil {
-		t.Fatal(err)
-	}
 	// XXX: specific to value returned by the dummy
-	if qResult.Exists != true {
-		panic(Fmt("Query error. Expected to find 'exists=true'. Got: %v", qResult))
+	if len(resQuery.Response.Value) == 0 {
+		panic(Fmt("Query error. Found no value"))
 	}
 }
 

From 7c15b54cccac574cfe673c473d4edff01c2503ec Mon Sep 17 00:00:00 2001
From: Jae Kwon <jae@tendermint.com>
Date: Sat, 28 Jan 2017 09:11:31 -0800
Subject: [PATCH 014/132] Fix test/app/dummy_test.sh

---
 test/app/dummy_test.sh | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/test/app/dummy_test.sh b/test/app/dummy_test.sh
index 27674222..adfcebb9 100644
--- a/test/app/dummy_test.sh
+++ b/test/app/dummy_test.sh
@@ -30,9 +30,9 @@ echo "... testing query with abci-cli"
 RESPONSE=`abci-cli query \"$KEY\"`
 
 set +e
-A=`echo $RESPONSE | grep '"exists":true'`
+A=`echo $RESPONSE | grep 'log: exists'`
 if [[ $? != 0 ]]; then
-	echo "Failed to find 'exists=true' for $KEY. Response:"
+	echo "Failed to find 'log: exists' for $KEY. Response:"
 	echo "$RESPONSE"
 	exit 1
 fi
@@ -41,9 +41,9 @@ set -e
 # we should not be able to look up the value
 RESPONSE=`abci-cli query \"$VALUE\"`
 set +e
-A=`echo $RESPONSE | grep '"exists":true'`
+A=`echo $RESPONSE | grep 'log: exists'`
 if [[ $? == 0 ]]; then
-	echo "Found 'exists=true' for $VALUE when we should not have. Response:"
+	echo "Found 'log: exists' for $VALUE when we should not have. Response:"
 	echo "$RESPONSE"
 	exit 1
 fi
@@ -53,28 +53,28 @@ set -e
 # test using the /abci_query
 #############################
 
-echo "... testing query with /abci_query"
+echo "... testing query with /abci_query 2"
 
 # we should be able to look up the key
-RESPONSE=`curl -s 127.0.0.1:46657/abci_query?query=$(toHex $KEY)`
-RESPONSE=`echo $RESPONSE | jq .result[1].result.Data | xxd -r -p`
+RESPONSE=`curl -s "127.0.0.1:46657/abci_query?path=\"\"&data=$(toHex $KEY)&prove=false"`
+RESPONSE=`echo $RESPONSE | jq .result[1].response.log`
 
 set +e
-A=`echo $RESPONSE | grep '"exists":true'`
+A=`echo $RESPONSE | grep 'exists'`
 if [[ $? != 0 ]]; then
-	echo "Failed to find 'exists=true' for $KEY. Response:"
+	echo "Failed to find 'exists' for $KEY. Response:"
 	echo "$RESPONSE"
 	exit 1
 fi
 set -e
 
 # we should not be able to look up the value
-RESPONSE=`curl -s 127.0.0.1:46657/abci_query?query=\"$(toHex $VALUE)\"`
-RESPONSE=`echo $RESPONSE | jq .result[1].result.Data | xxd -r -p`
+RESPONSE=`curl -s "127.0.0.1:46657/abci_query?path=\"\"&data=$(toHex $VALUE)&prove=false"`
+RESPONSE=`echo $RESPONSE | jq .result[1].response.log`
 set +e
-A=`echo $RESPONSE | grep '"exists":true'`
+A=`echo $RESPONSE | grep 'exists'`
 if [[ $? == 0 ]]; then
-	echo "Found 'exists=true' for $VALUE when we should not have. Response:"
+	echo "Found 'exists' for $VALUE when we should not have. Response:"
 	echo "$RESPONSE"
 	exit 1
 fi

From 9707cb447158bd307340078d9cc04fc8ef01b08f Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Sat, 28 Jan 2017 19:24:46 -0500
Subject: [PATCH 015/132] README for local testnets [ci skip]

---
 test/README.md     | 10 +++++----
 test/p2p/README.md | 54 ++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 60 insertions(+), 4 deletions(-)
 create mode 100644 test/p2p/README.md

diff --git a/test/README.md b/test/README.md
index 8333e27a..f53dc371 100644
--- a/test/README.md
+++ b/test/README.md
@@ -3,26 +3,28 @@
 The unit tests (ie. the `go test` s) can be run with `make test`.
 The integration tests can be run wtih `make test_integrations`.
 
-Running the integrations test will build a docker container with latest tendermint
+Running the integrations test will build a docker container with local version of tendermint
 and run the following tests in docker containers:
 
 - go tests, with --race
+	- includes test coverage
 - app tests
 	- dummy app over socket
 	- counter app over socket
 	- counter app over grpc
+- persistence tests
+	- crash tendermint at each of many predefined points, restart, and ensure it syncs properly with the app
 - p2p tests
 	- start a local dummy app testnet on a docker network (requires docker version 1.10+)
 	- send a tx on each node and ensure the state root is updated on all of them
+	- crash and restart nodes one at a time and ensure they can sync back up (via fastsync)
+	- crash and restart all nodes at once and ensure they can sync back up
 
 If on a `release-x.x.x` branch, we also run
 
 - `go test` for all our dependency libs (test/test_libs.sh)
 - network_testing - benchmark a mintnet based cloud deploy using netmon
 
-# Coverage
-
-TODO!
 
 
 
diff --git a/test/p2p/README.md b/test/p2p/README.md
new file mode 100644
index 00000000..5cd314d3
--- /dev/null
+++ b/test/p2p/README.md
@@ -0,0 +1,54 @@
+# Tendermint P2P Tests
+
+These scripts facilitate setting up and testing a local testnet using docker containers.
+
+Setup your own local testnet as follows.
+
+For consistency, we assume all commands are run from the Tendermint repository root (ie. $GOPATH/src/github.com/tendermint/tendermint).
+
+First, build the docker image:
+
+```
+docker build -t tendermint_tester -f ./test/docker/Dockerfile .
+```
+
+Now create the docker network:
+
+```
+docker network create --driver bridge --subnet 172.57.0.0/16 my_testnet
+```
+
+This gives us a new network with IP addresses in the rage `172.57.0.0 - 172.57.255.255`.
+Peers on the network can have any IP address in this range. 
+For our four node network, let's pick `172.57.0.101 - 172.57.0.104`.
+Since we use Tendermint's default listening port of 46656, our list of seed nodes will look like:
+
+```
+172.57.0.101:46656,172.57.0.102:46656,172.57.0.103:46656,172.57.0.104:46656
+```
+
+Now we can start up the peers. We already have config files setup in `test/p2p/data/`.
+Let's use a for-loop to start our peers:
+
+```
+for i in $(seq 1 4); do
+	docker run -d \
+	  --net=my_testnet\
+	  --ip="172.57.0.$((100 + $i))" \
+	  --name local_testnet_$i \
+	  --entrypoint tendermint \
+	  -e TMROOT=/go/src/github.com/tendermint/tendermint/test/p2p/data/mach$i/core \
+	  tendermint_tester node --seeds 172.57.0.101:46656,172.57.0.102:46656,172.57.0.103:46656,172.57.0.104:46656 --proxy_app=dummy
+done
+```
+
+If you now run `docker ps`, you'll see your containers!
+
+We can confirm they are making blocks by checking the `/status` message using `curl` and `jq` to pretty print the output json:
+
+```
+curl 172.57.0.101:46657/status | jq . 
+```
+
+
+

From 846e6d3bda13a3548fb22c6ca86b200e92c0374a Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Sun, 29 Jan 2017 13:38:54 -0800
Subject: [PATCH 016/132] fix readme links [ci skip]

---
 README.md | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 28730065..a1ba8fb3 100644
--- a/README.md
+++ b/README.md
@@ -27,9 +27,13 @@ To get started developing applications, see the [application developers guide](h
 
 ## Install
 
+To download pre-built binaries, see our [downloads page](https://tendermint.com/intro/getting-started/download).
+
+To install from source, you should be able to:
+
 `go get -u github.com/tendermint/tendermint/cmd/tendermint`
 
-For more details (or if it fails), see the [install guide](https://tendermint.com/intro/getting-started/install).
+For more details (or if it fails), see the [install guide](https://tendermint.com/docs/guides/install).
 
 ## Contributing
 

From 67ab574e9889c0641ae959296d391e3cadec55e3 Mon Sep 17 00:00:00 2001
From: Jae Kwon <jae@tendermint.com>
Date: Sun, 29 Jan 2017 13:50:04 -0800
Subject: [PATCH 017/132] Cleanup, add stub for VerifyCommitAny

---
 cmd/tendermint/run_node.go |  7 +++++--
 state/state.go             |  5 ++++-
 types/genesis.go           |  8 ++------
 types/validator_set.go     | 26 ++++++++++++++++++++++++++
 4 files changed, 37 insertions(+), 9 deletions(-)

diff --git a/cmd/tendermint/run_node.go b/cmd/tendermint/run_node.go
index 9f13533a..a6645154 100644
--- a/cmd/tendermint/run_node.go
+++ b/cmd/tendermint/run_node.go
@@ -34,9 +34,12 @@ func run_node(config cfg.Config) {
 			if err != nil {
 				Exit(Fmt("Couldn't read GenesisDoc file: %v", err))
 			}
-			genDoc := types.GenesisDocFromJSON(jsonBlob)
+			genDoc, err := types.GenesisDocFromJSON(jsonBlob)
+			if err != nil {
+				Exit(Fmt("Error reading GenesisDoc: %v", err))
+			}
 			if genDoc.ChainID == "" {
-				PanicSanity(Fmt("Genesis doc %v must include non-empty chain_id", genDocFile))
+				Exit(Fmt("Genesis doc %v must include non-empty chain_id", genDocFile))
 			}
 			config.Set("chain_id", genDoc.ChainID)
 		}
diff --git a/state/state.go b/state/state.go
index 455ba409..f4af8ee0 100644
--- a/state/state.go
+++ b/state/state.go
@@ -166,7 +166,10 @@ func MakeGenesisStateFromFile(db dbm.DB, genDocFile string) *State {
 	if err != nil {
 		Exit(Fmt("Couldn't read GenesisDoc file: %v", err))
 	}
-	genDoc := types.GenesisDocFromJSON(genDocJSON)
+	genDoc, err := types.GenesisDocFromJSON(genDocJSON)
+	if err != nil {
+		Exit(Fmt("Error reading GenesisDoc: %v", err))
+	}
 	return MakeGenesisState(db, genDoc)
 }
 
diff --git a/types/genesis.go b/types/genesis.go
index 9c81bc2d..3a039548 100644
--- a/types/genesis.go
+++ b/types/genesis.go
@@ -38,11 +38,7 @@ func (genDoc *GenesisDoc) SaveAs(file string) error {
 //------------------------------------------------------------
 // Make genesis state from file
 
-func GenesisDocFromJSON(jsonBlob []byte) (genState *GenesisDoc) {
-	var err error
-	wire.ReadJSONPtr(&genState, jsonBlob, &err)
-	if err != nil {
-		Exit(Fmt("Couldn't read GenesisDoc: %v", err))
-	}
+func GenesisDocFromJSON(jsonBlob []byte) (genDoc *GenesisDoc, err error) {
+	wire.ReadJSONPtr(&genDoc, jsonBlob, &err)
 	return
 }
diff --git a/types/validator_set.go b/types/validator_set.go
index 3f5a17d9..a3b2838c 100644
--- a/types/validator_set.go
+++ b/types/validator_set.go
@@ -252,6 +252,32 @@ func (valSet *ValidatorSet) VerifyCommit(chainID string, blockID BlockID, height
 	}
 }
 
+// Verify that +2/3 of this set had signed the given signBytes.
+// Unlike VerifyCommit(), this function can verify commits with differeent sets.
+func (valSet *ValidatorSet) VerifyCommitAny(chainID string, blockID BlockID, height int, commit *Commit) error {
+	panic("Not yet implemented")
+	/*
+			Start like:
+
+		FOR_LOOP:
+			for _, val := range vals {
+				if len(precommits) == 0 {
+					break FOR_LOOP
+				}
+				next := precommits[0]
+				switch bytes.Compare(val.Address(), next.ValidatorAddress) {
+				case -1:
+					continue FOR_LOOP
+				case 0:
+					signBytes := tm.SignBytes(next)
+					...
+				case 1:
+					... // error?
+				}
+			}
+	*/
+}
+
 func (valSet *ValidatorSet) String() string {
 	return valSet.StringIndented("")
 }

From c0e7d05b5c68030bd550f671cf9760dd6450e241 Mon Sep 17 00:00:00 2001
From: Anton Kaliaev <anton.kalyaev@gmail.com>
Date: Mon, 30 Jan 2017 11:59:35 +0400
Subject: [PATCH 018/132] rename tmbase image to tendermint

The main reason is people usually expect docker image to have the same
name as the repo. Plus, tendermint is cleaner than tmbase.

tmbase would make sense if we had multiple docker images within
tendermint, but we don't.
---
 DOCKER/README.md  | 12 ++++--------
 test/net/start.sh | 11 +++--------
 2 files changed, 7 insertions(+), 16 deletions(-)

diff --git a/DOCKER/README.md b/DOCKER/README.md
index e765fc52..8e660b98 100644
--- a/DOCKER/README.md
+++ b/DOCKER/README.md
@@ -1,24 +1,20 @@
 # Docker
 
-Tendermint uses docker for deployment of testnets via the [mintnet](github.com/tendermint/mintnet) tool. 
-
-For faster development iterations (ie. to avoid docker builds), 
-the dockerfile just sets up the OS, and tendermint is fetched/installed at runtime.
+Tendermint uses docker for deployment of testnets via the [mintnet](github.com/tendermint/mintnet) tool.
 
 For the deterministic docker builds used in testing, see the [tests directory](https://github.com/tendermint/tendermint/tree/master/test)
 
 # Build and run a docker image and container
 
-These are notes for the dev team. 
+These are notes for the dev team.
 
 ```
 # Build base Docker image
-# Make sure ./run.sh exists.
-docker build -t tendermint/tmbase -f Dockerfile .
+docker build -t "tendermint/tendermint" -t "tendermint/tendermint:0.8.0" -t "tendermint/tendermint:0.8" .
 
 # Log into dockerhub
 docker login
 
 # Push latest build to dockerhub
-docker push tendermint/tmbase
+docker push tendermint/tendermint
 ```
diff --git a/test/net/start.sh b/test/net/start.sh
index 1980280d..9e9ea3a0 100644
--- a/test/net/start.sh
+++ b/test/net/start.sh
@@ -17,18 +17,13 @@ if [[ "$MACH_PREFIX" == "" ]]; then
 fi
 set -u
 
-export TMHEAD=`git rev-parse --abbrev-ref HEAD`
-export TM_IMAGE="tendermint/tmbase"
-
-cd $GOPATH/src/github.com/tendermint/network_testing
+cd "$GOPATH/src/github.com/tendermint/network_testing"
 echo "... running network test $(pwd)"
-bash experiments/exp_throughput.sh $DATACENTER $VALSETSIZE $BLOCKSIZE $TX_SIZE $NTXS $MACH_PREFIX $RESULTSDIR $CLOUD_PROVIDER
+TMHEAD=$(git rev-parse --abbrev-ref HEAD) TM_IMAGE="tendermint/tendermint" bash experiments/exp_throughput.sh $DATACENTER $VALSETSIZE $BLOCKSIZE $TX_SIZE $NTXS $MACH_PREFIX $RESULTSDIR $CLOUD_PROVIDER
 
 # TODO: publish result!
 
 # cleanup
 
 echo "... destroying machines"
-mintnet destroy --machines $MACH_PREFIX[1-$VALSETSIZE] 
-
-
+mintnet destroy --machines $MACH_PREFIX[1-$VALSETSIZE]

From 65dfacac4b614b5e66495cd925a21ccb25e8c08b Mon Sep 17 00:00:00 2001
From: Anton Kaliaev <anton.kalyaev@gmail.com>
Date: Mon, 30 Jan 2017 12:02:24 +0400
Subject: [PATCH 019/132] do we really have to build in order to test?

---
 Makefile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index 424acccb..2b4ea598 100644
--- a/Makefile
+++ b/Makefile
@@ -20,18 +20,18 @@ build_race:
 dist:
 	@BUILD_TAGS='$(BUILD_TAGS)' sh -c "'$(CURDIR)/scripts/dist.sh'"
 
-test: build
+test:
 	@echo "--> Running go test"
 	@go test $(PACKAGES)
 
-test_race: build
+test_race:
 	@echo "--> Running go test --race"
 	@go test -race $(PACKAGES)
 
 test_integrations:
 	@bash ./test/test.sh
 
-test100: build
+test100:
 	@for i in {1..100}; do make test; done
 
 draw_deps:

From 1f527c5013e12cce80c8558f82a6c8b8424e15ef Mon Sep 17 00:00:00 2001
From: Anton Kaliaev <anton.kalyaev@gmail.com>
Date: Mon, 30 Jan 2017 12:03:09 +0400
Subject: [PATCH 020/132] fix go version in Vagrantfile

---
 Vagrantfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Vagrantfile b/Vagrantfile
index 8b65ff79..0acce1ac 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -16,7 +16,7 @@ Vagrant.configure("2") do |config|
     wget -qO- https://get.docker.com/ | sh
     usermod -a -G docker vagrant
 
-    curl -O https://storage.googleapis.com/golang/go1.6.linux-amd64.tar.gz
+    curl -O https://storage.googleapis.com/golang/go1.7.linux-amd64.tar.gz
     tar -xvf go1.7.linux-amd64.tar.gz
     mv go /usr/local
     echo 'export PATH=$PATH:/usr/local/go/bin' >> /home/vagrant/.profile
@@ -28,6 +28,6 @@ Vagrant.configure("2") do |config|
     ln -s /vagrant /home/vagrant/go/src/github.com/tendermint/tendermint
 
     su - vagrant -c 'curl https://glide.sh/get | sh'
-    su - vagrant -c 'cd /vagrant/ && glide install && make test'
+    su - vagrant -c 'cd /vagrant/ && make get_vendor_deps && make test'
   SHELL
 end

From f849e2c414622a8e5ec7a53a47e0fe19ebbf43b5 Mon Sep 17 00:00:00 2001
From: Anton Kaliaev <anton.kalyaev@gmail.com>
Date: Tue, 31 Jan 2017 11:30:54 +0400
Subject: [PATCH 021/132] copy instead of move (Fixes #385)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

```
mv: cannot move ‘./build/dist/windows_386.zip’ to ‘./build/dist/tendermint_0.8.0_windows_386.zip’: Permission denied
```
---
 scripts/dist.sh       |  6 ++++--
 scripts/dist_build.sh | 11 ++++-------
 2 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/scripts/dist.sh b/scripts/dist.sh
index 8da29299..14f0fef1 100755
--- a/scripts/dist.sh
+++ b/scripts/dist.sh
@@ -31,9 +31,11 @@ docker build -t tendermint/tendermint-builder scripts/tendermint-builder/
 docker run --rm -e "BUILD_TAGS=$BUILD_TAGS" -v "$(pwd)":/go/src/github.com/tendermint/tendermint tendermint/tendermint-builder ./scripts/dist_build.sh
 
 # Add "tendermint" and $VERSION prefix to package name.
-for FILENAME in $(find ./build/dist -mindepth 1 -maxdepth 1 -type f); do
+rm -rf ./build/dist
+mkdir -p ./build/dist
+for FILENAME in $(find ./build/pkg -mindepth 1 -maxdepth 1 -type f); do
   FILENAME=$(basename "$FILENAME")
-  mv "./build/dist/${FILENAME}" "./build/dist/tendermint_${VERSION}_${FILENAME}"
+	cp "./build/pkg/${FILENAME}" "./build/dist/tendermint_${VERSION}_${FILENAME}"
 done
 
 # Make the checksums.
diff --git a/scripts/dist_build.sh b/scripts/dist_build.sh
index 947a3202..f75faee5 100755
--- a/scripts/dist_build.sh
+++ b/scripts/dist_build.sh
@@ -20,8 +20,8 @@ XC_OS=${XC_OS:-"solaris darwin freebsd linux windows"}
 
 # Delete the old dir
 echo "==> Removing old directory..."
-rm -rf build/dist/*
-mkdir -p build/dist
+rm -rf build/pkg
+mkdir -p build/pkg
 
 # Make sure build tools are available.
 make tools
@@ -36,13 +36,13 @@ echo "==> Building..."
 		-arch="${XC_ARCH}" \
 		-osarch="!darwin/arm !solaris/amd64 !freebsd/amd64" \
 		-ldflags "-X ${GIT_IMPORT}.GitCommit='${GIT_COMMIT}' -X ${GIT_IMPORT}.GitDescribe='${GIT_DESCRIBE}'" \
-		-output "build/dist/{{.OS}}_{{.Arch}}/tendermint" \
+		-output "build/pkg/{{.OS}}_{{.Arch}}/tendermint" \
 		-tags="${BUILD_TAGS}" \
 		github.com/tendermint/tendermint/cmd/tendermint
 
 # Zip all the files.
 echo "==> Packaging..."
-for PLATFORM in $(find ./build/dist -mindepth 1 -maxdepth 1 -type d); do
+for PLATFORM in $(find ./build/pkg -mindepth 1 -maxdepth 1 -type d); do
 		OSARCH=$(basename "${PLATFORM}")
 		echo "--> ${OSARCH}"
 
@@ -51,7 +51,4 @@ for PLATFORM in $(find ./build/dist -mindepth 1 -maxdepth 1 -type d); do
 		popd >/dev/null 2>&1
 done
 
-# Remove build/dist/{{.OS}}_{{.Arch}} directories.
-rm -rf build/dist/*/
-
 exit 0

From 5fe04409164c00bcd3ae3976ba29cd8777ec0db2 Mon Sep 17 00:00:00 2001
From: Ethan Frey <ethanfrey@users.noreply.github.com>
Date: Thu, 2 Feb 2017 20:10:56 +0100
Subject: [PATCH 022/132] Strawman docs/architecture dir

---
 docs/architecture/ABCI.md   | 16 ++++++++++++++++
 docs/architecture/README.md | 15 +++++++++++++++
 docs/architecture/merkle.md | 17 +++++++++++++++++
 3 files changed, 48 insertions(+)
 create mode 100644 docs/architecture/ABCI.md
 create mode 100644 docs/architecture/README.md
 create mode 100644 docs/architecture/merkle.md

diff --git a/docs/architecture/ABCI.md b/docs/architecture/ABCI.md
new file mode 100644
index 00000000..1aef247a
--- /dev/null
+++ b/docs/architecture/ABCI.md
@@ -0,0 +1,16 @@
+# ABCI
+
+ABCI is an interface between the consensus/blockchain engine known as tendermint, and the application-specific business logic, known as an ABCi app.
+
+The tendermint core should run unchanged for all apps.  Each app can customize it, the supported transactions, queries, even the validator sets and hwo to handle staking / slashing stake. This customization is achieved by implementing the ABCi app to send the proper information to the tendermint engine to perform as directed.
+
+To understand this decision better, think of the design of the tendermint engine.
+
+* A blockchain is simply consensus on a unique global ordering of events.
+* This consensus can efficiently be implemented using BFT and PoS
+* This code can be generalized to easily support a large number of blockchains
+* The block-chain specific code, the interpretation of the individual events, can be implemented by a 3rd party app without touching the consensus engine core
+* Use an efficient, language-agnostic layer to implement this (ABCi)
+
+
+Bucky, please make this doc real.
diff --git a/docs/architecture/README.md b/docs/architecture/README.md
new file mode 100644
index 00000000..89cd14bf
--- /dev/null
+++ b/docs/architecture/README.md
@@ -0,0 +1,15 @@
+# Architecture Decision Records
+
+This is a location to record all high-level architecture decisions in the tendermin project.  Not the implementation details, but the reasoning that happened.  This should be refered to for guidance of the "right way" to extend the application.  And if we notice that the original decisions were lacking, we should have another open discussion, record the new decisions here, and then modify the code to match.
+
+This is like our guide and mentor when Jae and Bucky are offline.... The concept comes from a [blog post](https://product.reverb.com/documenting-architecture-decisions-the-reverb-way-a3563bb24bd0#.78xhdix6t) that resonated among the team when Anton shared it.
+
+Each section of the code can have it's own markdown file in this directory, and please add a link to the readme.
+
+## Sections
+
+* [ABCI](./ABCI.md)
+* [go-merkle / merkleeyes](./merkle.md)
+* basecoin
+* tendermint core (multiple sections)
+* ???
diff --git a/docs/architecture/merkle.md b/docs/architecture/merkle.md
new file mode 100644
index 00000000..fba8c517
--- /dev/null
+++ b/docs/architecture/merkle.md
@@ -0,0 +1,17 @@
+# Merkle data stores
+
+To allow the efficient creation of an ABCi app, tendermint wishes to provide a reference implemention of a key-value store that provides merkle proofs of the data.  These proofs then quickly allow the ABCi app to provide an apphash to the consensus engine, as well as a full proof to any client.
+
+This engine is currently implemented in `go-merkle` with `merkleeyes` providing a language-agnostic binding via ABCi.  It uses `go-db` bindings internally to persist data to leveldb.
+
+What are some of the requirements of this store:
+
+* It must support efficient key-value operations (get/set/delete)
+* It must support persistance.
+* We must only persist complete blocks, so when we come up after a crash we are at the state of block N or N+1, but not in-between these two states.
+* It must allow us to read/write from one uncommited state (working state), while serving other queries from the last commited state. And a way to determine which one to serve for each client.
+* It must allow us to hold references to old state, to allow providing proofs from 20 blocks ago.  We can define some limits as to the maximum time to hold this data.
+* We provide in process binding in the go-lanaguage
+* We provide language-agnostic bindings when running the data store as it's own process.
+
+

From d6965504946705069b202f7f6ed032555529b89b Mon Sep 17 00:00:00 2001
From: Anton Kaliaev <anton.kalyaev@gmail.com>
Date: Fri, 3 Feb 2017 13:01:40 +0400
Subject: [PATCH 023/132] add CONTRIBUTING.md [ci skip] [circle skip]

---
 CONTRIBUTING.md | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 CONTRIBUTING.md

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
new file mode 100644
index 00000000..a55d7a15
--- /dev/null
+++ b/CONTRIBUTING.md
@@ -0,0 +1,16 @@
+# Contributing guidelines
+
+**Thanks for considering making contributions to Tendermint!**
+
+Please follow standard github best practices: fork the repo, **branch from the
+tip of develop**, make some commits, test your code changes with `make test`,
+and submit a pull request to develop.
+
+See the [open issues](https://github.com/tendermint/tendermint/issues) for
+things we need help with!
+
+Please make sure to use `gofmt` before every commit - the easiest way to do
+this is have your editor run it for you upon saving a file.
+
+You can read the full guide [on our
+site](https://tendermint.com/docs/guides/contributing).

From e5ef3e30198ec178fa78dd2e36dc12a4b2b7d20e Mon Sep 17 00:00:00 2001
From: Anton Kaliaev <anton.kalyaev@gmail.com>
Date: Fri, 3 Feb 2017 12:29:18 +0400
Subject: [PATCH 024/132] add ISSUE_TEMPLATE for github [ci skip] [circle skip]

---
 .github/ISSUE_TEMPLATE | 41 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 41 insertions(+)
 create mode 100644 .github/ISSUE_TEMPLATE

diff --git a/.github/ISSUE_TEMPLATE b/.github/ISSUE_TEMPLATE
new file mode 100644
index 00000000..6b0b11f0
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE
@@ -0,0 +1,41 @@
+<!-- Thanks for filing an issue! Before hitting the button, please answer these questions.-->
+
+**Is this a BUG REPORT or FEATURE REQUEST?** (choose one):
+
+<!--
+If this is a BUG REPORT, please:
+  - Fill in as much of the template below as you can.
+
+If this is a FEATURE REQUEST, please:
+  - Describe *in detail* the feature/behavior/change you'd like to see.
+
+In both cases, be ready for followup questions, and please respond in a timely
+manner. We might ask you to provide additional logs and data (tendermint & app)
+in a case of bug.
+-->
+
+**Tendermint version** (use `tendermint version` or `git rev-parse --verify HEAD` if installed from source):
+
+
+**ABCI app** (name for built-in, URL for self-written if it's publicly available):
+
+
+**Merkleeyes version** (use `git rev-parse --verify HEAD`, skip if you don't use it):
+
+
+**Environment**:
+- **OS** (e.g. from /etc/os-release):
+- **Install tools**:
+- **Others**:
+
+
+**What happened**:
+
+
+**What you expected to happen**:
+
+
+**How to reproduce it** (as minimally and precisely as possible):
+
+
+**Anything else do we need to know**:

From b31e5d776404199b5bdc49dc11377b1321b3db9f Mon Sep 17 00:00:00 2001
From: Ethan Frey <ethanfrey@users.noreply.github.com>
Date: Fri, 3 Feb 2017 15:39:35 +0100
Subject: [PATCH 025/132] Starting some thoughts on merkle storage

---
 docs/architecture/README.md      |  1 +
 docs/architecture/merkle-frey.md | 75 ++++++++++++++++++++++++++++++++
 2 files changed, 76 insertions(+)
 create mode 100644 docs/architecture/merkle-frey.md

diff --git a/docs/architecture/README.md b/docs/architecture/README.md
index 89cd14bf..dd97a5b7 100644
--- a/docs/architecture/README.md
+++ b/docs/architecture/README.md
@@ -10,6 +10,7 @@ Each section of the code can have it's own markdown file in this directory, and
 
 * [ABCI](./ABCI.md)
 * [go-merkle / merkleeyes](./merkle.md)
+* [Frey's thoughts on the data store](./merkle-frey.md)
 * basecoin
 * tendermint core (multiple sections)
 * ???
diff --git a/docs/architecture/merkle-frey.md b/docs/architecture/merkle-frey.md
new file mode 100644
index 00000000..e6306111
--- /dev/null
+++ b/docs/architecture/merkle-frey.md
@@ -0,0 +1,75 @@
+# Merkle data stores - Frey's proposal
+
+To allow the efficient creation of an ABCi app, tendermint wishes to provide a reference implemention of a key-value store that provides merkle proofs of the data.  These proofs then quickly allow the ABCi app to provide an apphash to the consensus engine, as well as a full proof to any client.
+
+This is equivalent to building a database, and I would propose designing it from the API first, then looking how to implement this (or make an adaptor from the API to existing implementations). Once we agree on the functionality and the interface, we can implement the API bindings, and then work on building adaptors to existince merkle-ized data stores, or modifying the stores to support this interface
+
+## TL;DR
+
+* **Interface**
+  * This interface should be domain-specific - ie. designed just for this use case
+  * It should present a simple go interface for embedding the data store in-process
+  * It should create a gRPC/protobuf API for calling from any client
+  * It should provide and maintain client adaptors from our in-process interface to gRPC client calls for at least golang and java (maybe more languages?)
+  * It should provide and maintain server adaptors from our gRPC calls to the in-process interface for golang at least (unless there is another server we wish to support)
+* **State**
+  * There are two concepts of state, "committed state" and "working state"
+  * The working state is only accessible from the ABCi app, allows writing, but does not need to support proofs.
+  * When we commit the "working state", it becomes a new "commmited state" and has an immutible root hash, provides proofs, and can be exposed to external clients.
+* **Transactions**
+  * The database always allows creating a read-only transaction at the last "committed state", this transaction can serve read queries and proofs.
+  * The database maintains all data to serve these read transactions until they are closed by the client (or time out).  This allows the client(s) to determine how much old info is needed
+  * The database can only support *maximal* one writable transaction at a time.  This makes it easy to enforce serializability, and attempting to start a second writeable transaction may trigger a panic.
+* **Functionality**
+  * It must support efficient key-value operations (get/set/delete)
+  * It must support returning merkle proofs for any "committed state"
+  * It should support range queries on subsets of the key space if possible (ie. if the db doesn't hash keys)
+  * It should also support listening to changes to a desired key via pub-sub or similar method, so I can quickly notify you on a change to your balance without constant polling.
+  * It may support other db-specific query types as an extension to this interface, as long as all specified actions maintain their meaning.
+* **Persistence**
+  * It must support atomic persistance upon committing a new block.  That is, upon crash recovery, the state is guaranteed to represent the state at the end of a complete block (along with a note of which height it was).
+  * It must delay deletion of old data as long as there are open read-only transactions refering to it, thus we must maintain some sort of WAL to keep track of pending cleanup.
+  * When a transaction is closed, or when we recover from a crash, it should clean up all no longer needed data to avoid memory/storage leaks.
+* **Security and Auth**
+  * If we allow connections over gRPC, we must consider this issues and allow both encyption (SSL), and some basic auth rules to provent undesired access to the DB
+  * This is client-specific and does not need to be supported in the in-process, embeded version.
+
+## Details
+
+Here we go more in-depth in each of the sections, explaining the reasoning and more details on the desired behavior. This document is only the high-level architecture and should support multiple implementations.  When building out a specific implementation, a similar document should be provided for that repo, showing how it implements these concepts, and details about memory usage, storage, efficiency, etc.
+
+### Interface
+
+I will start with a simple go interface to illustrate the in-process interface.  Once there is agreement on how this looks, we can work out the gRPC bindings to support calling out of process.
+
+The use-case of allowing out-of-process calls is very powerful.  It we allow the ABCi app to maintain the only writable connections, we can guarantee that all transactions are only processed through the tendermint consensus engine.  We could then allow multiple "web server" machines "read-only" access and scale out the database reads, assuming the consensus engine, ABCi logic, and public key cryptography is more the bottleneck than the database.
+
+**TODO**
+
+### State
+
+**TODO**
+
+### Transactions
+
+**TODO**
+
+Points to add:
+
+* Subtransactions
+* Rollback and commit
+* Multiplexing various transactions over on stateless gRPC connection
+* Ping-ing and timeouts
+
+
+### Functionality
+
+**TODO**
+
+### Persistence
+
+**TODO**
+
+### Security
+
+**TODO**

From 5c2ead741cc0aa562fd05fdbee9499d6e59918f6 Mon Sep 17 00:00:00 2001
From: Ethan Frey <ethanfrey@users.noreply.github.com>
Date: Fri, 3 Feb 2017 18:54:19 +0100
Subject: [PATCH 026/132] Lots more text on transactions and interface

---
 docs/architecture/merkle-frey.md | 166 ++++++++++++++++++++++++++-----
 1 file changed, 142 insertions(+), 24 deletions(-)

diff --git a/docs/architecture/merkle-frey.md b/docs/architecture/merkle-frey.md
index e6306111..88993901 100644
--- a/docs/architecture/merkle-frey.md
+++ b/docs/architecture/merkle-frey.md
@@ -1,17 +1,15 @@
 # Merkle data stores - Frey's proposal
 
-To allow the efficient creation of an ABCi app, tendermint wishes to provide a reference implemention of a key-value store that provides merkle proofs of the data.  These proofs then quickly allow the ABCi app to provide an apphash to the consensus engine, as well as a full proof to any client.
-
-This is equivalent to building a database, and I would propose designing it from the API first, then looking how to implement this (or make an adaptor from the API to existing implementations). Once we agree on the functionality and the interface, we can implement the API bindings, and then work on building adaptors to existince merkle-ized data stores, or modifying the stores to support this interface
-
 ## TL;DR
 
-* **Interface**
-  * This interface should be domain-specific - ie. designed just for this use case
-  * It should present a simple go interface for embedding the data store in-process
-  * It should create a gRPC/protobuf API for calling from any client
-  * It should provide and maintain client adaptors from our in-process interface to gRPC client calls for at least golang and java (maybe more languages?)
-  * It should provide and maintain server adaptors from our gRPC calls to the in-process interface for golang at least (unless there is another server we wish to support)
+To allow the efficient creation of an ABCi app, tendermint wishes to provide a reference implemention of a key-value store that provides merkle proofs of the data.  These proofs then quickly allow the ABCi app to provide an apphash to the consensus engine, as well as a full proof to any client.
+
+This is equivalent to building a database, and I would propose designing it from the API first, then looking how to implement this (or make an adaptor from the API to existing implementations). Once we agree on the functionality and the interface, we can implement the API bindings, and then work on building adaptors to existince merkle-ized data stores, or modifying the stores to support this interface.
+
+We need to consider the API (both in-process and over the network), language bindings, maintaining handles to old state (and garbage collecting), persistence, security, providing merkle proofs, and general key-value store operations. To stay consistent with the blockchains "single global order of operations", this data store should only allow one connection at a time to have write access.
+
+## Overview
+
 * **State**
   * There are two concepts of state, "committed state" and "working state"
   * The working state is only accessible from the ABCi app, allows writing, but does not need to support proofs.
@@ -26,6 +24,12 @@ This is equivalent to building a database, and I would propose designing it from
   * It should support range queries on subsets of the key space if possible (ie. if the db doesn't hash keys)
   * It should also support listening to changes to a desired key via pub-sub or similar method, so I can quickly notify you on a change to your balance without constant polling.
   * It may support other db-specific query types as an extension to this interface, as long as all specified actions maintain their meaning.
+* **Interface**
+  * This interface should be domain-specific - ie. designed just for this use case
+  * It should present a simple go interface for embedding the data store in-process
+  * It should create a gRPC/protobuf API for calling from any client
+  * It should provide and maintain client adaptors from our in-process interface to gRPC client calls for at least golang and java (maybe more languages?)
+  * It should provide and maintain server adaptors from our gRPC calls to the in-process interface for golang at least (unless there is another server we wish to support)
 * **Persistence**
   * It must support atomic persistance upon committing a new block.  That is, upon crash recovery, the state is guaranteed to represent the state at the end of a complete block (along with a note of which height it was).
   * It must delay deletion of old data as long as there are open read-only transactions refering to it, thus we must maintain some sort of WAL to keep track of pending cleanup.
@@ -38,34 +42,148 @@ This is equivalent to building a database, and I would propose designing it from
 
 Here we go more in-depth in each of the sections, explaining the reasoning and more details on the desired behavior. This document is only the high-level architecture and should support multiple implementations.  When building out a specific implementation, a similar document should be provided for that repo, showing how it implements these concepts, and details about memory usage, storage, efficiency, etc.
 
-### Interface
-
-I will start with a simple go interface to illustrate the in-process interface.  Once there is agreement on how this looks, we can work out the gRPC bindings to support calling out of process.
-
-The use-case of allowing out-of-process calls is very powerful.  It we allow the ABCi app to maintain the only writable connections, we can guarantee that all transactions are only processed through the tendermint consensus engine.  We could then allow multiple "web server" machines "read-only" access and scale out the database reads, assuming the consensus engine, ABCi logic, and public key cryptography is more the bottleneck than the database.
-
-**TODO**
 
 ### State
 
-**TODO**
+The current ABCi interface avoids this question a bit and that has brought confusion.  If I use `merkleeyes` to store data, which state is returned from `Query`?  The current "working" state, which I would like to refer to in my ABCi application?  Or the last committed state, which I would like to return to a client's query?  Or an old state, which I may select based on height?
+
+Right now, `merkleeyes` implements `Query` like a normal ABCi app and only returns committed state, which has lead to problems and confusion.  Thus, we need to be explicit about which state we want to view.  Each viewer can then specify which state it wants to view.  This allows the app to query the workign state in DeliverTx, but the committed state in Query.
+
+We can easily provide two global references for "last committed" and "current working" states.  However, if we want to also allow querying of older commits... then we need some way to keep track of which ones are still in use, so we can garbage collect the unneeded ones. There is a non-trivial overhead in holdign references to all past states, but also a hardcoded solution (hold onto the last 5 commits) may not support all clients.  We should let the client define this somehow.
 
 ### Transactions
 
-**TODO**
+Transactions (in the typical database sense) are a clean and estabilished solution to this issue.  We can look at the [isolations levels](https://en.wikipedia.org/wiki/Isolation_(database_systems)#Serializable) which attempt to provide us things like "repeatable reads".  That means if we open a transaction, and query some data 100 times while other processes are writing to the db, we get the same result each time.  This transaction has a reference to its own local state from the time the transaction started. (We are refering to the highest isolation levels here, which correlate well this the blockchain use case).
 
-Points to add:
+If we implement a read-only transaction as a reference to state at the time of creation of that transaction, we can then hold these references to various snapshots, one per block that we are interested, and allow the client to multiplex queries and proofs from these various blocks.
 
-* Subtransactions
-* Rollback and commit
-* Multiplexing various transactions over on stateless gRPC connection
-* Ping-ing and timeouts
+If we continue using these concepts (which have informed 30+ years of server side design), we can add a few nice features to our write transactions. The first of which is `Rollback` and `Commit`.  That means all the changes we make in this transaction have no effect on the database until they are committed.  And until they are committed, we can always abort if we detect an anomaly, returning to the last committed state with a rollback.
 
+There is also a nice extension to this available on some database servers, basically, "nested" transactions or "savepoints".  This means that within one transaction, you can open a subtransaction/savepoint and continue work.  Later you have the option to commit or rollback all work since the savepoint/subtransaction.  And then continue with the main transaction.
+
+If you don't understand why this is useful, look at how basecoin needs to [hold cached state for AppTx](https://github.com/tendermint/basecoin/blob/master/state/execution.go#L126-L149), meaning that it rolls back all modifications if the AppTx returns an error. This was implemented as a wrapper in basecoin, but it is a reasonable thing to support in the DB interface itself (especially since the implementation becomes quite non-trivial as soon as you support range queries).
+
+To give a bit more reference to this concept in practice, read about [Savepoints in Postgesql](https://www.postgresql.org/docs/current/static/tutorial-transactions.html) ([reference](https://www.postgresql.org/docs/current/static/sql-savepoint.html)) or [Nesting transactions in SQL Server](http://dba-presents.com/index.php/databases/sql-server/43-nesting-transactions-and-save-transaction-command) (TL;DR: scroll to the bottom, section "Real nesting transactions with SAVE TRANSACTION")
 
 ### Functionality
 
 **TODO**
 
+### Go Interface
+
+I will start with a simple go interface to illustrate the in-process interface.  Once there is agreement on how this looks, we can work out the gRPC bindings to support calling out of process. These interfaces are not finalized code, but I think the demonstrate the concepts better than text and provide a strawman to get feedback.
+
+```
+// DB represents the committed state of a merkle-ized key-value store
+type DB interface {
+  // Snapshot returns a reference to last commited state to use for
+  // providing proofs, you must close it at the end to garbage collect
+  // the historical state we hold on to to make these proofs
+  Snapshot() Prover
+
+  // Start a transaction - only way to change state
+  // This will return an error if there is an open Transaction
+  Begin() (Transaction, error)
+
+  // These callbacks are triggered when the Transaction is Committed
+  // to the DB.  They can be used to eg. notify clients via websockets when
+  // their account balance changes.
+  AddListener(key []byte, listener Listener)
+  RemoveListener(listener Listener)
+}
+
+// DBReader represents a read-only connection to a snapshot of the db
+type DBReader interface {
+  // Queries on my local view
+  Has(key []byte) (bool, error)
+  Get(key []byte) (Model, error)
+  GetRange(start, end []byte, ascending bool, limit int) ([]Model, error)
+  Closer
+}
+
+// Prover is an interface that lets one query for Proofs, holding the
+// data at a specific location in memory
+type Prover interface {
+  DBReader
+
+  // Hash is the AppHash (RootHash) for this block
+  Hash() (hash []byte)
+
+  // Prove returns the data along with a merkle Proof
+  // Model and Proof are nil if not found
+  Prove(key []byte) (Model, Proof, error)
+}
+
+// Transaction is a set of state changes to the DB to be applied atomically.
+// There can only be one open transaction at a time, which may only have
+// maximum one subtransaction at a time.
+// In short, at any time, there is exactly one object that can write to the
+// DB, and we can use Subtransactions to group operations and roll them back
+// together (kind of like `types.KVCache` from basecoin)
+type Transaction interface {
+  DBReader
+
+  // Change the state - will raise error immediately if this Transaction
+  // is not holding the exclusive write lock
+  Set(model Model) (err error)
+  Remove(key []byte) (removed bool, err error)
+
+  // Subtransaction starts a new subtransaction, rollback will not affect the
+  // parent.  Only on Commit are the changes applied to this transaction.
+  // While the subtransaction exists, no write allowed on the parent.
+  // (You must Commit or Rollback the child to continue)
+  Subtransaction() Transaction
+
+  // Commit this transaction (or subtransaction), the parent reference is
+  // now updated.
+  // This only updates persistant store if the top level transaction commits
+  // (You may have any number of nested sub transactions)
+  Commit() error
+
+  // Rollback ends the transaction and throw away all transaction-local state,
+  // allowing the tree to prune those elements.
+  // The parent transaction now recovers the write lock.
+  Rollback()
+}
+
+// Listener registers callbacks on changes to the data store
+type Listener interface {
+  OnSet(key, value, oldValue []byte)
+  OnRemove(key, oldValue []byte)
+}
+
+// Proof represents a merkle proof for a key
+type Proof interface {
+  RootHash() []byte
+  Verify(key, value, root []byte) bool
+}
+
+type Model interface {
+  Key() []byte
+  Value() []byte
+}
+
+// Closer releases the reference to this state, allowing us to garbage collect
+// Make sure to call it before discarding.
+type Closer interface {
+  Close()
+}
+```
+
+### Remote Interface
+
+The use-case of allowing out-of-process calls is very powerful.  Not just to provide a powerful merkle-ready data store to non-go applications.
+
+It we allow the ABCi app to maintain the only writable connections, we can guarantee that all transactions are only processed through the tendermint consensus engine.  We could then allow multiple "web server" machines "read-only" access and scale out the database reads, assuming the consensus engine, ABCi logic, and public key cryptography is more the bottleneck than the database. We could even place the consensus engine, ABCi app, and data store on one machine, connected with unix sockets for security, and expose a tcp/ssl interface for reading the data, to scale out query processing over multiple machines.
+
+But returning our focus directly to the ABCi app (which is the most important use case).  An app may well want to maintain 100 or 1000 snapshots of different heights to allow people to easily query many proofs at a given height without race conditions (very important for IBC, ask Jae).  Thus, we should not require a separate TCP connection for each height, as this gets quite awkward with so many connections. Also, if we want to use gRPC, we should consider the connections potentially transient (although they are more efficient with keep-alive).
+
+Thus, the wire encoding of a transaction or a snapshot should simply return a unique id.  All methods on a `Prover` or `Transaction` over the wire can send this id along with the arguments for the method call.  And we just need a hash map on the server to map this id to a state.
+
+The only negative of not requiring a persistent tcp connection for each snapshot is there is no auto-detection if the client crashes without explicitly closing the connections.  Thus, I would suggest adding a `Ping` thread in the gRPC interface which keeps the Snapshot alive.  If no ping is received within a server-defined time, it may automatically close those transactions.  And if we consider a client with 500 snapshots that needs to ping each every 10 seconds, that is a lot of overhead, so we should design the ping to accept a list of IDs for the client and update them all.  Or associate all snapshots with a clientID and then just send the clientID in the ping.  (Please add other ideas on how to detect client crashes without persistent connections).
+
+To encourage adoption, we should provide a nice client that uses this gRPC interface (like we do with ABCi).  For go, the client may have the exact same interface as the in-process version, just that the error call may return network errors, not just illegal operations.  We should also add a client with a clean API for Java, since that seems to be popular among app developers in the current tendermint community.  Other bindings as we see the need in the server space.
+
 ### Persistence
 
 **TODO**

From 1bf5ae9c8d3332926a215165ec102336f8d794ce Mon Sep 17 00:00:00 2001
From: Ethan Frey <ethanfrey@users.noreply.github.com>
Date: Fri, 3 Feb 2017 19:44:08 +0100
Subject: [PATCH 027/132] Finish main argument on merkle data store
 architecture

---
 docs/architecture/merkle-frey.md | 60 ++++++++++++++++++++++----------
 1 file changed, 42 insertions(+), 18 deletions(-)

diff --git a/docs/architecture/merkle-frey.md b/docs/architecture/merkle-frey.md
index 88993901..46b90795 100644
--- a/docs/architecture/merkle-frey.md
+++ b/docs/architecture/merkle-frey.md
@@ -2,9 +2,9 @@
 
 ## TL;DR
 
-To allow the efficient creation of an ABCi app, tendermint wishes to provide a reference implemention of a key-value store that provides merkle proofs of the data.  These proofs then quickly allow the ABCi app to provide an apphash to the consensus engine, as well as a full proof to any client.
+To allow the efficient creation of an ABCi app, tendermint wishes to provide a reference implementation of a key-value store that provides merkle proofs of the data.  These proofs then quickly allow the ABCi app to provide an app hash to the consensus engine, as well as a full proof to any client.
 
-This is equivalent to building a database, and I would propose designing it from the API first, then looking how to implement this (or make an adaptor from the API to existing implementations). Once we agree on the functionality and the interface, we can implement the API bindings, and then work on building adaptors to existince merkle-ized data stores, or modifying the stores to support this interface.
+This is equivalent to building a database, and I would propose designing it from the API first, then looking how to implement this (or make an adapter from the API to existing implementations). Once we agree on the functionality and the interface, we can implement the API bindings, and then work on building adapters to existence merkle-ized data stores, or modifying the stores to support this interface.
 
 We need to consider the API (both in-process and over the network), language bindings, maintaining handles to old state (and garbage collecting), persistence, security, providing merkle proofs, and general key-value store operations. To stay consistent with the blockchains "single global order of operations", this data store should only allow one connection at a time to have write access.
 
@@ -13,11 +13,11 @@ We need to consider the API (both in-process and over the network), language bin
 * **State**
   * There are two concepts of state, "committed state" and "working state"
   * The working state is only accessible from the ABCi app, allows writing, but does not need to support proofs.
-  * When we commit the "working state", it becomes a new "commmited state" and has an immutible root hash, provides proofs, and can be exposed to external clients.
+  * When we commit the "working state", it becomes a new "committed state" and has an immutable root hash, provides proofs, and can be exposed to external clients.
 * **Transactions**
   * The database always allows creating a read-only transaction at the last "committed state", this transaction can serve read queries and proofs.
   * The database maintains all data to serve these read transactions until they are closed by the client (or time out).  This allows the client(s) to determine how much old info is needed
-  * The database can only support *maximal* one writable transaction at a time.  This makes it easy to enforce serializability, and attempting to start a second writeable transaction may trigger a panic.
+  * The database can only support *maximal* one writable transaction at a time.  This makes it easy to enforce serializability, and attempting to start a second writable transaction may trigger a panic.
 * **Functionality**
   * It must support efficient key-value operations (get/set/delete)
   * It must support returning merkle proofs for any "committed state"
@@ -28,15 +28,15 @@ We need to consider the API (both in-process and over the network), language bin
   * This interface should be domain-specific - ie. designed just for this use case
   * It should present a simple go interface for embedding the data store in-process
   * It should create a gRPC/protobuf API for calling from any client
-  * It should provide and maintain client adaptors from our in-process interface to gRPC client calls for at least golang and java (maybe more languages?)
-  * It should provide and maintain server adaptors from our gRPC calls to the in-process interface for golang at least (unless there is another server we wish to support)
+  * It should provide and maintain client adapters from our in-process interface to gRPC client calls for at least golang and Java (maybe more languages?)
+  * It should provide and maintain server adapters from our gRPC calls to the in-process interface for golang at least (unless there is another server we wish to support)
 * **Persistence**
-  * It must support atomic persistance upon committing a new block.  That is, upon crash recovery, the state is guaranteed to represent the state at the end of a complete block (along with a note of which height it was).
-  * It must delay deletion of old data as long as there are open read-only transactions refering to it, thus we must maintain some sort of WAL to keep track of pending cleanup.
+  * It must support atomic persistence upon committing a new block.  That is, upon crash recovery, the state is guaranteed to represent the state at the end of a complete block (along with a note of which height it was).
+  * It must delay deletion of old data as long as there are open read-only transactions referring to it, thus we must maintain some sort of WAL to keep track of pending cleanup.
   * When a transaction is closed, or when we recover from a crash, it should clean up all no longer needed data to avoid memory/storage leaks.
 * **Security and Auth**
-  * If we allow connections over gRPC, we must consider this issues and allow both encyption (SSL), and some basic auth rules to provent undesired access to the DB
-  * This is client-specific and does not need to be supported in the in-process, embeded version.
+  * If we allow connections over gRPC, we must consider this issues and allow both encryption (SSL), and some basic auth rules to prevent undesired access to the DB
+  * This is client-specific and does not need to be supported in the in-process, embedded version.
 
 ## Details
 
@@ -47,13 +47,13 @@ Here we go more in-depth in each of the sections, explaining the reasoning and m
 
 The current ABCi interface avoids this question a bit and that has brought confusion.  If I use `merkleeyes` to store data, which state is returned from `Query`?  The current "working" state, which I would like to refer to in my ABCi application?  Or the last committed state, which I would like to return to a client's query?  Or an old state, which I may select based on height?
 
-Right now, `merkleeyes` implements `Query` like a normal ABCi app and only returns committed state, which has lead to problems and confusion.  Thus, we need to be explicit about which state we want to view.  Each viewer can then specify which state it wants to view.  This allows the app to query the workign state in DeliverTx, but the committed state in Query.
+Right now, `merkleeyes` implements `Query` like a normal ABCi app and only returns committed state, which has lead to problems and confusion.  Thus, we need to be explicit about which state we want to view.  Each viewer can then specify which state it wants to view.  This allows the app to query the working state in DeliverTx, but the committed state in Query.
 
-We can easily provide two global references for "last committed" and "current working" states.  However, if we want to also allow querying of older commits... then we need some way to keep track of which ones are still in use, so we can garbage collect the unneeded ones. There is a non-trivial overhead in holdign references to all past states, but also a hardcoded solution (hold onto the last 5 commits) may not support all clients.  We should let the client define this somehow.
+We can easily provide two global references for "last committed" and "current working" states.  However, if we want to also allow querying of older commits... then we need some way to keep track of which ones are still in use, so we can garbage collect the unneeded ones. There is a non-trivial overhead in holding references to all past states, but also a hard-coded solution (hold onto the last 5 commits) may not support all clients.  We should let the client define this somehow.
 
 ### Transactions
 
-Transactions (in the typical database sense) are a clean and estabilished solution to this issue.  We can look at the [isolations levels](https://en.wikipedia.org/wiki/Isolation_(database_systems)#Serializable) which attempt to provide us things like "repeatable reads".  That means if we open a transaction, and query some data 100 times while other processes are writing to the db, we get the same result each time.  This transaction has a reference to its own local state from the time the transaction started. (We are refering to the highest isolation levels here, which correlate well this the blockchain use case).
+Transactions (in the typical database sense) are a clean and established solution to this issue.  We can look at the [isolations levels](https://en.wikipedia.org/wiki/Isolation_(database_systems)#Serializable) which attempt to provide us things like "repeatable reads".  That means if we open a transaction, and query some data 100 times while other processes are writing to the db, we get the same result each time.  This transaction has a reference to its own local state from the time the transaction started. (We are referring to the highest isolation levels here, which correlate well this the blockchain use case).
 
 If we implement a read-only transaction as a reference to state at the time of creation of that transaction, we can then hold these references to various snapshots, one per block that we are interested, and allow the client to multiplex queries and proofs from these various blocks.
 
@@ -63,11 +63,21 @@ There is also a nice extension to this available on some database servers, basic
 
 If you don't understand why this is useful, look at how basecoin needs to [hold cached state for AppTx](https://github.com/tendermint/basecoin/blob/master/state/execution.go#L126-L149), meaning that it rolls back all modifications if the AppTx returns an error. This was implemented as a wrapper in basecoin, but it is a reasonable thing to support in the DB interface itself (especially since the implementation becomes quite non-trivial as soon as you support range queries).
 
-To give a bit more reference to this concept in practice, read about [Savepoints in Postgesql](https://www.postgresql.org/docs/current/static/tutorial-transactions.html) ([reference](https://www.postgresql.org/docs/current/static/sql-savepoint.html)) or [Nesting transactions in SQL Server](http://dba-presents.com/index.php/databases/sql-server/43-nesting-transactions-and-save-transaction-command) (TL;DR: scroll to the bottom, section "Real nesting transactions with SAVE TRANSACTION")
+To give a bit more reference to this concept in practice, read about [Savepoints in Postgresql](https://www.postgresql.org/docs/current/static/tutorial-transactions.html) ([reference](https://www.postgresql.org/docs/current/static/sql-savepoint.html)) or [Nesting transactions in SQL Server](http://dba-presents.com/index.php/databases/sql-server/43-nesting-transactions-and-save-transaction-command) (TL;DR: scroll to the bottom, section "Real nesting transactions with SAVE TRANSACTION")
 
 ### Functionality
 
-**TODO**
+Merkle trees work with key-value pairs, so we should most importantly focus on the basic Key-Value operations.  That is `Get`, `Set`, and `Remove`. We also need to return a merkle proof for any key, along with a root hash of the tree for committing state to the blockchain. This is just the basic merkle-tree stuff.
+
+If it is possible with the implementation, it is nice to provide access to Range Queries.  That is, return all values where the key is between X and Y.  If you construct your keys wisely, it is possible to store lists (1:N) relations this way.  Eg, storing blog posts and the key is blog:`poster_id`:`sequence`, then I could search for all blog posts by a given `poster_id`, or even return just posts 10-19 from the given poster.
+
+The construction of a tree that supports range queries was one of the [design decisions of go-merkle](https://github.com/tendermint/go-merkle/blob/master/README.md).  It is also kind of possible with [ethereum's patricia trie](https://github.com/ethereum/wiki/wiki/Patricia-Tree) as long as the key is less than 32 bytes.
+
+In addition to range queries, there is one more nice feature that we could add to our data store - listening to events. Depending on your context, this is "reactive programming", "event emitters", "notifications", etc... But the basic concept is that a client can listen for all changes to a given key (or set of keys), and receive a notification when this happens. This is very important to avoid [repeated polling and wasted queries](http://resthooks.org/) when a client simply wants to [detect changes](https://www.rethinkdb.com/blog/realtime-web/).
+
+If the database provides access to some "listener" functionality, the app can choose to expose this to the external client via websockets, web hooks, http2 push events, android push notifications, etc, etc etc.... But if we want to support modern client functionality, let's add support for this reactive paradigm in our DB interface.
+
+**TODO** support for more advanced backends, eg. Bolt....
 
 ### Go Interface
 
@@ -76,7 +86,7 @@ I will start with a simple go interface to illustrate the in-process interface.
 ```
 // DB represents the committed state of a merkle-ized key-value store
 type DB interface {
-  // Snapshot returns a reference to last commited state to use for
+  // Snapshot returns a reference to last committed state to use for
   // providing proofs, you must close it at the end to garbage collect
   // the historical state we hold on to to make these proofs
   Snapshot() Prover
@@ -186,8 +196,22 @@ To encourage adoption, we should provide a nice client that uses this gRPC inter
 
 ### Persistence
 
-**TODO**
+Any data store worth it's name should not lose all data on a crash.  Even [redis provides some persistence](https://redis.io/topics/persistence) these days. Ideally, if the system crashes and restarts, it should have the data at the last block N that was committed.  If the system crash during the commit of block N+1, then the recovered state should either be block N or completely committed block N+1, but no partial state between the two.  Basically, the commit must be an atomic operation (even if updating 100's of records).
+
+To avoid a lot of headaches ourselves, we can use an existing data store, such as leveldb, which provides `WriteBatch` to group all operations.
+
+The other issue is cleaning up old state.  We cannot delete any information from our persistent store, as long as any snapshot holds a reference to it (or else we get some panics when the data we query is not there).  So, we need to store the outstanding deletions that we can perform when the snapshot is `Close`d.  In addition, we must consider the case that the data store crashes with open snapshots.  Thus, the info on outstanding deletions must also be persisted somewhere.  Something like a "delete-behind log" (the opposite of a "write ahead log").
+
+This is not a concern of the generic interface, but each implementation should take care to handle this well to avoid accumulation of unused references in the data store and eventual data bloat.
 
 ### Security
 
-**TODO**
+When allowing access out-of-process, we should provide different mechanisms to secure it.  The first is the choice of binding to a local unix socket or a tcp port.  The second is the optional use of ssl to encrypt the connection (very important over tcp).  The third is authentication to control access to the database.
+
+We may also want to consider the case of two server connections with different permissions, eg. a local unix socket that allows write access with no more credentials, and a public TCP connection with ssl and authentication that only provides read-only access.
+
+The use of ssl is quite easy in go, we just need to generate and sign a certificate, so it is nice to be able to disable it for dev machines, but it is very important for production.
+
+For authentication, let me sketch out a minimal solution. The server could just have a simple config file with key/bcrypt(password) pairs along with read/write permission level, and read that upon startup.  The client must provide a username and password in the HTTP headers when making the original HTTPS gRPC connection.
+
+This is super minimal to provide some protection. Things like LDAP, OAuth and single-sign on seem overkill and even potential security holes.  Maybe there is another solution somewhere in the middle.

From 36b5d86eda16ef17f1e00865ba646615515fa8c4 Mon Sep 17 00:00:00 2001
From: Ethan Frey <ethanfrey@users.noreply.github.com>
Date: Sun, 5 Feb 2017 16:29:37 +0100
Subject: [PATCH 028/132] [ci skip] [circle skip] Document research on backing
 dbs

---
 docs/architecture/merkle-frey.md | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/docs/architecture/merkle-frey.md b/docs/architecture/merkle-frey.md
index 46b90795..aca4ce32 100644
--- a/docs/architecture/merkle-frey.md
+++ b/docs/architecture/merkle-frey.md
@@ -204,6 +204,29 @@ The other issue is cleaning up old state.  We cannot delete any information from
 
 This is not a concern of the generic interface, but each implementation should take care to handle this well to avoid accumulation of unused references in the data store and eventual data bloat.
 
+#### Backing stores
+
+It is way outside the scope of this project to build our own database that is capable of efficiently storing the data, provide multiple read-only snapshots at once, and save it atomically. The best approach seems to select an existing database (best a simple one) that provides this functionality and build upon it, much like the current `go-merkle` implementation builds upon `leveldb`.  After some research here are winners and losers:
+
+**Winners**
+
+* Leveldb - [provides consistent snapshots](https://ayende.com/blog/161705/reviewing-leveldb-part-xiii-smile-and-here-is-your-snapshot), and [provides tooling for building ACID compliance](http://codeofrob.com/entries/writing-a-transaction-manager-on-top-of-leveldb.html)
+  * Note there are at least two solid implementations available in go - [goleveldb](https://github.com/syndtr/goleveldb) - a pure go implementation, and [levigo](https://github.com/jmhodges/levigo) - a go wrapper around leveldb.
+  * Goleveldb is much easier to compile and cross-compile (not requiring cgo), while levigo (or cleveldb) seems to provide a significant performance boosts (but I had trouble even running benchmarks)
+* PostgreSQL - fully supports these ACID semantics if you call `SET TRANSACTION ISOLATION LEVEL SERIALIZABLE` at the beginning of a transaction (tested)
+  * This may be total overkill unless we also want to make use of other features, like storing data in multiple columns with secondary indexes.
+  * Trillian can show an example of [how to store a merkle tree in sql](https://github.com/google/trillian/blob/master/storage/mysql/tree_storage.go)
+
+**Losers**
+
+* Bolt - open [read-only snapshots can block writing](https://github.com/boltdb/bolt/issues/378)
+* Mongo - [barely even supports atomic operations](https://docs.mongodb.com/manual/core/write-operations-atomicity/), much less multiple snapshots
+
+**To investigate**
+
+* [Trillian](https://github.com/google/trillian) - has a [persistent merkle tree interface](https://github.com/google/trillian/blob/master/storage/tree_storage.go) along with [backend storage with mysql](https://github.com/google/trillian/blob/master/storage/mysql/tree_storage.go), good inspiration for our design if not directly using it
+* [Moss](https://github.com/couchbase/moss) - another key-value store in go, seems similar to leveldb, maybe compare with performance tests?
+
 ### Security
 
 When allowing access out-of-process, we should provide different mechanisms to secure it.  The first is the choice of binding to a local unix socket or a tcp port.  The second is the optional use of ssl to encrypt the connection (very important over tcp).  The third is authentication to control access to the database.

From bf64dd21fdb193e54d8addaaaa2ecf7ac371de8c Mon Sep 17 00:00:00 2001
From: Anton Kaliaev <anton.kalyaev@gmail.com>
Date: Sun, 5 Feb 2017 17:58:08 +0400
Subject: [PATCH 029/132] [docker] move to alpine base image

813Mb => 29.5Mb
---
 DOCKER/Dockerfile | 31 ++++++++++++++++---------------
 1 file changed, 16 insertions(+), 15 deletions(-)

diff --git a/DOCKER/Dockerfile b/DOCKER/Dockerfile
index e640d4c0..947301e7 100644
--- a/DOCKER/Dockerfile
+++ b/DOCKER/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.7.4
+FROM alpine:3.5
 
 # This is the release of tendermint to pull in.
 ENV TM_VERSION 0.8.0
@@ -12,30 +12,31 @@ ENV DATA_ROOT /tendermint
 ENV TMROOT $DATA_ROOT
 
 # Set user right away for determinism
-RUN groupadd -r tmuser && \
-    useradd -r -s /bin/false -g tmuser tmuser
+RUN addgroup tmuser && \
+    adduser -S -G tmuser tmuser
 
 # Create directory for persistence and give our user ownership
 RUN mkdir -p $DATA_ROOT && \
     chown -R tmuser:tmuser $DATA_ROOT
 
-# TODO replace with downloading a binary (this will allow us to replace golang
-# base container with alpine|jessie - 600MB vs 50MB)
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-      git && \
-    rm -rf /var/lib/apt/lists/*
-RUN mkdir -p $GOPATH/src/github.com/tendermint/tendermint && \
-    cd $GOPATH/src/github.com/tendermint/tendermint && \
-    git clone https://github.com/tendermint/tendermint.git . && \
-    git fetch && \
-    git reset --hard v$TM_VERSION && \
-    make install
+# jq and curl used for extracting `pub_key` from private validator while
+# deploying tendermint with Kubernetes. It is nice to have bash so the users
+# could execute bash commands.
+RUN apk add --no-cache bash curl jq
+
+RUN apk add --no-cache openssl && \
+    wget https://s3-us-west-2.amazonaws.com/tendermint/${TM_VERSION}/tendermint_linux_amd64.zip && \
+    echo "83f6bd52055ebc93434a68263c6666a4de41e0e543d0b5a06ad461262c460f4c  tendermint_linux_amd64.zip" | sha256sum -c && \
+    unzip -d /bin tendermint_linux_amd64.zip && \
+    apk del openssl && \
+    rm -f tendermint_linux_amd64.zip
 
 # Expose the data directory as a volume since there's mutable state in there
 VOLUME $DATA_ROOT
 
+# p2p port
 EXPOSE 46656
+# rpc port
 EXPOSE 46657
 
 ENTRYPOINT ["tendermint"]

From d5d7286cb6c07c77bc95796b78dde2c0829721b0 Mon Sep 17 00:00:00 2001
From: Anton Kaliaev <anton.kalyaev@gmail.com>
Date: Fri, 3 Feb 2017 17:20:52 +0400
Subject: [PATCH 030/132] improve circleci config

- update docker machine version to 0.9.0
- save coverage.txt as artifact
- save docker logs as artifact
- test if coverage.txt exists
- 2>&1 redirect stderr to stdout in tests
---
 Makefile           |  6 +++---
 circle.yml         | 33 ++++++++++++++-------------------
 test/test.sh       |  2 +-
 test/test_cover.sh |  2 +-
 test/test_libs.sh  |  2 +-
 5 files changed, 20 insertions(+), 25 deletions(-)

diff --git a/Makefile b/Makefile
index 2b4ea598..dc22959c 100644
--- a/Makefile
+++ b/Makefile
@@ -22,11 +22,11 @@ dist:
 
 test:
 	@echo "--> Running go test"
-	@go test $(PACKAGES)
+	@go test -v $(PACKAGES)
 
 test_race:
 	@echo "--> Running go test --race"
-	@go test -race $(PACKAGES)
+	@go test -v -race $(PACKAGES)
 
 test_integrations:
 	@bash ./test/test.sh
@@ -48,7 +48,7 @@ get_deps:
 	@go get -d $(PACKAGES)
 	@go list -f '{{join .TestImports "\n"}}' ./... | \
 		grep -v /vendor/ | sort | uniq | \
-		xargs go get
+		xargs go get -d
 
 get_vendor_deps: tools
 	@rm -rf vendor/
diff --git a/circle.yml b/circle.yml
index d6f7abeb..e77b578b 100644
--- a/circle.yml
+++ b/circle.yml
@@ -1,38 +1,33 @@
+---
 machine:
   environment:
     MACH_PREFIX: tendermint-test-mach
-    GOPATH: /home/ubuntu/.go_workspace
-    REPO: $GOPATH/src/github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME
     DOCKER_VERSION: 1.10.0
-    DOCKER_MACHINE_VERSION: 0.6.0
+    DOCKER_MACHINE_VERSION: 0.9.0
+    GOPATH: "$HOME/.go_project"
+    PROJECT_PARENT_PATH: "$GOPATH/src/github.com/$CIRCLE_PROJECT_USERNAME"
+    PROJECT_PATH: "$PROJECT_PARENT_PATH/$CIRCLE_PROJECT_REPONAME"
   hosts:
-    circlehost: 127.0.0.1
     localhost: 127.0.0.1
 
-checkout:
-  post:
-    - rm -rf $REPO
-    - mkdir -p $HOME/.go_workspace/src/github.com/$CIRCLE_PROJECT_USERNAME
-    - mv $HOME/$CIRCLE_PROJECT_REPONAME $REPO
-    # - git submodule sync
-    # - git submodule update --init # use submodules
-
 dependencies:
   override:
-    - echo $MACH_PREFIX $GOPATH $REPO $DOCKER_VERSION $DOCKER_MACHINE_VERSION
     - curl -sSL https://s3.amazonaws.com/circle-downloads/install-circleci-docker.sh | sudo bash -s -- $DOCKER_VERSION
-    - sudo curl -sSL -o /usr/bin/docker-machine https://github.com/docker/machine/releases/download/v$DOCKER_MACHINE_VERSION/docker-machine-linux-x86_64; sudo chmod 0755 /usr/bin/docker-machine
     - sudo start docker
+    - sudo curl -sSL -o /usr/bin/docker-machine "https://github.com/docker/machine/releases/download/v$DOCKER_MACHINE_VERSION/docker-machine-`uname -s`-`uname -m`"; sudo chmod 0755 /usr/bin/docker-machine
+    - mkdir -p "$PROJECT_PARENT_PATH"
+    - ln -sf "$HOME/$CIRCLE_PROJECT_REPONAME/" "$PROJECT_PATH"
+  post:
     - go version
     - docker version
     - docker-machine version
 
 test:
   override:
-    - "cd $REPO && set -o pipefail && make test_integrations | tee ~/test_integrations.log":
+    - cd "$PROJECT_PATH" && set -o pipefail && make test_integrations 2>&1 | tee test_integrations.log:
         timeout: 1800
-    - "cp ~/test_integrations.log $CIRCLE_ARTIFACTS"
   post:
-    - "cd $REPO && bash <(curl -s https://codecov.io/bash)"
-
-
+    - cd "$PROJECT_PATH" && mv test_integrations.log "${CIRCLE_ARTIFACTS}"
+    - cd "$PROJECT_PATH" && bash <(curl -s https://codecov.io/bash) -f coverage.txt
+    - cd "$PROJECT_PATH" && [[ -f coverage.txt ]] && mv coverage.txt "${CIRCLE_ARTIFACTS}"
+    - sudo cp /var/log/upstart/docker.log "${CIRCLE_ARTIFACTS}"
diff --git a/test/test.sh b/test/test.sh
index 0e779a04..9e42235e 100644
--- a/test/test.sh
+++ b/test/test.sh
@@ -29,6 +29,6 @@ if [[ $(echo "$BRANCH" | grep "release-") != "" ]]; then
 	echo ""
 	echo "TODO: run network tests"
 	#echo "* branch $BRANCH; running mintnet/netmon throughput benchmark"
-	# TODO: replace mintnet 
+	# TODO: replace mintnet
 	#bash test/net/test.sh
 fi
diff --git a/test/test_cover.sh b/test/test_cover.sh
index 49992d63..60c84284 100644
--- a/test/test_cover.sh
+++ b/test/test_cover.sh
@@ -5,7 +5,7 @@ PKGS=$(go list github.com/tendermint/tendermint/... | grep -v /vendor/)
 set -e
 echo "mode: atomic" > coverage.txt
 for pkg in ${PKGS[@]}; do
-	go test -timeout 30m -race -coverprofile=profile.out -covermode=atomic $pkg
+	go test -v -timeout 30m -race -coverprofile=profile.out -covermode=atomic $pkg
 	if [ -f profile.out ]; then
 		tail -n +2 profile.out >> coverage.txt;
 		rm profile.out
diff --git a/test/test_libs.sh b/test/test_libs.sh
index 8692f5de..28ec0798 100644
--- a/test/test_libs.sh
+++ b/test/test_libs.sh
@@ -21,7 +21,7 @@ for lib in "${LIBS_GO_TEST[@]}"; do
 	bash scripts/glide/checkout.sh $GLIDE $lib
 
 	echo "Testing $lib ..."
-	go test --race github.com/tendermint/$lib/...
+	go test -v --race github.com/tendermint/$lib/...
 	if [[ "$?" != 0 ]]; then
 		echo "FAIL"
 		exit 1

From d4b3dde8537bc62a6ea3c9830ac3669118141a84 Mon Sep 17 00:00:00 2001
From: Ethan Frey <ethanfrey@users.noreply.github.com>
Date: Wed, 8 Feb 2017 19:28:25 +0100
Subject: [PATCH 031/132] make install now uses vendored dependencies

---
 Makefile | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index dc22959c..81edfd32 100644
--- a/Makefile
+++ b/Makefile
@@ -5,9 +5,9 @@ PACKAGES=$(shell go list ./... | grep -v '/vendor/')
 BUILD_TAGS?=tendermint
 TMROOT = $${TMROOT:-$$HOME/.tendermint}
 
-all: get_deps install test
+all: install test
 
-install: get_deps
+install: get_vendor_deps
 	@go install ./cmd/tendermint
 
 build:
@@ -50,12 +50,12 @@ get_deps:
 		grep -v /vendor/ | sort | uniq | \
 		xargs go get -d
 
-get_vendor_deps: tools
+get_vendor_deps: ensure_tools
 	@rm -rf vendor/
 	@echo "--> Running glide install"
 	@glide install
 
-update_deps:
+update_deps: tools
 	@echo "--> Updating dependencies"
 	@go get -d -u ./...
 
@@ -66,4 +66,8 @@ revision:
 tools:
 	go get -u -v $(GOTOOLS)
 
+ensure_tools:
+	go get $(GOTOOLS)
+
+
 .PHONY: install build build_race dist test test_race test_integrations test100 draw_deps list_deps get_deps get_vendor_deps update_deps revision tools

From ca21b6be935d71d0b6d5b0275333f3f0f446272b Mon Sep 17 00:00:00 2001
From: Anton Kaliaev <anton.kalyaev@gmail.com>
Date: Wed, 8 Feb 2017 22:56:17 +0400
Subject: [PATCH 032/132] update Vagrantfile

* glide now gets installed as a part of make get_vendor_deps
* rm go .tar.gz file
* autoremove garbage
* set LC_ALL to en_US.UTF-8
---
 Vagrantfile | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/Vagrantfile b/Vagrantfile
index 0acce1ac..aba80736 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -15,19 +15,22 @@ Vagrant.configure("2") do |config|
 
     wget -qO- https://get.docker.com/ | sh
     usermod -a -G docker vagrant
+    apt-get autoremove -y
 
     curl -O https://storage.googleapis.com/golang/go1.7.linux-amd64.tar.gz
     tar -xvf go1.7.linux-amd64.tar.gz
-    mv go /usr/local
-    echo 'export PATH=$PATH:/usr/local/go/bin' >> /home/vagrant/.profile
+    mv -f go /usr/local
+    rm -f go1.7.linux-amd64.tar.gz
     mkdir -p /home/vagrant/go/bin
     chown -R vagrant:vagrant /home/vagrant/go
-    echo 'export GOPATH=/home/vagrant/go' >> /home/vagrant/.profile
+    echo 'export PATH=$PATH:/usr/local/go/bin:/home/vagrant/go/bin' >> /home/vagrant/.bash_profile
+    echo 'export GOPATH=/home/vagrant/go' >> /home/vagrant/.bash_profile
+
+    echo 'export LC_ALL=en_US.UTF-8' >> /home/vagrant/.bash_profile
 
     mkdir -p /home/vagrant/go/src/github.com/tendermint
     ln -s /vagrant /home/vagrant/go/src/github.com/tendermint/tendermint
 
-    su - vagrant -c 'curl https://glide.sh/get | sh'
-    su - vagrant -c 'cd /vagrant/ && make get_vendor_deps && make test'
+    su - vagrant -c 'cd /home/vagrant/go/src/github.com/tendermint/tendermint && make get_vendor_deps'
   SHELL
 end

From 4896364952f5a24ef64525d1da1af082b184d3e0 Mon Sep 17 00:00:00 2001
From: Anton Kaliaev <anton.kalyaev@gmail.com>
Date: Wed, 8 Feb 2017 23:20:25 +0400
Subject: [PATCH 033/132] [cli] cleanup gen_validator output (Fixes #396)

so we can pipe result to a file:

```
tendermint gen_validator > example.json
```

before this we had to cut first 3 lines:

```
tendermint gen_validator | sed 1,3d > example.json
```
---
 cmd/tendermint/gen_validator.go | 12 ++----------
 1 file changed, 2 insertions(+), 10 deletions(-)

diff --git a/cmd/tendermint/gen_validator.go b/cmd/tendermint/gen_validator.go
index 37f0e299..0fc0e166 100644
--- a/cmd/tendermint/gen_validator.go
+++ b/cmd/tendermint/gen_validator.go
@@ -8,16 +8,8 @@ import (
 )
 
 func gen_validator() {
-
 	privValidator := types.GenPrivValidator()
 	privValidatorJSONBytes := wire.JSONBytesPretty(privValidator)
-	fmt.Printf(`Generated a new validator!
-Paste the following JSON into your %v file
-
-%v
-
-`,
-		config.GetString("priv_validator_file"),
-		string(privValidatorJSONBytes),
-	)
+	fmt.Printf(`%v
+`, string(privValidatorJSONBytes))
 }

From 7d91d4300b4e185d488e126ed161de3e4209eebe Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Mon, 13 Feb 2017 21:07:26 -0500
Subject: [PATCH 034/132] update glide

---
 consensus/mempool_test.go | 11 ++---------
 glide.lock                | 40 +++++++++++++++++++--------------------
 proxy/client.go           |  5 ++---
 3 files changed, 23 insertions(+), 33 deletions(-)

diff --git a/consensus/mempool_test.go b/consensus/mempool_test.go
index 4dd82a3f..e55dcf3e 100644
--- a/consensus/mempool_test.go
+++ b/consensus/mempool_test.go
@@ -114,6 +114,8 @@ func TestRmBadTx(t *testing.T) {
 
 // CounterApplication that maintains a mempool state and resets it upon commit
 type CounterApplication struct {
+	abci.BaseApplication
+
 	txCount        int
 	mempoolTxCount int
 }
@@ -126,10 +128,6 @@ func (app *CounterApplication) Info() abci.ResponseInfo {
 	return abci.ResponseInfo{Data: Fmt("txs:%v", app.txCount)}
 }
 
-func (app *CounterApplication) SetOption(key string, value string) (log string) {
-	return ""
-}
-
 func (app *CounterApplication) DeliverTx(tx []byte) abci.Result {
 	return runTx(tx, &app.txCount)
 }
@@ -160,8 +158,3 @@ func (app *CounterApplication) Commit() abci.Result {
 		return abci.NewResultOK(hash, "")
 	}
 }
-
-func (app *CounterApplication) Query(reqQuery abci.RequestQuery) (resQuery abci.ResponseQuery) {
-	resQuery.Log = "Query is not supported"
-	return
-}
diff --git a/glide.lock b/glide.lock
index a3cc726e..4fa40e9a 100644
--- a/glide.lock
+++ b/glide.lock
@@ -1,12 +1,10 @@
 hash: e283934fbbd221161d53a918db9e49db8c5be2b8929592b05ffe6b72c2ef0ab1
-updated: 2017-01-27T22:32:42.896956819-08:00
+updated: 2017-02-13T20:54:31.541721996-05:00
 imports:
 - name: github.com/btcsuite/btcd
-  version: afec1bd1245a4a19e6dfe1306974b733e7cbb9b8
+  version: d06c0bb181529331be8f8d9350288c420d9e60e4
   subpackages:
   - btcec
-- name: github.com/btcsuite/fastsha256
-  version: 637e656429416087660c84436a2a035d69d54e2e
 - name: github.com/BurntSushi/toml
   version: 99064174e013895bbd9b025c31100bd1d9b590ca
 - name: github.com/ebuchman/fail-test
@@ -14,27 +12,27 @@ imports:
 - name: github.com/go-stack/stack
   version: 100eb0c0a9c5b306ca2fb4f165df21d80ada4b82
 - name: github.com/gogo/protobuf
-  version: f9114dace7bd920b32f943b3c73fafbcbab2bf31
+  version: 9c5b7bafbfccf2b40d274e0496f3a09418a87af4
   subpackages:
   - proto
 - name: github.com/golang/protobuf
-  version: 1f49d83d9aa00e6ce4fc8258c71cc7786aec968a
+  version: 8ee79997227bf9b34611aee7946ae64735e6fd93
   subpackages:
   - proto
 - name: github.com/golang/snappy
-  version: d9eb7a3d35ec988b8585d4a0068e462c27d28380
+  version: 7db9049039a047d955fe8c19b83c8ff5abd765c7
 - name: github.com/gorilla/websocket
-  version: 3ab3a8b8831546bd18fd182c20687ca853b2bb13
+  version: 9bc973af0682dc73a22553a08bfe00ee6255f56f
 - name: github.com/jmhodges/levigo
   version: c42d9e0ca023e2198120196f842701bb4c55d7b9
 - name: github.com/mattn/go-colorable
-  version: ed8eb9e318d7a84ce5915b495b7d35e0cfe7b5a8
+  version: 5411d3eea5978e6cdc258b30de592b60df6aba96
 - name: github.com/mattn/go-isatty
-  version: 66b8e73f3f5cda9f96b69efd03dd3d7fc4a5cdb8
+  version: 281032e84ae07510239465db46bf442aa44b953a
 - name: github.com/spf13/pflag
-  version: 5ccb023bc27df288a957c5e994cd44fd19619465
+  version: 9ff6c6923cfffbcd502984b8e0c80539a94968b7
 - name: github.com/syndtr/goleveldb
-  version: 6ae1797c0b42b9323fc27ff7dcf568df88f2f33d
+  version: 23851d93a2292dcc56e71a18ec9e0624d84a0f65
   subpackages:
   - leveldb
   - leveldb/cache
@@ -49,12 +47,11 @@ imports:
   - leveldb/table
   - leveldb/util
 - name: github.com/tendermint/abci
-  version: 8df0bc3a40ccad0d2be10e33c62c404e65c92502
+  version: 2a3f6384950041b098d89e66b93c580ecd6d32ef
   subpackages:
   - client
   - example/counter
   - example/dummy
-  - example/nil
   - server
   - types
 - name: github.com/tendermint/ed25519
@@ -85,7 +82,7 @@ imports:
 - name: github.com/tendermint/go-logger
   version: cefb3a45c0bf3c493a04e9bcd9b1540528be59f2
 - name: github.com/tendermint/go-merkle
-  version: 653cb1f631528351ddbc359b994eb0c96f0341cd
+  version: 9f20e80cb188d07860caa70196dd7700659ec4a4
 - name: github.com/tendermint/go-p2p
   version: 67c9086b7458eb45b1970483decd01cd744c477a
   subpackages:
@@ -97,13 +94,13 @@ imports:
   - server
   - types
 - name: github.com/tendermint/go-wire
-  version: 2f3b7aafe21c80b19b6ee3210ecb3e3d07c7a471
+  version: 3216ec9d47bbdf8d4fc27d22169ea86a6688bc15
 - name: github.com/tendermint/log15
-  version: 9545b249b3aacafa97f79e0838b02b274adc6f5f
+  version: ae0f3d6450da9eac7074b439c8e1c3cabf0d5ce6
   subpackages:
   - term
 - name: golang.org/x/crypto
-  version: aa2481cbfe81d911eb62b642b7a6b5ec58bbea71
+  version: 453249f01cfeb54c3d549ddb75ff152ca243f9d8
   subpackages:
   - curve25519
   - nacl/box
@@ -114,20 +111,21 @@ imports:
   - ripemd160
   - salsa20/salsa
 - name: golang.org/x/net
-  version: cfe3c2a7525b50c3d707256e371c90938cfef98a
+  version: 61557ac0112b576429a0df080e1c2cef5dfbb642
   subpackages:
   - context
   - http2
   - http2/hpack
+  - idna
   - internal/timeseries
   - lex/httplex
   - trace
 - name: golang.org/x/sys
-  version: 30de6d19a3bd89a5f38ae4028e23aaa5582648af
+  version: e24f485414aeafb646f6fca458b0bf869c0880a1
   subpackages:
   - unix
 - name: google.golang.org/grpc
-  version: 50955793b0183f9de69bd78e2ec251cf20aab121
+  version: 4acc9108c18937815db3306a66990869d22aa980
   subpackages:
   - codes
   - credentials
diff --git a/proxy/client.go b/proxy/client.go
index d5c9ff7b..ea905198 100644
--- a/proxy/client.go
+++ b/proxy/client.go
@@ -4,11 +4,10 @@ import (
 	"fmt"
 	"sync"
 
-	cfg "github.com/tendermint/go-config"
 	abcicli "github.com/tendermint/abci/client"
 	"github.com/tendermint/abci/example/dummy"
-	nilapp "github.com/tendermint/abci/example/nil"
 	"github.com/tendermint/abci/types"
+	cfg "github.com/tendermint/go-config"
 )
 
 // NewABCIClient returns newly connected client
@@ -74,7 +73,7 @@ func DefaultClientCreator(config cfg.Config) ClientCreator {
 	case "persistent_dummy":
 		return NewLocalClientCreator(dummy.NewPersistentDummyApplication(config.GetString("db_dir")))
 	case "nilapp":
-		return NewLocalClientCreator(nilapp.NewNilApplication())
+		return NewLocalClientCreator(types.NewBaseApplication())
 	default:
 		mustConnect := false // loop retrying
 		return NewRemoteClientCreator(addr, transport, mustConnect)

From ac971c1a19582cf9546408c1c4f0dc0bef63142f Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Sun, 5 Feb 2017 15:19:03 -0500
Subject: [PATCH 035/132] some docs fixes

---
 docs/architecture/ABCI.md   | 2 +-
 docs/architecture/merkle.md | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/architecture/ABCI.md b/docs/architecture/ABCI.md
index 1aef247a..04d62c1c 100644
--- a/docs/architecture/ABCI.md
+++ b/docs/architecture/ABCI.md
@@ -2,7 +2,7 @@
 
 ABCI is an interface between the consensus/blockchain engine known as tendermint, and the application-specific business logic, known as an ABCi app.
 
-The tendermint core should run unchanged for all apps.  Each app can customize it, the supported transactions, queries, even the validator sets and hwo to handle staking / slashing stake. This customization is achieved by implementing the ABCi app to send the proper information to the tendermint engine to perform as directed.
+The tendermint core should run unchanged for all apps.  Each app can customize it, the supported transactions, queries, even the validator sets and how to handle staking / slashing stake. This customization is achieved by implementing the ABCi app to send the proper information to the tendermint engine to perform as directed.
 
 To understand this decision better, think of the design of the tendermint engine.
 
diff --git a/docs/architecture/merkle.md b/docs/architecture/merkle.md
index fba8c517..72998db8 100644
--- a/docs/architecture/merkle.md
+++ b/docs/architecture/merkle.md
@@ -11,7 +11,7 @@ What are some of the requirements of this store:
 * We must only persist complete blocks, so when we come up after a crash we are at the state of block N or N+1, but not in-between these two states.
 * It must allow us to read/write from one uncommited state (working state), while serving other queries from the last commited state. And a way to determine which one to serve for each client.
 * It must allow us to hold references to old state, to allow providing proofs from 20 blocks ago.  We can define some limits as to the maximum time to hold this data.
-* We provide in process binding in the go-lanaguage
+* We provide in process binding in Go
 * We provide language-agnostic bindings when running the data store as it's own process.
 
 

From 1275458c3f0e5da47b27215f8d8800856be6e93e Mon Sep 17 00:00:00 2001
From: Anton Kaliaev <anton.kalyaev@gmail.com>
Date: Tue, 14 Feb 2017 18:53:22 +0400
Subject: [PATCH 036/132] collect and add docker logs to CircleCI artifacts
 (Refs #387)

How: 1) we start syslog docker container 2) all other containers use
syslog logging driver to ship their logs to that container
---
 .gitignore       |  1 +
 circle.yml       |  4 ++--
 test/p2p/peer.sh |  4 ++++
 test/test.sh     | 45 +++++++++++++++++++++++++++++++--------------
 4 files changed, 38 insertions(+), 16 deletions(-)

diff --git a/.gitignore b/.gitignore
index 4d8d3395..c4694b3e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,4 +12,5 @@ remote_dump
 vendor
 .vagrant
 test/p2p/data/
+test/logs
 .glide
diff --git a/circle.yml b/circle.yml
index e77b578b..50ffbd01 100644
--- a/circle.yml
+++ b/circle.yml
@@ -29,5 +29,5 @@ test:
   post:
     - cd "$PROJECT_PATH" && mv test_integrations.log "${CIRCLE_ARTIFACTS}"
     - cd "$PROJECT_PATH" && bash <(curl -s https://codecov.io/bash) -f coverage.txt
-    - cd "$PROJECT_PATH" && [[ -f coverage.txt ]] && mv coverage.txt "${CIRCLE_ARTIFACTS}"
-    - sudo cp /var/log/upstart/docker.log "${CIRCLE_ARTIFACTS}"
+    - cd "$PROJECT_PATH" && mv coverage.txt "${CIRCLE_ARTIFACTS}"
+    - cd "$PROJECT_PATH" && cp test/logs/messages "${CIRCLE_ARTIFACTS}/docker_logs.txt"
diff --git a/test/p2p/peer.sh b/test/p2p/peer.sh
index 76314f58..19586b12 100644
--- a/test/p2p/peer.sh
+++ b/test/p2p/peer.sh
@@ -21,4 +21,8 @@ docker run -d \
   --name local_testnet_$ID \
   --entrypoint tendermint \
   -e TMROOT=/go/src/github.com/tendermint/tendermint/test/p2p/data/mach$ID/core \
+	--log-driver=syslog \
+	--log-opt syslog-address=udp://127.0.0.1:5514 \
+	--log-opt syslog-facility=daemon \
+	--log-opt tag="{{.Name}}" \
   $DOCKER_IMAGE node $SEEDS --proxy_app=$APP_PROXY
diff --git a/test/test.sh b/test/test.sh
index 9e42235e..2abafc32 100644
--- a/test/test.sh
+++ b/test/test.sh
@@ -1,17 +1,30 @@
 #! /bin/bash
 set -eu
 
-# Top Level Testing Script
-# See the github.com/tendermint/tendermint/test/README.md
+# Get the directory of where this script is.
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done
+DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
 
-echo ""
-echo "* building docker image"
-bash ./test/docker/build.sh
+LOGS_DIR="$DIR/logs"
+echo
+echo "* [$(date +"%T")] cleaning up $LOGS_DIR"
+rm -rf "$LOGS_DIR"
+mkdir -p "$LOGS_DIR"
 
-echo ""
-echo "* running go tests and app tests in docker container"
+echo
+echo "* [$(date +"%T")] starting rsyslog container"
+docker rm -f rsyslog || true
+docker run -d -v "$LOGS_DIR:/var/log/" -p 127.0.0.1:5514:514/udp --name rsyslog voxxit/rsyslog
+
+echo
+echo "* [$(date +"%T")] building docker image"
+bash "$DIR/docker/build.sh"
+
+echo
+echo "* [$(date +"%T")] running go tests and app tests in docker container"
 # sometimes its helpful to mount the local test folder
-# -v $GOPATH/src/github.com/tendermint/tendermint/test:/go/src/github.com/tendermint/tendermint/test
+# -v $DIR:/go/src/github.com/tendermint/tendermint/test
 docker run --name run_test -t tester bash test/run_test.sh
 
 # copy the coverage results out of docker container
@@ -19,16 +32,20 @@ docker cp run_test:/go/src/github.com/tendermint/tendermint/coverage.txt .
 
 # test basic network connectivity
 # by starting a local testnet and checking peers connect and make blocks
-echo ""
-echo "* running p2p tests on a local docker network"
-bash test/p2p/test.sh tester
+echo
+echo "* [$(date +"%T")] running p2p tests on a local docker network"
+bash "$DIR/p2p/test.sh" tester
 
 # only run the cloud benchmark for releases
-BRANCH=`git rev-parse --abbrev-ref HEAD`
+BRANCH=$(git rev-parse --abbrev-ref HEAD)
 if [[ $(echo "$BRANCH" | grep "release-") != "" ]]; then
-	echo ""
+	echo
 	echo "TODO: run network tests"
 	#echo "* branch $BRANCH; running mintnet/netmon throughput benchmark"
 	# TODO: replace mintnet
-	#bash test/net/test.sh
+	#bash "$DIR/net/test.sh"
 fi
+
+echo
+echo "* [$(date +"%T")] stopping rsyslog container"
+docker rm -f rsyslog

From b5bb1657d29341428de425ca2548187f3c9e0e54 Mon Sep 17 00:00:00 2001
From: Anton Kaliaev <anton.kalyaev@gmail.com>
Date: Tue, 14 Feb 2017 19:09:03 +0400
Subject: [PATCH 037/132] increase memory quota for Vagrant users

`make test_integrations` requires > 2Gi of memory
---
 Vagrantfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Vagrantfile b/Vagrantfile
index 0acce1ac..fbbdc70f 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -5,7 +5,7 @@ Vagrant.configure("2") do |config|
   config.vm.box = "ubuntu/trusty64"
 
   config.vm.provider "virtualbox" do |v|
-    v.memory = 2048
+    v.memory = 3072
     v.cpus = 2
   end
 

From ad354c4c483d7211a9ab2940833227af92815d0f Mon Sep 17 00:00:00 2001
From: Anton Kaliaev <anton.kalyaev@gmail.com>
Date: Tue, 14 Feb 2017 19:23:28 +0400
Subject: [PATCH 038/132] stop containers to avoid futher errors

Error:

```
Failed to remove container (rsyslog): Error response from daemon: Unable to remove filesystem
```
---
 test/p2p/test.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/test/p2p/test.sh b/test/p2p/test.sh
index 0f29aa19..4d91aaba 100644
--- a/test/p2p/test.sh
+++ b/test/p2p/test.sh
@@ -31,3 +31,6 @@ bash test/p2p/fast_sync/test.sh $DOCKER_IMAGE $NETWORK_NAME $N $PROXY_APP
 
 # test killing all peers
 bash test/p2p/kill_all/test.sh $DOCKER_IMAGE $NETWORK_NAME $N 3
+
+# stop the existing testnet and remove local network
+bash test/p2p/local_testnet_stop.sh $NETWORK_NAME $N

From 86f85525dd2d5e5e3ddf4fc7114c4c9a1e926b69 Mon Sep 17 00:00:00 2001
From: Anton Kaliaev <anton.kalyaev@gmail.com>
Date: Tue, 14 Feb 2017 20:30:15 +0400
Subject: [PATCH 039/132] dockerfile for develop branch

---
 DOCKER/Dockerfile.develop | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)
 create mode 100644 DOCKER/Dockerfile.develop

diff --git a/DOCKER/Dockerfile.develop b/DOCKER/Dockerfile.develop
new file mode 100644
index 00000000..adbe6c04
--- /dev/null
+++ b/DOCKER/Dockerfile.develop
@@ -0,0 +1,35 @@
+FROM alpine:3.5
+
+ENV DATA_ROOT /tendermint
+ENV TMROOT $DATA_ROOT
+
+RUN addgroup tmuser && \
+    adduser -S -G tmuser tmuser
+
+RUN mkdir -p $DATA_ROOT && \
+    chown -R tmuser:tmuser $DATA_ROOT
+
+RUN apk add --no-cache bash curl jq
+
+ENV GOPATH /go
+ENV PATH "$PATH:/go/bin"
+RUN mkdir -p /go/src/github.com/tendermint/tendermint && \
+    apk add --no-cache go build-base git && \
+    cd /go/src/github.com/tendermint/tendermint && \
+    git clone https://github.com/tendermint/tendermint . && \
+    git checkout develop && \
+    make get_vendor_deps && \
+    make install && \
+    glide cc && \ # 183Mb => 75Mb
+    cd - && \
+    rm -rf /go/src/github.com/tendermint/tendermint && \
+    apk del go build-base git
+
+VOLUME $DATA_ROOT
+
+EXPOSE 46656
+EXPOSE 46657
+
+ENTRYPOINT ["tendermint"]
+
+CMD ["node", "--moniker=`hostname`", "--proxy_app=dummy"]

From d0ca0cb308d6793b8d1ab48c4a4f29b7763b7951 Mon Sep 17 00:00:00 2001
From: Anton Kaliaev <anton.kalyaev@gmail.com>
Date: Tue, 14 Feb 2017 20:40:23 +0400
Subject: [PATCH 040/132] otherwise we end up with unmount errors due to busy
 disk

```
local_testnet_1
Failed to remove container (local_testnet_1): Error response from daemon: Unable to remove filesystem for 74e1104c705d4f21137bbe9e6710f33511948fc04d26475b482d1314fe70d537: remove /var/lib/docker/containers/74e1104c705d4f21137bbe9e6710f33511948fc04d26475b482d1314fe70d537/shm: device or resource busy
local_testnet_2
Failed to remove container (local_testnet_2): Error response from daemon: Unable to remove filesystem for 6f2ec44e01afd85875c1d9958afa957f1623ed9464df2b4fcac06feb8254145e: remove /var/lib/docker/containers/6f2ec44e01afd85875c1d9958afa957f1623ed9464df2b4fcac06feb8254145e/shm: device or resource busy
local_testnet_3
Failed to remove container (local_testnet_3): Error response from daemon: Unable to remove filesystem for 6160c73d45376920d90473a487a07f0e283e383011a0d23050b22cd82ab5d255: remove /var/lib/docker/containers/6160c73d45376920d90473a487a07f0e283e383011a0d23050b22cd82ab5d255/shm: device or resource busy
local_testnet_4
Failed to remove container (local_testnet_4): Error response from daemon: Driver btrfs failed to remove root filesystem c93a30d66ff3710617915a8f6075755ebc45aebf73569420225105cc81003e2f: Failed to destroy btrfs snapshot /var/lib/docker/btrfs/subvolumes for cac56069a3682b6d133e239f479fb82a67c4ccd09b877794d9f440a1125d59d6: operation not permitted

* [15:37:40] stopping rsyslog container
Failed to remove container (rsyslog): Error response from daemon: Driver btrfs failed to remove root filesystem 477287a170a48156e663c55092257064e1f14770d0b6c634a9972893636d9451: Failed to destroy btrfs snapshot /var/lib/docker/btrfs/subvolumes for 76b66d582a478743c099f479bf88eb334ad303ef607f91021dfa58d28c9161eb: operation not permitted
make: *** [test_integrations] Error 1
```
---
 test/p2p/test.sh | 3 ---
 test/test.sh     | 4 ----
 2 files changed, 7 deletions(-)

diff --git a/test/p2p/test.sh b/test/p2p/test.sh
index 4d91aaba..0f29aa19 100644
--- a/test/p2p/test.sh
+++ b/test/p2p/test.sh
@@ -31,6 +31,3 @@ bash test/p2p/fast_sync/test.sh $DOCKER_IMAGE $NETWORK_NAME $N $PROXY_APP
 
 # test killing all peers
 bash test/p2p/kill_all/test.sh $DOCKER_IMAGE $NETWORK_NAME $N 3
-
-# stop the existing testnet and remove local network
-bash test/p2p/local_testnet_stop.sh $NETWORK_NAME $N
diff --git a/test/test.sh b/test/test.sh
index 2abafc32..9fbfd7d3 100644
--- a/test/test.sh
+++ b/test/test.sh
@@ -45,7 +45,3 @@ if [[ $(echo "$BRANCH" | grep "release-") != "" ]]; then
 	# TODO: replace mintnet
 	#bash "$DIR/net/test.sh"
 fi
-
-echo
-echo "* [$(date +"%T")] stopping rsyslog container"
-docker rm -f rsyslog

From a756a9febae92bd03abe86ac19ab0b9cf8eb586c Mon Sep 17 00:00:00 2001
From: Anton Kaliaev <anton.kalyaev@gmail.com>
Date: Wed, 15 Feb 2017 00:49:43 +0400
Subject: [PATCH 041/132] add readme mirroring
 https://hub.docker.com/r/tendermint/tendermint/

---
 DOCKER/README.md | 61 +++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 48 insertions(+), 13 deletions(-)

diff --git a/DOCKER/README.md b/DOCKER/README.md
index 8e660b98..bbb66263 100644
--- a/DOCKER/README.md
+++ b/DOCKER/README.md
@@ -1,20 +1,55 @@
-# Docker
+# Supported tags and respective `Dockerfile` links
 
-Tendermint uses docker for deployment of testnets via the [mintnet](github.com/tendermint/mintnet) tool.
+- `0.8.0`, `0.8`, `latest` [(Dockerfile)](https://github.com/tendermint/tendermint/blob/bf64dd21fdb193e54d8addaaaa2ecf7ac371de8c/DOCKER/Dockerfile)
+- `develop` [(Dockerfile)]()
 
-For the deterministic docker builds used in testing, see the [tests directory](https://github.com/tendermint/tendermint/tree/master/test)
+`develop` tag points to the [develop](https://github.com/tendermint/tendermint/tree/develop) branch.
 
-# Build and run a docker image and container
+# Tendermint
 
-These are notes for the dev team.
+Tendermint Core is Byzantine Fault Tolerant (BFT) middleware that takes a state transition machine, written in any programming language, and securely replicates it on many machines.
+
+For more background, see the [introduction](https://tendermint.com/intro).
+
+To get started developing applications, see the [application developers guide](https://tendermint.com/docs/guides/app-development).
+
+# How to use this image
+
+## Start one instance of the Tendermint core with the `dummy` app
+
+A very simple example of a built-in app and Tendermint core in one container.
 
 ```
-# Build base Docker image
-docker build -t "tendermint/tendermint" -t "tendermint/tendermint:0.8.0" -t "tendermint/tendermint:0.8" .
-
-# Log into dockerhub
-docker login
-
-# Push latest build to dockerhub
-docker push tendermint/tendermint
+docker run -it --rm -v "/tmp:/tendermint" tendermint/tendermint init
+docker run -it --rm -v "/tmp:/tendermint" tendermint/tendermint
 ```
+
+## mintnet-kubernetes
+
+If you want to see many containers talking to each other, consider using [mintnet-kubernetes](https://github.com/tendermint/mintnet-kubernetes), which is a tool for running Tendermint-based applications on a Kubernetes cluster.
+
+# Supported Docker versions
+
+This image is officially supported on Docker version 1.13.1.
+
+Support for older versions (down to 1.6) is provided on a best-effort basis.
+
+Please see [the Docker installation documentation](https://docs.docker.com/installation/) for details on how to upgrade your Docker daemon.
+
+# License
+
+View [license information](https://raw.githubusercontent.com/tendermint/tendermint/master/LICENSE) for the software contained in this image.
+
+# User Feedback
+
+## Issues
+
+If you have any problems with or questions about this image, please contact us through a [GitHub](https://github.com/tendermint/tendermint/issues) issue. If the issue is related to a CVE, please check for [a `cve-tracker` issue on the `official-images` repository](https://github.com/docker-library/official-images/issues?q=label%3Acve-tracker) first.
+
+You can also reach the image maintainers via [Slack](http://forum.tendermint.com:3000/).
+
+## Contributing
+
+You are invited to contribute new features, fixes, or updates, large or small; we are always thrilled to receive pull requests, and do our best to process them as fast as we can.
+
+Before you start to code, we recommend discussing your plans through a [GitHub](https://github.com/tendermint/tendermint/issues) issue, especially for more ambitious contributions. This gives other contributors a chance to point you in the right direction, give you feedback on your design, and help you find out if someone else is working on the same thing.

From 6b499e2a8ba3500895ef58ab1184059e1ef2d38b Mon Sep 17 00:00:00 2001
From: Anton Kaliaev <anton.kalyaev@gmail.com>
Date: Wed, 15 Feb 2017 00:57:54 +0400
Subject: [PATCH 042/132] add Makefile for docker files [ci skip] [circleci
 skip]

---
 DOCKER/Makefile | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 DOCKER/Makefile

diff --git a/DOCKER/Makefile b/DOCKER/Makefile
new file mode 100644
index 00000000..09223301
--- /dev/null
+++ b/DOCKER/Makefile
@@ -0,0 +1,15 @@
+build:
+	# TAG=0.8.0 TAG_NO_PATCH=0.8
+	docker build -t "tendermint/tendermint" -t "tendermint/tendermint:$TAG" -t "tendermint/tendermint:$TAG_NO_PATCH" .
+
+push:
+	# TAG=0.8.0 TAG_NO_PATCH=0.8
+	docker push "tendermint/tendermint" "tendermint/tendermint:$TAG" "tendermint/tendermint:$TAG_NO_PATCH"
+
+build_develop:
+	docker build -t "tendermint/tendermint:develop" -f Dockerfile.develop .
+
+push_develop:
+	docker push "tendermint/tendermint:develop"
+
+.PHONY: build build_develop push push_develop

From e229c8c3d75d37d0dcd06a4313626d8df3477950 Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Thu, 9 Feb 2017 22:44:43 -0500
Subject: [PATCH 043/132] rpc: /commit

---
 rpc/core/blocks.go          | 23 +++++++++++++++++++++++
 rpc/core/pipe.go            |  4 +++-
 rpc/core/routes.go          |  9 +++++++++
 rpc/core/types/responses.go |  7 +++++++
 4 files changed, 42 insertions(+), 1 deletion(-)

diff --git a/rpc/core/blocks.go b/rpc/core/blocks.go
index 115ca2a9..f1e47d41 100644
--- a/rpc/core/blocks.go
+++ b/rpc/core/blocks.go
@@ -44,3 +44,26 @@ func Block(height int) (*ctypes.ResultBlock, error) {
 	block := blockStore.LoadBlock(height)
 	return &ctypes.ResultBlock{blockMeta, block}, nil
 }
+
+//-----------------------------------------------------------------------------
+
+func Commit(height int) (*ctypes.ResultCommit, error) {
+	if height == 0 {
+		return nil, fmt.Errorf("Height must be greater than 0")
+	}
+	storeHeight := blockStore.Height()
+	if height > storeHeight {
+		return nil, fmt.Errorf("Height must be less than or equal to the current blockchain height")
+	}
+
+	// If the next block has not been committed yet,
+	// use a non-canonical commit
+	if height == storeHeight+1 {
+		commit := blockStore.LoadSeenCommit(height)
+		return &ctypes.ResultCommit{commit, false}, nil
+	}
+
+	// Return the canonical commit (comes from the block at height+1)
+	commit := blockStore.LoadBlockCommit(height)
+	return &ctypes.ResultCommit{commit, true}, nil
+}
diff --git a/rpc/core/pipe.go b/rpc/core/pipe.go
index 356a2ff0..16bd8f20 100644
--- a/rpc/core/pipe.go
+++ b/rpc/core/pipe.go
@@ -5,10 +5,10 @@ import (
 	"github.com/tendermint/go-crypto"
 	"github.com/tendermint/go-p2p"
 
+	abci "github.com/tendermint/abci/types"
 	"github.com/tendermint/tendermint/consensus"
 	"github.com/tendermint/tendermint/proxy"
 	"github.com/tendermint/tendermint/types"
-	abci "github.com/tendermint/abci/types"
 )
 
 //-----------------------------------------------------
@@ -19,6 +19,8 @@ type BlockStore interface {
 	Height() int
 	LoadBlockMeta(height int) *types.BlockMeta
 	LoadBlock(height int) *types.Block
+	LoadSeenCommit(height int) *types.Commit
+	LoadBlockCommit(height int) *types.Commit
 }
 
 type Consensus interface {
diff --git a/rpc/core/routes.go b/rpc/core/routes.go
index 8396cd23..643b2bf0 100644
--- a/rpc/core/routes.go
+++ b/rpc/core/routes.go
@@ -18,6 +18,7 @@ var Routes = map[string]*rpc.RPCFunc{
 	"blockchain":           rpc.NewRPCFunc(BlockchainInfoResult, "minHeight,maxHeight"),
 	"genesis":              rpc.NewRPCFunc(GenesisResult, ""),
 	"block":                rpc.NewRPCFunc(BlockResult, "height"),
+	"commit":               rpc.NewRPCFunc(CommitResult, "height"),
 	"validators":           rpc.NewRPCFunc(ValidatorsResult, ""),
 	"dump_consensus_state": rpc.NewRPCFunc(DumpConsensusStateResult, ""),
 	"unconfirmed_txs":      rpc.NewRPCFunc(UnconfirmedTxsResult, ""),
@@ -107,6 +108,14 @@ func BlockResult(height int) (ctypes.TMResult, error) {
 	}
 }
 
+func CommitResult(height int) (ctypes.TMResult, error) {
+	if r, err := Commit(height); err != nil {
+		return nil, err
+	} else {
+		return r, nil
+	}
+}
+
 func ValidatorsResult() (ctypes.TMResult, error) {
 	if r, err := Validators(); err != nil {
 		return nil, err
diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go
index 1a5deaea..b68f169a 100644
--- a/rpc/core/types/responses.go
+++ b/rpc/core/types/responses.go
@@ -23,6 +23,11 @@ type ResultBlock struct {
 	Block     *types.Block     `json:"block"`
 }
 
+type ResultCommit struct {
+	Commit    *types.Commit `json:"commit"`
+	Canonical bool          `json:"canonical"`
+}
+
 type ResultStatus struct {
 	NodeInfo          *p2p.NodeInfo `json:"node_info"`
 	PubKey            crypto.PubKey `json:"pub_key"`
@@ -106,6 +111,7 @@ const (
 	ResultTypeGenesis        = byte(0x01)
 	ResultTypeBlockchainInfo = byte(0x02)
 	ResultTypeBlock          = byte(0x03)
+	ResultTypeCommit         = byte(0x04)
 
 	// 0x2 bytes are for the network
 	ResultTypeStatus    = byte(0x20)
@@ -148,6 +154,7 @@ var _ = wire.RegisterInterface(
 	wire.ConcreteType{&ResultGenesis{}, ResultTypeGenesis},
 	wire.ConcreteType{&ResultBlockchainInfo{}, ResultTypeBlockchainInfo},
 	wire.ConcreteType{&ResultBlock{}, ResultTypeBlock},
+	wire.ConcreteType{&ResultCommit{}, ResultTypeCommit},
 	wire.ConcreteType{&ResultStatus{}, ResultTypeStatus},
 	wire.ConcreteType{&ResultNetInfo{}, ResultTypeNetInfo},
 	wire.ConcreteType{&ResultDialSeeds{}, ResultTypeDialSeeds},

From 99b068b3136e3e70aba3612f36afd3288d817d96 Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Tue, 14 Feb 2017 15:33:14 -0500
Subject: [PATCH 044/132] BlockMeta uses BlockID

---
 blockchain/store.go         | 12 ++++++------
 consensus/reactor.go        |  6 +++---
 rpc/core/blocks.go          |  6 ++++--
 rpc/core/status.go          |  2 +-
 rpc/core/types/responses.go |  5 +++--
 state/execution.go          |  4 ++--
 state/execution_test.go     |  7 +++----
 types/block_meta.go         | 10 ++++------
 8 files changed, 26 insertions(+), 26 deletions(-)

diff --git a/blockchain/store.go b/blockchain/store.go
index db897465..ac7cfdaf 100644
--- a/blockchain/store.go
+++ b/blockchain/store.go
@@ -64,12 +64,12 @@ func (bs *BlockStore) LoadBlock(height int) *types.Block {
 	if r == nil {
 		return nil
 	}
-	meta := wire.ReadBinary(&types.BlockMeta{}, r, 0, &n, &err).(*types.BlockMeta)
+	blockMeta := wire.ReadBinary(&types.BlockMeta{}, r, 0, &n, &err).(*types.BlockMeta)
 	if err != nil {
 		PanicCrisis(Fmt("Error reading block meta: %v", err))
 	}
 	bytez := []byte{}
-	for i := 0; i < meta.PartsHeader.Total; i++ {
+	for i := 0; i < blockMeta.BlockID.PartsHeader.Total; i++ {
 		part := bs.LoadBlockPart(height, i)
 		bytez = append(bytez, part.Bytes...)
 	}
@@ -101,11 +101,11 @@ func (bs *BlockStore) LoadBlockMeta(height int) *types.BlockMeta {
 	if r == nil {
 		return nil
 	}
-	meta := wire.ReadBinary(&types.BlockMeta{}, r, 0, &n, &err).(*types.BlockMeta)
+	blockMeta := wire.ReadBinary(&types.BlockMeta{}, r, 0, &n, &err).(*types.BlockMeta)
 	if err != nil {
 		PanicCrisis(Fmt("Error reading block meta: %v", err))
 	}
-	return meta
+	return blockMeta
 }
 
 // The +2/3 and other Precommit-votes for block at `height`.
@@ -154,8 +154,8 @@ func (bs *BlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, s
 	}
 
 	// Save block meta
-	meta := types.NewBlockMeta(block, blockParts)
-	metaBytes := wire.BinaryBytes(meta)
+	blockMeta := types.NewBlockMeta(block, blockParts)
+	metaBytes := wire.BinaryBytes(blockMeta)
 	bs.db.Set(calcBlockMetaKey(height), metaBytes)
 
 	// Save block parts
diff --git a/consensus/reactor.go b/consensus/reactor.go
index 80e87efd..1e700a86 100644
--- a/consensus/reactor.go
+++ b/consensus/reactor.go
@@ -415,9 +415,9 @@ OUTER_LOOP:
 					log.Warn("Failed to load block meta", "peer height", prs.Height, "our height", rs.Height, "blockstore height", conR.conS.blockStore.Height(), "pv", conR.conS.privValidator)
 					time.Sleep(peerGossipSleepDuration)
 					continue OUTER_LOOP
-				} else if !blockMeta.PartsHeader.Equals(prs.ProposalBlockPartsHeader) {
+				} else if !blockMeta.BlockID.PartsHeader.Equals(prs.ProposalBlockPartsHeader) {
 					log.Info("Peer ProposalBlockPartsHeader mismatch, sleeping",
-						"peerHeight", prs.Height, "blockPartsHeader", blockMeta.PartsHeader, "peerBlockPartsHeader", prs.ProposalBlockPartsHeader)
+						"peerHeight", prs.Height, "blockPartsHeader", blockMeta.BlockID.PartsHeader, "peerBlockPartsHeader", prs.ProposalBlockPartsHeader)
 					time.Sleep(peerGossipSleepDuration)
 					continue OUTER_LOOP
 				}
@@ -425,7 +425,7 @@ OUTER_LOOP:
 				part := conR.conS.blockStore.LoadBlockPart(prs.Height, index)
 				if part == nil {
 					log.Warn("Could not load part", "index", index,
-						"peerHeight", prs.Height, "blockPartsHeader", blockMeta.PartsHeader, "peerBlockPartsHeader", prs.ProposalBlockPartsHeader)
+						"peerHeight", prs.Height, "blockPartsHeader", blockMeta.BlockID.PartsHeader, "peerBlockPartsHeader", prs.ProposalBlockPartsHeader)
 					time.Sleep(peerGossipSleepDuration)
 					continue OUTER_LOOP
 				}
diff --git a/rpc/core/blocks.go b/rpc/core/blocks.go
index f1e47d41..5006f897 100644
--- a/rpc/core/blocks.go
+++ b/rpc/core/blocks.go
@@ -56,14 +56,16 @@ func Commit(height int) (*ctypes.ResultCommit, error) {
 		return nil, fmt.Errorf("Height must be less than or equal to the current blockchain height")
 	}
 
+	header := blockStore.LoadBlockMeta(height).Header
+
 	// If the next block has not been committed yet,
 	// use a non-canonical commit
 	if height == storeHeight+1 {
 		commit := blockStore.LoadSeenCommit(height)
-		return &ctypes.ResultCommit{commit, false}, nil
+		return &ctypes.ResultCommit{header, commit, false}, nil
 	}
 
 	// Return the canonical commit (comes from the block at height+1)
 	commit := blockStore.LoadBlockCommit(height)
-	return &ctypes.ResultCommit{commit, true}, nil
+	return &ctypes.ResultCommit{header, commit, true}, nil
 }
diff --git a/rpc/core/status.go b/rpc/core/status.go
index 8edadf13..96ed46ea 100644
--- a/rpc/core/status.go
+++ b/rpc/core/status.go
@@ -15,7 +15,7 @@ func Status() (*ctypes.ResultStatus, error) {
 	)
 	if latestHeight != 0 {
 		latestBlockMeta = blockStore.LoadBlockMeta(latestHeight)
-		latestBlockHash = latestBlockMeta.Hash
+		latestBlockHash = latestBlockMeta.BlockID.Hash
 		latestAppHash = latestBlockMeta.Header.AppHash
 		latestBlockTime = latestBlockMeta.Header.Time.UnixNano()
 	}
diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go
index b68f169a..bcab4f59 100644
--- a/rpc/core/types/responses.go
+++ b/rpc/core/types/responses.go
@@ -24,8 +24,9 @@ type ResultBlock struct {
 }
 
 type ResultCommit struct {
-	Commit    *types.Commit `json:"commit"`
-	Canonical bool          `json:"canonical"`
+	Header          *types.Header `json:"header"`
+	Commit          *types.Commit `json:"commit"`
+	CanonicalCommit bool          `json:"canonical"`
 }
 
 type ResultStatus struct {
diff --git a/state/execution.go b/state/execution.go
index a7ba2399..3dbf5e28 100644
--- a/state/execution.go
+++ b/state/execution.go
@@ -6,12 +6,12 @@ import (
 
 	"github.com/ebuchman/fail-test"
 
+	abci "github.com/tendermint/abci/types"
 	. "github.com/tendermint/go-common"
 	cfg "github.com/tendermint/go-config"
 	"github.com/tendermint/go-crypto"
 	"github.com/tendermint/tendermint/proxy"
 	"github.com/tendermint/tendermint/types"
-	abci "github.com/tendermint/abci/types"
 )
 
 //--------------------------------------------------
@@ -393,7 +393,7 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, appConnCon
 		var eventCache types.Fireable // nil
 
 		// replay the latest block
-		return h.state.ApplyBlock(eventCache, appConnConsensus, block, blockMeta.PartsHeader, MockMempool{})
+		return h.state.ApplyBlock(eventCache, appConnConsensus, block, blockMeta.BlockID.PartsHeader, MockMempool{})
 	} else if storeBlockHeight != stateBlockHeight {
 		// unless we failed before committing or saving state (previous 2 case),
 		// the store and state should be at the same height!
diff --git a/state/execution_test.go b/state/execution_test.go
index 55a899a0..452e72e1 100644
--- a/state/execution_test.go
+++ b/state/execution_test.go
@@ -8,12 +8,12 @@ import (
 
 	"github.com/tendermint/tendermint/config/tendermint_test"
 	//	. "github.com/tendermint/go-common"
+	"github.com/tendermint/abci/example/dummy"
 	cfg "github.com/tendermint/go-config"
 	"github.com/tendermint/go-crypto"
 	dbm "github.com/tendermint/go-db"
 	"github.com/tendermint/tendermint/proxy"
 	"github.com/tendermint/tendermint/types"
-	"github.com/tendermint/abci/example/dummy"
 )
 
 var (
@@ -203,8 +203,7 @@ func (bs *mockBlockStore) LoadBlock(height int) *types.Block { return bs.chain[h
 func (bs *mockBlockStore) LoadBlockMeta(height int) *types.BlockMeta {
 	block := bs.chain[height-1]
 	return &types.BlockMeta{
-		Hash:        block.Hash(),
-		Header:      block.Header,
-		PartsHeader: block.MakePartSet(bs.config.GetInt("block_part_size")).Header(),
+		BlockID: types.BlockID{block.Hash(), block.MakePartSet(bs.config.GetInt("block_part_size")).Header()},
+		Header:  block.Header,
 	}
 }
diff --git a/types/block_meta.go b/types/block_meta.go
index b72c0c86..8e5bd43e 100644
--- a/types/block_meta.go
+++ b/types/block_meta.go
@@ -1,15 +1,13 @@
 package types
 
 type BlockMeta struct {
-	Hash        []byte        `json:"hash"`         // The block hash
-	Header      *Header       `json:"header"`       // The block's Header
-	PartsHeader PartSetHeader `json:"parts_header"` // The PartSetHeader, for transfer
+	BlockID BlockID `json:"block_id"` // the block hash and partsethash
+	Header  *Header `json:"header"`   // The block's Header
 }
 
 func NewBlockMeta(block *Block, blockParts *PartSet) *BlockMeta {
 	return &BlockMeta{
-		Hash:        block.Hash(),
-		Header:      block.Header,
-		PartsHeader: blockParts.Header(),
+		BlockID: BlockID{block.Hash(), blockParts.Header()},
+		Header:  block.Header,
 	}
 }

From 6f83f0bd4ea1322d19d4db08031f447f1551ddc1 Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Tue, 14 Feb 2017 17:09:12 -0500
Subject: [PATCH 045/132] glide update

---
 glide.lock | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/glide.lock b/glide.lock
index 4fa40e9a..c58c2a4d 100644
--- a/glide.lock
+++ b/glide.lock
@@ -1,5 +1,5 @@
 hash: e283934fbbd221161d53a918db9e49db8c5be2b8929592b05ffe6b72c2ef0ab1
-updated: 2017-02-13T20:54:31.541721996-05:00
+updated: 2017-02-14T17:07:40.028389369-05:00
 imports:
 - name: github.com/btcsuite/btcd
   version: d06c0bb181529331be8f8d9350288c420d9e60e4
@@ -12,7 +12,7 @@ imports:
 - name: github.com/go-stack/stack
   version: 100eb0c0a9c5b306ca2fb4f165df21d80ada4b82
 - name: github.com/gogo/protobuf
-  version: 9c5b7bafbfccf2b40d274e0496f3a09418a87af4
+  version: 2221ff550f109ae54cb617c0dc6ac62658c418d7
   subpackages:
   - proto
 - name: github.com/golang/protobuf
@@ -22,7 +22,7 @@ imports:
 - name: github.com/golang/snappy
   version: 7db9049039a047d955fe8c19b83c8ff5abd765c7
 - name: github.com/gorilla/websocket
-  version: 9bc973af0682dc73a22553a08bfe00ee6255f56f
+  version: 804cb600d06b10672f2fbc0a336a7bee507a428e
 - name: github.com/jmhodges/levigo
   version: c42d9e0ca023e2198120196f842701bb4c55d7b9
 - name: github.com/mattn/go-colorable
@@ -47,7 +47,7 @@ imports:
   - leveldb/table
   - leveldb/util
 - name: github.com/tendermint/abci
-  version: 2a3f6384950041b098d89e66b93c580ecd6d32ef
+  version: 31bdda27ad80e47d372e4b9e4acfd4c0ef81d3e4
   subpackages:
   - client
   - example/counter
@@ -125,7 +125,7 @@ imports:
   subpackages:
   - unix
 - name: google.golang.org/grpc
-  version: 4acc9108c18937815db3306a66990869d22aa980
+  version: cbcceb2942a489498cf22b2f918536e819d33f0a
   subpackages:
   - codes
   - credentials

From bb5688b1beae7eabb9627964efcd5f7c0262b1a5 Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Tue, 14 Feb 2017 17:09:47 -0500
Subject: [PATCH 046/132] make: dont use -v on go test

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 81edfd32..48270512 100644
--- a/Makefile
+++ b/Makefile
@@ -22,7 +22,7 @@ dist:
 
 test:
 	@echo "--> Running go test"
-	@go test -v $(PACKAGES)
+	@go test $(PACKAGES)
 
 test_race:
 	@echo "--> Running go test --race"

From a3898fae0f5b8037bc28c3f71f76ad660a6162d8 Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Thu, 16 Feb 2017 15:35:34 -0500
Subject: [PATCH 047/132] rpc: fix SeenCommit condition

---
 rpc/core/blocks.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rpc/core/blocks.go b/rpc/core/blocks.go
index 5006f897..65e47d12 100644
--- a/rpc/core/blocks.go
+++ b/rpc/core/blocks.go
@@ -60,7 +60,7 @@ func Commit(height int) (*ctypes.ResultCommit, error) {
 
 	// If the next block has not been committed yet,
 	// use a non-canonical commit
-	if height == storeHeight+1 {
+	if height == storeHeight {
 		commit := blockStore.LoadSeenCommit(height)
 		return &ctypes.ResultCommit{header, commit, false}, nil
 	}

From cbe6dbe7a1fb720a243c6d6234e9e011a052aaa2 Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Thu, 16 Feb 2017 17:56:45 -0500
Subject: [PATCH 048/132] handshake replay through consensus using mockApp

---
 consensus/replay_file.go |   2 +-
 consensus/state.go       |  18 ++++-
 mempool/mempool.go       |   8 +--
 node/node.go             |   2 +-
 rpc/core/pipe.go         |   2 +-
 state/execution.go       | 147 +++++++++++++++++++--------------------
 6 files changed, 92 insertions(+), 87 deletions(-)

diff --git a/consensus/replay_file.go b/consensus/replay_file.go
index 1c2e9cb0..5d674785 100644
--- a/consensus/replay_file.go
+++ b/consensus/replay_file.go
@@ -248,7 +248,7 @@ func newConsensusStateForReplay(config cfg.Config) *ConsensusState {
 	state := sm.MakeGenesisStateFromFile(stateDB, config.GetString("genesis_file"))
 
 	// Create proxyAppConn connection (consensus, mempool, query)
-	proxyApp := proxy.NewAppConns(config, proxy.DefaultClientCreator(config), sm.NewHandshaker(config, state, blockStore))
+	proxyApp := proxy.NewAppConns(config, proxy.DefaultClientCreator(config), sm.NewHandshaker(config, state, blockStore, ReplayLastBlock))
 	_, err := proxyApp.Start()
 	if err != nil {
 		Exit(Fmt("Error starting proxy app conns: %v", err))
diff --git a/consensus/state.go b/consensus/state.go
index 78e6c0ea..c872233d 100644
--- a/consensus/state.go
+++ b/consensus/state.go
@@ -15,7 +15,6 @@ import (
 	cfg "github.com/tendermint/go-config"
 	"github.com/tendermint/go-wire"
 	bc "github.com/tendermint/tendermint/blockchain"
-	mempl "github.com/tendermint/tendermint/mempool"
 	"github.com/tendermint/tendermint/proxy"
 	sm "github.com/tendermint/tendermint/state"
 	"github.com/tendermint/tendermint/types"
@@ -227,7 +226,7 @@ type ConsensusState struct {
 	config       cfg.Config
 	proxyAppConn proxy.AppConnConsensus
 	blockStore   *bc.BlockStore
-	mempool      *mempl.Mempool
+	mempool      sm.Mempool
 
 	privValidator PrivValidator // for signing votes
 
@@ -255,7 +254,20 @@ type ConsensusState struct {
 	done chan struct{}
 }
 
-func NewConsensusState(config cfg.Config, state *sm.State, proxyAppConn proxy.AppConnConsensus, blockStore *bc.BlockStore, mempool *mempl.Mempool) *ConsensusState {
+func ReplayLastBlock(config cfg.Config, state *sm.State, proxyApp proxy.AppConnConsensus, blockStore sm.BlockStore) {
+	mempool := sm.MockMempool{}
+	cs := NewConsensusState(config, state, proxyApp, blockStore.(*bc.BlockStore), mempool)
+
+	evsw := types.NewEventSwitch()
+	cs.SetEventSwitch(evsw)
+	newBlockCh := subscribeToEvent(evsw, "consensus-replay", types.EventStringNewBlock(), 0)
+
+	cs.Start()
+	<-newBlockCh
+	cs.Stop()
+}
+
+func NewConsensusState(config cfg.Config, state *sm.State, proxyAppConn proxy.AppConnConsensus, blockStore *bc.BlockStore, mempool sm.Mempool) *ConsensusState {
 	cs := &ConsensusState{
 		config:           config,
 		proxyAppConn:     proxyAppConn,
diff --git a/mempool/mempool.go b/mempool/mempool.go
index 6fe4d704..e960f520 100644
--- a/mempool/mempool.go
+++ b/mempool/mempool.go
@@ -7,13 +7,13 @@ import (
 	"sync/atomic"
 	"time"
 
+	abci "github.com/tendermint/abci/types"
 	auto "github.com/tendermint/go-autofile"
 	"github.com/tendermint/go-clist"
 	. "github.com/tendermint/go-common"
 	cfg "github.com/tendermint/go-config"
 	"github.com/tendermint/tendermint/proxy"
 	"github.com/tendermint/tendermint/types"
-	abci "github.com/tendermint/abci/types"
 )
 
 /*
@@ -249,7 +249,7 @@ func (mem *Mempool) resCbRecheck(req *abci.Request, res *abci.Response) {
 
 // Get the valid transactions remaining
 // If maxTxs is -1, there is no cap on returned transactions.
-func (mem *Mempool) Reap(maxTxs int) []types.Tx {
+func (mem *Mempool) Reap(maxTxs int) types.Txs {
 	mem.proxyMtx.Lock()
 	defer mem.proxyMtx.Unlock()
 
@@ -263,7 +263,7 @@ func (mem *Mempool) Reap(maxTxs int) []types.Tx {
 }
 
 // maxTxs: -1 means uncapped, 0 means none
-func (mem *Mempool) collectTxs(maxTxs int) []types.Tx {
+func (mem *Mempool) collectTxs(maxTxs int) types.Txs {
 	if maxTxs == 0 {
 		return []types.Tx{}
 	} else if maxTxs < 0 {
@@ -281,7 +281,7 @@ func (mem *Mempool) collectTxs(maxTxs int) []types.Tx {
 // Mempool will discard these txs.
 // NOTE: this should be called *after* block is committed by consensus.
 // NOTE: unsafe; Lock/Unlock must be managed by caller
-func (mem *Mempool) Update(height int, txs []types.Tx) {
+func (mem *Mempool) Update(height int, txs types.Txs) {
 	// TODO: check err ?
 	mem.proxyAppConn.FlushSync() // To flush async resCb calls e.g. from CheckTx
 
diff --git a/node/node.go b/node/node.go
index d9eeb1f4..8ccd47c0 100644
--- a/node/node.go
+++ b/node/node.go
@@ -64,7 +64,7 @@ func NewNode(config cfg.Config, privValidator *types.PrivValidator, clientCreato
 	state := sm.GetState(config, stateDB)
 
 	// Create the proxyApp, which manages connections (consensus, mempool, query)
-	proxyApp := proxy.NewAppConns(config, clientCreator, sm.NewHandshaker(config, state, blockStore))
+	proxyApp := proxy.NewAppConns(config, clientCreator, sm.NewHandshaker(config, state, blockStore, consensus.ReplayLastBlock))
 	if _, err := proxyApp.Start(); err != nil {
 		cmn.Exit(cmn.Fmt("Error starting proxy app connections: %v", err))
 	}
diff --git a/rpc/core/pipe.go b/rpc/core/pipe.go
index 16bd8f20..fb06c3ff 100644
--- a/rpc/core/pipe.go
+++ b/rpc/core/pipe.go
@@ -31,7 +31,7 @@ type Consensus interface {
 type Mempool interface {
 	Size() int
 	CheckTx(types.Tx, func(*abci.Response)) error
-	Reap(int) []types.Tx
+	Reap(int) types.Txs
 	Flush()
 }
 
diff --git a/state/execution.go b/state/execution.go
index 3dbf5e28..e4b7cff4 100644
--- a/state/execution.go
+++ b/state/execution.go
@@ -1,7 +1,6 @@
 package state
 
 import (
-	"bytes"
 	"errors"
 
 	"github.com/ebuchman/fail-test"
@@ -278,15 +277,20 @@ func (s *State) CommitStateUpdateMempool(proxyAppConn proxy.AppConnConsensus, bl
 type Mempool interface {
 	Lock()
 	Unlock()
-	Update(height int, txs []types.Tx)
+
+	CheckTx(types.Tx, func(*abci.Response)) error
+	Reap(int) types.Txs
+	Update(height int, txs types.Txs)
 }
 
 type MockMempool struct {
 }
 
-func (m MockMempool) Lock()                             {}
-func (m MockMempool) Unlock()                           {}
-func (m MockMempool) Update(height int, txs []types.Tx) {}
+func (m MockMempool) Lock()                                              {}
+func (m MockMempool) Unlock()                                            {}
+func (m MockMempool) CheckTx(tx types.Tx, cb func(*abci.Response)) error { return nil }
+func (m MockMempool) Reap(n int) types.Txs                               { return types.Txs{} }
+func (m MockMempool) Update(height int, txs types.Txs)                   {}
 
 //----------------------------------------------------------------
 // Handshake with app to sync to latest state of core by replaying blocks
@@ -298,16 +302,19 @@ type BlockStore interface {
 	LoadBlockMeta(height int) *types.BlockMeta
 }
 
+type blockReplayFunc func(cfg.Config, *State, proxy.AppConnConsensus, BlockStore)
+
 type Handshaker struct {
-	config cfg.Config
-	state  *State
-	store  BlockStore
+	config          cfg.Config
+	state           *State
+	store           BlockStore
+	replayLastBlock blockReplayFunc
 
 	nBlocks int // number of blocks applied to the state
 }
 
-func NewHandshaker(config cfg.Config, state *State, store BlockStore) *Handshaker {
-	return &Handshaker{config, state, store, 0}
+func NewHandshaker(config cfg.Config, state *State, store BlockStore, f blockReplayFunc) *Handshaker {
+	return &Handshaker{config, state, store, f, 0}
 }
 
 // TODO: retry the handshake/replay if it fails ?
@@ -326,7 +333,7 @@ func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error {
 	// TODO: check version
 
 	// replay blocks up to the latest in the blockstore
-	err = h.ReplayBlocks(appHash, blockHeight, proxyApp.Consensus())
+	err = h.ReplayBlocks(appHash, blockHeight, proxyApp)
 	if err != nil {
 		return errors.New(Fmt("Error on replay: %v", err))
 	}
@@ -340,7 +347,7 @@ func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error {
 }
 
 // Replay all blocks after blockHeight and ensure the result matches the current state.
-func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, appConnConsensus proxy.AppConnConsensus) error {
+func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, proxyApp proxy.AppConns) error {
 
 	storeBlockHeight := h.store.Height()
 	stateBlockHeight := h.state.LastBlockHeight
@@ -353,85 +360,71 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, appConnCon
 		return ErrAppBlockHeightTooHigh{storeBlockHeight, appBlockHeight}
 
 	} else if storeBlockHeight == appBlockHeight {
-		// We ran Commit, but if we crashed before state.Save(),
-		// load the intermediate state and update the state.AppHash.
-		// NOTE: If ABCI allowed rollbacks, we could just replay the
-		// block even though it's been committed
-		stateAppHash := h.state.AppHash
-		lastBlockAppHash := h.store.LoadBlock(storeBlockHeight).AppHash
+		// We already ran Commit, so run through consensus with mock app
+		mockApp := newMockProxyApp(appHash)
 
-		if bytes.Equal(stateAppHash, appHash) {
-			// we're all synced up
-			log.Debug("ABCI RelpayBlocks: Already synced")
-		} else if bytes.Equal(stateAppHash, lastBlockAppHash) {
-			// we crashed after commit and before saving state,
-			// so load the intermediate state and update the hash
-			h.state.LoadIntermediate()
-			h.state.AppHash = appHash
-			log.Debug("ABCI RelpayBlocks: Loaded intermediate state and updated state.AppHash")
-		} else {
-			PanicSanity(Fmt("Unexpected state.AppHash: state.AppHash %X; app.AppHash %X, lastBlock.AppHash %X", stateAppHash, appHash, lastBlockAppHash))
+		h.replayLastBlock(h.config, h.state, mockApp, h.store)
 
-		}
-		return nil
-
-	} else if storeBlockHeight == appBlockHeight+1 &&
-		storeBlockHeight == stateBlockHeight+1 {
+	} else if storeBlockHeight == appBlockHeight+1 {
 		// We crashed after saving the block
 		// but before Commit (both the state and app are behind),
-		// so just replay the block
+		// so run through consensus with the real app
+		h.replayLastBlock(h.config, h.state, proxyApp.Consensus(), h.store)
 
-		// check that the lastBlock.AppHash matches the state apphash
-		block := h.store.LoadBlock(storeBlockHeight)
-		if !bytes.Equal(block.Header.AppHash, appHash) {
-			return ErrLastStateMismatch{storeBlockHeight, block.Header.AppHash, appHash}
-		}
-
-		blockMeta := h.store.LoadBlockMeta(storeBlockHeight)
-
-		h.nBlocks += 1
-		var eventCache types.Fireable // nil
-
-		// replay the latest block
-		return h.state.ApplyBlock(eventCache, appConnConsensus, block, blockMeta.BlockID.PartsHeader, MockMempool{})
-	} else if storeBlockHeight != stateBlockHeight {
-		// unless we failed before committing or saving state (previous 2 case),
-		// the store and state should be at the same height!
-		PanicSanity(Fmt("Expected storeHeight (%d) and stateHeight (%d) to match.", storeBlockHeight, stateBlockHeight))
 	} else {
 		// store is more than one ahead,
 		// so app wants to replay many blocks
+		/*
 
-		// replay all blocks starting with appBlockHeight+1
-		var eventCache types.Fireable // nil
+			// replay all blocks starting with appBlockHeight+1
+			var eventCache types.Fireable // nil
 
-		// TODO: use stateBlockHeight instead and let the consensus state
-		// do the replay
+			// TODO: use stateBlockHeight instead and let the consensus state
+			// do the replay
 
-		var appHash []byte
-		for i := appBlockHeight + 1; i <= storeBlockHeight; i++ {
-			h.nBlocks += 1
-			block := h.store.LoadBlock(i)
-			_, err := execBlockOnProxyApp(eventCache, appConnConsensus, block)
-			if err != nil {
-				log.Warn("Error executing block on proxy app", "height", i, "err", err)
-				return err
+			var appHash []byte
+			for i := appBlockHeight + 1; i <= storeBlockHeight; i++ {
+				h.nBlocks += 1
+				block := h.store.LoadBlock(i)
+				_, err := execBlockOnProxyApp(eventCache, appConnConsensus, block)
+				if err != nil {
+					log.Warn("Error executing block on proxy app", "height", i, "err", err)
+					return err
+				}
+				// Commit block, get hash back
+				res := appConnConsensus.CommitSync()
+				if res.IsErr() {
+					log.Warn("Error in proxyAppConn.CommitSync", "error", res)
+					return res
+				}
+				if res.Log != "" {
+					log.Info("Commit.Log: " + res.Log)
+				}
+				appHash = res.Data
 			}
-			// Commit block, get hash back
-			res := appConnConsensus.CommitSync()
-			if res.IsErr() {
-				log.Warn("Error in proxyAppConn.CommitSync", "error", res)
-				return res
+			if !bytes.Equal(h.state.AppHash, appHash) {
+				return errors.New(Fmt("Tendermint state.AppHash does not match AppHash after replay. Got %X, expected %X", appHash, h.state.AppHash))
 			}
-			if res.Log != "" {
-				log.Info("Commit.Log: " + res.Log)
-			}
-			appHash = res.Data
-		}
-		if !bytes.Equal(h.state.AppHash, appHash) {
-			return errors.New(Fmt("Tendermint state.AppHash does not match AppHash after replay. Got %X, expected %X", appHash, h.state.AppHash))
-		}
+		*/
 		return nil
 	}
 	return nil
 }
+
+//--------------------------------------------------------------------------------
+
+func newMockProxyApp(appHash []byte) proxy.AppConnConsensus {
+	clientCreator := proxy.NewLocalClientCreator(&mockProxyApp{appHash: appHash})
+	cli, _ := clientCreator.NewABCIClient()
+	return proxy.NewAppConnConsensus(cli)
+}
+
+type mockProxyApp struct {
+	abci.BaseApplication
+
+	appHash []byte
+}
+
+func (mock *mockProxyApp) Commit() abci.Result {
+	return abci.NewResultOK(mock.appHash, "")
+}

From 6403b2f4681ab08eb1f34d225ed0232dd4dd6df4 Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Fri, 17 Feb 2017 10:51:05 -0500
Subject: [PATCH 049/132] fixes for handshake replay through consensus

---
 .../handshake_test.go_                        | 63 ++++++++++---------
 consensus/state.go                            | 33 +++-------
 node/node.go                                  | 10 ++-
 state/execution.go                            | 23 ++++---
 4 files changed, 67 insertions(+), 62 deletions(-)
 rename state/execution_test.go => consensus/handshake_test.go_ (78%)

diff --git a/state/execution_test.go b/consensus/handshake_test.go_
similarity index 78%
rename from state/execution_test.go
rename to consensus/handshake_test.go_
index 452e72e1..53d05284 100644
--- a/state/execution_test.go
+++ b/consensus/handshake_test.go_
@@ -1,4 +1,4 @@
-package state
+package consensus
 
 import (
 	"bytes"
@@ -13,6 +13,7 @@ import (
 	"github.com/tendermint/go-crypto"
 	dbm "github.com/tendermint/go-db"
 	"github.com/tendermint/tendermint/proxy"
+	sm "github.com/tendermint/tendermint/state"
 	"github.com/tendermint/tendermint/types"
 )
 
@@ -20,32 +21,25 @@ var (
 	privKey      = crypto.GenPrivKeyEd25519FromSecret([]byte("handshake_test"))
 	chainID      = "handshake_chain"
 	nBlocks      = 5
-	mempool      = MockMempool{}
+	mempool      = sm.MockMempool{}
 	testPartSize = 65536
 )
 
-//---------------------------------------
-// Test block execution
-
-func TestExecBlock(t *testing.T) {
-	// TODO
-}
-
 //---------------------------------------
 // Test handshake/replay
 
 // Sync from scratch
-func TestHandshakeReplayAll(t *testing.T) {
+func _TestHandshakeReplayAll(t *testing.T) {
 	testHandshakeReplay(t, 0)
 }
 
 // Sync many, not from scratch
-func TestHandshakeReplaySome(t *testing.T) {
+func _TestHandshakeReplaySome(t *testing.T) {
 	testHandshakeReplay(t, 1)
 }
 
 // Sync from lagging by one
-func TestHandshakeReplayOne(t *testing.T) {
+func _TestHandshakeReplayOne(t *testing.T) {
 	testHandshakeReplay(t, nBlocks-1)
 }
 
@@ -57,16 +51,18 @@ func TestHandshakeReplayNone(t *testing.T) {
 // Make some blocks. Start a fresh app and apply n blocks. Then restart the app and sync it up with the remaining blocks
 func testHandshakeReplay(t *testing.T, n int) {
 	config := tendermint_test.ResetConfig("proxy_test_")
+	config.Set("chain_id", chainID)
 
 	state, store := stateAndStore(config)
 	clientCreator := proxy.NewLocalClientCreator(dummy.NewPersistentDummyApplication(path.Join(config.GetString("db_dir"), "1")))
 	clientCreator2 := proxy.NewLocalClientCreator(dummy.NewPersistentDummyApplication(path.Join(config.GetString("db_dir"), "2")))
-	proxyApp := proxy.NewAppConns(config, clientCreator, NewHandshaker(config, state, store))
+	proxyApp := proxy.NewAppConns(config, clientCreator, sm.NewHandshaker(config, state, store, ReplayLastBlock))
 	if _, err := proxyApp.Start(); err != nil {
 		t.Fatalf("Error starting proxy app connections: %v", err)
 	}
-	chain := makeBlockchain(t, proxyApp, state)
+	chain, commits := makeBlockchain(t, proxyApp, state)
 	store.chain = chain //
+	store.commits = commits
 	latestAppHash := state.AppHash
 	proxyApp.Stop()
 
@@ -88,7 +84,7 @@ func testHandshakeReplay(t *testing.T, n int) {
 	}
 
 	// now start it with the handshake
-	handshaker := NewHandshaker(config, state, store)
+	handshaker := sm.NewHandshaker(config, state, store, ReplayLastBlock)
 	proxyApp = proxy.NewAppConns(config, clientCreator2, handshaker)
 	if _, err := proxyApp.Start(); err != nil {
 		t.Fatalf("Error starting proxy app connections: %v", err)
@@ -105,8 +101,8 @@ func testHandshakeReplay(t *testing.T, n int) {
 		t.Fatalf("Expected app hashes to match after handshake/replay. got %X, expected %X", res.LastBlockAppHash, latestAppHash)
 	}
 
-	if handshaker.nBlocks != nBlocks-n {
-		t.Fatalf("Expected handshake to sync %d blocks, got %d", nBlocks-n, handshaker.nBlocks)
+	if handshaker.NBlocks() != nBlocks-n {
+		t.Fatalf("Expected handshake to sync %d blocks, got %d", nBlocks-n, handshaker.NBlocks())
 	}
 
 }
@@ -139,7 +135,7 @@ func signCommit(height, round int, hash []byte, header types.PartSetHeader) *typ
 }
 
 // make a blockchain with one validator
-func makeBlockchain(t *testing.T, proxyApp proxy.AppConns, state *State) (blockchain []*types.Block) {
+func makeBlockchain(t *testing.T, proxyApp proxy.AppConns, state *sm.State) (blockchain []*types.Block, commits []*types.Commit) {
 
 	prevHash := state.LastBlockID.Hash
 	lastCommit := new(types.Commit)
@@ -151,7 +147,6 @@ func makeBlockchain(t *testing.T, proxyApp proxy.AppConns, state *State) (blockc
 		block, parts := types.MakeBlock(i, chainID, txsFunc(i), lastCommit,
 			prevBlockID, valHash, state.AppHash, testPartSize)
 		fmt.Println(i)
-		fmt.Println(prevBlockID)
 		fmt.Println(block.LastBlockID)
 		err := state.ApplyBlock(nil, proxyApp.Consensus(), block, block.MakePartSet(testPartSize).Header(), mempool)
 		if err != nil {
@@ -165,37 +160,40 @@ func makeBlockchain(t *testing.T, proxyApp proxy.AppConns, state *State) (blockc
 			t.Fatal(err)
 		}
 
-		blockchain = append(blockchain, block)
 		prevHash = block.Hash()
 		prevParts = parts.Header()
 		lastCommit = voteSet.MakeCommit()
 		prevBlockID = types.BlockID{prevHash, prevParts}
+
+		blockchain = append(blockchain, block)
+		commits = append(commits, lastCommit)
 	}
-	return blockchain
+	return blockchain, commits
 }
 
 // fresh state and mock store
-func stateAndStore(config cfg.Config) (*State, *mockBlockStore) {
+func stateAndStore(config cfg.Config) (*sm.State, *mockBlockStore) {
 	stateDB := dbm.NewMemDB()
-	return MakeGenesisState(stateDB, &types.GenesisDoc{
+	return sm.MakeGenesisState(stateDB, &types.GenesisDoc{
 		ChainID: chainID,
 		Validators: []types.GenesisValidator{
 			types.GenesisValidator{privKey.PubKey(), 10000, "test"},
 		},
 		AppHash: nil,
-	}), NewMockBlockStore(config, nil)
+	}), NewMockBlockStore(config)
 }
 
 //----------------------------------
 // mock block store
 
 type mockBlockStore struct {
-	config cfg.Config
-	chain  []*types.Block
+	config  cfg.Config
+	chain   []*types.Block
+	commits []*types.Commit
 }
 
-func NewMockBlockStore(config cfg.Config, chain []*types.Block) *mockBlockStore {
-	return &mockBlockStore{config, chain}
+func NewMockBlockStore(config cfg.Config) *mockBlockStore {
+	return &mockBlockStore{config, nil, nil}
 }
 
 func (bs *mockBlockStore) Height() int                       { return len(bs.chain) }
@@ -207,3 +205,12 @@ func (bs *mockBlockStore) LoadBlockMeta(height int) *types.BlockMeta {
 		Header:  block.Header,
 	}
 }
+func (bs *mockBlockStore) LoadBlockPart(height int, index int) *types.Part { return nil }
+func (bs *mockBlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit) {
+}
+func (bs *mockBlockStore) LoadBlockCommit(height int) *types.Commit {
+	return bs.commits[height-1]
+}
+func (bs *mockBlockStore) LoadSeenCommit(height int) *types.Commit {
+	return bs.commits[height-1]
+}
diff --git a/consensus/state.go b/consensus/state.go
index c872233d..231d74bb 100644
--- a/consensus/state.go
+++ b/consensus/state.go
@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-	"io"
 	"reflect"
 	"sync"
 	"time"
@@ -14,7 +13,6 @@ import (
 	. "github.com/tendermint/go-common"
 	cfg "github.com/tendermint/go-config"
 	"github.com/tendermint/go-wire"
-	bc "github.com/tendermint/tendermint/blockchain"
 	"github.com/tendermint/tendermint/proxy"
 	sm "github.com/tendermint/tendermint/state"
 	"github.com/tendermint/tendermint/types"
@@ -225,7 +223,7 @@ type ConsensusState struct {
 
 	config       cfg.Config
 	proxyAppConn proxy.AppConnConsensus
-	blockStore   *bc.BlockStore
+	blockStore   sm.BlockStore
 	mempool      sm.Mempool
 
 	privValidator PrivValidator // for signing votes
@@ -256,18 +254,20 @@ type ConsensusState struct {
 
 func ReplayLastBlock(config cfg.Config, state *sm.State, proxyApp proxy.AppConnConsensus, blockStore sm.BlockStore) {
 	mempool := sm.MockMempool{}
-	cs := NewConsensusState(config, state, proxyApp, blockStore.(*bc.BlockStore), mempool)
+	cs := NewConsensusState(config, state, proxyApp, blockStore, mempool)
 
 	evsw := types.NewEventSwitch()
+	evsw.Start()
 	cs.SetEventSwitch(evsw)
-	newBlockCh := subscribeToEvent(evsw, "consensus-replay", types.EventStringNewBlock(), 0)
+	newBlockCh := subscribeToEvent(evsw, "consensus-replay", types.EventStringNewBlock(), 1)
 
+	// run through the WAL, commit new block, stop
 	cs.Start()
 	<-newBlockCh
 	cs.Stop()
 }
 
-func NewConsensusState(config cfg.Config, state *sm.State, proxyAppConn proxy.AppConnConsensus, blockStore *bc.BlockStore, mempool sm.Mempool) *ConsensusState {
+func NewConsensusState(config cfg.Config, state *sm.State, proxyAppConn proxy.AppConnConsensus, blockStore sm.BlockStore, mempool sm.Mempool) *ConsensusState {
 	cs := &ConsensusState{
 		config:           config,
 		proxyAppConn:     proxyAppConn,
@@ -366,23 +366,6 @@ func (cs *ConsensusState) OnStart() error {
 		return err
 	}
 
-	// If the latest block was applied in the abci handshake,
-	// we may not have written the current height to the wal,
-	// so check here and write it if not found.
-	// TODO: remove this and run the handhsake/replay
-	// through the consensus state with a mock app
-	gr, found, err := cs.wal.group.Search("#HEIGHT: ", makeHeightSearchFunc(cs.Height))
-	if (err == io.EOF || !found) && cs.Step == RoundStepNewHeight {
-		log.Warn("Height not found in wal. Writing new height", "height", cs.Height)
-		rs := cs.RoundStateEvent()
-		cs.wal.Save(rs)
-	} else if err != nil {
-		return err
-	}
-	if gr != nil {
-		gr.Close()
-	}
-
 	// we need the timeoutRoutine for replay so
 	//  we don't block on the tick chan.
 	// NOTE: we will get a build up of garbage go routines
@@ -581,7 +564,6 @@ func (cs *ConsensusState) updateToState(state *sm.State) {
 
 	// Reset fields based on state.
 	validators := state.Validators
-	height := state.LastBlockHeight + 1 // Next desired block height
 	lastPrecommits := (*types.VoteSet)(nil)
 	if cs.CommitRound > -1 && cs.Votes != nil {
 		if !cs.Votes.Precommits(cs.CommitRound).HasTwoThirdsMajority() {
@@ -590,6 +572,9 @@ func (cs *ConsensusState) updateToState(state *sm.State) {
 		lastPrecommits = cs.Votes.Precommits(cs.CommitRound)
 	}
 
+	// Next desired block height
+	height := state.LastBlockHeight + 1
+
 	// RoundState fields
 	cs.updateHeight(height)
 	cs.updateRoundStep(0, RoundStepNewHeight)
diff --git a/node/node.go b/node/node.go
index 8ccd47c0..755658b7 100644
--- a/node/node.go
+++ b/node/node.go
@@ -63,15 +63,19 @@ func NewNode(config cfg.Config, privValidator *types.PrivValidator, clientCreato
 	stateDB := dbm.NewDB("state", config.GetString("db_backend"), config.GetString("db_dir"))
 	state := sm.GetState(config, stateDB)
 
+	// add the chainid and number of validators to the global config
+	config.Set("chain_id", state.ChainID)
+	config.Set("num_vals", state.Validators.Size())
+
 	// Create the proxyApp, which manages connections (consensus, mempool, query)
+	// and sync tendermint and the app by replaying any necessary blocks
 	proxyApp := proxy.NewAppConns(config, clientCreator, sm.NewHandshaker(config, state, blockStore, consensus.ReplayLastBlock))
 	if _, err := proxyApp.Start(); err != nil {
 		cmn.Exit(cmn.Fmt("Error starting proxy app connections: %v", err))
 	}
 
-	// add the chainid and number of validators to the global config
-	config.Set("chain_id", state.ChainID)
-	config.Set("num_vals", state.Validators.Size())
+	// reload the state (it may have been updated by the handshake)
+	state = sm.LoadState(stateDB)
 
 	// Generate node PrivKey
 	privKey := crypto.GenPrivKeyEd25519()
diff --git a/state/execution.go b/state/execution.go
index e4b7cff4..c767352f 100644
--- a/state/execution.go
+++ b/state/execution.go
@@ -298,8 +298,15 @@ func (m MockMempool) Update(height int, txs types.Txs)                   {}
 // TODO: Should we move blockchain/store.go to its own package?
 type BlockStore interface {
 	Height() int
-	LoadBlock(height int) *types.Block
+
 	LoadBlockMeta(height int) *types.BlockMeta
+	LoadBlock(height int) *types.Block
+	LoadBlockPart(height int, index int) *types.Part
+
+	SaveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit)
+
+	LoadBlockCommit(height int) *types.Commit
+	LoadSeenCommit(height int) *types.Commit
 }
 
 type blockReplayFunc func(cfg.Config, *State, proxy.AppConnConsensus, BlockStore)
@@ -317,6 +324,10 @@ func NewHandshaker(config cfg.Config, state *State, store BlockStore, f blockRep
 	return &Handshaker{config, state, store, f, 0}
 }
 
+func (h *Handshaker) NBlocks() int {
+	return h.nBlocks
+}
+
 // TODO: retry the handshake/replay if it fails ?
 func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error {
 	// handshake is done via info request on the query conn
@@ -338,9 +349,6 @@ func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error {
 		return errors.New(Fmt("Error on replay: %v", err))
 	}
 
-	// Save the state
-	h.state.Save()
-
 	// TODO: (on restart) replay mempool
 
 	return nil
@@ -359,16 +367,17 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, proxyApp p
 		// if the app is ahead, there's nothing we can do
 		return ErrAppBlockHeightTooHigh{storeBlockHeight, appBlockHeight}
 
-	} else if storeBlockHeight == appBlockHeight {
-		// We already ran Commit, so run through consensus with mock app
+	} else if storeBlockHeight == appBlockHeight && storeBlockHeight == stateBlockHeight+1 {
+		// We already ran Commit, but didn't save the state, so run through consensus with mock app
 		mockApp := newMockProxyApp(appHash)
-
+		log.Info("Replay last block using mock app")
 		h.replayLastBlock(h.config, h.state, mockApp, h.store)
 
 	} else if storeBlockHeight == appBlockHeight+1 {
 		// We crashed after saving the block
 		// but before Commit (both the state and app are behind),
 		// so run through consensus with the real app
+		log.Info("Replay last block using real app")
 		h.replayLastBlock(h.config, h.state, proxyApp.Consensus(), h.store)
 
 	} else {

From edc5e272dbacff29493b98860ff470c29f53f09a Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Fri, 17 Feb 2017 10:57:09 -0500
Subject: [PATCH 050/132] consensus: nice error msg if ApplyBlock fails

---
 consensus/state.go | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/consensus/state.go b/consensus/state.go
index 231d74bb..30ca8e9f 100644
--- a/consensus/state.go
+++ b/consensus/state.go
@@ -1232,7 +1232,8 @@ func (cs *ConsensusState) finalizeCommit(height int) {
 		seenCommit := precommits.MakeCommit()
 		cs.blockStore.SaveBlock(block, blockParts, seenCommit)
 	} else {
-		log.Warn("Why are we finalizeCommitting a block height we already have?", "height", block.Height)
+		// Happens during replay if we already saved the block but didn't commit
+		log.Notice("Calling finalizeCommit on already stored block", "height", block.Height)
 	}
 
 	fail.Fail() // XXX
@@ -1247,7 +1248,8 @@ func (cs *ConsensusState) finalizeCommit(height int) {
 	// NOTE: the block.AppHash wont reflect these txs until the next block
 	err := stateCopy.ApplyBlock(eventCache, cs.proxyAppConn, block, blockParts.Header(), cs.mempool)
 	if err != nil {
-		// TODO!
+		log.Warn("Error on ApplyBlock. Did the application crash? Please restart tendermint", "error", err)
+		return
 	}
 
 	fail.Fail() // XXX

From 3c5adebcd3f5817304baac652cb80a1f76ede7ac Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Fri, 17 Feb 2017 11:32:56 -0500
Subject: [PATCH 051/132] applyBlock to simplify replay of many blocks. still
 wip

---
 state/execution.go | 69 +++++++++++++++++++++++++---------------------
 1 file changed, 38 insertions(+), 31 deletions(-)

diff --git a/state/execution.go b/state/execution.go
index c767352f..88b80e48 100644
--- a/state/execution.go
+++ b/state/execution.go
@@ -272,6 +272,27 @@ func (s *State) CommitStateUpdateMempool(proxyAppConn proxy.AppConnConsensus, bl
 	return nil
 }
 
+// apply a nd commit a block, but with out all the state validation
+// returns the application root hash (result of abci.Commit)
+func applyBlock(appConnConsensus proxy.AppConnConsensus, block *types.Block) ([]byte, error) {
+	var eventCache types.Fireable // nil
+	_, err := execBlockOnProxyApp(eventCache, appConnConsensus, block)
+	if err != nil {
+		log.Warn("Error executing block on proxy app", "height", i, "err", err)
+		return nil, err
+	}
+	// Commit block, get hash back
+	res := appConnConsensus.CommitSync()
+	if res.IsErr() {
+		log.Warn("Error in proxyAppConn.CommitSync", "error", res)
+		return nil, res
+	}
+	if res.Log != "" {
+		log.Info("Commit.Log: " + res.Log)
+	}
+	return res.Data, nil
+}
+
 // Updates to the mempool need to be synchronized with committing a block
 // so apps can reset their transient state on Commit
 type Mempool interface {
@@ -382,39 +403,25 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, proxyApp p
 
 	} else {
 		// store is more than one ahead,
-		// so app wants to replay many blocks
-		/*
+		// so app wants to replay many blocks.
+		// replay all blocks from appBlockHeight+1 to storeBlockHeight-1.
+		// Replay the final block through consensus
 
-			// replay all blocks starting with appBlockHeight+1
-			var eventCache types.Fireable // nil
-
-			// TODO: use stateBlockHeight instead and let the consensus state
-			// do the replay
-
-			var appHash []byte
-			for i := appBlockHeight + 1; i <= storeBlockHeight; i++ {
-				h.nBlocks += 1
-				block := h.store.LoadBlock(i)
-				_, err := execBlockOnProxyApp(eventCache, appConnConsensus, block)
-				if err != nil {
-					log.Warn("Error executing block on proxy app", "height", i, "err", err)
-					return err
-				}
-				// Commit block, get hash back
-				res := appConnConsensus.CommitSync()
-				if res.IsErr() {
-					log.Warn("Error in proxyAppConn.CommitSync", "error", res)
-					return res
-				}
-				if res.Log != "" {
-					log.Info("Commit.Log: " + res.Log)
-				}
-				appHash = res.Data
+		var appHash []byte
+		var err error
+		for i := appBlockHeight + 1; i <= storeBlockHeight-1; i++ {
+			h.nBlocks += 1
+			block := h.store.LoadBlock(i)
+			appHash, err = applyBlock(proxyApp.Consensus(), block)
+			if err != nil {
+				return err
 			}
-			if !bytes.Equal(h.state.AppHash, appHash) {
-				return errors.New(Fmt("Tendermint state.AppHash does not match AppHash after replay. Got %X, expected %X", appHash, h.state.AppHash))
-			}
-		*/
+		}
+
+		h.replayLastBlock(h.config, h.state, proxyApp.Consensus(), h.store)
+		if !bytes.Equal(h.state.AppHash, appHash) {
+			return errors.New(Fmt("Tendermint state.AppHash does not match AppHash after replay. Got %X, expected %X", appHash, h.state.AppHash))
+		}
 		return nil
 	}
 	return nil

From 0bec99fbd417df47e0a307ca62fb78477a703e12 Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Fri, 17 Feb 2017 19:12:05 -0500
Subject: [PATCH 052/132] consensus: handshake replay test using wal

---
 cmd/tendermint/reset_priv_validator.go |   6 +-
 config/tendermint/config.go            |   2 +-
 config/tendermint_test/config.go       |   2 +-
 consensus/common_test.go               |   8 +-
 consensus/handshake_test.go_           | 216 --------------
 consensus/replay_test.go               | 377 +++++++++++++++++++++++--
 consensus/state.go                     |  11 +-
 consensus/test_data/build.sh           |  53 +++-
 consensus/test_data/many_blocks.cswal  |  65 +++++
 consensus/wal.go                       |   4 +-
 state/execution.go                     |  14 +-
 11 files changed, 496 insertions(+), 262 deletions(-)
 delete mode 100644 consensus/handshake_test.go_
 create mode 100644 consensus/test_data/many_blocks.cswal

diff --git a/cmd/tendermint/reset_priv_validator.go b/cmd/tendermint/reset_priv_validator.go
index 9ecbaa90..5bf3ba69 100644
--- a/cmd/tendermint/reset_priv_validator.go
+++ b/cmd/tendermint/reset_priv_validator.go
@@ -6,15 +6,15 @@ import (
 	"github.com/tendermint/tendermint/types"
 )
 
-// NOTE: this is totally unsafe.
+// XXX: this is totally unsafe.
 // it's only suitable for testnets.
 func reset_all() {
 	reset_priv_validator()
 	os.RemoveAll(config.GetString("db_dir"))
-	os.RemoveAll(config.GetString("cs_wal_dir"))
+	os.Remove(config.GetString("cs_wal_file"))
 }
 
-// NOTE: this is totally unsafe.
+// XXX: this is totally unsafe.
 // it's only suitable for testnets.
 func reset_priv_validator() {
 	// Get PrivValidator
diff --git a/config/tendermint/config.go b/config/tendermint/config.go
index ab4bf00b..c210d6e0 100644
--- a/config/tendermint/config.go
+++ b/config/tendermint/config.go
@@ -72,7 +72,7 @@ func GetConfig(rootDir string) cfg.Config {
 	mapConfig.SetDefault("grpc_laddr", "")
 	mapConfig.SetDefault("prof_laddr", "")
 	mapConfig.SetDefault("revision_file", rootDir+"/revision")
-	mapConfig.SetDefault("cs_wal_dir", rootDir+"/data/cs.wal")
+	mapConfig.SetDefault("cs_wal_file", rootDir+"/data/cs.wal/wal")
 	mapConfig.SetDefault("cs_wal_light", false)
 	mapConfig.SetDefault("filter_peers", false)
 
diff --git a/config/tendermint_test/config.go b/config/tendermint_test/config.go
index 930cf27e..55e3adb4 100644
--- a/config/tendermint_test/config.go
+++ b/config/tendermint_test/config.go
@@ -86,7 +86,7 @@ func ResetConfig(localPath string) cfg.Config {
 	mapConfig.SetDefault("grpc_laddr", "tcp://0.0.0.0:36658")
 	mapConfig.SetDefault("prof_laddr", "")
 	mapConfig.SetDefault("revision_file", rootDir+"/revision")
-	mapConfig.SetDefault("cs_wal_dir", rootDir+"/data/cs.wal")
+	mapConfig.SetDefault("cs_wal_file", rootDir+"/data/cs.wal/wal")
 	mapConfig.SetDefault("cs_wal_light", false)
 	mapConfig.SetDefault("filter_peers", false)
 
diff --git a/consensus/common_test.go b/consensus/common_test.go
index c345fe66..334c66dc 100644
--- a/consensus/common_test.go
+++ b/consensus/common_test.go
@@ -11,6 +11,8 @@ import (
 	"testing"
 	"time"
 
+	abcicli "github.com/tendermint/abci/client"
+	abci "github.com/tendermint/abci/types"
 	. "github.com/tendermint/go-common"
 	cfg "github.com/tendermint/go-config"
 	dbm "github.com/tendermint/go-db"
@@ -20,8 +22,6 @@ import (
 	mempl "github.com/tendermint/tendermint/mempool"
 	sm "github.com/tendermint/tendermint/state"
 	"github.com/tendermint/tendermint/types"
-	abcicli "github.com/tendermint/abci/client"
-	abci "github.com/tendermint/abci/types"
 
 	"github.com/tendermint/abci/example/counter"
 	"github.com/tendermint/abci/example/dummy"
@@ -320,7 +320,7 @@ func randConsensusNet(nValidators int, testName string, tickerFunc func() Timeou
 		state := sm.MakeGenesisState(db, genDoc)
 		state.Save()
 		thisConfig := tendermint_test.ResetConfig(Fmt("%s_%d", testName, i))
-		ensureDir(thisConfig.GetString("cs_wal_dir"), 0700) // dir for wal
+		ensureDir(path.Dir(thisConfig.GetString("cs_wal_file")), 0700) // dir for wal
 		css[i] = newConsensusStateWithConfig(thisConfig, state, privVals[i], appFunc())
 		css[i].SetTimeoutTicker(tickerFunc())
 	}
@@ -336,7 +336,7 @@ func randConsensusNetWithPeers(nValidators, nPeers int, testName string, tickerF
 		state := sm.MakeGenesisState(db, genDoc)
 		state.Save()
 		thisConfig := tendermint_test.ResetConfig(Fmt("%s_%d", testName, i))
-		ensureDir(thisConfig.GetString("cs_wal_dir"), 0700) // dir for wal
+		ensureDir(path.Dir(thisConfig.GetString("cs_wal_file")), 0700) // dir for wal
 		var privVal *types.PrivValidator
 		if i < nValidators {
 			privVal = privVals[i]
diff --git a/consensus/handshake_test.go_ b/consensus/handshake_test.go_
deleted file mode 100644
index 53d05284..00000000
--- a/consensus/handshake_test.go_
+++ /dev/null
@@ -1,216 +0,0 @@
-package consensus
-
-import (
-	"bytes"
-	"fmt"
-	"path"
-	"testing"
-
-	"github.com/tendermint/tendermint/config/tendermint_test"
-	//	. "github.com/tendermint/go-common"
-	"github.com/tendermint/abci/example/dummy"
-	cfg "github.com/tendermint/go-config"
-	"github.com/tendermint/go-crypto"
-	dbm "github.com/tendermint/go-db"
-	"github.com/tendermint/tendermint/proxy"
-	sm "github.com/tendermint/tendermint/state"
-	"github.com/tendermint/tendermint/types"
-)
-
-var (
-	privKey      = crypto.GenPrivKeyEd25519FromSecret([]byte("handshake_test"))
-	chainID      = "handshake_chain"
-	nBlocks      = 5
-	mempool      = sm.MockMempool{}
-	testPartSize = 65536
-)
-
-//---------------------------------------
-// Test handshake/replay
-
-// Sync from scratch
-func _TestHandshakeReplayAll(t *testing.T) {
-	testHandshakeReplay(t, 0)
-}
-
-// Sync many, not from scratch
-func _TestHandshakeReplaySome(t *testing.T) {
-	testHandshakeReplay(t, 1)
-}
-
-// Sync from lagging by one
-func _TestHandshakeReplayOne(t *testing.T) {
-	testHandshakeReplay(t, nBlocks-1)
-}
-
-// Sync from caught up
-func TestHandshakeReplayNone(t *testing.T) {
-	testHandshakeReplay(t, nBlocks)
-}
-
-// Make some blocks. Start a fresh app and apply n blocks. Then restart the app and sync it up with the remaining blocks
-func testHandshakeReplay(t *testing.T, n int) {
-	config := tendermint_test.ResetConfig("proxy_test_")
-	config.Set("chain_id", chainID)
-
-	state, store := stateAndStore(config)
-	clientCreator := proxy.NewLocalClientCreator(dummy.NewPersistentDummyApplication(path.Join(config.GetString("db_dir"), "1")))
-	clientCreator2 := proxy.NewLocalClientCreator(dummy.NewPersistentDummyApplication(path.Join(config.GetString("db_dir"), "2")))
-	proxyApp := proxy.NewAppConns(config, clientCreator, sm.NewHandshaker(config, state, store, ReplayLastBlock))
-	if _, err := proxyApp.Start(); err != nil {
-		t.Fatalf("Error starting proxy app connections: %v", err)
-	}
-	chain, commits := makeBlockchain(t, proxyApp, state)
-	store.chain = chain //
-	store.commits = commits
-	latestAppHash := state.AppHash
-	proxyApp.Stop()
-
-	if n > 0 {
-		// start a new app without handshake, play n blocks
-		proxyApp = proxy.NewAppConns(config, clientCreator2, nil)
-		if _, err := proxyApp.Start(); err != nil {
-			t.Fatalf("Error starting proxy app connections: %v", err)
-		}
-		state2, _ := stateAndStore(config)
-		for i := 0; i < n; i++ {
-			block := chain[i]
-			err := state2.ApplyBlock(nil, proxyApp.Consensus(), block, block.MakePartSet(testPartSize).Header(), mempool)
-			if err != nil {
-				t.Fatal(err)
-			}
-		}
-		proxyApp.Stop()
-	}
-
-	// now start it with the handshake
-	handshaker := sm.NewHandshaker(config, state, store, ReplayLastBlock)
-	proxyApp = proxy.NewAppConns(config, clientCreator2, handshaker)
-	if _, err := proxyApp.Start(); err != nil {
-		t.Fatalf("Error starting proxy app connections: %v", err)
-	}
-
-	// get the latest app hash from the app
-	res, err := proxyApp.Query().InfoSync()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// the app hash should be synced up
-	if !bytes.Equal(latestAppHash, res.LastBlockAppHash) {
-		t.Fatalf("Expected app hashes to match after handshake/replay. got %X, expected %X", res.LastBlockAppHash, latestAppHash)
-	}
-
-	if handshaker.NBlocks() != nBlocks-n {
-		t.Fatalf("Expected handshake to sync %d blocks, got %d", nBlocks-n, handshaker.NBlocks())
-	}
-
-}
-
-//--------------------------
-// utils for making blocks
-
-// make some bogus txs
-func txsFunc(blockNum int) (txs []types.Tx) {
-	for i := 0; i < 10; i++ {
-		txs = append(txs, types.Tx([]byte{byte(blockNum), byte(i)}))
-	}
-	return txs
-}
-
-// sign a commit vote
-func signCommit(height, round int, hash []byte, header types.PartSetHeader) *types.Vote {
-	vote := &types.Vote{
-		ValidatorIndex:   0,
-		ValidatorAddress: privKey.PubKey().Address(),
-		Height:           height,
-		Round:            round,
-		Type:             types.VoteTypePrecommit,
-		BlockID:          types.BlockID{hash, header},
-	}
-
-	sig := privKey.Sign(types.SignBytes(chainID, vote))
-	vote.Signature = sig
-	return vote
-}
-
-// make a blockchain with one validator
-func makeBlockchain(t *testing.T, proxyApp proxy.AppConns, state *sm.State) (blockchain []*types.Block, commits []*types.Commit) {
-
-	prevHash := state.LastBlockID.Hash
-	lastCommit := new(types.Commit)
-	prevParts := types.PartSetHeader{}
-	valHash := state.Validators.Hash()
-	prevBlockID := types.BlockID{prevHash, prevParts}
-
-	for i := 1; i < nBlocks+1; i++ {
-		block, parts := types.MakeBlock(i, chainID, txsFunc(i), lastCommit,
-			prevBlockID, valHash, state.AppHash, testPartSize)
-		fmt.Println(i)
-		fmt.Println(block.LastBlockID)
-		err := state.ApplyBlock(nil, proxyApp.Consensus(), block, block.MakePartSet(testPartSize).Header(), mempool)
-		if err != nil {
-			t.Fatal(i, err)
-		}
-
-		voteSet := types.NewVoteSet(chainID, i, 0, types.VoteTypePrecommit, state.Validators)
-		vote := signCommit(i, 0, block.Hash(), parts.Header())
-		_, err = voteSet.AddVote(vote)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		prevHash = block.Hash()
-		prevParts = parts.Header()
-		lastCommit = voteSet.MakeCommit()
-		prevBlockID = types.BlockID{prevHash, prevParts}
-
-		blockchain = append(blockchain, block)
-		commits = append(commits, lastCommit)
-	}
-	return blockchain, commits
-}
-
-// fresh state and mock store
-func stateAndStore(config cfg.Config) (*sm.State, *mockBlockStore) {
-	stateDB := dbm.NewMemDB()
-	return sm.MakeGenesisState(stateDB, &types.GenesisDoc{
-		ChainID: chainID,
-		Validators: []types.GenesisValidator{
-			types.GenesisValidator{privKey.PubKey(), 10000, "test"},
-		},
-		AppHash: nil,
-	}), NewMockBlockStore(config)
-}
-
-//----------------------------------
-// mock block store
-
-type mockBlockStore struct {
-	config  cfg.Config
-	chain   []*types.Block
-	commits []*types.Commit
-}
-
-func NewMockBlockStore(config cfg.Config) *mockBlockStore {
-	return &mockBlockStore{config, nil, nil}
-}
-
-func (bs *mockBlockStore) Height() int                       { return len(bs.chain) }
-func (bs *mockBlockStore) LoadBlock(height int) *types.Block { return bs.chain[height-1] }
-func (bs *mockBlockStore) LoadBlockMeta(height int) *types.BlockMeta {
-	block := bs.chain[height-1]
-	return &types.BlockMeta{
-		BlockID: types.BlockID{block.Hash(), block.MakePartSet(bs.config.GetInt("block_part_size")).Header()},
-		Header:  block.Header,
-	}
-}
-func (bs *mockBlockStore) LoadBlockPart(height int, index int) *types.Part { return nil }
-func (bs *mockBlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit) {
-}
-func (bs *mockBlockStore) LoadBlockCommit(height int) *types.Commit {
-	return bs.commits[height-1]
-}
-func (bs *mockBlockStore) LoadSeenCommit(height int) *types.Commit {
-	return bs.commits[height-1]
-}
diff --git a/consensus/replay_test.go b/consensus/replay_test.go
index d60fa9f2..a9123a8e 100644
--- a/consensus/replay_test.go
+++ b/consensus/replay_test.go
@@ -1,7 +1,10 @@
 package consensus
 
 import (
+	"bytes"
+	"errors"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
 	"path"
@@ -11,8 +14,14 @@ import (
 
 	"github.com/tendermint/tendermint/config/tendermint_test"
 
+	"github.com/tendermint/abci/example/dummy"
 	. "github.com/tendermint/go-common"
+	cfg "github.com/tendermint/go-config"
+	"github.com/tendermint/go-crypto"
+	dbm "github.com/tendermint/go-db"
 	"github.com/tendermint/go-wire"
+	"github.com/tendermint/tendermint/proxy"
+	sm "github.com/tendermint/tendermint/state"
 	"github.com/tendermint/tendermint/types"
 )
 
@@ -20,14 +29,23 @@ func init() {
 	config = tendermint_test.ResetConfig("consensus_replay_test")
 }
 
-// TODO: these tests ensure we can always recover from any state of the wal,
-// assuming it comes with a correct related state for the priv_validator.json.
-// It would be better to verify explicitly which states we can recover from without the wal
-// and which ones we need the wal for - then we'd also be able to only flush the
-// wal writer when we need to, instead of with every message.
+// These tests ensure we can always recover from failure at any part of the consensus process.
+// There are two general failure scenarios: failure during consensus, and failure while applying the block.
+// Only the latter interacts with the app and store,
+// but the former has to deal with restrictions on re-use of priv_validator keys.
+// The `WAL Tests` are for failures during the consensus;
+// the `Handshake Tests` are for failures in applying the block.
+// With the help of the WAL, we can recover from it all!
 
 var data_dir = path.Join(GoPath, "src/github.com/tendermint/tendermint/consensus", "test_data")
 
+//------------------------------------------------------------------------------------------
+// WAL Tests
+
+// TODO: It would be better to verify explicitly which states we can recover from without the wal
+// and which ones we need the wal for - then we'd also be able to only flush the
+// wal writer when we need to, instead of with every message.
+
 // the priv validator changes step at these lines for a block with 1 val and 1 part
 var baseStepChanges = []int{3, 6, 8}
 
@@ -85,18 +103,19 @@ func readWAL(p string) string {
 
 func writeWAL(walMsgs string) string {
 	tempDir := os.TempDir()
-	walDir := tempDir + "/wal" + RandStr(12)
+	walDir := path.Join(tempDir, "/wal"+RandStr(12))
+	walFile := path.Join(walDir, "wal")
 	// Create WAL directory
 	err := EnsureDir(walDir, 0700)
 	if err != nil {
 		panic(err)
 	}
 	// Write the needed WAL to file
-	err = WriteFile(walDir+"/wal", []byte(walMsgs), 0600)
+	err = WriteFile(walFile, []byte(walMsgs), 0600)
 	if err != nil {
 		panic(err)
 	}
-	return walDir
+	return walFile
 }
 
 func waitForBlock(newBlockCh chan interface{}, thisCase *testCase, i int) {
@@ -108,10 +127,10 @@ func waitForBlock(newBlockCh chan interface{}, thisCase *testCase, i int) {
 	}
 }
 
-func runReplayTest(t *testing.T, cs *ConsensusState, walDir string, newBlockCh chan interface{},
+func runReplayTest(t *testing.T, cs *ConsensusState, walFile string, newBlockCh chan interface{},
 	thisCase *testCase, i int) {
 
-	cs.config.Set("cs_wal_dir", walDir)
+	cs.config.Set("cs_wal_file", walFile)
 	cs.Start()
 	// Wait to make a new block.
 	// This is just a signal that we haven't halted; its not something contained in the WAL itself.
@@ -148,7 +167,7 @@ func setupReplayTest(thisCase *testCase, nLines int, crashAfter bool) (*Consensu
 	lastMsg := split[nLines]
 
 	// we write those lines up to (not including) one with the signature
-	walDir := writeWAL(strings.Join(split[:nLines], "\n") + "\n")
+	walFile := writeWAL(strings.Join(split[:nLines], "\n") + "\n")
 
 	cs := fixedConsensusStateDummy()
 
@@ -160,7 +179,7 @@ func setupReplayTest(thisCase *testCase, nLines int, crashAfter bool) (*Consensu
 
 	newBlockCh := subscribeToEvent(cs.evsw, "tester", types.EventStringNewBlock(), 1)
 
-	return cs, newBlockCh, lastMsg, walDir
+	return cs, newBlockCh, lastMsg, walFile
 }
 
 func readTimedWALMessage(t *testing.T, walMsg string) TimedWALMessage {
@@ -177,12 +196,12 @@ func readTimedWALMessage(t *testing.T, walMsg string) TimedWALMessage {
 // Test the log at every iteration, and set the privVal last step
 // as if the log was written after signing, before the crash
 
-func TestReplayCrashAfterWrite(t *testing.T) {
+func TestWALCrashAfterWrite(t *testing.T) {
 	for _, thisCase := range testCases {
 		split := strings.Split(thisCase.log, "\n")
 		for i := 0; i < len(split)-1; i++ {
-			cs, newBlockCh, _, walDir := setupReplayTest(thisCase, i+1, true)
-			runReplayTest(t, cs, walDir, newBlockCh, thisCase, i+1)
+			cs, newBlockCh, _, walFile := setupReplayTest(thisCase, i+1, true)
+			runReplayTest(t, cs, walFile, newBlockCh, thisCase, i+1)
 		}
 	}
 }
@@ -191,27 +210,27 @@ func TestReplayCrashAfterWrite(t *testing.T) {
 // Test the log as if we crashed after signing but before writing.
 // This relies on privValidator.LastSignature being set
 
-func TestReplayCrashBeforeWritePropose(t *testing.T) {
+func TestWALCrashBeforeWritePropose(t *testing.T) {
 	for _, thisCase := range testCases {
 		lineNum := thisCase.proposeLine
 		// setup replay test where last message is a proposal
-		cs, newBlockCh, proposalMsg, walDir := setupReplayTest(thisCase, lineNum, false)
+		cs, newBlockCh, proposalMsg, walFile := setupReplayTest(thisCase, lineNum, false)
 		msg := readTimedWALMessage(t, proposalMsg)
 		proposal := msg.Msg.(msgInfo).Msg.(*ProposalMessage)
 		// Set LastSig
 		toPV(cs.privValidator).LastSignBytes = types.SignBytes(cs.state.ChainID, proposal.Proposal)
 		toPV(cs.privValidator).LastSignature = proposal.Proposal.Signature
-		runReplayTest(t, cs, walDir, newBlockCh, thisCase, lineNum)
+		runReplayTest(t, cs, walFile, newBlockCh, thisCase, lineNum)
 	}
 }
 
-func TestReplayCrashBeforeWritePrevote(t *testing.T) {
+func TestWALCrashBeforeWritePrevote(t *testing.T) {
 	for _, thisCase := range testCases {
 		testReplayCrashBeforeWriteVote(t, thisCase, thisCase.prevoteLine, types.EventStringCompleteProposal())
 	}
 }
 
-func TestReplayCrashBeforeWritePrecommit(t *testing.T) {
+func TestWALCrashBeforeWritePrecommit(t *testing.T) {
 	for _, thisCase := range testCases {
 		testReplayCrashBeforeWriteVote(t, thisCase, thisCase.precommitLine, types.EventStringPolka())
 	}
@@ -219,7 +238,7 @@ func TestReplayCrashBeforeWritePrecommit(t *testing.T) {
 
 func testReplayCrashBeforeWriteVote(t *testing.T, thisCase *testCase, lineNum int, eventString string) {
 	// setup replay test where last message is a vote
-	cs, newBlockCh, voteMsg, walDir := setupReplayTest(thisCase, lineNum, false)
+	cs, newBlockCh, voteMsg, walFile := setupReplayTest(thisCase, lineNum, false)
 	types.AddListenerForEvent(cs.evsw, "tester", eventString, func(data types.TMEventData) {
 		msg := readTimedWALMessage(t, voteMsg)
 		vote := msg.Msg.(msgInfo).Msg.(*VoteMessage)
@@ -227,5 +246,319 @@ func testReplayCrashBeforeWriteVote(t *testing.T, thisCase *testCase, lineNum in
 		toPV(cs.privValidator).LastSignBytes = types.SignBytes(cs.state.ChainID, vote.Vote)
 		toPV(cs.privValidator).LastSignature = vote.Vote.Signature
 	})
-	runReplayTest(t, cs, walDir, newBlockCh, thisCase, lineNum)
+	runReplayTest(t, cs, walFile, newBlockCh, thisCase, lineNum)
+}
+
+//------------------------------------------------------------------------------------------
+// Handshake Tests
+
+var (
+	NUM_BLOCKS = 6 // number of blocks in the test_data/many_blocks.cswal
+	mempool    = sm.MockMempool{}
+
+	testPartSize int
+)
+
+//---------------------------------------
+// Test handshake/replay
+
+// Sync from scratch
+func TestHandshakeReplayAll(t *testing.T) {
+	testHandshakeReplay(t, 0)
+}
+
+// Sync many, not from scratch
+func TestHandshakeReplaySome(t *testing.T) {
+	testHandshakeReplay(t, 1)
+}
+
+// Sync from lagging by one
+func TestHandshakeReplayOne(t *testing.T) {
+	testHandshakeReplay(t, NUM_BLOCKS-1)
+}
+
+// Sync from caught up
+func TestHandshakeReplayNone(t *testing.T) {
+	testHandshakeReplay(t, NUM_BLOCKS)
+}
+
+// Make some blocks. Start a fresh app and apply nBlocks blocks. Then restart the app and sync it up with the remaining blocks
+func testHandshakeReplay(t *testing.T, nBlocks int) {
+	config := tendermint_test.ResetConfig("proxy_test_")
+	walFile := path.Join(data_dir, "many_blocks.cswal")
+	config.Set("cs_wal_file", walFile)
+	privVal := types.LoadPrivValidator(config.GetString("priv_validator_file"))
+	testPartSize = config.GetInt("block_part_size")
+
+	wal, err := NewWAL(walFile, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	chain, commits, err := makeBlockchainFromWAL(wal)
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	state, store := stateAndStore(config, privVal.PubKey)
+	store.chain = chain
+	store.commits = commits
+
+	// run the whole chain against this client to build up the tendermint state
+	clientCreator := proxy.NewLocalClientCreator(dummy.NewPersistentDummyApplication(path.Join(config.GetString("db_dir"), "1")))
+	proxyApp := proxy.NewAppConns(config, clientCreator, nil) // sm.NewHandshaker(config, state, store, ReplayLastBlock))
+	if _, err := proxyApp.Start(); err != nil {
+		t.Fatalf("Error starting proxy app connections: %v", err)
+	}
+	for _, block := range chain {
+		err := state.ApplyBlock(nil, proxyApp.Consensus(), block, block.MakePartSet(testPartSize).Header(), mempool)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	proxyApp.Stop()
+	latestAppHash := state.AppHash
+
+	// run nBlocks against a new client to build up the app state.
+	// use a throwaway tendermint state
+	clientCreator2 := proxy.NewLocalClientCreator(dummy.NewPersistentDummyApplication(path.Join(config.GetString("db_dir"), "2")))
+	if nBlocks > 0 {
+		// start a new app without handshake, play nBlocks blocks
+		proxyApp := proxy.NewAppConns(config, clientCreator2, nil)
+		if _, err := proxyApp.Start(); err != nil {
+			t.Fatalf("Error starting proxy app connections: %v", err)
+		}
+		state2, _ := stateAndStore(config, privVal.PubKey)
+		for i := 0; i < nBlocks; i++ {
+			block := chain[i]
+			err := state2.ApplyBlock(nil, proxyApp.Consensus(), block, block.MakePartSet(testPartSize).Header(), mempool)
+			if err != nil {
+				t.Fatal(err)
+			}
+		}
+		proxyApp.Stop()
+	}
+
+	// now start the app using the handshake - it should sync
+	handshaker := sm.NewHandshaker(config, state, store, ReplayLastBlock)
+	proxyApp = proxy.NewAppConns(config, clientCreator2, handshaker)
+	if _, err := proxyApp.Start(); err != nil {
+		t.Fatalf("Error starting proxy app connections: %v", err)
+	}
+
+	// get the latest app hash from the app
+	res, err := proxyApp.Query().InfoSync()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// the app hash should be synced up
+	if !bytes.Equal(latestAppHash, res.LastBlockAppHash) {
+		t.Fatalf("Expected app hashes to match after handshake/replay. got %X, expected %X", res.LastBlockAppHash, latestAppHash)
+	}
+
+	if handshaker.NBlocks() != NUM_BLOCKS-nBlocks {
+		t.Fatalf("Expected handshake to sync %d blocks, got %d", NUM_BLOCKS-nBlocks, handshaker.NBlocks())
+	}
+}
+
+//--------------------------
+// utils for making blocks
+
+func makeBlockchainFromWAL(wal *WAL) ([]*types.Block, []*types.Commit, error) {
+	// Search for height marker
+	gr, found, err := wal.group.Search("#HEIGHT: ", makeHeightSearchFunc(1))
+	if err != nil {
+		return nil, nil, err
+	}
+	if !found {
+		return nil, nil, errors.New(Fmt("WAL does not contain height %d.", 1))
+	}
+	defer gr.Close()
+
+	log.Notice("Build a blockchain by reading from the WAL")
+
+	var blockParts *types.PartSet
+	var blocks []*types.Block
+	var commits []*types.Commit
+	for {
+		line, err := gr.ReadLine()
+		if err != nil {
+			if err == io.EOF {
+				break
+			} else {
+				return nil, nil, err
+			}
+		}
+
+		piece, err := readPieceFromWAL([]byte(line))
+		if err != nil {
+			return nil, nil, err
+		}
+		if piece == nil {
+			continue
+		}
+
+		switch p := piece.(type) {
+		case *types.PartSetHeader:
+			// if its not the first one, we have a full block
+			if blockParts != nil {
+				var n int
+				block := wire.ReadBinary(&types.Block{}, blockParts.GetReader(), types.MaxBlockSize, &n, &err).(*types.Block)
+				blocks = append(blocks, block)
+			}
+			blockParts = types.NewPartSetFromHeader(*p)
+		case *types.Part:
+			_, err := blockParts.AddPart(p, false)
+			if err != nil {
+				return nil, nil, err
+			}
+		case *types.Vote:
+			if p.Type == types.VoteTypePrecommit {
+				commit := &types.Commit{
+					BlockID:    p.BlockID,
+					Precommits: []*types.Vote{p},
+				}
+				commits = append(commits, commit)
+			}
+		}
+	}
+	// grab the last block too
+	var n int
+	block := wire.ReadBinary(&types.Block{}, blockParts.GetReader(), types.MaxBlockSize, &n, &err).(*types.Block)
+	blocks = append(blocks, block)
+	return blocks, commits, nil
+}
+
+func readPieceFromWAL(msgBytes []byte) (interface{}, error) {
+	// Skip over empty and meta lines
+	if len(msgBytes) == 0 || msgBytes[0] == '#' {
+		return nil, nil
+	}
+	var err error
+	var msg TimedWALMessage
+	wire.ReadJSON(&msg, msgBytes, &err)
+	if err != nil {
+		fmt.Println("MsgBytes:", msgBytes, string(msgBytes))
+		return nil, fmt.Errorf("Error reading json data: %v", err)
+	}
+
+	// for logging
+	switch m := msg.Msg.(type) {
+	case msgInfo:
+		switch msg := m.Msg.(type) {
+		case *ProposalMessage:
+			return &msg.Proposal.BlockPartsHeader, nil
+		case *BlockPartMessage:
+			return msg.Part, nil
+		case *VoteMessage:
+			return msg.Vote, nil
+		}
+	}
+	return nil, nil
+}
+
+// make some bogus txs
+func txsFunc(blockNum int) (txs []types.Tx) {
+	for i := 0; i < 10; i++ {
+		txs = append(txs, types.Tx([]byte{byte(blockNum), byte(i)}))
+	}
+	return txs
+}
+
+// sign a commit vote
+func signCommit(chainID string, privVal *types.PrivValidator, height, round int, hash []byte, header types.PartSetHeader) *types.Vote {
+	vote := &types.Vote{
+		ValidatorIndex:   0,
+		ValidatorAddress: privVal.Address,
+		Height:           height,
+		Round:            round,
+		Type:             types.VoteTypePrecommit,
+		BlockID:          types.BlockID{hash, header},
+	}
+
+	sig := privVal.Sign(types.SignBytes(chainID, vote))
+	vote.Signature = sig
+	return vote
+}
+
+// make a blockchain with one validator
+func makeBlockchain(t *testing.T, chainID string, nBlocks int, privVal *types.PrivValidator, proxyApp proxy.AppConns, state *sm.State) (blockchain []*types.Block, commits []*types.Commit) {
+
+	prevHash := state.LastBlockID.Hash
+	lastCommit := new(types.Commit)
+	prevParts := types.PartSetHeader{}
+	valHash := state.Validators.Hash()
+	prevBlockID := types.BlockID{prevHash, prevParts}
+
+	for i := 1; i < nBlocks+1; i++ {
+		block, parts := types.MakeBlock(i, chainID, txsFunc(i), lastCommit,
+			prevBlockID, valHash, state.AppHash, testPartSize)
+		fmt.Println(i)
+		fmt.Println(block.LastBlockID)
+		err := state.ApplyBlock(nil, proxyApp.Consensus(), block, block.MakePartSet(testPartSize).Header(), mempool)
+		if err != nil {
+			t.Fatal(i, err)
+		}
+
+		voteSet := types.NewVoteSet(chainID, i, 0, types.VoteTypePrecommit, state.Validators)
+		vote := signCommit(chainID, privVal, i, 0, block.Hash(), parts.Header())
+		_, err = voteSet.AddVote(vote)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		prevHash = block.Hash()
+		prevParts = parts.Header()
+		lastCommit = voteSet.MakeCommit()
+		prevBlockID = types.BlockID{prevHash, prevParts}
+
+		blockchain = append(blockchain, block)
+		commits = append(commits, lastCommit)
+	}
+	return blockchain, commits
+}
+
+// fresh state and mock store
+func stateAndStore(config cfg.Config, pubKey crypto.PubKey) (*sm.State, *mockBlockStore) {
+	stateDB := dbm.NewMemDB()
+	return sm.MakeGenesisState(stateDB, &types.GenesisDoc{
+		ChainID: config.GetString("chain_id"),
+		Validators: []types.GenesisValidator{
+			types.GenesisValidator{pubKey, 10000, "test"},
+		},
+		AppHash: nil,
+	}), NewMockBlockStore(config)
+}
+
+//----------------------------------
+// mock block store
+
+type mockBlockStore struct {
+	config  cfg.Config
+	chain   []*types.Block
+	commits []*types.Commit
+}
+
+// TODO: NewBlockStore(db.NewMemDB) ...
+func NewMockBlockStore(config cfg.Config) *mockBlockStore {
+	return &mockBlockStore{config, nil, nil}
+}
+
+func (bs *mockBlockStore) Height() int                       { return len(bs.chain) }
+func (bs *mockBlockStore) LoadBlock(height int) *types.Block { return bs.chain[height-1] }
+func (bs *mockBlockStore) LoadBlockMeta(height int) *types.BlockMeta {
+	block := bs.chain[height-1]
+	return &types.BlockMeta{
+		BlockID: types.BlockID{block.Hash(), block.MakePartSet(bs.config.GetInt("block_part_size")).Header()},
+		Header:  block.Header,
+	}
+}
+func (bs *mockBlockStore) LoadBlockPart(height int, index int) *types.Part { return nil }
+func (bs *mockBlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit) {
+}
+func (bs *mockBlockStore) LoadBlockCommit(height int) *types.Commit {
+	return bs.commits[height-1]
+}
+func (bs *mockBlockStore) LoadSeenCommit(height int) *types.Commit {
+	return bs.commits[height-1]
 }
diff --git a/consensus/state.go b/consensus/state.go
index 30ca8e9f..a985fd2c 100644
--- a/consensus/state.go
+++ b/consensus/state.go
@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
+	"path"
 	"reflect"
 	"sync"
 	"time"
@@ -354,13 +355,13 @@ func (cs *ConsensusState) LoadCommit(height int) *types.Commit {
 func (cs *ConsensusState) OnStart() error {
 	cs.BaseService.OnStart()
 
-	walDir := cs.config.GetString("cs_wal_dir")
-	err := EnsureDir(walDir, 0700)
+	walFile := cs.config.GetString("cs_wal_file")
+	err := EnsureDir(path.Dir(walFile), 0700)
 	if err != nil {
 		log.Error("Error ensuring ConsensusState wal dir", "error", err.Error())
 		return err
 	}
-	err = cs.OpenWAL(walDir)
+	err = cs.OpenWAL(walFile)
 	if err != nil {
 		log.Error("Error loading ConsensusState wal", "error", err.Error())
 		return err
@@ -415,10 +416,10 @@ func (cs *ConsensusState) Wait() {
 }
 
 // Open file to log all consensus messages and timeouts for deterministic accountability
-func (cs *ConsensusState) OpenWAL(walDir string) (err error) {
+func (cs *ConsensusState) OpenWAL(walFile string) (err error) {
 	cs.mtx.Lock()
 	defer cs.mtx.Unlock()
-	wal, err := NewWAL(walDir, cs.config.GetBool("cs_wal_light"))
+	wal, err := NewWAL(walFile, cs.config.GetBool("cs_wal_light"))
 	if err != nil {
 		return err
 	}
diff --git a/consensus/test_data/build.sh b/consensus/test_data/build.sh
index 970eb783..ea0c9604 100644
--- a/consensus/test_data/build.sh
+++ b/consensus/test_data/build.sh
@@ -1,12 +1,13 @@
 #! /bin/bash
 
+# XXX: removes tendermint dir
+
 cd $GOPATH/src/github.com/tendermint/tendermint
 
 # specify a dir to copy
-# NOTE: eventually we should replace with `tendermint init --test`
+# TODO: eventually we should replace with `tendermint init --test`
 DIR=$HOME/.tendermint_test/consensus_state_test
 
-# XXX: remove tendermint dir
 rm -rf $HOME/.tendermint
 cp -r $DIR $HOME/.tendermint
 
@@ -18,6 +19,7 @@ function reset(){
 reset
 
 # empty block
+function empty_block(){
 tendermint node --proxy_app=dummy &> /dev/null &
 sleep 5
 killall tendermint
@@ -28,21 +30,40 @@ killall tendermint
 sed '/HEIGHT: 2/Q' ~/.tendermint/data/cs.wal/wal  > consensus/test_data/empty_block.cswal
 
 reset
+}
 
-# small block 1
+# many blocks
+function many_blocks(){
 bash scripts/txs/random.sh 1000 36657 &> /dev/null &
 PID=$!
 tendermint node --proxy_app=dummy &> /dev/null &
-sleep 5
+sleep 7
+killall tendermint
+kill -9 $PID
+
+sed '/HEIGHT: 7/Q' ~/.tendermint/data/cs.wal/wal  > consensus/test_data/many_blocks.cswal
+
+reset
+}
+
+
+# small block 1
+function small_block1(){
+bash scripts/txs/random.sh 1000 36657 &> /dev/null &
+PID=$!
+tendermint node --proxy_app=dummy &> /dev/null &
+sleep 10
 killall tendermint
 kill -9 $PID
 
 sed '/HEIGHT: 2/Q' ~/.tendermint/data/cs.wal/wal  > consensus/test_data/small_block1.cswal
 
 reset
+}
 
 
 # small block 2 (part size = 512)
+function small_block2(){
 echo "" >> ~/.tendermint/config.toml
 echo "block_part_size = 512" >> ~/.tendermint/config.toml
 bash scripts/txs/random.sh 1000 36657 &> /dev/null &
@@ -55,4 +76,28 @@ kill -9 $PID
 sed '/HEIGHT: 2/Q' ~/.tendermint/data/cs.wal/wal  > consensus/test_data/small_block2.cswal
 
 reset
+}
+
+
+
+case "$1" in 
+	"small_block1")
+		small_block1
+		;;
+	"small_block2")
+		small_block2
+		;;
+	"empty_block")
+		empty_block
+		;;
+	"many_blocks")
+		many_blocks
+		;;
+	*)
+		small_block1
+		small_block2
+		empty_block
+		many_blocks
+esac
+
 
diff --git a/consensus/test_data/many_blocks.cswal b/consensus/test_data/many_blocks.cswal
new file mode 100644
index 00000000..9ef06c32
--- /dev/null
+++ b/consensus/test_data/many_blocks.cswal
@@ -0,0 +1,65 @@
+#HEIGHT: 1
+{"time":"2017-02-17T23:54:19.013Z","msg":[3,{"duration":969121813,"height":1,"round":0,"step":1}]}
+{"time":"2017-02-17T23:54:19.014Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPropose"}]}
+{"time":"2017-02-17T23:54:19.014Z","msg":[2,{"msg":[17,{"Proposal":{"height":1,"round":0,"block_parts_header":{"total":1,"hash":"2E32C8D500E936D27A47FCE3FF4BE7C1AFB3FAE1"},"pol_round":-1,"pol_block_id":{"hash":"","parts":{"total":0,"hash":""}},"signature":[1,"105A5A834E9AE2FA2191CAB5CB20D63594BA7859BD3EB92F055C8A35476D71F0D89F9FD5B0FF030D021533C71A81BF6E8F026BF4A37FC637CF38CA35291A9D00"]}}],"peer_key":""}]}
+{"time":"2017-02-17T23:54:19.015Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":0,"bytes":"0101010F74656E6465726D696E745F74657374010114A438480B084B40017600000000011477134726D7D54ABC03516888951EBC652413B20B0114354594CBFC1A7BCA1AD0050ED6AA010023EADA3900010176011A3631363236333634333133313344363436333632363133313331011A3631363236333634333133323344363436333632363133313332011A3631363236333634333133333344363436333632363133313333011A3631363236333634333133343344363436333632363133313334011A3631363236333634333133353344363436333632363133313335011A3631363236333634333133363344363436333632363133313336011A3631363236333634333133373344363436333632363133313337011A3631363236333634333133383344363436333632363133313338011A3631363236333634333133393344363436333632363133313339011A3631363236333634333233303344363436333632363133323330011A3631363236333634333233313344363436333632363133323331011A3631363236333634333233323344363436333632363133323332011A3631363236333634333233333344363436333632363133323333011A3631363236333634333233343344363436333632363133323334011A3631363236333634333233353344363436333632363133323335011A3631363236333634333233363344363436333632363133323336011A3631363236333634333233373344363436333632363133323337011A3631363236333634333233383344363436333632363133323338011A3631363236333634333233393344363436333632363133323339011A3631363236333634333333303344363436333632363133333330011A3631363236333634333333313344363436333632363133333331011A3631363236333634333333323344363436333632363133333332011A3631363236333634333333333344363436333632363133333333011A3631363236333634333333343344363436333632363133333334011A3631363236333634333333353344363436333632363133333335011A3631363236333634333333363344363436333632363133333336011A3631363236333634333333373344363436333632363133333337011A3631363236333634333333383344363436333632363133333338011A3631363236333634333333393344363436333632363133333339011A3631363236333634333433303344363436333632363133343330011A3631363236333634333433313344363436333632363133343331011A3631363236333634333433323344363436333632363133343332011A3631363236333634333433333344363436333632363133343333011A3631363236333634333433343344363436333632363133343334011A3631363236333634333433353344363436333632363133343335011A3631363236333634333433363344363436333632363133343336011A3631363236333634333433373344363436333632363133343337011A3631363236333634333433383344363436333632363133343338011A3631363236333634333433393344363436333632363133343339011A3631363236333634333533303344363436333632363133353330011A3631363236333634333533313344363436333632363133353331011A3631363236333634333533323344363436333632363133353332011A3631363236333634333533333344363436333632363133353333011A3631363236333634333533343344363436333632363133353334011A3631363236333634333533353344363436333632363133353335011A3631363236333634333533363344363436333632363133353336011A3631363236333634333533373344363436333632363133353337011A3631363236333634333533383344363436333632363133353338011A3631363236333634333533393344363436333632363133353339011A3631363236333634333633303344363436333632363133363330011A3631363236333634333633313344363436333632363133363331011A3631363236333634333633323344363436333632363133363332011A3631363236333634333633333344363436333632363133363333011A3631363236333634333633343344363436333632363133363334011A3631363236333634333633353344363436333632363133363335011A3631363236333634333633363344363436333632363133363336011A3631363236333634333633373344363436333632363133363337011A3631363236333634333633383344363436333632363133363338011A3631363236333634333633393344363436333632363133363339011A3631363236333634333733303344363436333632363133373330011A3631363236333634333733313344363436333632363133373331011A3631363236333634333733323344363436333632363133373332011A3631363236333634333733333344363436333632363133373333011A3631363236333634333733343344363436333632363133373334011A3631363236333634333733353344363436333632363133373335011A3631363236333634333733363344363436333632363133373336011A3631363236333634333733373344363436333632363133373337011A3631363236333634333733383344363436333632363133373338011A3631363236333634333733393344363436333632363133373339011A3631363236333634333833303344363436333632363133383330011A3631363236333634333833313344363436333632363133383331011A3631363236333634333833323344363436333632363133383332011A3631363236333634333833333344363436333632363133383333011A3631363236333634333833343344363436333632363133383334011A3631363236333634333833353344363436333632363133383335011A3631363236333634333833363344363436333632363133383336011A3631363236333634333833373344363436333632363133383337011A3631363236333634333833383344363436333632363133383338011A3631363236333634333833393344363436333632363133383339011A3631363236333634333933303344363436333632363133393330011A3631363236333634333933313344363436333632363133393331011A3631363236333634333933323344363436333632363133393332011A3631363236333634333933333344363436333632363133393333011A3631363236333634333933343344363436333632363133393334011A3631363236333634333933353344363436333632363133393335011A3631363236333634333933363344363436333632363133393336011A3631363236333634333933373344363436333632363133393337011A3631363236333634333933383344363436333632363133393338011A3631363236333634333933393344363436333632363133393339011E363136323633363433313330333033443634363336323631333133303330011E363136323633363433313330333133443634363336323631333133303331011E363136323633363433313330333233443634363336323631333133303332011E363136323633363433313330333333443634363336323631333133303333011E363136323633363433313330333433443634363336323631333133303334011E363136323633363433313330333533443634363336323631333133303335011E363136323633363433313330333633443634363336323631333133303336011E363136323633363433313330333733443634363336323631333133303337011E363136323633363433313330333833443634363336323631333133303338011E363136323633363433313330333933443634363336323631333133303339011E363136323633363433313331333033443634363336323631333133313330011E363136323633363433313331333133443634363336323631333133313331011E363136323633363433313331333233443634363336323631333133313332011E363136323633363433313331333333443634363336323631333133313333011E363136323633363433313331333433443634363336323631333133313334011E363136323633363433313331333533443634363336323631333133313335011E363136323633363433313331333633443634363336323631333133313336011E363136323633363433313331333733443634363336323631333133313337011E363136323633363433313331333833443634363336323631333133313338011E363136323633363433313331333933443634363336323631333133313339011E363136323633363433313332333033443634363336323631333133323330011E363136323633363433313332333133443634363336323631333133323331011E363136323633363433313332333233443634363336323631333133323332011E363136323633363433313332333333443634363336323631333133323333011E363136323633363433313332333433443634363336323631333133323334011E363136323633363433313332333533443634363336323631333133323335011E363136323633363433313332333633443634363336323631333133323336011E363136323633363433313332333733443634363336323631333133323337011E3631363236333634333133323338334436343633363236313331333233380100000000","proof":{"aunts":[]}}}],"peer_key":""}]}
+{"time":"2017-02-17T23:54:19.016Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPrevote"}]}
+{"time":"2017-02-17T23:54:19.016Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":1,"round":0,"type":1,"block_id":{"hash":"3F32EE37F9EA674A2173CAD651836A8EE628B5C7","parts":{"total":1,"hash":"2E32C8D500E936D27A47FCE3FF4BE7C1AFB3FAE1"}},"signature":[1,"31851DA0008AFF4223245EDFCCF1AD7BE96F8D66F8BD02D87F06B2F800A9405413861877D08798F0F6297D29936F5380B352C82212D2EC6F0E194A8C22A1EB0E"]}}],"peer_key":""}]}
+{"time":"2017-02-17T23:54:19.016Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPrecommit"}]}
+{"time":"2017-02-17T23:54:19.016Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":1,"round":0,"type":2,"block_id":{"hash":"3F32EE37F9EA674A2173CAD651836A8EE628B5C7","parts":{"total":1,"hash":"2E32C8D500E936D27A47FCE3FF4BE7C1AFB3FAE1"}},"signature":[1,"2B1070A5AB9305612A3AE74A8036D82B5E49E0DBBFBC7D723DB985CC8A8E72A52FF8E34D85273FEB8B901945CA541FA5142C3C4D43A04E9205ACECF53FD19B01"]}}],"peer_key":""}]}
+{"time":"2017-02-17T23:54:19.017Z","msg":[1,{"height":1,"round":0,"step":"RoundStepCommit"}]}
+#HEIGHT: 2
+{"time":"2017-02-17T23:54:19.019Z","msg":[1,{"height":2,"round":0,"step":"RoundStepNewHeight"}]}
+{"time":"2017-02-17T23:54:20.017Z","msg":[3,{"duration":998073370,"height":2,"round":0,"step":1}]}
+{"time":"2017-02-17T23:54:20.018Z","msg":[1,{"height":2,"round":0,"step":"RoundStepPropose"}]}
+{"time":"2017-02-17T23:54:20.018Z","msg":[2,{"msg":[17,{"Proposal":{"height":2,"round":0,"block_parts_header":{"total":1,"hash":"D008E9014CDDEA8EC95E1E99E21333241BD52DFC"},"pol_round":-1,"pol_block_id":{"hash":"","parts":{"total":0,"hash":""}},"signature":[1,"03E06975CD5A83E2B6AADC82F0C5965BE13CCB589912B7CBEF847BDBED6E8EAEE0901C02FAE8BC96B269C4750E5BA5C351C587537E3C063358A7D769007D8509"]}}],"peer_key":""}]}
+{"time":"2017-02-17T23:54:20.019Z","msg":[2,{"msg":[19,{"Height":2,"Round":0,"Part":{"index":0,"bytes":"0101010F74656E6465726D696E745F74657374010214A4384846E01E40018101143F32EE37F9EA674A2173CAD651836A8EE628B5C7010101142E32C8D500E936D27A47FCE3FF4BE7C1AFB3FAE101142F7319866C6844639F566F5A201DE2C339EAF8670114A0DC0254E5A6831C03FB4E1BE09CB26AEAC5C73D0114354594CBFC1A7BCA1AD0050ED6AA010023EADA39011402B7F5E97ED0892197CF5779CA6B904B10C87E10010181011E363136323633363433313332333933443634363336323631333133323339011E363136323633363433313333333033443634363336323631333133333330011E363136323633363433313333333133443634363336323631333133333331011E363136323633363433313333333233443634363336323631333133333332011E363136323633363433313333333333443634363336323631333133333333011E363136323633363433313333333433443634363336323631333133333334011E363136323633363433313333333533443634363336323631333133333335011E363136323633363433313333333633443634363336323631333133333336011E363136323633363433313333333733443634363336323631333133333337011E363136323633363433313333333833443634363336323631333133333338011E363136323633363433313333333933443634363336323631333133333339011E363136323633363433313334333033443634363336323631333133343330011E363136323633363433313334333133443634363336323631333133343331011E363136323633363433313334333233443634363336323631333133343332011E363136323633363433313334333333443634363336323631333133343333011E363136323633363433313334333433443634363336323631333133343334011E363136323633363433313334333533443634363336323631333133343335011E363136323633363433313334333633443634363336323631333133343336011E363136323633363433313334333733443634363336323631333133343337011E363136323633363433313334333833443634363336323631333133343338011E363136323633363433313334333933443634363336323631333133343339011E363136323633363433313335333033443634363336323631333133353330011E363136323633363433313335333133443634363336323631333133353331011E363136323633363433313335333233443634363336323631333133353332011E363136323633363433313335333333443634363336323631333133353333011E363136323633363433313335333433443634363336323631333133353334011E363136323633363433313335333533443634363336323631333133353335011E363136323633363433313335333633443634363336323631333133353336011E363136323633363433313335333733443634363336323631333133353337011E363136323633363433313335333833443634363336323631333133353338011E363136323633363433313335333933443634363336323631333133353339011E363136323633363433313336333033443634363336323631333133363330011E363136323633363433313336333133443634363336323631333133363331011E363136323633363433313336333233443634363336323631333133363332011E363136323633363433313336333333443634363336323631333133363333011E363136323633363433313336333433443634363336323631333133363334011E363136323633363433313336333533443634363336323631333133363335011E363136323633363433313336333633443634363336323631333133363336011E363136323633363433313336333733443634363336323631333133363337011E363136323633363433313336333833443634363336323631333133363338011E363136323633363433313336333933443634363336323631333133363339011E363136323633363433313337333033443634363336323631333133373330011E363136323633363433313337333133443634363336323631333133373331011E363136323633363433313337333233443634363336323631333133373332011E363136323633363433313337333333443634363336323631333133373333011E363136323633363433313337333433443634363336323631333133373334011E363136323633363433313337333533443634363336323631333133373335011E363136323633363433313337333633443634363336323631333133373336011E363136323633363433313337333733443634363336323631333133373337011E363136323633363433313337333833443634363336323631333133373338011E363136323633363433313337333933443634363336323631333133373339011E363136323633363433313338333033443634363336323631333133383330011E363136323633363433313338333133443634363336323631333133383331011E363136323633363433313338333233443634363336323631333133383332011E363136323633363433313338333333443634363336323631333133383333011E363136323633363433313338333433443634363336323631333133383334011E363136323633363433313338333533443634363336323631333133383335011E363136323633363433313338333633443634363336323631333133383336011E363136323633363433313338333733443634363336323631333133383337011E363136323633363433313338333833443634363336323631333133383338011E363136323633363433313338333933443634363336323631333133383339011E363136323633363433313339333033443634363336323631333133393330011E363136323633363433313339333133443634363336323631333133393331011E363136323633363433313339333233443634363336323631333133393332011E363136323633363433313339333333443634363336323631333133393333011E363136323633363433313339333433443634363336323631333133393334011E363136323633363433313339333533443634363336323631333133393335011E363136323633363433313339333633443634363336323631333133393336011E363136323633363433313339333733443634363336323631333133393337011E363136323633363433313339333833443634363336323631333133393338011E363136323633363433313339333933443634363336323631333133393339011E363136323633363433323330333033443634363336323631333233303330011E363136323633363433323330333133443634363336323631333233303331011E363136323633363433323330333233443634363336323631333233303332011E363136323633363433323330333333443634363336323631333233303333011E363136323633363433323330333433443634363336323631333233303334011E363136323633363433323330333533443634363336323631333233303335011E363136323633363433323330333633443634363336323631333233303336011E363136323633363433323330333733443634363336323631333233303337011E363136323633363433323330333833443634363336323631333233303338011E363136323633363433323330333933443634363336323631333233303339011E363136323633363433323331333033443634363336323631333233313330011E363136323633363433323331333133443634363336323631333233313331011E363136323633363433323331333233443634363336323631333233313332011E363136323633363433323331333333443634363336323631333233313333011E363136323633363433323331333433443634363336323631333233313334011E363136323633363433323331333533443634363336323631333233313335011E363136323633363433323331333633443634363336323631333233313336011E363136323633363433323331333733443634363336323631333233313337011E363136323633363433323331333833443634363336323631333233313338011E363136323633363433323331333933443634363336323631333233313339011E363136323633363433323332333033443634363336323631333233323330011E363136323633363433323332333133443634363336323631333233323331011E363136323633363433323332333233443634363336323631333233323332011E363136323633363433323332333333443634363336323631333233323333011E363136323633363433323332333433443634363336323631333233323334011E363136323633363433323332333533443634363336323631333233323335011E363136323633363433323332333633443634363336323631333233323336011E363136323633363433323332333733443634363336323631333233323337011E363136323633363433323332333833443634363336323631333233323338011E363136323633363433323332333933443634363336323631333233323339011E363136323633363433323333333033443634363336323631333233333330011E363136323633363433323333333133443634363336323631333233333331011E363136323633363433323333333233443634363336323631333233333332011E363136323633363433323333333333443634363336323631333233333333011E363136323633363433323333333433443634363336323631333233333334011E363136323633363433323333333533443634363336323631333233333335011E363136323633363433323333333633443634363336323631333233333336011E363136323633363433323333333733443634363336323631333233333337011E363136323633363433323333333833443634363336323631333233333338011E363136323633363433323333333933443634363336323631333233333339011E363136323633363433323334333033443634363336323631333233343330011E363136323633363433323334333133443634363336323631333233343331011E363136323633363433323334333233443634363336323631333233343332011E363136323633363433323334333333443634363336323631333233343333011E363136323633363433323334333433443634363336323631333233343334011E363136323633363433323334333533443634363336323631333233343335011E363136323633363433323334333633443634363336323631333233343336011E363136323633363433323334333733443634363336323631333233343337011E363136323633363433323334333833443634363336323631333233343338011E363136323633363433323334333933443634363336323631333233343339011E363136323633363433323335333033443634363336323631333233353330011E363136323633363433323335333133443634363336323631333233353331011E363136323633363433323335333233443634363336323631333233353332011E363136323633363433323335333333443634363336323631333233353333011E363136323633363433323335333433443634363336323631333233353334011E363136323633363433323335333533443634363336323631333233353335011E363136323633363433323335333633443634363336323631333233353336011E3631363236333634333233353337334436343633363236313332333533370101143F32EE37F9EA674A2173CAD651836A8EE628B5C7010101142E32C8D500E936D27A47FCE3FF4BE7C1AFB3FAE10101010114D028C9981F7A87F3093672BF0D5B0E2A1B3ED456000101000201143F32EE37F9EA674A2173CAD651836A8EE628B5C7010101142E32C8D500E936D27A47FCE3FF4BE7C1AFB3FAE1012B1070A5AB9305612A3AE74A8036D82B5E49E0DBBFBC7D723DB985CC8A8E72A52FF8E34D85273FEB8B901945CA541FA5142C3C4D43A04E9205ACECF53FD19B01","proof":{"aunts":[]}}}],"peer_key":""}]}
+{"time":"2017-02-17T23:54:20.020Z","msg":[1,{"height":2,"round":0,"step":"RoundStepPrevote"}]}
+{"time":"2017-02-17T23:54:20.020Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":2,"round":0,"type":1,"block_id":{"hash":"32310D174A99844713693C9815D2CA660364E028","parts":{"total":1,"hash":"D008E9014CDDEA8EC95E1E99E21333241BD52DFC"}},"signature":[1,"E0289DE621820D9236632B4862BB4D1518A4B194C5AE8194192F375C9A52775A54A7F172A5D7A2014E404A1C3AFA386923E7A20329AFDDFA14655881C04A1A02"]}}],"peer_key":""}]}
+{"time":"2017-02-17T23:54:20.021Z","msg":[1,{"height":2,"round":0,"step":"RoundStepPrecommit"}]}
+{"time":"2017-02-17T23:54:20.021Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":2,"round":0,"type":2,"block_id":{"hash":"32310D174A99844713693C9815D2CA660364E028","parts":{"total":1,"hash":"D008E9014CDDEA8EC95E1E99E21333241BD52DFC"}},"signature":[1,"AA9F03D0707752301D7CBFCF4F0BCDBD666A46C1CAED3910BD64A3C5C2874AAF328172646C951C5E2FD962359C382A3CBBA2C73EC9B533668C6386995B83EC08"]}}],"peer_key":""}]}
+{"time":"2017-02-17T23:54:20.022Z","msg":[1,{"height":2,"round":0,"step":"RoundStepCommit"}]}
+#HEIGHT: 3
+{"time":"2017-02-17T23:54:20.025Z","msg":[1,{"height":3,"round":0,"step":"RoundStepNewHeight"}]}
+{"time":"2017-02-17T23:54:21.022Z","msg":[3,{"duration":997103974,"height":3,"round":0,"step":1}]}
+{"time":"2017-02-17T23:54:21.024Z","msg":[1,{"height":3,"round":0,"step":"RoundStepPropose"}]}
+{"time":"2017-02-17T23:54:21.024Z","msg":[2,{"msg":[17,{"Proposal":{"height":3,"round":0,"block_parts_header":{"total":1,"hash":"2E5DE5777A5AD899CD2531304F42A470509DE989"},"pol_round":-1,"pol_block_id":{"hash":"","parts":{"total":0,"hash":""}},"signature":[1,"5F6A6A8097BD6A1780568C7E064D932BC1F941E1D5AC408DE970C4EEDCCD939C0F163466D20F0E98A7599792341441422980C09D23E03009BD9CE565673C9704"]}}],"peer_key":""}]}
+{"time":"2017-02-17T23:54:21.024Z","msg":[2,{"msg":[19,{"Height":3,"Round":0,"Part":{"index":0,"bytes":"0101010F74656E6465726D696E745F74657374010314A4384882C73380017C011432310D174A99844713693C9815D2CA660364E02801010114D008E9014CDDEA8EC95E1E99E21333241BD52DFC0114ABAB9E28967792BDA02172D3CB99DA99A696738E0114513B0891921FB0A5F6471950AFE2598445153CFF0114354594CBFC1A7BCA1AD0050ED6AA010023EADA3901141FBD44B8259B2A6632F08F88BE5EC8C203075CD001017C011E363136323633363433323335333833443634363336323631333233353338011E363136323633363433323335333933443634363336323631333233353339011E363136323633363433323336333033443634363336323631333233363330011E363136323633363433323336333133443634363336323631333233363331011E363136323633363433323336333233443634363336323631333233363332011E363136323633363433323336333333443634363336323631333233363333011E363136323633363433323336333433443634363336323631333233363334011E363136323633363433323336333533443634363336323631333233363335011E363136323633363433323336333633443634363336323631333233363336011E363136323633363433323336333733443634363336323631333233363337011E363136323633363433323336333833443634363336323631333233363338011E363136323633363433323336333933443634363336323631333233363339011E363136323633363433323337333033443634363336323631333233373330011E363136323633363433323337333133443634363336323631333233373331011E363136323633363433323337333233443634363336323631333233373332011E363136323633363433323337333333443634363336323631333233373333011E363136323633363433323337333433443634363336323631333233373334011E363136323633363433323337333533443634363336323631333233373335011E363136323633363433323337333633443634363336323631333233373336011E363136323633363433323337333733443634363336323631333233373337011E363136323633363433323337333833443634363336323631333233373338011E363136323633363433323337333933443634363336323631333233373339011E363136323633363433323338333033443634363336323631333233383330011E363136323633363433323338333133443634363336323631333233383331011E363136323633363433323338333233443634363336323631333233383332011E363136323633363433323338333333443634363336323631333233383333011E363136323633363433323338333433443634363336323631333233383334011E363136323633363433323338333533443634363336323631333233383335011E363136323633363433323338333633443634363336323631333233383336011E363136323633363433323338333733443634363336323631333233383337011E363136323633363433323338333833443634363336323631333233383338011E363136323633363433323338333933443634363336323631333233383339011E363136323633363433323339333033443634363336323631333233393330011E363136323633363433323339333133443634363336323631333233393331011E363136323633363433323339333233443634363336323631333233393332011E363136323633363433323339333333443634363336323631333233393333011E363136323633363433323339333433443634363336323631333233393334011E363136323633363433323339333533443634363336323631333233393335011E363136323633363433323339333633443634363336323631333233393336011E363136323633363433323339333733443634363336323631333233393337011E363136323633363433323339333833443634363336323631333233393338011E363136323633363433323339333933443634363336323631333233393339011E363136323633363433333330333033443634363336323631333333303330011E363136323633363433333330333133443634363336323631333333303331011E363136323633363433333330333233443634363336323631333333303332011E363136323633363433333330333333443634363336323631333333303333011E363136323633363433333330333433443634363336323631333333303334011E363136323633363433333330333533443634363336323631333333303335011E363136323633363433333330333633443634363336323631333333303336011E363136323633363433333330333733443634363336323631333333303337011E363136323633363433333330333833443634363336323631333333303338011E363136323633363433333330333933443634363336323631333333303339011E363136323633363433333331333033443634363336323631333333313330011E363136323633363433333331333133443634363336323631333333313331011E363136323633363433333331333233443634363336323631333333313332011E363136323633363433333331333333443634363336323631333333313333011E363136323633363433333331333433443634363336323631333333313334011E363136323633363433333331333533443634363336323631333333313335011E363136323633363433333331333633443634363336323631333333313336011E363136323633363433333331333733443634363336323631333333313337011E363136323633363433333331333833443634363336323631333333313338011E363136323633363433333331333933443634363336323631333333313339011E363136323633363433333332333033443634363336323631333333323330011E363136323633363433333332333133443634363336323631333333323331011E363136323633363433333332333233443634363336323631333333323332011E363136323633363433333332333333443634363336323631333333323333011E363136323633363433333332333433443634363336323631333333323334011E363136323633363433333332333533443634363336323631333333323335011E363136323633363433333332333633443634363336323631333333323336011E363136323633363433333332333733443634363336323631333333323337011E363136323633363433333332333833443634363336323631333333323338011E363136323633363433333332333933443634363336323631333333323339011E363136323633363433333333333033443634363336323631333333333330011E363136323633363433333333333133443634363336323631333333333331011E363136323633363433333333333233443634363336323631333333333332011E363136323633363433333333333333443634363336323631333333333333011E363136323633363433333333333433443634363336323631333333333334011E363136323633363433333333333533443634363336323631333333333335011E363136323633363433333333333633443634363336323631333333333336011E363136323633363433333333333733443634363336323631333333333337011E363136323633363433333333333833443634363336323631333333333338011E363136323633363433333333333933443634363336323631333333333339011E363136323633363433333334333033443634363336323631333333343330011E363136323633363433333334333133443634363336323631333333343331011E363136323633363433333334333233443634363336323631333333343332011E363136323633363433333334333333443634363336323631333333343333011E363136323633363433333334333433443634363336323631333333343334011E363136323633363433333334333533443634363336323631333333343335011E363136323633363433333334333633443634363336323631333333343336011E363136323633363433333334333733443634363336323631333333343337011E363136323633363433333334333833443634363336323631333333343338011E363136323633363433333334333933443634363336323631333333343339011E363136323633363433333335333033443634363336323631333333353330011E363136323633363433333335333133443634363336323631333333353331011E363136323633363433333335333233443634363336323631333333353332011E363136323633363433333335333333443634363336323631333333353333011E363136323633363433333335333433443634363336323631333333353334011E363136323633363433333335333533443634363336323631333333353335011E363136323633363433333335333633443634363336323631333333353336011E363136323633363433333335333733443634363336323631333333353337011E363136323633363433333335333833443634363336323631333333353338011E363136323633363433333335333933443634363336323631333333353339011E363136323633363433333336333033443634363336323631333333363330011E363136323633363433333336333133443634363336323631333333363331011E363136323633363433333336333233443634363336323631333333363332011E363136323633363433333336333333443634363336323631333333363333011E363136323633363433333336333433443634363336323631333333363334011E363136323633363433333336333533443634363336323631333333363335011E363136323633363433333336333633443634363336323631333333363336011E363136323633363433333336333733443634363336323631333333363337011E363136323633363433333336333833443634363336323631333333363338011E363136323633363433333336333933443634363336323631333333363339011E363136323633363433333337333033443634363336323631333333373330011E363136323633363433333337333133443634363336323631333333373331011E363136323633363433333337333233443634363336323631333333373332011E363136323633363433333337333333443634363336323631333333373333011E363136323633363433333337333433443634363336323631333333373334011E363136323633363433333337333533443634363336323631333333373335011E363136323633363433333337333633443634363336323631333333373336011E363136323633363433333337333733443634363336323631333333373337011E363136323633363433333337333833443634363336323631333333373338011E363136323633363433333337333933443634363336323631333333373339011E363136323633363433333338333033443634363336323631333333383330011E36313632363336343333333833313344363436333632363133333338333101011432310D174A99844713693C9815D2CA660364E02801010114D008E9014CDDEA8EC95E1E99E21333241BD52DFC0101010114D028C9981F7A87F3093672BF0D5B0E2A1B3ED4560001020002011432310D174A99844713693C9815D2CA660364E02801010114D008E9014CDDEA8EC95E1E99E21333241BD52DFC01AA9F03D0707752301D7CBFCF4F0BCDBD666A46C1CAED3910BD64A3C5C2874AAF328172646C951C5E2FD962359C382A3CBBA2C73EC9B533668C6386995B83EC08","proof":{"aunts":[]}}}],"peer_key":""}]}
+{"time":"2017-02-17T23:54:21.026Z","msg":[1,{"height":3,"round":0,"step":"RoundStepPrevote"}]}
+{"time":"2017-02-17T23:54:21.026Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":3,"round":0,"type":1,"block_id":{"hash":"37AF6866DA8C3167CFC280FAE47B6ED441B00D5B","parts":{"total":1,"hash":"2E5DE5777A5AD899CD2531304F42A470509DE989"}},"signature":[1,"F0AAB604A8CE724453A378BBC66142C418464C3C0EC3EB2E15A1CB7524A92B9F36BE8A191238A4D317F542D999DF698B5C2A28D754240524FF8CCADA0947DE00"]}}],"peer_key":""}]}
+{"time":"2017-02-17T23:54:21.028Z","msg":[1,{"height":3,"round":0,"step":"RoundStepPrecommit"}]}
+{"time":"2017-02-17T23:54:21.028Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":3,"round":0,"type":2,"block_id":{"hash":"37AF6866DA8C3167CFC280FAE47B6ED441B00D5B","parts":{"total":1,"hash":"2E5DE5777A5AD899CD2531304F42A470509DE989"}},"signature":[1,"C900519E305EC03392E7D197D5FAB535DB240C9C0BA5375A1679C75BAAA07C7410C0EF43CF97D98F2C08A1D739667D5ACFF6233A1FAE75D3DA275AEA422EFD0F"]}}],"peer_key":""}]}
+{"time":"2017-02-17T23:54:21.028Z","msg":[1,{"height":3,"round":0,"step":"RoundStepCommit"}]}
+#HEIGHT: 4
+{"time":"2017-02-17T23:54:21.032Z","msg":[1,{"height":4,"round":0,"step":"RoundStepNewHeight"}]}
+{"time":"2017-02-17T23:54:22.028Z","msg":[3,{"duration":996302067,"height":4,"round":0,"step":1}]}
+{"time":"2017-02-17T23:54:22.030Z","msg":[1,{"height":4,"round":0,"step":"RoundStepPropose"}]}
+{"time":"2017-02-17T23:54:22.030Z","msg":[2,{"msg":[17,{"Proposal":{"height":4,"round":0,"block_parts_header":{"total":1,"hash":"24CEBCBEB833F56D47AD14354071B3B7A243068A"},"pol_round":-1,"pol_block_id":{"hash":"","parts":{"total":0,"hash":""}},"signature":[1,"CAECE2342987295CCB562C9B6AB0E296D0ECDBE0B40CDB5260B32DF07E07E7F30C4E815B76BC04B8E830143409E598F7BA24699F5B5A01A6237221C948A0920C"]}}],"peer_key":""}]}
+{"time":"2017-02-17T23:54:22.030Z","msg":[2,{"msg":[19,{"Height":4,"Round":0,"Part":{"index":0,"bytes":"0101010F74656E6465726D696E745F74657374010414A43848BECCCD400169011437AF6866DA8C3167CFC280FAE47B6ED441B00D5B010101142E5DE5777A5AD899CD2531304F42A470509DE9890114EA4CCD80AE261EF694AF4292F8AF5659FB66317301144FFC4AAC3EDFBB81DA9E9DDEC9D6A205AD49049E0114354594CBFC1A7BCA1AD0050ED6AA010023EADA390114E7563C252781F893D7A191A31566713888CCA3B2010169011E363136323633363433333338333233443634363336323631333333383332011E363136323633363433333338333333443634363336323631333333383333011E363136323633363433333338333433443634363336323631333333383334011E363136323633363433333338333533443634363336323631333333383335011E363136323633363433333338333633443634363336323631333333383336011E363136323633363433333338333733443634363336323631333333383337011E363136323633363433333338333833443634363336323631333333383338011E363136323633363433333338333933443634363336323631333333383339011E363136323633363433333339333033443634363336323631333333393330011E363136323633363433333339333133443634363336323631333333393331011E363136323633363433333339333233443634363336323631333333393332011E363136323633363433333339333333443634363336323631333333393333011E363136323633363433333339333433443634363336323631333333393334011E363136323633363433333339333533443634363336323631333333393335011E363136323633363433333339333633443634363336323631333333393336011E363136323633363433333339333733443634363336323631333333393337011E363136323633363433333339333833443634363336323631333333393338011E363136323633363433333339333933443634363336323631333333393339011E363136323633363433343330333033443634363336323631333433303330011E363136323633363433343330333133443634363336323631333433303331011E363136323633363433343330333233443634363336323631333433303332011E363136323633363433343330333333443634363336323631333433303333011E363136323633363433343330333433443634363336323631333433303334011E363136323633363433343330333533443634363336323631333433303335011E363136323633363433343330333633443634363336323631333433303336011E363136323633363433343330333733443634363336323631333433303337011E363136323633363433343330333833443634363336323631333433303338011E363136323633363433343330333933443634363336323631333433303339011E363136323633363433343331333033443634363336323631333433313330011E363136323633363433343331333133443634363336323631333433313331011E363136323633363433343331333233443634363336323631333433313332011E363136323633363433343331333333443634363336323631333433313333011E363136323633363433343331333433443634363336323631333433313334011E363136323633363433343331333533443634363336323631333433313335011E363136323633363433343331333633443634363336323631333433313336011E363136323633363433343331333733443634363336323631333433313337011E363136323633363433343331333833443634363336323631333433313338011E363136323633363433343331333933443634363336323631333433313339011E363136323633363433343332333033443634363336323631333433323330011E363136323633363433343332333133443634363336323631333433323331011E363136323633363433343332333233443634363336323631333433323332011E363136323633363433343332333333443634363336323631333433323333011E363136323633363433343332333433443634363336323631333433323334011E363136323633363433343332333533443634363336323631333433323335011E363136323633363433343332333633443634363336323631333433323336011E363136323633363433343332333733443634363336323631333433323337011E363136323633363433343332333833443634363336323631333433323338011E363136323633363433343332333933443634363336323631333433323339011E363136323633363433343333333033443634363336323631333433333330011E363136323633363433343333333133443634363336323631333433333331011E363136323633363433343333333233443634363336323631333433333332011E363136323633363433343333333333443634363336323631333433333333011E363136323633363433343333333433443634363336323631333433333334011E363136323633363433343333333533443634363336323631333433333335011E363136323633363433343333333633443634363336323631333433333336011E363136323633363433343333333733443634363336323631333433333337011E363136323633363433343333333833443634363336323631333433333338011E363136323633363433343333333933443634363336323631333433333339011E363136323633363433343334333033443634363336323631333433343330011E363136323633363433343334333133443634363336323631333433343331011E363136323633363433343334333233443634363336323631333433343332011E363136323633363433343334333333443634363336323631333433343333011E363136323633363433343334333433443634363336323631333433343334011E363136323633363433343334333533443634363336323631333433343335011E363136323633363433343334333633443634363336323631333433343336011E363136323633363433343334333733443634363336323631333433343337011E363136323633363433343334333833443634363336323631333433343338011E363136323633363433343334333933443634363336323631333433343339011E363136323633363433343335333033443634363336323631333433353330011E363136323633363433343335333133443634363336323631333433353331011E363136323633363433343335333233443634363336323631333433353332011E363136323633363433343335333333443634363336323631333433353333011E363136323633363433343335333433443634363336323631333433353334011E363136323633363433343335333533443634363336323631333433353335011E363136323633363433343335333633443634363336323631333433353336011E363136323633363433343335333733443634363336323631333433353337011E363136323633363433343335333833443634363336323631333433353338011E363136323633363433343335333933443634363336323631333433353339011E363136323633363433343336333033443634363336323631333433363330011E363136323633363433343336333133443634363336323631333433363331011E363136323633363433343336333233443634363336323631333433363332011E363136323633363433343336333333443634363336323631333433363333011E363136323633363433343336333433443634363336323631333433363334011E363136323633363433343336333533443634363336323631333433363335011E363136323633363433343336333633443634363336323631333433363336011E363136323633363433343336333733443634363336323631333433363337011E363136323633363433343336333833443634363336323631333433363338011E363136323633363433343336333933443634363336323631333433363339011E363136323633363433343337333033443634363336323631333433373330011E363136323633363433343337333133443634363336323631333433373331011E363136323633363433343337333233443634363336323631333433373332011E363136323633363433343337333333443634363336323631333433373333011E363136323633363433343337333433443634363336323631333433373334011E363136323633363433343337333533443634363336323631333433373335011E363136323633363433343337333633443634363336323631333433373336011E363136323633363433343337333733443634363336323631333433373337011E363136323633363433343337333833443634363336323631333433373338011E363136323633363433343337333933443634363336323631333433373339011E363136323633363433343338333033443634363336323631333433383330011E363136323633363433343338333133443634363336323631333433383331011E363136323633363433343338333233443634363336323631333433383332011E363136323633363433343338333333443634363336323631333433383333011E363136323633363433343338333433443634363336323631333433383334011E363136323633363433343338333533443634363336323631333433383335011E36313632363336343334333833363344363436333632363133343338333601011437AF6866DA8C3167CFC280FAE47B6ED441B00D5B010101142E5DE5777A5AD899CD2531304F42A470509DE9890101010114D028C9981F7A87F3093672BF0D5B0E2A1B3ED4560001030002011437AF6866DA8C3167CFC280FAE47B6ED441B00D5B010101142E5DE5777A5AD899CD2531304F42A470509DE98901C900519E305EC03392E7D197D5FAB535DB240C9C0BA5375A1679C75BAAA07C7410C0EF43CF97D98F2C08A1D739667D5ACFF6233A1FAE75D3DA275AEA422EFD0F","proof":{"aunts":[]}}}],"peer_key":""}]}
+{"time":"2017-02-17T23:54:22.032Z","msg":[1,{"height":4,"round":0,"step":"RoundStepPrevote"}]}
+{"time":"2017-02-17T23:54:22.032Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":4,"round":0,"type":1,"block_id":{"hash":"04715E223BF4327FFA9B0D5AD849B74A099D5DEC","parts":{"total":1,"hash":"24CEBCBEB833F56D47AD14354071B3B7A243068A"}},"signature":[1,"B1BFF3641FE1931C78A792540384B9D4CFC3D9008FD4988B24FAD872326C2A380A34F37610C6E076FA5B4DB9E4B3166B703B0429AF0BF1ABCCDB7B2EDB3C8F08"]}}],"peer_key":""}]}
+{"time":"2017-02-17T23:54:22.033Z","msg":[1,{"height":4,"round":0,"step":"RoundStepPrecommit"}]}
+{"time":"2017-02-17T23:54:22.033Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":4,"round":0,"type":2,"block_id":{"hash":"04715E223BF4327FFA9B0D5AD849B74A099D5DEC","parts":{"total":1,"hash":"24CEBCBEB833F56D47AD14354071B3B7A243068A"}},"signature":[1,"F544743F17479A61F94B0F68C63D254BD60493D78E818D48A5859133619AEE5E92C47CAD89C654DF64E0911C3152091E047555D5F14655D95B9681AE9B336505"]}}],"peer_key":""}]}
+{"time":"2017-02-17T23:54:22.034Z","msg":[1,{"height":4,"round":0,"step":"RoundStepCommit"}]}
+#HEIGHT: 5
+{"time":"2017-02-17T23:54:22.036Z","msg":[1,{"height":5,"round":0,"step":"RoundStepNewHeight"}]}
+{"time":"2017-02-17T23:54:23.034Z","msg":[3,{"duration":997096276,"height":5,"round":0,"step":1}]}
+{"time":"2017-02-17T23:54:23.035Z","msg":[1,{"height":5,"round":0,"step":"RoundStepPropose"}]}
+{"time":"2017-02-17T23:54:23.035Z","msg":[2,{"msg":[17,{"Proposal":{"height":5,"round":0,"block_parts_header":{"total":1,"hash":"A52BAA9C2E52E633A1605F4B930205613E3E7A2F"},"pol_round":-1,"pol_block_id":{"hash":"","parts":{"total":0,"hash":""}},"signature":[1,"32A96AA44440B6FDB28B590A029649CE37B0F1091B9E648658E910207BB2F96E4936102C63F3908942F1A45F52C01231680593FB3E53B8B29BF588A613116A0B"]}}],"peer_key":""}]}
+{"time":"2017-02-17T23:54:23.035Z","msg":[2,{"msg":[19,{"Height":5,"Round":0,"Part":{"index":0,"bytes":"0101010F74656E6465726D696E745F74657374010514A43848FAB3E280017A011404715E223BF4327FFA9B0D5AD849B74A099D5DEC0101011424CEBCBEB833F56D47AD14354071B3B7A243068A01144C1EF483E19426AC412E12F33E1D814FD019279801146440978BB85314393E824F6BBAFEE59FA1A5E30A0114354594CBFC1A7BCA1AD0050ED6AA010023EADA390114845C15D3FE4AA16021B85CD0F18E0188A673E73E01017A011E363136323633363433343338333733443634363336323631333433383337011E363136323633363433343338333833443634363336323631333433383338011E363136323633363433343338333933443634363336323631333433383339011E363136323633363433343339333033443634363336323631333433393330011E363136323633363433343339333133443634363336323631333433393331011E363136323633363433343339333233443634363336323631333433393332011E363136323633363433343339333333443634363336323631333433393333011E363136323633363433343339333433443634363336323631333433393334011E363136323633363433343339333533443634363336323631333433393335011E363136323633363433343339333633443634363336323631333433393336011E363136323633363433343339333733443634363336323631333433393337011E363136323633363433343339333833443634363336323631333433393338011E363136323633363433343339333933443634363336323631333433393339011E363136323633363433353330333033443634363336323631333533303330011E363136323633363433353330333133443634363336323631333533303331011E363136323633363433353330333233443634363336323631333533303332011E363136323633363433353330333333443634363336323631333533303333011E363136323633363433353330333433443634363336323631333533303334011E363136323633363433353330333533443634363336323631333533303335011E363136323633363433353330333633443634363336323631333533303336011E363136323633363433353330333733443634363336323631333533303337011E363136323633363433353330333833443634363336323631333533303338011E363136323633363433353330333933443634363336323631333533303339011E363136323633363433353331333033443634363336323631333533313330011E363136323633363433353331333133443634363336323631333533313331011E363136323633363433353331333233443634363336323631333533313332011E363136323633363433353331333333443634363336323631333533313333011E363136323633363433353331333433443634363336323631333533313334011E363136323633363433353331333533443634363336323631333533313335011E363136323633363433353331333633443634363336323631333533313336011E363136323633363433353331333733443634363336323631333533313337011E363136323633363433353331333833443634363336323631333533313338011E363136323633363433353331333933443634363336323631333533313339011E363136323633363433353332333033443634363336323631333533323330011E363136323633363433353332333133443634363336323631333533323331011E363136323633363433353332333233443634363336323631333533323332011E363136323633363433353332333333443634363336323631333533323333011E363136323633363433353332333433443634363336323631333533323334011E363136323633363433353332333533443634363336323631333533323335011E363136323633363433353332333633443634363336323631333533323336011E363136323633363433353332333733443634363336323631333533323337011E363136323633363433353332333833443634363336323631333533323338011E363136323633363433353332333933443634363336323631333533323339011E363136323633363433353333333033443634363336323631333533333330011E363136323633363433353333333133443634363336323631333533333331011E363136323633363433353333333233443634363336323631333533333332011E363136323633363433353333333333443634363336323631333533333333011E363136323633363433353333333433443634363336323631333533333334011E363136323633363433353333333533443634363336323631333533333335011E363136323633363433353333333633443634363336323631333533333336011E363136323633363433353333333733443634363336323631333533333337011E363136323633363433353333333833443634363336323631333533333338011E363136323633363433353333333933443634363336323631333533333339011E363136323633363433353334333033443634363336323631333533343330011E363136323633363433353334333133443634363336323631333533343331011E363136323633363433353334333233443634363336323631333533343332011E363136323633363433353334333333443634363336323631333533343333011E363136323633363433353334333433443634363336323631333533343334011E363136323633363433353334333533443634363336323631333533343335011E363136323633363433353334333633443634363336323631333533343336011E363136323633363433353334333733443634363336323631333533343337011E363136323633363433353334333833443634363336323631333533343338011E363136323633363433353334333933443634363336323631333533343339011E363136323633363433353335333033443634363336323631333533353330011E363136323633363433353335333133443634363336323631333533353331011E363136323633363433353335333233443634363336323631333533353332011E363136323633363433353335333333443634363336323631333533353333011E363136323633363433353335333433443634363336323631333533353334011E363136323633363433353335333533443634363336323631333533353335011E363136323633363433353335333633443634363336323631333533353336011E363136323633363433353335333733443634363336323631333533353337011E363136323633363433353335333833443634363336323631333533353338011E363136323633363433353335333933443634363336323631333533353339011E363136323633363433353336333033443634363336323631333533363330011E363136323633363433353336333133443634363336323631333533363331011E363136323633363433353336333233443634363336323631333533363332011E363136323633363433353336333333443634363336323631333533363333011E363136323633363433353336333433443634363336323631333533363334011E363136323633363433353336333533443634363336323631333533363335011E363136323633363433353336333633443634363336323631333533363336011E363136323633363433353336333733443634363336323631333533363337011E363136323633363433353336333833443634363336323631333533363338011E363136323633363433353336333933443634363336323631333533363339011E363136323633363433353337333033443634363336323631333533373330011E363136323633363433353337333133443634363336323631333533373331011E363136323633363433353337333233443634363336323631333533373332011E363136323633363433353337333333443634363336323631333533373333011E363136323633363433353337333433443634363336323631333533373334011E363136323633363433353337333533443634363336323631333533373335011E363136323633363433353337333633443634363336323631333533373336011E363136323633363433353337333733443634363336323631333533373337011E363136323633363433353337333833443634363336323631333533373338011E363136323633363433353337333933443634363336323631333533373339011E363136323633363433353338333033443634363336323631333533383330011E363136323633363433353338333133443634363336323631333533383331011E363136323633363433353338333233443634363336323631333533383332011E363136323633363433353338333333443634363336323631333533383333011E363136323633363433353338333433443634363336323631333533383334011E363136323633363433353338333533443634363336323631333533383335011E363136323633363433353338333633443634363336323631333533383336011E363136323633363433353338333733443634363336323631333533383337011E363136323633363433353338333833443634363336323631333533383338011E363136323633363433353338333933443634363336323631333533383339011E363136323633363433353339333033443634363336323631333533393330011E363136323633363433353339333133443634363336323631333533393331011E363136323633363433353339333233443634363336323631333533393332011E363136323633363433353339333333443634363336323631333533393333011E363136323633363433353339333433443634363336323631333533393334011E363136323633363433353339333533443634363336323631333533393335011E363136323633363433353339333633443634363336323631333533393336011E363136323633363433353339333733443634363336323631333533393337011E363136323633363433353339333833443634363336323631333533393338011E363136323633363433353339333933443634363336323631333533393339011E363136323633363433363330333033443634363336323631333633303330011E363136323633363433363330333133443634363336323631333633303331011E363136323633363433363330333233443634363336323631333633303332011E363136323633363433363330333333443634363336323631333633303333011E363136323633363433363330333433443634363336323631333633303334011E363136323633363433363330333533443634363336323631333633303335011E363136323633363433363330333633443634363336323631333633303336011E363136323633363433363330333733443634363336323631333633303337011E36313632363336343336333033383344363436333632363133363330333801011404715E223BF4327FFA9B0D5AD849B74A099D5DEC0101011424CEBCBEB833F56D47AD14354071B3B7A243068A0101010114D028C9981F7A87F3093672BF0D5B0E2A1B3ED4560001040002011404715E223BF4327FFA9B0D5AD849B74A099D5DEC0101011424CEBCBEB833F56D47AD14354071B3B7A243068A01F544743F17479A61F94B0F68C63D254BD60493D78E818D48A5859133619AEE5E92C47CAD89C654DF64E0911C3152091E047555D5F14655D95B9681AE9B336505","proof":{"aunts":[]}}}],"peer_key":""}]}
+{"time":"2017-02-17T23:54:23.037Z","msg":[1,{"height":5,"round":0,"step":"RoundStepPrevote"}]}
+{"time":"2017-02-17T23:54:23.037Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":5,"round":0,"type":1,"block_id":{"hash":"FDC6D837995BEBBBFCBF3E7D7CF44F8FDA448543","parts":{"total":1,"hash":"A52BAA9C2E52E633A1605F4B930205613E3E7A2F"}},"signature":[1,"684AB4918389E06ADD5DCC4EFCCD0464EAE2BC4212344D88694706837A4D47D484747C7B5906537181E0FBD35EF78EDF673E8492C6E875BB33934456A8254B03"]}}],"peer_key":""}]}
+{"time":"2017-02-17T23:54:23.038Z","msg":[1,{"height":5,"round":0,"step":"RoundStepPrecommit"}]}
+{"time":"2017-02-17T23:54:23.038Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":5,"round":0,"type":2,"block_id":{"hash":"FDC6D837995BEBBBFCBF3E7D7CF44F8FDA448543","parts":{"total":1,"hash":"A52BAA9C2E52E633A1605F4B930205613E3E7A2F"}},"signature":[1,"DF51D23D5D2C57598F67791D953A6C2D9FC5865A3048ADA4469B37500D2996B95732E0DC6F99EAEAEA12B4818CE355C7B701D16857D2AC767D740C2E30E9260C"]}}],"peer_key":""}]}
+{"time":"2017-02-17T23:54:23.038Z","msg":[1,{"height":5,"round":0,"step":"RoundStepCommit"}]}
+#HEIGHT: 6
+{"time":"2017-02-17T23:54:23.041Z","msg":[1,{"height":6,"round":0,"step":"RoundStepNewHeight"}]}
+{"time":"2017-02-17T23:54:24.038Z","msg":[3,{"duration":997341910,"height":6,"round":0,"step":1}]}
+{"time":"2017-02-17T23:54:24.040Z","msg":[1,{"height":6,"round":0,"step":"RoundStepPropose"}]}
+{"time":"2017-02-17T23:54:24.040Z","msg":[2,{"msg":[17,{"Proposal":{"height":6,"round":0,"block_parts_header":{"total":1,"hash":"EA1E4111198195006BF7C23322B1051BE6C11582"},"pol_round":-1,"pol_block_id":{"hash":"","parts":{"total":0,"hash":""}},"signature":[1,"571309E5959472CF453B83BB00F75BFAE9ACA8981279CBCBF19FD1A104BAD544D43A4F67FC54C17C9D51CEE821E4F514A1742FA5220EFF432C334D81B03B4C08"]}}],"peer_key":""}]}
+{"time":"2017-02-17T23:54:24.040Z","msg":[2,{"msg":[19,{"Height":6,"Round":0,"Part":{"index":0,"bytes":"0101010F74656E6465726D696E745F74657374010614A43849369AF7C0018C0114FDC6D837995BEBBBFCBF3E7D7CF44F8FDA44854301010114A52BAA9C2E52E633A1605F4B930205613E3E7A2F0114AD3EFC80659CF2DCB932CAAA832249B1100C7D0901149D3F259328643A0DA0B83CF23A1AC918965E393D0114354594CBFC1A7BCA1AD0050ED6AA010023EADA390114BFB5455B51A6694370771F5072E0204CE75E6C6901018C011E363136323633363433363330333933443634363336323631333633303339011E363136323633363433363331333033443634363336323631333633313330011E363136323633363433363331333133443634363336323631333633313331011E363136323633363433363331333233443634363336323631333633313332011E363136323633363433363331333333443634363336323631333633313333011E363136323633363433363331333433443634363336323631333633313334011E363136323633363433363331333533443634363336323631333633313335011E363136323633363433363331333633443634363336323631333633313336011E363136323633363433363331333733443634363336323631333633313337011E363136323633363433363331333833443634363336323631333633313338011E363136323633363433363331333933443634363336323631333633313339011E363136323633363433363332333033443634363336323631333633323330011E363136323633363433363332333133443634363336323631333633323331011E363136323633363433363332333233443634363336323631333633323332011E363136323633363433363332333333443634363336323631333633323333011E363136323633363433363332333433443634363336323631333633323334011E363136323633363433363332333533443634363336323631333633323335011E363136323633363433363332333633443634363336323631333633323336011E363136323633363433363332333733443634363336323631333633323337011E363136323633363433363332333833443634363336323631333633323338011E363136323633363433363332333933443634363336323631333633323339011E363136323633363433363333333033443634363336323631333633333330011E363136323633363433363333333133443634363336323631333633333331011E363136323633363433363333333233443634363336323631333633333332011E363136323633363433363333333333443634363336323631333633333333011E363136323633363433363333333433443634363336323631333633333334011E363136323633363433363333333533443634363336323631333633333335011E363136323633363433363333333633443634363336323631333633333336011E363136323633363433363333333733443634363336323631333633333337011E363136323633363433363333333833443634363336323631333633333338011E363136323633363433363333333933443634363336323631333633333339011E363136323633363433363334333033443634363336323631333633343330011E363136323633363433363334333133443634363336323631333633343331011E363136323633363433363334333233443634363336323631333633343332011E363136323633363433363334333333443634363336323631333633343333011E363136323633363433363334333433443634363336323631333633343334011E363136323633363433363334333533443634363336323631333633343335011E363136323633363433363334333633443634363336323631333633343336011E363136323633363433363334333733443634363336323631333633343337011E363136323633363433363334333833443634363336323631333633343338011E363136323633363433363334333933443634363336323631333633343339011E363136323633363433363335333033443634363336323631333633353330011E363136323633363433363335333133443634363336323631333633353331011E363136323633363433363335333233443634363336323631333633353332011E363136323633363433363335333333443634363336323631333633353333011E363136323633363433363335333433443634363336323631333633353334011E363136323633363433363335333533443634363336323631333633353335011E363136323633363433363335333633443634363336323631333633353336011E363136323633363433363335333733443634363336323631333633353337011E363136323633363433363335333833443634363336323631333633353338011E363136323633363433363335333933443634363336323631333633353339011E363136323633363433363336333033443634363336323631333633363330011E363136323633363433363336333133443634363336323631333633363331011E363136323633363433363336333233443634363336323631333633363332011E363136323633363433363336333333443634363336323631333633363333011E363136323633363433363336333433443634363336323631333633363334011E363136323633363433363336333533443634363336323631333633363335011E363136323633363433363336333633443634363336323631333633363336011E363136323633363433363336333733443634363336323631333633363337011E363136323633363433363336333833443634363336323631333633363338011E363136323633363433363336333933443634363336323631333633363339011E363136323633363433363337333033443634363336323631333633373330011E363136323633363433363337333133443634363336323631333633373331011E363136323633363433363337333233443634363336323631333633373332011E363136323633363433363337333333443634363336323631333633373333011E363136323633363433363337333433443634363336323631333633373334011E363136323633363433363337333533443634363336323631333633373335011E363136323633363433363337333633443634363336323631333633373336011E363136323633363433363337333733443634363336323631333633373337011E363136323633363433363337333833443634363336323631333633373338011E363136323633363433363337333933443634363336323631333633373339011E363136323633363433363338333033443634363336323631333633383330011E363136323633363433363338333133443634363336323631333633383331011E363136323633363433363338333233443634363336323631333633383332011E363136323633363433363338333333443634363336323631333633383333011E363136323633363433363338333433443634363336323631333633383334011E363136323633363433363338333533443634363336323631333633383335011E363136323633363433363338333633443634363336323631333633383336011E363136323633363433363338333733443634363336323631333633383337011E363136323633363433363338333833443634363336323631333633383338011E363136323633363433363338333933443634363336323631333633383339011E363136323633363433363339333033443634363336323631333633393330011E363136323633363433363339333133443634363336323631333633393331011E363136323633363433363339333233443634363336323631333633393332011E363136323633363433363339333333443634363336323631333633393333011E363136323633363433363339333433443634363336323631333633393334011E363136323633363433363339333533443634363336323631333633393335011E363136323633363433363339333633443634363336323631333633393336011E363136323633363433363339333733443634363336323631333633393337011E363136323633363433363339333833443634363336323631333633393338011E363136323633363433363339333933443634363336323631333633393339011E363136323633363433373330333033443634363336323631333733303330011E363136323633363433373330333133443634363336323631333733303331011E363136323633363433373330333233443634363336323631333733303332011E363136323633363433373330333333443634363336323631333733303333011E363136323633363433373330333433443634363336323631333733303334011E363136323633363433373330333533443634363336323631333733303335011E363136323633363433373330333633443634363336323631333733303336011E363136323633363433373330333733443634363336323631333733303337011E363136323633363433373330333833443634363336323631333733303338011E363136323633363433373330333933443634363336323631333733303339011E363136323633363433373331333033443634363336323631333733313330011E363136323633363433373331333133443634363336323631333733313331011E363136323633363433373331333233443634363336323631333733313332011E363136323633363433373331333333443634363336323631333733313333011E363136323633363433373331333433443634363336323631333733313334011E363136323633363433373331333533443634363336323631333733313335011E363136323633363433373331333633443634363336323631333733313336011E363136323633363433373331333733443634363336323631333733313337011E363136323633363433373331333833443634363336323631333733313338011E363136323633363433373331333933443634363336323631333733313339011E363136323633363433373332333033443634363336323631333733323330011E363136323633363433373332333133443634363336323631333733323331011E363136323633363433373332333233443634363336323631333733323332011E363136323633363433373332333333443634363336323631333733323333011E363136323633363433373332333433443634363336323631333733323334011E363136323633363433373332333533443634363336323631333733323335011E363136323633363433373332333633443634363336323631333733323336011E363136323633363433373332333733443634363336323631333733323337011E363136323633363433373332333833443634363336323631333733323338011E363136323633363433373332333933443634363336323631333733323339011E363136323633363433373333333033443634363336323631333733333330011E363136323633363433373333333133443634363336323631333733333331011E363136323633363433373333333233443634363336323631333733333332011E363136323633363433373333333333443634363336323631333733333333011E363136323633363433373333333433443634363336323631333733333334011E363136323633363433373333333533443634363336323631333733333335011E363136323633363433373333333633443634363336323631333733333336011E363136323633363433373333333733443634363336323631333733333337011E363136323633363433373333333833443634363336323631333733333338011E363136323633363433373333333933443634363336323631333733333339011E363136323633363433373334333033443634363336323631333733343330011E363136323633363433373334333133443634363336323631333733343331011E363136323633363433373334333233443634363336323631333733343332011E363136323633363433373334333333443634363336323631333733343333011E363136323633363433373334333433443634363336323631333733343334011E363136323633363433373334333533443634363336323631333733343335011E363136323633363433373334333633443634363336323631333733343336011E363136323633363433373334333733443634363336323631333733343337011E363136323633363433373334333833443634363336323631333733343338010114FDC6D837995BEBBBFCBF3E7D7CF44F8FDA44854301010114A52BAA9C2E52E633A1605F4B930205613E3E7A2F0101010114D028C9981F7A87F3093672BF0D5B0E2A1B3ED45600010500020114FDC6D837995BEBBBFCBF3E7D7CF44F8FDA44854301010114A52BAA9C2E52E633A1605F4B930205613E3E7A2F01DF51D23D5D2C57598F67791D953A6C2D9FC5865A3048ADA4469B37500D2996B95732E0DC6F99EAEAEA12B4818CE355C7B701D16857D2AC767D740C2E30E9260C","proof":{"aunts":[]}}}],"peer_key":""}]}
+{"time":"2017-02-17T23:54:24.041Z","msg":[1,{"height":6,"round":0,"step":"RoundStepPrevote"}]}
+{"time":"2017-02-17T23:54:24.041Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":6,"round":0,"type":1,"block_id":{"hash":"1F7C249FF99B67AC57C4EEC94C42E0B95C9AFB6B","parts":{"total":1,"hash":"EA1E4111198195006BF7C23322B1051BE6C11582"}},"signature":[1,"1F79910354E1F4ACA11FC16DBA1ED6F75063A15BF8093C4AAEF87F69B3990F65E51FFC8B35A409838ECD0FF3C26E87637B068D0DC7E5863D5F1CF97826222300"]}}],"peer_key":""}]}
+{"time":"2017-02-17T23:54:24.042Z","msg":[1,{"height":6,"round":0,"step":"RoundStepPrecommit"}]}
+{"time":"2017-02-17T23:54:24.042Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":6,"round":0,"type":2,"block_id":{"hash":"1F7C249FF99B67AC57C4EEC94C42E0B95C9AFB6B","parts":{"total":1,"hash":"EA1E4111198195006BF7C23322B1051BE6C11582"}},"signature":[1,"E7838F403E4D5E651317D8563355CB2E140409B1B471B4AC12EBF7085989228CFA062029DF78A405CF977925777177D876804D78D80DF2312977E6D804394A0E"]}}],"peer_key":""}]}
+{"time":"2017-02-17T23:54:24.042Z","msg":[1,{"height":6,"round":0,"step":"RoundStepCommit"}]}
diff --git a/consensus/wal.go b/consensus/wal.go
index 099e3c1a..99035ee2 100644
--- a/consensus/wal.go
+++ b/consensus/wal.go
@@ -40,8 +40,8 @@ type WAL struct {
 	light bool // ignore block parts
 }
 
-func NewWAL(walDir string, light bool) (*WAL, error) {
-	group, err := auto.OpenGroup(walDir + "/wal")
+func NewWAL(walFile string, light bool) (*WAL, error) {
+	group, err := auto.OpenGroup(walFile)
 	if err != nil {
 		return nil, err
 	}
diff --git a/state/execution.go b/state/execution.go
index 88b80e48..3ae0a9d3 100644
--- a/state/execution.go
+++ b/state/execution.go
@@ -1,6 +1,7 @@
 package state
 
 import (
+	"bytes"
 	"errors"
 
 	"github.com/ebuchman/fail-test"
@@ -278,7 +279,7 @@ func applyBlock(appConnConsensus proxy.AppConnConsensus, block *types.Block) ([]
 	var eventCache types.Fireable // nil
 	_, err := execBlockOnProxyApp(eventCache, appConnConsensus, block)
 	if err != nil {
-		log.Warn("Error executing block on proxy app", "height", i, "err", err)
+		log.Warn("Error executing block on proxy app", "height", block.Height, "err", err)
 		return nil, err
 	}
 	// Commit block, get hash back
@@ -388,13 +389,17 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, proxyApp p
 		// if the app is ahead, there's nothing we can do
 		return ErrAppBlockHeightTooHigh{storeBlockHeight, appBlockHeight}
 
+	} else if storeBlockHeight == appBlockHeight && storeBlockHeight == stateBlockHeight {
+		// all good!
+		return nil
+
 	} else if storeBlockHeight == appBlockHeight && storeBlockHeight == stateBlockHeight+1 {
 		// We already ran Commit, but didn't save the state, so run through consensus with mock app
 		mockApp := newMockProxyApp(appHash)
 		log.Info("Replay last block using mock app")
 		h.replayLastBlock(h.config, h.state, mockApp, h.store)
 
-	} else if storeBlockHeight == appBlockHeight+1 {
+	} else if storeBlockHeight == appBlockHeight+1 && storeBlockHeight == stateBlockHeight+1 {
 		// We crashed after saving the block
 		// but before Commit (both the state and app are behind),
 		// so run through consensus with the real app
@@ -409,7 +414,8 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, proxyApp p
 
 		var appHash []byte
 		var err error
-		for i := appBlockHeight + 1; i <= storeBlockHeight-1; i++ {
+		for i := appBlockHeight + 1; i <= storeBlockHeight; i++ {
+			log.Info("Applying block", "height", i)
 			h.nBlocks += 1
 			block := h.store.LoadBlock(i)
 			appHash, err = applyBlock(proxyApp.Consensus(), block)
@@ -418,7 +424,7 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, proxyApp p
 			}
 		}
 
-		h.replayLastBlock(h.config, h.state, proxyApp.Consensus(), h.store)
+		// h.replayLastBlock(h.config, h.state, proxyApp.Consensus(), h.store)
 		if !bytes.Equal(h.state.AppHash, appHash) {
 			return errors.New(Fmt("Tendermint state.AppHash does not match AppHash after replay. Got %X, expected %X", appHash, h.state.AppHash))
 		}

From 7228b11e3f63d27046b76cb95d89110c2c3e0ef2 Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Fri, 17 Feb 2017 19:13:35 -0500
Subject: [PATCH 053/132] state: remove StateIntermediate

---
 state/execution.go |  4 ----
 state/state.go     | 32 +-------------------------------
 2 files changed, 1 insertion(+), 35 deletions(-)

diff --git a/state/execution.go b/state/execution.go
index 3ae0a9d3..e03001da 100644
--- a/state/execution.go
+++ b/state/execution.go
@@ -54,10 +54,6 @@ func (s *State) ExecBlock(eventCache types.Fireable, proxyAppConn proxy.AppConnC
 	nextValSet.IncrementAccum(1)
 	s.SetBlockAndValidators(block.Header, blockPartsHeader, valSet, nextValSet)
 
-	// save state with updated height/blockhash/validators
-	// but stale apphash, in case we fail between Commit and Save
-	s.SaveIntermediate()
-
 	fail.Fail() // XXX
 
 	return nil
diff --git a/state/state.go b/state/state.go
index f4af8ee0..c4c6d748 100644
--- a/state/state.go
+++ b/state/state.go
@@ -14,8 +14,7 @@ import (
 )
 
 var (
-	stateKey             = []byte("stateKey")
-	stateIntermediateKey = []byte("stateIntermediateKey")
+	stateKey = []byte("stateKey")
 )
 
 //-----------------------------------------------------------------------------
@@ -82,35 +81,6 @@ func (s *State) Save() {
 	s.db.SetSync(stateKey, s.Bytes())
 }
 
-func (s *State) SaveIntermediate() {
-	s.mtx.Lock()
-	defer s.mtx.Unlock()
-	s.db.SetSync(stateIntermediateKey, s.Bytes())
-}
-
-// Load the intermediate state into the current state
-// and do some sanity checks
-func (s *State) LoadIntermediate() {
-	s2 := loadState(s.db, stateIntermediateKey)
-	if s.ChainID != s2.ChainID {
-		PanicSanity(Fmt("State mismatch for ChainID. Got %v, Expected %v", s2.ChainID, s.ChainID))
-	}
-
-	if s.LastBlockHeight+1 != s2.LastBlockHeight {
-		PanicSanity(Fmt("State mismatch for LastBlockHeight. Got %v, Expected %v", s2.LastBlockHeight, s.LastBlockHeight+1))
-	}
-
-	if !bytes.Equal(s.Validators.Hash(), s2.LastValidators.Hash()) {
-		PanicSanity(Fmt("State mismatch for LastValidators. Got %X, Expected %X", s2.LastValidators.Hash(), s.Validators.Hash()))
-	}
-
-	if !bytes.Equal(s.AppHash, s2.AppHash) {
-		PanicSanity(Fmt("State mismatch for AppHash. Got %X, Expected %X", s2.AppHash, s.AppHash))
-	}
-
-	s.setBlockAndValidators(s2.LastBlockHeight, s2.LastBlockID, s2.LastBlockTime, s2.Validators.Copy(), s2.LastValidators.Copy())
-}
-
 func (s *State) Equals(s2 *State) bool {
 	return bytes.Equal(s.Bytes(), s2.Bytes())
 }

From 44d472ddd3e5c7048deeb5022533abc28bc2c799 Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Sat, 18 Feb 2017 22:15:59 -0500
Subject: [PATCH 054/132] comments from review

---
 consensus/state.go | 1 +
 state/execution.go | 6 ++++--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/consensus/state.go b/consensus/state.go
index a985fd2c..1c864c6d 100644
--- a/consensus/state.go
+++ b/consensus/state.go
@@ -259,6 +259,7 @@ func ReplayLastBlock(config cfg.Config, state *sm.State, proxyApp proxy.AppConnC
 
 	evsw := types.NewEventSwitch()
 	evsw.Start()
+	defer evsw.Stop()
 	cs.SetEventSwitch(evsw)
 	newBlockCh := subscribeToEvent(evsw, "consensus-replay", types.EventStringNewBlock(), 1)
 
diff --git a/state/execution.go b/state/execution.go
index e03001da..71c2dd75 100644
--- a/state/execution.go
+++ b/state/execution.go
@@ -269,8 +269,8 @@ func (s *State) CommitStateUpdateMempool(proxyAppConn proxy.AppConnConsensus, bl
 	return nil
 }
 
-// apply a nd commit a block, but with out all the state validation
-// returns the application root hash (result of abci.Commit)
+// Apply and commit a block, but without all the state validation.
+// Returns the application root hash (result of abci.Commit)
 func applyBlock(appConnConsensus proxy.AppConnConsensus, block *types.Block) ([]byte, error) {
 	var eventCache types.Fireable // nil
 	_, err := execBlockOnProxyApp(eventCache, appConnConsensus, block)
@@ -420,7 +420,9 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, proxyApp p
 			}
 		}
 
+		// TODO: should we be playing the final block through the consensus, instead of using applyBlock?
 		// h.replayLastBlock(h.config, h.state, proxyApp.Consensus(), h.store)
+
 		if !bytes.Equal(h.state.AppHash, appHash) {
 			return errors.New(Fmt("Tendermint state.AppHash does not match AppHash after replay. Got %X, expected %X", appHash, h.state.AppHash))
 		}

From a99885cd439a02b5aa60e395f1499dcd12a247f1 Mon Sep 17 00:00:00 2001
From: Anton Kaliaev <anton.kalyaev@gmail.com>
Date: Mon, 20 Feb 2017 11:13:44 +0400
Subject: [PATCH 055/132] do not overwrite existing files when doing
 `tendermint init` (Fixes #410)

---
 cmd/tendermint/init.go | 39 +++++++++++++++++++++++++--------------
 1 file changed, 25 insertions(+), 14 deletions(-)

diff --git a/cmd/tendermint/init.go b/cmd/tendermint/init.go
index 8a7a7b09..5ec1efef 100644
--- a/cmd/tendermint/init.go
+++ b/cmd/tendermint/init.go
@@ -1,24 +1,35 @@
 package main
 
 import (
-	. "github.com/tendermint/go-common"
+	"os"
+
+	cmn "github.com/tendermint/go-common"
 	"github.com/tendermint/tendermint/types"
 )
 
 func init_files() {
-	privValidator := types.GenPrivValidator()
-	privValidator.SetFile(config.GetString("priv_validator_file"))
-	privValidator.Save()
+	privValFile := config.GetString("priv_validator_file")
+	if _, err := os.Stat(privValFile); os.IsNotExist(err) {
+		privValidator := types.GenPrivValidator()
+		privValidator.SetFile(privValFile)
+		privValidator.Save()
 
-	genDoc := types.GenesisDoc{
-		ChainID: Fmt("test-chain-%v", RandStr(6)),
+		genFile := config.GetString("genesis_file")
+
+		if _, err := os.Stat(genFile); os.IsNotExist(err) {
+			genDoc := types.GenesisDoc{
+				ChainID: cmn.Fmt("test-chain-%v", cmn.RandStr(6)),
+			}
+			genDoc.Validators = []types.GenesisValidator{types.GenesisValidator{
+				PubKey: privValidator.PubKey,
+				Amount: 10,
+			}}
+
+			genDoc.SaveAs(genFile)
+		}
+
+		log.Notice("Initialized tendermint", "genesis", config.GetString("genesis_file"), "priv_validator", config.GetString("priv_validator_file"))
+	} else {
+		log.Notice("Already initialized", "priv_validator", config.GetString("priv_validator_file"))
 	}
-	genDoc.Validators = []types.GenesisValidator{types.GenesisValidator{
-		PubKey: privValidator.PubKey,
-		Amount: 10,
-	}}
-
-	genDoc.SaveAs(config.GetString("genesis_file"))
-
-	log.Notice("Initialized tendermint", "genesis", config.GetString("genesis_file"), "priv_validator", config.GetString("priv_validator_file"))
 }

From bc678596729376465f56ac0a339a6799eef38e0a Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Mon, 20 Feb 2017 16:24:35 -0500
Subject: [PATCH 056/132] make ReplayBlocks logic exhaustive

---
 consensus/replay_test.go |  27 +++++---
 consensus/state.go       |  11 +--
 state/execution.go       | 145 ++++++++++++++++++++++++++-------------
 3 files changed, 121 insertions(+), 62 deletions(-)

diff --git a/consensus/replay_test.go b/consensus/replay_test.go
index a9123a8e..e8aa9e90 100644
--- a/consensus/replay_test.go
+++ b/consensus/replay_test.go
@@ -15,7 +15,7 @@ import (
 	"github.com/tendermint/tendermint/config/tendermint_test"
 
 	"github.com/tendermint/abci/example/dummy"
-	. "github.com/tendermint/go-common"
+	cmn "github.com/tendermint/go-common"
 	cfg "github.com/tendermint/go-config"
 	"github.com/tendermint/go-crypto"
 	dbm "github.com/tendermint/go-db"
@@ -37,7 +37,7 @@ func init() {
 // the `Handshake Tests` are for failures in applying the block.
 // With the help of the WAL, we can recover from it all!
 
-var data_dir = path.Join(GoPath, "src/github.com/tendermint/tendermint/consensus", "test_data")
+var data_dir = path.Join(cmn.GoPath, "src/github.com/tendermint/tendermint/consensus", "test_data")
 
 //------------------------------------------------------------------------------------------
 // WAL Tests
@@ -68,7 +68,7 @@ type testCase struct {
 
 func newTestCase(name string, stepChanges []int) *testCase {
 	if len(stepChanges) != 3 {
-		panic(Fmt("a full wal has 3 step changes! Got array %v", stepChanges))
+		panic(cmn.Fmt("a full wal has 3 step changes! Got array %v", stepChanges))
 	}
 	return &testCase{
 		name:    name,
@@ -103,15 +103,15 @@ func readWAL(p string) string {
 
 func writeWAL(walMsgs string) string {
 	tempDir := os.TempDir()
-	walDir := path.Join(tempDir, "/wal"+RandStr(12))
+	walDir := path.Join(tempDir, "/wal"+cmn.RandStr(12))
 	walFile := path.Join(walDir, "wal")
 	// Create WAL directory
-	err := EnsureDir(walDir, 0700)
+	err := cmn.EnsureDir(walDir, 0700)
 	if err != nil {
 		panic(err)
 	}
 	// Write the needed WAL to file
-	err = WriteFile(walFile, []byte(walMsgs), 0600)
+	err = cmn.WriteFile(walFile, []byte(walMsgs), 0600)
 	if err != nil {
 		panic(err)
 	}
@@ -123,7 +123,7 @@ func waitForBlock(newBlockCh chan interface{}, thisCase *testCase, i int) {
 	select {
 	case <-newBlockCh:
 	case <-after:
-		panic(Fmt("Timed out waiting for new block for case '%s' line %d", thisCase.name, i))
+		panic(cmn.Fmt("Timed out waiting for new block for case '%s' line %d", thisCase.name, i))
 	}
 }
 
@@ -156,7 +156,7 @@ func toPV(pv PrivValidator) *types.PrivValidator {
 
 func setupReplayTest(thisCase *testCase, nLines int, crashAfter bool) (*ConsensusState, chan interface{}, string, string) {
 	fmt.Println("-------------------------------------")
-	log.Notice(Fmt("Starting replay test %v (of %d lines of WAL). Crash after = %v", thisCase.name, nLines, crashAfter))
+	log.Notice(cmn.Fmt("Starting replay test %v (of %d lines of WAL). Crash after = %v", thisCase.name, nLines, crashAfter))
 
 	lineStep := nLines
 	if crashAfter {
@@ -285,8 +285,15 @@ func TestHandshakeReplayNone(t *testing.T) {
 // Make some blocks. Start a fresh app and apply nBlocks blocks. Then restart the app and sync it up with the remaining blocks
 func testHandshakeReplay(t *testing.T, nBlocks int) {
 	config := tendermint_test.ResetConfig("proxy_test_")
-	walFile := path.Join(data_dir, "many_blocks.cswal")
+
+	// copy the many_blocks file
+	walBody, err := cmn.ReadFile(path.Join(data_dir, "many_blocks.cswal"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	walFile := writeWAL(string(walBody))
 	config.Set("cs_wal_file", walFile)
+
 	privVal := types.LoadPrivValidator(config.GetString("priv_validator_file"))
 	testPartSize = config.GetInt("block_part_size")
 
@@ -371,7 +378,7 @@ func makeBlockchainFromWAL(wal *WAL) ([]*types.Block, []*types.Commit, error) {
 		return nil, nil, err
 	}
 	if !found {
-		return nil, nil, errors.New(Fmt("WAL does not contain height %d.", 1))
+		return nil, nil, errors.New(cmn.Fmt("WAL does not contain height %d.", 1))
 	}
 	defer gr.Close()
 
diff --git a/consensus/state.go b/consensus/state.go
index 1c864c6d..c077c412 100644
--- a/consensus/state.go
+++ b/consensus/state.go
@@ -253,7 +253,8 @@ type ConsensusState struct {
 	done chan struct{}
 }
 
-func ReplayLastBlock(config cfg.Config, state *sm.State, proxyApp proxy.AppConnConsensus, blockStore sm.BlockStore) {
+// Replay the last block through the consensus and return the AppHash after commit.
+func ReplayLastBlock(config cfg.Config, state *sm.State, proxyApp proxy.AppConnConsensus, blockStore sm.BlockStore) ([]byte, error) {
 	mempool := sm.MockMempool{}
 	cs := NewConsensusState(config, state, proxyApp, blockStore, mempool)
 
@@ -265,8 +266,10 @@ func ReplayLastBlock(config cfg.Config, state *sm.State, proxyApp proxy.AppConnC
 
 	// run through the WAL, commit new block, stop
 	cs.Start()
-	<-newBlockCh
+	<-newBlockCh // TODO: use a timeout and return err?
 	cs.Stop()
+
+	return cs.state.AppHash, nil
 }
 
 func NewConsensusState(config cfg.Config, state *sm.State, proxyAppConn proxy.AppConnConsensus, blockStore sm.BlockStore, mempool sm.Mempool) *ConsensusState {
@@ -1235,7 +1238,7 @@ func (cs *ConsensusState) finalizeCommit(height int) {
 		cs.blockStore.SaveBlock(block, blockParts, seenCommit)
 	} else {
 		// Happens during replay if we already saved the block but didn't commit
-		log.Notice("Calling finalizeCommit on already stored block", "height", block.Height)
+		log.Info("Calling finalizeCommit on already stored block", "height", block.Height)
 	}
 
 	fail.Fail() // XXX
@@ -1250,7 +1253,7 @@ func (cs *ConsensusState) finalizeCommit(height int) {
 	// NOTE: the block.AppHash wont reflect these txs until the next block
 	err := stateCopy.ApplyBlock(eventCache, cs.proxyAppConn, block, blockParts.Header(), cs.mempool)
 	if err != nil {
-		log.Warn("Error on ApplyBlock. Did the application crash? Please restart tendermint", "error", err)
+		log.Error("Error on ApplyBlock. Did the application crash? Please restart tendermint", "error", err)
 		return
 	}
 
diff --git a/state/execution.go b/state/execution.go
index 71c2dd75..7677a672 100644
--- a/state/execution.go
+++ b/state/execution.go
@@ -260,6 +260,7 @@ func (s *State) CommitStateUpdateMempool(proxyAppConn proxy.AppConnConsensus, bl
 		log.Debug("Commit.Log: " + res.Log)
 	}
 
+	log.Info("Committed state", "hash", res.Data)
 	// Set the state's new AppHash
 	s.AppHash = res.Data
 
@@ -327,7 +328,8 @@ type BlockStore interface {
 	LoadSeenCommit(height int) *types.Commit
 }
 
-type blockReplayFunc func(cfg.Config, *State, proxy.AppConnConsensus, BlockStore)
+// returns the apphash from Commit
+type blockReplayFunc func(cfg.Config, *State, proxy.AppConnConsensus, BlockStore) ([]byte, error)
 
 type Handshaker struct {
 	config          cfg.Config
@@ -362,73 +364,120 @@ func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error {
 	// TODO: check version
 
 	// replay blocks up to the latest in the blockstore
-	err = h.ReplayBlocks(appHash, blockHeight, proxyApp)
+	appHash, err = h.ReplayBlocks(appHash, blockHeight, proxyApp)
 	if err != nil {
 		return errors.New(Fmt("Error on replay: %v", err))
 	}
 
+	// NOTE: the h.state may now be behind the cs state
+
 	// TODO: (on restart) replay mempool
 
 	return nil
 }
 
 // Replay all blocks after blockHeight and ensure the result matches the current state.
-func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, proxyApp proxy.AppConns) error {
+// Returns the final AppHash or an error
+func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, proxyApp proxy.AppConns) ([]byte, error) {
 
 	storeBlockHeight := h.store.Height()
 	stateBlockHeight := h.state.LastBlockHeight
 	log.Notice("ABCI Replay Blocks", "appHeight", appBlockHeight, "storeHeight", storeBlockHeight, "stateHeight", stateBlockHeight)
 
+	// First handle edge cases and constraints on the storeBlockHeight
 	if storeBlockHeight == 0 {
-		return nil
+		return nil, nil
+
 	} else if storeBlockHeight < appBlockHeight {
-		// if the app is ahead, there's nothing we can do
-		return ErrAppBlockHeightTooHigh{storeBlockHeight, appBlockHeight}
+		// the app should never be ahead of the store (but this is under app's control)
+		return nil, ErrAppBlockHeightTooHigh{storeBlockHeight, appBlockHeight}
 
-	} else if storeBlockHeight == appBlockHeight && storeBlockHeight == stateBlockHeight {
-		// all good!
-		return nil
+	} else if storeBlockHeight < stateBlockHeight {
+		// the state should never be ahead of the store (this is under tendermint's control)
+		PanicSanity(Fmt("StateBlockHeight (%d) > StoreBlockHeight (%d)", stateBlockHeight, storeBlockHeight))
 
-	} else if storeBlockHeight == appBlockHeight && storeBlockHeight == stateBlockHeight+1 {
-		// We already ran Commit, but didn't save the state, so run through consensus with mock app
-		mockApp := newMockProxyApp(appHash)
-		log.Info("Replay last block using mock app")
-		h.replayLastBlock(h.config, h.state, mockApp, h.store)
-
-	} else if storeBlockHeight == appBlockHeight+1 && storeBlockHeight == stateBlockHeight+1 {
-		// We crashed after saving the block
-		// but before Commit (both the state and app are behind),
-		// so run through consensus with the real app
-		log.Info("Replay last block using real app")
-		h.replayLastBlock(h.config, h.state, proxyApp.Consensus(), h.store)
-
-	} else {
-		// store is more than one ahead,
-		// so app wants to replay many blocks.
-		// replay all blocks from appBlockHeight+1 to storeBlockHeight-1.
-		// Replay the final block through consensus
-
-		var appHash []byte
-		var err error
-		for i := appBlockHeight + 1; i <= storeBlockHeight; i++ {
-			log.Info("Applying block", "height", i)
-			h.nBlocks += 1
-			block := h.store.LoadBlock(i)
-			appHash, err = applyBlock(proxyApp.Consensus(), block)
-			if err != nil {
-				return err
-			}
-		}
-
-		// TODO: should we be playing the final block through the consensus, instead of using applyBlock?
-		// h.replayLastBlock(h.config, h.state, proxyApp.Consensus(), h.store)
-
-		if !bytes.Equal(h.state.AppHash, appHash) {
-			return errors.New(Fmt("Tendermint state.AppHash does not match AppHash after replay. Got %X, expected %X", appHash, h.state.AppHash))
-		}
-		return nil
+	} else if storeBlockHeight > stateBlockHeight+1 {
+		// store should be at most one ahead of the state (this is under tendermint's control)
+		PanicSanity(Fmt("StoreBlockHeight (%d) > StateBlockHeight + 1 (%d)", storeBlockHeight, stateBlockHeight+1))
 	}
-	return nil
+
+	// Now either store is equal to state, or one ahead.
+	// For each, consider all cases of where the app could be, given app <= store
+	if storeBlockHeight == stateBlockHeight {
+		// Tendermint ran Commit and saved the state.
+		// Either the app is asking for replay, or we're all synced up.
+		if appBlockHeight < storeBlockHeight {
+			// the app is behind, so replay blocks, but no need to go through WAL (state is already synced to store)
+			return h.replayBlocks(proxyApp, appBlockHeight, storeBlockHeight, false)
+
+		} else if appBlockHeight == storeBlockHeight {
+			// we're good!
+			return appHash, nil
+		}
+
+	} else if storeBlockHeight == stateBlockHeight+1 {
+		// We saved the block in the store but haven't updated the state,
+		// so we'll need to replay a block using the WAL.
+		if appBlockHeight < stateBlockHeight {
+			// the app is further behind than it should be, so replay blocks
+			// but leave the last block to go through the WAL
+			return h.replayBlocks(proxyApp, appBlockHeight, storeBlockHeight, true)
+
+		} else if appBlockHeight == stateBlockHeight {
+			// We haven't run Commit (both the state and app are one block behind),
+			// so run through consensus with the real app
+			log.Info("Replay last block using real app")
+			return h.replayLastBlock(h.config, h.state, proxyApp.Consensus(), h.store)
+
+		} else if appBlockHeight == storeBlockHeight {
+			// We ran Commit, but didn't save the state, so run through consensus with mock app
+			mockApp := newMockProxyApp(appHash)
+			log.Info("Replay last block using mock app")
+			return h.replayLastBlock(h.config, h.state, mockApp, h.store)
+		}
+
+	}
+
+	PanicSanity("Should never happen")
+	return nil, nil
+}
+
+func (h *Handshaker) replayBlocks(proxyApp proxy.AppConns, appBlockHeight, storeBlockHeight int, useReplayFunc bool) ([]byte, error) {
+	// App is further behind than it should be, so we need to replay blocks.
+	// We replay all blocks from appBlockHeight+1 to storeBlockHeight-1,
+	// and let the final block be replayed through ReplayBlocks.
+	// Note that we don't have an old version of the state,
+	// so we by-pass state validation using applyBlock here.
+
+	var appHash []byte
+	var err error
+	finalBlock := storeBlockHeight
+	if useReplayFunc {
+		finalBlock -= 1
+	}
+	for i := appBlockHeight + 1; i <= finalBlock; i++ {
+		log.Info("Applying block", "height", i)
+		h.nBlocks += 1
+		block := h.store.LoadBlock(i)
+		appHash, err = applyBlock(proxyApp.Consensus(), block)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if useReplayFunc {
+		// sync the final block
+		appHash, err = h.ReplayBlocks(appHash, finalBlock, proxyApp)
+		if err != nil {
+			return appHash, err
+		}
+	}
+
+	if !bytes.Equal(h.state.AppHash, appHash) {
+		return nil, errors.New(Fmt("Tendermint state.AppHash does not match AppHash after replay. Got %X, expected %X", appHash, h.state.AppHash))
+	}
+
+	return appHash, nil
 }
 
 //--------------------------------------------------------------------------------

From 756213c5f509425e9a253d707873cbe8504b7a28 Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Mon, 20 Feb 2017 17:08:38 -0500
Subject: [PATCH 057/132] check appHash

---
 consensus/state.go |  2 +-
 state/execution.go | 23 ++++++++++++-----------
 2 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/consensus/state.go b/consensus/state.go
index c077c412..8e721ed3 100644
--- a/consensus/state.go
+++ b/consensus/state.go
@@ -253,7 +253,7 @@ type ConsensusState struct {
 	done chan struct{}
 }
 
-// Replay the last block through the consensus and return the AppHash after commit.
+// Replay the last block through the consensus and return the AppHash from after Commit.
 func ReplayLastBlock(config cfg.Config, state *sm.State, proxyApp proxy.AppConnConsensus, blockStore sm.BlockStore) ([]byte, error) {
 	mempool := sm.MockMempool{}
 	cs := NewConsensusState(config, state, proxyApp, blockStore, mempool)
diff --git a/state/execution.go b/state/execution.go
index 7677a672..c1471bf8 100644
--- a/state/execution.go
+++ b/state/execution.go
@@ -364,19 +364,17 @@ func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error {
 	// TODO: check version
 
 	// replay blocks up to the latest in the blockstore
-	appHash, err = h.ReplayBlocks(appHash, blockHeight, proxyApp)
+	_, err = h.ReplayBlocks(appHash, blockHeight, proxyApp)
 	if err != nil {
 		return errors.New(Fmt("Error on replay: %v", err))
 	}
 
-	// NOTE: the h.state may now be behind the cs state
-
 	// TODO: (on restart) replay mempool
 
 	return nil
 }
 
-// Replay all blocks after blockHeight and ensure the result matches the current state.
+// Replay all blocks since appBlockHeight and ensure the result matches the current state.
 // Returns the final AppHash or an error
 func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, proxyApp proxy.AppConns) ([]byte, error) {
 
@@ -386,11 +384,11 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, proxyApp p
 
 	// First handle edge cases and constraints on the storeBlockHeight
 	if storeBlockHeight == 0 {
-		return nil, nil
+		return appHash, h.checkAppHash(appHash)
 
 	} else if storeBlockHeight < appBlockHeight {
 		// the app should never be ahead of the store (but this is under app's control)
-		return nil, ErrAppBlockHeightTooHigh{storeBlockHeight, appBlockHeight}
+		return appHash, ErrAppBlockHeightTooHigh{storeBlockHeight, appBlockHeight}
 
 	} else if storeBlockHeight < stateBlockHeight {
 		// the state should never be ahead of the store (this is under tendermint's control)
@@ -412,7 +410,7 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, proxyApp p
 
 		} else if appBlockHeight == storeBlockHeight {
 			// we're good!
-			return appHash, nil
+			return appHash, h.checkAppHash(appHash)
 		}
 
 	} else if storeBlockHeight == stateBlockHeight+1 {
@@ -473,11 +471,14 @@ func (h *Handshaker) replayBlocks(proxyApp proxy.AppConns, appBlockHeight, store
 		}
 	}
 
-	if !bytes.Equal(h.state.AppHash, appHash) {
-		return nil, errors.New(Fmt("Tendermint state.AppHash does not match AppHash after replay. Got %X, expected %X", appHash, h.state.AppHash))
-	}
+	return appHash, h.checkAppHash(appHash)
+}
 
-	return appHash, nil
+func (h *Handshaker) checkAppHash(appHash []byte) error {
+	if !bytes.Equal(h.state.AppHash, appHash) {
+		return errors.New(Fmt("Tendermint state.AppHash does not match AppHash after replay. Got %X, expected %X", appHash, h.state.AppHash))
+	}
+	return nil
 }
 
 //--------------------------------------------------------------------------------

From 2b1b8da58d9a07defc5d97baff6741d61bba397b Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Mon, 20 Feb 2017 18:41:29 -0500
Subject: [PATCH 058/132] test/persist: dont use log files on circle

---
 test/persist/test_failure_indices.sh | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/test/persist/test_failure_indices.sh b/test/persist/test_failure_indices.sh
index d828afe3..b2441235 100644
--- a/test/persist/test_failure_indices.sh
+++ b/test/persist/test_failure_indices.sh
@@ -6,19 +6,35 @@ export TMROOT=$HOME/.tendermint_persist
 rm -rf $TMROOT
 tendermint init
 
+TM_CMD="tendermint node --log_level=debug" # &> tendermint_${name}.log"
+DUMMY_CMD="dummy --persist $TMROOT/dummy" # &> dummy_${name}.log"
+
+
 function start_procs(){
 	name=$1
 	indexToFail=$2
 	echo "Starting persistent dummy and tendermint"
-	dummy --persist $TMROOT/dummy &> "dummy_${name}.log" &
+	if [[ "$CIRCLECI" == true ]]; then
+		$DUMMY_CMD &
+	else
+		$DUMMY_CMD &> "dummy_${name}.log" &
+	fi
 	PID_DUMMY=$!
 	if [[ "$indexToFail" == "" ]]; then
 		# run in background, dont fail
-		tendermint node --log_level=debug &> tendermint_${name}.log &
+		if [[ "$CIRCLECI" == true ]]; then
+			$TM_CMD &
+		else
+			$TM_CMD &> "tendermint_${name}.log" & 
+		fi
 		PID_TENDERMINT=$!
 	else
 		# run in foreground, fail
-		FAIL_TEST_INDEX=$indexToFail tendermint node --log_level=debug &> tendermint_${name}.log
+		if [[ "$CIRCLECI" == true ]]; then
+			FAIL_TEST_INDEX=$indexToFail $TM_CMD
+		else 
+			FAIL_TEST_INDEX=$indexToFail $TM_CMD &> "tendermint_${name}.log"
+		fi
 		PID_TENDERMINT=$!
 	fi
 }

From 1fa6e7f3b100760d788f0bbb15e178ca33f45130 Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Mon, 20 Feb 2017 18:51:00 -0500
Subject: [PATCH 059/132] test: shellcheck

---
 test/persist/test.sh                 |  2 +-
 test/persist/test_failure_indices.sh | 38 ++++++++++++++--------------
 test/run_test.sh                     |  4 +--
 test/test_cover.sh                   |  2 +-
 test/test_libs.sh                    | 12 ++++-----
 5 files changed, 29 insertions(+), 29 deletions(-)

diff --git a/test/persist/test.sh b/test/persist/test.sh
index b27394c5..e24e81de 100644
--- a/test/persist/test.sh
+++ b/test/persist/test.sh
@@ -1,5 +1,5 @@
 #! /bin/bash
 
-cd $GOPATH/src/github.com/tendermint/tendermint
+cd "$GOPATH/src/github.com/tendermint/tendermint"
 
 bash ./test/persist/test_failure_indices.sh
diff --git a/test/persist/test_failure_indices.sh b/test/persist/test_failure_indices.sh
index b2441235..8feb0005 100644
--- a/test/persist/test_failure_indices.sh
+++ b/test/persist/test_failure_indices.sh
@@ -3,7 +3,7 @@
 
 export TMROOT=$HOME/.tendermint_persist
 
-rm -rf $TMROOT
+rm -rf "$TMROOT"
 tendermint init
 
 TM_CMD="tendermint node --log_level=debug" # &> tendermint_${name}.log"
@@ -40,9 +40,9 @@ function start_procs(){
 }
 
 function kill_procs(){
-	kill -9 $PID_DUMMY $PID_TENDERMINT
-	wait $PID_DUMMY
-	wait $PID_TENDERMINT
+	kill -9 "$PID_DUMMY" "$PID_TENDERMINT"
+	wait "$PID_DUMMY"
+	wait "$PID_TENDERMINT"
 }
 
 
@@ -59,10 +59,10 @@ function send_txs(){
 
 	# send a bunch of txs over a few blocks
 	echo "Node is up, sending txs"
-	for i in `seq 1 5`; do
-		for j in `seq 1 100`; do
-			tx=`head -c 8 /dev/urandom | hexdump -ve '1/1 "%.2X"'`
-			curl -s $addr/broadcast_tx_async?tx=0x$tx &> /dev/null
+	for i in $(seq 1 5); do
+		for _ in $(seq 1 100); do
+			tx=$(head -c 8 /dev/urandom | hexdump -ve '1/1 "%.2X"')
+			curl -s "$addr/broadcast_tx_async?tx=0x$tx" &> /dev/null
 		done
 		sleep 1
 	done
@@ -70,33 +70,33 @@ function send_txs(){
 
 
 failsStart=0
-fails=`grep -r "fail.Fail" --include \*.go . | wc -l`
-failsEnd=$(($fails-1))
+fails=$(grep -r "fail.Fail" --include \*.go . | wc -l)
+failsEnd=$((fails-1))
 
-for failIndex in `seq $failsStart $failsEnd`; do
+for failIndex in $(seq $failsStart $failsEnd); do
 	echo ""
 	echo "* Test FailIndex $failIndex"
 	# test failure at failIndex
 
 	send_txs &
-	start_procs 1 $failIndex
+	start_procs 1 "$failIndex"
 
 	# tendermint should fail when it hits the fail index
-	kill -9 $PID_DUMMY
-	wait $PID_DUMMY
+	kill -9 "$PID_DUMMY"
+	wait "$PID_DUMMY"
 
 	start_procs 2
 
 	# wait for node to handshake and make a new block
 	addr="localhost:46657"
-	curl -s $addr/status > /dev/null
+	curl -s "$addr/status" > /dev/null
 	ERR=$?
 	i=0
 	while [ "$ERR" != 0 ]; do
 		sleep 1	
-		curl -s $addr/status > /dev/null
+		curl -s "$addr/status" > /dev/null
 		ERR=$?
-		i=$(($i + 1))
+		i=$((i + 1))
 		if [[ $i == 10 ]]; then
 			echo "Timed out waiting for tendermint to start"
 			exit 1
@@ -104,11 +104,11 @@ for failIndex in `seq $failsStart $failsEnd`; do
 	done
 
 	# wait for a new block
-	h1=`curl -s $addr/status | jq .result[1].latest_block_height`
+	h1=$(curl -s $addr/status | jq .result[1].latest_block_height)
 	h2=$h1
 	while [ "$h2" == "$h1" ]; do
 		sleep 1
-		h2=`curl -s $addr/status | jq .result[1].latest_block_height`
+		h2=$(curl -s $addr/status | jq .result[1].latest_block_height)
 	done
 
 	kill_procs
diff --git a/test/run_test.sh b/test/run_test.sh
index ba4e1b0e..fcc82d98 100644
--- a/test/run_test.sh
+++ b/test/run_test.sh
@@ -1,9 +1,9 @@
 #! /bin/bash
 set -e
 
-echo `pwd`
+pwd
 
-BRANCH=`git rev-parse --abbrev-ref HEAD`
+BRANCH=$(git rev-parse --abbrev-ref HEAD)
 echo "Current branch: $BRANCH"
 
 bash test/test_cover.sh
diff --git a/test/test_cover.sh b/test/test_cover.sh
index 60c84284..b1b754ba 100644
--- a/test/test_cover.sh
+++ b/test/test_cover.sh
@@ -5,7 +5,7 @@ PKGS=$(go list github.com/tendermint/tendermint/... | grep -v /vendor/)
 set -e
 echo "mode: atomic" > coverage.txt
 for pkg in ${PKGS[@]}; do
-	go test -v -timeout 30m -race -coverprofile=profile.out -covermode=atomic $pkg
+	go test -v -timeout 30m -race -coverprofile=profile.out -covermode=atomic "$pkg"
 	if [ -f profile.out ]; then
 		tail -n +2 profile.out >> coverage.txt;
 		rm profile.out
diff --git a/test/test_libs.sh b/test/test_libs.sh
index 28ec0798..4a531bf3 100644
--- a/test/test_libs.sh
+++ b/test/test_libs.sh
@@ -18,30 +18,30 @@ LIBS_MAKE_TEST=(go-rpc go-wire abci)
 for lib in "${LIBS_GO_TEST[@]}"; do
 
 	# checkout vendored version of lib
-	bash scripts/glide/checkout.sh $GLIDE $lib
+	bash scripts/glide/checkout.sh "$GLIDE" "$lib"
 
 	echo "Testing $lib ..."
-	go test -v --race github.com/tendermint/$lib/...
+	go test -v --race "github.com/tendermint/$lib/..."
 	if [[ "$?" != 0 ]]; then
 		echo "FAIL"
 		exit 1
 	fi
 done
 
-DIR=`pwd`
+DIR=$(pwd)
 for lib in "${LIBS_MAKE_TEST[@]}"; do
 
 	# checkout vendored version of lib
-	bash scripts/glide/checkout.sh $GLIDE $lib
+	bash scripts/glide/checkout.sh "$GLIDE" "$lib"
 
 	echo "Testing $lib ..."
-	cd $GOPATH/src/github.com/tendermint/$lib
+	cd "$GOPATH/src/github.com/tendermint/$lib"
 	make test
 	if [[ "$?" != 0 ]]; then
 		echo "FAIL"
 		exit 1
 	fi
-	cd $DIR
+	cd "$DIR"
 done
 
 echo ""

From 07656137780412a2341ec21ed1466647b656944a Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Mon, 20 Feb 2017 19:52:36 -0500
Subject: [PATCH 060/132] move handshake to consensus package

---
 consensus/replay.go      | 205 +++++++++++++++++++++++++++++++++++++++
 consensus/replay_file.go |   2 +-
 consensus/replay_test.go |   2 +-
 consensus/state.go       |  29 ------
 node/node.go             |   2 +-
 state/errors.go          |  32 +++---
 state/execution.go       | 181 +---------------------------------
 7 files changed, 228 insertions(+), 225 deletions(-)

diff --git a/consensus/replay.go b/consensus/replay.go
index d534827b..2ab84dc2 100644
--- a/consensus/replay.go
+++ b/consensus/replay.go
@@ -1,6 +1,7 @@
 package consensus
 
 import (
+	"bytes"
 	"errors"
 	"fmt"
 	"io"
@@ -9,13 +10,26 @@ import (
 	"strings"
 	"time"
 
+	abci "github.com/tendermint/abci/types"
 	auto "github.com/tendermint/go-autofile"
 	. "github.com/tendermint/go-common"
+	cfg "github.com/tendermint/go-config"
 	"github.com/tendermint/go-wire"
 
+	"github.com/tendermint/tendermint/proxy"
+	sm "github.com/tendermint/tendermint/state"
 	"github.com/tendermint/tendermint/types"
 )
 
+// Functionality to replay blocks and messages on recovery from a crash.
+// There are two general failure scenarios: failure during consensus, and failure while applying the block.
+// The former is handled by the WAL, the latter by the proxyApp Handshake on restart,
+// which ultimately hands off the work to the WAL.
+
+//-----------------------------------------
+// recover from failure during consensus
+// by replaying messages from the WAL
+
 // Unmarshal and apply a single message to the consensus state
 // as if it were received in receiveRoutine
 // Lines that start with "#" are ignored.
@@ -154,3 +168,194 @@ func makeHeightSearchFunc(height int) auto.SearchFunc {
 		}
 	}
 }
+
+//----------------------------------------------
+// Recover from failure during block processing
+// by handshaking with the app to figure out where
+// we were last and using the WAL to recover there
+
+// Replay the last block through the consensus and return the AppHash from after Commit.
+func replayLastBlock(config cfg.Config, state *sm.State, proxyApp proxy.AppConnConsensus, blockStore sm.BlockStore) ([]byte, error) {
+	mempool := sm.MockMempool{}
+	cs := NewConsensusState(config, state, proxyApp, blockStore, mempool)
+
+	evsw := types.NewEventSwitch()
+	evsw.Start()
+	defer evsw.Stop()
+	cs.SetEventSwitch(evsw)
+	newBlockCh := subscribeToEvent(evsw, "consensus-replay", types.EventStringNewBlock(), 1)
+
+	// run through the WAL, commit new block, stop
+	cs.Start()
+	<-newBlockCh // TODO: use a timeout and return err?
+	cs.Stop()
+
+	return cs.state.AppHash, nil
+}
+
+type Handshaker struct {
+	config cfg.Config
+	state  *sm.State
+	store  sm.BlockStore
+
+	nBlocks int // number of blocks applied to the state
+}
+
+func NewHandshaker(config cfg.Config, state *sm.State, store sm.BlockStore) *Handshaker {
+	return &Handshaker{config, state, store, 0}
+}
+
+func (h *Handshaker) NBlocks() int {
+	return h.nBlocks
+}
+
+// TODO: retry the handshake/replay if it fails ?
+func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error {
+	// handshake is done via info request on the query conn
+	res, err := proxyApp.Query().InfoSync()
+	if err != nil {
+		return errors.New(Fmt("Error calling Info: %v", err))
+	}
+
+	blockHeight := int(res.LastBlockHeight) // XXX: beware overflow
+	appHash := res.LastBlockAppHash
+
+	log.Notice("ABCI Handshake", "appHeight", blockHeight, "appHash", appHash)
+
+	// TODO: check version
+
+	// replay blocks up to the latest in the blockstore
+	_, err = h.ReplayBlocks(appHash, blockHeight, proxyApp)
+	if err != nil {
+		return errors.New(Fmt("Error on replay: %v", err))
+	}
+
+	// TODO: (on restart) replay mempool
+
+	return nil
+}
+
+// Replay all blocks since appBlockHeight and ensure the result matches the current state.
+// Returns the final AppHash or an error
+func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, proxyApp proxy.AppConns) ([]byte, error) {
+
+	storeBlockHeight := h.store.Height()
+	stateBlockHeight := h.state.LastBlockHeight
+	log.Notice("ABCI Replay Blocks", "appHeight", appBlockHeight, "storeHeight", storeBlockHeight, "stateHeight", stateBlockHeight)
+
+	// First handle edge cases and constraints on the storeBlockHeight
+	if storeBlockHeight == 0 {
+		return appHash, h.checkAppHash(appHash)
+
+	} else if storeBlockHeight < appBlockHeight {
+		// the app should never be ahead of the store (but this is under app's control)
+		return appHash, sm.ErrAppBlockHeightTooHigh{storeBlockHeight, appBlockHeight}
+
+	} else if storeBlockHeight < stateBlockHeight {
+		// the state should never be ahead of the store (this is under tendermint's control)
+		PanicSanity(Fmt("StateBlockHeight (%d) > StoreBlockHeight (%d)", stateBlockHeight, storeBlockHeight))
+
+	} else if storeBlockHeight > stateBlockHeight+1 {
+		// store should be at most one ahead of the state (this is under tendermint's control)
+		PanicSanity(Fmt("StoreBlockHeight (%d) > StateBlockHeight + 1 (%d)", storeBlockHeight, stateBlockHeight+1))
+	}
+
+	// Now either store is equal to state, or one ahead.
+	// For each, consider all cases of where the app could be, given app <= store
+	if storeBlockHeight == stateBlockHeight {
+		// Tendermint ran Commit and saved the state.
+		// Either the app is asking for replay, or we're all synced up.
+		if appBlockHeight < storeBlockHeight {
+			// the app is behind, so replay blocks, but no need to go through WAL (state is already synced to store)
+			return h.replayBlocks(proxyApp, appBlockHeight, storeBlockHeight, false)
+
+		} else if appBlockHeight == storeBlockHeight {
+			// we're good!
+			return appHash, h.checkAppHash(appHash)
+		}
+
+	} else if storeBlockHeight == stateBlockHeight+1 {
+		// We saved the block in the store but haven't updated the state,
+		// so we'll need to replay a block using the WAL.
+		if appBlockHeight < stateBlockHeight {
+			// the app is further behind than it should be, so replay blocks
+			// but leave the last block to go through the WAL
+			return h.replayBlocks(proxyApp, appBlockHeight, storeBlockHeight, true)
+
+		} else if appBlockHeight == stateBlockHeight {
+			// We haven't run Commit (both the state and app are one block behind),
+			// so run through consensus with the real app
+			log.Info("Replay last block using real app")
+			return replayLastBlock(h.config, h.state, proxyApp.Consensus(), h.store)
+
+		} else if appBlockHeight == storeBlockHeight {
+			// We ran Commit, but didn't save the state, so run through consensus with mock app
+			mockApp := newMockProxyApp(appHash)
+			log.Info("Replay last block using mock app")
+			return replayLastBlock(h.config, h.state, mockApp, h.store)
+		}
+
+	}
+
+	PanicSanity("Should never happen")
+	return nil, nil
+}
+
+func (h *Handshaker) replayBlocks(proxyApp proxy.AppConns, appBlockHeight, storeBlockHeight int, useReplayFunc bool) ([]byte, error) {
+	// App is further behind than it should be, so we need to replay blocks.
+	// We replay all blocks from appBlockHeight+1 to storeBlockHeight-1,
+	// and let the final block be replayed through ReplayBlocks.
+	// Note that we don't have an old version of the state,
+	// so we by-pass state validation using applyBlock here.
+
+	var appHash []byte
+	var err error
+	finalBlock := storeBlockHeight
+	if useReplayFunc {
+		finalBlock -= 1
+	}
+	for i := appBlockHeight + 1; i <= finalBlock; i++ {
+		log.Info("Applying block", "height", i)
+		h.nBlocks += 1
+		block := h.store.LoadBlock(i)
+		appHash, err = sm.ApplyBlock(proxyApp.Consensus(), block)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if useReplayFunc {
+		// sync the final block
+		appHash, err = h.ReplayBlocks(appHash, finalBlock, proxyApp)
+		if err != nil {
+			return appHash, err
+		}
+	}
+
+	return appHash, h.checkAppHash(appHash)
+}
+
+func (h *Handshaker) checkAppHash(appHash []byte) error {
+	if !bytes.Equal(h.state.AppHash, appHash) {
+		return errors.New(Fmt("Tendermint state.AppHash does not match AppHash after replay. Got %X, expected %X", appHash, h.state.AppHash))
+	}
+	return nil
+}
+
+//--------------------------------------------------------------------------------
+
+func newMockProxyApp(appHash []byte) proxy.AppConnConsensus {
+	clientCreator := proxy.NewLocalClientCreator(&mockProxyApp{appHash: appHash})
+	cli, _ := clientCreator.NewABCIClient()
+	return proxy.NewAppConnConsensus(cli)
+}
+
+type mockProxyApp struct {
+	abci.BaseApplication
+
+	appHash []byte
+}
+
+func (mock *mockProxyApp) Commit() abci.Result {
+	return abci.NewResultOK(mock.appHash, "")
+}
diff --git a/consensus/replay_file.go b/consensus/replay_file.go
index 5d674785..6ff38088 100644
--- a/consensus/replay_file.go
+++ b/consensus/replay_file.go
@@ -248,7 +248,7 @@ func newConsensusStateForReplay(config cfg.Config) *ConsensusState {
 	state := sm.MakeGenesisStateFromFile(stateDB, config.GetString("genesis_file"))
 
 	// Create proxyAppConn connection (consensus, mempool, query)
-	proxyApp := proxy.NewAppConns(config, proxy.DefaultClientCreator(config), sm.NewHandshaker(config, state, blockStore, ReplayLastBlock))
+	proxyApp := proxy.NewAppConns(config, proxy.DefaultClientCreator(config), NewHandshaker(config, state, blockStore))
 	_, err := proxyApp.Start()
 	if err != nil {
 		Exit(Fmt("Error starting proxy app conns: %v", err))
diff --git a/consensus/replay_test.go b/consensus/replay_test.go
index e8aa9e90..3016f104 100644
--- a/consensus/replay_test.go
+++ b/consensus/replay_test.go
@@ -346,7 +346,7 @@ func testHandshakeReplay(t *testing.T, nBlocks int) {
 	}
 
 	// now start the app using the handshake - it should sync
-	handshaker := sm.NewHandshaker(config, state, store, ReplayLastBlock)
+	handshaker := NewHandshaker(config, state, store)
 	proxyApp = proxy.NewAppConns(config, clientCreator2, handshaker)
 	if _, err := proxyApp.Start(); err != nil {
 		t.Fatalf("Error starting proxy app connections: %v", err)
diff --git a/consensus/state.go b/consensus/state.go
index 8e721ed3..dc1324e3 100644
--- a/consensus/state.go
+++ b/consensus/state.go
@@ -253,25 +253,6 @@ type ConsensusState struct {
 	done chan struct{}
 }
 
-// Replay the last block through the consensus and return the AppHash from after Commit.
-func ReplayLastBlock(config cfg.Config, state *sm.State, proxyApp proxy.AppConnConsensus, blockStore sm.BlockStore) ([]byte, error) {
-	mempool := sm.MockMempool{}
-	cs := NewConsensusState(config, state, proxyApp, blockStore, mempool)
-
-	evsw := types.NewEventSwitch()
-	evsw.Start()
-	defer evsw.Stop()
-	cs.SetEventSwitch(evsw)
-	newBlockCh := subscribeToEvent(evsw, "consensus-replay", types.EventStringNewBlock(), 1)
-
-	// run through the WAL, commit new block, stop
-	cs.Start()
-	<-newBlockCh // TODO: use a timeout and return err?
-	cs.Stop()
-
-	return cs.state.AppHash, nil
-}
-
 func NewConsensusState(config cfg.Config, state *sm.State, proxyAppConn proxy.AppConnConsensus, blockStore sm.BlockStore, mempool sm.Mempool) *ConsensusState {
 	cs := &ConsensusState{
 		config:           config,
@@ -624,11 +605,6 @@ func (cs *ConsensusState) newStep() {
 //-----------------------------------------
 // the main go routines
 
-// a nice idea but probably more trouble than its worth
-func (cs *ConsensusState) stopTimer() {
-	cs.timeoutTicker.Stop()
-}
-
 // receiveRoutine handles messages which may cause state transitions.
 // it's argument (n) is the number of messages to process before exiting - use 0 to run forever
 // It keeps the RoundState and is the only thing that updates it.
@@ -767,7 +743,6 @@ func (cs *ConsensusState) enterNewRound(height int, round int) {
 	if now := time.Now(); cs.StartTime.After(now) {
 		log.Warn("Need to set a buffer and log.Warn() here for sanity.", "startTime", cs.StartTime, "now", now)
 	}
-	// cs.stopTimer()
 
 	log.Notice(Fmt("enterNewRound(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
 
@@ -947,8 +922,6 @@ func (cs *ConsensusState) enterPrevote(height int, round int) {
 		// TODO: catchup event?
 	}
 
-	// cs.stopTimer()
-
 	log.Info(Fmt("enterPrevote(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
 
 	// Sign and broadcast vote as necessary
@@ -1022,8 +995,6 @@ func (cs *ConsensusState) enterPrecommit(height int, round int) {
 		return
 	}
 
-	// cs.stopTimer()
-
 	log.Info(Fmt("enterPrecommit(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
 
 	defer func() {
diff --git a/node/node.go b/node/node.go
index 755658b7..deaa9e16 100644
--- a/node/node.go
+++ b/node/node.go
@@ -69,7 +69,7 @@ func NewNode(config cfg.Config, privValidator *types.PrivValidator, clientCreato
 
 	// Create the proxyApp, which manages connections (consensus, mempool, query)
 	// and sync tendermint and the app by replaying any necessary blocks
-	proxyApp := proxy.NewAppConns(config, clientCreator, sm.NewHandshaker(config, state, blockStore, consensus.ReplayLastBlock))
+	proxyApp := proxy.NewAppConns(config, clientCreator, consensus.NewHandshaker(config, state, blockStore))
 	if _, err := proxyApp.Start(); err != nil {
 		cmn.Exit(cmn.Fmt("Error starting proxy app connections: %v", err))
 	}
diff --git a/state/errors.go b/state/errors.go
index 0d0eae14..32a9351c 100644
--- a/state/errors.go
+++ b/state/errors.go
@@ -9,47 +9,47 @@ type (
 	ErrProxyAppConn error
 
 	ErrUnknownBlock struct {
-		height int
+		Height int
 	}
 
 	ErrBlockHashMismatch struct {
-		coreHash []byte
-		appHash  []byte
-		height   int
+		CoreHash []byte
+		AppHash  []byte
+		Height   int
 	}
 
 	ErrAppBlockHeightTooHigh struct {
-		coreHeight int
-		appHeight  int
+		CoreHeight int
+		AppHeight  int
 	}
 
 	ErrLastStateMismatch struct {
-		height int
-		core   []byte
-		app    []byte
+		Height int
+		Core   []byte
+		App    []byte
 	}
 
 	ErrStateMismatch struct {
-		got      *State
-		expected *State
+		Got      *State
+		Expected *State
 	}
 )
 
 func (e ErrUnknownBlock) Error() string {
-	return Fmt("Could not find block #%d", e.height)
+	return Fmt("Could not find block #%d", e.Height)
 }
 
 func (e ErrBlockHashMismatch) Error() string {
-	return Fmt("App block hash (%X) does not match core block hash (%X) for height %d", e.appHash, e.coreHash, e.height)
+	return Fmt("App block hash (%X) does not match core block hash (%X) for height %d", e.AppHash, e.CoreHash, e.Height)
 }
 
 func (e ErrAppBlockHeightTooHigh) Error() string {
-	return Fmt("App block height (%d) is higher than core (%d)", e.appHeight, e.coreHeight)
+	return Fmt("App block height (%d) is higher than core (%d)", e.AppHeight, e.CoreHeight)
 }
 func (e ErrLastStateMismatch) Error() string {
-	return Fmt("Latest tendermint block (%d) LastAppHash (%X) does not match app's AppHash (%X)", e.height, e.core, e.app)
+	return Fmt("Latest tendermint block (%d) LastAppHash (%X) does not match app's AppHash (%X)", e.Height, e.Core, e.App)
 }
 
 func (e ErrStateMismatch) Error() string {
-	return Fmt("State after replay does not match saved state. Got ----\n%v\nExpected ----\n%v\n", e.got, e.expected)
+	return Fmt("State after replay does not match saved state. Got ----\n%v\nExpected ----\n%v\n", e.Got, e.Expected)
 }
diff --git a/state/execution.go b/state/execution.go
index c1471bf8..3e7ad91b 100644
--- a/state/execution.go
+++ b/state/execution.go
@@ -1,14 +1,12 @@
 package state
 
 import (
-	"bytes"
 	"errors"
 
 	"github.com/ebuchman/fail-test"
 
 	abci "github.com/tendermint/abci/types"
 	. "github.com/tendermint/go-common"
-	cfg "github.com/tendermint/go-config"
 	"github.com/tendermint/go-crypto"
 	"github.com/tendermint/tendermint/proxy"
 	"github.com/tendermint/tendermint/types"
@@ -272,7 +270,7 @@ func (s *State) CommitStateUpdateMempool(proxyAppConn proxy.AppConnConsensus, bl
 
 // Apply and commit a block, but without all the state validation.
 // Returns the application root hash (result of abci.Commit)
-func applyBlock(appConnConsensus proxy.AppConnConsensus, block *types.Block) ([]byte, error) {
+func ApplyBlock(appConnConsensus proxy.AppConnConsensus, block *types.Block) ([]byte, error) {
 	var eventCache types.Fireable // nil
 	_, err := execBlockOnProxyApp(eventCache, appConnConsensus, block)
 	if err != nil {
@@ -291,6 +289,9 @@ func applyBlock(appConnConsensus proxy.AppConnConsensus, block *types.Block) ([]
 	return res.Data, nil
 }
 
+//------------------------------------------------------
+// blockchain services types
+
 // Updates to the mempool need to be synchronized with committing a block
 // so apps can reset their transient state on Commit
 type Mempool interface {
@@ -311,9 +312,6 @@ func (m MockMempool) CheckTx(tx types.Tx, cb func(*abci.Response)) error { retur
 func (m MockMempool) Reap(n int) types.Txs                               { return types.Txs{} }
 func (m MockMempool) Update(height int, txs types.Txs)                   {}
 
-//----------------------------------------------------------------
-// Handshake with app to sync to latest state of core by replaying blocks
-
 // TODO: Should we move blockchain/store.go to its own package?
 type BlockStore interface {
 	Height() int
@@ -327,174 +325,3 @@ type BlockStore interface {
 	LoadBlockCommit(height int) *types.Commit
 	LoadSeenCommit(height int) *types.Commit
 }
-
-// returns the apphash from Commit
-type blockReplayFunc func(cfg.Config, *State, proxy.AppConnConsensus, BlockStore) ([]byte, error)
-
-type Handshaker struct {
-	config          cfg.Config
-	state           *State
-	store           BlockStore
-	replayLastBlock blockReplayFunc
-
-	nBlocks int // number of blocks applied to the state
-}
-
-func NewHandshaker(config cfg.Config, state *State, store BlockStore, f blockReplayFunc) *Handshaker {
-	return &Handshaker{config, state, store, f, 0}
-}
-
-func (h *Handshaker) NBlocks() int {
-	return h.nBlocks
-}
-
-// TODO: retry the handshake/replay if it fails ?
-func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error {
-	// handshake is done via info request on the query conn
-	res, err := proxyApp.Query().InfoSync()
-	if err != nil {
-		return errors.New(Fmt("Error calling Info: %v", err))
-	}
-
-	blockHeight := int(res.LastBlockHeight) // XXX: beware overflow
-	appHash := res.LastBlockAppHash
-
-	log.Notice("ABCI Handshake", "appHeight", blockHeight, "appHash", appHash)
-
-	// TODO: check version
-
-	// replay blocks up to the latest in the blockstore
-	_, err = h.ReplayBlocks(appHash, blockHeight, proxyApp)
-	if err != nil {
-		return errors.New(Fmt("Error on replay: %v", err))
-	}
-
-	// TODO: (on restart) replay mempool
-
-	return nil
-}
-
-// Replay all blocks since appBlockHeight and ensure the result matches the current state.
-// Returns the final AppHash or an error
-func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, proxyApp proxy.AppConns) ([]byte, error) {
-
-	storeBlockHeight := h.store.Height()
-	stateBlockHeight := h.state.LastBlockHeight
-	log.Notice("ABCI Replay Blocks", "appHeight", appBlockHeight, "storeHeight", storeBlockHeight, "stateHeight", stateBlockHeight)
-
-	// First handle edge cases and constraints on the storeBlockHeight
-	if storeBlockHeight == 0 {
-		return appHash, h.checkAppHash(appHash)
-
-	} else if storeBlockHeight < appBlockHeight {
-		// the app should never be ahead of the store (but this is under app's control)
-		return appHash, ErrAppBlockHeightTooHigh{storeBlockHeight, appBlockHeight}
-
-	} else if storeBlockHeight < stateBlockHeight {
-		// the state should never be ahead of the store (this is under tendermint's control)
-		PanicSanity(Fmt("StateBlockHeight (%d) > StoreBlockHeight (%d)", stateBlockHeight, storeBlockHeight))
-
-	} else if storeBlockHeight > stateBlockHeight+1 {
-		// store should be at most one ahead of the state (this is under tendermint's control)
-		PanicSanity(Fmt("StoreBlockHeight (%d) > StateBlockHeight + 1 (%d)", storeBlockHeight, stateBlockHeight+1))
-	}
-
-	// Now either store is equal to state, or one ahead.
-	// For each, consider all cases of where the app could be, given app <= store
-	if storeBlockHeight == stateBlockHeight {
-		// Tendermint ran Commit and saved the state.
-		// Either the app is asking for replay, or we're all synced up.
-		if appBlockHeight < storeBlockHeight {
-			// the app is behind, so replay blocks, but no need to go through WAL (state is already synced to store)
-			return h.replayBlocks(proxyApp, appBlockHeight, storeBlockHeight, false)
-
-		} else if appBlockHeight == storeBlockHeight {
-			// we're good!
-			return appHash, h.checkAppHash(appHash)
-		}
-
-	} else if storeBlockHeight == stateBlockHeight+1 {
-		// We saved the block in the store but haven't updated the state,
-		// so we'll need to replay a block using the WAL.
-		if appBlockHeight < stateBlockHeight {
-			// the app is further behind than it should be, so replay blocks
-			// but leave the last block to go through the WAL
-			return h.replayBlocks(proxyApp, appBlockHeight, storeBlockHeight, true)
-
-		} else if appBlockHeight == stateBlockHeight {
-			// We haven't run Commit (both the state and app are one block behind),
-			// so run through consensus with the real app
-			log.Info("Replay last block using real app")
-			return h.replayLastBlock(h.config, h.state, proxyApp.Consensus(), h.store)
-
-		} else if appBlockHeight == storeBlockHeight {
-			// We ran Commit, but didn't save the state, so run through consensus with mock app
-			mockApp := newMockProxyApp(appHash)
-			log.Info("Replay last block using mock app")
-			return h.replayLastBlock(h.config, h.state, mockApp, h.store)
-		}
-
-	}
-
-	PanicSanity("Should never happen")
-	return nil, nil
-}
-
-func (h *Handshaker) replayBlocks(proxyApp proxy.AppConns, appBlockHeight, storeBlockHeight int, useReplayFunc bool) ([]byte, error) {
-	// App is further behind than it should be, so we need to replay blocks.
-	// We replay all blocks from appBlockHeight+1 to storeBlockHeight-1,
-	// and let the final block be replayed through ReplayBlocks.
-	// Note that we don't have an old version of the state,
-	// so we by-pass state validation using applyBlock here.
-
-	var appHash []byte
-	var err error
-	finalBlock := storeBlockHeight
-	if useReplayFunc {
-		finalBlock -= 1
-	}
-	for i := appBlockHeight + 1; i <= finalBlock; i++ {
-		log.Info("Applying block", "height", i)
-		h.nBlocks += 1
-		block := h.store.LoadBlock(i)
-		appHash, err = applyBlock(proxyApp.Consensus(), block)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	if useReplayFunc {
-		// sync the final block
-		appHash, err = h.ReplayBlocks(appHash, finalBlock, proxyApp)
-		if err != nil {
-			return appHash, err
-		}
-	}
-
-	return appHash, h.checkAppHash(appHash)
-}
-
-func (h *Handshaker) checkAppHash(appHash []byte) error {
-	if !bytes.Equal(h.state.AppHash, appHash) {
-		return errors.New(Fmt("Tendermint state.AppHash does not match AppHash after replay. Got %X, expected %X", appHash, h.state.AppHash))
-	}
-	return nil
-}
-
-//--------------------------------------------------------------------------------
-
-func newMockProxyApp(appHash []byte) proxy.AppConnConsensus {
-	clientCreator := proxy.NewLocalClientCreator(&mockProxyApp{appHash: appHash})
-	cli, _ := clientCreator.NewABCIClient()
-	return proxy.NewAppConnConsensus(cli)
-}
-
-type mockProxyApp struct {
-	abci.BaseApplication
-
-	appHash []byte
-}
-
-func (mock *mockProxyApp) Commit() abci.Result {
-	return abci.NewResultOK(mock.appHash, "")
-}

From f9df4294f34eccbdd1bea8c524f1b48f8b7d7cb4 Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Mon, 20 Feb 2017 20:09:15 -0500
Subject: [PATCH 061/132] move some interfaces to types/services.go

---
 blockchain/reactor.go    |  2 +-
 consensus/replay.go      |  8 +++---
 consensus/replay_test.go |  2 +-
 consensus/state.go       |  6 ++---
 rpc/core/pipe.go         | 39 +++++++++-------------------
 state/execution.go       | 41 ++---------------------------
 types/services.go        | 56 ++++++++++++++++++++++++++++++++++++++++
 7 files changed, 79 insertions(+), 75 deletions(-)
 create mode 100644 types/services.go

diff --git a/blockchain/reactor.go b/blockchain/reactor.go
index bfa671d0..4a7f21d0 100644
--- a/blockchain/reactor.go
+++ b/blockchain/reactor.go
@@ -242,7 +242,7 @@ FOR_LOOP:
 					// NOTE: we could improve performance if we
 					// didn't make the app commit to disk every block
 					// ... but we would need a way to get the hash without it persisting
-					err := bcR.state.ApplyBlock(bcR.evsw, bcR.proxyAppConn, first, firstPartsHeader, sm.MockMempool{})
+					err := bcR.state.ApplyBlock(bcR.evsw, bcR.proxyAppConn, first, firstPartsHeader, types.MockMempool{})
 					if err != nil {
 						// TODO This is bad, are we zombie?
 						PanicQ(Fmt("Failed to process committed block (%d:%X): %v", first.Height, first.Hash(), err))
diff --git a/consensus/replay.go b/consensus/replay.go
index 2ab84dc2..07308bf1 100644
--- a/consensus/replay.go
+++ b/consensus/replay.go
@@ -175,8 +175,8 @@ func makeHeightSearchFunc(height int) auto.SearchFunc {
 // we were last and using the WAL to recover there
 
 // Replay the last block through the consensus and return the AppHash from after Commit.
-func replayLastBlock(config cfg.Config, state *sm.State, proxyApp proxy.AppConnConsensus, blockStore sm.BlockStore) ([]byte, error) {
-	mempool := sm.MockMempool{}
+func replayLastBlock(config cfg.Config, state *sm.State, proxyApp proxy.AppConnConsensus, blockStore types.BlockStore) ([]byte, error) {
+	mempool := types.MockMempool{}
 	cs := NewConsensusState(config, state, proxyApp, blockStore, mempool)
 
 	evsw := types.NewEventSwitch()
@@ -196,12 +196,12 @@ func replayLastBlock(config cfg.Config, state *sm.State, proxyApp proxy.AppConnC
 type Handshaker struct {
 	config cfg.Config
 	state  *sm.State
-	store  sm.BlockStore
+	store  types.BlockStore
 
 	nBlocks int // number of blocks applied to the state
 }
 
-func NewHandshaker(config cfg.Config, state *sm.State, store sm.BlockStore) *Handshaker {
+func NewHandshaker(config cfg.Config, state *sm.State, store types.BlockStore) *Handshaker {
 	return &Handshaker{config, state, store, 0}
 }
 
diff --git a/consensus/replay_test.go b/consensus/replay_test.go
index 3016f104..2d812de9 100644
--- a/consensus/replay_test.go
+++ b/consensus/replay_test.go
@@ -254,7 +254,7 @@ func testReplayCrashBeforeWriteVote(t *testing.T, thisCase *testCase, lineNum in
 
 var (
 	NUM_BLOCKS = 6 // number of blocks in the test_data/many_blocks.cswal
-	mempool    = sm.MockMempool{}
+	mempool    = types.MockMempool{}
 
 	testPartSize int
 )
diff --git a/consensus/state.go b/consensus/state.go
index dc1324e3..882d5c87 100644
--- a/consensus/state.go
+++ b/consensus/state.go
@@ -224,8 +224,8 @@ type ConsensusState struct {
 
 	config       cfg.Config
 	proxyAppConn proxy.AppConnConsensus
-	blockStore   sm.BlockStore
-	mempool      sm.Mempool
+	blockStore   types.BlockStore
+	mempool      types.Mempool
 
 	privValidator PrivValidator // for signing votes
 
@@ -253,7 +253,7 @@ type ConsensusState struct {
 	done chan struct{}
 }
 
-func NewConsensusState(config cfg.Config, state *sm.State, proxyAppConn proxy.AppConnConsensus, blockStore sm.BlockStore, mempool sm.Mempool) *ConsensusState {
+func NewConsensusState(config cfg.Config, state *sm.State, proxyAppConn proxy.AppConnConsensus, blockStore types.BlockStore, mempool types.Mempool) *ConsensusState {
 	cs := &ConsensusState{
 		config:           config,
 		proxyAppConn:     proxyAppConn,
diff --git a/rpc/core/pipe.go b/rpc/core/pipe.go
index fb06c3ff..40ef7081 100644
--- a/rpc/core/pipe.go
+++ b/rpc/core/pipe.go
@@ -5,36 +5,19 @@ import (
 	"github.com/tendermint/go-crypto"
 	"github.com/tendermint/go-p2p"
 
-	abci "github.com/tendermint/abci/types"
 	"github.com/tendermint/tendermint/consensus"
 	"github.com/tendermint/tendermint/proxy"
 	"github.com/tendermint/tendermint/types"
 )
 
-//-----------------------------------------------------
-// Interfaces for use by RPC
-// NOTE: these methods must be thread safe!
-
-type BlockStore interface {
-	Height() int
-	LoadBlockMeta(height int) *types.BlockMeta
-	LoadBlock(height int) *types.Block
-	LoadSeenCommit(height int) *types.Commit
-	LoadBlockCommit(height int) *types.Commit
-}
+//----------------------------------------------
+// These interfaces are used by RPC and must be thread safe
 
 type Consensus interface {
 	GetValidators() (int, []*types.Validator)
 	GetRoundState() *consensus.RoundState
 }
 
-type Mempool interface {
-	Size() int
-	CheckTx(types.Tx, func(*abci.Response)) error
-	Reap(int) types.Txs
-	Flush()
-}
-
 type P2P interface {
 	Listeners() []p2p.Listener
 	Peers() p2p.IPeerSet
@@ -44,16 +27,18 @@ type P2P interface {
 	DialSeeds([]string)
 }
 
+//----------------------------------------------
+
 var (
 	// external, thread safe interfaces
 	eventSwitch   types.EventSwitch
 	proxyAppQuery proxy.AppConnQuery
 	config        cfg.Config
 
-	// interfaces defined above
-	blockStore     BlockStore
+	// interfaces defined in types and above
+	blockStore     types.BlockStore
+	mempool        types.Mempool
 	consensusState Consensus
-	mempool        Mempool
 	p2pSwitch      P2P
 
 	// objects
@@ -69,18 +54,18 @@ func SetEventSwitch(evsw types.EventSwitch) {
 	eventSwitch = evsw
 }
 
-func SetBlockStore(bs BlockStore) {
+func SetBlockStore(bs types.BlockStore) {
 	blockStore = bs
 }
 
+func SetMempool(mem types.Mempool) {
+	mempool = mem
+}
+
 func SetConsensusState(cs Consensus) {
 	consensusState = cs
 }
 
-func SetMempool(mem Mempool) {
-	mempool = mem
-}
-
 func SetSwitch(sw P2P) {
 	p2pSwitch = sw
 }
diff --git a/state/execution.go b/state/execution.go
index 3e7ad91b..aa911301 100644
--- a/state/execution.go
+++ b/state/execution.go
@@ -223,7 +223,7 @@ func (s *State) validateBlock(block *types.Block) error {
 
 // Execute and commit block against app, save block and state
 func (s *State) ApplyBlock(eventCache types.Fireable, proxyAppConn proxy.AppConnConsensus,
-	block *types.Block, partsHeader types.PartSetHeader, mempool Mempool) error {
+	block *types.Block, partsHeader types.PartSetHeader, mempool types.Mempool) error {
 
 	// Run the block on the State:
 	// + update validator sets
@@ -244,7 +244,7 @@ func (s *State) ApplyBlock(eventCache types.Fireable, proxyAppConn proxy.AppConn
 // mempool must be locked during commit and update
 // because state is typically reset on Commit and old txs must be replayed
 // against committed state before new txs are run in the mempool, lest they be invalid
-func (s *State) CommitStateUpdateMempool(proxyAppConn proxy.AppConnConsensus, block *types.Block, mempool Mempool) error {
+func (s *State) CommitStateUpdateMempool(proxyAppConn proxy.AppConnConsensus, block *types.Block, mempool types.Mempool) error {
 	mempool.Lock()
 	defer mempool.Unlock()
 
@@ -288,40 +288,3 @@ func ApplyBlock(appConnConsensus proxy.AppConnConsensus, block *types.Block) ([]
 	}
 	return res.Data, nil
 }
-
-//------------------------------------------------------
-// blockchain services types
-
-// Updates to the mempool need to be synchronized with committing a block
-// so apps can reset their transient state on Commit
-type Mempool interface {
-	Lock()
-	Unlock()
-
-	CheckTx(types.Tx, func(*abci.Response)) error
-	Reap(int) types.Txs
-	Update(height int, txs types.Txs)
-}
-
-type MockMempool struct {
-}
-
-func (m MockMempool) Lock()                                              {}
-func (m MockMempool) Unlock()                                            {}
-func (m MockMempool) CheckTx(tx types.Tx, cb func(*abci.Response)) error { return nil }
-func (m MockMempool) Reap(n int) types.Txs                               { return types.Txs{} }
-func (m MockMempool) Update(height int, txs types.Txs)                   {}
-
-// TODO: Should we move blockchain/store.go to its own package?
-type BlockStore interface {
-	Height() int
-
-	LoadBlockMeta(height int) *types.BlockMeta
-	LoadBlock(height int) *types.Block
-	LoadBlockPart(height int, index int) *types.Part
-
-	SaveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit)
-
-	LoadBlockCommit(height int) *types.Commit
-	LoadSeenCommit(height int) *types.Commit
-}
diff --git a/types/services.go b/types/services.go
new file mode 100644
index 00000000..ee20487e
--- /dev/null
+++ b/types/services.go
@@ -0,0 +1,56 @@
+package types
+
+import (
+	abci "github.com/tendermint/abci/types"
+)
+
+//------------------------------------------------------
+// blockchain services types
+// NOTE: Interfaces used by RPC must be thread safe!
+//------------------------------------------------------
+
+//------------------------------------------------------
+// mempool
+
+// Updates to the mempool need to be synchronized with committing a block
+// so apps can reset their transient state on Commit
+type Mempool interface {
+	Lock()
+	Unlock()
+
+	Size() int
+	CheckTx(Tx, func(*abci.Response)) error
+	Reap(int) Txs
+	Update(height int, txs Txs)
+	Flush()
+}
+
+type MockMempool struct {
+}
+
+func (m MockMempool) Lock()                                        {}
+func (m MockMempool) Unlock()                                      {}
+func (m MockMempool) Size() int                                    { return 0 }
+func (m MockMempool) CheckTx(tx Tx, cb func(*abci.Response)) error { return nil }
+func (m MockMempool) Reap(n int) Txs                               { return Txs{} }
+func (m MockMempool) Update(height int, txs Txs)                   {}
+func (m MockMempool) Flush()                                       {}
+
+//------------------------------------------------------
+// blockstore
+
+type BlockStoreRPC interface {
+	Height() int
+
+	LoadBlockMeta(height int) *BlockMeta
+	LoadBlock(height int) *Block
+	LoadBlockPart(height int, index int) *Part
+
+	LoadBlockCommit(height int) *Commit
+	LoadSeenCommit(height int) *Commit
+}
+
+type BlockStore interface {
+	BlockStoreRPC
+	SaveBlock(block *Block, blockParts *PartSet, seenCommit *Commit)
+}

From 0c4b6cd0716c3385cf71ae686e5196a0ee63a27c Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Mon, 20 Feb 2017 21:45:53 -0500
Subject: [PATCH 062/132] consensus: more handshake replay tests

---
 consensus/replay.go      |  55 +++++++--------
 consensus/replay_test.go | 146 +++++++++++++++++++++++++++++----------
 2 files changed, 136 insertions(+), 65 deletions(-)

diff --git a/consensus/replay.go b/consensus/replay.go
index 07308bf1..f117e66d 100644
--- a/consensus/replay.go
+++ b/consensus/replay.go
@@ -174,25 +174,6 @@ func makeHeightSearchFunc(height int) auto.SearchFunc {
 // by handshaking with the app to figure out where
 // we were last and using the WAL to recover there
 
-// Replay the last block through the consensus and return the AppHash from after Commit.
-func replayLastBlock(config cfg.Config, state *sm.State, proxyApp proxy.AppConnConsensus, blockStore types.BlockStore) ([]byte, error) {
-	mempool := types.MockMempool{}
-	cs := NewConsensusState(config, state, proxyApp, blockStore, mempool)
-
-	evsw := types.NewEventSwitch()
-	evsw.Start()
-	defer evsw.Stop()
-	cs.SetEventSwitch(evsw)
-	newBlockCh := subscribeToEvent(evsw, "consensus-replay", types.EventStringNewBlock(), 1)
-
-	// run through the WAL, commit new block, stop
-	cs.Start()
-	<-newBlockCh // TODO: use a timeout and return err?
-	cs.Stop()
-
-	return cs.state.AppHash, nil
-}
-
 type Handshaker struct {
 	config cfg.Config
 	state  *sm.State
@@ -286,13 +267,13 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, proxyApp p
 			// We haven't run Commit (both the state and app are one block behind),
 			// so run through consensus with the real app
 			log.Info("Replay last block using real app")
-			return replayLastBlock(h.config, h.state, proxyApp.Consensus(), h.store)
+			return h.replayLastBlock(proxyApp.Consensus())
 
 		} else if appBlockHeight == storeBlockHeight {
 			// We ran Commit, but didn't save the state, so run through consensus with mock app
 			mockApp := newMockProxyApp(appHash)
 			log.Info("Replay last block using mock app")
-			return replayLastBlock(h.config, h.state, mockApp, h.store)
+			return h.replayLastBlock(mockApp)
 		}
 
 	}
@@ -316,28 +297,48 @@ func (h *Handshaker) replayBlocks(proxyApp proxy.AppConns, appBlockHeight, store
 	}
 	for i := appBlockHeight + 1; i <= finalBlock; i++ {
 		log.Info("Applying block", "height", i)
-		h.nBlocks += 1
 		block := h.store.LoadBlock(i)
 		appHash, err = sm.ApplyBlock(proxyApp.Consensus(), block)
 		if err != nil {
 			return nil, err
 		}
+
+		h.nBlocks += 1
 	}
 
 	if useReplayFunc {
 		// sync the final block
-		appHash, err = h.ReplayBlocks(appHash, finalBlock, proxyApp)
-		if err != nil {
-			return appHash, err
-		}
+		return h.ReplayBlocks(appHash, finalBlock, proxyApp)
 	}
 
 	return appHash, h.checkAppHash(appHash)
 }
 
+// Replay the last block through the consensus and return the AppHash from after Commit.
+func (h *Handshaker) replayLastBlock(proxyApp proxy.AppConnConsensus) ([]byte, error) {
+	mempool := types.MockMempool{}
+	cs := NewConsensusState(h.config, h.state, proxyApp, h.store, mempool)
+
+	evsw := types.NewEventSwitch()
+	evsw.Start()
+	defer evsw.Stop()
+	cs.SetEventSwitch(evsw)
+	newBlockCh := subscribeToEvent(evsw, "consensus-replay", types.EventStringNewBlock(), 1)
+
+	// run through the WAL, commit new block, stop
+	cs.Start()
+	<-newBlockCh // TODO: use a timeout and return err?
+	cs.Stop()
+
+	h.nBlocks += 1
+
+	return cs.state.AppHash, nil
+}
+
 func (h *Handshaker) checkAppHash(appHash []byte) error {
 	if !bytes.Equal(h.state.AppHash, appHash) {
-		return errors.New(Fmt("Tendermint state.AppHash does not match AppHash after replay. Got %X, expected %X", appHash, h.state.AppHash))
+		panic(errors.New(Fmt("Tendermint state.AppHash does not match AppHash after replay. Got %X, expected %X", appHash, h.state.AppHash)).Error())
+		return nil
 	}
 	return nil
 }
diff --git a/consensus/replay_test.go b/consensus/replay_test.go
index 2d812de9..c70b60fa 100644
--- a/consensus/replay_test.go
+++ b/consensus/replay_test.go
@@ -262,28 +262,41 @@ var (
 //---------------------------------------
 // Test handshake/replay
 
+// 0 - all synced up
+// 1 - saved block but app and state are behind
+// 2 - save block and committed but state is behind
+var modes = []uint{0, 1, 2}
+
 // Sync from scratch
 func TestHandshakeReplayAll(t *testing.T) {
-	testHandshakeReplay(t, 0)
+	for _, m := range modes {
+		testHandshakeReplay(t, 0, m)
+	}
 }
 
 // Sync many, not from scratch
 func TestHandshakeReplaySome(t *testing.T) {
-	testHandshakeReplay(t, 1)
+	for _, m := range modes {
+		testHandshakeReplay(t, 1, m)
+	}
 }
 
 // Sync from lagging by one
 func TestHandshakeReplayOne(t *testing.T) {
-	testHandshakeReplay(t, NUM_BLOCKS-1)
+	for _, m := range modes {
+		testHandshakeReplay(t, NUM_BLOCKS-1, m)
+	}
 }
 
 // Sync from caught up
 func TestHandshakeReplayNone(t *testing.T) {
-	testHandshakeReplay(t, NUM_BLOCKS)
+	for _, m := range modes {
+		testHandshakeReplay(t, NUM_BLOCKS, m)
+	}
 }
 
 // Make some blocks. Start a fresh app and apply nBlocks blocks. Then restart the app and sync it up with the remaining blocks
-func testHandshakeReplay(t *testing.T, nBlocks int) {
+func testHandshakeReplay(t *testing.T, nBlocks int, mode uint) {
 	config := tendermint_test.ResetConfig("proxy_test_")
 
 	// copy the many_blocks file
@@ -310,44 +323,23 @@ func testHandshakeReplay(t *testing.T, nBlocks int) {
 	store.chain = chain
 	store.commits = commits
 
-	// run the whole chain against this client to build up the tendermint state
-	clientCreator := proxy.NewLocalClientCreator(dummy.NewPersistentDummyApplication(path.Join(config.GetString("db_dir"), "1")))
-	proxyApp := proxy.NewAppConns(config, clientCreator, nil) // sm.NewHandshaker(config, state, store, ReplayLastBlock))
-	if _, err := proxyApp.Start(); err != nil {
-		t.Fatalf("Error starting proxy app connections: %v", err)
-	}
-	for _, block := range chain {
-		err := state.ApplyBlock(nil, proxyApp.Consensus(), block, block.MakePartSet(testPartSize).Header(), mempool)
-		if err != nil {
-			t.Fatal(err)
-		}
-	}
-	proxyApp.Stop()
-	latestAppHash := state.AppHash
+	// run the chain through state.ApplyBlock to build up the tendermint state
+	latestAppHash := buildTMStateFromChain(config, state, chain, mode)
 
-	// run nBlocks against a new client to build up the app state.
-	// use a throwaway tendermint state
-	clientCreator2 := proxy.NewLocalClientCreator(dummy.NewPersistentDummyApplication(path.Join(config.GetString("db_dir"), "2")))
+	// make a new client creator
+	dummyApp := dummy.NewPersistentDummyApplication(path.Join(config.GetString("db_dir"), "2"))
+	clientCreator2 := proxy.NewLocalClientCreator(dummyApp)
 	if nBlocks > 0 {
-		// start a new app without handshake, play nBlocks blocks
+		// run nBlocks against a new client to build up the app state.
+		// use a throwaway tendermint state
 		proxyApp := proxy.NewAppConns(config, clientCreator2, nil)
-		if _, err := proxyApp.Start(); err != nil {
-			t.Fatalf("Error starting proxy app connections: %v", err)
-		}
-		state2, _ := stateAndStore(config, privVal.PubKey)
-		for i := 0; i < nBlocks; i++ {
-			block := chain[i]
-			err := state2.ApplyBlock(nil, proxyApp.Consensus(), block, block.MakePartSet(testPartSize).Header(), mempool)
-			if err != nil {
-				t.Fatal(err)
-			}
-		}
-		proxyApp.Stop()
+		state, _ := stateAndStore(config, privVal.PubKey)
+		buildAppStateFromChain(proxyApp, state, chain, nBlocks, mode)
 	}
 
 	// now start the app using the handshake - it should sync
 	handshaker := NewHandshaker(config, state, store)
-	proxyApp = proxy.NewAppConns(config, clientCreator2, handshaker)
+	proxyApp := proxy.NewAppConns(config, clientCreator2, handshaker)
 	if _, err := proxyApp.Start(); err != nil {
 		t.Fatalf("Error starting proxy app connections: %v", err)
 	}
@@ -363,9 +355,87 @@ func testHandshakeReplay(t *testing.T, nBlocks int) {
 		t.Fatalf("Expected app hashes to match after handshake/replay. got %X, expected %X", res.LastBlockAppHash, latestAppHash)
 	}
 
-	if handshaker.NBlocks() != NUM_BLOCKS-nBlocks {
-		t.Fatalf("Expected handshake to sync %d blocks, got %d", NUM_BLOCKS-nBlocks, handshaker.NBlocks())
+	expectedBlocksToSync := NUM_BLOCKS - nBlocks
+	if nBlocks == NUM_BLOCKS && mode > 0 {
+		expectedBlocksToSync += 1
+	} else if nBlocks > 0 && mode == 1 {
+		expectedBlocksToSync += 1
 	}
+
+	if handshaker.NBlocks() != expectedBlocksToSync {
+		t.Fatalf("Expected handshake to sync %d blocks, got %d", expectedBlocksToSync, handshaker.NBlocks())
+	}
+}
+
+func applyBlock(st *sm.State, blk *types.Block, proxyApp proxy.AppConns) {
+	err := st.ApplyBlock(nil, proxyApp.Consensus(), blk, blk.MakePartSet(testPartSize).Header(), mempool)
+	if err != nil {
+		panic(err)
+	}
+}
+
+func buildAppStateFromChain(proxyApp proxy.AppConns,
+	state *sm.State, chain []*types.Block, nBlocks int, mode uint) {
+	// start a new app without handshake, play nBlocks blocks
+	if _, err := proxyApp.Start(); err != nil {
+		panic(err)
+	}
+	defer proxyApp.Stop()
+	switch mode {
+	case 0:
+		for i := 0; i < nBlocks; i++ {
+			block := chain[i]
+			applyBlock(state, block, proxyApp)
+		}
+	case 1, 2:
+		for i := 0; i < nBlocks-1; i++ {
+			block := chain[i]
+			applyBlock(state, block, proxyApp)
+		}
+
+		if mode == 2 {
+			// update the dummy height and apphash
+			// as if we ran commit but not
+			applyBlock(state, chain[nBlocks-1], proxyApp)
+		}
+	}
+
+}
+
+func buildTMStateFromChain(config cfg.Config, state *sm.State, chain []*types.Block, mode uint) []byte {
+	// run the whole chain against this client to build up the tendermint state
+	clientCreator := proxy.NewLocalClientCreator(dummy.NewPersistentDummyApplication(path.Join(config.GetString("db_dir"), "1")))
+	proxyApp := proxy.NewAppConns(config, clientCreator, nil) // sm.NewHandshaker(config, state, store, ReplayLastBlock))
+	if _, err := proxyApp.Start(); err != nil {
+		panic(err)
+	}
+	defer proxyApp.Stop()
+
+	var latestAppHash []byte
+
+	switch mode {
+	case 0:
+		// sync right up
+		for _, block := range chain {
+			applyBlock(state, block, proxyApp)
+		}
+
+		latestAppHash = state.AppHash
+	case 1, 2:
+		// sync up to the penultimate as if we stored the block.
+		// whether we commit or not depends on the appHash
+		for _, block := range chain[:len(chain)-1] {
+			applyBlock(state, block, proxyApp)
+		}
+
+		// apply the final block to a state copy so we can
+		// get the right next appHash but keep the state back
+		stateCopy := state.Copy()
+		applyBlock(stateCopy, chain[len(chain)-1], proxyApp)
+		latestAppHash = stateCopy.AppHash
+	}
+
+	return latestAppHash
 }
 
 //--------------------------

From f3da6d23cb8a73e41acd2dfc8804c8db84e85b35 Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Mon, 20 Feb 2017 22:53:14 -0500
Subject: [PATCH 063/132] test: forward CIRCLECI var through docker

---
 test/test.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/test/test.sh b/test/test.sh
index 9fbfd7d3..69c4199d 100644
--- a/test/test.sh
+++ b/test/test.sh
@@ -25,7 +25,11 @@ echo
 echo "* [$(date +"%T")] running go tests and app tests in docker container"
 # sometimes its helpful to mount the local test folder
 # -v $DIR:/go/src/github.com/tendermint/tendermint/test
-docker run --name run_test -t tester bash test/run_test.sh
+if [[ "$CIRCLECI" == true ]]; then
+	docker run --name run_test -e CIRCLECI=true -t tester bash test/run_test.sh
+else
+	docker run --name run_test -t tester bash test/run_test.sh
+fi
 
 # copy the coverage results out of docker container
 docker cp run_test:/go/src/github.com/tendermint/tendermint/coverage.txt .

From d754d210cda3505ef5db23f93901b091c2692254 Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Tue, 21 Feb 2017 01:28:58 -0500
Subject: [PATCH 064/132] test: only use syslog on circle

---
 test/p2p/peer.sh                     | 36 +++++++++++++++++++---------
 test/persist/test_failure_indices.sh | 25 +------------------
 test/test.sh                         | 12 ++++++----
 test/utils/txs.sh                    | 23 ++++++++++++++++++
 4 files changed, 57 insertions(+), 39 deletions(-)
 create mode 100644 test/utils/txs.sh

diff --git a/test/p2p/peer.sh b/test/p2p/peer.sh
index 19586b12..7ee942bb 100644
--- a/test/p2p/peer.sh
+++ b/test/p2p/peer.sh
@@ -13,16 +13,30 @@ if [[ "$SEEDS" != "" ]]; then
 	SEEDS=" --seeds $SEEDS "
 fi
 
+set +eu
+
 echo "starting tendermint peer ID=$ID"
 # start tendermint container on the network
-docker run -d \
-  --net=$NETWORK_NAME \
-  --ip=$(test/p2p/ip.sh $ID) \
-  --name local_testnet_$ID \
-  --entrypoint tendermint \
-  -e TMROOT=/go/src/github.com/tendermint/tendermint/test/p2p/data/mach$ID/core \
-	--log-driver=syslog \
-	--log-opt syslog-address=udp://127.0.0.1:5514 \
-	--log-opt syslog-facility=daemon \
-	--log-opt tag="{{.Name}}" \
-  $DOCKER_IMAGE node $SEEDS --proxy_app=$APP_PROXY
+if [[ "$CIRCLECI" == true ]]; then
+	set -u
+	docker run -d \
+	  --net=$NETWORK_NAME \
+	  --ip=$(test/p2p/ip.sh $ID) \
+	  --name local_testnet_$ID \
+	  --entrypoint tendermint \
+	  -e TMROOT=/go/src/github.com/tendermint/tendermint/test/p2p/data/mach$ID/core \
+		--log-driver=syslog \
+		--log-opt syslog-address=udp://127.0.0.1:5514 \
+		--log-opt syslog-facility=daemon \
+		--log-opt tag="{{.Name}}" \
+	  $DOCKER_IMAGE node $SEEDS --proxy_app=$APP_PROXY
+else
+	set -u
+	docker run -d \
+	  --net=$NETWORK_NAME \
+	  --ip=$(test/p2p/ip.sh $ID) \
+	  --name local_testnet_$ID \
+	  --entrypoint tendermint \
+	  -e TMROOT=/go/src/github.com/tendermint/tendermint/test/p2p/data/mach$ID/core \
+	  $DOCKER_IMAGE node $SEEDS --proxy_app=$APP_PROXY
+fi
diff --git a/test/persist/test_failure_indices.sh b/test/persist/test_failure_indices.sh
index 8feb0005..af29f637 100644
--- a/test/persist/test_failure_indices.sh
+++ b/test/persist/test_failure_indices.sh
@@ -46,29 +46,6 @@ function kill_procs(){
 }
 
 
-# wait till node is up, send txs
-function send_txs(){
-	addr="127.0.0.1:46657"
-	curl -s $addr/status > /dev/null
-	ERR=$?
-	while [ "$ERR" != 0 ]; do
-		sleep 1	
-		curl -s $addr/status > /dev/null
-		ERR=$?
-	done
-
-	# send a bunch of txs over a few blocks
-	echo "Node is up, sending txs"
-	for i in $(seq 1 5); do
-		for _ in $(seq 1 100); do
-			tx=$(head -c 8 /dev/urandom | hexdump -ve '1/1 "%.2X"')
-			curl -s "$addr/broadcast_tx_async?tx=0x$tx" &> /dev/null
-		done
-		sleep 1
-	done
-}
-
-
 failsStart=0
 fails=$(grep -r "fail.Fail" --include \*.go . | wc -l)
 failsEnd=$((fails-1))
@@ -78,7 +55,7 @@ for failIndex in $(seq $failsStart $failsEnd); do
 	echo "* Test FailIndex $failIndex"
 	# test failure at failIndex
 
-	send_txs &
+	bash ./test/utils/txs.sh "localhost:46657" &
 	start_procs 1 "$failIndex"
 
 	# tendermint should fail when it hits the fail index
diff --git a/test/test.sh b/test/test.sh
index 69c4199d..1c1faca2 100644
--- a/test/test.sh
+++ b/test/test.sh
@@ -12,10 +12,14 @@ echo "* [$(date +"%T")] cleaning up $LOGS_DIR"
 rm -rf "$LOGS_DIR"
 mkdir -p "$LOGS_DIR"
 
-echo
-echo "* [$(date +"%T")] starting rsyslog container"
-docker rm -f rsyslog || true
-docker run -d -v "$LOGS_DIR:/var/log/" -p 127.0.0.1:5514:514/udp --name rsyslog voxxit/rsyslog
+set +u
+if [[ "$CIRCLECI" == true ]]; then
+	set -u
+	echo
+	echo "* [$(date +"%T")] starting rsyslog container"
+	docker rm -f rsyslog || true
+	docker run -d -v "$LOGS_DIR:/var/log/" -p 127.0.0.1:5514:514/udp --name rsyslog voxxit/rsyslog
+fi
 
 echo
 echo "* [$(date +"%T")] building docker image"
diff --git a/test/utils/txs.sh b/test/utils/txs.sh
new file mode 100644
index 00000000..c49d12c0
--- /dev/null
+++ b/test/utils/txs.sh
@@ -0,0 +1,23 @@
+#! /bin/bash
+set -u
+
+# wait till node is up, send txs
+ADDR=$1 #="127.0.0.1:46657"
+curl -s $ADDR/status > /dev/null
+ERR=$?
+while [ "$ERR" != 0 ]; do
+	sleep 1	
+	curl -s $ADDR/status > /dev/null
+	ERR=$?
+done
+
+# send a bunch of txs over a few blocks
+echo "Node is up, sending txs"
+for i in $(seq 1 5); do
+	for _ in $(seq 1 100); do
+		tx=$(head -c 8 /dev/urandom | hexdump -ve '1/1 "%.2X"')
+		curl -s "$ADDR/broadcast_tx_async?tx=0x$tx" &> /dev/null
+	done
+	echo "sent 100"
+	sleep 1
+done

From 6dbe9febce198df4468a4dcd7915208db5fab5e4 Mon Sep 17 00:00:00 2001
From: Anton Kaliaev <anton.kalyaev@gmail.com>
Date: Tue, 17 Jan 2017 20:58:27 +0400
Subject: [PATCH 065/132] log warning if peer send failed (Refs #174)

make lint happy

remove dead code

remove not needed go-common dependency

check peer.Send failures (Refs #174)
---
 blockchain/reactor.go       | 53 ++++++++++++++++++++-----------------
 consensus/byzantine_test.go |  2 +-
 consensus/reactor.go        | 45 ++++++++++++++++++++-----------
 mempool/reactor.go          |  5 ++--
 4 files changed, 61 insertions(+), 44 deletions(-)

diff --git a/blockchain/reactor.go b/blockchain/reactor.go
index 4a7f21d0..fda49db4 100644
--- a/blockchain/reactor.go
+++ b/blockchain/reactor.go
@@ -3,11 +3,10 @@ package blockchain
 import (
 	"bytes"
 	"errors"
-	"fmt"
 	"reflect"
 	"time"
 
-	. "github.com/tendermint/go-common"
+	cmn "github.com/tendermint/go-common"
 	cfg "github.com/tendermint/go-config"
 	"github.com/tendermint/go-p2p"
 	"github.com/tendermint/go-wire"
@@ -17,7 +16,9 @@ import (
 )
 
 const (
-	BlockchainChannel      = byte(0x40)
+	// BlockchainChannel is a channel for blocks and status updates (`BlockStore` height)
+	BlockchainChannel = byte(0x40)
+
 	defaultChannelCapacity = 100
 	defaultSleepIntervalMS = 500
 	trySyncIntervalMS      = 100
@@ -55,12 +56,13 @@ type BlockchainReactor struct {
 	evsw types.EventSwitch
 }
 
+// NewBlockchainReactor returns new reactor instance.
 func NewBlockchainReactor(config cfg.Config, state *sm.State, proxyAppConn proxy.AppConnConsensus, store *BlockStore, fastSync bool) *BlockchainReactor {
 	if state.LastBlockHeight == store.Height()-1 {
-		store.height -= 1 // XXX HACK, make this better
+		store.height-- // XXX HACK, make this better
 	}
 	if state.LastBlockHeight != store.Height() {
-		PanicSanity(Fmt("state (%v) and store (%v) height mismatch", state.LastBlockHeight, store.Height()))
+		cmn.PanicSanity(cmn.Fmt("state (%v) and store (%v) height mismatch", state.LastBlockHeight, store.Height()))
 	}
 	requestsCh := make(chan BlockRequest, defaultChannelCapacity)
 	timeoutsCh := make(chan string, defaultChannelCapacity)
@@ -83,6 +85,7 @@ func NewBlockchainReactor(config cfg.Config, state *sm.State, proxyAppConn proxy
 	return bcR
 }
 
+// OnStart implements BaseService
 func (bcR *BlockchainReactor) OnStart() error {
 	bcR.BaseReactor.OnStart()
 	if bcR.fastSync {
@@ -95,12 +98,13 @@ func (bcR *BlockchainReactor) OnStart() error {
 	return nil
 }
 
+// OnStop implements BaseService
 func (bcR *BlockchainReactor) OnStop() {
 	bcR.BaseReactor.OnStop()
 	bcR.pool.Stop()
 }
 
-// Implements Reactor
+// GetChannels implements Reactor
 func (bcR *BlockchainReactor) GetChannels() []*p2p.ChannelDescriptor {
 	return []*p2p.ChannelDescriptor{
 		&p2p.ChannelDescriptor{
@@ -111,19 +115,20 @@ func (bcR *BlockchainReactor) GetChannels() []*p2p.ChannelDescriptor {
 	}
 }
 
-// Implements Reactor
+// AddPeer implements Reactor by sending our state to peer.
 func (bcR *BlockchainReactor) AddPeer(peer *p2p.Peer) {
-	// Send peer our state.
-	peer.Send(BlockchainChannel, struct{ BlockchainMessage }{&bcStatusResponseMessage{bcR.store.Height()}})
+	if !peer.Send(BlockchainChannel, struct{ BlockchainMessage }{&bcStatusResponseMessage{bcR.store.Height()}}) {
+		log.Warn("Failed to send our state to peer", "peer", peer)
+		// doing nothing, will try later in `poolRoutine`
+	}
 }
 
-// Implements Reactor
+// RemovePeer implements Reactor by removing peer from the pool.
 func (bcR *BlockchainReactor) RemovePeer(peer *p2p.Peer, reason interface{}) {
-	// Remove peer from the pool.
 	bcR.pool.RemovePeer(peer.Key)
 }
 
-// Implements Reactor
+// Receive implements Reactor by handling 4 types of messages (look below).
 func (bcR *BlockchainReactor) Receive(chID byte, src *p2p.Peer, msgBytes []byte) {
 	_, msg, err := DecodeMessage(msgBytes)
 	if err != nil {
@@ -159,7 +164,7 @@ func (bcR *BlockchainReactor) Receive(chID byte, src *p2p.Peer, msgBytes []byte)
 		// Got a peer status. Unverified.
 		bcR.pool.SetPeerHeight(src.Key, msg.Height)
 	default:
-		log.Warn(Fmt("Unknown message type %v", reflect.TypeOf(msg)))
+		log.Warn(cmn.Fmt("Unknown message type %v", reflect.TypeOf(msg)))
 	}
 }
 
@@ -245,7 +250,7 @@ FOR_LOOP:
 					err := bcR.state.ApplyBlock(bcR.evsw, bcR.proxyAppConn, first, firstPartsHeader, types.MockMempool{})
 					if err != nil {
 						// TODO This is bad, are we zombie?
-						PanicQ(Fmt("Failed to process committed block (%d:%X): %v", first.Height, first.Hash(), err))
+						cmn.PanicQ(cmn.Fmt("Failed to process committed block (%d:%X): %v", first.Height, first.Hash(), err))
 					}
 					bcR.state.Save()
 				}
@@ -257,17 +262,13 @@ FOR_LOOP:
 	}
 }
 
-func (bcR *BlockchainReactor) BroadcastStatusResponse() error {
-	bcR.Switch.Broadcast(BlockchainChannel, struct{ BlockchainMessage }{&bcStatusResponseMessage{bcR.store.Height()}})
-	return nil
-}
-
+// BroadcastStatusRequest broadcasts `BlockStore` height.
 func (bcR *BlockchainReactor) BroadcastStatusRequest() error {
 	bcR.Switch.Broadcast(BlockchainChannel, struct{ BlockchainMessage }{&bcStatusRequestMessage{bcR.store.Height()}})
 	return nil
 }
 
-// implements events.Eventable
+// SetEventSwitch implements events.Eventable
 func (bcR *BlockchainReactor) SetEventSwitch(evsw types.EventSwitch) {
 	bcR.evsw = evsw
 }
@@ -282,6 +283,7 @@ const (
 	msgTypeStatusRequest  = byte(0x21)
 )
 
+// BlockchainMessage is a generic message for this reactor.
 type BlockchainMessage interface{}
 
 var _ = wire.RegisterInterface(
@@ -292,6 +294,7 @@ var _ = wire.RegisterInterface(
 	wire.ConcreteType{&bcStatusRequestMessage{}, msgTypeStatusRequest},
 )
 
+// DecodeMessage decodes BlockchainMessage.
 // TODO: ensure that bz is completely read.
 func DecodeMessage(bz []byte) (msgType byte, msg BlockchainMessage, err error) {
 	msgType = bz[0]
@@ -299,7 +302,7 @@ func DecodeMessage(bz []byte) (msgType byte, msg BlockchainMessage, err error) {
 	r := bytes.NewReader(bz)
 	msg = wire.ReadBinary(struct{ BlockchainMessage }{}, r, maxBlockchainResponseSize, &n, &err).(struct{ BlockchainMessage }).BlockchainMessage
 	if err != nil && n != len(bz) {
-		err = errors.New("DecodeMessage() had bytes left over.")
+		err = errors.New("DecodeMessage() had bytes left over")
 	}
 	return
 }
@@ -311,7 +314,7 @@ type bcBlockRequestMessage struct {
 }
 
 func (m *bcBlockRequestMessage) String() string {
-	return fmt.Sprintf("[bcBlockRequestMessage %v]", m.Height)
+	return cmn.Fmt("[bcBlockRequestMessage %v]", m.Height)
 }
 
 //-------------------------------------
@@ -322,7 +325,7 @@ type bcBlockResponseMessage struct {
 }
 
 func (m *bcBlockResponseMessage) String() string {
-	return fmt.Sprintf("[bcBlockResponseMessage %v]", m.Block.Height)
+	return cmn.Fmt("[bcBlockResponseMessage %v]", m.Block.Height)
 }
 
 //-------------------------------------
@@ -332,7 +335,7 @@ type bcStatusRequestMessage struct {
 }
 
 func (m *bcStatusRequestMessage) String() string {
-	return fmt.Sprintf("[bcStatusRequestMessage %v]", m.Height)
+	return cmn.Fmt("[bcStatusRequestMessage %v]", m.Height)
 }
 
 //-------------------------------------
@@ -342,5 +345,5 @@ type bcStatusResponseMessage struct {
 }
 
 func (m *bcStatusResponseMessage) String() string {
-	return fmt.Sprintf("[bcStatusResponseMessage %v]", m.Height)
+	return cmn.Fmt("[bcStatusResponseMessage %v]", m.Height)
 }
diff --git a/consensus/byzantine_test.go b/consensus/byzantine_test.go
index 396c8c07..cd62f3f0 100644
--- a/consensus/byzantine_test.go
+++ b/consensus/byzantine_test.go
@@ -242,7 +242,7 @@ func (br *ByzantineReactor) AddPeer(peer *p2p.Peer) {
 	// Send our state to peer.
 	// If we're fast_syncing, broadcast a RoundStepMessage later upon SwitchToConsensus().
 	if !br.reactor.fastSync {
-		br.reactor.sendNewRoundStepMessage(peer)
+		br.reactor.sendNewRoundStepMessages(peer)
 	}
 }
 func (br *ByzantineReactor) RemovePeer(peer *p2p.Peer, reason interface{}) {
diff --git a/consensus/reactor.go b/consensus/reactor.go
index 1e700a86..c79dcc81 100644
--- a/consensus/reactor.go
+++ b/consensus/reactor.go
@@ -127,7 +127,7 @@ func (conR *ConsensusReactor) AddPeer(peer *p2p.Peer) {
 	// Send our state to peer.
 	// If we're fast_syncing, broadcast a RoundStepMessage later upon SwitchToConsensus().
 	if !conR.fastSync {
-		conR.sendNewRoundStepMessage(peer)
+		conR.sendNewRoundStepMessages(peer)
 	}
 }
 
@@ -201,7 +201,6 @@ func (conR *ConsensusReactor) Receive(chID byte, src *p2p.Peer, msgBytes []byte)
 				BlockID: msg.BlockID,
 				Votes:   ourVotes,
 			}})
-
 		default:
 			log.Warn(Fmt("Unknown message type %v", reflect.TypeOf(msg)))
 		}
@@ -365,14 +364,20 @@ func makeRoundStepMessages(rs *RoundState) (nrsMsg *NewRoundStepMessage, csMsg *
 	return
 }
 
-func (conR *ConsensusReactor) sendNewRoundStepMessage(peer *p2p.Peer) {
+func (conR *ConsensusReactor) sendNewRoundStepMessages(peer *p2p.Peer) {
+	log := log.New("peer", peer)
+
 	rs := conR.conS.GetRoundState()
 	nrsMsg, csMsg := makeRoundStepMessages(rs)
 	if nrsMsg != nil {
-		peer.Send(StateChannel, struct{ ConsensusMessage }{nrsMsg})
+		if !peer.Send(StateChannel, struct{ ConsensusMessage }{nrsMsg}) {
+			log.Warn("Failed to send NewRoundStepMessage to peer")
+		}
 	}
 	if csMsg != nil {
-		peer.Send(StateChannel, struct{ ConsensusMessage }{csMsg})
+		if !peer.Send(StateChannel, struct{ ConsensusMessage }{csMsg}) {
+			log.Warn("Failed to send RoundStepCommit to peer")
+		}
 	}
 }
 
@@ -399,8 +404,11 @@ OUTER_LOOP:
 					Round:  rs.Round,  // This tells peer that this part applies to us.
 					Part:   part,
 				}
-				peer.Send(DataChannel, struct{ ConsensusMessage }{msg})
-				ps.SetHasProposalBlockPart(prs.Height, prs.Round, index)
+				if peer.Send(DataChannel, struct{ ConsensusMessage }{msg}) {
+					ps.SetHasProposalBlockPart(prs.Height, prs.Round, index)
+				} else {
+					log.Warn("Failed to send BlockPartMessage to peer")
+				}
 				continue OUTER_LOOP
 			}
 		}
@@ -435,8 +443,11 @@ OUTER_LOOP:
 					Round:  prs.Round,  // Not our height, so it doesn't matter.
 					Part:   part,
 				}
-				peer.Send(DataChannel, struct{ ConsensusMessage }{msg})
-				ps.SetHasProposalBlockPart(prs.Height, prs.Round, index)
+				if peer.Send(DataChannel, struct{ ConsensusMessage }{msg}) {
+					ps.SetHasProposalBlockPart(prs.Height, prs.Round, index)
+				} else {
+					log.Warn("Failed to send BlockPartMessage to peer")
+				}
 				continue OUTER_LOOP
 			} else {
 				//log.Info("No parts to send in catch-up, sleeping")
@@ -462,8 +473,11 @@ OUTER_LOOP:
 			// Proposal: share the proposal metadata with peer.
 			{
 				msg := &ProposalMessage{Proposal: rs.Proposal}
-				peer.Send(DataChannel, struct{ ConsensusMessage }{msg})
-				ps.SetHasProposal(rs.Proposal)
+				if peer.Send(DataChannel, struct{ ConsensusMessage }{msg}) {
+					ps.SetHasProposal(rs.Proposal)
+				} else {
+					log.Warn("Failed to send ProposalMessage to peer")
+				}
 			}
 			// ProposalPOL: lets peer know which POL votes we have so far.
 			// Peer must receive ProposalMessage first.
@@ -475,7 +489,9 @@ OUTER_LOOP:
 					ProposalPOLRound: rs.Proposal.POLRound,
 					ProposalPOL:      rs.Votes.Prevotes(rs.Proposal.POLRound).BitArray(),
 				}
-				peer.Send(DataChannel, struct{ ConsensusMessage }{msg})
+				if !peer.Send(DataChannel, struct{ ConsensusMessage }{msg}) {
+					log.Warn("Failed to send ProposalPOLMessage to peer")
+				}
 			}
 			continue OUTER_LOOP
 		}
@@ -806,13 +822,12 @@ func (ps *PeerState) SetHasProposalBlockPart(height int, round int, index int) {
 	ps.ProposalBlockParts.SetIndex(index, true)
 }
 
-// Convenience function to send vote to peer.
+// PickVoteToSend sends vote to peer.
 // Returns true if vote was sent.
 func (ps *PeerState) PickSendVote(votes types.VoteSetReader) (ok bool) {
 	if vote, ok := ps.PickVoteToSend(votes); ok {
 		msg := &VoteMessage{vote}
-		ps.Peer.Send(VoteChannel, struct{ ConsensusMessage }{msg})
-		return true
+		return ps.Peer.Send(VoteChannel, struct{ ConsensusMessage }{msg})
 	}
 	return false
 }
diff --git a/mempool/reactor.go b/mempool/reactor.go
index 0c5cc9f8..4531edee 100644
--- a/mempool/reactor.go
+++ b/mempool/reactor.go
@@ -6,13 +6,12 @@ import (
 	"reflect"
 	"time"
 
+	abci "github.com/tendermint/abci/types"
 	"github.com/tendermint/go-clist"
-	. "github.com/tendermint/go-common"
 	cfg "github.com/tendermint/go-config"
 	"github.com/tendermint/go-p2p"
 	"github.com/tendermint/go-wire"
 	"github.com/tendermint/tendermint/types"
-	abci "github.com/tendermint/abci/types"
 )
 
 const (
@@ -80,7 +79,7 @@ func (memR *MempoolReactor) Receive(chID byte, src *p2p.Peer, msgBytes []byte) {
 		}
 		// broadcasting happens from go routines per peer
 	default:
-		log.Warn(Fmt("Unknown message type %v", reflect.TypeOf(msg)))
+		log.Warn(fmt.Sprintf("Unknown message type %v", reflect.TypeOf(msg)))
 	}
 }
 

From 2c724d5eee2c236ba2c3bb00d7443d8807fe29fb Mon Sep 17 00:00:00 2001
From: Anton Kaliaev <anton.kalyaev@gmail.com>
Date: Tue, 21 Feb 2017 13:21:55 +0400
Subject: [PATCH 066/132] remove warning messages in favor of "Send failed"

---
 blockchain/reactor.go |  1 -
 consensus/reactor.go  | 20 +++-----------------
 2 files changed, 3 insertions(+), 18 deletions(-)

diff --git a/blockchain/reactor.go b/blockchain/reactor.go
index fda49db4..90258825 100644
--- a/blockchain/reactor.go
+++ b/blockchain/reactor.go
@@ -118,7 +118,6 @@ func (bcR *BlockchainReactor) GetChannels() []*p2p.ChannelDescriptor {
 // AddPeer implements Reactor by sending our state to peer.
 func (bcR *BlockchainReactor) AddPeer(peer *p2p.Peer) {
 	if !peer.Send(BlockchainChannel, struct{ BlockchainMessage }{&bcStatusResponseMessage{bcR.store.Height()}}) {
-		log.Warn("Failed to send our state to peer", "peer", peer)
 		// doing nothing, will try later in `poolRoutine`
 	}
 }
diff --git a/consensus/reactor.go b/consensus/reactor.go
index c79dcc81..c3b1c590 100644
--- a/consensus/reactor.go
+++ b/consensus/reactor.go
@@ -365,19 +365,13 @@ func makeRoundStepMessages(rs *RoundState) (nrsMsg *NewRoundStepMessage, csMsg *
 }
 
 func (conR *ConsensusReactor) sendNewRoundStepMessages(peer *p2p.Peer) {
-	log := log.New("peer", peer)
-
 	rs := conR.conS.GetRoundState()
 	nrsMsg, csMsg := makeRoundStepMessages(rs)
 	if nrsMsg != nil {
-		if !peer.Send(StateChannel, struct{ ConsensusMessage }{nrsMsg}) {
-			log.Warn("Failed to send NewRoundStepMessage to peer")
-		}
+		peer.Send(StateChannel, struct{ ConsensusMessage }{nrsMsg})
 	}
 	if csMsg != nil {
-		if !peer.Send(StateChannel, struct{ ConsensusMessage }{csMsg}) {
-			log.Warn("Failed to send RoundStepCommit to peer")
-		}
+		peer.Send(StateChannel, struct{ ConsensusMessage }{csMsg})
 	}
 }
 
@@ -406,8 +400,6 @@ OUTER_LOOP:
 				}
 				if peer.Send(DataChannel, struct{ ConsensusMessage }{msg}) {
 					ps.SetHasProposalBlockPart(prs.Height, prs.Round, index)
-				} else {
-					log.Warn("Failed to send BlockPartMessage to peer")
 				}
 				continue OUTER_LOOP
 			}
@@ -445,8 +437,6 @@ OUTER_LOOP:
 				}
 				if peer.Send(DataChannel, struct{ ConsensusMessage }{msg}) {
 					ps.SetHasProposalBlockPart(prs.Height, prs.Round, index)
-				} else {
-					log.Warn("Failed to send BlockPartMessage to peer")
 				}
 				continue OUTER_LOOP
 			} else {
@@ -475,8 +465,6 @@ OUTER_LOOP:
 				msg := &ProposalMessage{Proposal: rs.Proposal}
 				if peer.Send(DataChannel, struct{ ConsensusMessage }{msg}) {
 					ps.SetHasProposal(rs.Proposal)
-				} else {
-					log.Warn("Failed to send ProposalMessage to peer")
 				}
 			}
 			// ProposalPOL: lets peer know which POL votes we have so far.
@@ -489,9 +477,7 @@ OUTER_LOOP:
 					ProposalPOLRound: rs.Proposal.POLRound,
 					ProposalPOL:      rs.Votes.Prevotes(rs.Proposal.POLRound).BitArray(),
 				}
-				if !peer.Send(DataChannel, struct{ ConsensusMessage }{msg}) {
-					log.Warn("Failed to send ProposalPOLMessage to peer")
-				}
+				peer.Send(DataChannel, struct{ ConsensusMessage }{msg})
 			}
 			continue OUTER_LOOP
 		}

From 7fab31fbe3e0a3f835d5216f44c96b26f1918da5 Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Tue, 21 Feb 2017 11:18:40 -0500
Subject: [PATCH 067/132] test: more logging

---
 test/p2p/kill_all/check_peers.sh | 1 +
 test/p2p/peer.sh                 | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/test/p2p/kill_all/check_peers.sh b/test/p2p/kill_all/check_peers.sh
index 3c925079..d085a025 100644
--- a/test/p2p/kill_all/check_peers.sh
+++ b/test/p2p/kill_all/check_peers.sh
@@ -40,6 +40,7 @@ for i in $(seq 2 "$NUM_OF_PEERS"); do
     ((attempt++))
     if [ "$attempt" -ge $MAX_ATTEMPTS_TO_CATCH_UP ] ; then
       echo "$attempt unsuccessful attempts were made to catch up"
+      curl -s "$addr/dump_consensus_state" | jq .result[1]
       exit 1
     fi
 
diff --git a/test/p2p/peer.sh b/test/p2p/peer.sh
index 7ee942bb..611f5ac2 100644
--- a/test/p2p/peer.sh
+++ b/test/p2p/peer.sh
@@ -29,7 +29,7 @@ if [[ "$CIRCLECI" == true ]]; then
 		--log-opt syslog-address=udp://127.0.0.1:5514 \
 		--log-opt syslog-facility=daemon \
 		--log-opt tag="{{.Name}}" \
-	  $DOCKER_IMAGE node $SEEDS --proxy_app=$APP_PROXY
+	  $DOCKER_IMAGE node $SEEDS --log_level=debug --proxy_app=$APP_PROXY
 else
 	set -u
 	docker run -d \
@@ -38,5 +38,5 @@ else
 	  --name local_testnet_$ID \
 	  --entrypoint tendermint \
 	  -e TMROOT=/go/src/github.com/tendermint/tendermint/test/p2p/data/mach$ID/core \
-	  $DOCKER_IMAGE node $SEEDS --proxy_app=$APP_PROXY
+	  $DOCKER_IMAGE node $SEEDS --log_level=info --proxy_app=$APP_PROXY
 fi

From 7c1e79cbc54fd02fe0db51f38b49ae7924a88342 Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Tue, 21 Feb 2017 14:14:08 -0500
Subject: [PATCH 068/132] test/persists: wait for tendermint proc

---
 test/persist/test_failure_indices.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/persist/test_failure_indices.sh b/test/persist/test_failure_indices.sh
index af29f637..81721e65 100644
--- a/test/persist/test_failure_indices.sh
+++ b/test/persist/test_failure_indices.sh
@@ -61,6 +61,7 @@ for failIndex in $(seq $failsStart $failsEnd); do
 	# tendermint should fail when it hits the fail index
 	kill -9 "$PID_DUMMY"
 	wait "$PID_DUMMY"
+	wait "$PID_TENDERMINT"
 
 	start_procs 2
 

From f73f53c486f0571396e414f338902cee4607f51d Mon Sep 17 00:00:00 2001
From: Anton Kaliaev <anton.kalyaev@gmail.com>
Date: Thu, 23 Feb 2017 12:17:20 +0400
Subject: [PATCH 069/132] [Dockerfile.develop] fix error "/bin/sh:  #: not
 found" when building

---
 DOCKER/Dockerfile.develop | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/DOCKER/Dockerfile.develop b/DOCKER/Dockerfile.develop
index adbe6c04..19e101ad 100644
--- a/DOCKER/Dockerfile.develop
+++ b/DOCKER/Dockerfile.develop
@@ -20,7 +20,7 @@ RUN mkdir -p /go/src/github.com/tendermint/tendermint && \
     git checkout develop && \
     make get_vendor_deps && \
     make install && \
-    glide cc && \ # 183Mb => 75Mb
+    glide cc && \
     cd - && \
     rm -rf /go/src/github.com/tendermint/tendermint && \
     apk del go build-base git

From d38a6e329f3c47be55c90ed213f8737fefd62802 Mon Sep 17 00:00:00 2001
From: Anton Kaliaev <anton.kalyaev@gmail.com>
Date: Fri, 24 Feb 2017 13:30:46 +0400
Subject: [PATCH 070/132] verbose output when get_deps

---
 Makefile | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 48270512..bfdaf007 100644
--- a/Makefile
+++ b/Makefile
@@ -45,10 +45,11 @@ list_deps:
 		xargs go list -f '{{if not .Standard}}{{.ImportPath}}{{end}}'
 
 get_deps:
-	@go get -d $(PACKAGES)
+	@echo "--> Running go get"
+	@go get -v -d $(PACKAGES)
 	@go list -f '{{join .TestImports "\n"}}' ./... | \
 		grep -v /vendor/ | sort | uniq | \
-		xargs go get -d
+		xargs go get -v -d
 
 get_vendor_deps: ensure_tools
 	@rm -rf vendor/

From 1f1dcead3d87d6af6c7ccd958ff9a268e1dcc695 Mon Sep 17 00:00:00 2001
From: Anton Kaliaev <anton.kalyaev@gmail.com>
Date: Fri, 24 Feb 2017 13:31:27 +0400
Subject: [PATCH 071/132] [Vagrantfile] go 1.8 and permissions fix

---
 Vagrantfile | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/Vagrantfile b/Vagrantfile
index 5b9b239c..a3b32974 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -17,12 +17,12 @@ Vagrant.configure("2") do |config|
     usermod -a -G docker vagrant
     apt-get autoremove -y
 
-    curl -O https://storage.googleapis.com/golang/go1.7.linux-amd64.tar.gz
-    tar -xvf go1.7.linux-amd64.tar.gz
-    mv -f go /usr/local
-    rm -f go1.7.linux-amd64.tar.gz
+    curl -O https://storage.googleapis.com/golang/go1.8.linux-amd64.tar.gz
+    tar -xvf go1.8.linux-amd64.tar.gz
+    rm -rf /usr/local/go
+    mv go /usr/local
+    rm -f go1.8.linux-amd64.tar.gz
     mkdir -p /home/vagrant/go/bin
-    chown -R vagrant:vagrant /home/vagrant/go
     echo 'export PATH=$PATH:/usr/local/go/bin:/home/vagrant/go/bin' >> /home/vagrant/.bash_profile
     echo 'export GOPATH=/home/vagrant/go' >> /home/vagrant/.bash_profile
 
@@ -31,6 +31,8 @@ Vagrant.configure("2") do |config|
     mkdir -p /home/vagrant/go/src/github.com/tendermint
     ln -s /vagrant /home/vagrant/go/src/github.com/tendermint/tendermint
 
+    chown -R vagrant:vagrant /home/vagrant/go
+
     su - vagrant -c 'cd /home/vagrant/go/src/github.com/tendermint/tendermint && make get_vendor_deps'
   SHELL
 end

From d971416d12558edad0cc852ec365a18a5e418f88 Mon Sep 17 00:00:00 2001
From: Ethan Frey <ethanfrey@users.noreply.github.com>
Date: Tue, 21 Feb 2017 19:36:18 +0100
Subject: [PATCH 072/132] collision merge of light-client code

---
 rpc/client/client.go    | 178 ++++++++++++++++++++++++++++++++++++
 rpc/test/client_test.go |  39 ++++----
 rpc/test/grpc_test.go   |   2 +-
 rpc/test/helpers.go     | 195 +++++++++++++---------------------------
 rpc/test/helpers_old.go | 157 ++++++++++++++++++++++++++++++++
 rpc/test/main_test.go   |  33 +++++++
 rpc/test/rpc_test.go    | 130 +++++++++++++++++++++++++++
 7 files changed, 581 insertions(+), 153 deletions(-)
 create mode 100644 rpc/client/client.go
 create mode 100644 rpc/test/helpers_old.go
 create mode 100644 rpc/test/main_test.go
 create mode 100644 rpc/test/rpc_test.go

diff --git a/rpc/client/client.go b/rpc/client/client.go
new file mode 100644
index 00000000..b2d4ee21
--- /dev/null
+++ b/rpc/client/client.go
@@ -0,0 +1,178 @@
+package rpcclient
+
+import (
+	"encoding/json"
+
+	"github.com/pkg/errors"
+	"github.com/tendermint/go-rpc/client"
+	ctypes "github.com/tendermint/tendermint/rpc/core/types"
+	"github.com/tendermint/tendermint/types"
+)
+
+type HTTPClient struct {
+	remote   string
+	endpoint string
+	rpc      *rpcclient.ClientJSONRPC
+	ws       *rpcclient.WSClient
+}
+
+func New(remote, wsEndpoint string) *HTTPClient {
+	return &HTTPClient{
+		rpc:      rpcclient.NewClientJSONRPC(remote),
+		remote:   remote,
+		endpoint: wsEndpoint,
+	}
+}
+
+func (c *HTTPClient) Status() (*ctypes.ResultStatus, error) {
+	tmResult := new(ctypes.TMResult)
+	_, err := c.rpc.Call("status", []interface{}{}, tmResult)
+	if err != nil {
+		return nil, errors.Wrap(err, "Status")
+	}
+	// note: panics if rpc doesn't match.  okay???
+	return (*tmResult).(*ctypes.ResultStatus), nil
+}
+
+func (c *HTTPClient) ABCIInfo() (*ctypes.ResultABCIInfo, error) {
+	tmResult := new(ctypes.TMResult)
+	_, err := c.rpc.Call("abci_info", []interface{}{}, tmResult)
+	if err != nil {
+		return nil, errors.Wrap(err, "ABCIInfo")
+	}
+	return (*tmResult).(*ctypes.ResultABCIInfo), nil
+}
+
+func (c *HTTPClient) ABCIQuery(path string, data []byte, prove bool) (*ctypes.ResultABCIQuery, error) {
+	tmResult := new(ctypes.TMResult)
+	_, err := c.rpc.Call("abci_query", []interface{}{path, data, prove}, tmResult)
+	if err != nil {
+		return nil, errors.Wrap(err, "ABCIQuery")
+	}
+	return (*tmResult).(*ctypes.ResultABCIQuery), nil
+}
+
+func (c *HTTPClient) BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) {
+	return c.broadcastTX("broadcast_tx_commit", tx)
+}
+
+func (c *HTTPClient) BroadcastTxAsync(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) {
+	return c.broadcastTX("broadcast_tx_async", tx)
+}
+
+func (c *HTTPClient) BroadcastTxSync(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) {
+	return c.broadcastTX("broadcast_tx_sync", tx)
+}
+
+func (c *HTTPClient) broadcastTX(route string, tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) {
+	tmResult := new(ctypes.TMResult)
+	_, err := c.rpc.Call(route, []interface{}{tx}, tmResult)
+	if err != nil {
+		return nil, errors.Wrap(err, route)
+	}
+	return (*tmResult).(*ctypes.ResultBroadcastTxCommit), nil
+}
+
+func (c *HTTPClient) NetInfo() (*ctypes.ResultNetInfo, error) {
+	tmResult := new(ctypes.TMResult)
+	_, err := c.rpc.Call("net_info", nil, tmResult)
+	if err != nil {
+		return nil, errors.Wrap(err, "NetInfo")
+	}
+	return (*tmResult).(*ctypes.ResultNetInfo), nil
+}
+
+func (c *HTTPClient) DialSeeds(seeds []string) (*ctypes.ResultDialSeeds, error) {
+	tmResult := new(ctypes.TMResult)
+	// TODO: is this the correct way to marshall seeds?
+	_, err := c.rpc.Call("dial_seeds", []interface{}{seeds}, tmResult)
+	if err != nil {
+		return nil, errors.Wrap(err, "DialSeeds")
+	}
+	return (*tmResult).(*ctypes.ResultDialSeeds), nil
+}
+
+func (c *HTTPClient) BlockchainInfo(minHeight, maxHeight int) (*ctypes.ResultBlockchainInfo, error) {
+	tmResult := new(ctypes.TMResult)
+	_, err := c.rpc.Call("blockchain", []interface{}{minHeight, maxHeight}, tmResult)
+	if err != nil {
+		return nil, errors.Wrap(err, "BlockchainInfo")
+	}
+	return (*tmResult).(*ctypes.ResultBlockchainInfo), nil
+}
+
+func (c *HTTPClient) Genesis() (*ctypes.ResultGenesis, error) {
+	tmResult := new(ctypes.TMResult)
+	_, err := c.rpc.Call("genesis", nil, tmResult)
+	if err != nil {
+		return nil, errors.Wrap(err, "Genesis")
+	}
+	return (*tmResult).(*ctypes.ResultGenesis), nil
+}
+
+func (c *HTTPClient) Block(height int) (*ctypes.ResultBlock, error) {
+	tmResult := new(ctypes.TMResult)
+	_, err := c.rpc.Call("block", []interface{}{height}, tmResult)
+	if err != nil {
+		return nil, errors.Wrap(err, "Block")
+	}
+	return (*tmResult).(*ctypes.ResultBlock), nil
+}
+
+func (c *HTTPClient) Commit(height int) (*ctypes.ResultCommit, error) {
+	tmResult := new(ctypes.TMResult)
+	_, err := c.rpc.Call("commit", []interface{}{height}, tmResult)
+	if err != nil {
+		return nil, errors.Wrap(err, "Commit")
+	}
+	return (*tmResult).(*ctypes.ResultCommit), nil
+}
+
+func (c *HTTPClient) Validators() (*ctypes.ResultValidators, error) {
+	tmResult := new(ctypes.TMResult)
+	_, err := c.rpc.Call("validators", nil, tmResult)
+	if err != nil {
+		return nil, errors.Wrap(err, "Validators")
+	}
+	return (*tmResult).(*ctypes.ResultValidators), nil
+}
+
+/** websocket event stuff here... **/
+
+// StartWebsocket starts up a websocket and a listener goroutine
+// if already started, do nothing
+func (c *HTTPClient) StartWebsocket() error {
+	var err error
+	if c.ws == nil {
+		ws := rpcclient.NewWSClient(c.remote, c.endpoint)
+		_, err = ws.Start()
+		if err == nil {
+			c.ws = ws
+		}
+	}
+	return errors.Wrap(err, "StartWebsocket")
+}
+
+// StopWebsocket stops the websocket connection
+func (c *HTTPClient) StopWebsocket() {
+	if c.ws != nil {
+		c.ws.Stop()
+		c.ws = nil
+	}
+}
+
+// GetEventChannels returns the results and error channel from the websocket
+func (c *HTTPClient) GetEventChannels() (chan json.RawMessage, chan error) {
+	if c.ws == nil {
+		return nil, nil
+	}
+	return c.ws.ResultsCh, c.ws.ErrorsCh
+}
+
+func (c *HTTPClient) Subscribe(event string) error {
+	return errors.Wrap(c.ws.Subscribe(event), "Subscribe")
+}
+
+func (c *HTTPClient) Unsubscribe(event string) error {
+	return errors.Wrap(c.ws.Unsubscribe(event), "Unsubscribe")
+}
diff --git a/rpc/test/client_test.go b/rpc/test/client_test.go
index b17252df..a173bf29 100644
--- a/rpc/test/client_test.go
+++ b/rpc/test/client_test.go
@@ -24,7 +24,7 @@ import (
 
 func TestURIStatus(t *testing.T) {
 	tmResult := new(ctypes.TMResult)
-	_, err := clientURI.Call("status", map[string]interface{}{}, tmResult)
+	_, err := GetURIClient().Call("status", map[string]interface{}{}, tmResult)
 	if err != nil {
 		panic(err)
 	}
@@ -33,7 +33,7 @@ func TestURIStatus(t *testing.T) {
 
 func TestJSONStatus(t *testing.T) {
 	tmResult := new(ctypes.TMResult)
-	_, err := clientJSON.Call("status", []interface{}{}, tmResult)
+	_, err := GetJSONClient().Call("status", []interface{}{}, tmResult)
 	if err != nil {
 		panic(err)
 	}
@@ -41,6 +41,8 @@ func TestJSONStatus(t *testing.T) {
 }
 
 func testStatus(t *testing.T, statusI interface{}) {
+	chainID := GetConfig().GetString("chain_id")
+
 	tmRes := statusI.(*ctypes.TMResult)
 	status := (*tmRes).(*ctypes.ResultStatus)
 	if status.NodeInfo.Network != chainID {
@@ -68,7 +70,7 @@ func TestURIBroadcastTxSync(t *testing.T) {
 	defer config.Set("block_size", -1)
 	tmResult := new(ctypes.TMResult)
 	tx := randBytes()
-	_, err := clientURI.Call("broadcast_tx_sync", map[string]interface{}{"tx": tx}, tmResult)
+	_, err := GetURIClient().Call("broadcast_tx_sync", map[string]interface{}{"tx": tx}, tmResult)
 	if err != nil {
 		panic(err)
 	}
@@ -80,7 +82,7 @@ func TestJSONBroadcastTxSync(t *testing.T) {
 	defer config.Set("block_size", -1)
 	tmResult := new(ctypes.TMResult)
 	tx := randBytes()
-	_, err := clientJSON.Call("broadcast_tx_sync", []interface{}{tx}, tmResult)
+	_, err := GetJSONClient().Call("broadcast_tx_sync", []interface{}{tx}, tmResult)
 	if err != nil {
 		panic(err)
 	}
@@ -118,13 +120,10 @@ func testTxKV() ([]byte, []byte, []byte) {
 func sendTx() ([]byte, []byte) {
 	tmResult := new(ctypes.TMResult)
 	k, v, tx := testTxKV()
-	_, err := clientJSON.Call("broadcast_tx_commit", []interface{}{tx}, tmResult)
+	_, err := GetJSONClient().Call("broadcast_tx_commit", []interface{}{tx}, tmResult)
 	if err != nil {
 		panic(err)
 	}
-	fmt.Println("SENT TX", tx)
-	fmt.Printf("SENT TX %X\n", tx)
-	fmt.Printf("k %X; v %X", k, v)
 	return k, v
 }
 
@@ -132,7 +131,7 @@ func TestURIABCIQuery(t *testing.T) {
 	k, v := sendTx()
 	time.Sleep(time.Second)
 	tmResult := new(ctypes.TMResult)
-	_, err := clientURI.Call("abci_query", map[string]interface{}{"path": "", "data": k, "prove": false}, tmResult)
+	_, err := GetURIClient().Call("abci_query", map[string]interface{}{"path": "", "data": k, "prove": false}, tmResult)
 	if err != nil {
 		panic(err)
 	}
@@ -142,7 +141,7 @@ func TestURIABCIQuery(t *testing.T) {
 func TestJSONABCIQuery(t *testing.T) {
 	k, v := sendTx()
 	tmResult := new(ctypes.TMResult)
-	_, err := clientJSON.Call("abci_query", []interface{}{"", k, false}, tmResult)
+	_, err := GetJSONClient().Call("abci_query", []interface{}{"", k, false}, tmResult)
 	if err != nil {
 		panic(err)
 	}
@@ -168,7 +167,7 @@ func testABCIQuery(t *testing.T, statusI interface{}, value []byte) {
 func TestURIBroadcastTxCommit(t *testing.T) {
 	tmResult := new(ctypes.TMResult)
 	tx := randBytes()
-	_, err := clientURI.Call("broadcast_tx_commit", map[string]interface{}{"tx": tx}, tmResult)
+	_, err := GetURIClient().Call("broadcast_tx_commit", map[string]interface{}{"tx": tx}, tmResult)
 	if err != nil {
 		panic(err)
 	}
@@ -178,7 +177,7 @@ func TestURIBroadcastTxCommit(t *testing.T) {
 func TestJSONBroadcastTxCommit(t *testing.T) {
 	tmResult := new(ctypes.TMResult)
 	tx := randBytes()
-	_, err := clientJSON.Call("broadcast_tx_commit", []interface{}{tx}, tmResult)
+	_, err := GetJSONClient().Call("broadcast_tx_commit", []interface{}{tx}, tmResult)
 	if err != nil {
 		panic(err)
 	}
@@ -211,13 +210,13 @@ var wsTyp = "JSONRPC"
 
 // make a simple connection to the server
 func TestWSConnect(t *testing.T) {
-	wsc := newWSClient(t)
+	wsc := GetWSClient()
 	wsc.Stop()
 }
 
 // receive a new block message
 func TestWSNewBlock(t *testing.T) {
-	wsc := newWSClient(t)
+	wsc := GetWSClient()
 	eid := types.EventStringNewBlock()
 	subscribe(t, wsc, eid)
 	defer func() {
@@ -225,7 +224,7 @@ func TestWSNewBlock(t *testing.T) {
 		wsc.Stop()
 	}()
 	waitForEvent(t, wsc, eid, true, func() {}, func(eid string, b interface{}) error {
-		fmt.Println("Check:", b)
+		// fmt.Println("Check:", b)
 		return nil
 	})
 }
@@ -235,7 +234,7 @@ func TestWSBlockchainGrowth(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping test in short mode.")
 	}
-	wsc := newWSClient(t)
+	wsc := GetWSClient()
 	eid := types.EventStringNewBlock()
 	subscribe(t, wsc, eid)
 	defer func() {
@@ -263,7 +262,7 @@ func TestWSBlockchainGrowth(t *testing.T) {
 }
 
 func TestWSTxEvent(t *testing.T) {
-	wsc := newWSClient(t)
+	wsc := GetWSClient()
 	tx := randBytes()
 
 	// listen for the tx I am about to submit
@@ -276,7 +275,7 @@ func TestWSTxEvent(t *testing.T) {
 
 	// send an tx
 	tmResult := new(ctypes.TMResult)
-	_, err := clientJSON.Call("broadcast_tx_sync", []interface{}{tx}, tmResult)
+	_, err := GetJSONClient().Call("broadcast_tx_sync", []interface{}{tx}, tmResult)
 	if err != nil {
 		t.Fatal("Error submitting event")
 	}
@@ -341,7 +340,7 @@ var testCasesUnsafeSetConfig = [][]string{
 func TestURIUnsafeSetConfig(t *testing.T) {
 	for _, testCase := range testCasesUnsafeSetConfig {
 		tmResult := new(ctypes.TMResult)
-		_, err := clientURI.Call("unsafe_set_config", map[string]interface{}{
+		_, err := GetURIClient().Call("unsafe_set_config", map[string]interface{}{
 			"type":  testCase[0],
 			"key":   testCase[1],
 			"value": testCase[2],
@@ -356,7 +355,7 @@ func TestURIUnsafeSetConfig(t *testing.T) {
 func TestJSONUnsafeSetConfig(t *testing.T) {
 	for _, testCase := range testCasesUnsafeSetConfig {
 		tmResult := new(ctypes.TMResult)
-		_, err := clientJSON.Call("unsafe_set_config", []interface{}{testCase[0], testCase[1], testCase[2]}, tmResult)
+		_, err := GetJSONClient().Call("unsafe_set_config", []interface{}{testCase[0], testCase[1], testCase[2]}, tmResult)
 		if err != nil {
 			panic(err)
 		}
diff --git a/rpc/test/grpc_test.go b/rpc/test/grpc_test.go
index 73ba22a7..fa57b1fb 100644
--- a/rpc/test/grpc_test.go
+++ b/rpc/test/grpc_test.go
@@ -11,7 +11,7 @@ import (
 //-------------------------------------------
 
 func TestBroadcastTx(t *testing.T) {
-	res, err := clientGRPC.BroadcastTx(context.Background(), &core_grpc.RequestBroadcastTx{[]byte("this is a tx")})
+	res, err := GetGRPCClient().BroadcastTx(context.Background(), &core_grpc.RequestBroadcastTx{[]byte("this is a tx")})
 	if err != nil {
 		t.Fatal(err)
 	}
diff --git a/rpc/test/helpers.go b/rpc/test/helpers.go
index 76fcd4b3..287228e0 100644
--- a/rpc/test/helpers.go
+++ b/rpc/test/helpers.go
@@ -1,160 +1,91 @@
 package rpctest
 
 import (
-	"testing"
-	"time"
+	"fmt"
 
-	. "github.com/tendermint/go-common"
+	logger "github.com/tendermint/go-logger"
+
+	abci "github.com/tendermint/abci/types"
 	cfg "github.com/tendermint/go-config"
-	"github.com/tendermint/go-wire"
-
 	client "github.com/tendermint/go-rpc/client"
 	"github.com/tendermint/tendermint/config/tendermint_test"
 	nm "github.com/tendermint/tendermint/node"
-	ctypes "github.com/tendermint/tendermint/rpc/core/types"
-	"github.com/tendermint/tendermint/rpc/grpc"
+	"github.com/tendermint/tendermint/proxy"
+	rpcclient "github.com/tendermint/tendermint/rpc/client"
+	core_grpc "github.com/tendermint/tendermint/rpc/grpc"
+	"github.com/tendermint/tendermint/types"
 )
 
-// global variables for use across all tests
 var (
-	config            cfg.Config
-	node              *nm.Node
-	chainID           string
-	rpcAddr           string
-	requestAddr       string
-	websocketAddr     string
-	websocketEndpoint string
-	grpcAddr          string
-	clientURI         *client.ClientURI
-	clientJSON        *client.ClientJSONRPC
-	clientGRPC        core_grpc.BroadcastAPIClient
+	config cfg.Config
+	node   *nm.Node
 )
 
-// initialize config and create new node
-func init() {
-	config = tendermint_test.ResetConfig("rpc_test_client_test")
-	chainID = config.GetString("chain_id")
-	rpcAddr = config.GetString("rpc_laddr")
-	grpcAddr = config.GetString("grpc_laddr")
-	requestAddr = rpcAddr
-	websocketAddr = rpcAddr
-	websocketEndpoint = "/websocket"
+const tmLogLevel = "error"
 
-	clientURI = client.NewClientURI(requestAddr)
-	clientJSON = client.NewClientJSONRPC(requestAddr)
-	clientGRPC = core_grpc.StartGRPCClient(grpcAddr)
-
-	// TODO: change consensus/state.go timeouts to be shorter
-
-	// start a node
-	ready := make(chan struct{})
-	go newNode(ready)
-	<-ready
+// GetConfig returns a config for the test cases as a singleton
+func GetConfig() cfg.Config {
+	if config == nil {
+		config = tendermint_test.ResetConfig("rpc_test_client_test")
+		// Shut up the logging
+		logger.SetLogLevel(tmLogLevel)
+	}
+	return config
 }
 
-// create a new node and sleep forever
-func newNode(ready chan struct{}) {
-	// Create & start node
-	node = nm.NewNodeDefault(config)
-	node.Start()
-
-	time.Sleep(time.Second)
-
-	ready <- struct{}{}
-
-	// Sleep forever
-	ch := make(chan struct{})
-	<-ch
+// GetClient gets a rpc client pointing to the test tendermint rpc
+func GetClient() *rpcclient.HTTPClient {
+	rpcAddr := GetConfig().GetString("rpc_laddr")
+	return rpcclient.New(rpcAddr, "/websocket")
 }
 
-//--------------------------------------------------------------------------------
-// Utilities for testing the websocket service
+// GetURIClient gets a uri client pointing to the test tendermint rpc
+func GetURIClient() *client.ClientURI {
+	rpcAddr := GetConfig().GetString("rpc_laddr")
+	return client.NewClientURI(rpcAddr)
+}
 
-// create a new connection
-func newWSClient(t *testing.T) *client.WSClient {
-	wsc := client.NewWSClient(websocketAddr, websocketEndpoint)
+// GetJSONClient gets a http/json client pointing to the test tendermint rpc
+func GetJSONClient() *client.ClientJSONRPC {
+	rpcAddr := GetConfig().GetString("rpc_laddr")
+	return client.NewClientJSONRPC(rpcAddr)
+}
+
+func GetGRPCClient() core_grpc.BroadcastAPIClient {
+	grpcAddr := config.GetString("grpc_laddr")
+	return core_grpc.StartGRPCClient(grpcAddr)
+}
+
+func GetWSClient() *client.WSClient {
+	rpcAddr := GetConfig().GetString("rpc_laddr")
+	wsc := client.NewWSClient(rpcAddr, "/websocket")
 	if _, err := wsc.Start(); err != nil {
 		panic(err)
 	}
 	return wsc
 }
 
-// subscribe to an event
-func subscribe(t *testing.T, wsc *client.WSClient, eventid string) {
-	if err := wsc.Subscribe(eventid); err != nil {
-		panic(err)
-	}
+// StartTendermint starts a test tendermint server in a go routine and returns when it is initialized
+// TODO: can one pass an Application in????
+func StartTendermint(app abci.Application) *nm.Node {
+	// start a node
+	fmt.Println("Starting Tendermint...")
+
+	node = NewTendermint(app)
+	fmt.Println("Tendermint running!")
+	return node
 }
 
-// unsubscribe from an event
-func unsubscribe(t *testing.T, wsc *client.WSClient, eventid string) {
-	if err := wsc.Unsubscribe(eventid); err != nil {
-		panic(err)
-	}
+// NewTendermint creates a new tendermint server and sleeps forever
+func NewTendermint(app abci.Application) *nm.Node {
+	// Create & start node
+	config := GetConfig()
+	privValidatorFile := config.GetString("priv_validator_file")
+	privValidator := types.LoadOrGenPrivValidator(privValidatorFile)
+	papp := proxy.NewLocalClientCreator(app)
+	node := nm.NewNode(config, privValidator, papp)
+
+	// node.Start now does everything including the RPC server
+	node.Start()
+	return node
 }
-
-// wait for an event; do things that might trigger events, and check them when they are received
-// the check function takes an event id and the byte slice read off the ws
-func waitForEvent(t *testing.T, wsc *client.WSClient, eventid string, dieOnTimeout bool, f func(), check func(string, interface{}) error) {
-	// go routine to wait for webscoket msg
-	goodCh := make(chan interface{})
-	errCh := make(chan error)
-
-	// Read message
-	go func() {
-		var err error
-	LOOP:
-		for {
-			select {
-			case r := <-wsc.ResultsCh:
-				result := new(ctypes.TMResult)
-				wire.ReadJSONPtr(result, r, &err)
-				if err != nil {
-					errCh <- err
-					break LOOP
-				}
-				event, ok := (*result).(*ctypes.ResultEvent)
-				if ok && event.Name == eventid {
-					goodCh <- event.Data
-					break LOOP
-				}
-			case err := <-wsc.ErrorsCh:
-				errCh <- err
-				break LOOP
-			case <-wsc.Quit:
-				break LOOP
-			}
-		}
-	}()
-
-	// do stuff (transactions)
-	f()
-
-	// wait for an event or timeout
-	timeout := time.NewTimer(10 * time.Second)
-	select {
-	case <-timeout.C:
-		if dieOnTimeout {
-			wsc.Stop()
-			panic(Fmt("%s event was not received in time", eventid))
-		}
-		// else that's great, we didn't hear the event
-		// and we shouldn't have
-	case eventData := <-goodCh:
-		if dieOnTimeout {
-			// message was received and expected
-			// run the check
-			if err := check(eventid, eventData); err != nil {
-				panic(err) // Show the stack trace.
-			}
-		} else {
-			wsc.Stop()
-			panic(Fmt("%s event was not expected", eventid))
-		}
-	case err := <-errCh:
-		panic(err) // Show the stack trace.
-
-	}
-}
-
-//--------------------------------------------------------------------------------
diff --git a/rpc/test/helpers_old.go b/rpc/test/helpers_old.go
new file mode 100644
index 00000000..8795fb5d
--- /dev/null
+++ b/rpc/test/helpers_old.go
@@ -0,0 +1,157 @@
+package rpctest
+
+import (
+	"testing"
+	"time"
+
+	. "github.com/tendermint/go-common"
+	"github.com/tendermint/go-wire"
+
+	client "github.com/tendermint/go-rpc/client"
+	ctypes "github.com/tendermint/tendermint/rpc/core/types"
+)
+
+/*
+// global variables for use across all tests
+var (
+	config            cfg.Config
+	node              *nm.Node
+	chainID           string
+	rpcAddr           string
+	requestAddr       string
+	websocketAddr     string
+	websocketEndpoint string
+	grpcAddr          string
+	clientURI         *client.ClientURI
+	clientJSON        *client.ClientJSONRPC
+	clientGRPC        core_grpc.BroadcastAPIClient
+)
+
+// initialize config and create new node
+func init() {
+	config = tendermint_test.ResetConfig("rpc_test_client_test")
+	chainID = config.GetString("chain_id")
+	rpcAddr = config.GetString("rpc_laddr")
+	grpcAddr = config.GetString("grpc_laddr")
+	requestAddr = rpcAddr
+	websocketAddr = rpcAddr
+	websocketEndpoint = "/websocket"
+
+	clientURI = client.NewClientURI(requestAddr)
+	clientJSON = client.NewClientJSONRPC(requestAddr)
+	clientGRPC = core_grpc.StartGRPCClient(grpcAddr)
+
+	// TODO: change consensus/state.go timeouts to be shorter
+
+	// start a node
+	ready := make(chan struct{})
+	go newNode(ready)
+	<-ready
+}
+
+// create a new node and sleep forever
+func newNode(ready chan struct{}) {
+	// Create & start node
+	node = nm.NewNodeDefault(config)
+	node.Start()
+
+	time.Sleep(time.Second)
+
+	ready <- struct{}{}
+
+	// Sleep forever
+	ch := make(chan struct{})
+	<-ch
+}
+
+//--------------------------------------------------------------------------------
+// Utilities for testing the websocket service
+
+// create a new connection
+func newWSClient(t *testing.T) *client.WSClient {
+	wsc := client.NewWSClient(websocketAddr, websocketEndpoint)
+	if _, err := wsc.Start(); err != nil {
+		panic(err)
+	}
+	return wsc
+}
+*/
+// subscribe to an event
+func subscribe(t *testing.T, wsc *client.WSClient, eventid string) {
+	if err := wsc.Subscribe(eventid); err != nil {
+		panic(err)
+	}
+}
+
+// unsubscribe from an event
+func unsubscribe(t *testing.T, wsc *client.WSClient, eventid string) {
+	if err := wsc.Unsubscribe(eventid); err != nil {
+		panic(err)
+	}
+}
+
+// wait for an event; do things that might trigger events, and check them when they are received
+// the check function takes an event id and the byte slice read off the ws
+func waitForEvent(t *testing.T, wsc *client.WSClient, eventid string, dieOnTimeout bool, f func(), check func(string, interface{}) error) {
+	// go routine to wait for webscoket msg
+	goodCh := make(chan interface{})
+	errCh := make(chan error)
+
+	// Read message
+	go func() {
+		var err error
+	LOOP:
+		for {
+			select {
+			case r := <-wsc.ResultsCh:
+				result := new(ctypes.TMResult)
+				wire.ReadJSONPtr(result, r, &err)
+				if err != nil {
+					errCh <- err
+					break LOOP
+				}
+				event, ok := (*result).(*ctypes.ResultEvent)
+				if ok && event.Name == eventid {
+					goodCh <- event.Data
+					break LOOP
+				}
+			case err := <-wsc.ErrorsCh:
+				errCh <- err
+				break LOOP
+			case <-wsc.Quit:
+				break LOOP
+			}
+		}
+	}()
+
+	// do stuff (transactions)
+	f()
+
+	// wait for an event or timeout
+	timeout := time.NewTimer(10 * time.Second)
+	select {
+	case <-timeout.C:
+		if dieOnTimeout {
+			wsc.Stop()
+			panic(Fmt("%s event was not received in time", eventid))
+		}
+		// else that's great, we didn't hear the event
+		// and we shouldn't have
+	case eventData := <-goodCh:
+		if dieOnTimeout {
+			// message was received and expected
+			// run the check
+			if err := check(eventid, eventData); err != nil {
+				panic(err) // Show the stack trace.
+			}
+		} else {
+			wsc.Stop()
+			panic(Fmt("%s event was not expected", eventid))
+		}
+	case err := <-errCh:
+		panic(err) // Show the stack trace.
+
+	}
+}
+
+//--------------------------------------------------------------------------------
diff --git a/rpc/test/main_test.go b/rpc/test/main_test.go
new file mode 100644
index 00000000..e78cd17a
--- /dev/null
+++ b/rpc/test/main_test.go
@@ -0,0 +1,33 @@
+/*
+package tests contain integration tests and helper functions for testing
+the RPC interface
+
+In particular, it allows us to spin up a tendermint node in process, with
+a live RPC server, which we can use to verify our rpc calls.  It provides
+all data structures, enabling us to do more complex tests (like node_test.go)
+that introspect the blocks themselves to validate signatures and the like.
+
+It currently only spins up one node, it would be interesting to expand it
+to multiple nodes to see the real effects of validating partially signed
+blocks.
+*/
+package rpctest
+
+import (
+	"os"
+	"testing"
+
+	"github.com/tendermint/abci/example/dummy"
+)
+
+func TestMain(m *testing.M) {
+	// start a tendermint node (and merkleeyes) in the background to test against
+	app := dummy.NewDummyApplication()
+	node := StartTendermint(app)
+	code := m.Run()
+
+	// and shut down proper at the end
+	node.Stop()
+	node.Wait()
+	os.Exit(code)
+}
diff --git a/rpc/test/rpc_test.go b/rpc/test/rpc_test.go
new file mode 100644
index 00000000..3896fffd
--- /dev/null
+++ b/rpc/test/rpc_test.go
@@ -0,0 +1,130 @@
+package rpctest
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	// "github.com/stretchr/testify/require"
+	// merkle "github.com/tendermint/go-merkle"
+	// "github.com/tendermint/tendermint/types"
+)
+
+// Make sure status is correct (we connect properly)
+func TestStatus(t *testing.T) {
+	c := GetClient()
+	status, err := c.Status()
+	if assert.Nil(t, err) {
+		assert.Equal(t, GetConfig().GetString("chain_id"), status.NodeInfo.Network)
+	}
+}
+
+/*
+// Make some app checks
+func TestAppCalls(t *testing.T) {
+	assert, require := assert.New(t), require.New(t)
+	c := GetClient()
+	_, err := c.Block(1)
+	assert.NotNil(err) // no block yet
+	k, v, tx := TestTxKV()
+	_, err = c.BroadcastTxCommit(tx)
+	require.Nil(err)
+	// wait before querying
+	time.Sleep(time.Second * 2)
+	qres, err := c.ABCIQuery("/key", k, false)
+	if assert.Nil(err) && assert.True(qres.Response.Code.IsOK()) {
+		data := qres.Response
+		// assert.Equal(k, data.GetKey())  // only returned for proofs
+		assert.Equal(v, data.GetValue())
+	}
+	// and we can even check the block is added
+	block, err := c.Block(3)
+	assert.Nil(err) // now it's good :)
+	appHash := block.BlockMeta.Header.AppHash
+	assert.True(len(appHash) > 0)
+
+	// and get the corresponding commit with the same apphash
+	commit, err := c.Commit(3)
+	assert.Nil(err) // now it's good :)
+	cappHash := commit.Header.AppHash
+	assert.Equal(appHash, cappHash)
+	assert.NotNil(commit.Commit)
+
+	// compare the commits (note Commit(2) has commit from Block(3))
+	commit2, err := c.Commit(2)
+	assert.Nil(err) // now it's good :)
+	assert.Equal(block.Block.LastCommit, commit2.Commit)
+
+	// and we got a proof that works!
+	pres, err := c.ABCIQuery("/key", k, true)
+	if assert.Nil(err) && assert.True(pres.Response.Code.IsOK()) {
+		proof, err := merkle.ReadProof(pres.Response.GetProof())
+		if assert.Nil(err) {
+			key := pres.Response.GetKey()
+			value := pres.Response.GetValue()
+			assert.Equal(appHash, proof.RootHash)
+			valid := proof.Verify(key, value, appHash)
+			assert.True(valid)
+		}
+	}
+}
+
+// run most calls just to make sure no syntax errors
+func TestNoErrors(t *testing.T) {
+	assert := assert.New(t)
+	c := GetClient()
+	_, err := c.NetInfo()
+	assert.Nil(err)
+	_, err = c.BlockchainInfo(0, 4)
+	assert.Nil(err)
+	// TODO: check with a valid height
+	_, err = c.Block(1000)
+	assert.NotNil(err)
+	// maybe this is an error???
+	// _, err = c.DialSeeds([]string{"one", "two"})
+	// assert.Nil(err)
+	gen, err := c.Genesis()
+	if assert.Nil(err) {
+		assert.Equal(GetConfig().GetString("chain_id"), gen.Genesis.ChainID)
+	}
+}
+
+func TestSubscriptions(t *testing.T) {
+	assert, require := assert.New(t), require.New(t)
+	c := GetClient()
+	err := c.StartWebsocket()
+	require.Nil(err)
+	defer c.StopWebsocket()
+
+	// subscribe to a transaction event
+	_, _, tx := TestTxKV()
+	// this causes a panic in tendermint core!!!
+	eventType := types.EventStringTx(types.Tx(tx))
+	c.Subscribe(eventType)
+	read := 0
+
+	// set up a listener
+	r, e := c.GetEventChannels()
+	go func() {
+		// read one event in the background
+		select {
+		case <-r:
+			// TODO: actually parse this or something
+			read += 1
+		case err := <-e:
+			panic(err)
+		}
+	}()
+
+	// make sure nothing has happened yet.
+	assert.Equal(0, read)
+
+	// send a tx and wait for it to propogate
+	_, err = c.BroadcastTxCommit(tx)
+	assert.Nil(err, string(tx))
+	// wait before querying
+	time.Sleep(time.Second)
+
+	// now make sure the event arrived
+	assert.Equal(1, read)
+}
+*/

From 2c75c9daf985bd918d461402030159f33884e6da Mon Sep 17 00:00:00 2001
From: Ethan Frey <ethanfrey@users.noreply.github.com>
Date: Tue, 21 Feb 2017 19:57:10 +0100
Subject: [PATCH 073/132] Clean up tests, remove panics

---
 rpc/test/client_test.go | 162 ++++++++++++++--------------------------
 rpc/test/grpc_test.go   |  14 ++--
 rpc/test/helpers_old.go |  15 ++--
 rpc/test/rpc_test.go    |   3 +-
 4 files changed, 67 insertions(+), 127 deletions(-)

diff --git a/rpc/test/client_test.go b/rpc/test/client_test.go
index a173bf29..1141de4e 100644
--- a/rpc/test/client_test.go
+++ b/rpc/test/client_test.go
@@ -8,6 +8,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	abci "github.com/tendermint/abci/types"
 	. "github.com/tendermint/go-common"
 	ctypes "github.com/tendermint/tendermint/rpc/core/types"
@@ -25,18 +27,14 @@ import (
 func TestURIStatus(t *testing.T) {
 	tmResult := new(ctypes.TMResult)
 	_, err := GetURIClient().Call("status", map[string]interface{}{}, tmResult)
-	if err != nil {
-		panic(err)
-	}
+	require.Nil(t, err)
 	testStatus(t, tmResult)
 }
 
 func TestJSONStatus(t *testing.T) {
 	tmResult := new(ctypes.TMResult)
 	_, err := GetJSONClient().Call("status", []interface{}{}, tmResult)
-	if err != nil {
-		panic(err)
-	}
+	require.Nil(t, err)
 	testStatus(t, tmResult)
 }
 
@@ -45,23 +43,18 @@ func testStatus(t *testing.T, statusI interface{}) {
 
 	tmRes := statusI.(*ctypes.TMResult)
 	status := (*tmRes).(*ctypes.ResultStatus)
-	if status.NodeInfo.Network != chainID {
-		panic(Fmt("ChainID mismatch: got %s expected %s",
-			status.NodeInfo.Network, chainID))
-	}
+	assert.Equal(t, chainID, status.NodeInfo.Network)
 }
 
 //--------------------------------------------------------------------------------
 // broadcast tx sync
 
 // random bytes (excluding byte('='))
-func randBytes() []byte {
+func randBytes(t *testing.T) []byte {
 	n := rand.Intn(10) + 2
 	buf := make([]byte, n)
 	_, err := crand.Read(buf)
-	if err != nil {
-		panic(err)
-	}
+	require.Nil(t, err)
 	return bytes.Replace(buf, []byte("="), []byte{100}, -1)
 }
 
@@ -69,11 +62,9 @@ func TestURIBroadcastTxSync(t *testing.T) {
 	config.Set("block_size", 0)
 	defer config.Set("block_size", -1)
 	tmResult := new(ctypes.TMResult)
-	tx := randBytes()
+	tx := randBytes(t)
 	_, err := GetURIClient().Call("broadcast_tx_sync", map[string]interface{}{"tx": tx}, tmResult)
-	if err != nil {
-		panic(err)
-	}
+	require.Nil(t, err)
 	testBroadcastTxSync(t, tmResult, tx)
 }
 
@@ -81,84 +72,64 @@ func TestJSONBroadcastTxSync(t *testing.T) {
 	config.Set("block_size", 0)
 	defer config.Set("block_size", -1)
 	tmResult := new(ctypes.TMResult)
-	tx := randBytes()
+	tx := randBytes(t)
 	_, err := GetJSONClient().Call("broadcast_tx_sync", []interface{}{tx}, tmResult)
-	if err != nil {
-		panic(err)
-	}
+	require.Nil(t, err)
 	testBroadcastTxSync(t, tmResult, tx)
 }
 
 func testBroadcastTxSync(t *testing.T, resI interface{}, tx []byte) {
 	tmRes := resI.(*ctypes.TMResult)
 	res := (*tmRes).(*ctypes.ResultBroadcastTx)
-	if res.Code != abci.CodeType_OK {
-		panic(Fmt("BroadcastTxSync got non-zero exit code: %v. %X; %s", res.Code, res.Data, res.Log))
-	}
+	require.Equal(t, abci.CodeType_OK, res.Code)
 	mem := node.MempoolReactor().Mempool
-	if mem.Size() != 1 {
-		panic(Fmt("Mempool size should have been 1. Got %d", mem.Size()))
-	}
-
+	require.Equal(t, 1, mem.Size())
 	txs := mem.Reap(1)
-	if !bytes.Equal(txs[0], tx) {
-		panic(Fmt("Tx in mempool does not match test tx. Got %X, expected %X", txs[0], tx))
-	}
-
+	require.EqualValues(t, tx, txs[0])
 	mem.Flush()
 }
 
 //--------------------------------------------------------------------------------
 // query
 
-func testTxKV() ([]byte, []byte, []byte) {
-	k := randBytes()
-	v := randBytes()
+func testTxKV(t *testing.T) ([]byte, []byte, []byte) {
+	k := randBytes(t)
+	v := randBytes(t)
 	return k, v, []byte(Fmt("%s=%s", k, v))
 }
 
-func sendTx() ([]byte, []byte) {
+func sendTx(t *testing.T) ([]byte, []byte) {
 	tmResult := new(ctypes.TMResult)
-	k, v, tx := testTxKV()
+	k, v, tx := testTxKV(t)
 	_, err := GetJSONClient().Call("broadcast_tx_commit", []interface{}{tx}, tmResult)
-	if err != nil {
-		panic(err)
-	}
+	require.Nil(t, err)
 	return k, v
 }
 
 func TestURIABCIQuery(t *testing.T) {
-	k, v := sendTx()
+	k, v := sendTx(t)
 	time.Sleep(time.Second)
 	tmResult := new(ctypes.TMResult)
 	_, err := GetURIClient().Call("abci_query", map[string]interface{}{"path": "", "data": k, "prove": false}, tmResult)
-	if err != nil {
-		panic(err)
-	}
+	require.Nil(t, err)
 	testABCIQuery(t, tmResult, v)
 }
 
 func TestJSONABCIQuery(t *testing.T) {
-	k, v := sendTx()
+	k, v := sendTx(t)
 	tmResult := new(ctypes.TMResult)
 	_, err := GetJSONClient().Call("abci_query", []interface{}{"", k, false}, tmResult)
-	if err != nil {
-		panic(err)
-	}
+	require.Nil(t, err)
 	testABCIQuery(t, tmResult, v)
 }
 
 func testABCIQuery(t *testing.T, statusI interface{}, value []byte) {
 	tmRes := statusI.(*ctypes.TMResult)
 	resQuery := (*tmRes).(*ctypes.ResultABCIQuery)
-	if !resQuery.Response.Code.IsOK() {
-		panic(Fmt("Query returned an err: %v", resQuery))
-	}
+	require.EqualValues(t, 0, resQuery.Response.Code)
 
 	// XXX: specific to value returned by the dummy
-	if len(resQuery.Response.Value) == 0 {
-		panic(Fmt("Query error. Found no value"))
-	}
+	require.NotEqual(t, 0, len(resQuery.Response.Value))
 }
 
 //--------------------------------------------------------------------------------
@@ -166,40 +137,30 @@ func testABCIQuery(t *testing.T, statusI interface{}, value []byte) {
 
 func TestURIBroadcastTxCommit(t *testing.T) {
 	tmResult := new(ctypes.TMResult)
-	tx := randBytes()
+	tx := randBytes(t)
 	_, err := GetURIClient().Call("broadcast_tx_commit", map[string]interface{}{"tx": tx}, tmResult)
-	if err != nil {
-		panic(err)
-	}
+	require.Nil(t, err)
 	testBroadcastTxCommit(t, tmResult, tx)
 }
 
 func TestJSONBroadcastTxCommit(t *testing.T) {
 	tmResult := new(ctypes.TMResult)
-	tx := randBytes()
+	tx := randBytes(t)
 	_, err := GetJSONClient().Call("broadcast_tx_commit", []interface{}{tx}, tmResult)
-	if err != nil {
-		panic(err)
-	}
+	require.Nil(t, err)
 	testBroadcastTxCommit(t, tmResult, tx)
 }
 
 func testBroadcastTxCommit(t *testing.T, resI interface{}, tx []byte) {
+	require := require.New(t)
 	tmRes := resI.(*ctypes.TMResult)
 	res := (*tmRes).(*ctypes.ResultBroadcastTxCommit)
 	checkTx := res.CheckTx
-	if checkTx.Code != abci.CodeType_OK {
-		panic(Fmt("BroadcastTxCommit got non-zero exit code from CheckTx: %v. %X; %s", checkTx.Code, checkTx.Data, checkTx.Log))
-	}
+	require.Equal(abci.CodeType_OK, checkTx.Code)
 	deliverTx := res.DeliverTx
-	if deliverTx.Code != abci.CodeType_OK {
-		panic(Fmt("BroadcastTxCommit got non-zero exit code from CheckTx: %v. %X; %s", deliverTx.Code, deliverTx.Data, deliverTx.Log))
-	}
+	require.Equal(abci.CodeType_OK, deliverTx.Code)
 	mem := node.MempoolReactor().Mempool
-	if mem.Size() != 0 {
-		panic(Fmt("Mempool size should have been 0. Got %d", mem.Size()))
-	}
-
+	require.Equal(0, mem.Size())
 	// TODO: find tx in block
 }
 
@@ -218,9 +179,10 @@ func TestWSConnect(t *testing.T) {
 func TestWSNewBlock(t *testing.T) {
 	wsc := GetWSClient()
 	eid := types.EventStringNewBlock()
-	subscribe(t, wsc, eid)
+	require.Nil(t, wsc.Subscribe(eid))
+
 	defer func() {
-		unsubscribe(t, wsc, eid)
+		require.Nil(t, wsc.Unsubscribe(eid))
 		wsc.Stop()
 	}()
 	waitForEvent(t, wsc, eid, true, func() {}, func(eid string, b interface{}) error {
@@ -236,9 +198,10 @@ func TestWSBlockchainGrowth(t *testing.T) {
 	}
 	wsc := GetWSClient()
 	eid := types.EventStringNewBlock()
-	subscribe(t, wsc, eid)
+	require.Nil(t, wsc.Subscribe(eid))
+
 	defer func() {
-		unsubscribe(t, wsc, eid)
+		require.Nil(t, wsc.Unsubscribe(eid))
 		wsc.Stop()
 	}()
 
@@ -262,35 +225,29 @@ func TestWSBlockchainGrowth(t *testing.T) {
 }
 
 func TestWSTxEvent(t *testing.T) {
+	require := require.New(t)
 	wsc := GetWSClient()
-	tx := randBytes()
+	tx := randBytes(t)
 
 	// listen for the tx I am about to submit
 	eid := types.EventStringTx(types.Tx(tx))
-	subscribe(t, wsc, eid)
+	require.Nil(wsc.Subscribe(eid))
+
 	defer func() {
-		unsubscribe(t, wsc, eid)
+		require.Nil(wsc.Unsubscribe(eid))
 		wsc.Stop()
 	}()
 
 	// send an tx
 	tmResult := new(ctypes.TMResult)
 	_, err := GetJSONClient().Call("broadcast_tx_sync", []interface{}{tx}, tmResult)
-	if err != nil {
-		t.Fatal("Error submitting event")
-	}
+	require.Nil(err)
 
 	waitForEvent(t, wsc, eid, true, func() {}, func(eid string, b interface{}) error {
 		evt, ok := b.(types.EventDataTx)
-		if !ok {
-			t.Fatal("Got wrong event type", b)
-		}
-		if bytes.Compare([]byte(evt.Tx), tx) != 0 {
-			t.Error("Event returned different tx")
-		}
-		if evt.Code != abci.CodeType_OK {
-			t.Error("Event returned tx error code", evt.Code)
-		}
+		require.True(ok, "Got wrong event type: %#v", b)
+		require.Equal(tx, []byte(evt.Tx), "Returned different tx")
+		require.Equal(abci.CodeType_OK, evt.Code)
 		return nil
 	})
 }
@@ -345,9 +302,7 @@ func TestURIUnsafeSetConfig(t *testing.T) {
 			"key":   testCase[1],
 			"value": testCase[2],
 		}, tmResult)
-		if err != nil {
-			panic(err)
-		}
+		require.Nil(t, err)
 	}
 	testUnsafeSetConfig(t)
 }
@@ -356,26 +311,19 @@ func TestJSONUnsafeSetConfig(t *testing.T) {
 	for _, testCase := range testCasesUnsafeSetConfig {
 		tmResult := new(ctypes.TMResult)
 		_, err := GetJSONClient().Call("unsafe_set_config", []interface{}{testCase[0], testCase[1], testCase[2]}, tmResult)
-		if err != nil {
-			panic(err)
-		}
+		require.Nil(t, err)
 	}
 	testUnsafeSetConfig(t)
 }
 
 func testUnsafeSetConfig(t *testing.T) {
+	require := require.New(t)
 	s := config.GetString("key1")
-	if s != stringVal {
-		panic(Fmt("got %v, expected %v", s, stringVal))
-	}
+	require.Equal(stringVal, s)
 
 	i := config.GetInt("key2")
-	if i != intVal {
-		panic(Fmt("got %v, expected %v", i, intVal))
-	}
+	require.Equal(intVal, i)
 
 	b := config.GetBool("key3")
-	if b != boolVal {
-		panic(Fmt("got %v, expected %v", b, boolVal))
-	}
+	require.Equal(boolVal, b)
 }
diff --git a/rpc/test/grpc_test.go b/rpc/test/grpc_test.go
index fa57b1fb..4935a09d 100644
--- a/rpc/test/grpc_test.go
+++ b/rpc/test/grpc_test.go
@@ -5,20 +5,16 @@ import (
 
 	"golang.org/x/net/context"
 
+	"github.com/stretchr/testify/require"
 	"github.com/tendermint/tendermint/rpc/grpc"
 )
 
 //-------------------------------------------
 
 func TestBroadcastTx(t *testing.T) {
+	require := require.New(t)
 	res, err := GetGRPCClient().BroadcastTx(context.Background(), &core_grpc.RequestBroadcastTx{[]byte("this is a tx")})
-	if err != nil {
-		t.Fatal(err)
-	}
-	if res.CheckTx.Code != 0 {
-		t.Fatalf("Non-zero check tx code: %d", res.CheckTx.Code)
-	}
-	if res.DeliverTx.Code != 0 {
-		t.Fatalf("Non-zero append tx code: %d", res.DeliverTx.Code)
-	}
+	require.Nil(err)
+	require.EqualValues(0, res.CheckTx.Code)
+	require.EqualValues(0, res.DeliverTx.Code)
 }
diff --git a/rpc/test/helpers_old.go b/rpc/test/helpers_old.go
index 8795fb5d..a5d2f41d 100644
--- a/rpc/test/helpers_old.go
+++ b/rpc/test/helpers_old.go
@@ -4,6 +4,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/require"
 	. "github.com/tendermint/go-common"
 	"github.com/tendermint/go-wire"
 
@@ -76,18 +77,15 @@ func newWSClient(t *testing.T) *client.WSClient {
 	return wsc
 }
 */
+
 // subscribe to an event
 func subscribe(t *testing.T, wsc *client.WSClient, eventid string) {
-	if err := wsc.Subscribe(eventid); err != nil {
-		panic(err)
-	}
+	require.Nil(t, wsc.Subscribe(eventid))
 }
 
 // unsubscribe from an event
 func unsubscribe(t *testing.T, wsc *client.WSClient, eventid string) {
-	if err := wsc.Unsubscribe(eventid); err != nil {
-		panic(err)
-	}
+	require.Nil(t, wsc.Unsubscribe(eventid))
 }
 
 // wait for an event; do things that might trigger events, and check them when they are received
@@ -141,16 +139,13 @@ func waitForEvent(t *testing.T, wsc *client.WSClient, eventid string, dieOnTimeo
 		if dieOnTimeout {
 			// message was received and expected
 			// run the check
-			if err := check(eventid, eventData); err != nil {
-				panic(err) // Show the stack trace.
-			}
+			require.Nil(t, check(eventid, eventData))
 		} else {
 			wsc.Stop()
 			panic(Fmt("%s event was not expected", eventid))
 		}
 	case err := <-errCh:
 		panic(err) // Show the stack trace.
-
 	}
 }
 
diff --git a/rpc/test/rpc_test.go b/rpc/test/rpc_test.go
index 3896fffd..50a74f37 100644
--- a/rpc/test/rpc_test.go
+++ b/rpc/test/rpc_test.go
@@ -12,9 +12,10 @@ import (
 // Make sure status is correct (we connect properly)
 func TestStatus(t *testing.T) {
 	c := GetClient()
+	chainID := GetConfig().GetString("chain_id")
 	status, err := c.Status()
 	if assert.Nil(t, err) {
-		assert.Equal(t, GetConfig().GetString("chain_id"), status.NodeInfo.Network)
+		assert.Equal(t, chainID, status.NodeInfo.Network)
 	}
 }
 

From c9d36cd71393c13c040375c8029ceab54a97f5ce Mon Sep 17 00:00:00 2001
From: Ethan Frey <ethanfrey@users.noreply.github.com>
Date: Tue, 21 Feb 2017 22:32:17 +0100
Subject: [PATCH 074/132] Add dependencies,  pull out HTTPClient test code

---
 glide.lock                       |  12 ++-
 glide.yaml                       |  10 ++
 rpc/client/app_test.go           |  45 +++++++++
 rpc/client/client.go             |   2 +-
 rpc/client/main_test.go          |  21 +++++
 rpc/{test => client}/rpc_test.go |  29 +++---
 rpc/test/helpers.go              |  71 +++++++++++++++
 rpc/test/helpers_old.go          | 152 -------------------------------
 8 files changed, 174 insertions(+), 168 deletions(-)
 create mode 100644 rpc/client/app_test.go
 create mode 100644 rpc/client/main_test.go
 rename rpc/{test => client}/rpc_test.go (85%)
 delete mode 100644 rpc/test/helpers_old.go

diff --git a/glide.lock b/glide.lock
index c58c2a4d..759553ea 100644
--- a/glide.lock
+++ b/glide.lock
@@ -137,4 +137,14 @@ imports:
   - stats
   - tap
   - transport
-testImports: []
+testImports:
+- name: github.com/tendermint/merkleeyes
+  version: 87b0a111a716f1495f30ce58bd469e36ac220e09
+  subpackages:
+  - app
+- name: github.com/stretchr/testify
+  version: 69483b4bd14f5845b5a1e55bca19e954e827f1d0
+  subpackages:
+  - assert
+  - require
+
diff --git a/glide.yaml b/glide.yaml
index d51903d5..55c85962 100644
--- a/glide.yaml
+++ b/glide.yaml
@@ -37,3 +37,13 @@ import:
 - package: golang.org/x/crypto
   subpackages:
   - ripemd160
+testImport:
+- package: github.com/stretchr/testify
+  version: ^1.1.4
+  subpackages:
+  - assert
+  - require
+- package: github.com/tendermint/merkleeyes
+  version: develop
+  subpackages:
+  - app
diff --git a/rpc/client/app_test.go b/rpc/client/app_test.go
new file mode 100644
index 00000000..b360dba3
--- /dev/null
+++ b/rpc/client/app_test.go
@@ -0,0 +1,45 @@
+package client_test
+
+import (
+	"math/rand"
+
+	meapp "github.com/tendermint/merkleeyes/app"
+
+	wire "github.com/tendermint/go-wire"
+)
+
+// MakeTxKV returns a text transaction, allong with expected key, value pair
+func MakeTxKV() ([]byte, []byte, []byte) {
+	k := RandAsciiBytes(8)
+	v := RandAsciiBytes(8)
+	return k, v, makeSet(k, v)
+}
+
+// blatently copied from merkleeyes/app/app_test.go
+// constructs a "set" transaction
+func makeSet(key, value []byte) []byte {
+	tx := make([]byte, 1+wire.ByteSliceSize(key)+wire.ByteSliceSize(value))
+	buf := tx
+	buf[0] = meapp.WriteSet // Set TypeByte
+	buf = buf[1:]
+	n, err := wire.PutByteSlice(buf, key)
+	if err != nil {
+		panic(err)
+	}
+	buf = buf[n:]
+	n, err = wire.PutByteSlice(buf, value)
+	if err != nil {
+		panic(err)
+	}
+	return tx
+}
+
+const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+
+func RandAsciiBytes(n int) []byte {
+	b := make([]byte, n)
+	for i := range b {
+		b[i] = letterBytes[rand.Intn(len(letterBytes))]
+	}
+	return b
+}
diff --git a/rpc/client/client.go b/rpc/client/client.go
index b2d4ee21..a319b9bc 100644
--- a/rpc/client/client.go
+++ b/rpc/client/client.go
@@ -1,4 +1,4 @@
-package rpcclient
+package client
 
 import (
 	"encoding/json"
diff --git a/rpc/client/main_test.go b/rpc/client/main_test.go
new file mode 100644
index 00000000..5119ac41
--- /dev/null
+++ b/rpc/client/main_test.go
@@ -0,0 +1,21 @@
+package client_test
+
+import (
+	"os"
+	"testing"
+
+	meapp "github.com/tendermint/merkleeyes/app"
+	rpctest "github.com/tendermint/tendermint/rpc/test"
+)
+
+func TestMain(m *testing.M) {
+	// start a tendermint node (and merkleeyes) in the background to test against
+	app := meapp.NewMerkleEyesApp("", 100)
+	node := rpctest.StartTendermint(app)
+	code := m.Run()
+
+	// and shut down proper at the end
+	node.Stop()
+	node.Wait()
+	os.Exit(code)
+}
diff --git a/rpc/test/rpc_test.go b/rpc/client/rpc_test.go
similarity index 85%
rename from rpc/test/rpc_test.go
rename to rpc/client/rpc_test.go
index 50a74f37..6711c220 100644
--- a/rpc/test/rpc_test.go
+++ b/rpc/client/rpc_test.go
@@ -1,32 +1,33 @@
-package rpctest
+package client_test
 
 import (
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
-	// "github.com/stretchr/testify/require"
-	// merkle "github.com/tendermint/go-merkle"
-	// "github.com/tendermint/tendermint/types"
+	"github.com/stretchr/testify/require"
+	merkle "github.com/tendermint/go-merkle"
+	rpctest "github.com/tendermint/tendermint/rpc/test"
+	"github.com/tendermint/tendermint/types"
 )
 
 // Make sure status is correct (we connect properly)
 func TestStatus(t *testing.T) {
-	c := GetClient()
-	chainID := GetConfig().GetString("chain_id")
+	c := rpctest.GetClient()
+	chainID := rpctest.GetConfig().GetString("chain_id")
 	status, err := c.Status()
 	if assert.Nil(t, err) {
 		assert.Equal(t, chainID, status.NodeInfo.Network)
 	}
 }
 
-/*
 // Make some app checks
 func TestAppCalls(t *testing.T) {
 	assert, require := assert.New(t), require.New(t)
-	c := GetClient()
+	c := rpctest.GetClient()
 	_, err := c.Block(1)
 	assert.NotNil(err) // no block yet
-	k, v, tx := TestTxKV()
+	k, v, tx := MakeTxKV()
 	_, err = c.BroadcastTxCommit(tx)
 	require.Nil(err)
 	// wait before querying
@@ -72,7 +73,7 @@ func TestAppCalls(t *testing.T) {
 // run most calls just to make sure no syntax errors
 func TestNoErrors(t *testing.T) {
 	assert := assert.New(t)
-	c := GetClient()
+	c := rpctest.GetClient()
 	_, err := c.NetInfo()
 	assert.Nil(err)
 	_, err = c.BlockchainInfo(0, 4)
@@ -85,19 +86,20 @@ func TestNoErrors(t *testing.T) {
 	// assert.Nil(err)
 	gen, err := c.Genesis()
 	if assert.Nil(err) {
-		assert.Equal(GetConfig().GetString("chain_id"), gen.Genesis.ChainID)
+		chainID := rpctest.GetConfig().GetString("chain_id")
+		assert.Equal(chainID, gen.Genesis.ChainID)
 	}
 }
 
 func TestSubscriptions(t *testing.T) {
 	assert, require := assert.New(t), require.New(t)
-	c := GetClient()
+	c := rpctest.GetClient()
 	err := c.StartWebsocket()
 	require.Nil(err)
 	defer c.StopWebsocket()
 
 	// subscribe to a transaction event
-	_, _, tx := TestTxKV()
+	_, _, tx := MakeTxKV()
 	// this causes a panic in tendermint core!!!
 	eventType := types.EventStringTx(types.Tx(tx))
 	c.Subscribe(eventType)
@@ -128,4 +130,3 @@ func TestSubscriptions(t *testing.T) {
 	// now make sure the event arrived
 	assert.Equal(1, read)
 }
-*/
diff --git a/rpc/test/helpers.go b/rpc/test/helpers.go
index 287228e0..0c80f62a 100644
--- a/rpc/test/helpers.go
+++ b/rpc/test/helpers.go
@@ -2,8 +2,12 @@ package rpctest
 
 import (
 	"fmt"
+	"testing"
+	"time"
 
+	"github.com/stretchr/testify/require"
 	logger "github.com/tendermint/go-logger"
+	wire "github.com/tendermint/go-wire"
 
 	abci "github.com/tendermint/abci/types"
 	cfg "github.com/tendermint/go-config"
@@ -12,6 +16,7 @@ import (
 	nm "github.com/tendermint/tendermint/node"
 	"github.com/tendermint/tendermint/proxy"
 	rpcclient "github.com/tendermint/tendermint/rpc/client"
+	ctypes "github.com/tendermint/tendermint/rpc/core/types"
 	core_grpc "github.com/tendermint/tendermint/rpc/grpc"
 	"github.com/tendermint/tendermint/types"
 )
@@ -89,3 +94,69 @@ func NewTendermint(app abci.Application) *nm.Node {
 	node.Start()
 	return node
 }
+
+//--------------------------------------------------------------------------------
+// Utilities for testing the websocket service
+
+// wait for an event; do things that might trigger events, and check them when they are received
+// the check function takes an event id and the byte slice read off the ws
+func waitForEvent(t *testing.T, wsc *client.WSClient, eventid string, dieOnTimeout bool, f func(), check func(string, interface{}) error) {
+	// go routine to wait for webscoket msg
+	goodCh := make(chan interface{})
+	errCh := make(chan error)
+
+	// Read message
+	go func() {
+		var err error
+	LOOP:
+		for {
+			select {
+			case r := <-wsc.ResultsCh:
+				result := new(ctypes.TMResult)
+				wire.ReadJSONPtr(result, r, &err)
+				if err != nil {
+					errCh <- err
+					break LOOP
+				}
+				event, ok := (*result).(*ctypes.ResultEvent)
+				if ok && event.Name == eventid {
+					goodCh <- event.Data
+					break LOOP
+				}
+			case err := <-wsc.ErrorsCh:
+				errCh <- err
+				break LOOP
+			case <-wsc.Quit:
+				break LOOP
+			}
+		}
+	}()
+
+	// do stuff (transactions)
+	f()
+
+	// wait for an event or timeout
+	timeout := time.NewTimer(10 * time.Second)
+	select {
+	case <-timeout.C:
+		if dieOnTimeout {
+			wsc.Stop()
+			require.True(t, false, "%s event was not received in time", eventid)
+		}
+		// else that's great, we didn't hear the event
+		// and we shouldn't have
+	case eventData := <-goodCh:
+		if dieOnTimeout {
+			// message was received and expected
+			// run the check
+			require.Nil(t, check(eventid, eventData))
+		} else {
+			wsc.Stop()
+			require.True(t, false, "%s event was not expected", eventid)
+		}
+	case err := <-errCh:
+		panic(err) // Show the stack trace.
+	}
+}
+
+//--------------------------------------------------------------------------------
diff --git a/rpc/test/helpers_old.go b/rpc/test/helpers_old.go
deleted file mode 100644
index a5d2f41d..00000000
--- a/rpc/test/helpers_old.go
+++ /dev/null
@@ -1,152 +0,0 @@
-package rpctest
-
-import (
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/require"
-	. "github.com/tendermint/go-common"
-	"github.com/tendermint/go-wire"
-
-	client "github.com/tendermint/go-rpc/client"
-	ctypes "github.com/tendermint/tendermint/rpc/core/types"
-)
-
-/*
-// global variables for use across all tests
-var (
-	config            cfg.Config
-	node              *nm.Node
-	chainID           string
-	rpcAddr           string
-	requestAddr       string
-	websocketAddr     string
-	websocketEndpoint string
-	grpcAddr          string
-	clientURI         *client.ClientURI
-	clientJSON        *client.ClientJSONRPC
-	clientGRPC        core_grpc.BroadcastAPIClient
-)
-
-// initialize config and create new node
-func init() {
-	config = tendermint_test.ResetConfig("rpc_test_client_test")
-	chainID = config.GetString("chain_id")
-	rpcAddr = config.GetString("rpc_laddr")
-	grpcAddr = config.GetString("grpc_laddr")
-	requestAddr = rpcAddr
-	websocketAddr = rpcAddr
-	websocketEndpoint = "/websocket"
-
-	clientURI = client.NewClientURI(requestAddr)
-	clientJSON = client.NewClientJSONRPC(requestAddr)
-	clientGRPC = core_grpc.StartGRPCClient(grpcAddr)
-
-	// TODO: change consensus/state.go timeouts to be shorter
-
-	// start a node
-	ready := make(chan struct{})
-	go newNode(ready)
-	<-ready
-}
-
-// create a new node and sleep forever
-func newNode(ready chan struct{}) {
-	// Create & start node
-	node = nm.NewNodeDefault(config)
-	node.Start()
-
-	time.Sleep(time.Second)
-
-	ready <- struct{}{}
-
-	// Sleep forever
-	ch := make(chan struct{})
-	<-ch
-}
-
-//--------------------------------------------------------------------------------
-// Utilities for testing the websocket service
-
-// create a new connection
-func newWSClient(t *testing.T) *client.WSClient {
-	wsc := client.NewWSClient(websocketAddr, websocketEndpoint)
-	if _, err := wsc.Start(); err != nil {
-		panic(err)
-	}
-	return wsc
-}
-*/
-
-// subscribe to an event
-func subscribe(t *testing.T, wsc *client.WSClient, eventid string) {
-	require.Nil(t, wsc.Subscribe(eventid))
-}
-
-// unsubscribe from an event
-func unsubscribe(t *testing.T, wsc *client.WSClient, eventid string) {
-	require.Nil(t, wsc.Unsubscribe(eventid))
-}
-
-// wait for an event; do things that might trigger events, and check them when they are received
-// the check function takes an event id and the byte slice read off the ws
-func waitForEvent(t *testing.T, wsc *client.WSClient, eventid string, dieOnTimeout bool, f func(), check func(string, interface{}) error) {
-	// go routine to wait for webscoket msg
-	goodCh := make(chan interface{})
-	errCh := make(chan error)
-
-	// Read message
-	go func() {
-		var err error
-	LOOP:
-		for {
-			select {
-			case r := <-wsc.ResultsCh:
-				result := new(ctypes.TMResult)
-				wire.ReadJSONPtr(result, r, &err)
-				if err != nil {
-					errCh <- err
-					break LOOP
-				}
-				event, ok := (*result).(*ctypes.ResultEvent)
-				if ok && event.Name == eventid {
-					goodCh <- event.Data
-					break LOOP
-				}
-			case err := <-wsc.ErrorsCh:
-				errCh <- err
-				break LOOP
-			case <-wsc.Quit:
-				break LOOP
-			}
-		}
-	}()
-
-	// do stuff (transactions)
-	f()
-
-	// wait for an event or timeout
-	timeout := time.NewTimer(10 * time.Second)
-	select {
-	case <-timeout.C:
-		if dieOnTimeout {
-			wsc.Stop()
-			panic(Fmt("%s event was not received in time", eventid))
-		}
-		// else that's great, we didn't hear the event
-		// and we shouldn't have
-	case eventData := <-goodCh:
-		if dieOnTimeout {
-			// message was received and expected
-			// run the check
-			require.Nil(t, check(eventid, eventData))
-		} else {
-			wsc.Stop()
-			panic(Fmt("%s event was not expected", eventid))
-		}
-	case err := <-errCh:
-		panic(err) // Show the stack trace.
-	}
-}
-
-//--------------------------------------------------------------------------------

From 5ea3f24304cd4c7c837f13eb5ba0a8ba79f87793 Mon Sep 17 00:00:00 2001
From: Ethan Frey <ethanfrey@users.noreply.github.com>
Date: Tue, 21 Feb 2017 22:39:04 +0100
Subject: [PATCH 075/132] test info

---
 rpc/client/rpc_test.go | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/rpc/client/rpc_test.go b/rpc/client/rpc_test.go
index 6711c220..3b6f0260 100644
--- a/rpc/client/rpc_test.go
+++ b/rpc/client/rpc_test.go
@@ -1,6 +1,7 @@
 package client_test
 
 import (
+	"strings"
 	"testing"
 	"time"
 
@@ -21,6 +22,17 @@ func TestStatus(t *testing.T) {
 	}
 }
 
+// Make sure info is correct (we connect properly)
+func TestInfo(t *testing.T) {
+	c := rpctest.GetClient()
+	status, err := c.Status()
+	require.Nil(t, err)
+	info, err := c.ABCIInfo()
+	require.Nil(t, err)
+	assert.EqualValues(t, status.LatestBlockHeight, info.Response.LastBlockHeight)
+	assert.True(t, strings.HasPrefix(info.Response.Data, "size:"))
+}
+
 // Make some app checks
 func TestAppCalls(t *testing.T) {
 	assert, require := assert.New(t), require.New(t)

From 64feaa05f4852ff0155c461e4bf831cd78dfe2ff Mon Sep 17 00:00:00 2001
From: Ethan Frey <ethanfrey@users.noreply.github.com>
Date: Wed, 22 Feb 2017 13:46:55 +0100
Subject: [PATCH 076/132] Full test coverage for rpc client

---
 rpc/client/rpc_test.go | 93 ++++++++++++++++++++++++++++--------------
 1 file changed, 62 insertions(+), 31 deletions(-)

diff --git a/rpc/client/rpc_test.go b/rpc/client/rpc_test.go
index 3b6f0260..4aca05e8 100644
--- a/rpc/client/rpc_test.go
+++ b/rpc/client/rpc_test.go
@@ -17,22 +17,59 @@ func TestStatus(t *testing.T) {
 	c := rpctest.GetClient()
 	chainID := rpctest.GetConfig().GetString("chain_id")
 	status, err := c.Status()
-	if assert.Nil(t, err) {
-		assert.Equal(t, chainID, status.NodeInfo.Network)
-	}
+	require.Nil(t, err, "%+v", err)
+	assert.Equal(t, chainID, status.NodeInfo.Network)
 }
 
 // Make sure info is correct (we connect properly)
 func TestInfo(t *testing.T) {
 	c := rpctest.GetClient()
 	status, err := c.Status()
-	require.Nil(t, err)
+	require.Nil(t, err, "%+v", err)
 	info, err := c.ABCIInfo()
-	require.Nil(t, err)
+	require.Nil(t, err, "%+v", err)
 	assert.EqualValues(t, status.LatestBlockHeight, info.Response.LastBlockHeight)
 	assert.True(t, strings.HasPrefix(info.Response.Data, "size:"))
 }
 
+func TestNetInfo(t *testing.T) {
+	c := rpctest.GetClient()
+	netinfo, err := c.NetInfo()
+	require.Nil(t, err, "%+v", err)
+	assert.True(t, netinfo.Listening)
+	assert.Equal(t, 0, len(netinfo.Peers))
+}
+
+func TestDialSeeds(t *testing.T) {
+	c := rpctest.GetClient()
+	// FIXME: fix server so it doesn't panic on invalid input
+	_, err := c.DialSeeds([]string{"12.34.56.78:12345"})
+	require.Nil(t, err, "%+v", err)
+}
+
+func TestGenesisAndValidators(t *testing.T) {
+	c := rpctest.GetClient()
+	chainID := rpctest.GetConfig().GetString("chain_id")
+
+	// make sure this is the right genesis file
+	gen, err := c.Genesis()
+	require.Nil(t, err, "%+v", err)
+	assert.Equal(t, chainID, gen.Genesis.ChainID)
+	// get the genesis validator
+	require.Equal(t, 1, len(gen.Genesis.Validators))
+	gval := gen.Genesis.Validators[0]
+
+	// get the current validators
+	vals, err := c.Validators()
+	require.Nil(t, err, "%+v", err)
+	require.Equal(t, 1, len(vals.Validators))
+	val := vals.Validators[0]
+
+	// make sure the current set is also the genesis set
+	assert.Equal(t, gval.Amount, val.VotingPower)
+	assert.Equal(t, gval.PubKey, val.PubKey)
+}
+
 // Make some app checks
 func TestAppCalls(t *testing.T) {
 	assert, require := assert.New(t), require.New(t)
@@ -41,9 +78,9 @@ func TestAppCalls(t *testing.T) {
 	assert.NotNil(err) // no block yet
 	k, v, tx := MakeTxKV()
 	_, err = c.BroadcastTxCommit(tx)
-	require.Nil(err)
+	require.Nil(err, "%+v", err)
 	// wait before querying
-	time.Sleep(time.Second * 2)
+	time.Sleep(time.Second * 1)
 	qres, err := c.ABCIQuery("/key", k, false)
 	if assert.Nil(err) && assert.True(qres.Response.Code.IsOK()) {
 		data := qres.Response
@@ -52,20 +89,35 @@ func TestAppCalls(t *testing.T) {
 	}
 	// and we can even check the block is added
 	block, err := c.Block(3)
-	assert.Nil(err) // now it's good :)
+	require.Nil(err, "%+v", err)
 	appHash := block.BlockMeta.Header.AppHash
 	assert.True(len(appHash) > 0)
+	assert.EqualValues(3, block.BlockMeta.Header.Height)
+
+	// check blockchain info, now that we know there is info
+	// TODO: is this commented somewhere that they are returned
+	// in order of descending height???
+	info, err := c.BlockchainInfo(1, 3)
+	require.Nil(err, "%+v", err)
+	assert.True(info.LastHeight > 2)
+	if assert.Equal(3, len(info.BlockMetas)) {
+		lastMeta := info.BlockMetas[0]
+		assert.EqualValues(3, lastMeta.Header.Height)
+		bMeta := block.BlockMeta
+		assert.Equal(bMeta.Header.AppHash, lastMeta.Header.AppHash)
+		assert.Equal(bMeta.BlockID, lastMeta.BlockID)
+	}
 
 	// and get the corresponding commit with the same apphash
 	commit, err := c.Commit(3)
-	assert.Nil(err) // now it's good :)
+	require.Nil(err, "%+v", err)
 	cappHash := commit.Header.AppHash
 	assert.Equal(appHash, cappHash)
 	assert.NotNil(commit.Commit)
 
 	// compare the commits (note Commit(2) has commit from Block(3))
 	commit2, err := c.Commit(2)
-	assert.Nil(err) // now it's good :)
+	require.Nil(err, "%+v", err)
 	assert.Equal(block.Block.LastCommit, commit2.Commit)
 
 	// and we got a proof that works!
@@ -82,27 +134,6 @@ func TestAppCalls(t *testing.T) {
 	}
 }
 
-// run most calls just to make sure no syntax errors
-func TestNoErrors(t *testing.T) {
-	assert := assert.New(t)
-	c := rpctest.GetClient()
-	_, err := c.NetInfo()
-	assert.Nil(err)
-	_, err = c.BlockchainInfo(0, 4)
-	assert.Nil(err)
-	// TODO: check with a valid height
-	_, err = c.Block(1000)
-	assert.NotNil(err)
-	// maybe this is an error???
-	// _, err = c.DialSeeds([]string{"one", "two"})
-	// assert.Nil(err)
-	gen, err := c.Genesis()
-	if assert.Nil(err) {
-		chainID := rpctest.GetConfig().GetString("chain_id")
-		assert.Equal(chainID, gen.Genesis.ChainID)
-	}
-}
-
 func TestSubscriptions(t *testing.T) {
 	assert, require := assert.New(t), require.New(t)
 	c := rpctest.GetClient()

From a0bdae4f9c07632f32c096cc95081273de15e269 Mon Sep 17 00:00:00 2001
From: Ethan Frey <ethanfrey@users.noreply.github.com>
Date: Wed, 22 Feb 2017 14:15:13 +0100
Subject: [PATCH 077/132] Added missing pkg/errors dependency

---
 glide.lock | 2 ++
 glide.yaml | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/glide.lock b/glide.lock
index 759553ea..3d72ddc0 100644
--- a/glide.lock
+++ b/glide.lock
@@ -29,6 +29,8 @@ imports:
   version: 5411d3eea5978e6cdc258b30de592b60df6aba96
 - name: github.com/mattn/go-isatty
   version: 281032e84ae07510239465db46bf442aa44b953a
+- name: github.com/pkg/errors
+  version: 248dadf4e9068a0b3e79f02ed0a610d935de5302
 - name: github.com/spf13/pflag
   version: 9ff6c6923cfffbcd502984b8e0c80539a94968b7
 - name: github.com/syndtr/goleveldb
diff --git a/glide.yaml b/glide.yaml
index 55c85962..9a7e6744 100644
--- a/glide.yaml
+++ b/glide.yaml
@@ -34,6 +34,8 @@ import:
   - proto
 - package: github.com/gorilla/websocket
 - package: github.com/spf13/pflag
+- package: github.com/pkg/errors
+  version: ^0.8.0
 - package: golang.org/x/crypto
   subpackages:
   - ripemd160

From bf1ee89b270892b06f763579f9bbf1d760fae639 Mon Sep 17 00:00:00 2001
From: Ethan Frey <ethanfrey@users.noreply.github.com>
Date: Wed, 22 Feb 2017 14:19:24 +0100
Subject: [PATCH 078/132] Moved httpclient into subpackage

---
 rpc/client/{ => http}/app_test.go  |  2 +-
 rpc/client/{ => http}/client.go    |  2 +-
 rpc/client/{ => http}/main_test.go |  2 +-
 rpc/client/{ => http}/rpc_test.go  | 23 +++++++++++++++--------
 rpc/test/helpers.go                |  7 -------
 5 files changed, 18 insertions(+), 18 deletions(-)
 rename rpc/client/{ => http}/app_test.go (97%)
 rename rpc/client/{ => http}/client.go (99%)
 rename rpc/client/{ => http}/main_test.go (94%)
 rename rpc/client/{ => http}/rpc_test.go (91%)

diff --git a/rpc/client/app_test.go b/rpc/client/http/app_test.go
similarity index 97%
rename from rpc/client/app_test.go
rename to rpc/client/http/app_test.go
index b360dba3..2d599cbc 100644
--- a/rpc/client/app_test.go
+++ b/rpc/client/http/app_test.go
@@ -1,4 +1,4 @@
-package client_test
+package httpclient_test
 
 import (
 	"math/rand"
diff --git a/rpc/client/client.go b/rpc/client/http/client.go
similarity index 99%
rename from rpc/client/client.go
rename to rpc/client/http/client.go
index a319b9bc..773b313a 100644
--- a/rpc/client/client.go
+++ b/rpc/client/http/client.go
@@ -1,4 +1,4 @@
-package client
+package httpclient
 
 import (
 	"encoding/json"
diff --git a/rpc/client/main_test.go b/rpc/client/http/main_test.go
similarity index 94%
rename from rpc/client/main_test.go
rename to rpc/client/http/main_test.go
index 5119ac41..0e324df4 100644
--- a/rpc/client/main_test.go
+++ b/rpc/client/http/main_test.go
@@ -1,4 +1,4 @@
-package client_test
+package httpclient_test
 
 import (
 	"os"
diff --git a/rpc/client/rpc_test.go b/rpc/client/http/rpc_test.go
similarity index 91%
rename from rpc/client/rpc_test.go
rename to rpc/client/http/rpc_test.go
index 4aca05e8..3a7f8540 100644
--- a/rpc/client/rpc_test.go
+++ b/rpc/client/http/rpc_test.go
@@ -1,4 +1,4 @@
-package client_test
+package httpclient_test
 
 import (
 	"strings"
@@ -8,13 +8,20 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	merkle "github.com/tendermint/go-merkle"
+	httpclient "github.com/tendermint/tendermint/rpc/client/http"
 	rpctest "github.com/tendermint/tendermint/rpc/test"
 	"github.com/tendermint/tendermint/types"
 )
 
+// GetClient gets a rpc client pointing to the test tendermint rpc
+func GetClient() *httpclient.HTTPClient {
+	rpcAddr := rpctest.GetConfig().GetString("rpc_laddr")
+	return httpclient.New(rpcAddr, "/websocket")
+}
+
 // Make sure status is correct (we connect properly)
 func TestStatus(t *testing.T) {
-	c := rpctest.GetClient()
+	c := GetClient()
 	chainID := rpctest.GetConfig().GetString("chain_id")
 	status, err := c.Status()
 	require.Nil(t, err, "%+v", err)
@@ -23,7 +30,7 @@ func TestStatus(t *testing.T) {
 
 // Make sure info is correct (we connect properly)
 func TestInfo(t *testing.T) {
-	c := rpctest.GetClient()
+	c := GetClient()
 	status, err := c.Status()
 	require.Nil(t, err, "%+v", err)
 	info, err := c.ABCIInfo()
@@ -33,7 +40,7 @@ func TestInfo(t *testing.T) {
 }
 
 func TestNetInfo(t *testing.T) {
-	c := rpctest.GetClient()
+	c := GetClient()
 	netinfo, err := c.NetInfo()
 	require.Nil(t, err, "%+v", err)
 	assert.True(t, netinfo.Listening)
@@ -41,14 +48,14 @@ func TestNetInfo(t *testing.T) {
 }
 
 func TestDialSeeds(t *testing.T) {
-	c := rpctest.GetClient()
+	c := GetClient()
 	// FIXME: fix server so it doesn't panic on invalid input
 	_, err := c.DialSeeds([]string{"12.34.56.78:12345"})
 	require.Nil(t, err, "%+v", err)
 }
 
 func TestGenesisAndValidators(t *testing.T) {
-	c := rpctest.GetClient()
+	c := GetClient()
 	chainID := rpctest.GetConfig().GetString("chain_id")
 
 	// make sure this is the right genesis file
@@ -73,7 +80,7 @@ func TestGenesisAndValidators(t *testing.T) {
 // Make some app checks
 func TestAppCalls(t *testing.T) {
 	assert, require := assert.New(t), require.New(t)
-	c := rpctest.GetClient()
+	c := GetClient()
 	_, err := c.Block(1)
 	assert.NotNil(err) // no block yet
 	k, v, tx := MakeTxKV()
@@ -136,7 +143,7 @@ func TestAppCalls(t *testing.T) {
 
 func TestSubscriptions(t *testing.T) {
 	assert, require := assert.New(t), require.New(t)
-	c := rpctest.GetClient()
+	c := GetClient()
 	err := c.StartWebsocket()
 	require.Nil(err)
 	defer c.StopWebsocket()
diff --git a/rpc/test/helpers.go b/rpc/test/helpers.go
index 0c80f62a..c028fa5a 100644
--- a/rpc/test/helpers.go
+++ b/rpc/test/helpers.go
@@ -15,7 +15,6 @@ import (
 	"github.com/tendermint/tendermint/config/tendermint_test"
 	nm "github.com/tendermint/tendermint/node"
 	"github.com/tendermint/tendermint/proxy"
-	rpcclient "github.com/tendermint/tendermint/rpc/client"
 	ctypes "github.com/tendermint/tendermint/rpc/core/types"
 	core_grpc "github.com/tendermint/tendermint/rpc/grpc"
 	"github.com/tendermint/tendermint/types"
@@ -38,12 +37,6 @@ func GetConfig() cfg.Config {
 	return config
 }
 
-// GetClient gets a rpc client pointing to the test tendermint rpc
-func GetClient() *rpcclient.HTTPClient {
-	rpcAddr := GetConfig().GetString("rpc_laddr")
-	return rpcclient.New(rpcAddr, "/websocket")
-}
-
 // GetURIClient gets a uri client pointing to the test tendermint rpc
 func GetURIClient() *client.ClientURI {
 	rpcAddr := GetConfig().GetString("rpc_laddr")

From 7c26be3242fc34bc3f2cc9feee7e570460093c86 Mon Sep 17 00:00:00 2001
From: Ethan Frey <ethanfrey@users.noreply.github.com>
Date: Wed, 22 Feb 2017 14:41:10 +0100
Subject: [PATCH 079/132] Begin implementation of local client

---
 node/node.go                  |   8 +-
 rpc/client/http/client.go     |  57 +++++++----
 rpc/client/http/rpc_test.go   |   2 +-
 rpc/client/local/app_test.go  |  45 +++++++++
 rpc/client/local/client.go    | 134 +++++++++++++++++++++++++
 rpc/client/local/main_test.go |  21 ++++
 rpc/client/local/rpc_test.go  | 183 ++++++++++++++++++++++++++++++++++
 rpc/test/helpers.go           |   4 +
 8 files changed, 429 insertions(+), 25 deletions(-)
 create mode 100644 rpc/client/local/app_test.go
 create mode 100644 rpc/client/local/client.go
 create mode 100644 rpc/client/local/main_test.go
 create mode 100644 rpc/client/local/rpc_test.go

diff --git a/node/node.go b/node/node.go
index deaa9e16..6ee02771 100644
--- a/node/node.go
+++ b/node/node.go
@@ -246,9 +246,10 @@ func (n *Node) AddListener(l p2p.Listener) {
 	n.sw.AddListener(l)
 }
 
-func (n *Node) startRPC() ([]net.Listener, error) {
+// ConfigureRPC sets all variables in rpccore so they will serve
+// rpc calls from this node
+func (n *Node) ConfigureRPC() {
 	rpccore.SetConfig(n.config)
-
 	rpccore.SetEventSwitch(n.evsw)
 	rpccore.SetBlockStore(n.blockStore)
 	rpccore.SetConsensusState(n.consensusState)
@@ -257,7 +258,10 @@ func (n *Node) startRPC() ([]net.Listener, error) {
 	rpccore.SetPubKey(n.privValidator.PubKey)
 	rpccore.SetGenesisDoc(n.genesisDoc)
 	rpccore.SetProxyAppQuery(n.proxyApp.Query())
+}
 
+func (n *Node) startRPC() ([]net.Listener, error) {
+	n.ConfigureRPC()
 	listenAddrs := strings.Split(n.config.GetString("rpc_laddr"), ",")
 
 	// we may expose the rpc over both a unix and tcp socket
diff --git a/rpc/client/http/client.go b/rpc/client/http/client.go
index 773b313a..c0c8d00e 100644
--- a/rpc/client/http/client.go
+++ b/rpc/client/http/client.go
@@ -1,3 +1,12 @@
+/*
+package httpclient returns a Client implementation that communicates
+with a tendermint node over json rpc and websockets.
+
+This is the main implementation you probably want to use in
+production code.  There are other implementations when calling
+the tendermint node in-process (local), or when you want to mock
+out the server for test code (mock).
+*/
 package httpclient
 
 import (
@@ -9,22 +18,26 @@ import (
 	"github.com/tendermint/tendermint/types"
 )
 
-type HTTPClient struct {
+// Client is a Client implementation that communicates over
+// JSONRPC
+type Client struct {
 	remote   string
 	endpoint string
 	rpc      *rpcclient.ClientJSONRPC
 	ws       *rpcclient.WSClient
 }
 
-func New(remote, wsEndpoint string) *HTTPClient {
-	return &HTTPClient{
+// New takes a remote endpoint in the form tcp://<host>:<port>
+// and the websocket path (which always seems to be "/websocket")
+func New(remote, wsEndpoint string) *Client {
+	return &Client{
 		rpc:      rpcclient.NewClientJSONRPC(remote),
 		remote:   remote,
 		endpoint: wsEndpoint,
 	}
 }
 
-func (c *HTTPClient) Status() (*ctypes.ResultStatus, error) {
+func (c *Client) Status() (*ctypes.ResultStatus, error) {
 	tmResult := new(ctypes.TMResult)
 	_, err := c.rpc.Call("status", []interface{}{}, tmResult)
 	if err != nil {
@@ -34,7 +47,7 @@ func (c *HTTPClient) Status() (*ctypes.ResultStatus, error) {
 	return (*tmResult).(*ctypes.ResultStatus), nil
 }
 
-func (c *HTTPClient) ABCIInfo() (*ctypes.ResultABCIInfo, error) {
+func (c *Client) ABCIInfo() (*ctypes.ResultABCIInfo, error) {
 	tmResult := new(ctypes.TMResult)
 	_, err := c.rpc.Call("abci_info", []interface{}{}, tmResult)
 	if err != nil {
@@ -43,7 +56,7 @@ func (c *HTTPClient) ABCIInfo() (*ctypes.ResultABCIInfo, error) {
 	return (*tmResult).(*ctypes.ResultABCIInfo), nil
 }
 
-func (c *HTTPClient) ABCIQuery(path string, data []byte, prove bool) (*ctypes.ResultABCIQuery, error) {
+func (c *Client) ABCIQuery(path string, data []byte, prove bool) (*ctypes.ResultABCIQuery, error) {
 	tmResult := new(ctypes.TMResult)
 	_, err := c.rpc.Call("abci_query", []interface{}{path, data, prove}, tmResult)
 	if err != nil {
@@ -52,19 +65,19 @@ func (c *HTTPClient) ABCIQuery(path string, data []byte, prove bool) (*ctypes.Re
 	return (*tmResult).(*ctypes.ResultABCIQuery), nil
 }
 
-func (c *HTTPClient) BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) {
+func (c *Client) BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) {
 	return c.broadcastTX("broadcast_tx_commit", tx)
 }
 
-func (c *HTTPClient) BroadcastTxAsync(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) {
+func (c *Client) BroadcastTxAsync(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) {
 	return c.broadcastTX("broadcast_tx_async", tx)
 }
 
-func (c *HTTPClient) BroadcastTxSync(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) {
+func (c *Client) BroadcastTxSync(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) {
 	return c.broadcastTX("broadcast_tx_sync", tx)
 }
 
-func (c *HTTPClient) broadcastTX(route string, tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) {
+func (c *Client) broadcastTX(route string, tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) {
 	tmResult := new(ctypes.TMResult)
 	_, err := c.rpc.Call(route, []interface{}{tx}, tmResult)
 	if err != nil {
@@ -73,7 +86,7 @@ func (c *HTTPClient) broadcastTX(route string, tx types.Tx) (*ctypes.ResultBroad
 	return (*tmResult).(*ctypes.ResultBroadcastTxCommit), nil
 }
 
-func (c *HTTPClient) NetInfo() (*ctypes.ResultNetInfo, error) {
+func (c *Client) NetInfo() (*ctypes.ResultNetInfo, error) {
 	tmResult := new(ctypes.TMResult)
 	_, err := c.rpc.Call("net_info", nil, tmResult)
 	if err != nil {
@@ -82,7 +95,7 @@ func (c *HTTPClient) NetInfo() (*ctypes.ResultNetInfo, error) {
 	return (*tmResult).(*ctypes.ResultNetInfo), nil
 }
 
-func (c *HTTPClient) DialSeeds(seeds []string) (*ctypes.ResultDialSeeds, error) {
+func (c *Client) DialSeeds(seeds []string) (*ctypes.ResultDialSeeds, error) {
 	tmResult := new(ctypes.TMResult)
 	// TODO: is this the correct way to marshall seeds?
 	_, err := c.rpc.Call("dial_seeds", []interface{}{seeds}, tmResult)
@@ -92,7 +105,7 @@ func (c *HTTPClient) DialSeeds(seeds []string) (*ctypes.ResultDialSeeds, error)
 	return (*tmResult).(*ctypes.ResultDialSeeds), nil
 }
 
-func (c *HTTPClient) BlockchainInfo(minHeight, maxHeight int) (*ctypes.ResultBlockchainInfo, error) {
+func (c *Client) BlockchainInfo(minHeight, maxHeight int) (*ctypes.ResultBlockchainInfo, error) {
 	tmResult := new(ctypes.TMResult)
 	_, err := c.rpc.Call("blockchain", []interface{}{minHeight, maxHeight}, tmResult)
 	if err != nil {
@@ -101,7 +114,7 @@ func (c *HTTPClient) BlockchainInfo(minHeight, maxHeight int) (*ctypes.ResultBlo
 	return (*tmResult).(*ctypes.ResultBlockchainInfo), nil
 }
 
-func (c *HTTPClient) Genesis() (*ctypes.ResultGenesis, error) {
+func (c *Client) Genesis() (*ctypes.ResultGenesis, error) {
 	tmResult := new(ctypes.TMResult)
 	_, err := c.rpc.Call("genesis", nil, tmResult)
 	if err != nil {
@@ -110,7 +123,7 @@ func (c *HTTPClient) Genesis() (*ctypes.ResultGenesis, error) {
 	return (*tmResult).(*ctypes.ResultGenesis), nil
 }
 
-func (c *HTTPClient) Block(height int) (*ctypes.ResultBlock, error) {
+func (c *Client) Block(height int) (*ctypes.ResultBlock, error) {
 	tmResult := new(ctypes.TMResult)
 	_, err := c.rpc.Call("block", []interface{}{height}, tmResult)
 	if err != nil {
@@ -119,7 +132,7 @@ func (c *HTTPClient) Block(height int) (*ctypes.ResultBlock, error) {
 	return (*tmResult).(*ctypes.ResultBlock), nil
 }
 
-func (c *HTTPClient) Commit(height int) (*ctypes.ResultCommit, error) {
+func (c *Client) Commit(height int) (*ctypes.ResultCommit, error) {
 	tmResult := new(ctypes.TMResult)
 	_, err := c.rpc.Call("commit", []interface{}{height}, tmResult)
 	if err != nil {
@@ -128,7 +141,7 @@ func (c *HTTPClient) Commit(height int) (*ctypes.ResultCommit, error) {
 	return (*tmResult).(*ctypes.ResultCommit), nil
 }
 
-func (c *HTTPClient) Validators() (*ctypes.ResultValidators, error) {
+func (c *Client) Validators() (*ctypes.ResultValidators, error) {
 	tmResult := new(ctypes.TMResult)
 	_, err := c.rpc.Call("validators", nil, tmResult)
 	if err != nil {
@@ -141,7 +154,7 @@ func (c *HTTPClient) Validators() (*ctypes.ResultValidators, error) {
 
 // StartWebsocket starts up a websocket and a listener goroutine
 // if already started, do nothing
-func (c *HTTPClient) StartWebsocket() error {
+func (c *Client) StartWebsocket() error {
 	var err error
 	if c.ws == nil {
 		ws := rpcclient.NewWSClient(c.remote, c.endpoint)
@@ -154,7 +167,7 @@ func (c *HTTPClient) StartWebsocket() error {
 }
 
 // StopWebsocket stops the websocket connection
-func (c *HTTPClient) StopWebsocket() {
+func (c *Client) StopWebsocket() {
 	if c.ws != nil {
 		c.ws.Stop()
 		c.ws = nil
@@ -162,17 +175,17 @@ func (c *HTTPClient) StopWebsocket() {
 }
 
 // GetEventChannels returns the results and error channel from the websocket
-func (c *HTTPClient) GetEventChannels() (chan json.RawMessage, chan error) {
+func (c *Client) GetEventChannels() (chan json.RawMessage, chan error) {
 	if c.ws == nil {
 		return nil, nil
 	}
 	return c.ws.ResultsCh, c.ws.ErrorsCh
 }
 
-func (c *HTTPClient) Subscribe(event string) error {
+func (c *Client) Subscribe(event string) error {
 	return errors.Wrap(c.ws.Subscribe(event), "Subscribe")
 }
 
-func (c *HTTPClient) Unsubscribe(event string) error {
+func (c *Client) Unsubscribe(event string) error {
 	return errors.Wrap(c.ws.Unsubscribe(event), "Unsubscribe")
 }
diff --git a/rpc/client/http/rpc_test.go b/rpc/client/http/rpc_test.go
index 3a7f8540..09a26096 100644
--- a/rpc/client/http/rpc_test.go
+++ b/rpc/client/http/rpc_test.go
@@ -14,7 +14,7 @@ import (
 )
 
 // GetClient gets a rpc client pointing to the test tendermint rpc
-func GetClient() *httpclient.HTTPClient {
+func GetClient() *httpclient.Client {
 	rpcAddr := rpctest.GetConfig().GetString("rpc_laddr")
 	return httpclient.New(rpcAddr, "/websocket")
 }
diff --git a/rpc/client/local/app_test.go b/rpc/client/local/app_test.go
new file mode 100644
index 00000000..bd16b375
--- /dev/null
+++ b/rpc/client/local/app_test.go
@@ -0,0 +1,45 @@
+package localclient_test
+
+import (
+	"math/rand"
+
+	meapp "github.com/tendermint/merkleeyes/app"
+
+	wire "github.com/tendermint/go-wire"
+)
+
+// MakeTxKV returns a text transaction, allong with expected key, value pair
+func MakeTxKV() ([]byte, []byte, []byte) {
+	k := RandAsciiBytes(8)
+	v := RandAsciiBytes(8)
+	return k, v, makeSet(k, v)
+}
+
+// blatently copied from merkleeyes/app/app_test.go
+// constructs a "set" transaction
+func makeSet(key, value []byte) []byte {
+	tx := make([]byte, 1+wire.ByteSliceSize(key)+wire.ByteSliceSize(value))
+	buf := tx
+	buf[0] = meapp.WriteSet // Set TypeByte
+	buf = buf[1:]
+	n, err := wire.PutByteSlice(buf, key)
+	if err != nil {
+		panic(err)
+	}
+	buf = buf[n:]
+	n, err = wire.PutByteSlice(buf, value)
+	if err != nil {
+		panic(err)
+	}
+	return tx
+}
+
+const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+
+func RandAsciiBytes(n int) []byte {
+	b := make([]byte, n)
+	for i := range b {
+		b[i] = letterBytes[rand.Intn(len(letterBytes))]
+	}
+	return b
+}
diff --git a/rpc/client/local/client.go b/rpc/client/local/client.go
new file mode 100644
index 00000000..3817e2df
--- /dev/null
+++ b/rpc/client/local/client.go
@@ -0,0 +1,134 @@
+/*
+package localclient returns a Client implementation that
+directly executes the rpc functions on a given node.
+
+This implementation is useful for:
+
+* Running tests against a node in-process without the overhead
+of going through an http server
+* Communication between an ABCI app and tendermin core when they
+are compiled in process.
+
+For real clients, you probably want the "http" package.  For more
+powerful control during testing, you probably want the "mock" package.
+*/
+package localclient
+
+import (
+	nm "github.com/tendermint/tendermint/node"
+	"github.com/tendermint/tendermint/rpc/core"
+	ctypes "github.com/tendermint/tendermint/rpc/core/types"
+	"github.com/tendermint/tendermint/types"
+)
+
+type Client struct {
+	node *nm.Node
+}
+
+// New configures this to call the Node directly.
+//
+// Note that given how rpc/core works with package singletons, that
+// you can only have one node per process.  So make sure test cases
+// don't run in parallel, or try to simulate an entire network in
+// one process...
+func New(node *nm.Node) Client {
+	node.ConfigureRPC()
+	return Client{
+		node: node,
+	}
+}
+
+func (c Client) Status() (*ctypes.ResultStatus, error) {
+	return core.Status()
+}
+
+func (c Client) ABCIInfo() (*ctypes.ResultABCIInfo, error) {
+	return core.ABCIInfo()
+}
+
+func (c Client) ABCIQuery(path string, data []byte, prove bool) (*ctypes.ResultABCIQuery, error) {
+	return core.ABCIQuery(path, data, prove)
+}
+
+func (c Client) BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) {
+	return core.BroadcastTxCommit(tx)
+}
+
+func (c Client) BroadcastTxAsync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) {
+	return core.BroadcastTxAsync(tx)
+}
+
+func (c Client) BroadcastTxSync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) {
+	return core.BroadcastTxSync(tx)
+}
+
+func (c Client) NetInfo() (*ctypes.ResultNetInfo, error) {
+	return core.NetInfo()
+}
+
+func (c Client) DialSeeds(seeds []string) (*ctypes.ResultDialSeeds, error) {
+	return core.UnsafeDialSeeds(seeds)
+}
+
+func (c Client) BlockchainInfo(minHeight, maxHeight int) (*ctypes.ResultBlockchainInfo, error) {
+	return core.BlockchainInfo(minHeight, maxHeight)
+}
+
+func (c Client) Genesis() (*ctypes.ResultGenesis, error) {
+	return core.Genesis()
+}
+
+func (c Client) Block(height int) (*ctypes.ResultBlock, error) {
+	return core.Block(height)
+}
+
+func (c Client) Commit(height int) (*ctypes.ResultCommit, error) {
+	return core.Commit(height)
+}
+
+func (c Client) Validators() (*ctypes.ResultValidators, error) {
+	return core.Validators()
+}
+
+/** websocket event stuff here... **/
+
+/*
+// StartWebsocket starts up a websocket and a listener goroutine
+// if already started, do nothing
+func (c Client) StartWebsocket() error {
+	var err error
+	if c.ws == nil {
+		ws := rpcclient.NewWSClient(c.remote, c.endpoint)
+		_, err = ws.Start()
+		if err == nil {
+			c.ws = ws
+		}
+	}
+	return errors.Wrap(err, "StartWebsocket")
+}
+
+// StopWebsocket stops the websocket connection
+func (c Client) StopWebsocket() {
+	if c.ws != nil {
+		c.ws.Stop()
+		c.ws = nil
+	}
+}
+
+// GetEventChannels returns the results and error channel from the websocket
+func (c Client) GetEventChannels() (chan json.RawMessage, chan error) {
+	if c.ws == nil {
+		return nil, nil
+	}
+	return c.ws.ResultsCh, c.ws.ErrorsCh
+}
+
+func (c Client) Subscribe(event string) error {
+	return errors.Wrap(c.ws.Subscribe(event), "Subscribe")
+}
+
+func (c Client) Unsubscribe(event string) error {
+	return errors.Wrap(c.ws.Unsubscribe(event), "Unsubscribe")
+}
+
+*/
diff --git a/rpc/client/local/main_test.go b/rpc/client/local/main_test.go
new file mode 100644
index 00000000..5e0a0c23
--- /dev/null
+++ b/rpc/client/local/main_test.go
@@ -0,0 +1,21 @@
+package localclient_test
+
+import (
+	"os"
+	"testing"
+
+	meapp "github.com/tendermint/merkleeyes/app"
+	rpctest "github.com/tendermint/tendermint/rpc/test"
+)
+
+func TestMain(m *testing.M) {
+	// start a tendermint node (and merkleeyes) in the background to test against
+	app := meapp.NewMerkleEyesApp("", 100)
+	node := rpctest.StartTendermint(app)
+	code := m.Run()
+
+	// and shut down proper at the end
+	node.Stop()
+	node.Wait()
+	os.Exit(code)
+}
diff --git a/rpc/client/local/rpc_test.go b/rpc/client/local/rpc_test.go
new file mode 100644
index 00000000..13fc5547
--- /dev/null
+++ b/rpc/client/local/rpc_test.go
@@ -0,0 +1,183 @@
+package localclient_test
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	merkle "github.com/tendermint/go-merkle"
+	localclient "github.com/tendermint/tendermint/rpc/client/local"
+	rpctest "github.com/tendermint/tendermint/rpc/test"
+)
+
+// GetClient gets a rpc client pointing to the test tendermint rpc
+func GetClient() localclient.Client {
+	node := rpctest.GetNode()
+	return localclient.New(node)
+}
+
+// Make sure status is correct (we connect properly)
+func TestStatus(t *testing.T) {
+	c := GetClient()
+	chainID := rpctest.GetConfig().GetString("chain_id")
+	status, err := c.Status()
+	require.Nil(t, err, "%+v", err)
+	assert.Equal(t, chainID, status.NodeInfo.Network)
+}
+
+// Make sure info is correct (we connect properly)
+func TestInfo(t *testing.T) {
+	c := GetClient()
+	status, err := c.Status()
+	require.Nil(t, err, "%+v", err)
+	info, err := c.ABCIInfo()
+	require.Nil(t, err, "%+v", err)
+	assert.EqualValues(t, status.LatestBlockHeight, info.Response.LastBlockHeight)
+	assert.True(t, strings.HasPrefix(info.Response.Data, "size:"))
+}
+
+func TestNetInfo(t *testing.T) {
+	c := GetClient()
+	netinfo, err := c.NetInfo()
+	require.Nil(t, err, "%+v", err)
+	assert.True(t, netinfo.Listening)
+	assert.Equal(t, 0, len(netinfo.Peers))
+}
+
+func TestDialSeeds(t *testing.T) {
+	c := GetClient()
+	// FIXME: fix server so it doesn't panic on invalid input
+	_, err := c.DialSeeds([]string{"12.34.56.78:12345"})
+	require.Nil(t, err, "%+v", err)
+}
+
+func TestGenesisAndValidators(t *testing.T) {
+	c := GetClient()
+	chainID := rpctest.GetConfig().GetString("chain_id")
+
+	// make sure this is the right genesis file
+	gen, err := c.Genesis()
+	require.Nil(t, err, "%+v", err)
+	assert.Equal(t, chainID, gen.Genesis.ChainID)
+	// get the genesis validator
+	require.Equal(t, 1, len(gen.Genesis.Validators))
+	gval := gen.Genesis.Validators[0]
+
+	// get the current validators
+	vals, err := c.Validators()
+	require.Nil(t, err, "%+v", err)
+	require.Equal(t, 1, len(vals.Validators))
+	val := vals.Validators[0]
+
+	// make sure the current set is also the genesis set
+	assert.Equal(t, gval.Amount, val.VotingPower)
+	assert.Equal(t, gval.PubKey, val.PubKey)
+}
+
+// Make some app checks
+func TestAppCalls(t *testing.T) {
+	assert, require := assert.New(t), require.New(t)
+	c := GetClient()
+	_, err := c.Block(1)
+	assert.NotNil(err) // no block yet
+	k, v, tx := MakeTxKV()
+	_, err = c.BroadcastTxCommit(tx)
+	require.Nil(err, "%+v", err)
+	// wait before querying
+	time.Sleep(time.Second * 1)
+	qres, err := c.ABCIQuery("/key", k, false)
+	if assert.Nil(err) && assert.True(qres.Response.Code.IsOK()) {
+		data := qres.Response
+		// assert.Equal(k, data.GetKey())  // only returned for proofs
+		assert.Equal(v, data.GetValue())
+	}
+	// and we can even check the block is added
+	block, err := c.Block(3)
+	require.Nil(err, "%+v", err)
+	appHash := block.BlockMeta.Header.AppHash
+	assert.True(len(appHash) > 0)
+	assert.EqualValues(3, block.BlockMeta.Header.Height)
+
+	// check blockchain info, now that we know there is info
+	// TODO: is this commented somewhere that they are returned
+	// in order of descending height???
+	info, err := c.BlockchainInfo(1, 3)
+	require.Nil(err, "%+v", err)
+	assert.True(info.LastHeight > 2)
+	if assert.Equal(3, len(info.BlockMetas)) {
+		lastMeta := info.BlockMetas[0]
+		assert.EqualValues(3, lastMeta.Header.Height)
+		bMeta := block.BlockMeta
+		assert.Equal(bMeta.Header.AppHash, lastMeta.Header.AppHash)
+		assert.Equal(bMeta.BlockID, lastMeta.BlockID)
+	}
+
+	// and get the corresponding commit with the same apphash
+	commit, err := c.Commit(3)
+	require.Nil(err, "%+v", err)
+	cappHash := commit.Header.AppHash
+	assert.Equal(appHash, cappHash)
+	assert.NotNil(commit.Commit)
+
+	// compare the commits (note Commit(2) has commit from Block(3))
+	commit2, err := c.Commit(2)
+	require.Nil(err, "%+v", err)
+	assert.Equal(block.Block.LastCommit, commit2.Commit)
+
+	// and we got a proof that works!
+	pres, err := c.ABCIQuery("/key", k, true)
+	if assert.Nil(err) && assert.True(pres.Response.Code.IsOK()) {
+		proof, err := merkle.ReadProof(pres.Response.GetProof())
+		if assert.Nil(err) {
+			key := pres.Response.GetKey()
+			value := pres.Response.GetValue()
+			assert.Equal(appHash, proof.RootHash)
+			valid := proof.Verify(key, value, appHash)
+			assert.True(valid)
+		}
+	}
+}
+
+/*
+func TestSubscriptions(t *testing.T) {
+	assert, require := assert.New(t), require.New(t)
+	c := GetClient()
+	err := c.StartWebsocket()
+	require.Nil(err)
+	defer c.StopWebsocket()
+
+	// subscribe to a transaction event
+	_, _, tx := MakeTxKV()
+	// this causes a panic in tendermint core!!!
+	eventType := types.EventStringTx(types.Tx(tx))
+	c.Subscribe(eventType)
+	read := 0
+
+	// set up a listener
+	r, e := c.GetEventChannels()
+	go func() {
+		// read one event in the background
+		select {
+		case <-r:
+			// TODO: actually parse this or something
+			read += 1
+		case err := <-e:
+			panic(err)
+		}
+	}()
+
+	// make sure nothing has happened yet.
+	assert.Equal(0, read)
+
+	// send a tx and wait for it to propogate
+	_, err = c.BroadcastTxCommit(tx)
+	assert.Nil(err, string(tx))
+	// wait before querying
+	time.Sleep(time.Second)
+
+	// now make sure the event arrived
+	assert.Equal(1, read)
+}
+*/
diff --git a/rpc/test/helpers.go b/rpc/test/helpers.go
index c028fa5a..e835a393 100644
--- a/rpc/test/helpers.go
+++ b/rpc/test/helpers.go
@@ -37,6 +37,10 @@ func GetConfig() cfg.Config {
 	return config
 }
 
+func GetNode() *nm.Node {
+	return node
+}
+
 // GetURIClient gets a uri client pointing to the test tendermint rpc
 func GetURIClient() *client.ClientURI {
 	rpcAddr := GetConfig().GetString("rpc_laddr")

From ce044dbb763bd6367a4e25d1d9e347a3a653515a Mon Sep 17 00:00:00 2001
From: Ethan Frey <ethanfrey@users.noreply.github.com>
Date: Wed, 22 Feb 2017 14:54:13 +0100
Subject: [PATCH 080/132] Extracted Clients into a consistent interface, fixed
 type issue in http.Client

---
 rpc/client/http/client.go  | 20 +++++++++++----
 rpc/client/interface.go    | 52 ++++++++++++++++++++++++++++++++++++++
 rpc/client/local/client.go |  5 ++++
 3 files changed, 72 insertions(+), 5 deletions(-)
 create mode 100644 rpc/client/interface.go

diff --git a/rpc/client/http/client.go b/rpc/client/http/client.go
index c0c8d00e..f5716267 100644
--- a/rpc/client/http/client.go
+++ b/rpc/client/http/client.go
@@ -14,6 +14,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/tendermint/go-rpc/client"
+	"github.com/tendermint/tendermint/rpc/client"
 	ctypes "github.com/tendermint/tendermint/rpc/core/types"
 	"github.com/tendermint/tendermint/types"
 )
@@ -37,6 +38,10 @@ func New(remote, wsEndpoint string) *Client {
 	}
 }
 
+func (c *Client) _assertIsClient() client.Client {
+	return c
+}
+
 func (c *Client) Status() (*ctypes.ResultStatus, error) {
 	tmResult := new(ctypes.TMResult)
 	_, err := c.rpc.Call("status", []interface{}{}, tmResult)
@@ -66,24 +71,29 @@ func (c *Client) ABCIQuery(path string, data []byte, prove bool) (*ctypes.Result
 }
 
 func (c *Client) BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) {
-	return c.broadcastTX("broadcast_tx_commit", tx)
+	tmResult := new(ctypes.TMResult)
+	_, err := c.rpc.Call("broadcast_tx_commit", []interface{}{tx}, tmResult)
+	if err != nil {
+		return nil, errors.Wrap(err, "broadcast_tx_commit")
+	}
+	return (*tmResult).(*ctypes.ResultBroadcastTxCommit), nil
 }
 
-func (c *Client) BroadcastTxAsync(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) {
+func (c *Client) BroadcastTxAsync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) {
 	return c.broadcastTX("broadcast_tx_async", tx)
 }
 
-func (c *Client) BroadcastTxSync(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) {
+func (c *Client) BroadcastTxSync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) {
 	return c.broadcastTX("broadcast_tx_sync", tx)
 }
 
-func (c *Client) broadcastTX(route string, tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) {
+func (c *Client) broadcastTX(route string, tx types.Tx) (*ctypes.ResultBroadcastTx, error) {
 	tmResult := new(ctypes.TMResult)
 	_, err := c.rpc.Call(route, []interface{}{tx}, tmResult)
 	if err != nil {
 		return nil, errors.Wrap(err, route)
 	}
-	return (*tmResult).(*ctypes.ResultBroadcastTxCommit), nil
+	return (*tmResult).(*ctypes.ResultBroadcastTx), nil
 }
 
 func (c *Client) NetInfo() (*ctypes.ResultNetInfo, error) {
diff --git a/rpc/client/interface.go b/rpc/client/interface.go
new file mode 100644
index 00000000..33f5dace
--- /dev/null
+++ b/rpc/client/interface.go
@@ -0,0 +1,52 @@
+/*
+package client provides a general purpose interface for connecting
+to a tendermint node, as well as higher-level functionality.
+
+The main implementation for production code is http, which connects
+via http to the jsonrpc interface of the tendermint node.
+
+For connecting to a node running in the same process (eg. when
+compiling the abci app in the same process), you can use the local
+implementation.
+
+For mocking out server responses during testing to see behavior for
+arbitrary return values, use the mock package.
+
+In addition to the Client interface, which should be used externally
+for maximum flexibility and testability, this package also provides
+a wrapper that accepts any Client implementation and adds some
+higher-level functionality.
+*/
+package client
+
+import (
+	ctypes "github.com/tendermint/tendermint/rpc/core/types"
+	"github.com/tendermint/tendermint/types"
+)
+
+type Client interface {
+	// general chain info
+	Status() (*ctypes.ResultStatus, error)
+	NetInfo() (*ctypes.ResultNetInfo, error)
+	Genesis() (*ctypes.ResultGenesis, error)
+
+	// reading from abci app
+	ABCIInfo() (*ctypes.ResultABCIInfo, error)
+	ABCIQuery(path string, data []byte, prove bool) (*ctypes.ResultABCIQuery, error)
+
+	// writing to abci app
+	BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error)
+	BroadcastTxAsync(tx types.Tx) (*ctypes.ResultBroadcastTx, error)
+	BroadcastTxSync(tx types.Tx) (*ctypes.ResultBroadcastTx, error)
+
+	// validating block info
+	BlockchainInfo(minHeight, maxHeight int) (*ctypes.ResultBlockchainInfo, error)
+	Block(height int) (*ctypes.ResultBlock, error)
+	Commit(height int) (*ctypes.ResultCommit, error)
+	Validators() (*ctypes.ResultValidators, error)
+
+	// TODO: add some sort of generic subscription mechanism...
+
+	// remove this???
+	// DialSeeds(seeds []string) (*ctypes.ResultDialSeeds, error)
+}
diff --git a/rpc/client/local/client.go b/rpc/client/local/client.go
index 3817e2df..f050b6b1 100644
--- a/rpc/client/local/client.go
+++ b/rpc/client/local/client.go
@@ -16,6 +16,7 @@ package localclient
 
 import (
 	nm "github.com/tendermint/tendermint/node"
+	"github.com/tendermint/tendermint/rpc/client"
 	"github.com/tendermint/tendermint/rpc/core"
 	ctypes "github.com/tendermint/tendermint/rpc/core/types"
 	"github.com/tendermint/tendermint/types"
@@ -38,6 +39,10 @@ func New(node *nm.Node) Client {
 	}
 }
 
+func (c Client) _assertIsClient() client.Client {
+	return c
+}
+
 func (c Client) Status() (*ctypes.ResultStatus, error) {
 	return core.Status()
 }

From df172fa840ee75fa90d4898e94fffed7805462a4 Mon Sep 17 00:00:00 2001
From: Ethan Frey <ethanfrey@users.noreply.github.com>
Date: Wed, 22 Feb 2017 16:39:01 +0100
Subject: [PATCH 081/132] Provide mock interfaces for calling abci app over
 tendermint rpc

---
 rpc/client/interface.go      |  49 ++++++---
 rpc/client/mock/abci.go      | 186 +++++++++++++++++++++++++++++++++++
 rpc/client/mock/abci_test.go | 142 ++++++++++++++++++++++++++
 rpc/client/mock/client.go    | 169 +++++++++++++++++++++++++++++++
 4 files changed, 535 insertions(+), 11 deletions(-)
 create mode 100644 rpc/client/mock/abci.go
 create mode 100644 rpc/client/mock/abci_test.go
 create mode 100644 rpc/client/mock/client.go

diff --git a/rpc/client/interface.go b/rpc/client/interface.go
index 33f5dace..0683d469 100644
--- a/rpc/client/interface.go
+++ b/rpc/client/interface.go
@@ -24,12 +24,10 @@ import (
 	"github.com/tendermint/tendermint/types"
 )
 
-type Client interface {
-	// general chain info
-	Status() (*ctypes.ResultStatus, error)
-	NetInfo() (*ctypes.ResultNetInfo, error)
-	Genesis() (*ctypes.ResultGenesis, error)
-
+// ABCIClient groups together the functionality that principally
+// affects the ABCI app. In many cases this will be all we want,
+// so we can accept an interface which is easier to mock
+type ABCIClient interface {
 	// reading from abci app
 	ABCIInfo() (*ctypes.ResultABCIInfo, error)
 	ABCIQuery(path string, data []byte, prove bool) (*ctypes.ResultABCIQuery, error)
@@ -38,15 +36,44 @@ type Client interface {
 	BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error)
 	BroadcastTxAsync(tx types.Tx) (*ctypes.ResultBroadcastTx, error)
 	BroadcastTxSync(tx types.Tx) (*ctypes.ResultBroadcastTx, error)
+}
 
-	// validating block info
-	BlockchainInfo(minHeight, maxHeight int) (*ctypes.ResultBlockchainInfo, error)
+// SignClient groups together the interfaces need to get valid
+// signatures and prove anything about the chain
+type SignClient interface {
 	Block(height int) (*ctypes.ResultBlock, error)
 	Commit(height int) (*ctypes.ResultCommit, error)
 	Validators() (*ctypes.ResultValidators, error)
+}
+
+// NetworkClient is general info about the network state.  May not
+// be needed usually.
+//
+// Not included in the Client interface, but generally implemented
+// by concrete implementations.
+type NetworkClient interface {
+	NetInfo() (*ctypes.ResultNetInfo, error)
+	// remove this???
+	DialSeeds(seeds []string) (*ctypes.ResultDialSeeds, error)
+}
+
+// HistoryClient shows us data from genesis to now in large chunks.
+type HistoryClient interface {
+	Genesis() (*ctypes.ResultGenesis, error)
+	BlockchainInfo(minHeight, maxHeight int) (*ctypes.ResultBlockchainInfo, error)
+}
+
+type StatusClient interface {
+	// general chain info
+	Status() (*ctypes.ResultStatus, error)
+}
+
+type Client interface {
+	ABCIClient
+	SignClient
+	HistoryClient
+	StatusClient
+	// Note: doesn't include NetworkClient, is it important??
 
 	// TODO: add some sort of generic subscription mechanism...
-
-	// remove this???
-	// DialSeeds(seeds []string) (*ctypes.ResultDialSeeds, error)
 }
diff --git a/rpc/client/mock/abci.go b/rpc/client/mock/abci.go
new file mode 100644
index 00000000..95d6fc0a
--- /dev/null
+++ b/rpc/client/mock/abci.go
@@ -0,0 +1,186 @@
+package mock
+
+import (
+	abci "github.com/tendermint/abci/types"
+	"github.com/tendermint/tendermint/rpc/client"
+	ctypes "github.com/tendermint/tendermint/rpc/core/types"
+	"github.com/tendermint/tendermint/types"
+)
+
+// ABCIApp will send all abci related request to the named app,
+// so you can test app behavior from a client without needing
+// an entire tendermint node
+type ABCIApp struct {
+	App abci.Application
+}
+
+func (a ABCIApp) _assertABCIClient() client.ABCIClient {
+	return a
+}
+
+func (a ABCIApp) ABCIInfo() (*ctypes.ResultABCIInfo, error) {
+	return &ctypes.ResultABCIInfo{a.App.Info()}, nil
+}
+
+func (a ABCIApp) ABCIQuery(path string, data []byte, prove bool) (*ctypes.ResultABCIQuery, error) {
+	q := a.App.Query(abci.RequestQuery{data, path, 0, prove})
+	return &ctypes.ResultABCIQuery{q}, nil
+}
+
+func (a ABCIApp) BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) {
+	res := ctypes.ResultBroadcastTxCommit{}
+	c := a.App.CheckTx(tx)
+	res.CheckTx = &abci.ResponseCheckTx{c.Code, c.Data, c.Log}
+	if !c.IsOK() {
+		return &res, nil
+	}
+	d := a.App.DeliverTx(tx)
+	res.DeliverTx = &abci.ResponseDeliverTx{d.Code, d.Data, d.Log}
+	return &res, nil
+}
+
+func (a ABCIApp) BroadcastTxAsync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) {
+	d := a.App.DeliverTx(tx)
+	return &ctypes.ResultBroadcastTx{d.Code, d.Data, d.Log}, nil
+}
+
+func (a ABCIApp) BroadcastTxSync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) {
+	d := a.App.DeliverTx(tx)
+	return &ctypes.ResultBroadcastTx{d.Code, d.Data, d.Log}, nil
+}
+
+// ABCIMock will send all abci related request to the named app,
+// so you can test app behavior from a client without needing
+// an entire tendermint node
+type ABCIMock struct {
+	Info            Call
+	Query           Call
+	BroadcastCommit Call
+	Broadcast       Call
+}
+
+func (m ABCIMock) _assertABCIClient() client.ABCIClient {
+	return m
+}
+
+func (m ABCIMock) ABCIInfo() (*ctypes.ResultABCIInfo, error) {
+	res, err := m.Info.GetResponse(nil)
+	if err != nil {
+		return nil, err
+	}
+	return &ctypes.ResultABCIInfo{res.(abci.ResponseInfo)}, nil
+}
+
+func (m ABCIMock) ABCIQuery(path string, data []byte, prove bool) (*ctypes.ResultABCIQuery, error) {
+	res, err := m.Query.GetResponse(QueryArgs{path, data, prove})
+	if err != nil {
+		return nil, err
+	}
+	return &ctypes.ResultABCIQuery{res.(abci.ResponseQuery)}, nil
+}
+
+func (m ABCIMock) BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) {
+	res, err := m.BroadcastCommit.GetResponse(tx)
+	if err != nil {
+		return nil, err
+	}
+	return res.(*ctypes.ResultBroadcastTxCommit), nil
+}
+
+func (m ABCIMock) BroadcastTxAsync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) {
+	res, err := m.Broadcast.GetResponse(tx)
+	if err != nil {
+		return nil, err
+	}
+	return res.(*ctypes.ResultBroadcastTx), nil
+}
+
+func (m ABCIMock) BroadcastTxSync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) {
+	res, err := m.Broadcast.GetResponse(tx)
+	if err != nil {
+		return nil, err
+	}
+	return res.(*ctypes.ResultBroadcastTx), nil
+}
+
+// ABCIRecorder can wrap another type (ABCIApp, ABCIMock, or Client)
+// and record all ABCI related calls.
+type ABCIRecorder struct {
+	Client client.ABCIClient
+	Calls  []Call
+}
+
+func NewABCIRecorder(client client.ABCIClient) *ABCIRecorder {
+	return &ABCIRecorder{
+		Client: client,
+		Calls:  []Call{},
+	}
+}
+
+func (r *ABCIRecorder) _assertABCIClient() client.ABCIClient {
+	return r
+}
+
+type QueryArgs struct {
+	Path  string
+	Data  []byte
+	Prove bool
+}
+
+func (r *ABCIRecorder) addCall(call Call) {
+	r.Calls = append(r.Calls, call)
+}
+
+func (r *ABCIRecorder) ABCIInfo() (*ctypes.ResultABCIInfo, error) {
+	res, err := r.Client.ABCIInfo()
+	r.addCall(Call{
+		Name:     "abci_info",
+		Response: res,
+		Error:    err,
+	})
+	return res, err
+}
+
+func (r *ABCIRecorder) ABCIQuery(path string, data []byte, prove bool) (*ctypes.ResultABCIQuery, error) {
+	res, err := r.Client.ABCIQuery(path, data, prove)
+	r.addCall(Call{
+		Name:     "abci_query",
+		Args:     QueryArgs{path, data, prove},
+		Response: res,
+		Error:    err,
+	})
+	return res, err
+}
+
+func (r *ABCIRecorder) BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) {
+	res, err := r.Client.BroadcastTxCommit(tx)
+	r.addCall(Call{
+		Name:     "broadcast_tx_commit",
+		Args:     tx,
+		Response: res,
+		Error:    err,
+	})
+	return res, err
+}
+
+func (r *ABCIRecorder) BroadcastTxAsync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) {
+	res, err := r.Client.BroadcastTxAsync(tx)
+	r.addCall(Call{
+		Name:     "broadcast_tx_async",
+		Args:     tx,
+		Response: res,
+		Error:    err,
+	})
+	return res, err
+}
+
+func (r *ABCIRecorder) BroadcastTxSync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) {
+	res, err := r.Client.BroadcastTxSync(tx)
+	r.addCall(Call{
+		Name:     "broadcast_tx_sync",
+		Args:     tx,
+		Response: res,
+		Error:    err,
+	})
+	return res, err
+}
diff --git a/rpc/client/mock/abci_test.go b/rpc/client/mock/abci_test.go
new file mode 100644
index 00000000..50e767ed
--- /dev/null
+++ b/rpc/client/mock/abci_test.go
@@ -0,0 +1,142 @@
+package mock_test
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	abci "github.com/tendermint/abci/types"
+	ctypes "github.com/tendermint/tendermint/rpc/core/types"
+	"github.com/tendermint/tendermint/types"
+
+	"github.com/tendermint/tendermint/rpc/client/mock"
+)
+
+func TestABCIMock(t *testing.T) {
+	assert, require := assert.New(t), require.New(t)
+
+	key, value := []byte("foo"), []byte("bar")
+	height := uint64(10)
+	goodTx := types.Tx{0x01, 0xff}
+	badTx := types.Tx{0x12, 0x21}
+
+	m := mock.ABCIMock{
+		Info: mock.Call{Error: errors.New("foobar")},
+		Query: mock.Call{Response: abci.ResponseQuery{
+			Key:    key,
+			Value:  value,
+			Height: height,
+		}},
+		// Broadcast commit depends on call
+		BroadcastCommit: mock.Call{
+			Args: goodTx,
+			Response: &ctypes.ResultBroadcastTxCommit{
+				CheckTx:   &abci.ResponseCheckTx{Data: []byte("stand")},
+				DeliverTx: &abci.ResponseDeliverTx{Data: []byte("deliver")},
+			},
+			Error: errors.New("bad tx"),
+		},
+		Broadcast: mock.Call{Error: errors.New("must commit")},
+	}
+
+	// now, let's try to make some calls
+	_, err := m.ABCIInfo()
+	require.NotNil(err)
+	assert.Equal("foobar", err.Error())
+
+	// query always returns the response
+	query, err := m.ABCIQuery("/", nil, false)
+	require.Nil(err)
+	require.NotNil(query)
+	assert.Equal(key, query.Response.GetKey())
+	assert.Equal(value, query.Response.GetValue())
+	assert.Equal(height, query.Response.GetHeight())
+
+	// non-commit calls always return errors
+	_, err = m.BroadcastTxSync(goodTx)
+	require.NotNil(err)
+	assert.Equal("must commit", err.Error())
+	_, err = m.BroadcastTxAsync(goodTx)
+	require.NotNil(err)
+	assert.Equal("must commit", err.Error())
+
+	// commit depends on the input
+	_, err = m.BroadcastTxCommit(badTx)
+	require.NotNil(err)
+	assert.Equal("bad tx", err.Error())
+	bres, err := m.BroadcastTxCommit(goodTx)
+	require.Nil(err, "%+v", err)
+	assert.EqualValues(0, bres.CheckTx.Code)
+	assert.EqualValues("stand", bres.CheckTx.Data)
+	assert.EqualValues("deliver", bres.DeliverTx.Data)
+}
+
+func TestABCIRecorder(t *testing.T) {
+	assert, require := assert.New(t), require.New(t)
+	m := mock.ABCIMock{
+		Info: mock.Call{Response: abci.ResponseInfo{
+			Data:    "data",
+			Version: "v0.9.9",
+		}},
+		Query:           mock.Call{Error: errors.New("query")},
+		Broadcast:       mock.Call{Error: errors.New("broadcast")},
+		BroadcastCommit: mock.Call{Error: errors.New("broadcast_commit")},
+	}
+	r := mock.NewABCIRecorder(m)
+
+	require.Equal(0, len(r.Calls))
+
+	r.ABCIInfo()
+	r.ABCIQuery("path", []byte("data"), true)
+	require.Equal(2, len(r.Calls))
+
+	info := r.Calls[0]
+	assert.Equal("abci_info", info.Name)
+	assert.Nil(info.Error)
+	assert.Nil(info.Args)
+	require.NotNil(info.Response)
+	ir, ok := info.Response.(*ctypes.ResultABCIInfo)
+	require.True(ok)
+	assert.Equal("data", ir.Response.Data)
+	assert.Equal("v0.9.9", ir.Response.Version)
+
+	query := r.Calls[1]
+	assert.Equal("abci_query", query.Name)
+	assert.Nil(query.Response)
+	require.NotNil(query.Error)
+	assert.Equal("query", query.Error.Error())
+	require.NotNil(query.Args)
+	qa, ok := query.Args.(mock.QueryArgs)
+	require.True(ok)
+	assert.Equal("path", qa.Path)
+	assert.EqualValues("data", qa.Data)
+	assert.True(qa.Prove)
+
+	// now add some broadcasts
+	txs := []types.Tx{{1}, {2}, {3}}
+	r.BroadcastTxCommit(txs[0])
+	r.BroadcastTxSync(txs[1])
+	r.BroadcastTxAsync(txs[2])
+
+	require.Equal(5, len(r.Calls))
+
+	bc := r.Calls[2]
+	assert.Equal("broadcast_tx_commit", bc.Name)
+	assert.Nil(bc.Response)
+	require.NotNil(bc.Error)
+	assert.EqualValues(bc.Args, txs[0])
+
+	bs := r.Calls[3]
+	assert.Equal("broadcast_tx_sync", bs.Name)
+	assert.Nil(bs.Response)
+	require.NotNil(bs.Error)
+	assert.EqualValues(bs.Args, txs[1])
+
+	ba := r.Calls[4]
+	assert.Equal("broadcast_tx_async", ba.Name)
+	assert.Nil(ba.Response)
+	require.NotNil(ba.Error)
+	assert.EqualValues(ba.Args, txs[2])
+
+}
diff --git a/rpc/client/mock/client.go b/rpc/client/mock/client.go
new file mode 100644
index 00000000..ddeb1219
--- /dev/null
+++ b/rpc/client/mock/client.go
@@ -0,0 +1,169 @@
+/*
+package mock returns a Client implementation that
+accepts various (mock) implementations of the various methods.
+
+This implementation is useful for using in tests, when you don't
+need a real server, but want a high-level of control about
+the server response you want to mock (eg. error handling),
+or if you just want to record the calls to verify in your tests.
+
+For real clients, you probably want the "http" package.  If you
+want to directly call a tendermint node in process, you can use the
+"local" package.
+*/
+package mock
+
+import (
+	"reflect"
+
+	"github.com/tendermint/tendermint/rpc/client"
+	"github.com/tendermint/tendermint/rpc/core"
+	ctypes "github.com/tendermint/tendermint/rpc/core/types"
+	"github.com/tendermint/tendermint/types"
+)
+
+// Client wraps arbitrary implementations of the various interfaces.
+//
+// We provide a few choices to mock out each one in this package.
+// Nothing hidden here, so no New function, just construct it from
+// some parts, and swap them out them during the tests.
+type Client struct {
+	client.ABCIClient
+	client.SignClient
+	client.HistoryClient
+	client.StatusClient
+}
+
+func (c Client) _assertIsClient() client.Client {
+	return c
+}
+
+// Call is used by recorders to save a call and response.
+// It can also be used to configure mock responses.
+//
+type Call struct {
+	Name     string
+	Args     interface{}
+	Response interface{}
+	Error    error
+}
+
+// GetResponse will generate the apporiate response for us, when
+// using the Call struct to configure a Mock handler.
+//
+// When configuring a response, if only one of Response or Error is
+// set then that will always be returned. If both are set, then
+// we return Response if the Args match the set args, Error otherwise.
+func (c Call) GetResponse(args interface{}) (interface{}, error) {
+	// handle the case with no response
+	if c.Response == nil {
+		if c.Error == nil {
+			panic("Misconfigured call, you must set either Response or Error")
+		}
+		return nil, c.Error
+	}
+	// response without error
+	if c.Error == nil {
+		return c.Response, nil
+	}
+	// have both, we must check args....
+	if reflect.DeepEqual(args, c.Args) {
+		return c.Response, nil
+	}
+	return nil, c.Error
+}
+
+func (c Client) Status() (*ctypes.ResultStatus, error) {
+	return core.Status()
+}
+
+func (c Client) ABCIInfo() (*ctypes.ResultABCIInfo, error) {
+	return core.ABCIInfo()
+}
+
+func (c Client) ABCIQuery(path string, data []byte, prove bool) (*ctypes.ResultABCIQuery, error) {
+	return core.ABCIQuery(path, data, prove)
+}
+
+func (c Client) BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) {
+	return core.BroadcastTxCommit(tx)
+}
+
+func (c Client) BroadcastTxAsync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) {
+	return core.BroadcastTxAsync(tx)
+}
+
+func (c Client) BroadcastTxSync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) {
+	return core.BroadcastTxSync(tx)
+}
+
+func (c Client) NetInfo() (*ctypes.ResultNetInfo, error) {
+	return core.NetInfo()
+}
+
+func (c Client) DialSeeds(seeds []string) (*ctypes.ResultDialSeeds, error) {
+	return core.UnsafeDialSeeds(seeds)
+}
+
+func (c Client) BlockchainInfo(minHeight, maxHeight int) (*ctypes.ResultBlockchainInfo, error) {
+	return core.BlockchainInfo(minHeight, maxHeight)
+}
+
+func (c Client) Genesis() (*ctypes.ResultGenesis, error) {
+	return core.Genesis()
+}
+
+func (c Client) Block(height int) (*ctypes.ResultBlock, error) {
+	return core.Block(height)
+}
+
+func (c Client) Commit(height int) (*ctypes.ResultCommit, error) {
+	return core.Commit(height)
+}
+
+func (c Client) Validators() (*ctypes.ResultValidators, error) {
+	return core.Validators()
+}
+
+/** websocket event stuff here... **/
+
+/*
+// StartWebsocket starts up a websocket and a listener goroutine
+// if already started, do nothing
+func (c Client) StartWebsocket() error {
+	var err error
+	if c.ws == nil {
+		ws := rpcclient.NewWSClient(c.remote, c.endpoint)
+		_, err = ws.Start()
+		if err == nil {
+			c.ws = ws
+		}
+	}
+	return errors.Wrap(err, "StartWebsocket")
+}
+
+// StopWebsocket stops the websocket connection
+func (c Client) StopWebsocket() {
+	if c.ws != nil {
+		c.ws.Stop()
+		c.ws = nil
+	}
+}
+
+// GetEventChannels returns the results and error channel from the websocket
+func (c Client) GetEventChannels() (chan json.RawMessage, chan error) {
+	if c.ws == nil {
+		return nil, nil
+	}
+	return c.ws.ResultsCh, c.ws.ErrorsCh
+}
+
+func (c Client) Subscribe(event string) error {
+	return errors.Wrap(c.ws.Subscribe(event), "Subscribe")
+}
+
+func (c Client) Unsubscribe(event string) error {
+	return errors.Wrap(c.ws.Unsubscribe(event), "Unsubscribe")
+}
+
+*/

From 0905332f1d1efd750d62e45e9b00e6474e7a4b66 Mon Sep 17 00:00:00 2001
From: Ethan Frey <ethanfrey@users.noreply.github.com>
Date: Wed, 22 Feb 2017 16:49:45 +0100
Subject: [PATCH 082/132] MockClient for real abci app

---
 Makefile                     |  4 ++--
 rpc/client/mock/abci_test.go | 26 +++++++++++++++++++++++++-
 2 files changed, 27 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index bfdaf007..9985c39a 100644
--- a/Makefile
+++ b/Makefile
@@ -22,11 +22,11 @@ dist:
 
 test:
 	@echo "--> Running go test"
-	@go test $(PACKAGES)
+	@go test -p 1 $(PACKAGES)
 
 test_race:
 	@echo "--> Running go test --race"
-	@go test -v -race $(PACKAGES)
+	@go test -p 1 -v -race $(PACKAGES)
 
 test_integrations:
 	@bash ./test/test.sh
diff --git a/rpc/client/mock/abci_test.go b/rpc/client/mock/abci_test.go
index 50e767ed..ce9a1272 100644
--- a/rpc/client/mock/abci_test.go
+++ b/rpc/client/mock/abci_test.go
@@ -2,10 +2,12 @@ package mock_test
 
 import (
 	"errors"
+	"fmt"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"github.com/tendermint/abci/example/dummy"
 	abci "github.com/tendermint/abci/types"
 	ctypes "github.com/tendermint/tendermint/rpc/core/types"
 	"github.com/tendermint/tendermint/types"
@@ -138,5 +140,27 @@ func TestABCIRecorder(t *testing.T) {
 	assert.Nil(ba.Response)
 	require.NotNil(ba.Error)
 	assert.EqualValues(ba.Args, txs[2])
-
+}
+
+func TestABCIApp(t *testing.T) {
+	assert, require := assert.New(t), require.New(t)
+	app := dummy.NewDummyApplication()
+	m := mock.ABCIApp{app}
+
+	// get some info
+	info, err := m.ABCIInfo()
+	require.Nil(err)
+	assert.Equal(`{"size":0}`, info.Response.GetData())
+
+	// add a key
+	key, value := "foo", "bar"
+	tx := fmt.Sprintf("%s=%s", key, value)
+	res, err := m.BroadcastTxSync(types.Tx(tx))
+	require.Nil(err)
+	assert.True(res.Code.IsOK())
+
+	// check the key
+	qres, err := m.ABCIQuery("/key", []byte(key), false)
+	require.Nil(err)
+	assert.EqualValues(value, qres.Response.Value)
 }

From 70f19e809b9363d5e9964db9ef42cfc165121a9b Mon Sep 17 00:00:00 2001
From: Ethan Frey <ethanfrey@users.noreply.github.com>
Date: Wed, 22 Feb 2017 17:39:53 +0100
Subject: [PATCH 083/132] Add MockStatus, WaitForHeight helper

---
 rpc/client/helpers.go          | 31 ++++++++++++++
 rpc/client/helpers_test.go     | 75 ++++++++++++++++++++++++++++++++++
 rpc/client/mock/abci_test.go   |  3 +-
 rpc/client/mock/status.go      | 55 +++++++++++++++++++++++++
 rpc/client/mock/status_test.go | 45 ++++++++++++++++++++
 5 files changed, 208 insertions(+), 1 deletion(-)
 create mode 100644 rpc/client/helpers.go
 create mode 100644 rpc/client/helpers_test.go
 create mode 100644 rpc/client/mock/status.go
 create mode 100644 rpc/client/mock/status_test.go

diff --git a/rpc/client/helpers.go b/rpc/client/helpers.go
new file mode 100644
index 00000000..555fa386
--- /dev/null
+++ b/rpc/client/helpers.go
@@ -0,0 +1,31 @@
+package client
+
+import (
+	"time"
+
+	"github.com/pkg/errors"
+)
+
+// Wait for height will poll status at reasonable intervals until
+// the block at the given height is available.
+func WaitForHeight(c StatusClient, h int) error {
+	wait := 1
+	for wait > 0 {
+		s, err := c.Status()
+		if err != nil {
+			return err
+		}
+		wait = h - s.LatestBlockHeight
+		if wait > 10 {
+			return errors.Errorf("Waiting for %d block... aborting", wait)
+		} else if wait > 0 {
+			// estimate of wait time....
+			// wait half a second for the next block (in progress)
+			// plus one second for every full block
+			delay := time.Duration(wait-1)*time.Second + 500*time.Millisecond
+			time.Sleep(delay)
+		}
+	}
+	// guess we waited long enough
+	return nil
+}
diff --git a/rpc/client/helpers_test.go b/rpc/client/helpers_test.go
new file mode 100644
index 00000000..9b3607a8
--- /dev/null
+++ b/rpc/client/helpers_test.go
@@ -0,0 +1,75 @@
+package client_test
+
+import (
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/tendermint/tendermint/rpc/client"
+	"github.com/tendermint/tendermint/rpc/client/mock"
+	ctypes "github.com/tendermint/tendermint/rpc/core/types"
+)
+
+func TestWaitForHeight(t *testing.T) {
+	assert, require := assert.New(t), require.New(t)
+
+	// test with error result - immediate failure
+	m := &mock.StatusMock{
+		Call: mock.Call{
+			Error: errors.New("bye"),
+		},
+	}
+	r := mock.NewStatusRecorder(m)
+
+	// connection failure always leads to error
+	err := client.WaitForHeight(r, 8)
+	require.NotNil(err)
+	require.Equal("bye", err.Error())
+	// we called status once to check
+	require.Equal(1, len(r.Calls))
+
+	// now set current block height to 10
+	m.Call = mock.Call{
+		Response: &ctypes.ResultStatus{LatestBlockHeight: 10},
+	}
+
+	// we will not wait for more than 10 blocks
+	err = client.WaitForHeight(r, 40)
+	require.NotNil(err)
+	require.True(strings.Contains(err.Error(), "aborting"))
+	// we called status once more to check
+	require.Equal(2, len(r.Calls))
+
+	// waiting for the past returns immediately
+	err = client.WaitForHeight(r, 5)
+	require.Nil(err)
+	// we called status once more to check
+	require.Equal(3, len(r.Calls))
+
+	// next tick in background while we wait
+	go func() {
+		time.Sleep(500 * time.Millisecond)
+		m.Call.Response = &ctypes.ResultStatus{LatestBlockHeight: 15}
+	}()
+
+	// we wait for a few blocks
+	err = client.WaitForHeight(r, 12)
+	require.Nil(err)
+	// we called status once to check
+	require.Equal(5, len(r.Calls))
+
+	pre := r.Calls[3]
+	require.Nil(pre.Error)
+	prer, ok := pre.Response.(*ctypes.ResultStatus)
+	require.True(ok)
+	assert.Equal(10, prer.LatestBlockHeight)
+
+	post := r.Calls[4]
+	require.Nil(post.Error)
+	postr, ok := post.Response.(*ctypes.ResultStatus)
+	require.True(ok)
+	assert.Equal(15, postr.LatestBlockHeight)
+}
diff --git a/rpc/client/mock/abci_test.go b/rpc/client/mock/abci_test.go
index ce9a1272..77ad496b 100644
--- a/rpc/client/mock/abci_test.go
+++ b/rpc/client/mock/abci_test.go
@@ -1,10 +1,11 @@
 package mock_test
 
 import (
-	"errors"
 	"fmt"
 	"testing"
 
+	"github.com/pkg/errors"
+
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/tendermint/abci/example/dummy"
diff --git a/rpc/client/mock/status.go b/rpc/client/mock/status.go
new file mode 100644
index 00000000..af0f5335
--- /dev/null
+++ b/rpc/client/mock/status.go
@@ -0,0 +1,55 @@
+package mock
+
+import (
+	"github.com/tendermint/tendermint/rpc/client"
+	ctypes "github.com/tendermint/tendermint/rpc/core/types"
+)
+
+// StatusMock returns the result specified by the Call
+type StatusMock struct {
+	Call
+}
+
+func (m *StatusMock) _assertStatusClient() client.StatusClient {
+	return m
+}
+
+func (m *StatusMock) Status() (*ctypes.ResultStatus, error) {
+	res, err := m.GetResponse(nil)
+	if err != nil {
+		return nil, err
+	}
+	return res.(*ctypes.ResultStatus), nil
+}
+
+// StatusRecorder can wrap another type (StatusMock, full client)
+// and record the status calls
+type StatusRecorder struct {
+	Client client.StatusClient
+	Calls  []Call
+}
+
+func NewStatusRecorder(client client.StatusClient) *StatusRecorder {
+	return &StatusRecorder{
+		Client: client,
+		Calls:  []Call{},
+	}
+}
+
+func (r *StatusRecorder) _assertStatusClient() client.StatusClient {
+	return r
+}
+
+func (r *StatusRecorder) addCall(call Call) {
+	r.Calls = append(r.Calls, call)
+}
+
+func (r *StatusRecorder) Status() (*ctypes.ResultStatus, error) {
+	res, err := r.Client.Status()
+	r.addCall(Call{
+		Name:     "status",
+		Response: res,
+		Error:    err,
+	})
+	return res, err
+}
diff --git a/rpc/client/mock/status_test.go b/rpc/client/mock/status_test.go
new file mode 100644
index 00000000..3e695cd5
--- /dev/null
+++ b/rpc/client/mock/status_test.go
@@ -0,0 +1,45 @@
+package mock_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	ctypes "github.com/tendermint/tendermint/rpc/core/types"
+
+	"github.com/tendermint/tendermint/rpc/client/mock"
+)
+
+func TestStatus(t *testing.T) {
+	assert, require := assert.New(t), require.New(t)
+
+	m := &mock.StatusMock{
+		Call: mock.Call{
+			Response: &ctypes.ResultStatus{
+				LatestBlockHash:   []byte("block"),
+				LatestAppHash:     []byte("app"),
+				LatestBlockHeight: 10,
+			}},
+	}
+
+	r := mock.NewStatusRecorder(m)
+	require.Equal(0, len(r.Calls))
+
+	// make sure response works proper
+	status, err := r.Status()
+	require.Nil(err, "%+v", err)
+	assert.EqualValues("block", status.LatestBlockHash)
+	assert.EqualValues(10, status.LatestBlockHeight)
+
+	// make sure recorder works properly
+	require.Equal(1, len(r.Calls))
+	rs := r.Calls[0]
+	assert.Equal("status", rs.Name)
+	assert.Nil(rs.Args)
+	assert.Nil(rs.Error)
+	require.NotNil(rs.Response)
+	st, ok := rs.Response.(*ctypes.ResultStatus)
+	require.True(ok)
+	assert.EqualValues("block", st.LatestBlockHash)
+	assert.EqualValues(10, st.LatestBlockHeight)
+}

From 26f4b5c98e7a7e4b3a400c124c6e92808cb2380d Mon Sep 17 00:00:00 2001
From: Ethan Frey <ethanfrey@users.noreply.github.com>
Date: Wed, 22 Feb 2017 17:51:18 +0100
Subject: [PATCH 084/132] Clean up package names

---
 rpc/client/http/app_test.go   | 2 +-
 rpc/client/http/client.go     | 4 ++--
 rpc/client/http/main_test.go  | 2 +-
 rpc/client/http/rpc_test.go   | 8 ++++----
 rpc/client/local/app_test.go  | 2 +-
 rpc/client/local/client.go    | 4 ++--
 rpc/client/local/main_test.go | 2 +-
 rpc/client/local/rpc_test.go  | 8 ++++----
 8 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/rpc/client/http/app_test.go b/rpc/client/http/app_test.go
index 2d599cbc..f9034144 100644
--- a/rpc/client/http/app_test.go
+++ b/rpc/client/http/app_test.go
@@ -1,4 +1,4 @@
-package httpclient_test
+package http_test
 
 import (
 	"math/rand"
diff --git a/rpc/client/http/client.go b/rpc/client/http/client.go
index f5716267..b8fb0c4e 100644
--- a/rpc/client/http/client.go
+++ b/rpc/client/http/client.go
@@ -1,5 +1,5 @@
 /*
-package httpclient returns a Client implementation that communicates
+package http returns a Client implementation that communicates
 with a tendermint node over json rpc and websockets.
 
 This is the main implementation you probably want to use in
@@ -7,7 +7,7 @@ production code.  There are other implementations when calling
 the tendermint node in-process (local), or when you want to mock
 out the server for test code (mock).
 */
-package httpclient
+package http
 
 import (
 	"encoding/json"
diff --git a/rpc/client/http/main_test.go b/rpc/client/http/main_test.go
index 0e324df4..a5e1887a 100644
--- a/rpc/client/http/main_test.go
+++ b/rpc/client/http/main_test.go
@@ -1,4 +1,4 @@
-package httpclient_test
+package http_test
 
 import (
 	"os"
diff --git a/rpc/client/http/rpc_test.go b/rpc/client/http/rpc_test.go
index 09a26096..adb42057 100644
--- a/rpc/client/http/rpc_test.go
+++ b/rpc/client/http/rpc_test.go
@@ -1,4 +1,4 @@
-package httpclient_test
+package http_test
 
 import (
 	"strings"
@@ -8,15 +8,15 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	merkle "github.com/tendermint/go-merkle"
-	httpclient "github.com/tendermint/tendermint/rpc/client/http"
+	"github.com/tendermint/tendermint/rpc/client/http"
 	rpctest "github.com/tendermint/tendermint/rpc/test"
 	"github.com/tendermint/tendermint/types"
 )
 
 // GetClient gets a rpc client pointing to the test tendermint rpc
-func GetClient() *httpclient.Client {
+func GetClient() *http.Client {
 	rpcAddr := rpctest.GetConfig().GetString("rpc_laddr")
-	return httpclient.New(rpcAddr, "/websocket")
+	return http.New(rpcAddr, "/websocket")
 }
 
 // Make sure status is correct (we connect properly)
diff --git a/rpc/client/local/app_test.go b/rpc/client/local/app_test.go
index bd16b375..898399ce 100644
--- a/rpc/client/local/app_test.go
+++ b/rpc/client/local/app_test.go
@@ -1,4 +1,4 @@
-package localclient_test
+package local_test
 
 import (
 	"math/rand"
diff --git a/rpc/client/local/client.go b/rpc/client/local/client.go
index f050b6b1..5a6a699e 100644
--- a/rpc/client/local/client.go
+++ b/rpc/client/local/client.go
@@ -1,5 +1,5 @@
 /*
-package localclient returns a Client implementation that
+package local returns a Client implementation that
 directly executes the rpc functions on a given node.
 
 This implementation is useful for:
@@ -12,7 +12,7 @@ are compiled in process.
 For real clients, you probably want the "http" package.  For more
 powerful control during testing, you probably want the "mock" package.
 */
-package localclient
+package local
 
 import (
 	nm "github.com/tendermint/tendermint/node"
diff --git a/rpc/client/local/main_test.go b/rpc/client/local/main_test.go
index 5e0a0c23..2a6930c3 100644
--- a/rpc/client/local/main_test.go
+++ b/rpc/client/local/main_test.go
@@ -1,4 +1,4 @@
-package localclient_test
+package local_test
 
 import (
 	"os"
diff --git a/rpc/client/local/rpc_test.go b/rpc/client/local/rpc_test.go
index 13fc5547..1a5a3829 100644
--- a/rpc/client/local/rpc_test.go
+++ b/rpc/client/local/rpc_test.go
@@ -1,4 +1,4 @@
-package localclient_test
+package local_test
 
 import (
 	"strings"
@@ -8,14 +8,14 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	merkle "github.com/tendermint/go-merkle"
-	localclient "github.com/tendermint/tendermint/rpc/client/local"
+	"github.com/tendermint/tendermint/rpc/client/local"
 	rpctest "github.com/tendermint/tendermint/rpc/test"
 )
 
 // GetClient gets a rpc client pointing to the test tendermint rpc
-func GetClient() localclient.Client {
+func GetClient() local.Client {
 	node := rpctest.GetNode()
-	return localclient.New(node)
+	return local.New(node)
 }
 
 // Make sure status is correct (we connect properly)

From f7f7cf576a5a8c11f2e85337d4716fa2a18b6b19 Mon Sep 17 00:00:00 2001
From: Ethan Frey <ethanfrey@users.noreply.github.com>
Date: Wed, 22 Feb 2017 18:32:16 +0100
Subject: [PATCH 085/132] Fix race condition in websocket event listening

---
 rpc/client/http/rpc_test.go | 54 ++++++++++++++++++++++++-------------
 1 file changed, 35 insertions(+), 19 deletions(-)

diff --git a/rpc/client/http/rpc_test.go b/rpc/client/http/rpc_test.go
index adb42057..b2c0c6e9 100644
--- a/rpc/client/http/rpc_test.go
+++ b/rpc/client/http/rpc_test.go
@@ -1,6 +1,7 @@
 package http_test
 
 import (
+	"encoding/json"
 	"strings"
 	"testing"
 	"time"
@@ -9,6 +10,7 @@ import (
 	"github.com/stretchr/testify/require"
 	merkle "github.com/tendermint/go-merkle"
 	"github.com/tendermint/tendermint/rpc/client/http"
+	ctypes "github.com/tendermint/tendermint/rpc/core/types"
 	rpctest "github.com/tendermint/tendermint/rpc/test"
 	"github.com/tendermint/tendermint/types"
 )
@@ -142,7 +144,7 @@ func TestAppCalls(t *testing.T) {
 }
 
 func TestSubscriptions(t *testing.T) {
-	assert, require := assert.New(t), require.New(t)
+	require := require.New(t)
 	c := GetClient()
 	err := c.StartWebsocket()
 	require.Nil(err)
@@ -153,30 +155,44 @@ func TestSubscriptions(t *testing.T) {
 	// this causes a panic in tendermint core!!!
 	eventType := types.EventStringTx(types.Tx(tx))
 	c.Subscribe(eventType)
-	read := 0
 
 	// set up a listener
 	r, e := c.GetEventChannels()
 	go func() {
-		// read one event in the background
-		select {
-		case <-r:
-			// TODO: actually parse this or something
-			read += 1
-		case err := <-e:
-			panic(err)
-		}
+		// send a tx and wait for it to propogate
+		_, err = c.BroadcastTxCommit(tx)
+		require.Nil(err, string(tx))
 	}()
 
-	// make sure nothing has happened yet.
-	assert.Equal(0, read)
+	checkData := func(data []byte, kind byte) {
+		x := []interface{}{}
+		err := json.Unmarshal(data, &x)
+		require.Nil(err)
+		// gotta love wire's json format
+		require.EqualValues(kind, x[0])
+	}
 
-	// send a tx and wait for it to propogate
-	_, err = c.BroadcastTxCommit(tx)
-	assert.Nil(err, string(tx))
-	// wait before querying
-	time.Sleep(time.Second)
+	res := <-r
+	checkData(res, ctypes.ResultTypeSubscribe)
+
+	// read one event, must be success
+	select {
+	case res := <-r:
+		checkData(res, ctypes.ResultTypeEvent)
+		// this is good.. let's get the data... ugh...
+		// result := new(ctypes.TMResult)
+		// wire.ReadJSON(result, res, &err)
+		// require.Nil(err, "%+v", err)
+		// event, ok := (*result).(*ctypes.ResultEvent)
+		// require.True(ok)
+		// assert.Equal("foo", event.Name)
+		// data, ok := event.Data.(types.EventDataTx)
+		// require.True(ok)
+		// assert.EqualValues(0, data.Code)
+		// assert.EqualValues(tx, data.Tx)
+	case err := <-e:
+		// this is a failure
+		require.Nil(err)
+	}
 
-	// now make sure the event arrived
-	assert.Equal(1, read)
 }

From d92a5b1074eb96cf49d1d3ca7b6dc9a75360b297 Mon Sep 17 00:00:00 2001
From: Ethan Frey <ethanfrey@users.noreply.github.com>
Date: Wed, 22 Feb 2017 19:14:46 +0100
Subject: [PATCH 086/132] Reworked WaitForHeight to avoid race conditions

---
 rpc/client/helpers.go      | 44 +++++++++++++++++++++++++++-----------
 rpc/client/helpers_test.go | 19 ++++++++--------
 2 files changed, 41 insertions(+), 22 deletions(-)

diff --git a/rpc/client/helpers.go b/rpc/client/helpers.go
index 555fa386..bf8860e7 100644
--- a/rpc/client/helpers.go
+++ b/rpc/client/helpers.go
@@ -6,26 +6,44 @@ import (
 	"github.com/pkg/errors"
 )
 
+// Waiter is informed of current height, decided whether to quit early
+type Waiter func(delta int) (abort error)
+
+// DefaultWaitStrategy is the standard backoff algorithm,
+// but you can plug in another one
+func DefaultWaitStrategy(delta int) (abort error) {
+	if delta > 10 {
+		return errors.Errorf("Waiting for %d blocks... aborting", delta)
+	} else if delta > 0 {
+		// estimate of wait time....
+		// wait half a second for the next block (in progress)
+		// plus one second for every full block
+		delay := time.Duration(delta-1)*time.Second + 500*time.Millisecond
+		time.Sleep(delay)
+	}
+	return nil
+}
+
 // Wait for height will poll status at reasonable intervals until
 // the block at the given height is available.
-func WaitForHeight(c StatusClient, h int) error {
-	wait := 1
-	for wait > 0 {
+//
+// If waiter is nil, we use DefaultWaitStrategy, but you can also
+// provide your own implementation
+func WaitForHeight(c StatusClient, h int, waiter Waiter) error {
+	if waiter == nil {
+		waiter = DefaultWaitStrategy
+	}
+	delta := 1
+	for delta > 0 {
 		s, err := c.Status()
 		if err != nil {
 			return err
 		}
-		wait = h - s.LatestBlockHeight
-		if wait > 10 {
-			return errors.Errorf("Waiting for %d block... aborting", wait)
-		} else if wait > 0 {
-			// estimate of wait time....
-			// wait half a second for the next block (in progress)
-			// plus one second for every full block
-			delay := time.Duration(wait-1)*time.Second + 500*time.Millisecond
-			time.Sleep(delay)
+		delta = h - s.LatestBlockHeight
+		// wait for the time, or abort early
+		if err := waiter(delta); err != nil {
+			return err
 		}
 	}
-	// guess we waited long enough
 	return nil
 }
diff --git a/rpc/client/helpers_test.go b/rpc/client/helpers_test.go
index 9b3607a8..fe186122 100644
--- a/rpc/client/helpers_test.go
+++ b/rpc/client/helpers_test.go
@@ -4,7 +4,6 @@ import (
 	"errors"
 	"strings"
 	"testing"
-	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -25,7 +24,7 @@ func TestWaitForHeight(t *testing.T) {
 	r := mock.NewStatusRecorder(m)
 
 	// connection failure always leads to error
-	err := client.WaitForHeight(r, 8)
+	err := client.WaitForHeight(r, 8, nil)
 	require.NotNil(err)
 	require.Equal("bye", err.Error())
 	// we called status once to check
@@ -37,26 +36,28 @@ func TestWaitForHeight(t *testing.T) {
 	}
 
 	// we will not wait for more than 10 blocks
-	err = client.WaitForHeight(r, 40)
+	err = client.WaitForHeight(r, 40, nil)
 	require.NotNil(err)
 	require.True(strings.Contains(err.Error(), "aborting"))
 	// we called status once more to check
 	require.Equal(2, len(r.Calls))
 
 	// waiting for the past returns immediately
-	err = client.WaitForHeight(r, 5)
+	err = client.WaitForHeight(r, 5, nil)
 	require.Nil(err)
 	// we called status once more to check
 	require.Equal(3, len(r.Calls))
 
-	// next tick in background while we wait
-	go func() {
-		time.Sleep(500 * time.Millisecond)
+	// since we can't update in a background goroutine (test --race)
+	// we use the callback to update the status height
+	myWaiter := func(delta int) error {
+		// update the height for the next call
 		m.Call.Response = &ctypes.ResultStatus{LatestBlockHeight: 15}
-	}()
+		return client.DefaultWaitStrategy(delta)
+	}
 
 	// we wait for a few blocks
-	err = client.WaitForHeight(r, 12)
+	err = client.WaitForHeight(r, 12, myWaiter)
 	require.Nil(err)
 	// we called status once to check
 	require.Equal(5, len(r.Calls))

From 42a9b847ec81449b9c39d09c0533d1ad0b42b579 Mon Sep 17 00:00:00 2001
From: Ethan Frey <ethanfrey@users.noreply.github.com>
Date: Wed, 22 Feb 2017 20:01:32 +0100
Subject: [PATCH 087/132] Make all client tests safe to run in parallel

---
 Makefile                    |  4 ++--
 rpc/client/http/rpc_test.go | 29 +++++++++++++++++++++--------
 rpc/test/helpers.go         | 37 ++++++++++++++++++++++++++++++++++++-
 3 files changed, 59 insertions(+), 11 deletions(-)

diff --git a/Makefile b/Makefile
index 9985c39a..bfdaf007 100644
--- a/Makefile
+++ b/Makefile
@@ -22,11 +22,11 @@ dist:
 
 test:
 	@echo "--> Running go test"
-	@go test -p 1 $(PACKAGES)
+	@go test $(PACKAGES)
 
 test_race:
 	@echo "--> Running go test --race"
-	@go test -p 1 -v -race $(PACKAGES)
+	@go test -v -race $(PACKAGES)
 
 test_integrations:
 	@bash ./test/test.sh
diff --git a/rpc/client/http/rpc_test.go b/rpc/client/http/rpc_test.go
index b2c0c6e9..4f48c7a2 100644
--- a/rpc/client/http/rpc_test.go
+++ b/rpc/client/http/rpc_test.go
@@ -38,7 +38,7 @@ func TestInfo(t *testing.T) {
 	info, err := c.ABCIInfo()
 	require.Nil(t, err, "%+v", err)
 	assert.EqualValues(t, status.LatestBlockHeight, info.Response.LastBlockHeight)
-	assert.True(t, strings.HasPrefix(info.Response.Data, "size:"))
+	assert.True(t, strings.HasPrefix(info.Response.Data, "size"))
 }
 
 func TestNetInfo(t *testing.T) {
@@ -83,8 +83,18 @@ func TestGenesisAndValidators(t *testing.T) {
 func TestAppCalls(t *testing.T) {
 	assert, require := assert.New(t), require.New(t)
 	c := GetClient()
-	_, err := c.Block(1)
+
+	// get an offset of height to avoid racing and guessing
+	s, err := c.Status()
+	require.Nil(err)
+	// sh is start height or status height
+	sh := s.LatestBlockHeight
+
+	// look for the future
+	_, err = c.Block(sh + 2)
 	assert.NotNil(err) // no block yet
+
+	// write something
 	k, v, tx := MakeTxKV()
 	_, err = c.BroadcastTxCommit(tx)
 	require.Nil(err, "%+v", err)
@@ -96,36 +106,39 @@ func TestAppCalls(t *testing.T) {
 		// assert.Equal(k, data.GetKey())  // only returned for proofs
 		assert.Equal(v, data.GetValue())
 	}
+	// +/- 1 making my head hurt
+	h := int(qres.Response.Height) - 1
+
 	// and we can even check the block is added
-	block, err := c.Block(3)
+	block, err := c.Block(h)
 	require.Nil(err, "%+v", err)
 	appHash := block.BlockMeta.Header.AppHash
 	assert.True(len(appHash) > 0)
-	assert.EqualValues(3, block.BlockMeta.Header.Height)
+	assert.EqualValues(h, block.BlockMeta.Header.Height)
 
 	// check blockchain info, now that we know there is info
 	// TODO: is this commented somewhere that they are returned
 	// in order of descending height???
-	info, err := c.BlockchainInfo(1, 3)
+	info, err := c.BlockchainInfo(h-2, h)
 	require.Nil(err, "%+v", err)
 	assert.True(info.LastHeight > 2)
 	if assert.Equal(3, len(info.BlockMetas)) {
 		lastMeta := info.BlockMetas[0]
-		assert.EqualValues(3, lastMeta.Header.Height)
+		assert.EqualValues(h, lastMeta.Header.Height)
 		bMeta := block.BlockMeta
 		assert.Equal(bMeta.Header.AppHash, lastMeta.Header.AppHash)
 		assert.Equal(bMeta.BlockID, lastMeta.BlockID)
 	}
 
 	// and get the corresponding commit with the same apphash
-	commit, err := c.Commit(3)
+	commit, err := c.Commit(h)
 	require.Nil(err, "%+v", err)
 	cappHash := commit.Header.AppHash
 	assert.Equal(appHash, cappHash)
 	assert.NotNil(commit.Commit)
 
 	// compare the commits (note Commit(2) has commit from Block(3))
-	commit2, err := c.Commit(2)
+	commit2, err := c.Commit(h - 1)
 	require.Nil(err, "%+v", err)
 	assert.Equal(block.Block.LastCommit, commit2.Commit)
 
diff --git a/rpc/test/helpers.go b/rpc/test/helpers.go
index e835a393..f51c83f9 100644
--- a/rpc/test/helpers.go
+++ b/rpc/test/helpers.go
@@ -2,6 +2,10 @@ package rpctest
 
 import (
 	"fmt"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"strings"
 	"testing"
 	"time"
 
@@ -27,12 +31,43 @@ var (
 
 const tmLogLevel = "error"
 
+// f**ing long, but unique for each test
+func makePathname() string {
+	// get path
+	p, err := os.Getwd()
+	if err != nil {
+		panic(err)
+	}
+	fmt.Println(p)
+	sep := string(filepath.Separator)
+	return strings.Replace(p, sep, "_", -1)
+}
+
+func randPort() int {
+	// returns between base and base + spread
+	base, spread := 20000, 20000
+	return base + rand.Intn(spread)
+}
+
+func makeAddrs() (string, string, string) {
+	start := randPort()
+	return fmt.Sprintf("tcp://0.0.0.0:%d", start),
+		fmt.Sprintf("tcp://0.0.0.0:%d", start+1),
+		fmt.Sprintf("tcp://0.0.0.0:%d", start+2)
+}
+
 // GetConfig returns a config for the test cases as a singleton
 func GetConfig() cfg.Config {
 	if config == nil {
-		config = tendermint_test.ResetConfig("rpc_test_client_test")
+		pathname := makePathname()
+		config = tendermint_test.ResetConfig(pathname)
 		// Shut up the logging
 		logger.SetLogLevel(tmLogLevel)
+		// and we use random ports to run in parallel
+		tm, rpc, grpc := makeAddrs()
+		config.Set("node_laddr", tm)
+		config.Set("rpc_laddr", rpc)
+		config.Set("grpc_laddr", grpc)
 	}
 	return config
 }

From cd9ee9d84b7f7daa678e42411ac31a0713f43230 Mon Sep 17 00:00:00 2001
From: Ethan Frey <ethanfrey@users.noreply.github.com>
Date: Wed, 22 Feb 2017 22:52:16 +0100
Subject: [PATCH 088/132] cleanup

---
 rpc/client/local/main_test.go | 15 ++++++++-------
 rpc/client/local/rpc_test.go  |  4 +---
 rpc/test/helpers.go           | 15 ++-------------
 rpc/test/main_test.go         |  5 ++++-
 4 files changed, 15 insertions(+), 24 deletions(-)

diff --git a/rpc/client/local/main_test.go b/rpc/client/local/main_test.go
index 2a6930c3..5797843f 100644
--- a/rpc/client/local/main_test.go
+++ b/rpc/client/local/main_test.go
@@ -5,17 +5,18 @@ import (
 	"testing"
 
 	meapp "github.com/tendermint/merkleeyes/app"
+	nm "github.com/tendermint/tendermint/node"
 	rpctest "github.com/tendermint/tendermint/rpc/test"
 )
 
-func TestMain(m *testing.M) {
-	// start a tendermint node (and merkleeyes) in the background to test against
-	app := meapp.NewMerkleEyesApp("", 100)
-	node := rpctest.StartTendermint(app)
-	code := m.Run()
+var node *nm.Node
 
+func TestMain(m *testing.M) {
+	// configure a node, but don't start the server
+	app := meapp.NewMerkleEyesApp("", 100)
+	node = rpctest.StartTendermint(app)
+
+	code := m.Run()
 	// and shut down proper at the end
-	node.Stop()
-	node.Wait()
 	os.Exit(code)
 }
diff --git a/rpc/client/local/rpc_test.go b/rpc/client/local/rpc_test.go
index 1a5a3829..069db06f 100644
--- a/rpc/client/local/rpc_test.go
+++ b/rpc/client/local/rpc_test.go
@@ -14,17 +14,15 @@ import (
 
 // GetClient gets a rpc client pointing to the test tendermint rpc
 func GetClient() local.Client {
-	node := rpctest.GetNode()
 	return local.New(node)
 }
 
 // Make sure status is correct (we connect properly)
 func TestStatus(t *testing.T) {
 	c := GetClient()
-	chainID := rpctest.GetConfig().GetString("chain_id")
 	status, err := c.Status()
 	require.Nil(t, err, "%+v", err)
-	assert.Equal(t, chainID, status.NodeInfo.Network)
+	require.NotNil(t, status.PubKey)
 }
 
 // Make sure info is correct (we connect properly)
diff --git a/rpc/test/helpers.go b/rpc/test/helpers.go
index f51c83f9..cf64ac15 100644
--- a/rpc/test/helpers.go
+++ b/rpc/test/helpers.go
@@ -26,7 +26,6 @@ import (
 
 var (
 	config cfg.Config
-	node   *nm.Node
 )
 
 const tmLogLevel = "error"
@@ -72,10 +71,6 @@ func GetConfig() cfg.Config {
 	return config
 }
 
-func GetNode() *nm.Node {
-	return node
-}
-
 // GetURIClient gets a uri client pointing to the test tendermint rpc
 func GetURIClient() *client.ClientURI {
 	rpcAddr := GetConfig().GetString("rpc_laddr")
@@ -103,12 +98,9 @@ func GetWSClient() *client.WSClient {
 }
 
 // StartTendermint starts a test tendermint server in a go routine and returns when it is initialized
-// TODO: can one pass an Application in????
 func StartTendermint(app abci.Application) *nm.Node {
-	// start a node
-	fmt.Println("Starting Tendermint...")
-
-	node = NewTendermint(app)
+	node := NewTendermint(app)
+	node.Start()
 	fmt.Println("Tendermint running!")
 	return node
 }
@@ -121,9 +113,6 @@ func NewTendermint(app abci.Application) *nm.Node {
 	privValidator := types.LoadOrGenPrivValidator(privValidatorFile)
 	papp := proxy.NewLocalClientCreator(app)
 	node := nm.NewNode(config, privValidator, papp)
-
-	// node.Start now does everything including the RPC server
-	node.Start()
 	return node
 }
 
diff --git a/rpc/test/main_test.go b/rpc/test/main_test.go
index e78cd17a..f95176d9 100644
--- a/rpc/test/main_test.go
+++ b/rpc/test/main_test.go
@@ -18,12 +18,15 @@ import (
 	"testing"
 
 	"github.com/tendermint/abci/example/dummy"
+	nm "github.com/tendermint/tendermint/node"
 )
 
+var node *nm.Node
+
 func TestMain(m *testing.M) {
 	// start a tendermint node (and merkleeyes) in the background to test against
 	app := dummy.NewDummyApplication()
-	node := StartTendermint(app)
+	node = StartTendermint(app)
 	code := m.Run()
 
 	// and shut down proper at the end

From 9693795c4cc45617dc336562957dc06960ff69a1 Mon Sep 17 00:00:00 2001
From: Ethan Frey <ethanfrey@users.noreply.github.com>
Date: Thu, 23 Feb 2017 13:43:28 +0100
Subject: [PATCH 089/132] Move common code to merkleeyes/testutil

---
 glide.lock                  |  2 +-
 rpc/client/http/app_test.go | 45 -------------------------------------
 rpc/client/http/rpc_test.go |  6 ++---
 3 files changed, 4 insertions(+), 49 deletions(-)
 delete mode 100644 rpc/client/http/app_test.go

diff --git a/glide.lock b/glide.lock
index 3d72ddc0..28082d2c 100644
--- a/glide.lock
+++ b/glide.lock
@@ -141,7 +141,7 @@ imports:
   - transport
 testImports:
 - name: github.com/tendermint/merkleeyes
-  version: 87b0a111a716f1495f30ce58bd469e36ac220e09
+  version: acd8e9c42e1d819c51e9e1cd3870ea4d94b167f5
   subpackages:
   - app
 - name: github.com/stretchr/testify
diff --git a/rpc/client/http/app_test.go b/rpc/client/http/app_test.go
deleted file mode 100644
index f9034144..00000000
--- a/rpc/client/http/app_test.go
+++ /dev/null
@@ -1,45 +0,0 @@
-package http_test
-
-import (
-	"math/rand"
-
-	meapp "github.com/tendermint/merkleeyes/app"
-
-	wire "github.com/tendermint/go-wire"
-)
-
-// MakeTxKV returns a text transaction, allong with expected key, value pair
-func MakeTxKV() ([]byte, []byte, []byte) {
-	k := RandAsciiBytes(8)
-	v := RandAsciiBytes(8)
-	return k, v, makeSet(k, v)
-}
-
-// blatently copied from merkleeyes/app/app_test.go
-// constructs a "set" transaction
-func makeSet(key, value []byte) []byte {
-	tx := make([]byte, 1+wire.ByteSliceSize(key)+wire.ByteSliceSize(value))
-	buf := tx
-	buf[0] = meapp.WriteSet // Set TypeByte
-	buf = buf[1:]
-	n, err := wire.PutByteSlice(buf, key)
-	if err != nil {
-		panic(err)
-	}
-	buf = buf[n:]
-	n, err = wire.PutByteSlice(buf, value)
-	if err != nil {
-		panic(err)
-	}
-	return tx
-}
-
-const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
-
-func RandAsciiBytes(n int) []byte {
-	b := make([]byte, n)
-	for i := range b {
-		b[i] = letterBytes[rand.Intn(len(letterBytes))]
-	}
-	return b
-}
diff --git a/rpc/client/http/rpc_test.go b/rpc/client/http/rpc_test.go
index 4f48c7a2..96585377 100644
--- a/rpc/client/http/rpc_test.go
+++ b/rpc/client/http/rpc_test.go
@@ -9,6 +9,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	merkle "github.com/tendermint/go-merkle"
+	merktest "github.com/tendermint/merkleeyes/testutil"
 	"github.com/tendermint/tendermint/rpc/client/http"
 	ctypes "github.com/tendermint/tendermint/rpc/core/types"
 	rpctest "github.com/tendermint/tendermint/rpc/test"
@@ -95,7 +96,7 @@ func TestAppCalls(t *testing.T) {
 	assert.NotNil(err) // no block yet
 
 	// write something
-	k, v, tx := MakeTxKV()
+	k, v, tx := merktest.MakeTxKV()
 	_, err = c.BroadcastTxCommit(tx)
 	require.Nil(err, "%+v", err)
 	// wait before querying
@@ -164,8 +165,7 @@ func TestSubscriptions(t *testing.T) {
 	defer c.StopWebsocket()
 
 	// subscribe to a transaction event
-	_, _, tx := MakeTxKV()
-	// this causes a panic in tendermint core!!!
+	_, _, tx := merktest.MakeTxKV()
 	eventType := types.EventStringTx(types.Tx(tx))
 	c.Subscribe(eventType)
 

From 202146e4cef2575af665869e15ee7b3069bab7a0 Mon Sep 17 00:00:00 2001
From: Ethan Frey <ethanfrey@users.noreply.github.com>
Date: Thu, 23 Feb 2017 13:50:05 +0100
Subject: [PATCH 090/132] Fix up checktx/delivertx in broadcastTx(A)sync

---
 rpc/client/mock/abci.go      | 16 ++++++++++++----
 rpc/client/mock/abci_test.go |  6 ++++--
 2 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/rpc/client/mock/abci.go b/rpc/client/mock/abci.go
index 95d6fc0a..13e18741 100644
--- a/rpc/client/mock/abci.go
+++ b/rpc/client/mock/abci.go
@@ -40,13 +40,21 @@ func (a ABCIApp) BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit
 }
 
 func (a ABCIApp) BroadcastTxAsync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) {
-	d := a.App.DeliverTx(tx)
-	return &ctypes.ResultBroadcastTx{d.Code, d.Data, d.Log}, nil
+	c := a.App.CheckTx(tx)
+	// and this gets writen in a background thread...
+	if c.IsOK() {
+		go func() { a.App.DeliverTx(tx) }()
+	}
+	return &ctypes.ResultBroadcastTx{c.Code, c.Data, c.Log}, nil
 }
 
 func (a ABCIApp) BroadcastTxSync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) {
-	d := a.App.DeliverTx(tx)
-	return &ctypes.ResultBroadcastTx{d.Code, d.Data, d.Log}, nil
+	c := a.App.CheckTx(tx)
+	// and this gets writen in a background thread...
+	if c.IsOK() {
+		go func() { a.App.DeliverTx(tx) }()
+	}
+	return &ctypes.ResultBroadcastTx{c.Code, c.Data, c.Log}, nil
 }
 
 // ABCIMock will send all abci related request to the named app,
diff --git a/rpc/client/mock/abci_test.go b/rpc/client/mock/abci_test.go
index 77ad496b..823752ca 100644
--- a/rpc/client/mock/abci_test.go
+++ b/rpc/client/mock/abci_test.go
@@ -156,9 +156,11 @@ func TestABCIApp(t *testing.T) {
 	// add a key
 	key, value := "foo", "bar"
 	tx := fmt.Sprintf("%s=%s", key, value)
-	res, err := m.BroadcastTxSync(types.Tx(tx))
+	res, err := m.BroadcastTxCommit(types.Tx(tx))
 	require.Nil(err)
-	assert.True(res.Code.IsOK())
+	assert.True(res.CheckTx.Code.IsOK())
+	require.NotNil(res.DeliverTx)
+	assert.True(res.DeliverTx.Code.IsOK())
 
 	// check the key
 	qres, err := m.ABCIQuery("/key", []byte(key), false)

From 931af6a0725d7b0e756777dee92404005cd290c6 Mon Sep 17 00:00:00 2001
From: Ethan Frey <ethanfrey@users.noreply.github.com>
Date: Thu, 23 Feb 2017 14:16:59 +0100
Subject: [PATCH 091/132] Combine local and http into client package, unify
 tests with table-driven tests

---
 rpc/client/http/rpc_test.go                   | 211 ----------------
 rpc/client/{http/client.go => httpclient.go}  |  73 +++---
 rpc/client/interface.go                       |  14 +-
 rpc/client/local/app_test.go                  |  45 ----
 rpc/client/local/main_test.go                 |  22 --
 rpc/client/local/rpc_test.go                  | 181 --------------
 .../{local/client.go => localclient.go}       |  63 +++--
 rpc/client/{http => }/main_test.go            |   7 +-
 rpc/client/rpc_test.go                        | 236 ++++++++++++++++++
 9 files changed, 314 insertions(+), 538 deletions(-)
 delete mode 100644 rpc/client/http/rpc_test.go
 rename rpc/client/{http/client.go => httpclient.go} (72%)
 delete mode 100644 rpc/client/local/app_test.go
 delete mode 100644 rpc/client/local/main_test.go
 delete mode 100644 rpc/client/local/rpc_test.go
 rename rpc/client/{local/client.go => localclient.go} (60%)
 rename rpc/client/{http => }/main_test.go (75%)
 create mode 100644 rpc/client/rpc_test.go

diff --git a/rpc/client/http/rpc_test.go b/rpc/client/http/rpc_test.go
deleted file mode 100644
index 96585377..00000000
--- a/rpc/client/http/rpc_test.go
+++ /dev/null
@@ -1,211 +0,0 @@
-package http_test
-
-import (
-	"encoding/json"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	merkle "github.com/tendermint/go-merkle"
-	merktest "github.com/tendermint/merkleeyes/testutil"
-	"github.com/tendermint/tendermint/rpc/client/http"
-	ctypes "github.com/tendermint/tendermint/rpc/core/types"
-	rpctest "github.com/tendermint/tendermint/rpc/test"
-	"github.com/tendermint/tendermint/types"
-)
-
-// GetClient gets a rpc client pointing to the test tendermint rpc
-func GetClient() *http.Client {
-	rpcAddr := rpctest.GetConfig().GetString("rpc_laddr")
-	return http.New(rpcAddr, "/websocket")
-}
-
-// Make sure status is correct (we connect properly)
-func TestStatus(t *testing.T) {
-	c := GetClient()
-	chainID := rpctest.GetConfig().GetString("chain_id")
-	status, err := c.Status()
-	require.Nil(t, err, "%+v", err)
-	assert.Equal(t, chainID, status.NodeInfo.Network)
-}
-
-// Make sure info is correct (we connect properly)
-func TestInfo(t *testing.T) {
-	c := GetClient()
-	status, err := c.Status()
-	require.Nil(t, err, "%+v", err)
-	info, err := c.ABCIInfo()
-	require.Nil(t, err, "%+v", err)
-	assert.EqualValues(t, status.LatestBlockHeight, info.Response.LastBlockHeight)
-	assert.True(t, strings.HasPrefix(info.Response.Data, "size"))
-}
-
-func TestNetInfo(t *testing.T) {
-	c := GetClient()
-	netinfo, err := c.NetInfo()
-	require.Nil(t, err, "%+v", err)
-	assert.True(t, netinfo.Listening)
-	assert.Equal(t, 0, len(netinfo.Peers))
-}
-
-func TestDialSeeds(t *testing.T) {
-	c := GetClient()
-	// FIXME: fix server so it doesn't panic on invalid input
-	_, err := c.DialSeeds([]string{"12.34.56.78:12345"})
-	require.Nil(t, err, "%+v", err)
-}
-
-func TestGenesisAndValidators(t *testing.T) {
-	c := GetClient()
-	chainID := rpctest.GetConfig().GetString("chain_id")
-
-	// make sure this is the right genesis file
-	gen, err := c.Genesis()
-	require.Nil(t, err, "%+v", err)
-	assert.Equal(t, chainID, gen.Genesis.ChainID)
-	// get the genesis validator
-	require.Equal(t, 1, len(gen.Genesis.Validators))
-	gval := gen.Genesis.Validators[0]
-
-	// get the current validators
-	vals, err := c.Validators()
-	require.Nil(t, err, "%+v", err)
-	require.Equal(t, 1, len(vals.Validators))
-	val := vals.Validators[0]
-
-	// make sure the current set is also the genesis set
-	assert.Equal(t, gval.Amount, val.VotingPower)
-	assert.Equal(t, gval.PubKey, val.PubKey)
-}
-
-// Make some app checks
-func TestAppCalls(t *testing.T) {
-	assert, require := assert.New(t), require.New(t)
-	c := GetClient()
-
-	// get an offset of height to avoid racing and guessing
-	s, err := c.Status()
-	require.Nil(err)
-	// sh is start height or status height
-	sh := s.LatestBlockHeight
-
-	// look for the future
-	_, err = c.Block(sh + 2)
-	assert.NotNil(err) // no block yet
-
-	// write something
-	k, v, tx := merktest.MakeTxKV()
-	_, err = c.BroadcastTxCommit(tx)
-	require.Nil(err, "%+v", err)
-	// wait before querying
-	time.Sleep(time.Second * 1)
-	qres, err := c.ABCIQuery("/key", k, false)
-	if assert.Nil(err) && assert.True(qres.Response.Code.IsOK()) {
-		data := qres.Response
-		// assert.Equal(k, data.GetKey())  // only returned for proofs
-		assert.Equal(v, data.GetValue())
-	}
-	// +/- 1 making my head hurt
-	h := int(qres.Response.Height) - 1
-
-	// and we can even check the block is added
-	block, err := c.Block(h)
-	require.Nil(err, "%+v", err)
-	appHash := block.BlockMeta.Header.AppHash
-	assert.True(len(appHash) > 0)
-	assert.EqualValues(h, block.BlockMeta.Header.Height)
-
-	// check blockchain info, now that we know there is info
-	// TODO: is this commented somewhere that they are returned
-	// in order of descending height???
-	info, err := c.BlockchainInfo(h-2, h)
-	require.Nil(err, "%+v", err)
-	assert.True(info.LastHeight > 2)
-	if assert.Equal(3, len(info.BlockMetas)) {
-		lastMeta := info.BlockMetas[0]
-		assert.EqualValues(h, lastMeta.Header.Height)
-		bMeta := block.BlockMeta
-		assert.Equal(bMeta.Header.AppHash, lastMeta.Header.AppHash)
-		assert.Equal(bMeta.BlockID, lastMeta.BlockID)
-	}
-
-	// and get the corresponding commit with the same apphash
-	commit, err := c.Commit(h)
-	require.Nil(err, "%+v", err)
-	cappHash := commit.Header.AppHash
-	assert.Equal(appHash, cappHash)
-	assert.NotNil(commit.Commit)
-
-	// compare the commits (note Commit(2) has commit from Block(3))
-	commit2, err := c.Commit(h - 1)
-	require.Nil(err, "%+v", err)
-	assert.Equal(block.Block.LastCommit, commit2.Commit)
-
-	// and we got a proof that works!
-	pres, err := c.ABCIQuery("/key", k, true)
-	if assert.Nil(err) && assert.True(pres.Response.Code.IsOK()) {
-		proof, err := merkle.ReadProof(pres.Response.GetProof())
-		if assert.Nil(err) {
-			key := pres.Response.GetKey()
-			value := pres.Response.GetValue()
-			assert.Equal(appHash, proof.RootHash)
-			valid := proof.Verify(key, value, appHash)
-			assert.True(valid)
-		}
-	}
-}
-
-func TestSubscriptions(t *testing.T) {
-	require := require.New(t)
-	c := GetClient()
-	err := c.StartWebsocket()
-	require.Nil(err)
-	defer c.StopWebsocket()
-
-	// subscribe to a transaction event
-	_, _, tx := merktest.MakeTxKV()
-	eventType := types.EventStringTx(types.Tx(tx))
-	c.Subscribe(eventType)
-
-	// set up a listener
-	r, e := c.GetEventChannels()
-	go func() {
-		// send a tx and wait for it to propogate
-		_, err = c.BroadcastTxCommit(tx)
-		require.Nil(err, string(tx))
-	}()
-
-	checkData := func(data []byte, kind byte) {
-		x := []interface{}{}
-		err := json.Unmarshal(data, &x)
-		require.Nil(err)
-		// gotta love wire's json format
-		require.EqualValues(kind, x[0])
-	}
-
-	res := <-r
-	checkData(res, ctypes.ResultTypeSubscribe)
-
-	// read one event, must be success
-	select {
-	case res := <-r:
-		checkData(res, ctypes.ResultTypeEvent)
-		// this is good.. let's get the data... ugh...
-		// result := new(ctypes.TMResult)
-		// wire.ReadJSON(result, res, &err)
-		// require.Nil(err, "%+v", err)
-		// event, ok := (*result).(*ctypes.ResultEvent)
-		// require.True(ok)
-		// assert.Equal("foo", event.Name)
-		// data, ok := event.Data.(types.EventDataTx)
-		// require.True(ok)
-		// assert.EqualValues(0, data.Code)
-		// assert.EqualValues(tx, data.Tx)
-	case err := <-e:
-		// this is a failure
-		require.Nil(err)
-	}
-
-}
diff --git a/rpc/client/http/client.go b/rpc/client/httpclient.go
similarity index 72%
rename from rpc/client/http/client.go
rename to rpc/client/httpclient.go
index b8fb0c4e..ea81199f 100644
--- a/rpc/client/http/client.go
+++ b/rpc/client/httpclient.go
@@ -1,5 +1,16 @@
+package client
+
+import (
+	"encoding/json"
+
+	"github.com/pkg/errors"
+	"github.com/tendermint/go-rpc/client"
+	ctypes "github.com/tendermint/tendermint/rpc/core/types"
+	"github.com/tendermint/tendermint/types"
+)
+
 /*
-package http returns a Client implementation that communicates
+HTTP is a Client implementation that communicates
 with a tendermint node over json rpc and websockets.
 
 This is the main implementation you probably want to use in
@@ -7,21 +18,7 @@ production code.  There are other implementations when calling
 the tendermint node in-process (local), or when you want to mock
 out the server for test code (mock).
 */
-package http
-
-import (
-	"encoding/json"
-
-	"github.com/pkg/errors"
-	"github.com/tendermint/go-rpc/client"
-	"github.com/tendermint/tendermint/rpc/client"
-	ctypes "github.com/tendermint/tendermint/rpc/core/types"
-	"github.com/tendermint/tendermint/types"
-)
-
-// Client is a Client implementation that communicates over
-// JSONRPC
-type Client struct {
+type HTTP struct {
 	remote   string
 	endpoint string
 	rpc      *rpcclient.ClientJSONRPC
@@ -30,19 +27,19 @@ type Client struct {
 
 // New takes a remote endpoint in the form tcp://<host>:<port>
 // and the websocket path (which always seems to be "/websocket")
-func New(remote, wsEndpoint string) *Client {
-	return &Client{
+func NewHTTP(remote, wsEndpoint string) *HTTP {
+	return &HTTP{
 		rpc:      rpcclient.NewClientJSONRPC(remote),
 		remote:   remote,
 		endpoint: wsEndpoint,
 	}
 }
 
-func (c *Client) _assertIsClient() client.Client {
+func (c *HTTP) _assertIsClient() Client {
 	return c
 }
 
-func (c *Client) Status() (*ctypes.ResultStatus, error) {
+func (c *HTTP) Status() (*ctypes.ResultStatus, error) {
 	tmResult := new(ctypes.TMResult)
 	_, err := c.rpc.Call("status", []interface{}{}, tmResult)
 	if err != nil {
@@ -52,7 +49,7 @@ func (c *Client) Status() (*ctypes.ResultStatus, error) {
 	return (*tmResult).(*ctypes.ResultStatus), nil
 }
 
-func (c *Client) ABCIInfo() (*ctypes.ResultABCIInfo, error) {
+func (c *HTTP) ABCIInfo() (*ctypes.ResultABCIInfo, error) {
 	tmResult := new(ctypes.TMResult)
 	_, err := c.rpc.Call("abci_info", []interface{}{}, tmResult)
 	if err != nil {
@@ -61,7 +58,7 @@ func (c *Client) ABCIInfo() (*ctypes.ResultABCIInfo, error) {
 	return (*tmResult).(*ctypes.ResultABCIInfo), nil
 }
 
-func (c *Client) ABCIQuery(path string, data []byte, prove bool) (*ctypes.ResultABCIQuery, error) {
+func (c *HTTP) ABCIQuery(path string, data []byte, prove bool) (*ctypes.ResultABCIQuery, error) {
 	tmResult := new(ctypes.TMResult)
 	_, err := c.rpc.Call("abci_query", []interface{}{path, data, prove}, tmResult)
 	if err != nil {
@@ -70,7 +67,7 @@ func (c *Client) ABCIQuery(path string, data []byte, prove bool) (*ctypes.Result
 	return (*tmResult).(*ctypes.ResultABCIQuery), nil
 }
 
-func (c *Client) BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) {
+func (c *HTTP) BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) {
 	tmResult := new(ctypes.TMResult)
 	_, err := c.rpc.Call("broadcast_tx_commit", []interface{}{tx}, tmResult)
 	if err != nil {
@@ -79,15 +76,15 @@ func (c *Client) BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit
 	return (*tmResult).(*ctypes.ResultBroadcastTxCommit), nil
 }
 
-func (c *Client) BroadcastTxAsync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) {
+func (c *HTTP) BroadcastTxAsync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) {
 	return c.broadcastTX("broadcast_tx_async", tx)
 }
 
-func (c *Client) BroadcastTxSync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) {
+func (c *HTTP) BroadcastTxSync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) {
 	return c.broadcastTX("broadcast_tx_sync", tx)
 }
 
-func (c *Client) broadcastTX(route string, tx types.Tx) (*ctypes.ResultBroadcastTx, error) {
+func (c *HTTP) broadcastTX(route string, tx types.Tx) (*ctypes.ResultBroadcastTx, error) {
 	tmResult := new(ctypes.TMResult)
 	_, err := c.rpc.Call(route, []interface{}{tx}, tmResult)
 	if err != nil {
@@ -96,7 +93,7 @@ func (c *Client) broadcastTX(route string, tx types.Tx) (*ctypes.ResultBroadcast
 	return (*tmResult).(*ctypes.ResultBroadcastTx), nil
 }
 
-func (c *Client) NetInfo() (*ctypes.ResultNetInfo, error) {
+func (c *HTTP) NetInfo() (*ctypes.ResultNetInfo, error) {
 	tmResult := new(ctypes.TMResult)
 	_, err := c.rpc.Call("net_info", nil, tmResult)
 	if err != nil {
@@ -105,7 +102,7 @@ func (c *Client) NetInfo() (*ctypes.ResultNetInfo, error) {
 	return (*tmResult).(*ctypes.ResultNetInfo), nil
 }
 
-func (c *Client) DialSeeds(seeds []string) (*ctypes.ResultDialSeeds, error) {
+func (c *HTTP) DialSeeds(seeds []string) (*ctypes.ResultDialSeeds, error) {
 	tmResult := new(ctypes.TMResult)
 	// TODO: is this the correct way to marshall seeds?
 	_, err := c.rpc.Call("dial_seeds", []interface{}{seeds}, tmResult)
@@ -115,7 +112,7 @@ func (c *Client) DialSeeds(seeds []string) (*ctypes.ResultDialSeeds, error) {
 	return (*tmResult).(*ctypes.ResultDialSeeds), nil
 }
 
-func (c *Client) BlockchainInfo(minHeight, maxHeight int) (*ctypes.ResultBlockchainInfo, error) {
+func (c *HTTP) BlockchainInfo(minHeight, maxHeight int) (*ctypes.ResultBlockchainInfo, error) {
 	tmResult := new(ctypes.TMResult)
 	_, err := c.rpc.Call("blockchain", []interface{}{minHeight, maxHeight}, tmResult)
 	if err != nil {
@@ -124,7 +121,7 @@ func (c *Client) BlockchainInfo(minHeight, maxHeight int) (*ctypes.ResultBlockch
 	return (*tmResult).(*ctypes.ResultBlockchainInfo), nil
 }
 
-func (c *Client) Genesis() (*ctypes.ResultGenesis, error) {
+func (c *HTTP) Genesis() (*ctypes.ResultGenesis, error) {
 	tmResult := new(ctypes.TMResult)
 	_, err := c.rpc.Call("genesis", nil, tmResult)
 	if err != nil {
@@ -133,7 +130,7 @@ func (c *Client) Genesis() (*ctypes.ResultGenesis, error) {
 	return (*tmResult).(*ctypes.ResultGenesis), nil
 }
 
-func (c *Client) Block(height int) (*ctypes.ResultBlock, error) {
+func (c *HTTP) Block(height int) (*ctypes.ResultBlock, error) {
 	tmResult := new(ctypes.TMResult)
 	_, err := c.rpc.Call("block", []interface{}{height}, tmResult)
 	if err != nil {
@@ -142,7 +139,7 @@ func (c *Client) Block(height int) (*ctypes.ResultBlock, error) {
 	return (*tmResult).(*ctypes.ResultBlock), nil
 }
 
-func (c *Client) Commit(height int) (*ctypes.ResultCommit, error) {
+func (c *HTTP) Commit(height int) (*ctypes.ResultCommit, error) {
 	tmResult := new(ctypes.TMResult)
 	_, err := c.rpc.Call("commit", []interface{}{height}, tmResult)
 	if err != nil {
@@ -151,7 +148,7 @@ func (c *Client) Commit(height int) (*ctypes.ResultCommit, error) {
 	return (*tmResult).(*ctypes.ResultCommit), nil
 }
 
-func (c *Client) Validators() (*ctypes.ResultValidators, error) {
+func (c *HTTP) Validators() (*ctypes.ResultValidators, error) {
 	tmResult := new(ctypes.TMResult)
 	_, err := c.rpc.Call("validators", nil, tmResult)
 	if err != nil {
@@ -164,7 +161,7 @@ func (c *Client) Validators() (*ctypes.ResultValidators, error) {
 
 // StartWebsocket starts up a websocket and a listener goroutine
 // if already started, do nothing
-func (c *Client) StartWebsocket() error {
+func (c *HTTP) StartWebsocket() error {
 	var err error
 	if c.ws == nil {
 		ws := rpcclient.NewWSClient(c.remote, c.endpoint)
@@ -177,7 +174,7 @@ func (c *Client) StartWebsocket() error {
 }
 
 // StopWebsocket stops the websocket connection
-func (c *Client) StopWebsocket() {
+func (c *HTTP) StopWebsocket() {
 	if c.ws != nil {
 		c.ws.Stop()
 		c.ws = nil
@@ -185,17 +182,17 @@ func (c *Client) StopWebsocket() {
 }
 
 // GetEventChannels returns the results and error channel from the websocket
-func (c *Client) GetEventChannels() (chan json.RawMessage, chan error) {
+func (c *HTTP) GetEventChannels() (chan json.RawMessage, chan error) {
 	if c.ws == nil {
 		return nil, nil
 	}
 	return c.ws.ResultsCh, c.ws.ErrorsCh
 }
 
-func (c *Client) Subscribe(event string) error {
+func (c *HTTP) Subscribe(event string) error {
 	return errors.Wrap(c.ws.Subscribe(event), "Subscribe")
 }
 
-func (c *Client) Unsubscribe(event string) error {
+func (c *HTTP) Unsubscribe(event string) error {
 	return errors.Wrap(c.ws.Unsubscribe(event), "Unsubscribe")
 }
diff --git a/rpc/client/interface.go b/rpc/client/interface.go
index 0683d469..b50950e4 100644
--- a/rpc/client/interface.go
+++ b/rpc/client/interface.go
@@ -1,21 +1,21 @@
 /*
-package client provides a general purpose interface for connecting
+package client provides a general purpose interface (Client) for connecting
 to a tendermint node, as well as higher-level functionality.
 
-The main implementation for production code is http, which connects
-via http to the jsonrpc interface of the tendermint node.
+The main implementation for production code is client.HTTP, which
+connects via http to the jsonrpc interface of the tendermint node.
 
 For connecting to a node running in the same process (eg. when
-compiling the abci app in the same process), you can use the local
+compiling the abci app in the same process), you can use the client.Local
 implementation.
 
 For mocking out server responses during testing to see behavior for
 arbitrary return values, use the mock package.
 
 In addition to the Client interface, which should be used externally
-for maximum flexibility and testability, this package also provides
-a wrapper that accepts any Client implementation and adds some
-higher-level functionality.
+for maximum flexibility and testability, and two implementations,
+this package also provides helper functions that work on any Client
+implementation.
 */
 package client
 
diff --git a/rpc/client/local/app_test.go b/rpc/client/local/app_test.go
deleted file mode 100644
index 898399ce..00000000
--- a/rpc/client/local/app_test.go
+++ /dev/null
@@ -1,45 +0,0 @@
-package local_test
-
-import (
-	"math/rand"
-
-	meapp "github.com/tendermint/merkleeyes/app"
-
-	wire "github.com/tendermint/go-wire"
-)
-
-// MakeTxKV returns a text transaction, allong with expected key, value pair
-func MakeTxKV() ([]byte, []byte, []byte) {
-	k := RandAsciiBytes(8)
-	v := RandAsciiBytes(8)
-	return k, v, makeSet(k, v)
-}
-
-// blatently copied from merkleeyes/app/app_test.go
-// constructs a "set" transaction
-func makeSet(key, value []byte) []byte {
-	tx := make([]byte, 1+wire.ByteSliceSize(key)+wire.ByteSliceSize(value))
-	buf := tx
-	buf[0] = meapp.WriteSet // Set TypeByte
-	buf = buf[1:]
-	n, err := wire.PutByteSlice(buf, key)
-	if err != nil {
-		panic(err)
-	}
-	buf = buf[n:]
-	n, err = wire.PutByteSlice(buf, value)
-	if err != nil {
-		panic(err)
-	}
-	return tx
-}
-
-const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
-
-func RandAsciiBytes(n int) []byte {
-	b := make([]byte, n)
-	for i := range b {
-		b[i] = letterBytes[rand.Intn(len(letterBytes))]
-	}
-	return b
-}
diff --git a/rpc/client/local/main_test.go b/rpc/client/local/main_test.go
deleted file mode 100644
index 5797843f..00000000
--- a/rpc/client/local/main_test.go
+++ /dev/null
@@ -1,22 +0,0 @@
-package local_test
-
-import (
-	"os"
-	"testing"
-
-	meapp "github.com/tendermint/merkleeyes/app"
-	nm "github.com/tendermint/tendermint/node"
-	rpctest "github.com/tendermint/tendermint/rpc/test"
-)
-
-var node *nm.Node
-
-func TestMain(m *testing.M) {
-	// configure a node, but don't start the server
-	app := meapp.NewMerkleEyesApp("", 100)
-	node = rpctest.StartTendermint(app)
-
-	code := m.Run()
-	// and shut down proper at the end
-	os.Exit(code)
-}
diff --git a/rpc/client/local/rpc_test.go b/rpc/client/local/rpc_test.go
deleted file mode 100644
index 069db06f..00000000
--- a/rpc/client/local/rpc_test.go
+++ /dev/null
@@ -1,181 +0,0 @@
-package local_test
-
-import (
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	merkle "github.com/tendermint/go-merkle"
-	"github.com/tendermint/tendermint/rpc/client/local"
-	rpctest "github.com/tendermint/tendermint/rpc/test"
-)
-
-// GetClient gets a rpc client pointing to the test tendermint rpc
-func GetClient() local.Client {
-	return local.New(node)
-}
-
-// Make sure status is correct (we connect properly)
-func TestStatus(t *testing.T) {
-	c := GetClient()
-	status, err := c.Status()
-	require.Nil(t, err, "%+v", err)
-	require.NotNil(t, status.PubKey)
-}
-
-// Make sure info is correct (we connect properly)
-func TestInfo(t *testing.T) {
-	c := GetClient()
-	status, err := c.Status()
-	require.Nil(t, err, "%+v", err)
-	info, err := c.ABCIInfo()
-	require.Nil(t, err, "%+v", err)
-	assert.EqualValues(t, status.LatestBlockHeight, info.Response.LastBlockHeight)
-	assert.True(t, strings.HasPrefix(info.Response.Data, "size:"))
-}
-
-func TestNetInfo(t *testing.T) {
-	c := GetClient()
-	netinfo, err := c.NetInfo()
-	require.Nil(t, err, "%+v", err)
-	assert.True(t, netinfo.Listening)
-	assert.Equal(t, 0, len(netinfo.Peers))
-}
-
-func TestDialSeeds(t *testing.T) {
-	c := GetClient()
-	// FIXME: fix server so it doesn't panic on invalid input
-	_, err := c.DialSeeds([]string{"12.34.56.78:12345"})
-	require.Nil(t, err, "%+v", err)
-}
-
-func TestGenesisAndValidators(t *testing.T) {
-	c := GetClient()
-	chainID := rpctest.GetConfig().GetString("chain_id")
-
-	// make sure this is the right genesis file
-	gen, err := c.Genesis()
-	require.Nil(t, err, "%+v", err)
-	assert.Equal(t, chainID, gen.Genesis.ChainID)
-	// get the genesis validator
-	require.Equal(t, 1, len(gen.Genesis.Validators))
-	gval := gen.Genesis.Validators[0]
-
-	// get the current validators
-	vals, err := c.Validators()
-	require.Nil(t, err, "%+v", err)
-	require.Equal(t, 1, len(vals.Validators))
-	val := vals.Validators[0]
-
-	// make sure the current set is also the genesis set
-	assert.Equal(t, gval.Amount, val.VotingPower)
-	assert.Equal(t, gval.PubKey, val.PubKey)
-}
-
-// Make some app checks
-func TestAppCalls(t *testing.T) {
-	assert, require := assert.New(t), require.New(t)
-	c := GetClient()
-	_, err := c.Block(1)
-	assert.NotNil(err) // no block yet
-	k, v, tx := MakeTxKV()
-	_, err = c.BroadcastTxCommit(tx)
-	require.Nil(err, "%+v", err)
-	// wait before querying
-	time.Sleep(time.Second * 1)
-	qres, err := c.ABCIQuery("/key", k, false)
-	if assert.Nil(err) && assert.True(qres.Response.Code.IsOK()) {
-		data := qres.Response
-		// assert.Equal(k, data.GetKey())  // only returned for proofs
-		assert.Equal(v, data.GetValue())
-	}
-	// and we can even check the block is added
-	block, err := c.Block(3)
-	require.Nil(err, "%+v", err)
-	appHash := block.BlockMeta.Header.AppHash
-	assert.True(len(appHash) > 0)
-	assert.EqualValues(3, block.BlockMeta.Header.Height)
-
-	// check blockchain info, now that we know there is info
-	// TODO: is this commented somewhere that they are returned
-	// in order of descending height???
-	info, err := c.BlockchainInfo(1, 3)
-	require.Nil(err, "%+v", err)
-	assert.True(info.LastHeight > 2)
-	if assert.Equal(3, len(info.BlockMetas)) {
-		lastMeta := info.BlockMetas[0]
-		assert.EqualValues(3, lastMeta.Header.Height)
-		bMeta := block.BlockMeta
-		assert.Equal(bMeta.Header.AppHash, lastMeta.Header.AppHash)
-		assert.Equal(bMeta.BlockID, lastMeta.BlockID)
-	}
-
-	// and get the corresponding commit with the same apphash
-	commit, err := c.Commit(3)
-	require.Nil(err, "%+v", err)
-	cappHash := commit.Header.AppHash
-	assert.Equal(appHash, cappHash)
-	assert.NotNil(commit.Commit)
-
-	// compare the commits (note Commit(2) has commit from Block(3))
-	commit2, err := c.Commit(2)
-	require.Nil(err, "%+v", err)
-	assert.Equal(block.Block.LastCommit, commit2.Commit)
-
-	// and we got a proof that works!
-	pres, err := c.ABCIQuery("/key", k, true)
-	if assert.Nil(err) && assert.True(pres.Response.Code.IsOK()) {
-		proof, err := merkle.ReadProof(pres.Response.GetProof())
-		if assert.Nil(err) {
-			key := pres.Response.GetKey()
-			value := pres.Response.GetValue()
-			assert.Equal(appHash, proof.RootHash)
-			valid := proof.Verify(key, value, appHash)
-			assert.True(valid)
-		}
-	}
-}
-
-/*
-func TestSubscriptions(t *testing.T) {
-	assert, require := assert.New(t), require.New(t)
-	c := GetClient()
-	err := c.StartWebsocket()
-	require.Nil(err)
-	defer c.StopWebsocket()
-
-	// subscribe to a transaction event
-	_, _, tx := MakeTxKV()
-	// this causes a panic in tendermint core!!!
-	eventType := types.EventStringTx(types.Tx(tx))
-	c.Subscribe(eventType)
-	read := 0
-
-	// set up a listener
-	r, e := c.GetEventChannels()
-	go func() {
-		// read one event in the background
-		select {
-		case <-r:
-			// TODO: actually parse this or something
-			read += 1
-		case err := <-e:
-			panic(err)
-		}
-	}()
-
-	// make sure nothing has happened yet.
-	assert.Equal(0, read)
-
-	// send a tx and wait for it to propogate
-	_, err = c.BroadcastTxCommit(tx)
-	assert.Nil(err, string(tx))
-	// wait before querying
-	time.Sleep(time.Second)
-
-	// now make sure the event arrived
-	assert.Equal(1, read)
-}
-*/
diff --git a/rpc/client/local/client.go b/rpc/client/localclient.go
similarity index 60%
rename from rpc/client/local/client.go
rename to rpc/client/localclient.go
index 5a6a699e..eb180a6e 100644
--- a/rpc/client/local/client.go
+++ b/rpc/client/localclient.go
@@ -1,6 +1,15 @@
+package client
+
+import (
+	nm "github.com/tendermint/tendermint/node"
+	"github.com/tendermint/tendermint/rpc/core"
+	ctypes "github.com/tendermint/tendermint/rpc/core/types"
+	"github.com/tendermint/tendermint/types"
+)
+
 /*
-package local returns a Client implementation that
-directly executes the rpc functions on a given node.
+Local is a Client implementation that directly executes the rpc
+functions on a given node, without going through HTTP or GRPC
 
 This implementation is useful for:
 
@@ -9,89 +18,79 @@ of going through an http server
 * Communication between an ABCI app and tendermin core when they
 are compiled in process.
 
-For real clients, you probably want the "http" package.  For more
-powerful control during testing, you probably want the "mock" package.
+For real clients, you probably want to use client.HTTP.  For more
+powerful control during testing, you probably want the "client/mock" package.
 */
-package local
-
-import (
-	nm "github.com/tendermint/tendermint/node"
-	"github.com/tendermint/tendermint/rpc/client"
-	"github.com/tendermint/tendermint/rpc/core"
-	ctypes "github.com/tendermint/tendermint/rpc/core/types"
-	"github.com/tendermint/tendermint/types"
-)
-
-type Client struct {
+type Local struct {
 	node *nm.Node
 }
 
-// New configures this to call the Node directly.
+// NewLocal configures a client that calls the Node directly.
 //
 // Note that given how rpc/core works with package singletons, that
 // you can only have one node per process.  So make sure test cases
 // don't run in parallel, or try to simulate an entire network in
 // one process...
-func New(node *nm.Node) Client {
+func NewLocal(node *nm.Node) Local {
 	node.ConfigureRPC()
-	return Client{
+	return Local{
 		node: node,
 	}
 }
 
-func (c Client) _assertIsClient() client.Client {
+func (c Local) _assertIsClient() Client {
 	return c
 }
 
-func (c Client) Status() (*ctypes.ResultStatus, error) {
+func (c Local) Status() (*ctypes.ResultStatus, error) {
 	return core.Status()
 }
 
-func (c Client) ABCIInfo() (*ctypes.ResultABCIInfo, error) {
+func (c Local) ABCIInfo() (*ctypes.ResultABCIInfo, error) {
 	return core.ABCIInfo()
 }
 
-func (c Client) ABCIQuery(path string, data []byte, prove bool) (*ctypes.ResultABCIQuery, error) {
+func (c Local) ABCIQuery(path string, data []byte, prove bool) (*ctypes.ResultABCIQuery, error) {
 	return core.ABCIQuery(path, data, prove)
 }
 
-func (c Client) BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) {
+func (c Local) BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) {
 	return core.BroadcastTxCommit(tx)
 }
 
-func (c Client) BroadcastTxAsync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) {
+func (c Local) BroadcastTxAsync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) {
 	return core.BroadcastTxAsync(tx)
 }
 
-func (c Client) BroadcastTxSync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) {
+func (c Local) BroadcastTxSync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) {
 	return core.BroadcastTxSync(tx)
 }
 
-func (c Client) NetInfo() (*ctypes.ResultNetInfo, error) {
+func (c Local) NetInfo() (*ctypes.ResultNetInfo, error) {
 	return core.NetInfo()
 }
 
-func (c Client) DialSeeds(seeds []string) (*ctypes.ResultDialSeeds, error) {
+func (c Local) DialSeeds(seeds []string) (*ctypes.ResultDialSeeds, error) {
 	return core.UnsafeDialSeeds(seeds)
 }
 
-func (c Client) BlockchainInfo(minHeight, maxHeight int) (*ctypes.ResultBlockchainInfo, error) {
+func (c Local) BlockchainInfo(minHeight, maxHeight int) (*ctypes.ResultBlockchainInfo, error) {
 	return core.BlockchainInfo(minHeight, maxHeight)
 }
 
-func (c Client) Genesis() (*ctypes.ResultGenesis, error) {
+func (c Local) Genesis() (*ctypes.ResultGenesis, error) {
 	return core.Genesis()
 }
 
-func (c Client) Block(height int) (*ctypes.ResultBlock, error) {
+func (c Local) Block(height int) (*ctypes.ResultBlock, error) {
 	return core.Block(height)
 }
 
-func (c Client) Commit(height int) (*ctypes.ResultCommit, error) {
+func (c Local) Commit(height int) (*ctypes.ResultCommit, error) {
 	return core.Commit(height)
 }
 
-func (c Client) Validators() (*ctypes.ResultValidators, error) {
+func (c Local) Validators() (*ctypes.ResultValidators, error) {
 	return core.Validators()
 }
 
diff --git a/rpc/client/http/main_test.go b/rpc/client/main_test.go
similarity index 75%
rename from rpc/client/http/main_test.go
rename to rpc/client/main_test.go
index a5e1887a..5ec911e5 100644
--- a/rpc/client/http/main_test.go
+++ b/rpc/client/main_test.go
@@ -1,17 +1,20 @@
-package http_test
+package client_test
 
 import (
 	"os"
 	"testing"
 
 	meapp "github.com/tendermint/merkleeyes/app"
+	nm "github.com/tendermint/tendermint/node"
 	rpctest "github.com/tendermint/tendermint/rpc/test"
 )
 
+var node *nm.Node
+
 func TestMain(m *testing.M) {
 	// start a tendermint node (and merkleeyes) in the background to test against
 	app := meapp.NewMerkleEyesApp("", 100)
-	node := rpctest.StartTendermint(app)
+	node = rpctest.StartTendermint(app)
 	code := m.Run()
 
 	// and shut down proper at the end
diff --git a/rpc/client/rpc_test.go b/rpc/client/rpc_test.go
new file mode 100644
index 00000000..f37386c3
--- /dev/null
+++ b/rpc/client/rpc_test.go
@@ -0,0 +1,236 @@
+package client_test
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	merkle "github.com/tendermint/go-merkle"
+	merktest "github.com/tendermint/merkleeyes/testutil"
+	"github.com/tendermint/tendermint/rpc/client"
+	ctypes "github.com/tendermint/tendermint/rpc/core/types"
+	rpctest "github.com/tendermint/tendermint/rpc/test"
+	"github.com/tendermint/tendermint/types"
+)
+
+func getHTTPClient() *client.HTTP {
+	rpcAddr := rpctest.GetConfig().GetString("rpc_laddr")
+	return client.NewHTTP(rpcAddr, "/websocket")
+}
+
+func getLocalClient() client.Local {
+	return client.NewLocal(node)
+}
+
+// GetClients returns a slice of clients for table-driven tests
+func GetClients() []client.Client {
+	return []client.Client{
+		getHTTPClient(),
+		getLocalClient(),
+	}
+}
+
+// Make sure status is correct (we connect properly)
+func TestStatus(t *testing.T) {
+	for i, c := range GetClients() {
+		chainID := rpctest.GetConfig().GetString("chain_id")
+		status, err := c.Status()
+		require.Nil(t, err, "%d: %+v", i, err)
+		assert.Equal(t, chainID, status.NodeInfo.Network)
+	}
+}
+
+// Make sure info is correct (we connect properly)
+func TestInfo(t *testing.T) {
+	for i, c := range GetClients() {
+		// status, err := c.Status()
+		// require.Nil(t, err, "%+v", err)
+		info, err := c.ABCIInfo()
+		require.Nil(t, err, "%d: %+v", i, err)
+		// TODO: this is not correct - fix merkleeyes!
+		// assert.EqualValues(t, status.LatestBlockHeight, info.Response.LastBlockHeight)
+		assert.True(t, strings.HasPrefix(info.Response.Data, "size"))
+	}
+}
+
+func TestNetInfo(t *testing.T) {
+	for i, c := range GetClients() {
+		nc, ok := c.(client.NetworkClient)
+		require.True(t, ok, "%d", i)
+		netinfo, err := nc.NetInfo()
+		require.Nil(t, err, "%d: %+v", i, err)
+		assert.True(t, netinfo.Listening)
+		assert.Equal(t, 0, len(netinfo.Peers))
+	}
+}
+
+func TestDialSeeds(t *testing.T) {
+	for i, c := range GetClients() {
+		// FIXME: fix server so it doesn't panic on invalid input
+		nc, ok := c.(client.NetworkClient)
+		require.True(t, ok, "%d", i)
+		_, err := nc.DialSeeds([]string{"12.34.56.78:12345"})
+		require.Nil(t, err, "%d: %+v", i, err)
+	}
+}
+
+func TestGenesisAndValidators(t *testing.T) {
+	for i, c := range GetClients() {
+		chainID := rpctest.GetConfig().GetString("chain_id")
+
+		// make sure this is the right genesis file
+		gen, err := c.Genesis()
+		require.Nil(t, err, "%d: %+v", i, err)
+		assert.Equal(t, chainID, gen.Genesis.ChainID)
+		// get the genesis validator
+		require.Equal(t, 1, len(gen.Genesis.Validators))
+		gval := gen.Genesis.Validators[0]
+
+		// get the current validators
+		vals, err := c.Validators()
+		require.Nil(t, err, "%d: %+v", i, err)
+		require.Equal(t, 1, len(vals.Validators))
+		val := vals.Validators[0]
+
+		// make sure the current set is also the genesis set
+		assert.Equal(t, gval.Amount, val.VotingPower)
+		assert.Equal(t, gval.PubKey, val.PubKey)
+	}
+}
+
+// Make some app checks
+func TestAppCalls(t *testing.T) {
+	assert, require := assert.New(t), require.New(t)
+	for i, c := range GetClients() {
+
+		// get an offset of height to avoid racing and guessing
+		s, err := c.Status()
+		require.Nil(err, "%d: %+v", i, err)
+		// sh is start height or status height
+		sh := s.LatestBlockHeight
+
+		// look for the future
+		_, err = c.Block(sh + 2)
+		assert.NotNil(err) // no block yet
+
+		// write something
+		k, v, tx := merktest.MakeTxKV()
+		_, err = c.BroadcastTxCommit(tx)
+		require.Nil(err, "%d: %+v", i, err)
+		// wait before querying
+		time.Sleep(time.Second * 1)
+		qres, err := c.ABCIQuery("/key", k, false)
+		if assert.Nil(err) && assert.True(qres.Response.Code.IsOK()) {
+			data := qres.Response
+			// assert.Equal(k, data.GetKey())  // only returned for proofs
+			assert.Equal(v, data.GetValue())
+		}
+		// +/- 1 making my head hurt
+		h := int(qres.Response.Height) - 1
+
+		// and we can even check the block is added
+		block, err := c.Block(h)
+		require.Nil(err, "%d: %+v", i, err)
+		appHash := block.BlockMeta.Header.AppHash
+		assert.True(len(appHash) > 0)
+		assert.EqualValues(h, block.BlockMeta.Header.Height)
+
+		// check blockchain info, now that we know there is info
+		// TODO: is this commented somewhere that they are returned
+		// in order of descending height???
+		info, err := c.BlockchainInfo(h-2, h)
+		require.Nil(err, "%d: %+v", i, err)
+		assert.True(info.LastHeight > 2)
+		if assert.Equal(3, len(info.BlockMetas)) {
+			lastMeta := info.BlockMetas[0]
+			assert.EqualValues(h, lastMeta.Header.Height)
+			bMeta := block.BlockMeta
+			assert.Equal(bMeta.Header.AppHash, lastMeta.Header.AppHash)
+			assert.Equal(bMeta.BlockID, lastMeta.BlockID)
+		}
+
+		// and get the corresponding commit with the same apphash
+		commit, err := c.Commit(h)
+		require.Nil(err, "%d: %+v", i, err)
+		cappHash := commit.Header.AppHash
+		assert.Equal(appHash, cappHash)
+		assert.NotNil(commit.Commit)
+
+		// compare the commits (note Commit(2) has commit from Block(3))
+		commit2, err := c.Commit(h - 1)
+		require.Nil(err, "%d: %+v", i, err)
+		assert.Equal(block.Block.LastCommit, commit2.Commit)
+
+		// and we got a proof that works!
+		pres, err := c.ABCIQuery("/key", k, true)
+		if assert.Nil(err) && assert.True(pres.Response.Code.IsOK()) {
+			proof, err := merkle.ReadProof(pres.Response.GetProof())
+			if assert.Nil(err) {
+				key := pres.Response.GetKey()
+				value := pres.Response.GetValue()
+				assert.Equal(appHash, proof.RootHash)
+				valid := proof.Verify(key, value, appHash)
+				assert.True(valid)
+			}
+		}
+	}
+}
+
+// TestSubscriptions only works for HTTPClient
+//
+// TODO: generalize this functionality -> Local and Client
+func TestSubscriptions(t *testing.T) {
+	require := require.New(t)
+	c := getHTTPClient()
+	err := c.StartWebsocket()
+	require.Nil(err)
+	defer c.StopWebsocket()
+
+	// subscribe to a transaction event
+	_, _, tx := merktest.MakeTxKV()
+	eventType := types.EventStringTx(types.Tx(tx))
+	c.Subscribe(eventType)
+
+	// set up a listener
+	r, e := c.GetEventChannels()
+	go func() {
+		// send a tx and wait for it to propogate
+		_, err = c.BroadcastTxCommit(tx)
+		require.Nil(err, string(tx))
+	}()
+
+	checkData := func(data []byte, kind byte) {
+		x := []interface{}{}
+		err := json.Unmarshal(data, &x)
+		require.Nil(err)
+		// gotta love wire's json format
+		require.EqualValues(kind, x[0])
+	}
+
+	res := <-r
+	checkData(res, ctypes.ResultTypeSubscribe)
+
+	// read one event, must be success
+	select {
+	case res := <-r:
+		checkData(res, ctypes.ResultTypeEvent)
+		// this is good.. let's get the data... ugh...
+		// result := new(ctypes.TMResult)
+		// wire.ReadJSON(result, res, &err)
+		// require.Nil(err, "%+v", err)
+		// event, ok := (*result).(*ctypes.ResultEvent)
+		// require.True(ok)
+		// assert.Equal("foo", event.Name)
+		// data, ok := event.Data.(types.EventDataTx)
+		// require.True(ok)
+		// assert.EqualValues(0, data.Code)
+		// assert.EqualValues(tx, data.Tx)
+	case err := <-e:
+		// this is a failure
+		require.Nil(err)
+	}
+
+}

From 98450ee2db39a7ccc0701eaa66c5094c71227e16 Mon Sep 17 00:00:00 2001
From: Ethan Frey <ethanfrey@users.noreply.github.com>
Date: Fri, 24 Feb 2017 17:04:38 +0100
Subject: [PATCH 092/132] Client: DumpConsensusState, not DialSeeds. Cleanup

---
 rpc/client/httpclient.go  | 13 ++++++----
 rpc/client/interface.go   |  3 +--
 rpc/client/localclient.go | 51 ++++++---------------------------------
 rpc/client/mock/client.go | 43 ---------------------------------
 rpc/client/rpc_test.go    |  6 +++--
 5 files changed, 21 insertions(+), 95 deletions(-)

diff --git a/rpc/client/httpclient.go b/rpc/client/httpclient.go
index ea81199f..6245333c 100644
--- a/rpc/client/httpclient.go
+++ b/rpc/client/httpclient.go
@@ -39,6 +39,10 @@ func (c *HTTP) _assertIsClient() Client {
 	return c
 }
 
+func (c *HTTP) _assertIsNetworkClient() NetworkClient {
+	return c
+}
+
 func (c *HTTP) Status() (*ctypes.ResultStatus, error) {
 	tmResult := new(ctypes.TMResult)
 	_, err := c.rpc.Call("status", []interface{}{}, tmResult)
@@ -102,14 +106,13 @@ func (c *HTTP) NetInfo() (*ctypes.ResultNetInfo, error) {
 	return (*tmResult).(*ctypes.ResultNetInfo), nil
 }
 
-func (c *HTTP) DialSeeds(seeds []string) (*ctypes.ResultDialSeeds, error) {
+func (c *HTTP) DumpConsensusState() (*ctypes.ResultDumpConsensusState, error) {
 	tmResult := new(ctypes.TMResult)
-	// TODO: is this the correct way to marshall seeds?
-	_, err := c.rpc.Call("dial_seeds", []interface{}{seeds}, tmResult)
+	_, err := c.rpc.Call("dump_consensus_state", nil, tmResult)
 	if err != nil {
-		return nil, errors.Wrap(err, "DialSeeds")
+		return nil, errors.Wrap(err, "DumpConsensusState")
 	}
-	return (*tmResult).(*ctypes.ResultDialSeeds), nil
+	return (*tmResult).(*ctypes.ResultDumpConsensusState), nil
 }
 
 func (c *HTTP) BlockchainInfo(minHeight, maxHeight int) (*ctypes.ResultBlockchainInfo, error) {
diff --git a/rpc/client/interface.go b/rpc/client/interface.go
index b50950e4..5dffe30b 100644
--- a/rpc/client/interface.go
+++ b/rpc/client/interface.go
@@ -53,8 +53,7 @@ type SignClient interface {
 // by concrete implementations.
 type NetworkClient interface {
 	NetInfo() (*ctypes.ResultNetInfo, error)
-	// remove this???
-	DialSeeds(seeds []string) (*ctypes.ResultDialSeeds, error)
+	DumpConsensusState() (*ctypes.ResultDumpConsensusState, error)
 }
 
 // HistoryClient shows us data from genesis to now in large chunks.
diff --git a/rpc/client/localclient.go b/rpc/client/localclient.go
index eb180a6e..fc533f15 100644
--- a/rpc/client/localclient.go
+++ b/rpc/client/localclient.go
@@ -42,6 +42,10 @@ func (c Local) _assertIsClient() Client {
 	return c
 }
 
+func (c Local) _assertIsNetworkClient() NetworkClient {
+	return c
+}
+
 func (c Local) Status() (*ctypes.ResultStatus, error) {
 	return core.Status()
 }
@@ -70,6 +74,10 @@ func (c Local) NetInfo() (*ctypes.ResultNetInfo, error) {
 	return core.NetInfo()
 }
 
+func (c Local) DumpConsensusState() (*ctypes.ResultDumpConsensusState, error) {
+	return core.DumpConsensusState()
+}
+
 func (c Local) DialSeeds(seeds []string) (*ctypes.ResultDialSeeds, error) {
 	return core.UnsafeDialSeeds(seeds)
 }
@@ -93,46 +101,3 @@ func (c Local) Commit(height int) (*ctypes.ResultCommit, error) {
 func (c Local) Validators() (*ctypes.ResultValidators, error) {
 	return core.Validators()
 }
-
-/** websocket event stuff here... **/
-
-/*
-// StartWebsocket starts up a websocket and a listener goroutine
-// if already started, do nothing
-func (c Client) StartWebsocket() error {
-	var err error
-	if c.ws == nil {
-		ws := rpcclient.NewWSClient(c.remote, c.endpoint)
-		_, err = ws.Start()
-		if err == nil {
-			c.ws = ws
-		}
-	}
-	return errors.Wrap(err, "StartWebsocket")
-}
-
-// StopWebsocket stops the websocket connection
-func (c Client) StopWebsocket() {
-	if c.ws != nil {
-		c.ws.Stop()
-		c.ws = nil
-	}
-}
-
-// GetEventChannels returns the results and error channel from the websocket
-func (c Client) GetEventChannels() (chan json.RawMessage, chan error) {
-	if c.ws == nil {
-		return nil, nil
-	}
-	return c.ws.ResultsCh, c.ws.ErrorsCh
-}
-
-func (c Client) Subscribe(event string) error {
-	return errors.Wrap(c.ws.Subscribe(event), "Subscribe")
-}
-
-func (c Client) Unsubscribe(event string) error {
-	return errors.Wrap(c.ws.Unsubscribe(event), "Unsubscribe")
-}
-
-*/
diff --git a/rpc/client/mock/client.go b/rpc/client/mock/client.go
index ddeb1219..0dcba718 100644
--- a/rpc/client/mock/client.go
+++ b/rpc/client/mock/client.go
@@ -124,46 +124,3 @@ func (c Client) Commit(height int) (*ctypes.ResultCommit, error) {
 func (c Client) Validators() (*ctypes.ResultValidators, error) {
 	return core.Validators()
 }
-
-/** websocket event stuff here... **/
-
-/*
-// StartWebsocket starts up a websocket and a listener goroutine
-// if already started, do nothing
-func (c Client) StartWebsocket() error {
-	var err error
-	if c.ws == nil {
-		ws := rpcclient.NewWSClient(c.remote, c.endpoint)
-		_, err = ws.Start()
-		if err == nil {
-			c.ws = ws
-		}
-	}
-	return errors.Wrap(err, "StartWebsocket")
-}
-
-// StopWebsocket stops the websocket connection
-func (c Client) StopWebsocket() {
-	if c.ws != nil {
-		c.ws.Stop()
-		c.ws = nil
-	}
-}
-
-// GetEventChannels returns the results and error channel from the websocket
-func (c Client) GetEventChannels() (chan json.RawMessage, chan error) {
-	if c.ws == nil {
-		return nil, nil
-	}
-	return c.ws.ResultsCh, c.ws.ErrorsCh
-}
-
-func (c Client) Subscribe(event string) error {
-	return errors.Wrap(c.ws.Subscribe(event), "Subscribe")
-}
-
-func (c Client) Unsubscribe(event string) error {
-	return errors.Wrap(c.ws.Unsubscribe(event), "Unsubscribe")
-}
-
-*/
diff --git a/rpc/client/rpc_test.go b/rpc/client/rpc_test.go
index f37386c3..d9670814 100644
--- a/rpc/client/rpc_test.go
+++ b/rpc/client/rpc_test.go
@@ -67,13 +67,15 @@ func TestNetInfo(t *testing.T) {
 	}
 }
 
-func TestDialSeeds(t *testing.T) {
+func TestDumpConsensusState(t *testing.T) {
 	for i, c := range GetClients() {
 		// FIXME: fix server so it doesn't panic on invalid input
 		nc, ok := c.(client.NetworkClient)
 		require.True(t, ok, "%d", i)
-		_, err := nc.DialSeeds([]string{"12.34.56.78:12345"})
+		cons, err := nc.DumpConsensusState()
 		require.Nil(t, err, "%d: %+v", i, err)
+		assert.NotEmpty(t, cons.RoundState)
+		assert.Empty(t, cons.PeerRoundStates)
 	}
 }
 

From 9be306490409488b7628aa74d31e3b18069f9329 Mon Sep 17 00:00:00 2001
From: Ethan Frey <ethanfrey@users.noreply.github.com>
Date: Fri, 24 Feb 2017 17:51:19 +0100
Subject: [PATCH 093/132] Expose and test EventSwitch in client.Local

---
 rpc/client/event_test.go  | 58 +++++++++++++++++++++++++++++++++++++++
 rpc/client/interface.go   | 26 +++++++++---------
 rpc/client/localclient.go |  4 ++-
 3 files changed, 74 insertions(+), 14 deletions(-)
 create mode 100644 rpc/client/event_test.go

diff --git a/rpc/client/event_test.go b/rpc/client/event_test.go
new file mode 100644
index 00000000..912949e0
--- /dev/null
+++ b/rpc/client/event_test.go
@@ -0,0 +1,58 @@
+package client_test
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	events "github.com/tendermint/go-events"
+	"github.com/tendermint/tendermint/types"
+)
+
+func TestEvents(t *testing.T) {
+	require := require.New(t)
+	for i, c := range GetClients() {
+		// test if this client implements event switch as well.
+		evsw, ok := c.(types.EventSwitch)
+		// TODO: assert this for all clients when it is suported
+		// if !assert.True(ok, "%d: %v", i, c) {
+		// 	continue
+		// }
+		if !ok {
+			continue
+		}
+
+		// start for this test it if it wasn't already running
+		if !evsw.IsRunning() {
+			// if so, then we start it, listen, and stop it.
+			st, err := evsw.Start()
+			require.Nil(err, "%d: %+v", i, err)
+			require.True(st, "%d", i)
+			// defer evsw.Stop()
+		}
+
+		// let's wait for the next header...
+		listener := "fooz"
+		event, timeout := make(chan events.EventData, 1), make(chan bool, 1)
+		// start timeout count-down
+		go func() {
+			time.Sleep(1 * time.Second)
+			timeout <- true
+		}()
+
+		// register for the next header event
+		evsw.AddListenerForEvent(listener, types.EventStringNewBlockHeader(), func(data events.EventData) {
+			event <- data
+		})
+		// make sure to unregister after the test is over
+		defer evsw.RemoveListener(listener)
+
+		select {
+		case <-timeout:
+			require.True(false, "%d: a timeout waiting for event", i)
+		case evt := <-event:
+			_, ok := evt.(types.EventDataNewBlockHeader)
+			require.True(ok, "%d: %#v", i, evt)
+		}
+	}
+}
diff --git a/rpc/client/interface.go b/rpc/client/interface.go
index 5dffe30b..50f06511 100644
--- a/rpc/client/interface.go
+++ b/rpc/client/interface.go
@@ -46,16 +46,6 @@ type SignClient interface {
 	Validators() (*ctypes.ResultValidators, error)
 }
 
-// NetworkClient is general info about the network state.  May not
-// be needed usually.
-//
-// Not included in the Client interface, but generally implemented
-// by concrete implementations.
-type NetworkClient interface {
-	NetInfo() (*ctypes.ResultNetInfo, error)
-	DumpConsensusState() (*ctypes.ResultDumpConsensusState, error)
-}
-
 // HistoryClient shows us data from genesis to now in large chunks.
 type HistoryClient interface {
 	Genesis() (*ctypes.ResultGenesis, error)
@@ -67,12 +57,22 @@ type StatusClient interface {
 	Status() (*ctypes.ResultStatus, error)
 }
 
+// Client wraps most important rpc calls a client would make
+// if you want to listen for events, test if it also
+// implements events.EventSwitch
 type Client interface {
 	ABCIClient
 	SignClient
 	HistoryClient
 	StatusClient
-	// Note: doesn't include NetworkClient, is it important??
-
-	// TODO: add some sort of generic subscription mechanism...
+}
+
+// NetworkClient is general info about the network state.  May not
+// be needed usually.
+//
+// Not included in the Client interface, but generally implemented
+// by concrete implementations.
+type NetworkClient interface {
+	NetInfo() (*ctypes.ResultNetInfo, error)
+	DumpConsensusState() (*ctypes.ResultDumpConsensusState, error)
 }
diff --git a/rpc/client/localclient.go b/rpc/client/localclient.go
index fc533f15..0a83db05 100644
--- a/rpc/client/localclient.go
+++ b/rpc/client/localclient.go
@@ -23,6 +23,7 @@ powerful control during testing, you probably want the "client/mock" package.
 */
 type Local struct {
 	node *nm.Node
+	types.EventSwitch
 }
 
 // NewLocal configures a client that calls the Node directly.
@@ -34,7 +35,8 @@ type Local struct {
 func NewLocal(node *nm.Node) Local {
 	node.ConfigureRPC()
 	return Local{
-		node: node,
+		node:        node,
+		EventSwitch: node.EventSwitch(),
 	}
 }
 

From 175bb329e4347bcb760ee978763d208213674504 Mon Sep 17 00:00:00 2001
From: Ethan Frey <ethanfrey@users.noreply.github.com>
Date: Fri, 24 Feb 2017 19:15:22 +0100
Subject: [PATCH 094/132] Expose EventSwitch on top of websocket client

---
 rpc/client/event_test.go |  17 ++---
 rpc/client/httpclient.go | 140 +++++++++++++++++++++++++++++++--------
 rpc/client/rpc_test.go   | 119 +++++++++++++++++----------------
 3 files changed, 180 insertions(+), 96 deletions(-)

diff --git a/rpc/client/event_test.go b/rpc/client/event_test.go
index 912949e0..863d23d4 100644
--- a/rpc/client/event_test.go
+++ b/rpc/client/event_test.go
@@ -4,6 +4,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	events "github.com/tendermint/go-events"
 	"github.com/tendermint/tendermint/types"
@@ -12,13 +13,10 @@ import (
 func TestEvents(t *testing.T) {
 	require := require.New(t)
 	for i, c := range GetClients() {
+		// for i, c := range []client.Client{getLocalClient()} {
 		// test if this client implements event switch as well.
 		evsw, ok := c.(types.EventSwitch)
-		// TODO: assert this for all clients when it is suported
-		// if !assert.True(ok, "%d: %v", i, c) {
-		// 	continue
-		// }
-		if !ok {
+		if !assert.True(t, ok, "%d: %v", i, c) {
 			continue
 		}
 
@@ -28,12 +26,12 @@ func TestEvents(t *testing.T) {
 			st, err := evsw.Start()
 			require.Nil(err, "%d: %+v", i, err)
 			require.True(st, "%d", i)
-			// defer evsw.Stop()
+			defer evsw.Stop()
 		}
 
 		// let's wait for the next header...
 		listener := "fooz"
-		event, timeout := make(chan events.EventData, 1), make(chan bool, 1)
+		event, timeout := make(chan events.EventData, 10), make(chan bool, 1)
 		// start timeout count-down
 		go func() {
 			time.Sleep(1 * time.Second)
@@ -41,10 +39,13 @@ func TestEvents(t *testing.T) {
 		}()
 
 		// register for the next header event
-		evsw.AddListenerForEvent(listener, types.EventStringNewBlockHeader(), func(data events.EventData) {
+		evtTyp := types.EventStringNewBlockHeader()
+		evsw.AddListenerForEvent(listener, evtTyp, func(data events.EventData) {
 			event <- data
 		})
 		// make sure to unregister after the test is over
+		// TODO: don't require both!
+		defer evsw.RemoveListenerForEvent(listener, evtTyp)
 		defer evsw.RemoveListener(listener)
 
 		select {
diff --git a/rpc/client/httpclient.go b/rpc/client/httpclient.go
index 6245333c..f07775eb 100644
--- a/rpc/client/httpclient.go
+++ b/rpc/client/httpclient.go
@@ -1,10 +1,12 @@
 package client
 
 import (
-	"encoding/json"
+	"fmt"
 
 	"github.com/pkg/errors"
+	events "github.com/tendermint/go-events"
 	"github.com/tendermint/go-rpc/client"
+	wire "github.com/tendermint/go-wire"
 	ctypes "github.com/tendermint/tendermint/rpc/core/types"
 	"github.com/tendermint/tendermint/types"
 )
@@ -19,10 +21,9 @@ the tendermint node in-process (local), or when you want to mock
 out the server for test code (mock).
 */
 type HTTP struct {
-	remote   string
-	endpoint string
-	rpc      *rpcclient.ClientJSONRPC
-	ws       *rpcclient.WSClient
+	remote string
+	rpc    *rpcclient.ClientJSONRPC
+	*WSEvents
 }
 
 // New takes a remote endpoint in the form tcp://<host>:<port>
@@ -31,7 +32,7 @@ func NewHTTP(remote, wsEndpoint string) *HTTP {
 	return &HTTP{
 		rpc:      rpcclient.NewClientJSONRPC(remote),
 		remote:   remote,
-		endpoint: wsEndpoint,
+		WSEvents: newWSEvents(remote, wsEndpoint),
 	}
 }
 
@@ -43,6 +44,10 @@ func (c *HTTP) _assertIsNetworkClient() NetworkClient {
 	return c
 }
 
+func (c *HTTP) _assertIsEventSwitch() types.EventSwitch {
+	return c
+}
+
 func (c *HTTP) Status() (*ctypes.ResultStatus, error) {
 	tmResult := new(ctypes.TMResult)
 	_, err := c.rpc.Call("status", []interface{}{}, tmResult)
@@ -162,40 +167,119 @@ func (c *HTTP) Validators() (*ctypes.ResultValidators, error) {
 
 /** websocket event stuff here... **/
 
-// StartWebsocket starts up a websocket and a listener goroutine
-// if already started, do nothing
-func (c *HTTP) StartWebsocket() error {
-	var err error
-	if c.ws == nil {
-		ws := rpcclient.NewWSClient(c.remote, c.endpoint)
+type WSEvents struct {
+	types.EventSwitch
+	remote   string
+	endpoint string
+	ws       *rpcclient.WSClient
+	quit     chan bool
+}
+
+func newWSEvents(remote, endpoint string) *WSEvents {
+	return &WSEvents{
+		EventSwitch: types.NewEventSwitch(),
+		endpoint:    endpoint,
+		remote:      remote,
+		quit:        make(chan bool, 1),
+	}
+}
+
+func (w *WSEvents) _assertIsEventSwitch() types.EventSwitch {
+	return w
+}
+
+// Start is the only way I could think the extend OnStart from
+// events.eventSwitch.  If only it wasn't private...
+// BaseService.Start -> eventSwitch.OnStart -> WSEvents.Start
+func (w *WSEvents) Start() (bool, error) {
+	st, err := w.EventSwitch.Start()
+	// if we did start, then OnStart here...
+	if st && err == nil {
+		ws := rpcclient.NewWSClient(w.remote, w.endpoint)
 		_, err = ws.Start()
 		if err == nil {
-			c.ws = ws
+			w.ws = ws
+			go w.eventListener()
 		}
 	}
-	return errors.Wrap(err, "StartWebsocket")
+	return st, errors.Wrap(err, "StartWSEvent")
 }
 
-// StopWebsocket stops the websocket connection
-func (c *HTTP) StopWebsocket() {
-	if c.ws != nil {
-		c.ws.Stop()
-		c.ws = nil
+// Stop wraps the BaseService/eventSwitch actions as Start does
+func (w *WSEvents) Stop() bool {
+	stop := w.EventSwitch.Stop()
+	if stop {
+		// send a message to quit to stop the eventListener
+		w.quit <- true
+		w.ws.Stop()
+	}
+	return stop
+}
+
+/** TODO: more intelligent subscriptions! **/
+func (w *WSEvents) AddListenerForEvent(listenerID, event string, cb events.EventCallback) {
+	w.subscribe(event)
+	w.EventSwitch.AddListenerForEvent(listenerID, event, cb)
+}
+
+func (w *WSEvents) RemoveListenerForEvent(event string, listenerID string) {
+	w.unsubscribe(event)
+	w.EventSwitch.RemoveListenerForEvent(event, listenerID)
+}
+
+func (w *WSEvents) RemoveListener(listenerID string) {
+	w.EventSwitch.RemoveListener(listenerID)
+}
+
+// eventListener is an infinite loop pulling all websocket events
+// and pushing them to the EventSwitch.
+//
+// the goroutine only stops by closing quit
+func (w *WSEvents) eventListener() {
+	for {
+		select {
+		case res := <-w.ws.ResultsCh:
+			// res is json.RawMessage
+			err := w.parseEvent(res)
+			if err != nil {
+				// FIXME: better logging/handling of errors??
+				fmt.Printf("ws result: %+v\n", err)
+			}
+		case err := <-w.ws.ErrorsCh:
+			// FIXME: better logging/handling of errors??
+			fmt.Printf("ws err: %+v\n", err)
+		case <-w.quit:
+			// only way to finish this method
+			return
+		}
 	}
 }
 
-// GetEventChannels returns the results and error channel from the websocket
-func (c *HTTP) GetEventChannels() (chan json.RawMessage, chan error) {
-	if c.ws == nil {
-		return nil, nil
+// parseEvent unmarshals the json message and converts it into
+// some implementation of types.TMEventData, and sends it off
+// on the merry way to the EventSwitch
+func (w *WSEvents) parseEvent(data []byte) (err error) {
+	result := new(ctypes.TMResult)
+	wire.ReadJSONPtr(result, data, &err)
+	if err != nil {
+		return err
 	}
-	return c.ws.ResultsCh, c.ws.ErrorsCh
+	event, ok := (*result).(*ctypes.ResultEvent)
+	if !ok {
+		// ignore silently (eg. subscribe, unsubscribe and maybe other events)
+		return nil
+		// or report loudly???
+		// return errors.Errorf("unknown message: %#v", *result)
+	}
+	// looks good!  let's fire this baby!
+	w.EventSwitch.FireEvent(event.Name, event.Data)
+	return nil
 }
 
-func (c *HTTP) Subscribe(event string) error {
-	return errors.Wrap(c.ws.Subscribe(event), "Subscribe")
+func (w *WSEvents) subscribe(event string) error {
+	return errors.Wrap(w.ws.Subscribe(event), "Subscribe")
 }
 
-func (c *HTTP) Unsubscribe(event string) error {
-	return errors.Wrap(c.ws.Unsubscribe(event), "Unsubscribe")
+func (w *WSEvents) unsubscribe(event string) error {
+	return errors.Wrap(w.ws.Unsubscribe(event), "Unsubscribe")
 }
diff --git a/rpc/client/rpc_test.go b/rpc/client/rpc_test.go
index d9670814..4c9e5abf 100644
--- a/rpc/client/rpc_test.go
+++ b/rpc/client/rpc_test.go
@@ -1,7 +1,6 @@
 package client_test
 
 import (
-	"encoding/json"
 	"strings"
 	"testing"
 	"time"
@@ -11,9 +10,7 @@ import (
 	merkle "github.com/tendermint/go-merkle"
 	merktest "github.com/tendermint/merkleeyes/testutil"
 	"github.com/tendermint/tendermint/rpc/client"
-	ctypes "github.com/tendermint/tendermint/rpc/core/types"
 	rpctest "github.com/tendermint/tendermint/rpc/test"
-	"github.com/tendermint/tendermint/types"
 )
 
 func getHTTPClient() *client.HTTP {
@@ -67,17 +64,19 @@ func TestNetInfo(t *testing.T) {
 	}
 }
 
-func TestDumpConsensusState(t *testing.T) {
-	for i, c := range GetClients() {
-		// FIXME: fix server so it doesn't panic on invalid input
-		nc, ok := c.(client.NetworkClient)
-		require.True(t, ok, "%d", i)
-		cons, err := nc.DumpConsensusState()
-		require.Nil(t, err, "%d: %+v", i, err)
-		assert.NotEmpty(t, cons.RoundState)
-		assert.Empty(t, cons.PeerRoundStates)
-	}
-}
+// FIXME: This seems to trigger a race condition with client.Local
+// go test -v -race . -run=DumpCons
+// func TestDumpConsensusState(t *testing.T) {
+// 	for i, c := range GetClients() {
+// 		// FIXME: fix server so it doesn't panic on invalid input
+// 		nc, ok := c.(client.NetworkClient)
+// 		require.True(t, ok, "%d", i)
+// 		cons, err := nc.DumpConsensusState()
+// 		require.Nil(t, err, "%d: %+v", i, err)
+// 		assert.NotEmpty(t, cons.RoundState)
+// 		assert.Empty(t, cons.PeerRoundStates)
+// 	}
+// }
 
 func TestGenesisAndValidators(t *testing.T) {
 	for i, c := range GetClients() {
@@ -184,55 +183,55 @@ func TestAppCalls(t *testing.T) {
 // TestSubscriptions only works for HTTPClient
 //
 // TODO: generalize this functionality -> Local and Client
-func TestSubscriptions(t *testing.T) {
-	require := require.New(t)
-	c := getHTTPClient()
-	err := c.StartWebsocket()
-	require.Nil(err)
-	defer c.StopWebsocket()
+// func TestSubscriptions(t *testing.T) {
+// 	require := require.New(t)
+// 	c := getHTTPClient()
+// 	err := c.StartWebsocket()
+// 	require.Nil(err)
+// 	defer c.StopWebsocket()
 
-	// subscribe to a transaction event
-	_, _, tx := merktest.MakeTxKV()
-	eventType := types.EventStringTx(types.Tx(tx))
-	c.Subscribe(eventType)
+// 	// subscribe to a transaction event
+// 	_, _, tx := merktest.MakeTxKV()
+// 	eventType := types.EventStringTx(types.Tx(tx))
+// 	c.Subscribe(eventType)
 
-	// set up a listener
-	r, e := c.GetEventChannels()
-	go func() {
-		// send a tx and wait for it to propogate
-		_, err = c.BroadcastTxCommit(tx)
-		require.Nil(err, string(tx))
-	}()
+// 	// set up a listener
+// 	r, e := c.GetEventChannels()
+// 	go func() {
+// 		// send a tx and wait for it to propogate
+// 		_, err = c.BroadcastTxCommit(tx)
+// 		require.Nil(err, string(tx))
+// 	}()
 
-	checkData := func(data []byte, kind byte) {
-		x := []interface{}{}
-		err := json.Unmarshal(data, &x)
-		require.Nil(err)
-		// gotta love wire's json format
-		require.EqualValues(kind, x[0])
-	}
+// 	checkData := func(data []byte, kind byte) {
+// 		x := []interface{}{}
+// 		err := json.Unmarshal(data, &x)
+// 		require.Nil(err)
+// 		// gotta love wire's json format
+// 		require.EqualValues(kind, x[0])
+// 	}
 
-	res := <-r
-	checkData(res, ctypes.ResultTypeSubscribe)
+// 	res := <-r
+// 	checkData(res, ctypes.ResultTypeSubscribe)
 
-	// read one event, must be success
-	select {
-	case res := <-r:
-		checkData(res, ctypes.ResultTypeEvent)
-		// this is good.. let's get the data... ugh...
-		// result := new(ctypes.TMResult)
-		// wire.ReadJSON(result, res, &err)
-		// require.Nil(err, "%+v", err)
-		// event, ok := (*result).(*ctypes.ResultEvent)
-		// require.True(ok)
-		// assert.Equal("foo", event.Name)
-		// data, ok := event.Data.(types.EventDataTx)
-		// require.True(ok)
-		// assert.EqualValues(0, data.Code)
-		// assert.EqualValues(tx, data.Tx)
-	case err := <-e:
-		// this is a failure
-		require.Nil(err)
-	}
+// 	// read one event, must be success
+// 	select {
+// 	case res := <-r:
+// 		checkData(res, ctypes.ResultTypeEvent)
+// 		// this is good.. let's get the data... ugh...
+// 		// result := new(ctypes.TMResult)
+// 		// wire.ReadJSON(result, res, &err)
+// 		// require.Nil(err, "%+v", err)
+// 		// event, ok := (*result).(*ctypes.ResultEvent)
+// 		// require.True(ok)
+// 		// assert.Equal("foo", event.Name)
+// 		// data, ok := event.Data.(types.EventDataTx)
+// 		// require.True(ok)
+// 		// assert.EqualValues(0, data.Code)
+// 		// assert.EqualValues(tx, data.Tx)
+// 	case err := <-e:
+// 		// this is a failure
+// 		require.Nil(err)
+// 	}
 
-}
+// }

From 6282fad51816404e1e26e7c672a73d30b99f1aa1 Mon Sep 17 00:00:00 2001
From: Ethan Frey <ethanfrey@users.noreply.github.com>
Date: Fri, 24 Feb 2017 19:59:40 +0100
Subject: [PATCH 095/132] Clean up event switch add helper function

---
 rpc/client/event_test.go | 75 +++++++++++++++++++++++++---------------
 rpc/client/helpers.go    | 35 +++++++++++++++++++
 rpc/client/httpclient.go |  8 ++++-
 rpc/client/rpc_test.go   | 56 ------------------------------
 4 files changed, 89 insertions(+), 85 deletions(-)

diff --git a/rpc/client/event_test.go b/rpc/client/event_test.go
index 863d23d4..f43a47e9 100644
--- a/rpc/client/event_test.go
+++ b/rpc/client/event_test.go
@@ -6,14 +6,14 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	events "github.com/tendermint/go-events"
+	merktest "github.com/tendermint/merkleeyes/testutil"
+	"github.com/tendermint/tendermint/rpc/client"
 	"github.com/tendermint/tendermint/types"
 )
 
-func TestEvents(t *testing.T) {
+func TestHeaderEvents(t *testing.T) {
 	require := require.New(t)
 	for i, c := range GetClients() {
-		// for i, c := range []client.Client{getLocalClient()} {
 		// test if this client implements event switch as well.
 		evsw, ok := c.(types.EventSwitch)
 		if !assert.True(t, ok, "%d: %v", i, c) {
@@ -29,31 +29,50 @@ func TestEvents(t *testing.T) {
 			defer evsw.Stop()
 		}
 
-		// let's wait for the next header...
-		listener := "fooz"
-		event, timeout := make(chan events.EventData, 10), make(chan bool, 1)
-		// start timeout count-down
-		go func() {
-			time.Sleep(1 * time.Second)
-			timeout <- true
-		}()
-
-		// register for the next header event
 		evtTyp := types.EventStringNewBlockHeader()
-		evsw.AddListenerForEvent(listener, evtTyp, func(data events.EventData) {
-			event <- data
-		})
-		// make sure to unregister after the test is over
-		// TODO: don't require both!
-		defer evsw.RemoveListenerForEvent(listener, evtTyp)
-		defer evsw.RemoveListener(listener)
-
-		select {
-		case <-timeout:
-			require.True(false, "%d: a timeout waiting for event", i)
-		case evt := <-event:
-			_, ok := evt.(types.EventDataNewBlockHeader)
-			require.True(ok, "%d: %#v", i, evt)
-		}
+		evt, err := client.WaitForOneEvent(evsw, evtTyp, 1*time.Second)
+		require.Nil(err, "%d: %+v", i, err)
+		_, ok = evt.(types.EventDataNewBlockHeader)
+		require.True(ok, "%d: %#v", i, evt)
+		// TODO: more checks...
+	}
+}
+
+func TestTxEvents(t *testing.T) {
+	require := require.New(t)
+	for i, c := range GetClients() {
+		// test if this client implements event switch as well.
+		evsw, ok := c.(types.EventSwitch)
+		if !assert.True(t, ok, "%d: %v", i, c) {
+			continue
+		}
+
+		// start for this test it if it wasn't already running
+		if !evsw.IsRunning() {
+			// if so, then we start it, listen, and stop it.
+			st, err := evsw.Start()
+			require.Nil(err, "%d: %+v", i, err)
+			require.True(st, "%d", i)
+			defer evsw.Stop()
+		}
+
+		// make the tx
+		_, _, tx := merktest.MakeTxKV()
+		evtTyp := types.EventStringTx(types.Tx(tx))
+
+		// send async
+		txres, err := c.BroadcastTxAsync(tx)
+		require.Nil(err, "%+v", err)
+		require.True(txres.Code.IsOK())
+
+		// and wait for confirmation
+		evt, err := client.WaitForOneEvent(evsw, evtTyp, 1*time.Second)
+		require.Nil(err, "%d: %+v", i, err)
+		// and make sure it has the proper info
+		txe, ok := evt.(types.EventDataTx)
+		require.True(ok, "%d: %#v", i, evt)
+		// make sure this is the proper tx
+		require.EqualValues(tx, txe.Tx)
+		require.True(txe.Code.IsOK())
 	}
 }
diff --git a/rpc/client/helpers.go b/rpc/client/helpers.go
index bf8860e7..5af1fe20 100644
--- a/rpc/client/helpers.go
+++ b/rpc/client/helpers.go
@@ -4,6 +4,8 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
+	cmn "github.com/tendermint/go-common"
+	events "github.com/tendermint/go-events"
 )
 
 // Waiter is informed of current height, decided whether to quit early
@@ -47,3 +49,36 @@ func WaitForHeight(c StatusClient, h int, waiter Waiter) error {
 	}
 	return nil
 }
+
+// WaitForOneEvent subscribes to a websocket event for the given
+// event time and returns upon receiving it one time, or
+// when the timeout duration has expired.
+//
+// This handles subscribing and unsubscribing under the hood
+func WaitForOneEvent(evsw events.EventSwitch,
+	evtTyp string, timeout time.Duration) (events.EventData, error) {
+	listener := cmn.RandStr(12)
+
+	evts, quit := make(chan events.EventData, 10), make(chan bool, 1)
+	// start timeout count-down
+	go func() {
+		time.Sleep(1 * time.Second)
+		quit <- true
+	}()
+
+	// register for the next event of this type
+	evsw.AddListenerForEvent(listener, evtTyp, func(data events.EventData) {
+		evts <- data
+	})
+	// make sure to unregister after the test is over
+	// TODO: don't require both!
+	defer evsw.RemoveListenerForEvent(listener, evtTyp)
+	defer evsw.RemoveListener(listener)
+
+	select {
+	case <-quit:
+		return nil, errors.New("timed out waiting for event")
+	case evt := <-evts:
+		return evt, nil
+	}
+}
diff --git a/rpc/client/httpclient.go b/rpc/client/httpclient.go
index f07775eb..4eb94f5a 100644
--- a/rpc/client/httpclient.go
+++ b/rpc/client/httpclient.go
@@ -173,6 +173,7 @@ type WSEvents struct {
 	endpoint string
 	ws       *rpcclient.WSClient
 	quit     chan bool
+	done     chan bool
 }
 
 func newWSEvents(remote, endpoint string) *WSEvents {
@@ -181,6 +182,7 @@ func newWSEvents(remote, endpoint string) *WSEvents {
 		endpoint:    endpoint,
 		remote:      remote,
 		quit:        make(chan bool, 1),
+		done:        make(chan bool, 1),
 	}
 }
 
@@ -211,7 +213,9 @@ func (w *WSEvents) Stop() bool {
 	if stop {
 		// send a message to quit to stop the eventListener
 		w.quit <- true
+		<-w.done
 		w.ws.Stop()
+		w.ws = nil
 	}
 	return stop
 }
@@ -249,7 +253,9 @@ func (w *WSEvents) eventListener() {
 			// FIXME: better logging/handling of errors??
 			fmt.Printf("ws err: %+v\n", err)
 		case <-w.quit:
-			// only way to finish this method
+			// send a message so we can wait for the routine to exit
+			// before cleaning up the w.ws stuff
+			w.done <- true
 			return
 		}
 	}
diff --git a/rpc/client/rpc_test.go b/rpc/client/rpc_test.go
index 4c9e5abf..11c7f726 100644
--- a/rpc/client/rpc_test.go
+++ b/rpc/client/rpc_test.go
@@ -179,59 +179,3 @@ func TestAppCalls(t *testing.T) {
 		}
 	}
 }
-
-// TestSubscriptions only works for HTTPClient
-//
-// TODO: generalize this functionality -> Local and Client
-// func TestSubscriptions(t *testing.T) {
-// 	require := require.New(t)
-// 	c := getHTTPClient()
-// 	err := c.StartWebsocket()
-// 	require.Nil(err)
-// 	defer c.StopWebsocket()
-
-// 	// subscribe to a transaction event
-// 	_, _, tx := merktest.MakeTxKV()
-// 	eventType := types.EventStringTx(types.Tx(tx))
-// 	c.Subscribe(eventType)
-
-// 	// set up a listener
-// 	r, e := c.GetEventChannels()
-// 	go func() {
-// 		// send a tx and wait for it to propogate
-// 		_, err = c.BroadcastTxCommit(tx)
-// 		require.Nil(err, string(tx))
-// 	}()
-
-// 	checkData := func(data []byte, kind byte) {
-// 		x := []interface{}{}
-// 		err := json.Unmarshal(data, &x)
-// 		require.Nil(err)
-// 		// gotta love wire's json format
-// 		require.EqualValues(kind, x[0])
-// 	}
-
-// 	res := <-r
-// 	checkData(res, ctypes.ResultTypeSubscribe)
-
-// 	// read one event, must be success
-// 	select {
-// 	case res := <-r:
-// 		checkData(res, ctypes.ResultTypeEvent)
-// 		// this is good.. let's get the data... ugh...
-// 		// result := new(ctypes.TMResult)
-// 		// wire.ReadJSON(result, res, &err)
-// 		// require.Nil(err, "%+v", err)
-// 		// event, ok := (*result).(*ctypes.ResultEvent)
-// 		// require.True(ok)
-// 		// assert.Equal("foo", event.Name)
-// 		// data, ok := event.Data.(types.EventDataTx)
-// 		// require.True(ok)
-// 		// assert.EqualValues(0, data.Code)
-// 		// assert.EqualValues(tx, data.Tx)
-// 	case err := <-e:
-// 		// this is a failure
-// 		require.Nil(err)
-// 	}
-
-// }

From 4fead237f082020e3d52b3bb0ab26e38d4267d62 Mon Sep 17 00:00:00 2001
From: Ethan Frey <ethanfrey@users.noreply.github.com>
Date: Fri, 24 Feb 2017 21:26:17 +0100
Subject: [PATCH 096/132] Client embeds EventSwitch, client.HTTP properly
 un/subscribes events over websocket

---
 rpc/client/event_test.go  | 31 +++++-----------
 rpc/client/helpers.go     | 15 +++++---
 rpc/client/httpclient.go  | 78 ++++++++++++++++++++++++++++++++++-----
 rpc/client/interface.go   |  4 ++
 rpc/client/mock/client.go |  2 +
 5 files changed, 93 insertions(+), 37 deletions(-)

diff --git a/rpc/client/event_test.go b/rpc/client/event_test.go
index f43a47e9..cc421ad9 100644
--- a/rpc/client/event_test.go
+++ b/rpc/client/event_test.go
@@ -4,7 +4,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	merktest "github.com/tendermint/merkleeyes/testutil"
 	"github.com/tendermint/tendermint/rpc/client"
@@ -14,25 +13,19 @@ import (
 func TestHeaderEvents(t *testing.T) {
 	require := require.New(t)
 	for i, c := range GetClients() {
-		// test if this client implements event switch as well.
-		evsw, ok := c.(types.EventSwitch)
-		if !assert.True(t, ok, "%d: %v", i, c) {
-			continue
-		}
-
 		// start for this test it if it wasn't already running
-		if !evsw.IsRunning() {
+		if !c.IsRunning() {
 			// if so, then we start it, listen, and stop it.
-			st, err := evsw.Start()
+			st, err := c.Start()
 			require.Nil(err, "%d: %+v", i, err)
 			require.True(st, "%d", i)
-			defer evsw.Stop()
+			defer c.Stop()
 		}
 
 		evtTyp := types.EventStringNewBlockHeader()
-		evt, err := client.WaitForOneEvent(evsw, evtTyp, 1*time.Second)
+		evt, err := client.WaitForOneEvent(c, evtTyp, 1*time.Second)
 		require.Nil(err, "%d: %+v", i, err)
-		_, ok = evt.(types.EventDataNewBlockHeader)
+		_, ok := evt.(types.EventDataNewBlockHeader)
 		require.True(ok, "%d: %#v", i, evt)
 		// TODO: more checks...
 	}
@@ -41,19 +34,13 @@ func TestHeaderEvents(t *testing.T) {
 func TestTxEvents(t *testing.T) {
 	require := require.New(t)
 	for i, c := range GetClients() {
-		// test if this client implements event switch as well.
-		evsw, ok := c.(types.EventSwitch)
-		if !assert.True(t, ok, "%d: %v", i, c) {
-			continue
-		}
-
 		// start for this test it if it wasn't already running
-		if !evsw.IsRunning() {
+		if !c.IsRunning() {
 			// if so, then we start it, listen, and stop it.
-			st, err := evsw.Start()
+			st, err := c.Start()
 			require.Nil(err, "%d: %+v", i, err)
 			require.True(st, "%d", i)
-			defer evsw.Stop()
+			defer c.Stop()
 		}
 
 		// make the tx
@@ -66,7 +53,7 @@ func TestTxEvents(t *testing.T) {
 		require.True(txres.Code.IsOK())
 
 		// and wait for confirmation
-		evt, err := client.WaitForOneEvent(evsw, evtTyp, 1*time.Second)
+		evt, err := client.WaitForOneEvent(c, evtTyp, 1*time.Second)
 		require.Nil(err, "%d: %+v", i, err)
 		// and make sure it has the proper info
 		txe, ok := evt.(types.EventDataTx)
diff --git a/rpc/client/helpers.go b/rpc/client/helpers.go
index 5af1fe20..89da434b 100644
--- a/rpc/client/helpers.go
+++ b/rpc/client/helpers.go
@@ -6,6 +6,7 @@ import (
 	"github.com/pkg/errors"
 	cmn "github.com/tendermint/go-common"
 	events "github.com/tendermint/go-events"
+	"github.com/tendermint/tendermint/types"
 )
 
 // Waiter is informed of current height, decided whether to quit early
@@ -55,8 +56,8 @@ func WaitForHeight(c StatusClient, h int, waiter Waiter) error {
 // when the timeout duration has expired.
 //
 // This handles subscribing and unsubscribing under the hood
-func WaitForOneEvent(evsw events.EventSwitch,
-	evtTyp string, timeout time.Duration) (events.EventData, error) {
+func WaitForOneEvent(evsw types.EventSwitch,
+	evtTyp string, timeout time.Duration) (types.TMEventData, error) {
 	listener := cmn.RandStr(12)
 
 	evts, quit := make(chan events.EventData, 10), make(chan bool, 1)
@@ -71,14 +72,18 @@ func WaitForOneEvent(evsw events.EventSwitch,
 		evts <- data
 	})
 	// make sure to unregister after the test is over
-	// TODO: don't require both!
-	defer evsw.RemoveListenerForEvent(listener, evtTyp)
+	// TODO: why doesn't the other call work???
+	// defer evsw.RemoveListenerForEvent(listener, evtTyp)
 	defer evsw.RemoveListener(listener)
 
 	select {
 	case <-quit:
 		return nil, errors.New("timed out waiting for event")
 	case evt := <-evts:
-		return evt, nil
+		tmevt, ok := evt.(types.TMEventData)
+		if ok {
+			return tmevt, nil
+		}
+		return nil, errors.Errorf("Got unexpected event type: %#v", evt)
 	}
 }
diff --git a/rpc/client/httpclient.go b/rpc/client/httpclient.go
index 4eb94f5a..bb4e6d3a 100644
--- a/rpc/client/httpclient.go
+++ b/rpc/client/httpclient.go
@@ -172,8 +172,17 @@ type WSEvents struct {
 	remote   string
 	endpoint string
 	ws       *rpcclient.WSClient
-	quit     chan bool
-	done     chan bool
+
+	// used for signaling the goroutine that feeds ws -> EventSwitch
+	quit chan bool
+	done chan bool
+
+	// used to maintain counts of actively listened events
+	// so we can properly subscribe/unsubscribe
+	// FIXME: thread-safety???
+	// FIXME: reuse code from go-events???
+	evtCount  map[string]int      // count how many time each event is subscribed
+	listeners map[string][]string // keep track of which events each listener is listening to
 }
 
 func newWSEvents(remote, endpoint string) *WSEvents {
@@ -183,6 +192,8 @@ func newWSEvents(remote, endpoint string) *WSEvents {
 		remote:      remote,
 		quit:        make(chan bool, 1),
 		done:        make(chan bool, 1),
+		evtCount:    map[string]int{},
+		listeners:   map[string][]string{},
 	}
 }
 
@@ -222,16 +233,57 @@ func (w *WSEvents) Stop() bool {
 
 /** TODO: more intelligent subscriptions! **/
 func (w *WSEvents) AddListenerForEvent(listenerID, event string, cb events.EventCallback) {
-	w.subscribe(event)
+	// no one listening -> subscribe
+	if w.evtCount[event] == 0 {
+		w.subscribe(event)
+	}
+	// if this listener was already listening to this event, return early
+	for _, s := range w.listeners[listenerID] {
+		if event == s {
+			return
+		}
+	}
+	// otherwise, add this event to this listener
+	w.evtCount[event] += 1
+	w.listeners[listenerID] = append(w.listeners[listenerID], event)
 	w.EventSwitch.AddListenerForEvent(listenerID, event, cb)
 }
 
 func (w *WSEvents) RemoveListenerForEvent(event string, listenerID string) {
-	w.unsubscribe(event)
+	// if this listener is listening already, splice it out
+	found := false
+	l := w.listeners[listenerID]
+	for i, s := range l {
+		if event == s {
+			found = true
+			w.listeners[listenerID] = append(l[:i], l[i+1:]...)
+			break
+		}
+	}
+	// if the listener wasn't already listening to the event, exit early
+	if !found {
+		return
+	}
+
+	// now we can update the subscriptions
+	w.evtCount[event] -= 1
+	if w.evtCount[event] == 0 {
+		w.unsubscribe(event)
+	}
 	w.EventSwitch.RemoveListenerForEvent(event, listenerID)
 }
 
 func (w *WSEvents) RemoveListener(listenerID string) {
+	// remove all counts for this listener
+	for _, s := range w.listeners[listenerID] {
+		w.evtCount[s] -= 1
+		if w.evtCount[s] == 0 {
+			w.unsubscribe(s)
+		}
+	}
+	w.listeners[listenerID] = nil
+
+	// then let the switch do it's magic
 	w.EventSwitch.RemoveListener(listenerID)
 }
 
@@ -274,18 +326,24 @@ func (w *WSEvents) parseEvent(data []byte) (err error) {
 	if !ok {
 		// ignore silently (eg. subscribe, unsubscribe and maybe other events)
 		return nil
-		// or report loudly???
-		// return errors.Errorf("unknown message: %#v", *result)
 	}
 	// looks good!  let's fire this baby!
 	w.EventSwitch.FireEvent(event.Name, event.Data)
 	return nil
 }
 
-func (w *WSEvents) subscribe(event string) error {
-	return errors.Wrap(w.ws.Subscribe(event), "Subscribe")
+// no way of exposing these failures, so we panic.
+// is this right?  or silently ignore???
+func (w *WSEvents) subscribe(event string) {
+	err := w.ws.Subscribe(event)
+	if err != nil {
+		panic(err)
+	}
 }
 
-func (w *WSEvents) unsubscribe(event string) error {
-	return errors.Wrap(w.ws.Unsubscribe(event), "Unsubscribe")
+func (w *WSEvents) unsubscribe(event string) {
+	err := w.ws.Unsubscribe(event)
+	if err != nil {
+		panic(err)
+	}
 }
diff --git a/rpc/client/interface.go b/rpc/client/interface.go
index 50f06511..9a5ba668 100644
--- a/rpc/client/interface.go
+++ b/rpc/client/interface.go
@@ -65,6 +65,10 @@ type Client interface {
 	SignClient
 	HistoryClient
 	StatusClient
+
+	// this Client is reactive, you can subscribe to any TMEventData
+	// type, given the proper string. see tendermint/types/events.go
+	types.EventSwitch
 }
 
 // NetworkClient is general info about the network state.  May not
diff --git a/rpc/client/mock/client.go b/rpc/client/mock/client.go
index 0dcba718..a3cecfca 100644
--- a/rpc/client/mock/client.go
+++ b/rpc/client/mock/client.go
@@ -32,6 +32,8 @@ type Client struct {
 	client.SignClient
 	client.HistoryClient
 	client.StatusClient
+	// create a mock with types.NewEventSwitch()
+	types.EventSwitch
 }
 
 func (c Client) _assertIsClient() client.Client {

From d56cb2ab4b222367f5cff3722763c9aabc4dfdc1 Mon Sep 17 00:00:00 2001
From: Ethan Frey <ethanfrey@users.noreply.github.com>
Date: Fri, 24 Feb 2017 23:40:05 +0100
Subject: [PATCH 097/132] fix typo

---
 rpc/client/helpers.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rpc/client/helpers.go b/rpc/client/helpers.go
index 89da434b..d73c4ece 100644
--- a/rpc/client/helpers.go
+++ b/rpc/client/helpers.go
@@ -63,7 +63,7 @@ func WaitForOneEvent(evsw types.EventSwitch,
 	evts, quit := make(chan events.EventData, 10), make(chan bool, 1)
 	// start timeout count-down
 	go func() {
-		time.Sleep(1 * time.Second)
+		time.Sleep(timeout)
 		quit <- true
 	}()
 

From 21501815dd8ad9e316d436b3ab7b2dac28807c41 Mon Sep 17 00:00:00 2001
From: Ethan Frey <ethanfrey@users.noreply.github.com>
Date: Mon, 27 Feb 2017 14:55:29 +0100
Subject: [PATCH 098/132] Fix EventSwitch usage in WaitForOneEvent

---
 rpc/client/helpers.go | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/rpc/client/helpers.go b/rpc/client/helpers.go
index d73c4ece..bd00c143 100644
--- a/rpc/client/helpers.go
+++ b/rpc/client/helpers.go
@@ -72,9 +72,8 @@ func WaitForOneEvent(evsw types.EventSwitch,
 		evts <- data
 	})
 	// make sure to unregister after the test is over
-	// TODO: why doesn't the other call work???
-	// defer evsw.RemoveListenerForEvent(listener, evtTyp)
-	defer evsw.RemoveListener(listener)
+	defer evsw.RemoveListenerForEvent(evtTyp, listener)
+	// defer evsw.RemoveListener(listener)  // this also works
 
 	select {
 	case <-quit:

From 64566543070d53d5c0fb9087b506582517de50e5 Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Thu, 2 Mar 2017 13:18:13 -0500
Subject: [PATCH 099/132] test: add extra kill after fail index triggered

---
 test/persist/test_failure_indices.sh | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/test/persist/test_failure_indices.sh b/test/persist/test_failure_indices.sh
index 81721e65..701d8c1f 100644
--- a/test/persist/test_failure_indices.sh
+++ b/test/persist/test_failure_indices.sh
@@ -58,10 +58,9 @@ for failIndex in $(seq $failsStart $failsEnd); do
 	bash ./test/utils/txs.sh "localhost:46657" &
 	start_procs 1 "$failIndex"
 
-	# tendermint should fail when it hits the fail index
-	kill -9 "$PID_DUMMY"
-	wait "$PID_DUMMY"
-	wait "$PID_TENDERMINT"
+	# tendermint should already have paniced when it hits the fail index
+	# but kill -9 for OS cleanup
+	kill_procs
 
 	start_procs 2
 

From 8ba79252c8b1569c6d0c471488c118b0ff120589 Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Thu, 2 Mar 2017 20:47:07 -0500
Subject: [PATCH 100/132] types: use mtx on PartSet.String()

---
 consensus/state.go      |  2 ++
 proxy/multi_app_conn.go |  8 ++++----
 rpc/client/rpc_test.go  | 22 +++++++++++-----------
 rpc/core/abci.go        |  1 +
 types/part_set.go       |  2 ++
 5 files changed, 20 insertions(+), 15 deletions(-)

diff --git a/consensus/state.go b/consensus/state.go
index 882d5c87..f3fb7d16 100644
--- a/consensus/state.go
+++ b/consensus/state.go
@@ -122,6 +122,8 @@ func (rs RoundStepType) String() string {
 //-----------------------------------------------------------------------------
 
 // Immutable when returned from ConsensusState.GetRoundState()
+// TODO: Actually, only the top pointer is copied,
+// so access to field pointers is still racey
 type RoundState struct {
 	Height             int // Height we are working on
 	Round              int
diff --git a/proxy/multi_app_conn.go b/proxy/multi_app_conn.go
index 2c93152b..353ade35 100644
--- a/proxy/multi_app_conn.go
+++ b/proxy/multi_app_conn.go
@@ -1,7 +1,7 @@
 package proxy
 
 import (
-	. "github.com/tendermint/go-common"
+	cmn "github.com/tendermint/go-common"
 	cfg "github.com/tendermint/go-config"
 )
 
@@ -9,7 +9,7 @@ import (
 
 // Tendermint's interface to the application consists of multiple connections
 type AppConns interface {
-	Service
+	cmn.Service
 
 	Mempool() AppConnMempool
 	Consensus() AppConnConsensus
@@ -32,7 +32,7 @@ type Handshaker interface {
 // which ensures the app and tendermint are synced.
 // TODO: on app restart, clients must reboot together
 type multiAppConn struct {
-	BaseService
+	cmn.BaseService
 
 	config cfg.Config
 
@@ -52,7 +52,7 @@ func NewMultiAppConn(config cfg.Config, clientCreator ClientCreator, handshaker
 		handshaker:    handshaker,
 		clientCreator: clientCreator,
 	}
-	multiAppConn.BaseService = *NewBaseService(log, "multiAppConn", multiAppConn)
+	multiAppConn.BaseService = *cmn.NewBaseService(log, "multiAppConn", multiAppConn)
 	return multiAppConn
 }
 
diff --git a/rpc/client/rpc_test.go b/rpc/client/rpc_test.go
index 11c7f726..68148593 100644
--- a/rpc/client/rpc_test.go
+++ b/rpc/client/rpc_test.go
@@ -66,17 +66,17 @@ func TestNetInfo(t *testing.T) {
 
 // FIXME: This seems to trigger a race condition with client.Local
 // go test -v -race . -run=DumpCons
-// func TestDumpConsensusState(t *testing.T) {
-// 	for i, c := range GetClients() {
-// 		// FIXME: fix server so it doesn't panic on invalid input
-// 		nc, ok := c.(client.NetworkClient)
-// 		require.True(t, ok, "%d", i)
-// 		cons, err := nc.DumpConsensusState()
-// 		require.Nil(t, err, "%d: %+v", i, err)
-// 		assert.NotEmpty(t, cons.RoundState)
-// 		assert.Empty(t, cons.PeerRoundStates)
-// 	}
-// }
+func TestDumpConsensusState(t *testing.T) {
+	for i, c := range GetClients() {
+		// FIXME: fix server so it doesn't panic on invalid input
+		nc, ok := c.(client.NetworkClient)
+		require.True(t, ok, "%d", i)
+		cons, err := nc.DumpConsensusState()
+		require.Nil(t, err, "%d: %+v", i, err)
+		assert.NotEmpty(t, cons.RoundState)
+		assert.Empty(t, cons.PeerRoundStates)
+	}
+}
 
 func TestGenesisAndValidators(t *testing.T) {
 	for i, c := range GetClients() {
diff --git a/rpc/core/abci.go b/rpc/core/abci.go
index cb748fe0..95772726 100644
--- a/rpc/core/abci.go
+++ b/rpc/core/abci.go
@@ -16,6 +16,7 @@ func ABCIQuery(path string, data []byte, prove bool) (*ctypes.ResultABCIQuery, e
 	if err != nil {
 		return nil, err
 	}
+	log.Info("ABCIQuery", "path", path, "data", data, "result", resQuery)
 	return &ctypes.ResultABCIQuery{resQuery}, nil
 }
 
diff --git a/types/part_set.go b/types/part_set.go
index c2152440..3a5ee26a 100644
--- a/types/part_set.go
+++ b/types/part_set.go
@@ -268,6 +268,8 @@ func (ps *PartSet) StringShort() string {
 	if ps == nil {
 		return "nil-PartSet"
 	} else {
+		ps.mtx.Lock()
+		defer ps.mtx.Unlock()
 		return fmt.Sprintf("(%v of %v)", ps.Count(), ps.Total())
 	}
 }

From 692691938ccabc48f5bc312bb37f820a66634ec3 Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Thu, 2 Mar 2017 23:57:28 -0500
Subject: [PATCH 101/132] remove comment

---
 rpc/client/rpc_test.go | 2 --
 1 file changed, 2 deletions(-)

diff --git a/rpc/client/rpc_test.go b/rpc/client/rpc_test.go
index 68148593..c5f32d97 100644
--- a/rpc/client/rpc_test.go
+++ b/rpc/client/rpc_test.go
@@ -64,8 +64,6 @@ func TestNetInfo(t *testing.T) {
 	}
 }
 
-// FIXME: This seems to trigger a race condition with client.Local
-// go test -v -race . -run=DumpCons
 func TestDumpConsensusState(t *testing.T) {
 	for i, c := range GetClients() {
 		// FIXME: fix server so it doesn't panic on invalid input

From 4747f14a2cb683fe78e40b70103dd4bf203be15b Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Thu, 2 Mar 2017 23:58:55 -0500
Subject: [PATCH 102/132] version bump to 0.9.0

---
 version/version.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/version/version.go b/version/version.go
index 395087d7..074876b9 100644
--- a/version/version.go
+++ b/version/version.go
@@ -1,7 +1,7 @@
 package version
 
 const Maj = "0"
-const Min = "8"
+const Min = "9"
 const Fix = "0"
 
-const Version = "0.8.0"
+const Version = "0.9.0"

From 7098e5c7eb622dc4914c21537f9f2c0a2de51256 Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Fri, 3 Mar 2017 01:13:23 -0500
Subject: [PATCH 103/132] glide update

---
 glide.lock | 61 ++++++++++++++++++++++++++++++++----------------------
 1 file changed, 36 insertions(+), 25 deletions(-)

diff --git a/glide.lock b/glide.lock
index 28082d2c..e47b8d2f 100644
--- a/glide.lock
+++ b/glide.lock
@@ -1,5 +1,5 @@
-hash: e283934fbbd221161d53a918db9e49db8c5be2b8929592b05ffe6b72c2ef0ab1
-updated: 2017-02-14T17:07:40.028389369-05:00
+hash: 9f2553f32b158ce61be7169d2b92e63a77a01cdd619629cb6c0064c196a7fa9c
+updated: 2017-03-03T01:22:09.83538904-05:00
 imports:
 - name: github.com/btcsuite/btcd
   version: d06c0bb181529331be8f8d9350288c420d9e60e4
@@ -7,8 +7,12 @@ imports:
   - btcec
 - name: github.com/BurntSushi/toml
   version: 99064174e013895bbd9b025c31100bd1d9b590ca
+- name: github.com/davecgh/go-spew
+  version: 6d212800a42e8ab5c146b8ace3490ee17e5225f9
+  subpackages:
+  - spew
 - name: github.com/ebuchman/fail-test
-  version: c1eddaa09da2b4017351245b0d43234955276798
+  version: 13f91f14c826314205cdbed1ec8ac8bf08e03381
 - name: github.com/go-stack/stack
   version: 100eb0c0a9c5b306ca2fb4f165df21d80ada4b82
 - name: github.com/gogo/protobuf
@@ -20,19 +24,28 @@ imports:
   subpackages:
   - proto
 - name: github.com/golang/snappy
-  version: 7db9049039a047d955fe8c19b83c8ff5abd765c7
+  version: d9eb7a3d35ec988b8585d4a0068e462c27d28380
 - name: github.com/gorilla/websocket
   version: 804cb600d06b10672f2fbc0a336a7bee507a428e
 - name: github.com/jmhodges/levigo
   version: c42d9e0ca023e2198120196f842701bb4c55d7b9
 - name: github.com/mattn/go-colorable
-  version: 5411d3eea5978e6cdc258b30de592b60df6aba96
+  version: d228849504861217f796da67fae4f6e347643f15
 - name: github.com/mattn/go-isatty
-  version: 281032e84ae07510239465db46bf442aa44b953a
+  version: 30a891c33c7cde7b02a981314b4228ec99380cca
 - name: github.com/pkg/errors
-  version: 248dadf4e9068a0b3e79f02ed0a610d935de5302
+  version: 645ef00459ed84a119197bfb8d8205042c6df63d
+- name: github.com/pmezard/go-difflib
+  version: d8ed2627bdf02c080bf22230dbb337003b7aba2d
+  subpackages:
+  - difflib
 - name: github.com/spf13/pflag
   version: 9ff6c6923cfffbcd502984b8e0c80539a94968b7
+- name: github.com/stretchr/testify
+  version: 69483b4bd14f5845b5a1e55bca19e954e827f1d0
+  subpackages:
+  - assert
+  - require
 - name: github.com/syndtr/goleveldb
   version: 23851d93a2292dcc56e71a18ec9e0624d84a0f65
   subpackages:
@@ -49,7 +62,7 @@ imports:
   - leveldb/table
   - leveldb/util
 - name: github.com/tendermint/abci
-  version: 31bdda27ad80e47d372e4b9e4acfd4c0ef81d3e4
+  version: c1f5a963821ab5d7c32f7ab109ed75fa929937e6
   subpackages:
   - client
   - example/counter
@@ -62,21 +75,23 @@ imports:
   - edwards25519
   - extra25519
 - name: github.com/tendermint/go-autofile
-  version: 0416e0aa9c68205aa44844096f9f151ada9d0405
+  version: 48b17de82914e1ec2f134ce823ba426337d2c518
 - name: github.com/tendermint/go-clist
   version: 3baa390bbaf7634251c42ad69a8682e7e3990552
 - name: github.com/tendermint/go-common
-  version: 339e135776142939d82bc8e699db0bf391fd938d
+  version: 6141dc6eedab067f7e25707a237218ef2c0dc273
   subpackages:
   - test
 - name: github.com/tendermint/go-config
   version: e64b424499acd0eb9856b88e10c0dff41628c0d6
 - name: github.com/tendermint/go-crypto
-  version: 4b11d62bdb324027ea01554e5767b71174680ba0
+  version: 3f47cfac5fcd9e0f1727c7db980b3559913b3e3a
+- name: github.com/tendermint/go-data
+  version: 32271140e8fd5abdbb22e268d7a02421fa382f0b
 - name: github.com/tendermint/go-db
-  version: 2645626c33d8702739e52a61a55d705c2dfe4530
+  version: 72f6dacd22a686cdf7fcd60286503e3aceda77ba
 - name: github.com/tendermint/go-events
-  version: 2337086736a6adeb2de6f66739b66ecd77535997
+  version: f8ffbfb2be3483e9e7927495590a727f51c0c11f
 - name: github.com/tendermint/go-flowrate
   version: a20c98e61957faa93b4014fbd902f20ab9317a6a
   subpackages:
@@ -84,25 +99,25 @@ imports:
 - name: github.com/tendermint/go-logger
   version: cefb3a45c0bf3c493a04e9bcd9b1540528be59f2
 - name: github.com/tendermint/go-merkle
-  version: 9f20e80cb188d07860caa70196dd7700659ec4a4
+  version: 714d4d04557fd068a7c2a1748241ce8428015a96
 - name: github.com/tendermint/go-p2p
-  version: 67c9086b7458eb45b1970483decd01cd744c477a
+  version: 3d98f675f30dc4796546b8b890f895926152fa8d
   subpackages:
   - upnp
 - name: github.com/tendermint/go-rpc
-  version: 6177eb8398ebd4613fbecb71fd96d7c7d97303ec
+  version: fcea0cda21f64889be00a0f4b6d13266b1a76ee7
   subpackages:
   - client
   - server
   - types
 - name: github.com/tendermint/go-wire
-  version: 3216ec9d47bbdf8d4fc27d22169ea86a6688bc15
+  version: f530b7af7a8b06e612c2063bff6ace49060a085e
 - name: github.com/tendermint/log15
   version: ae0f3d6450da9eac7074b439c8e1c3cabf0d5ce6
   subpackages:
   - term
 - name: golang.org/x/crypto
-  version: 453249f01cfeb54c3d549ddb75ff152ca243f9d8
+  version: 7c6cc321c680f03b9ef0764448e780704f486b51
   subpackages:
   - curve25519
   - nacl/box
@@ -123,7 +138,7 @@ imports:
   - lex/httplex
   - trace
 - name: golang.org/x/sys
-  version: e24f485414aeafb646f6fca458b0bf869c0880a1
+  version: d75a52659825e75fff6158388dddc6a5b04f9ba5
   subpackages:
   - unix
 - name: google.golang.org/grpc
@@ -144,9 +159,5 @@ testImports:
   version: acd8e9c42e1d819c51e9e1cd3870ea4d94b167f5
   subpackages:
   - app
-- name: github.com/stretchr/testify
-  version: 69483b4bd14f5845b5a1e55bca19e954e827f1d0
-  subpackages:
-  - assert
-  - require
-
+  - client
+  - testutil

From b13924701e10955a19059e39af36507ed90b838c Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Fri, 3 Mar 2017 14:00:51 -0500
Subject: [PATCH 104/132] test/persist: wait for ports to be freed

---
 test/docker/Dockerfile               |  2 +-
 test/persist/test_failure_indices.sh | 29 ++++++++++++++++++++--
 test/run_test.sh                     | 20 ++++++++--------
 test/test.sh                         | 36 ++++++++++++++--------------
 4 files changed, 56 insertions(+), 31 deletions(-)

diff --git a/test/docker/Dockerfile b/test/docker/Dockerfile
index e69f1ca6..bec59bf9 100644
--- a/test/docker/Dockerfile
+++ b/test/docker/Dockerfile
@@ -4,7 +4,7 @@ FROM golang:1.7.4
 # Grab deps (jq, hexdump, xxd, killall)
 RUN apt-get update && \
   apt-get install -y --no-install-recommends \
-  jq bsdmainutils vim-common psmisc 
+  jq bsdmainutils vim-common psmisc netcat
 
 # Setup tendermint repo with vendored dependencies
 # but without code - docker caching prevents reinstall on code change!
diff --git a/test/persist/test_failure_indices.sh b/test/persist/test_failure_indices.sh
index 701d8c1f..ba09464f 100644
--- a/test/persist/test_failure_indices.sh
+++ b/test/persist/test_failure_indices.sh
@@ -43,6 +43,31 @@ function kill_procs(){
 	kill -9 "$PID_DUMMY" "$PID_TENDERMINT"
 	wait "$PID_DUMMY"
 	wait "$PID_TENDERMINT"
+
+	# wait for the ports to be released 
+	wait_for_port 46656
+	wait_for_port 46657
+}
+
+# wait for port to be available
+function wait_for_port() {
+	port=$1
+	# this will succeed while port is bound
+	nc -z 127.0.0.1 $port
+	ERR=$?
+	i=0
+	while [ "$ERR" == 0 ]; do
+		echo "... port $port is still bound. waiting ..."
+		sleep 1	
+		nc -z 127.0.0.1 $port 
+		ERR=$?
+		i=$((i + 1))
+		if [[ $i == 10 ]]; then
+			echo "Timed out waiting for port to be released"
+			exit 1
+		fi
+	done
+	echo "... port $port is free!"
 }
 
 
@@ -58,8 +83,8 @@ for failIndex in $(seq $failsStart $failsEnd); do
 	bash ./test/utils/txs.sh "localhost:46657" &
 	start_procs 1 "$failIndex"
 
-	# tendermint should already have paniced when it hits the fail index
-	# but kill -9 for OS cleanup
+	# tendermint should already have exited when it hits the fail index
+	# but kill -9 for good measure
 	kill_procs
 
 	start_procs 2
diff --git a/test/run_test.sh b/test/run_test.sh
index fcc82d98..4bf69ed7 100644
--- a/test/run_test.sh
+++ b/test/run_test.sh
@@ -6,17 +6,17 @@ pwd
 BRANCH=$(git rev-parse --abbrev-ref HEAD)
 echo "Current branch: $BRANCH"
 
-bash test/test_cover.sh
-
-# run the app tests
-bash test/app/test.sh
+#bash test/test_cover.sh
+#
+## run the app tests
+#bash test/app/test.sh
 
 # run the persistence test
 bash test/persist/test.sh
 
-if [[ "$BRANCH" == "master" || $(echo "$BRANCH" | grep "release-") != "" ]]; then
-	echo ""
-	echo "* branch $BRANCH; testing libs"
-	# checkout every github.com/tendermint dir and run its tests
-	bash test/test_libs.sh
-fi
+#if [[ "$BRANCH" == "master" || $(echo "$BRANCH" | grep "release-") != "" ]]; then
+#	echo ""
+#	echo "* branch $BRANCH; testing libs"
+#	# checkout every github.com/tendermint dir and run its tests
+#	bash test/test_libs.sh
+#fi
diff --git a/test/test.sh b/test/test.sh
index 1c1faca2..d90fb68d 100644
--- a/test/test.sh
+++ b/test/test.sh
@@ -35,21 +35,21 @@ else
 	docker run --name run_test -t tester bash test/run_test.sh
 fi
 
-# copy the coverage results out of docker container
-docker cp run_test:/go/src/github.com/tendermint/tendermint/coverage.txt .
-
-# test basic network connectivity
-# by starting a local testnet and checking peers connect and make blocks
-echo
-echo "* [$(date +"%T")] running p2p tests on a local docker network"
-bash "$DIR/p2p/test.sh" tester
-
-# only run the cloud benchmark for releases
-BRANCH=$(git rev-parse --abbrev-ref HEAD)
-if [[ $(echo "$BRANCH" | grep "release-") != "" ]]; then
-	echo
-	echo "TODO: run network tests"
-	#echo "* branch $BRANCH; running mintnet/netmon throughput benchmark"
-	# TODO: replace mintnet
-	#bash "$DIR/net/test.sh"
-fi
+## copy the coverage results out of docker container
+#docker cp run_test:/go/src/github.com/tendermint/tendermint/coverage.txt .
+#
+## test basic network connectivity
+## by starting a local testnet and checking peers connect and make blocks
+#echo
+#echo "* [$(date +"%T")] running p2p tests on a local docker network"
+#bash "$DIR/p2p/test.sh" tester
+#
+## only run the cloud benchmark for releases
+#BRANCH=$(git rev-parse --abbrev-ref HEAD)
+#if [[ $(echo "$BRANCH" | grep "release-") != "" ]]; then
+#	echo
+#	echo "TODO: run network tests"
+#	#echo "* branch $BRANCH; running mintnet/netmon throughput benchmark"
+#	# TODO: replace mintnet
+#	#bash "$DIR/net/test.sh"
+#fi

From 8192bb0aaf83e18df1f1b9d80adff2add75fb071 Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Fri, 3 Mar 2017 19:28:17 -0500
Subject: [PATCH 105/132] stop rpc listeners in node.OnStop()

---
 node/node.go                         | 11 ++++++++++-
 test/persist/test_failure_indices.sh |  4 ++--
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/node/node.go b/node/node.go
index 6ee02771..b739396d 100644
--- a/node/node.go
+++ b/node/node.go
@@ -44,6 +44,7 @@ type Node struct {
 	genesisDoc       *types.GenesisDoc
 	privKey          crypto.PrivKeyEd25519
 	proxyApp         proxy.AppConns
+	rpcListeners     []net.Listener
 }
 
 func NewNodeDefault(config cfg.Config) *Node {
@@ -208,10 +209,11 @@ func (n *Node) OnStart() error {
 
 	// Run the RPC server
 	if n.config.GetString("rpc_laddr") != "" {
-		_, err := n.startRPC()
+		listeners, err := n.startRPC()
 		if err != nil {
 			return err
 		}
+		n.rpcListeners = listeners
 	}
 
 	return nil
@@ -230,6 +232,13 @@ func (n *Node) OnStop() {
 	log.Notice("Stopping Node")
 	// TODO: gracefully disconnect from peers.
 	n.sw.Stop()
+
+	for _, l := range n.rpcListeners {
+		log.Info("Closing rpc listener", "listener", l)
+		if err := l.Close(); err != nil {
+			log.Error("Error closing listener", "listener", l, "error", err)
+		}
+	}
 }
 
 // Add the event switch to reactors, mempool, etc.
diff --git a/test/persist/test_failure_indices.sh b/test/persist/test_failure_indices.sh
index ba09464f..04118c72 100644
--- a/test/persist/test_failure_indices.sh
+++ b/test/persist/test_failure_indices.sh
@@ -45,8 +45,8 @@ function kill_procs(){
 	wait "$PID_TENDERMINT"
 
 	# wait for the ports to be released 
-	wait_for_port 46656
-	wait_for_port 46657
+	#wait_for_port 46656
+	#wait_for_port 46657
 }
 
 # wait for port to be available

From 8352ec4e5ddcf24bf731c3139bc7fb68add0bb9d Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Fri, 3 Mar 2017 19:58:17 -0500
Subject: [PATCH 106/132] circle sigh

---
 test/persist/test_failure_indices.sh |  7 ++++--
 test/run_test.sh                     | 20 ++++++++--------
 test/test.sh                         | 36 ++++++++++++++--------------
 3 files changed, 33 insertions(+), 30 deletions(-)

diff --git a/test/persist/test_failure_indices.sh b/test/persist/test_failure_indices.sh
index 04118c72..0d0b3b2b 100644
--- a/test/persist/test_failure_indices.sh
+++ b/test/persist/test_failure_indices.sh
@@ -45,8 +45,11 @@ function kill_procs(){
 	wait "$PID_TENDERMINT"
 
 	# wait for the ports to be released 
-	#wait_for_port 46656
-	#wait_for_port 46657
+	wait_for_port 46656
+	wait_for_port 46657
+	
+	# XXX: sometimes the port is still bound :(
+	sleep 2
 }
 
 # wait for port to be available
diff --git a/test/run_test.sh b/test/run_test.sh
index 4bf69ed7..fcc82d98 100644
--- a/test/run_test.sh
+++ b/test/run_test.sh
@@ -6,17 +6,17 @@ pwd
 BRANCH=$(git rev-parse --abbrev-ref HEAD)
 echo "Current branch: $BRANCH"
 
-#bash test/test_cover.sh
-#
-## run the app tests
-#bash test/app/test.sh
+bash test/test_cover.sh
+
+# run the app tests
+bash test/app/test.sh
 
 # run the persistence test
 bash test/persist/test.sh
 
-#if [[ "$BRANCH" == "master" || $(echo "$BRANCH" | grep "release-") != "" ]]; then
-#	echo ""
-#	echo "* branch $BRANCH; testing libs"
-#	# checkout every github.com/tendermint dir and run its tests
-#	bash test/test_libs.sh
-#fi
+if [[ "$BRANCH" == "master" || $(echo "$BRANCH" | grep "release-") != "" ]]; then
+	echo ""
+	echo "* branch $BRANCH; testing libs"
+	# checkout every github.com/tendermint dir and run its tests
+	bash test/test_libs.sh
+fi
diff --git a/test/test.sh b/test/test.sh
index d90fb68d..1c1faca2 100644
--- a/test/test.sh
+++ b/test/test.sh
@@ -35,21 +35,21 @@ else
 	docker run --name run_test -t tester bash test/run_test.sh
 fi
 
-## copy the coverage results out of docker container
-#docker cp run_test:/go/src/github.com/tendermint/tendermint/coverage.txt .
-#
-## test basic network connectivity
-## by starting a local testnet and checking peers connect and make blocks
-#echo
-#echo "* [$(date +"%T")] running p2p tests on a local docker network"
-#bash "$DIR/p2p/test.sh" tester
-#
-## only run the cloud benchmark for releases
-#BRANCH=$(git rev-parse --abbrev-ref HEAD)
-#if [[ $(echo "$BRANCH" | grep "release-") != "" ]]; then
-#	echo
-#	echo "TODO: run network tests"
-#	#echo "* branch $BRANCH; running mintnet/netmon throughput benchmark"
-#	# TODO: replace mintnet
-#	#bash "$DIR/net/test.sh"
-#fi
+# copy the coverage results out of docker container
+docker cp run_test:/go/src/github.com/tendermint/tendermint/coverage.txt .
+
+# test basic network connectivity
+# by starting a local testnet and checking peers connect and make blocks
+echo
+echo "* [$(date +"%T")] running p2p tests on a local docker network"
+bash "$DIR/p2p/test.sh" tester
+
+# only run the cloud benchmark for releases
+BRANCH=$(git rev-parse --abbrev-ref HEAD)
+if [[ $(echo "$BRANCH" | grep "release-") != "" ]]; then
+	echo
+	echo "TODO: run network tests"
+	#echo "* branch $BRANCH; running mintnet/netmon throughput benchmark"
+	# TODO: replace mintnet
+	#bash "$DIR/net/test.sh"
+fi

From f5c4fdc82abd1cb6d15b09cd0e93abc21937264e Mon Sep 17 00:00:00 2001
From: rigelrozanski <rigel.rozanski@gmail.com>
Date: Fri, 3 Mar 2017 01:27:41 -0500
Subject: [PATCH 107/132] seeds fix

---
 glide.lock       | 14 +++++++-------
 glide.yaml       |  2 +-
 rpc/core/net.go  |  4 ++--
 rpc/core/pipe.go |  2 +-
 4 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/glide.lock b/glide.lock
index e47b8d2f..e282aac7 100644
--- a/glide.lock
+++ b/glide.lock
@@ -2,7 +2,7 @@ hash: 9f2553f32b158ce61be7169d2b92e63a77a01cdd619629cb6c0064c196a7fa9c
 updated: 2017-03-03T01:22:09.83538904-05:00
 imports:
 - name: github.com/btcsuite/btcd
-  version: d06c0bb181529331be8f8d9350288c420d9e60e4
+  version: 583684b21bfbde9b5fc4403916fd7c807feb0289
   subpackages:
   - btcec
 - name: github.com/BurntSushi/toml
@@ -16,17 +16,17 @@ imports:
 - name: github.com/go-stack/stack
   version: 100eb0c0a9c5b306ca2fb4f165df21d80ada4b82
 - name: github.com/gogo/protobuf
-  version: 2221ff550f109ae54cb617c0dc6ac62658c418d7
+  version: d66ead2957a1d163be783c18c25bc388a201cdd0
   subpackages:
   - proto
 - name: github.com/golang/protobuf
-  version: 8ee79997227bf9b34611aee7946ae64735e6fd93
+  version: 69b215d01a5606c843240eab4937eab3acee6530
   subpackages:
   - proto
 - name: github.com/golang/snappy
   version: d9eb7a3d35ec988b8585d4a0068e462c27d28380
 - name: github.com/gorilla/websocket
-  version: 804cb600d06b10672f2fbc0a336a7bee507a428e
+  version: b258b4fadb573ac412f187b9f31974ea99d32f50
 - name: github.com/jmhodges/levigo
   version: c42d9e0ca023e2198120196f842701bb4c55d7b9
 - name: github.com/mattn/go-colorable
@@ -47,7 +47,7 @@ imports:
   - assert
   - require
 - name: github.com/syndtr/goleveldb
-  version: 23851d93a2292dcc56e71a18ec9e0624d84a0f65
+  version: 3c5717caf1475fd25964109a0fc640bd150fce43
   subpackages:
   - leveldb
   - leveldb/cache
@@ -128,7 +128,7 @@ imports:
   - ripemd160
   - salsa20/salsa
 - name: golang.org/x/net
-  version: 61557ac0112b576429a0df080e1c2cef5dfbb642
+  version: 906cda9512f77671ab44f8c8563b13a8e707b230
   subpackages:
   - context
   - http2
@@ -142,7 +142,7 @@ imports:
   subpackages:
   - unix
 - name: google.golang.org/grpc
-  version: cbcceb2942a489498cf22b2f918536e819d33f0a
+  version: 1dab93372523195731c738b0f0cb4e452228e959
   subpackages:
   - codes
   - credentials
diff --git a/glide.yaml b/glide.yaml
index 9a7e6744..3cf2c36f 100644
--- a/glide.yaml
+++ b/glide.yaml
@@ -19,7 +19,7 @@ import:
 - package: github.com/tendermint/go-merkle
   version: develop
 - package: github.com/tendermint/go-p2p
-  version: develop
+  version: seedsfix
 - package: github.com/tendermint/go-rpc
   version: develop
 - package: github.com/tendermint/go-wire
diff --git a/rpc/core/net.go b/rpc/core/net.go
index 9e9f027e..3e9526d2 100644
--- a/rpc/core/net.go
+++ b/rpc/core/net.go
@@ -32,8 +32,8 @@ func NetInfo() (*ctypes.ResultNetInfo, error) {
 // Dial given list of seeds
 func UnsafeDialSeeds(seeds []string) (*ctypes.ResultDialSeeds, error) {
 	// starts go routines to dial each seed after random delays
-	p2pSwitch.DialSeeds(seeds)
-	return &ctypes.ResultDialSeeds{}, nil
+	err := p2pSwitch.DialSeeds(seeds)
+	return &ctypes.ResultDialSeeds{}, err
 }
 
 //-----------------------------------------------------------------------------
diff --git a/rpc/core/pipe.go b/rpc/core/pipe.go
index 40ef7081..fae2630c 100644
--- a/rpc/core/pipe.go
+++ b/rpc/core/pipe.go
@@ -24,7 +24,7 @@ type P2P interface {
 	NumPeers() (outbound, inbound, dialig int)
 	NodeInfo() *p2p.NodeInfo
 	IsListening() bool
-	DialSeeds([]string)
+	DialSeeds([]string) error
 }
 
 //----------------------------------------------

From c7386b139bd9ca058f91f435879a2cba1fc51f6f Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Sat, 4 Mar 2017 22:05:14 -0500
Subject: [PATCH 108/132] glide update

---
 glide.lock | 40 ++++++++++++++++++++--------------------
 glide.yaml | 10 +++++-----
 2 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/glide.lock b/glide.lock
index e282aac7..fba1f097 100644
--- a/glide.lock
+++ b/glide.lock
@@ -1,8 +1,8 @@
-hash: 9f2553f32b158ce61be7169d2b92e63a77a01cdd619629cb6c0064c196a7fa9c
-updated: 2017-03-03T01:22:09.83538904-05:00
+hash: 238f5a4399605b0d6eb35930b04b70395c70562fbc3861b95bb3a92c0d4cf8dd
+updated: 2017-03-04T22:04:42.038691605-05:00
 imports:
 - name: github.com/btcsuite/btcd
-  version: 583684b21bfbde9b5fc4403916fd7c807feb0289
+  version: d06c0bb181529331be8f8d9350288c420d9e60e4
   subpackages:
   - btcec
 - name: github.com/BurntSushi/toml
@@ -16,17 +16,17 @@ imports:
 - name: github.com/go-stack/stack
   version: 100eb0c0a9c5b306ca2fb4f165df21d80ada4b82
 - name: github.com/gogo/protobuf
-  version: d66ead2957a1d163be783c18c25bc388a201cdd0
+  version: 2221ff550f109ae54cb617c0dc6ac62658c418d7
   subpackages:
   - proto
 - name: github.com/golang/protobuf
-  version: 69b215d01a5606c843240eab4937eab3acee6530
+  version: 8ee79997227bf9b34611aee7946ae64735e6fd93
   subpackages:
   - proto
 - name: github.com/golang/snappy
   version: d9eb7a3d35ec988b8585d4a0068e462c27d28380
 - name: github.com/gorilla/websocket
-  version: b258b4fadb573ac412f187b9f31974ea99d32f50
+  version: 804cb600d06b10672f2fbc0a336a7bee507a428e
 - name: github.com/jmhodges/levigo
   version: c42d9e0ca023e2198120196f842701bb4c55d7b9
 - name: github.com/mattn/go-colorable
@@ -47,7 +47,7 @@ imports:
   - assert
   - require
 - name: github.com/syndtr/goleveldb
-  version: 3c5717caf1475fd25964109a0fc640bd150fce43
+  version: 23851d93a2292dcc56e71a18ec9e0624d84a0f65
   subpackages:
   - leveldb
   - leveldb/cache
@@ -62,7 +62,7 @@ imports:
   - leveldb/table
   - leveldb/util
 - name: github.com/tendermint/abci
-  version: c1f5a963821ab5d7c32f7ab109ed75fa929937e6
+  version: 0b1ee4b67318047d20a90bc6a8b0119a15c7c771
   subpackages:
   - client
   - example/counter
@@ -83,13 +83,13 @@ imports:
   subpackages:
   - test
 - name: github.com/tendermint/go-config
-  version: e64b424499acd0eb9856b88e10c0dff41628c0d6
+  version: 28765c18222cd3171d5290f1cc69a9cf710ac5ae
 - name: github.com/tendermint/go-crypto
   version: 3f47cfac5fcd9e0f1727c7db980b3559913b3e3a
 - name: github.com/tendermint/go-data
   version: 32271140e8fd5abdbb22e268d7a02421fa382f0b
 - name: github.com/tendermint/go-db
-  version: 72f6dacd22a686cdf7fcd60286503e3aceda77ba
+  version: 286cbbd99d9ff76681b7c18cc506d476ccf68725
 - name: github.com/tendermint/go-events
   version: f8ffbfb2be3483e9e7927495590a727f51c0c11f
 - name: github.com/tendermint/go-flowrate
@@ -101,7 +101,7 @@ imports:
 - name: github.com/tendermint/go-merkle
   version: 714d4d04557fd068a7c2a1748241ce8428015a96
 - name: github.com/tendermint/go-p2p
-  version: 3d98f675f30dc4796546b8b890f895926152fa8d
+  version: 17e6ae813fc8399ab96d32dc6b392eb1be8dbe8d
   subpackages:
   - upnp
 - name: github.com/tendermint/go-rpc
@@ -116,6 +116,12 @@ imports:
   version: ae0f3d6450da9eac7074b439c8e1c3cabf0d5ce6
   subpackages:
   - term
+- name: github.com/tendermint/merkleeyes
+  version: acd8e9c42e1d819c51e9e1cd3870ea4d94b167f5
+  subpackages:
+  - app
+  - client
+  - testutil
 - name: golang.org/x/crypto
   version: 7c6cc321c680f03b9ef0764448e780704f486b51
   subpackages:
@@ -128,7 +134,7 @@ imports:
   - ripemd160
   - salsa20/salsa
 - name: golang.org/x/net
-  version: 906cda9512f77671ab44f8c8563b13a8e707b230
+  version: 61557ac0112b576429a0df080e1c2cef5dfbb642
   subpackages:
   - context
   - http2
@@ -142,7 +148,7 @@ imports:
   subpackages:
   - unix
 - name: google.golang.org/grpc
-  version: 1dab93372523195731c738b0f0cb4e452228e959
+  version: cbcceb2942a489498cf22b2f918536e819d33f0a
   subpackages:
   - codes
   - credentials
@@ -154,10 +160,4 @@ imports:
   - stats
   - tap
   - transport
-testImports:
-- name: github.com/tendermint/merkleeyes
-  version: acd8e9c42e1d819c51e9e1cd3870ea4d94b167f5
-  subpackages:
-  - app
-  - client
-  - testutil
+testImports: []
diff --git a/glide.yaml b/glide.yaml
index 3cf2c36f..fd63b995 100644
--- a/glide.yaml
+++ b/glide.yaml
@@ -19,7 +19,7 @@ import:
 - package: github.com/tendermint/go-merkle
   version: develop
 - package: github.com/tendermint/go-p2p
-  version: seedsfix
+  version: develop
 - package: github.com/tendermint/go-rpc
   version: develop
 - package: github.com/tendermint/go-wire
@@ -29,6 +29,10 @@ import:
 - package: github.com/tendermint/go-flowrate
 - package: github.com/tendermint/log15
 - package: github.com/tendermint/ed25519
+- package: github.com/tendermint/merkleeyes
+  version: develop
+  subpackages:
+  - app
 - package: github.com/gogo/protobuf
   subpackages:
   - proto
@@ -45,7 +49,3 @@ testImport:
   subpackages:
   - assert
   - require
-- package: github.com/tendermint/merkleeyes
-  version: develop
-  subpackages:
-  - app

From 027cb8dc6bcfc5b331d6b1ffc256890124bf5fe0 Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Sat, 4 Mar 2017 22:11:44 -0500
Subject: [PATCH 109/132] glide: use versions where applicable

---
 glide.lock | 8 ++++----
 glide.yaml | 2 ++
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/glide.lock b/glide.lock
index fba1f097..c024b391 100644
--- a/glide.lock
+++ b/glide.lock
@@ -1,5 +1,5 @@
-hash: 238f5a4399605b0d6eb35930b04b70395c70562fbc3861b95bb3a92c0d4cf8dd
-updated: 2017-03-04T22:04:42.038691605-05:00
+hash: 41f8fec708e98b7f8c4804be46008493199fa45e89b2d5dc237fd65fe431c62f
+updated: 2017-03-04T22:10:20.337283524-05:00
 imports:
 - name: github.com/btcsuite/btcd
   version: d06c0bb181529331be8f8d9350288c420d9e60e4
@@ -16,7 +16,7 @@ imports:
 - name: github.com/go-stack/stack
   version: 100eb0c0a9c5b306ca2fb4f165df21d80ada4b82
 - name: github.com/gogo/protobuf
-  version: 2221ff550f109ae54cb617c0dc6ac62658c418d7
+  version: 909568be09de550ed094403c2bf8a261b5bb730a
   subpackages:
   - proto
 - name: github.com/golang/protobuf
@@ -26,7 +26,7 @@ imports:
 - name: github.com/golang/snappy
   version: d9eb7a3d35ec988b8585d4a0068e462c27d28380
 - name: github.com/gorilla/websocket
-  version: 804cb600d06b10672f2fbc0a336a7bee507a428e
+  version: 3ab3a8b8831546bd18fd182c20687ca853b2bb13
 - name: github.com/jmhodges/levigo
   version: c42d9e0ca023e2198120196f842701bb4c55d7b9
 - name: github.com/mattn/go-colorable
diff --git a/glide.yaml b/glide.yaml
index fd63b995..e4029380 100644
--- a/glide.yaml
+++ b/glide.yaml
@@ -34,9 +34,11 @@ import:
   subpackages:
   - app
 - package: github.com/gogo/protobuf
+  version: ^0.3
   subpackages:
   - proto
 - package: github.com/gorilla/websocket
+  version: ^1.1.0
 - package: github.com/spf13/pflag
 - package: github.com/pkg/errors
   version: ^0.8.0

From eef9124d1b16b2d686e4d6c3b0cbeec445d817c2 Mon Sep 17 00:00:00 2001
From: Anton Kalyaev <anton.kalyaev@gmail.com>
Date: Fri, 13 Jan 2017 13:16:50 +0400
Subject: [PATCH 110/132] fix typo

---
 blockchain/pool.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/blockchain/pool.go b/blockchain/pool.go
index 0886035b..ef673b34 100644
--- a/blockchain/pool.go
+++ b/blockchain/pool.go
@@ -26,7 +26,7 @@ var peerTimeoutSeconds = time.Duration(15) // not const so we can override with
 	in sequence from peers that reported higher heights than ours.
 	Every so often we ask peers what height they're on so we can keep going.
 
-	Requests are continuously made for blocks of heigher heights until
+	Requests are continuously made for blocks of higher heights until
 	the limits. If most of the requests have no available peers, and we
 	are not at peer limits, we can probably switch to consensus reactor
 */

From 163fe1731b9ad3cbcd532a23d103fcbf8e31cea5 Mon Sep 17 00:00:00 2001
From: Anton Kalyaev <anton.kalyaev@gmail.com>
Date: Fri, 13 Jan 2017 19:27:31 +0400
Subject: [PATCH 111/132] test p2p pex reactor (Refs #335)

---
 test/p2p/fast_sync/test_peer.sh |  2 +-
 test/p2p/local_testnet_start.sh |  2 +-
 test/p2p/peer.sh                |  9 +++------
 test/p2p/pex/check_peer.sh      | 17 +++++++++++++++++
 test/p2p/pex/test.sh            | 33 +++++++++++++++++++++++++++++++++
 test/p2p/test.sh                |  5 ++++-
 6 files changed, 59 insertions(+), 9 deletions(-)
 create mode 100644 test/p2p/pex/check_peer.sh
 create mode 100644 test/p2p/pex/test.sh

diff --git a/test/p2p/fast_sync/test_peer.sh b/test/p2p/fast_sync/test_peer.sh
index a065ea5c..cab6fc6f 100644
--- a/test/p2p/fast_sync/test_peer.sh
+++ b/test/p2p/fast_sync/test_peer.sh
@@ -27,7 +27,7 @@ SEEDS="$(test/p2p/ip.sh 1):46656"
 for j in `seq 2 $N`; do
 	SEEDS="$SEEDS,$(test/p2p/ip.sh $j):46656"
 done
-bash test/p2p/peer.sh $DOCKER_IMAGE $NETWORK_NAME $ID $PROXY_APP $SEEDS
+bash test/p2p/peer.sh $DOCKER_IMAGE $NETWORK_NAME $ID $PROXY_APP "--seeds $SEEDS"
 
 # wait for peer to sync and check the app hash
 bash test/p2p/client.sh $DOCKER_IMAGE $NETWORK_NAME fs_$ID "test/p2p/fast_sync/check_peer.sh $ID"
diff --git a/test/p2p/local_testnet_start.sh b/test/p2p/local_testnet_start.sh
index 4dd2ab05..3b842480 100644
--- a/test/p2p/local_testnet_start.sh
+++ b/test/p2p/local_testnet_start.sh
@@ -18,5 +18,5 @@ done
 echo "Seeds: $seeds"
 
 for i in `seq 1 $N`; do
-	bash test/p2p/peer.sh $DOCKER_IMAGE $NETWORK_NAME $i $APP_PROXY $seeds
+	bash test/p2p/peer.sh $DOCKER_IMAGE $NETWORK_NAME $i $APP_PROXY "--seeds $seeds --pex"
 done
diff --git a/test/p2p/peer.sh b/test/p2p/peer.sh
index 611f5ac2..adeb01a1 100644
--- a/test/p2p/peer.sh
+++ b/test/p2p/peer.sh
@@ -7,11 +7,8 @@ ID=$3
 APP_PROXY=$4
 
 set +u
-SEEDS=$5
+NODE_FLAGS=$5
 set -u
-if [[ "$SEEDS" != "" ]]; then
-	SEEDS=" --seeds $SEEDS "
-fi
 
 set +eu
 
@@ -29,7 +26,7 @@ if [[ "$CIRCLECI" == true ]]; then
 		--log-opt syslog-address=udp://127.0.0.1:5514 \
 		--log-opt syslog-facility=daemon \
 		--log-opt tag="{{.Name}}" \
-	  $DOCKER_IMAGE node $SEEDS --log_level=debug --proxy_app=$APP_PROXY
+  	  $DOCKER_IMAGE node $NODE_FLAGS --log_level=debug --proxy_app=$APP_PROXY
 else
 	set -u
 	docker run -d \
@@ -38,5 +35,5 @@ else
 	  --name local_testnet_$ID \
 	  --entrypoint tendermint \
 	  -e TMROOT=/go/src/github.com/tendermint/tendermint/test/p2p/data/mach$ID/core \
-	  $DOCKER_IMAGE node $SEEDS --log_level=info --proxy_app=$APP_PROXY
+  	  $DOCKER_IMAGE node $NODE_FLAGS --log_level=info --proxy_app=$APP_PROXY
 fi
diff --git a/test/p2p/pex/check_peer.sh b/test/p2p/pex/check_peer.sh
new file mode 100644
index 00000000..ceabd2ac
--- /dev/null
+++ b/test/p2p/pex/check_peer.sh
@@ -0,0 +1,17 @@
+#! /bin/bash
+set -u
+
+ID=$1
+N=$2
+
+addr=$(test/p2p/ip.sh "$ID"):46657
+
+echo "2. wait until peer $ID connects to other nodes using pex reactor"
+peers_count="0"
+while [[ "$peers_count" -lt "$((N-1))" ]]; do
+  sleep 1
+  peers_count=$(curl -s "$addr/net_info" | jq ".result[1].peers | length")
+  echo "... peers count = $peers_count, expected = $((N-1))"
+done
+
+echo "... successful"
diff --git a/test/p2p/pex/test.sh b/test/p2p/pex/test.sh
new file mode 100644
index 00000000..84d832b9
--- /dev/null
+++ b/test/p2p/pex/test.sh
@@ -0,0 +1,33 @@
+#! /bin/bash
+set -eu
+
+DOCKER_IMAGE=$1
+NETWORK_NAME=$2
+N=$3
+PROXY_APP=$4
+
+ID=1
+
+echo "----------------------------------------------------------------------"
+echo "Testing pex creates the addrbook and uses it if seeds are not provided"
+echo "(assuming peers are started with pex enabled)"
+
+echo "1. restart peer $ID"
+docker stop "local_testnet_$ID"
+# preserce addrbook.json
+docker cp "local_testnet_$ID:/go/src/github.com/tendermint/tendermint/test/p2p/data/mach1/core/addrbook.json" "/tmp/addrbook.json"
+docker rm -vf "local_testnet_$ID"
+
+# NOTE that we do not provide seeds this time
+bash test/p2p/peer.sh "$DOCKER_IMAGE" "$NETWORK_NAME" "$ID" "$PROXY_APP" "--pex"
+docker cp "/tmp/addrbook.json" "local_testnet_$ID:/go/src/github.com/tendermint/tendermint/test/p2p/data/mach1/core/addrbook.json"
+echo "with the following addrbook:"
+docker exec "local_testnet_$ID" cat "/go/src/github.com/tendermint/tendermint/test/p2p/data/mach1/core/addrbook.json"
+echo ""
+
+# if the client runs forever, it means addrbook wasn't saved or was empty
+bash test/p2p/client.sh "$DOCKER_IMAGE" "$NETWORK_NAME" "pex_$ID" "test/p2p/pex/check_peer.sh $ID $N"
+
+echo ""
+echo "PASS"
+echo ""
diff --git a/test/p2p/test.sh b/test/p2p/test.sh
index 0f29aa19..1157ac06 100644
--- a/test/p2p/test.sh
+++ b/test/p2p/test.sh
@@ -29,5 +29,8 @@ bash test/p2p/client.sh $DOCKER_IMAGE $NETWORK_NAME ab "test/p2p/atomic_broadcas
 # for each node, kill it and readd via fast sync
 bash test/p2p/fast_sync/test.sh $DOCKER_IMAGE $NETWORK_NAME $N $PROXY_APP
 
-# test killing all peers
+# test killing all peers 3 times
 bash test/p2p/kill_all/test.sh $DOCKER_IMAGE $NETWORK_NAME $N 3
+
+# test pex
+bash test/p2p/pex/test.sh $DOCKER_IMAGE $NETWORK_NAME $N $PROXY_APP

From 4090a31d194a5aeec1cfd9432feb1df3656191f5 Mon Sep 17 00:00:00 2001
From: Anton Kaliaev <anton.kalyaev@gmail.com>
Date: Sat, 14 Jan 2017 00:47:13 +0400
Subject: [PATCH 112/132] save seeds to addrBook (Refs #335)

---
 node/node.go | 82 ++++++++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 80 insertions(+), 2 deletions(-)

diff --git a/node/node.go b/node/node.go
index b739396d..f14fc3ec 100644
--- a/node/node.go
+++ b/node/node.go
@@ -119,10 +119,8 @@ func NewNode(config cfg.Config, privValidator *types.PrivValidator, clientCreato
 	sw.AddReactor("CONSENSUS", consensusReactor)
 
 	// Optionally, start the pex reactor
-	// TODO: this is a dev feature, it needs some love
 	if config.GetBool("pex_reactor") {
 		addrBook := p2p.NewAddrBook(config.GetString("addrbook_file"), config.GetBool("addrbook_strict"))
-		addrBook.Start()
 		pexReactor := p2p.NewPEXReactor(addrBook)
 		sw.AddReactor("PEX", pexReactor)
 	}
@@ -374,6 +372,86 @@ func makeNodeInfo(config cfg.Config, sw *p2p.Switch, privKey crypto.PrivKeyEd255
 	return nodeInfo
 }
 
+//------------------------------------------------------------------------------
+
+// Users wishing to:
+//	* use an external signer for their validators
+//	* supply an in-proc abci app
+// should fork tendermint/tendermint and implement RunNode to
+// call NewNode with their custom priv validator and/or custom
+// proxy.ClientCreator interface
+func RunNode(config cfg.Config) {
+	// Wait until the genesis doc becomes available
+	genDocFile := config.GetString("genesis_file")
+	if !FileExists(genDocFile) {
+		log.Notice(Fmt("Waiting for genesis file %v...", genDocFile))
+		for {
+			time.Sleep(time.Second)
+			if !FileExists(genDocFile) {
+				continue
+			}
+			jsonBlob, err := ioutil.ReadFile(genDocFile)
+			if err != nil {
+				Exit(Fmt("Couldn't read GenesisDoc file: %v", err))
+			}
+			genDoc := types.GenesisDocFromJSON(jsonBlob)
+			if genDoc.ChainID == "" {
+				PanicSanity(Fmt("Genesis doc %v must include non-empty chain_id", genDocFile))
+			}
+			config.Set("chain_id", genDoc.ChainID)
+		}
+	}
+
+	// Create & start node
+	n := NewNodeDefault(config)
+
+	protocol, address := ProtocolAndAddress(config.GetString("node_laddr"))
+	l := p2p.NewDefaultListener(protocol, address, config.GetBool("skip_upnp"))
+	n.AddListener(l)
+	err := n.Start()
+	if err != nil {
+		Exit(Fmt("Failed to start node: %v", err))
+	}
+
+	log.Notice("Started node", "nodeInfo", n.sw.NodeInfo())
+
+	if config.GetString("seeds") != "" {
+		seeds := strings.Split(config.GetString("seeds"), ",")
+
+		if config.GetBool("pex_reactor") {
+			// add seeds to `addrBook` to avoid losing
+			r := n.sw.Reactor("PEX").(*p2p.PEXReactor)
+			ourAddr := n.NodeInfo().ListenAddr
+			for _, s := range seeds {
+				// do not add ourselves
+				if s == ourAddr {
+					continue
+				}
+
+				addr := p2p.NewNetAddressString(s)
+				r.AddPeerAddress(addr, p2p.NewNetAddressString(ourAddr))
+			}
+			r.SaveAddrBook()
+		}
+
+		// dial out
+		n.sw.DialSeeds(seeds)
+	}
+
+	// Run the RPC server.
+	if config.GetString("rpc_laddr") != "" {
+		_, err := n.StartRPC()
+		if err != nil {
+			PanicCrisis(err)
+		}
+	}
+
+	// Sleep forever and then...
+	TrapSignal(func() {
+		n.Stop()
+	})
+}
+
 func (n *Node) NodeInfo() *p2p.NodeInfo {
 	return n.sw.NodeInfo()
 }

From af5cd5cc7522729b72747d3cbf5b7293df3c634c Mon Sep 17 00:00:00 2001
From: Anton Kaliaev <anton.kalyaev@gmail.com>
Date: Tue, 17 Jan 2017 23:08:30 +0400
Subject: [PATCH 113/132] [p2p tests] test other peers connect to us

if we have neither seeds nor the addrbook
---
 test/p2p/pex/test.sh | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/test/p2p/pex/test.sh b/test/p2p/pex/test.sh
index 84d832b9..45779603 100644
--- a/test/p2p/pex/test.sh
+++ b/test/p2p/pex/test.sh
@@ -18,7 +18,7 @@ docker stop "local_testnet_$ID"
 docker cp "local_testnet_$ID:/go/src/github.com/tendermint/tendermint/test/p2p/data/mach1/core/addrbook.json" "/tmp/addrbook.json"
 docker rm -vf "local_testnet_$ID"
 
-# NOTE that we do not provide seeds this time
+# NOTE that we do not provide seeds
 bash test/p2p/peer.sh "$DOCKER_IMAGE" "$NETWORK_NAME" "$ID" "$PROXY_APP" "--pex"
 docker cp "/tmp/addrbook.json" "local_testnet_$ID:/go/src/github.com/tendermint/tendermint/test/p2p/data/mach1/core/addrbook.json"
 echo "with the following addrbook:"
@@ -28,6 +28,20 @@ echo ""
 # if the client runs forever, it means addrbook wasn't saved or was empty
 bash test/p2p/client.sh "$DOCKER_IMAGE" "$NETWORK_NAME" "pex_$ID" "test/p2p/pex/check_peer.sh $ID $N"
 
+echo "----------------------------------------------------------------------"
+echo "Testing other peers connect to us if we have neither seeds nor the addrbook"
+echo "(assuming peers are started with pex enabled)"
+
+echo "1. restart peer $ID"
+docker stop "local_testnet_$ID"
+docker rm -vf "local_testnet_$ID"
+
+# NOTE that we do not provide seeds
+bash test/p2p/peer.sh "$DOCKER_IMAGE" "$NETWORK_NAME" "$ID" "$PROXY_APP" "--pex"
+
+# if the client runs forever, it means other peers have removed us from their books (which should not happen)
+bash test/p2p/client.sh "$DOCKER_IMAGE" "$NETWORK_NAME" "pex_$ID" "test/p2p/pex/check_peer.sh $ID $N"
+
 echo ""
 echo "PASS"
 echo ""

From 05d8cd50b5f5c5049a1d87b9371457140d1b066c Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Sat, 4 Mar 2017 23:25:12 -0500
Subject: [PATCH 114/132] update glide and node.go for update to p2p.AddrBook

---
 cmd/tendermint/run_node.go |   6 +-
 glide.lock                 |   4 +-
 node/node.go               | 152 +++++++++++++------------------------
 3 files changed, 58 insertions(+), 104 deletions(-)

diff --git a/cmd/tendermint/run_node.go b/cmd/tendermint/run_node.go
index a6645154..1940a605 100644
--- a/cmd/tendermint/run_node.go
+++ b/cmd/tendermint/run_node.go
@@ -13,9 +13,9 @@ import (
 // Users wishing to:
 //	* Use an external signer for their validators
 //	* Supply an in-proc abci app
-// should import tendermint/tendermint and implement their own RunNode to
-// call NewNode with their custom priv validator and/or custom
-// proxy.ClientCreator interface
+// should import github.com/tendermint/tendermint/node and implement
+// their own run_node to call node.NewNode (instead of node.NewNodeDefault)
+// with their custom priv validator and/or custom proxy.ClientCreator
 func run_node(config cfg.Config) {
 
 	// Wait until the genesis doc becomes available
diff --git a/glide.lock b/glide.lock
index c024b391..af517336 100644
--- a/glide.lock
+++ b/glide.lock
@@ -1,5 +1,5 @@
 hash: 41f8fec708e98b7f8c4804be46008493199fa45e89b2d5dc237fd65fe431c62f
-updated: 2017-03-04T22:10:20.337283524-05:00
+updated: 2017-03-04T23:34:29.802454496-05:00
 imports:
 - name: github.com/btcsuite/btcd
   version: d06c0bb181529331be8f8d9350288c420d9e60e4
@@ -101,7 +101,7 @@ imports:
 - name: github.com/tendermint/go-merkle
   version: 714d4d04557fd068a7c2a1748241ce8428015a96
 - name: github.com/tendermint/go-p2p
-  version: 17e6ae813fc8399ab96d32dc6b392eb1be8dbe8d
+  version: 56eebb95eede5f7cf742ecaefa83bed884df99ec
   subpackages:
   - upnp
 - name: github.com/tendermint/go-rpc
diff --git a/node/node.go b/node/node.go
index f14fc3ec..2e817766 100644
--- a/node/node.go
+++ b/node/node.go
@@ -32,19 +32,25 @@ import (
 type Node struct {
 	cmn.BaseService
 
-	config           cfg.Config
-	sw               *p2p.Switch
-	evsw             types.EventSwitch
-	blockStore       *bc.BlockStore
-	bcReactor        *bc.BlockchainReactor
-	mempoolReactor   *mempl.MempoolReactor
-	consensusState   *consensus.ConsensusState
-	consensusReactor *consensus.ConsensusReactor
-	privValidator    *types.PrivValidator
-	genesisDoc       *types.GenesisDoc
-	privKey          crypto.PrivKeyEd25519
-	proxyApp         proxy.AppConns
-	rpcListeners     []net.Listener
+	// config
+	config        cfg.Config           // user config
+	genesisDoc    *types.GenesisDoc    // initial validator set
+	privValidator *types.PrivValidator // local node's validator key
+
+	// network
+	privKey  crypto.PrivKeyEd25519 // local node's p2p key
+	sw       *p2p.Switch           // p2p connections
+	addrBook *p2p.AddrBook         // known peers
+
+	// services
+	evsw             types.EventSwitch           // pub/sub for services
+	blockStore       *bc.BlockStore              // store the blockchain to disk
+	bcReactor        *bc.BlockchainReactor       // for fast-syncing
+	mempoolReactor   *mempl.MempoolReactor       // for gossipping transactions
+	consensusState   *consensus.ConsensusState   // latest consensus state
+	consensusReactor *consensus.ConsensusReactor // for participating in the consensus
+	proxyApp         proxy.AppConns              // connection to the application
+	rpcListeners     []net.Listener              // rpc servers
 }
 
 func NewNodeDefault(config cfg.Config) *Node {
@@ -119,8 +125,9 @@ func NewNode(config cfg.Config, privValidator *types.PrivValidator, clientCreato
 	sw.AddReactor("CONSENSUS", consensusReactor)
 
 	// Optionally, start the pex reactor
+	var addrBook *p2p.AddrBook
 	if config.GetBool("pex_reactor") {
-		addrBook := p2p.NewAddrBook(config.GetString("addrbook_file"), config.GetBool("addrbook_strict"))
+		addrBook = p2p.NewAddrBook(config.GetString("addrbook_file"), config.GetBool("addrbook_strict"))
 		pexReactor := p2p.NewPEXReactor(addrBook)
 		sw.AddReactor("PEX", pexReactor)
 	}
@@ -166,17 +173,20 @@ func NewNode(config cfg.Config, privValidator *types.PrivValidator, clientCreato
 	}
 
 	node := &Node{
-		config:           config,
-		sw:               sw,
+		config:        config,
+		genesisDoc:    state.GenesisDoc,
+		privValidator: privValidator,
+
+		privKey:  privKey,
+		sw:       sw,
+		addrBook: addrBook,
+
 		evsw:             eventSwitch,
 		blockStore:       blockStore,
 		bcReactor:        bcReactor,
 		mempoolReactor:   mempoolReactor,
 		consensusState:   consensusState,
 		consensusReactor: consensusReactor,
-		privValidator:    privValidator,
-		genesisDoc:       state.GenesisDoc,
-		privKey:          privKey,
 		proxyApp:         proxyApp,
 	}
 	node.BaseService = *cmn.NewBaseService(log, "Node", node)
@@ -199,10 +209,32 @@ func (n *Node) OnStart() error {
 		return err
 	}
 
-	// Dial out of seed nodes exist
+	// If seeds exist, add them to the address book and dial out
 	if n.config.GetString("seeds") != "" {
 		seeds := strings.Split(n.config.GetString("seeds"), ",")
-		n.sw.DialSeeds(seeds)
+
+		if n.config.GetBool("pex_reactor") {
+			// add seeds to `addrBook` to avoid losing
+			ourAddrS := n.NodeInfo().ListenAddr
+			ourAddr, _ := p2p.NewNetAddressString(ourAddrS)
+			for _, s := range seeds {
+				// do not add ourselves
+				if s == ourAddrS {
+					continue
+				}
+
+				addr, err := p2p.NewNetAddressString(s)
+				if err != nil {
+					n.addrBook.AddAddress(addr, ourAddr)
+				}
+			}
+			n.addrBook.Save()
+		}
+
+		// dial out
+		if err := n.sw.DialSeeds(seeds); err != nil {
+			return err
+		}
 	}
 
 	// Run the RPC server
@@ -374,84 +406,6 @@ func makeNodeInfo(config cfg.Config, sw *p2p.Switch, privKey crypto.PrivKeyEd255
 
 //------------------------------------------------------------------------------
 
-// Users wishing to:
-//	* use an external signer for their validators
-//	* supply an in-proc abci app
-// should fork tendermint/tendermint and implement RunNode to
-// call NewNode with their custom priv validator and/or custom
-// proxy.ClientCreator interface
-func RunNode(config cfg.Config) {
-	// Wait until the genesis doc becomes available
-	genDocFile := config.GetString("genesis_file")
-	if !FileExists(genDocFile) {
-		log.Notice(Fmt("Waiting for genesis file %v...", genDocFile))
-		for {
-			time.Sleep(time.Second)
-			if !FileExists(genDocFile) {
-				continue
-			}
-			jsonBlob, err := ioutil.ReadFile(genDocFile)
-			if err != nil {
-				Exit(Fmt("Couldn't read GenesisDoc file: %v", err))
-			}
-			genDoc := types.GenesisDocFromJSON(jsonBlob)
-			if genDoc.ChainID == "" {
-				PanicSanity(Fmt("Genesis doc %v must include non-empty chain_id", genDocFile))
-			}
-			config.Set("chain_id", genDoc.ChainID)
-		}
-	}
-
-	// Create & start node
-	n := NewNodeDefault(config)
-
-	protocol, address := ProtocolAndAddress(config.GetString("node_laddr"))
-	l := p2p.NewDefaultListener(protocol, address, config.GetBool("skip_upnp"))
-	n.AddListener(l)
-	err := n.Start()
-	if err != nil {
-		Exit(Fmt("Failed to start node: %v", err))
-	}
-
-	log.Notice("Started node", "nodeInfo", n.sw.NodeInfo())
-
-	if config.GetString("seeds") != "" {
-		seeds := strings.Split(config.GetString("seeds"), ",")
-
-		if config.GetBool("pex_reactor") {
-			// add seeds to `addrBook` to avoid losing
-			r := n.sw.Reactor("PEX").(*p2p.PEXReactor)
-			ourAddr := n.NodeInfo().ListenAddr
-			for _, s := range seeds {
-				// do not add ourselves
-				if s == ourAddr {
-					continue
-				}
-
-				addr := p2p.NewNetAddressString(s)
-				r.AddPeerAddress(addr, p2p.NewNetAddressString(ourAddr))
-			}
-			r.SaveAddrBook()
-		}
-
-		// dial out
-		n.sw.DialSeeds(seeds)
-	}
-
-	// Run the RPC server.
-	if config.GetString("rpc_laddr") != "" {
-		_, err := n.StartRPC()
-		if err != nil {
-			PanicCrisis(err)
-		}
-	}
-
-	// Sleep forever and then...
-	TrapSignal(func() {
-		n.Stop()
-	})
-}
-
 func (n *Node) NodeInfo() *p2p.NodeInfo {
 	return n.sw.NodeInfo()
 }

From a5ce4f6c36114d1582931f718d29a61991b17f2d Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Sun, 5 Mar 2017 00:19:53 -0500
Subject: [PATCH 115/132] update glide

---
 glide.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/glide.lock b/glide.lock
index af517336..a72ced40 100644
--- a/glide.lock
+++ b/glide.lock
@@ -1,5 +1,5 @@
 hash: 41f8fec708e98b7f8c4804be46008493199fa45e89b2d5dc237fd65fe431c62f
-updated: 2017-03-04T23:34:29.802454496-05:00
+updated: 2017-03-05T00:18:48.136517571-05:00
 imports:
 - name: github.com/btcsuite/btcd
   version: d06c0bb181529331be8f8d9350288c420d9e60e4
@@ -83,13 +83,13 @@ imports:
   subpackages:
   - test
 - name: github.com/tendermint/go-config
-  version: 28765c18222cd3171d5290f1cc69a9cf710ac5ae
+  version: 620dcbbd7d587cf3599dedbf329b64311b0c307a
 - name: github.com/tendermint/go-crypto
   version: 3f47cfac5fcd9e0f1727c7db980b3559913b3e3a
 - name: github.com/tendermint/go-data
   version: 32271140e8fd5abdbb22e268d7a02421fa382f0b
 - name: github.com/tendermint/go-db
-  version: 286cbbd99d9ff76681b7c18cc506d476ccf68725
+  version: eac3f2bc147023957c8bf69432a4e6c4dc5c3f72
 - name: github.com/tendermint/go-events
   version: f8ffbfb2be3483e9e7927495590a727f51c0c11f
 - name: github.com/tendermint/go-flowrate

From d93d75497213e504deff76541e6f2a40e899f359 Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Sun, 5 Mar 2017 02:03:22 -0500
Subject: [PATCH 116/132] test/p2p/fast_sync: use --pex on restart

---
 test/p2p/fast_sync/test_peer.sh | 2 +-
 test/p2p/pex/test.sh            | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/p2p/fast_sync/test_peer.sh b/test/p2p/fast_sync/test_peer.sh
index cab6fc6f..615ae9e0 100644
--- a/test/p2p/fast_sync/test_peer.sh
+++ b/test/p2p/fast_sync/test_peer.sh
@@ -27,7 +27,7 @@ SEEDS="$(test/p2p/ip.sh 1):46656"
 for j in `seq 2 $N`; do
 	SEEDS="$SEEDS,$(test/p2p/ip.sh $j):46656"
 done
-bash test/p2p/peer.sh $DOCKER_IMAGE $NETWORK_NAME $ID $PROXY_APP "--seeds $SEEDS"
+bash test/p2p/peer.sh $DOCKER_IMAGE $NETWORK_NAME $ID $PROXY_APP "--seeds $SEEDS --pex"
 
 # wait for peer to sync and check the app hash
 bash test/p2p/client.sh $DOCKER_IMAGE $NETWORK_NAME fs_$ID "test/p2p/fast_sync/check_peer.sh $ID"
diff --git a/test/p2p/pex/test.sh b/test/p2p/pex/test.sh
index 45779603..3c624e27 100644
--- a/test/p2p/pex/test.sh
+++ b/test/p2p/pex/test.sh
@@ -14,7 +14,7 @@ echo "(assuming peers are started with pex enabled)"
 
 echo "1. restart peer $ID"
 docker stop "local_testnet_$ID"
-# preserce addrbook.json
+# preserve addrbook.json
 docker cp "local_testnet_$ID:/go/src/github.com/tendermint/tendermint/test/p2p/data/mach1/core/addrbook.json" "/tmp/addrbook.json"
 docker rm -vf "local_testnet_$ID"
 

From de0153a1c4efc29a513113d5bf4f6909c65e9be7 Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Sun, 5 Mar 2017 02:04:09 -0500
Subject: [PATCH 117/132] consensus: some more informative logging

---
 consensus/replay.go    | 2 ++
 consensus/state.go     | 4 ++--
 types/block.go         | 2 +-
 types/validator_set.go | 4 ++--
 4 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/consensus/replay.go b/consensus/replay.go
index f117e66d..4b35a107 100644
--- a/consensus/replay.go
+++ b/consensus/replay.go
@@ -211,6 +211,8 @@ func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error {
 		return errors.New(Fmt("Error on replay: %v", err))
 	}
 
+	log.Notice("Completed ABCI Handshake - Tendermint and App are synced", "appHeight", blockHeight, "appHash", appHash)
+
 	// TODO: (on restart) replay mempool
 
 	return nil
diff --git a/consensus/state.go b/consensus/state.go
index f3fb7d16..3077439e 100644
--- a/consensus/state.go
+++ b/consensus/state.go
@@ -1160,13 +1160,13 @@ func (cs *ConsensusState) tryFinalizeCommit(height int) {
 
 	blockID, ok := cs.Votes.Precommits(cs.CommitRound).TwoThirdsMajority()
 	if !ok || len(blockID.Hash) == 0 {
-		log.Warn("Attempt to finalize failed. There was no +2/3 majority, or +2/3 was for <nil>.")
+		log.Warn("Attempt to finalize failed. There was no +2/3 majority, or +2/3 was for <nil>.", "height", height)
 		return
 	}
 	if !cs.ProposalBlock.HashesTo(blockID.Hash) {
 		// TODO: this happens every time if we're not a validator (ugly logs)
 		// TODO: ^^ wait, why does it matter that we're a validator?
-		log.Warn("Attempt to finalize failed. We don't have the commit block.")
+		log.Warn("Attempt to finalize failed. We don't have the commit block.", "height", height, "proposal-block", cs.ProposalBlock.Hash(), "commit-block", blockID.Hash)
 		return
 	}
 	//	go
diff --git a/types/block.go b/types/block.go
index a19afaac..61d25f6e 100644
--- a/types/block.go
+++ b/types/block.go
@@ -96,7 +96,7 @@ func (b *Block) FillHeader() {
 // If the block is incomplete, block hash is nil for safety.
 func (b *Block) Hash() []byte {
 	// fmt.Println(">>", b.Data)
-	if b.Header == nil || b.Data == nil || b.LastCommit == nil {
+	if b == nil || b.Header == nil || b.Data == nil || b.LastCommit == nil {
 		return nil
 	}
 	b.FillHeader()
diff --git a/types/validator_set.go b/types/validator_set.go
index a3b2838c..ce44b328 100644
--- a/types/validator_set.go
+++ b/types/validator_set.go
@@ -6,7 +6,7 @@ import (
 	"sort"
 	"strings"
 
-	. "github.com/tendermint/go-common"
+	cmn "github.com/tendermint/go-common"
 	"github.com/tendermint/go-merkle"
 )
 
@@ -48,7 +48,7 @@ func NewValidatorSet(vals []*Validator) *ValidatorSet {
 // TODO: mind the overflow when times and votingPower shares too large.
 func (valSet *ValidatorSet) IncrementAccum(times int) {
 	// Add VotingPower * times to each validator and order into heap.
-	validatorsHeap := NewHeap()
+	validatorsHeap := cmn.NewHeap()
 	for _, val := range valSet.Validators {
 		val.Accum += int64(val.VotingPower) * int64(times) // TODO: mind overflow
 		validatorsHeap.Push(val, accumComparable(val.Accum))

From 2037d2631ab5d37458b6a6e42ceb00718ca29e2c Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Sun, 5 Mar 2017 03:40:36 -0500
Subject: [PATCH 118/132] fix race

---
 consensus/mempool_test.go | 13 +++++--------
 glide.lock                |  2 +-
 2 files changed, 6 insertions(+), 9 deletions(-)

diff --git a/consensus/mempool_test.go b/consensus/mempool_test.go
index e55dcf3e..6bfdfda9 100644
--- a/consensus/mempool_test.go
+++ b/consensus/mempool_test.go
@@ -81,15 +81,12 @@ func TestRmBadTx(t *testing.T) {
 		// check for the tx
 		for {
 			time.Sleep(time.Second)
-			select {
-			case <-ch:
-			default:
-				txs := cs.mempool.Reap(1)
-				if len(txs) == 0 {
-					ch <- struct{}{}
-				}
-
+			txs := cs.mempool.Reap(1)
+			if len(txs) == 0 {
+				ch <- struct{}{}
+				return
 			}
+
 		}
 	}()
 
diff --git a/glide.lock b/glide.lock
index a72ced40..a3ce63dd 100644
--- a/glide.lock
+++ b/glide.lock
@@ -79,7 +79,7 @@ imports:
 - name: github.com/tendermint/go-clist
   version: 3baa390bbaf7634251c42ad69a8682e7e3990552
 - name: github.com/tendermint/go-common
-  version: 6141dc6eedab067f7e25707a237218ef2c0dc273
+  version: dcb015dff6c7af21e65c8e2f3b450df19d38c777
   subpackages:
   - test
 - name: github.com/tendermint/go-config

From 749df0536f111aa1961f67c1b026cbd6596635dd Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Sun, 5 Mar 2017 14:59:02 -0500
Subject: [PATCH 119/132] test/docker: install abci apps first

---
 consensus/replay.go    |  2 +-
 test/docker/Dockerfile | 11 ++++++++---
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/consensus/replay.go b/consensus/replay.go
index 4b35a107..6c4e65a0 100644
--- a/consensus/replay.go
+++ b/consensus/replay.go
@@ -253,7 +253,7 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, proxyApp p
 			return h.replayBlocks(proxyApp, appBlockHeight, storeBlockHeight, false)
 
 		} else if appBlockHeight == storeBlockHeight {
-			// we're good!
+			// We're good!
 			return appHash, h.checkAppHash(appHash)
 		}
 
diff --git a/test/docker/Dockerfile b/test/docker/Dockerfile
index bec59bf9..389158e5 100644
--- a/test/docker/Dockerfile
+++ b/test/docker/Dockerfile
@@ -6,10 +6,16 @@ RUN apt-get update && \
   apt-get install -y --no-install-recommends \
   jq bsdmainutils vim-common psmisc netcat
 
-# Setup tendermint repo with vendored dependencies
-# but without code - docker caching prevents reinstall on code change!
+# Setup tendermint repo 
 ENV REPO $GOPATH/src/github.com/tendermint/tendermint
 WORKDIR $REPO
+
+# Install the apps
+ADD scripts/install_abci_apps.sh install_abci_apps.sh
+RUN bash install_abci_apps.sh
+
+# Install the vendored dependencies before copying code
+# docker caching prevents reinstall on code change!
 ADD glide.yaml glide.yaml
 ADD glide.lock glide.lock
 ADD Makefile Makefile
@@ -19,7 +25,6 @@ RUN make get_vendor_deps
 COPY . $REPO
 
 RUN go install ./cmd/tendermint
-RUN bash scripts/install_abci_apps.sh
 
 # expose the volume for debugging
 VOLUME $REPO

From 55602b9be669982b2ccdcafb79be9d703a5cb078 Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Sun, 5 Mar 2017 15:05:36 -0500
Subject: [PATCH 120/132] failing ProposerSelection test

---
 types/validator_set_test.go | 49 ++++++++++++++++++++++++++++++++-----
 1 file changed, 43 insertions(+), 6 deletions(-)

diff --git a/types/validator_set_test.go b/types/validator_set_test.go
index 9107e77e..76a32f46 100644
--- a/types/validator_set_test.go
+++ b/types/validator_set_test.go
@@ -1,23 +1,23 @@
 package types
 
 import (
-	. "github.com/tendermint/go-common"
-	"github.com/tendermint/go-crypto"
-
 	"bytes"
 	"strings"
 	"testing"
+
+	cmn "github.com/tendermint/go-common"
+	"github.com/tendermint/go-crypto"
 )
 
 func randPubKey() crypto.PubKeyEd25519 {
 	var pubKey [32]byte
-	copy(pubKey[:], RandBytes(32))
+	copy(pubKey[:], cmn.RandBytes(32))
 	return crypto.PubKeyEd25519(pubKey)
 }
 
 func randValidator_() *Validator {
-	val := NewValidator(randPubKey(), RandInt64())
-	val.Accum = RandInt64()
+	val := NewValidator(randPubKey(), cmn.RandInt64())
+	val.Accum = cmn.RandInt64()
 	return val
 }
 
@@ -138,6 +138,43 @@ func TestProposerSelection2(t *testing.T) {
 	}
 }
 
+func TestProposerSelection3(t *testing.T) {
+	vset := NewValidatorSet([]*Validator{
+		newValidator([]byte("a"), 1),
+		newValidator([]byte("b"), 1),
+		newValidator([]byte("c"), 1),
+		newValidator([]byte("d"), 1),
+	})
+
+	proposerOrder := make([]*Validator, 4)
+	for i := 0; i < 4; i++ {
+		proposerOrder[i] = vset.Proposer()
+		vset.IncrementAccum(1)
+	}
+
+	// i for the loop
+	// j for the times
+	// we should go in order for ever, despite occasional IncrementAccums with times > 1
+	var i, j int
+	for ; i < 1000; i++ {
+		got := vset.Proposer().Address
+		expected := proposerOrder[j%4].Address
+		if !bytes.Equal(got, expected) {
+			t.Fatalf(cmn.Fmt("vset.Proposer (%X) does not match expected proposer (%X) for (%d, %d)", got, expected, i, j))
+		}
+
+		// times is usually 1
+		times := 1
+		if cmn.RandInt()%2 > 0 {
+			// sometimes its up to 5
+			times = cmn.RandInt() % 5
+		}
+		vset.IncrementAccum(times)
+
+		j += times
+	}
+}
+
 func BenchmarkValidatorSetCopy(b *testing.B) {
 	b.StopTimer()
 	vset := NewValidatorSet([]*Validator{})

From 0fa34f7f67c8292f5d9af95150494d5646cd752e Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Sun, 5 Mar 2017 15:24:15 -0500
Subject: [PATCH 121/132] fix ProposerSelection by persisting proposer

---
 types/validator_set.go      | 67 ++++++++++++++++++++++++++----------
 types/validator_set_test.go | 68 ++++++++++++++++++++++---------------
 2 files changed, 91 insertions(+), 44 deletions(-)

diff --git a/types/validator_set.go b/types/validator_set.go
index ce44b328..7d872450 100644
--- a/types/validator_set.go
+++ b/types/validator_set.go
@@ -8,6 +8,7 @@ import (
 
 	cmn "github.com/tendermint/go-common"
 	"github.com/tendermint/go-merkle"
+	"github.com/tendermint/go-wire"
 )
 
 // ValidatorSet represent a set of *Validator at a given height.
@@ -22,10 +23,10 @@ import (
 // TODO: consider validator Accum overflow
 // TODO: move valset into an iavl tree where key is 'blockbonded|pubkey'
 type ValidatorSet struct {
-	Validators []*Validator // NOTE: persisted via reflect, must be exported.
+	Validators   []*Validator // NOTE: persisted via reflect, must be exported.
+	LastProposer *Validator
 
 	// cached (unexported)
-	proposer         *Validator
 	totalVotingPower int64
 }
 
@@ -42,26 +43,28 @@ func NewValidatorSet(vals []*Validator) *ValidatorSet {
 	if vals != nil {
 		vs.IncrementAccum(1)
 	}
+
 	return vs
 }
 
+// incrementAccum and update the proposer
 // TODO: mind the overflow when times and votingPower shares too large.
 func (valSet *ValidatorSet) IncrementAccum(times int) {
 	// Add VotingPower * times to each validator and order into heap.
 	validatorsHeap := cmn.NewHeap()
 	for _, val := range valSet.Validators {
 		val.Accum += int64(val.VotingPower) * int64(times) // TODO: mind overflow
-		validatorsHeap.Push(val, accumComparable(val.Accum))
+		validatorsHeap.Push(val, accumComparable{val})
 	}
 
-	// Decrement the validator with most accum, times times.
+	// Decrement the validator with most accum times times
 	for i := 0; i < times; i++ {
 		mostest := validatorsHeap.Peek().(*Validator)
 		if i == times-1 {
-			valSet.proposer = mostest
+			valSet.LastProposer = mostest
 		}
 		mostest.Accum -= int64(valSet.TotalVotingPower())
-		validatorsHeap.Update(mostest, accumComparable(mostest.Accum))
+		validatorsHeap.Update(mostest, accumComparable{mostest})
 	}
 }
 
@@ -73,7 +76,7 @@ func (valSet *ValidatorSet) Copy() *ValidatorSet {
 	}
 	return &ValidatorSet{
 		Validators:       validators,
-		proposer:         valSet.proposer,
+		LastProposer:     valSet.LastProposer,
 		totalVotingPower: valSet.totalVotingPower,
 	}
 }
@@ -118,12 +121,20 @@ func (valSet *ValidatorSet) Proposer() (proposer *Validator) {
 	if len(valSet.Validators) == 0 {
 		return nil
 	}
-	if valSet.proposer == nil {
-		for _, val := range valSet.Validators {
-			valSet.proposer = valSet.proposer.CompareAccum(val)
+	if valSet.LastProposer == nil {
+		valSet.LastProposer = valSet.findProposer()
+	}
+	return valSet.LastProposer.Copy()
+}
+
+func (valSet *ValidatorSet) findProposer() *Validator {
+	var proposer *Validator
+	for _, val := range valSet.Validators {
+		if proposer == nil || !bytes.Equal(val.Address, proposer.Address) {
+			proposer = proposer.CompareAccum(val)
 		}
 	}
-	return valSet.proposer.Copy()
+	return proposer
 }
 
 func (valSet *ValidatorSet) Hash() []byte {
@@ -145,7 +156,7 @@ func (valSet *ValidatorSet) Add(val *Validator) (added bool) {
 	if idx == len(valSet.Validators) {
 		valSet.Validators = append(valSet.Validators, val)
 		// Invalidate cache
-		valSet.proposer = nil
+		valSet.LastProposer = nil
 		valSet.totalVotingPower = 0
 		return true
 	} else if bytes.Compare(valSet.Validators[idx].Address, val.Address) == 0 {
@@ -157,7 +168,7 @@ func (valSet *ValidatorSet) Add(val *Validator) (added bool) {
 		copy(newValidators[idx+1:], valSet.Validators[idx:])
 		valSet.Validators = newValidators
 		// Invalidate cache
-		valSet.proposer = nil
+		valSet.LastProposer = nil
 		valSet.totalVotingPower = 0
 		return true
 	}
@@ -170,7 +181,7 @@ func (valSet *ValidatorSet) Update(val *Validator) (updated bool) {
 	} else {
 		valSet.Validators[index] = val.Copy()
 		// Invalidate cache
-		valSet.proposer = nil
+		valSet.LastProposer = nil
 		valSet.totalVotingPower = 0
 		return true
 	}
@@ -190,7 +201,7 @@ func (valSet *ValidatorSet) Remove(address []byte) (val *Validator, removed bool
 		}
 		valSet.Validators = newValidators
 		// Invalidate cache
-		valSet.proposer = nil
+		valSet.LastProposer = nil
 		valSet.totalVotingPower = 0
 		return removedVal, true
 	}
@@ -278,6 +289,24 @@ func (valSet *ValidatorSet) VerifyCommitAny(chainID string, blockID BlockID, hei
 	*/
 }
 
+func (valSet *ValidatorSet) ToBytes() []byte {
+	buf, n, err := new(bytes.Buffer), new(int), new(error)
+	wire.WriteBinary(valSet, buf, n, err)
+	if *err != nil {
+		cmn.PanicCrisis(*err)
+	}
+	return buf.Bytes()
+}
+
+func (valSet *ValidatorSet) FromBytes(b []byte) {
+	r, n, err := bytes.NewReader(b), new(int), new(error)
+	wire.ReadBinary(valSet, r, 0, n, err)
+	if *err != nil {
+		// DATA HAS BEEN CORRUPTED OR THE SPEC HAS CHANGED
+		cmn.PanicCrisis(*err)
+	}
+}
+
 func (valSet *ValidatorSet) String() string {
 	return valSet.StringIndented("")
 }
@@ -325,11 +354,15 @@ func (vs ValidatorsByAddress) Swap(i, j int) {
 //-------------------------------------
 // Use with Heap for sorting validators by accum
 
-type accumComparable int64
+type accumComparable struct {
+	*Validator
+}
 
 // We want to find the validator with the greatest accum.
 func (ac accumComparable) Less(o interface{}) bool {
-	return int64(ac) > int64(o.(accumComparable))
+	other := o.(accumComparable).Validator
+	larger := ac.CompareAccum(other)
+	return bytes.Equal(larger.Address, ac.Address)
 }
 
 //----------------------------------------
diff --git a/types/validator_set_test.go b/types/validator_set_test.go
index 76a32f46..c37fc568 100644
--- a/types/validator_set_test.go
+++ b/types/validator_set_test.go
@@ -44,7 +44,7 @@ func TestCopy(t *testing.T) {
 	}
 }
 
-func TestProposerSelection(t *testing.T) {
+func TestProposerSelection1(t *testing.T) {
 	vset := NewValidatorSet([]*Validator{
 		newValidator([]byte("foo"), 1000),
 		newValidator([]byte("bar"), 300),
@@ -67,74 +67,76 @@ func newValidator(address []byte, power int64) *Validator {
 }
 
 func TestProposerSelection2(t *testing.T) {
-	addr1 := []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
-	addr2 := []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}
-	addr3 := []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2}
+	addr0 := []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
+	addr1 := []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}
+	addr2 := []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2}
 
 	// when all voting power is same, we go in order of addresses
-	val1, val2, val3 := newValidator(addr1, 100), newValidator(addr2, 100), newValidator(addr3, 100)
-	valList := []*Validator{val1, val2, val3}
+	val0, val1, val2 := newValidator(addr0, 100), newValidator(addr1, 100), newValidator(addr2, 100)
+	valList := []*Validator{val0, val1, val2}
 	vals := NewValidatorSet(valList)
 	for i := 0; i < len(valList)*5; i++ {
-		ii := i % len(valList)
+		ii := (i) % len(valList)
 		prop := vals.Proposer()
 		if !bytes.Equal(prop.Address, valList[ii].Address) {
-			t.Fatalf("Expected %X. Got %X", valList[ii].Address, prop.Address)
+			t.Fatalf("(%d): Expected %X. Got %X", i, valList[ii].Address, prop.Address)
 		}
 		vals.IncrementAccum(1)
 	}
 
 	// One validator has more than the others, but not enough to propose twice in a row
-	*val3 = *newValidator(addr3, 400)
+	*val2 = *newValidator(addr2, 400)
 	vals = NewValidatorSet(valList)
+	// vals.IncrementAccum(1)
 	prop := vals.Proposer()
-	if !bytes.Equal(prop.Address, addr3) {
+	if !bytes.Equal(prop.Address, addr2) {
 		t.Fatalf("Expected address with highest voting power to be first proposer. Got %X", prop.Address)
 	}
 	vals.IncrementAccum(1)
 	prop = vals.Proposer()
-	if !bytes.Equal(prop.Address, addr1) {
+	if !bytes.Equal(prop.Address, addr0) {
 		t.Fatalf("Expected smallest address to be validator. Got %X", prop.Address)
 	}
 
 	// One validator has more than the others, and enough to be proposer twice in a row
-	*val3 = *newValidator(addr3, 401)
+	*val2 = *newValidator(addr2, 401)
 	vals = NewValidatorSet(valList)
 	prop = vals.Proposer()
-	if !bytes.Equal(prop.Address, addr3) {
+	if !bytes.Equal(prop.Address, addr2) {
 		t.Fatalf("Expected address with highest voting power to be first proposer. Got %X", prop.Address)
 	}
 	vals.IncrementAccum(1)
 	prop = vals.Proposer()
-	if !bytes.Equal(prop.Address, addr3) {
+	if !bytes.Equal(prop.Address, addr2) {
 		t.Fatalf("Expected address with highest voting power to be second proposer. Got %X", prop.Address)
 	}
 	vals.IncrementAccum(1)
 	prop = vals.Proposer()
-	if !bytes.Equal(prop.Address, addr1) {
+	if !bytes.Equal(prop.Address, addr0) {
 		t.Fatalf("Expected smallest address to be validator. Got %X", prop.Address)
 	}
 
 	// each validator should be the proposer a proportional number of times
-	val1, val2, val3 = newValidator(addr1, 4), newValidator(addr2, 5), newValidator(addr3, 3)
-	valList = []*Validator{val1, val2, val3}
+	val0, val1, val2 = newValidator(addr0, 4), newValidator(addr1, 5), newValidator(addr2, 3)
+	valList = []*Validator{val0, val1, val2}
 	propCount := make([]int, 3)
 	vals = NewValidatorSet(valList)
-	for i := 0; i < 120; i++ {
+	N := 1
+	for i := 0; i < 120*N; i++ {
 		prop := vals.Proposer()
 		ii := prop.Address[19]
 		propCount[ii] += 1
 		vals.IncrementAccum(1)
 	}
 
-	if propCount[0] != 40 {
-		t.Fatalf("Expected prop count for validator with 4/12 of voting power to be 40/120. Got %d/120", propCount[0])
+	if propCount[0] != 40*N {
+		t.Fatalf("Expected prop count for validator with 4/12 of voting power to be %d/%d. Got %d/%d", 40*N, 120*N, propCount[0], 120*N)
 	}
-	if propCount[1] != 50 {
-		t.Fatalf("Expected prop count for validator with 5/12 of voting power to be 50/120. Got %d/120", propCount[1])
+	if propCount[1] != 50*N {
+		t.Fatalf("Expected prop count for validator with 5/12 of voting power to be %d/%d. Got %d/%d", 50*N, 120*N, propCount[1], 120*N)
 	}
-	if propCount[2] != 30 {
-		t.Fatalf("Expected prop count for validator with 3/12 of voting power to be 30/120. Got %d/120", propCount[2])
+	if propCount[2] != 30*N {
+		t.Fatalf("Expected prop count for validator with 3/12 of voting power to be %d/%d. Got %d/%d", 30*N, 120*N, propCount[2], 120*N)
 	}
 }
 
@@ -154,18 +156,30 @@ func TestProposerSelection3(t *testing.T) {
 
 	// i for the loop
 	// j for the times
-	// we should go in order for ever, despite occasional IncrementAccums with times > 1
+	// we should go in order for ever, despite some IncrementAccums with times > 1
 	var i, j int
-	for ; i < 1000; i++ {
+	for ; i < 10000; i++ {
 		got := vset.Proposer().Address
 		expected := proposerOrder[j%4].Address
 		if !bytes.Equal(got, expected) {
 			t.Fatalf(cmn.Fmt("vset.Proposer (%X) does not match expected proposer (%X) for (%d, %d)", got, expected, i, j))
 		}
 
+		// serialize, deserialize, check proposer
+		b := vset.ToBytes()
+		vset.FromBytes(b)
+
+		computed := vset.Proposer() // findProposer()
+		if i != 0 {
+			if !bytes.Equal(got, computed.Address) {
+				t.Fatalf(cmn.Fmt("vset.Proposer (%X) does not match computed proposer (%X) for (%d, %d)", got, computed.Address, i, j))
+			}
+		}
+
 		// times is usually 1
 		times := 1
-		if cmn.RandInt()%2 > 0 {
+		mod := (cmn.RandInt() % 5) + 1
+		if cmn.RandInt()%mod > 0 {
 			// sometimes its up to 5
 			times = cmn.RandInt() % 5
 		}

From e4e70ece3f717475a3ad7e91ab6aef73e4eb3743 Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Sun, 5 Mar 2017 20:39:52 -0500
Subject: [PATCH 122/132] test: fix docker and apps

---
 test/app/dummy_test.sh |  8 ++++----
 test/docker/Dockerfile |  8 ++++----
 test/run_test.sh       |  5 +++--
 test/test.sh           | 15 +++++++++++----
 4 files changed, 22 insertions(+), 14 deletions(-)

diff --git a/test/app/dummy_test.sh b/test/app/dummy_test.sh
index adfcebb9..0449bc49 100644
--- a/test/app/dummy_test.sh
+++ b/test/app/dummy_test.sh
@@ -30,9 +30,9 @@ echo "... testing query with abci-cli"
 RESPONSE=`abci-cli query \"$KEY\"`
 
 set +e
-A=`echo $RESPONSE | grep 'log: exists'`
+A=`echo $RESPONSE | grep "$VALUE"`
 if [[ $? != 0 ]]; then
-	echo "Failed to find 'log: exists' for $KEY. Response:"
+	echo "Failed to find $VALUE for $KEY. Response:"
 	echo "$RESPONSE"
 	exit 1
 fi
@@ -41,9 +41,9 @@ set -e
 # we should not be able to look up the value
 RESPONSE=`abci-cli query \"$VALUE\"`
 set +e
-A=`echo $RESPONSE | grep 'log: exists'`
+A=`echo $RESPONSE | grep $VALUE`
 if [[ $? == 0 ]]; then
-	echo "Found 'log: exists' for $VALUE when we should not have. Response:"
+	echo "Found '$VALUE' for $VALUE when we should not have. Response:"
 	echo "$RESPONSE"
 	exit 1
 fi
diff --git a/test/docker/Dockerfile b/test/docker/Dockerfile
index 389158e5..829f5e45 100644
--- a/test/docker/Dockerfile
+++ b/test/docker/Dockerfile
@@ -10,10 +10,6 @@ RUN apt-get update && \
 ENV REPO $GOPATH/src/github.com/tendermint/tendermint
 WORKDIR $REPO
 
-# Install the apps
-ADD scripts/install_abci_apps.sh install_abci_apps.sh
-RUN bash install_abci_apps.sh
-
 # Install the vendored dependencies before copying code
 # docker caching prevents reinstall on code change!
 ADD glide.yaml glide.yaml
@@ -21,6 +17,10 @@ ADD glide.lock glide.lock
 ADD Makefile Makefile
 RUN make get_vendor_deps
 
+# Install the apps
+ADD scripts scripts
+RUN bash scripts/install_abci_apps.sh
+
 # Now copy in the code
 COPY . $REPO
 
diff --git a/test/run_test.sh b/test/run_test.sh
index fcc82d98..b505126e 100644
--- a/test/run_test.sh
+++ b/test/run_test.sh
@@ -6,12 +6,13 @@ pwd
 BRANCH=$(git rev-parse --abbrev-ref HEAD)
 echo "Current branch: $BRANCH"
 
+# run the go unit tests with coverage
 bash test/test_cover.sh
 
-# run the app tests
+# run the app tests using bash
 bash test/app/test.sh
 
-# run the persistence test
+# run the persistence tests using bash
 bash test/persist/test.sh
 
 if [[ "$BRANCH" == "master" || $(echo "$BRANCH" | grep "release-") != "" ]]; then
diff --git a/test/test.sh b/test/test.sh
index 1c1faca2..2e164fb3 100644
--- a/test/test.sh
+++ b/test/test.sh
@@ -12,18 +12,25 @@ echo "* [$(date +"%T")] cleaning up $LOGS_DIR"
 rm -rf "$LOGS_DIR"
 mkdir -p "$LOGS_DIR"
 
+set +e
+echo
+echo "* [$(date +"%T")] removing run_test container"
+docker rm -vf run_test
+set -e
+
 set +u
 if [[ "$CIRCLECI" == true ]]; then
-	set -u
 	echo
 	echo "* [$(date +"%T")] starting rsyslog container"
 	docker rm -f rsyslog || true
 	docker run -d -v "$LOGS_DIR:/var/log/" -p 127.0.0.1:5514:514/udp --name rsyslog voxxit/rsyslog
 fi
 
-echo
-echo "* [$(date +"%T")] building docker image"
-bash "$DIR/docker/build.sh"
+if [[ "$SKIP_BUILD" == "" ]]; then
+	echo
+	echo "* [$(date +"%T")] building docker image"
+	bash "$DIR/docker/build.sh"
+fi
 
 echo
 echo "* [$(date +"%T")] running go tests and app tests in docker container"

From cfc7fed31cb13881bb1a81dac65e971ca24929e6 Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Sun, 5 Mar 2017 23:12:27 -0500
Subject: [PATCH 123/132] test/pex: dial_seeds

---
 test/p2p/local_testnet_start.sh | 16 ++++++-----
 test/p2p/pex/dial_seeds.sh      | 31 ++++++++++++++++++++
 test/p2p/pex/test.sh            | 42 ++++-----------------------
 test/p2p/pex/test_addrbook.sh   | 51 +++++++++++++++++++++++++++++++++
 test/p2p/pex/test_dial_seeds.sh | 32 +++++++++++++++++++++
 test/p2p/seeds.sh               | 12 ++++++++
 test/p2p/test.sh                |  4 ++-
 7 files changed, 143 insertions(+), 45 deletions(-)
 create mode 100644 test/p2p/pex/dial_seeds.sh
 create mode 100644 test/p2p/pex/test_addrbook.sh
 create mode 100644 test/p2p/pex/test_dial_seeds.sh
 create mode 100644 test/p2p/seeds.sh

diff --git a/test/p2p/local_testnet_start.sh b/test/p2p/local_testnet_start.sh
index 3b842480..9a280c59 100644
--- a/test/p2p/local_testnet_start.sh
+++ b/test/p2p/local_testnet_start.sh
@@ -6,17 +6,19 @@ NETWORK_NAME=$2
 N=$3
 APP_PROXY=$4
 
+set +u
+SEEDS=$5
+if [[ "$SEEDS" != "" ]]; then
+	echo "Seeds: $SEEDS"
+	SEEDS="--seeds $SEEDS"
+fi
+set -u
+
 cd $GOPATH/src/github.com/tendermint/tendermint
 
 # create docker network
 docker network create --driver bridge --subnet 172.57.0.0/16 $NETWORK_NAME
 
-seeds="$(test/p2p/ip.sh 1):46656"
-for i in `seq 2 $N`; do
-	seeds="$seeds,$(test/p2p/ip.sh $i):46656"
-done
-echo "Seeds: $seeds"
-
 for i in `seq 1 $N`; do
-	bash test/p2p/peer.sh $DOCKER_IMAGE $NETWORK_NAME $i $APP_PROXY "--seeds $seeds --pex"
+	bash test/p2p/peer.sh $DOCKER_IMAGE $NETWORK_NAME $i $APP_PROXY "$SEEDS --pex"
 done
diff --git a/test/p2p/pex/dial_seeds.sh b/test/p2p/pex/dial_seeds.sh
new file mode 100644
index 00000000..15c22af6
--- /dev/null
+++ b/test/p2p/pex/dial_seeds.sh
@@ -0,0 +1,31 @@
+#! /bin/bash 
+set -u
+
+N=$1
+
+cd $GOPATH/src/github.com/tendermint/tendermint
+
+echo "Waiting for nodes to come online"
+for i in `seq 1 $N`; do
+	addr=$(test/p2p/ip.sh $i):46657
+	curl -s $addr/status > /dev/null
+	ERR=$?
+	while [ "$ERR" != 0 ]; do
+		sleep 1	
+		curl -s $addr/status > /dev/null
+		ERR=$?
+	done
+	echo "... node $i is up"
+done
+
+set -e
+# seeds need quotes
+seeds="\"$(test/p2p/ip.sh 1):46656\""
+for i in `seq 2 $N`; do
+	seeds="$seeds,\"$(test/p2p/ip.sh $i):46656\""
+done
+echo $seeds
+
+echo $seeds
+IP=$(test/p2p/ip.sh 1)
+curl --data-urlencode "seeds=[$seeds]" "$IP:46657/dial_seeds"
diff --git a/test/p2p/pex/test.sh b/test/p2p/pex/test.sh
index 3c624e27..d54d8135 100644
--- a/test/p2p/pex/test.sh
+++ b/test/p2p/pex/test.sh
@@ -6,42 +6,10 @@ NETWORK_NAME=$2
 N=$3
 PROXY_APP=$4
 
-ID=1
+cd $GOPATH/src/github.com/tendermint/tendermint
 
-echo "----------------------------------------------------------------------"
-echo "Testing pex creates the addrbook and uses it if seeds are not provided"
-echo "(assuming peers are started with pex enabled)"
+echo "Test reconnecting from the address book"
+bash test/p2p/pex/test_addrbook.sh $DOCKER_IMAGE $NETWORK_NAME $N $PROXY_APP
 
-echo "1. restart peer $ID"
-docker stop "local_testnet_$ID"
-# preserve addrbook.json
-docker cp "local_testnet_$ID:/go/src/github.com/tendermint/tendermint/test/p2p/data/mach1/core/addrbook.json" "/tmp/addrbook.json"
-docker rm -vf "local_testnet_$ID"
-
-# NOTE that we do not provide seeds
-bash test/p2p/peer.sh "$DOCKER_IMAGE" "$NETWORK_NAME" "$ID" "$PROXY_APP" "--pex"
-docker cp "/tmp/addrbook.json" "local_testnet_$ID:/go/src/github.com/tendermint/tendermint/test/p2p/data/mach1/core/addrbook.json"
-echo "with the following addrbook:"
-docker exec "local_testnet_$ID" cat "/go/src/github.com/tendermint/tendermint/test/p2p/data/mach1/core/addrbook.json"
-echo ""
-
-# if the client runs forever, it means addrbook wasn't saved or was empty
-bash test/p2p/client.sh "$DOCKER_IMAGE" "$NETWORK_NAME" "pex_$ID" "test/p2p/pex/check_peer.sh $ID $N"
-
-echo "----------------------------------------------------------------------"
-echo "Testing other peers connect to us if we have neither seeds nor the addrbook"
-echo "(assuming peers are started with pex enabled)"
-
-echo "1. restart peer $ID"
-docker stop "local_testnet_$ID"
-docker rm -vf "local_testnet_$ID"
-
-# NOTE that we do not provide seeds
-bash test/p2p/peer.sh "$DOCKER_IMAGE" "$NETWORK_NAME" "$ID" "$PROXY_APP" "--pex"
-
-# if the client runs forever, it means other peers have removed us from their books (which should not happen)
-bash test/p2p/client.sh "$DOCKER_IMAGE" "$NETWORK_NAME" "pex_$ID" "test/p2p/pex/check_peer.sh $ID $N"
-
-echo ""
-echo "PASS"
-echo ""
+echo "Test connecting via /dial_seeds"
+bash test/p2p/pex/test_dial_seeds.sh $DOCKER_IMAGE $NETWORK_NAME $N $PROXY_APP
diff --git a/test/p2p/pex/test_addrbook.sh b/test/p2p/pex/test_addrbook.sh
new file mode 100644
index 00000000..c472dff6
--- /dev/null
+++ b/test/p2p/pex/test_addrbook.sh
@@ -0,0 +1,51 @@
+#! /bin/bash
+set -eu
+
+DOCKER_IMAGE=$1
+NETWORK_NAME=$2
+N=$3
+PROXY_APP=$4
+
+ID=1
+
+echo "----------------------------------------------------------------------"
+echo "Testing pex creates the addrbook and uses it if seeds are not provided"
+echo "(assuming peers are started with pex enabled)"
+
+echo "1. restart peer $ID"
+docker stop "local_testnet_$ID"
+# preserve addrbook.json
+docker cp "local_testnet_$ID:/go/src/github.com/tendermint/tendermint/test/p2p/data/mach1/core/addrbook.json" "/tmp/addrbook.json"
+set +e #CIRCLE
+docker rm -vf "local_testnet_$ID"
+set -e
+
+# NOTE that we do not provide seeds
+bash test/p2p/peer.sh "$DOCKER_IMAGE" "$NETWORK_NAME" "$ID" "$PROXY_APP" "--pex"
+docker cp "/tmp/addrbook.json" "local_testnet_$ID:/go/src/github.com/tendermint/tendermint/test/p2p/data/mach1/core/addrbook.json"
+echo "with the following addrbook:"
+docker exec "local_testnet_$ID" cat "/go/src/github.com/tendermint/tendermint/test/p2p/data/mach1/core/addrbook.json"
+echo ""
+
+# if the client runs forever, it means addrbook wasn't saved or was empty
+bash test/p2p/client.sh "$DOCKER_IMAGE" "$NETWORK_NAME" "pex_$ID" "test/p2p/pex/check_peer.sh $ID $N"
+
+echo "----------------------------------------------------------------------"
+echo "Testing other peers connect to us if we have neither seeds nor the addrbook"
+echo "(assuming peers are started with pex enabled)"
+
+echo "1. restart peer $ID"
+docker stop "local_testnet_$ID"
+set +e #CIRCLE
+docker rm -vf "local_testnet_$ID"
+set -e 
+
+# NOTE that we do not provide seeds
+bash test/p2p/peer.sh "$DOCKER_IMAGE" "$NETWORK_NAME" "$ID" "$PROXY_APP" "--pex"
+
+# if the client runs forever, it means other peers have removed us from their books (which should not happen)
+bash test/p2p/client.sh "$DOCKER_IMAGE" "$NETWORK_NAME" "pex_$ID" "test/p2p/pex/check_peer.sh $ID $N"
+
+echo ""
+echo "PASS"
+echo ""
diff --git a/test/p2p/pex/test_dial_seeds.sh b/test/p2p/pex/test_dial_seeds.sh
new file mode 100644
index 00000000..5b0bfffb
--- /dev/null
+++ b/test/p2p/pex/test_dial_seeds.sh
@@ -0,0 +1,32 @@
+#! /bin/bash
+set -eu
+
+DOCKER_IMAGE=$1
+NETWORK_NAME=$2
+N=$3
+PROXY_APP=$4
+
+ID=1
+
+cd $GOPATH/src/github.com/tendermint/tendermint
+
+echo "----------------------------------------------------------------------"
+echo "Testing full network connection using one /dial_seeds call"
+echo "(assuming peers are started with pex enabled)"
+
+# stop the existing testnet and remove local network
+set +e
+bash test/p2p/local_testnet_stop.sh $NETWORK_NAME $N
+set -e
+
+# start the testnet on a local network
+# NOTE we re-use the same network for all tests
+SEEDS=""
+bash test/p2p/local_testnet_start.sh $DOCKER_IMAGE $NETWORK_NAME $N $PROXY_APP $SEEDS
+
+# dial seeds from one node
+bash test/p2p/client.sh $DOCKER_IMAGE $NETWORK_NAME dial_seeds "test/p2p/pex/dial_seeds.sh $N"
+
+# test basic connectivity and consensus
+# start client container and check the num peers and height for all nodes
+bash test/p2p/client.sh $DOCKER_IMAGE $NETWORK_NAME basic "test/p2p/basic/test.sh $N"
diff --git a/test/p2p/seeds.sh b/test/p2p/seeds.sh
new file mode 100644
index 00000000..471476ad
--- /dev/null
+++ b/test/p2p/seeds.sh
@@ -0,0 +1,12 @@
+#! /bin/bash
+set -eu
+
+N=$1
+
+cd $GOPATH/src/github.com/tendermint/tendermint
+
+seeds="$(test/p2p/ip.sh 1):46656"
+for i in `seq 2 $N`; do
+	seeds="$seeds,$(test/p2p/ip.sh $i):46656"
+done
+echo $seeds
diff --git a/test/p2p/test.sh b/test/p2p/test.sh
index 1157ac06..0522a422 100644
--- a/test/p2p/test.sh
+++ b/test/p2p/test.sh
@@ -13,9 +13,11 @@ set +e
 bash test/p2p/local_testnet_stop.sh $NETWORK_NAME $N
 set -e
 
+SEEDS=$(bash test/p2p/seeds.sh $N)
+
 # start the testnet on a local network
 # NOTE we re-use the same network for all tests
-bash test/p2p/local_testnet_start.sh $DOCKER_IMAGE $NETWORK_NAME $N $PROXY_APP
+bash test/p2p/local_testnet_start.sh $DOCKER_IMAGE $NETWORK_NAME $N $PROXY_APP $SEEDS
 
 # test basic connectivity and consensus
 # start client container and check the num peers and height for all nodes

From 1208296dc0d1338359fc1a64ab5c47a3e894afd6 Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Sun, 5 Mar 2017 23:13:34 -0500
Subject: [PATCH 124/132] DialSeeds takes an AddrBook

---
 glide.lock       |  4 ++--
 node/node.go     | 42 ++++++++++++------------------------------
 rpc/core/net.go  |  3 ++-
 rpc/core/pipe.go | 11 ++++++++---
 4 files changed, 24 insertions(+), 36 deletions(-)

diff --git a/glide.lock b/glide.lock
index a3ce63dd..216871f8 100644
--- a/glide.lock
+++ b/glide.lock
@@ -1,5 +1,5 @@
 hash: 41f8fec708e98b7f8c4804be46008493199fa45e89b2d5dc237fd65fe431c62f
-updated: 2017-03-05T00:18:48.136517571-05:00
+updated: 2017-03-05T22:59:46.61454297-05:00
 imports:
 - name: github.com/btcsuite/btcd
   version: d06c0bb181529331be8f8d9350288c420d9e60e4
@@ -101,7 +101,7 @@ imports:
 - name: github.com/tendermint/go-merkle
   version: 714d4d04557fd068a7c2a1748241ce8428015a96
 - name: github.com/tendermint/go-p2p
-  version: 56eebb95eede5f7cf742ecaefa83bed884df99ec
+  version: beb3eda438fbbfc3b566ead4be5705474a12e50a
   subpackages:
   - upnp
 - name: github.com/tendermint/go-rpc
diff --git a/node/node.go b/node/node.go
index 2e817766..f9c69c28 100644
--- a/node/node.go
+++ b/node/node.go
@@ -211,28 +211,9 @@ func (n *Node) OnStart() error {
 
 	// If seeds exist, add them to the address book and dial out
 	if n.config.GetString("seeds") != "" {
-		seeds := strings.Split(n.config.GetString("seeds"), ",")
-
-		if n.config.GetBool("pex_reactor") {
-			// add seeds to `addrBook` to avoid losing
-			ourAddrS := n.NodeInfo().ListenAddr
-			ourAddr, _ := p2p.NewNetAddressString(ourAddrS)
-			for _, s := range seeds {
-				// do not add ourselves
-				if s == ourAddrS {
-					continue
-				}
-
-				addr, err := p2p.NewNetAddressString(s)
-				if err != nil {
-					n.addrBook.AddAddress(addr, ourAddr)
-				}
-			}
-			n.addrBook.Save()
-		}
-
 		// dial out
-		if err := n.sw.DialSeeds(seeds); err != nil {
+		seeds := strings.Split(n.config.GetString("seeds"), ",")
+		if err := n.DialSeeds(seeds); err != nil {
 			return err
 		}
 	}
@@ -249,13 +230,6 @@ func (n *Node) OnStart() error {
 	return nil
 }
 
-func (n *Node) RunForever() {
-	// Sleep forever and then...
-	cmn.TrapSignal(func() {
-		n.Stop()
-	})
-}
-
 func (n *Node) OnStop() {
 	n.BaseService.OnStop()
 
@@ -271,6 +245,13 @@ func (n *Node) OnStop() {
 	}
 }
 
+func (n *Node) RunForever() {
+	// Sleep forever and then...
+	cmn.TrapSignal(func() {
+		n.Stop()
+	})
+}
+
 // Add the event switch to reactors, mempool, etc.
 func SetEventSwitch(evsw types.EventSwitch, eventables ...types.Eventable) {
 	for _, e := range eventables {
@@ -296,6 +277,7 @@ func (n *Node) ConfigureRPC() {
 	rpccore.SetSwitch(n.sw)
 	rpccore.SetPubKey(n.privValidator.PubKey)
 	rpccore.SetGenesisDoc(n.genesisDoc)
+	rpccore.SetAddrBook(n.addrBook)
 	rpccore.SetProxyAppQuery(n.proxyApp.Query())
 }
 
@@ -410,8 +392,8 @@ func (n *Node) NodeInfo() *p2p.NodeInfo {
 	return n.sw.NodeInfo()
 }
 
-func (n *Node) DialSeeds(seeds []string) {
-	n.sw.DialSeeds(seeds)
+func (n *Node) DialSeeds(seeds []string) error {
+	return n.sw.DialSeeds(n.addrBook, seeds)
 }
 
 // Defaults to tcp
diff --git a/rpc/core/net.go b/rpc/core/net.go
index 3e9526d2..df491ba8 100644
--- a/rpc/core/net.go
+++ b/rpc/core/net.go
@@ -32,7 +32,8 @@ func NetInfo() (*ctypes.ResultNetInfo, error) {
 // Dial given list of seeds
 func UnsafeDialSeeds(seeds []string) (*ctypes.ResultDialSeeds, error) {
 	// starts go routines to dial each seed after random delays
-	err := p2pSwitch.DialSeeds(seeds)
+	log.Info("DialSeeds", "addrBook", addrBook, "seeds", seeds)
+	err := p2pSwitch.DialSeeds(addrBook, seeds)
 	return &ctypes.ResultDialSeeds{}, err
 }
 
diff --git a/rpc/core/pipe.go b/rpc/core/pipe.go
index fae2630c..123b13dd 100644
--- a/rpc/core/pipe.go
+++ b/rpc/core/pipe.go
@@ -24,7 +24,7 @@ type P2P interface {
 	NumPeers() (outbound, inbound, dialig int)
 	NodeInfo() *p2p.NodeInfo
 	IsListening() bool
-	DialSeeds([]string) error
+	DialSeeds(*p2p.AddrBook, []string) error
 }
 
 //----------------------------------------------
@@ -42,8 +42,9 @@ var (
 	p2pSwitch      P2P
 
 	// objects
-	pubKey crypto.PubKey
-	genDoc *types.GenesisDoc // cache the genesis structure
+	pubKey   crypto.PubKey
+	genDoc   *types.GenesisDoc // cache the genesis structure
+	addrBook *p2p.AddrBook
 )
 
 func SetConfig(c cfg.Config) {
@@ -78,6 +79,10 @@ func SetGenesisDoc(doc *types.GenesisDoc) {
 	genDoc = doc
 }
 
+func SetAddrBook(book *p2p.AddrBook) {
+	addrBook = book
+}
+
 func SetProxyAppQuery(appConn proxy.AppConnQuery) {
 	proxyAppQuery = appConn
 }

From b1cd67771185329ec7688fdd3f65f11a55d0e7b3 Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Sun, 5 Mar 2017 23:28:42 -0500
Subject: [PATCH 125/132] types: valSet LastProposer->Proposer and
 Proposer()->GetProposer()

---
 consensus/state.go          |  8 ++++----
 consensus/state_test.go     |  6 +++---
 types/validator_set.go      | 28 ++++++++++++++--------------
 types/validator_set_test.go | 22 +++++++++++-----------
 4 files changed, 32 insertions(+), 32 deletions(-)

diff --git a/consensus/state.go b/consensus/state.go
index 3077439e..23eaff74 100644
--- a/consensus/state.go
+++ b/consensus/state.go
@@ -806,10 +806,10 @@ func (cs *ConsensusState) enterPropose(height int, round int) {
 		return
 	}
 
-	if !bytes.Equal(cs.Validators.Proposer().Address, cs.privValidator.GetAddress()) {
-		log.Info("enterPropose: Not our turn to propose", "proposer", cs.Validators.Proposer().Address, "privValidator", cs.privValidator)
+	if !bytes.Equal(cs.Validators.GetProposer().Address, cs.privValidator.GetAddress()) {
+		log.Info("enterPropose: Not our turn to propose", "proposer", cs.Validators.GetProposer().Address, "privValidator", cs.privValidator)
 	} else {
-		log.Info("enterPropose: Our turn to propose", "proposer", cs.Validators.Proposer().Address, "privValidator", cs.privValidator)
+		log.Info("enterPropose: Our turn to propose", "proposer", cs.Validators.GetProposer().Address, "privValidator", cs.privValidator)
 		cs.decideProposal(height, round)
 
 	}
@@ -1283,7 +1283,7 @@ func (cs *ConsensusState) defaultSetProposal(proposal *types.Proposal) error {
 	}
 
 	// Verify signature
-	if !cs.Validators.Proposer().PubKey.VerifyBytes(types.SignBytes(cs.state.ChainID, proposal), proposal.Signature) {
+	if !cs.Validators.GetProposer().PubKey.VerifyBytes(types.SignBytes(cs.state.ChainID, proposal), proposal.Signature) {
 		return ErrInvalidProposalSignature
 	}
 
diff --git a/consensus/state_test.go b/consensus/state_test.go
index d3eef59f..b7d9a42d 100644
--- a/consensus/state_test.go
+++ b/consensus/state_test.go
@@ -65,7 +65,7 @@ func TestProposerSelection0(t *testing.T) {
 	<-newRoundCh
 
 	// lets commit a block and ensure proposer for the next height is correct
-	prop := cs1.GetRoundState().Validators.Proposer()
+	prop := cs1.GetRoundState().Validators.GetProposer()
 	if !bytes.Equal(prop.Address, cs1.privValidator.GetAddress()) {
 		t.Fatalf("expected proposer to be validator %d. Got %X", 0, prop.Address)
 	}
@@ -79,7 +79,7 @@ func TestProposerSelection0(t *testing.T) {
 	// wait for new round so next validator is set
 	<-newRoundCh
 
-	prop = cs1.GetRoundState().Validators.Proposer()
+	prop = cs1.GetRoundState().Validators.GetProposer()
 	if !bytes.Equal(prop.Address, vss[1].Address) {
 		panic(Fmt("expected proposer to be validator %d. Got %X", 1, prop.Address))
 	}
@@ -100,7 +100,7 @@ func TestProposerSelection2(t *testing.T) {
 
 	// everyone just votes nil. we get a new proposer each round
 	for i := 0; i < len(vss); i++ {
-		prop := cs1.GetRoundState().Validators.Proposer()
+		prop := cs1.GetRoundState().Validators.GetProposer()
 		if !bytes.Equal(prop.Address, vss[(i+2)%len(vss)].Address) {
 			panic(Fmt("expected proposer to be validator %d. Got %X", (i+2)%len(vss), prop.Address))
 		}
diff --git a/types/validator_set.go b/types/validator_set.go
index 7d872450..b997b471 100644
--- a/types/validator_set.go
+++ b/types/validator_set.go
@@ -16,15 +16,15 @@ import (
 // The index is in order of .Address, so the indices are fixed
 // for all rounds of a given blockchain height.
 // On the other hand, the .AccumPower of each validator and
-// the designated .Proposer() of a set changes every round,
+// the designated .GetProposer() of a set changes every round,
 // upon calling .IncrementAccum().
 // NOTE: Not goroutine-safe.
 // NOTE: All get/set to validators should copy the value for safety.
 // TODO: consider validator Accum overflow
 // TODO: move valset into an iavl tree where key is 'blockbonded|pubkey'
 type ValidatorSet struct {
-	Validators   []*Validator // NOTE: persisted via reflect, must be exported.
-	LastProposer *Validator
+	Validators []*Validator // NOTE: persisted via reflect, must be exported.
+	Proposer   *Validator
 
 	// cached (unexported)
 	totalVotingPower int64
@@ -61,7 +61,7 @@ func (valSet *ValidatorSet) IncrementAccum(times int) {
 	for i := 0; i < times; i++ {
 		mostest := validatorsHeap.Peek().(*Validator)
 		if i == times-1 {
-			valSet.LastProposer = mostest
+			valSet.Proposer = mostest
 		}
 		mostest.Accum -= int64(valSet.TotalVotingPower())
 		validatorsHeap.Update(mostest, accumComparable{mostest})
@@ -76,7 +76,7 @@ func (valSet *ValidatorSet) Copy() *ValidatorSet {
 	}
 	return &ValidatorSet{
 		Validators:       validators,
-		LastProposer:     valSet.LastProposer,
+		Proposer:         valSet.Proposer,
 		totalVotingPower: valSet.totalVotingPower,
 	}
 }
@@ -117,14 +117,14 @@ func (valSet *ValidatorSet) TotalVotingPower() int64 {
 	return valSet.totalVotingPower
 }
 
-func (valSet *ValidatorSet) Proposer() (proposer *Validator) {
+func (valSet *ValidatorSet) GetProposer() (proposer *Validator) {
 	if len(valSet.Validators) == 0 {
 		return nil
 	}
-	if valSet.LastProposer == nil {
-		valSet.LastProposer = valSet.findProposer()
+	if valSet.Proposer == nil {
+		valSet.Proposer = valSet.findProposer()
 	}
-	return valSet.LastProposer.Copy()
+	return valSet.Proposer.Copy()
 }
 
 func (valSet *ValidatorSet) findProposer() *Validator {
@@ -156,7 +156,7 @@ func (valSet *ValidatorSet) Add(val *Validator) (added bool) {
 	if idx == len(valSet.Validators) {
 		valSet.Validators = append(valSet.Validators, val)
 		// Invalidate cache
-		valSet.LastProposer = nil
+		valSet.Proposer = nil
 		valSet.totalVotingPower = 0
 		return true
 	} else if bytes.Compare(valSet.Validators[idx].Address, val.Address) == 0 {
@@ -168,7 +168,7 @@ func (valSet *ValidatorSet) Add(val *Validator) (added bool) {
 		copy(newValidators[idx+1:], valSet.Validators[idx:])
 		valSet.Validators = newValidators
 		// Invalidate cache
-		valSet.LastProposer = nil
+		valSet.Proposer = nil
 		valSet.totalVotingPower = 0
 		return true
 	}
@@ -181,7 +181,7 @@ func (valSet *ValidatorSet) Update(val *Validator) (updated bool) {
 	} else {
 		valSet.Validators[index] = val.Copy()
 		// Invalidate cache
-		valSet.LastProposer = nil
+		valSet.Proposer = nil
 		valSet.totalVotingPower = 0
 		return true
 	}
@@ -201,7 +201,7 @@ func (valSet *ValidatorSet) Remove(address []byte) (val *Validator, removed bool
 		}
 		valSet.Validators = newValidators
 		// Invalidate cache
-		valSet.LastProposer = nil
+		valSet.Proposer = nil
 		valSet.totalVotingPower = 0
 		return removedVal, true
 	}
@@ -325,7 +325,7 @@ func (valSet *ValidatorSet) StringIndented(indent string) string {
 %s  Validators:
 %s    %v
 %s}`,
-		indent, valSet.Proposer().String(),
+		indent, valSet.GetProposer().String(),
 		indent,
 		indent, strings.Join(valStrings, "\n"+indent+"    "),
 		indent)
diff --git a/types/validator_set_test.go b/types/validator_set_test.go
index c37fc568..bc7fef79 100644
--- a/types/validator_set_test.go
+++ b/types/validator_set_test.go
@@ -52,7 +52,7 @@ func TestProposerSelection1(t *testing.T) {
 	})
 	proposers := []string{}
 	for i := 0; i < 99; i++ {
-		val := vset.Proposer()
+		val := vset.GetProposer()
 		proposers = append(proposers, string(val.Address))
 		vset.IncrementAccum(1)
 	}
@@ -77,7 +77,7 @@ func TestProposerSelection2(t *testing.T) {
 	vals := NewValidatorSet(valList)
 	for i := 0; i < len(valList)*5; i++ {
 		ii := (i) % len(valList)
-		prop := vals.Proposer()
+		prop := vals.GetProposer()
 		if !bytes.Equal(prop.Address, valList[ii].Address) {
 			t.Fatalf("(%d): Expected %X. Got %X", i, valList[ii].Address, prop.Address)
 		}
@@ -88,12 +88,12 @@ func TestProposerSelection2(t *testing.T) {
 	*val2 = *newValidator(addr2, 400)
 	vals = NewValidatorSet(valList)
 	// vals.IncrementAccum(1)
-	prop := vals.Proposer()
+	prop := vals.GetProposer()
 	if !bytes.Equal(prop.Address, addr2) {
 		t.Fatalf("Expected address with highest voting power to be first proposer. Got %X", prop.Address)
 	}
 	vals.IncrementAccum(1)
-	prop = vals.Proposer()
+	prop = vals.GetProposer()
 	if !bytes.Equal(prop.Address, addr0) {
 		t.Fatalf("Expected smallest address to be validator. Got %X", prop.Address)
 	}
@@ -101,17 +101,17 @@ func TestProposerSelection2(t *testing.T) {
 	// One validator has more than the others, and enough to be proposer twice in a row
 	*val2 = *newValidator(addr2, 401)
 	vals = NewValidatorSet(valList)
-	prop = vals.Proposer()
+	prop = vals.GetProposer()
 	if !bytes.Equal(prop.Address, addr2) {
 		t.Fatalf("Expected address with highest voting power to be first proposer. Got %X", prop.Address)
 	}
 	vals.IncrementAccum(1)
-	prop = vals.Proposer()
+	prop = vals.GetProposer()
 	if !bytes.Equal(prop.Address, addr2) {
 		t.Fatalf("Expected address with highest voting power to be second proposer. Got %X", prop.Address)
 	}
 	vals.IncrementAccum(1)
-	prop = vals.Proposer()
+	prop = vals.GetProposer()
 	if !bytes.Equal(prop.Address, addr0) {
 		t.Fatalf("Expected smallest address to be validator. Got %X", prop.Address)
 	}
@@ -123,7 +123,7 @@ func TestProposerSelection2(t *testing.T) {
 	vals = NewValidatorSet(valList)
 	N := 1
 	for i := 0; i < 120*N; i++ {
-		prop := vals.Proposer()
+		prop := vals.GetProposer()
 		ii := prop.Address[19]
 		propCount[ii] += 1
 		vals.IncrementAccum(1)
@@ -150,7 +150,7 @@ func TestProposerSelection3(t *testing.T) {
 
 	proposerOrder := make([]*Validator, 4)
 	for i := 0; i < 4; i++ {
-		proposerOrder[i] = vset.Proposer()
+		proposerOrder[i] = vset.GetProposer()
 		vset.IncrementAccum(1)
 	}
 
@@ -159,7 +159,7 @@ func TestProposerSelection3(t *testing.T) {
 	// we should go in order for ever, despite some IncrementAccums with times > 1
 	var i, j int
 	for ; i < 10000; i++ {
-		got := vset.Proposer().Address
+		got := vset.GetProposer().Address
 		expected := proposerOrder[j%4].Address
 		if !bytes.Equal(got, expected) {
 			t.Fatalf(cmn.Fmt("vset.Proposer (%X) does not match expected proposer (%X) for (%d, %d)", got, expected, i, j))
@@ -169,7 +169,7 @@ func TestProposerSelection3(t *testing.T) {
 		b := vset.ToBytes()
 		vset.FromBytes(b)
 
-		computed := vset.Proposer() // findProposer()
+		computed := vset.GetProposer() // findGetProposer()
 		if i != 0 {
 			if !bytes.Equal(got, computed.Address) {
 				t.Fatalf(cmn.Fmt("vset.Proposer (%X) does not match computed proposer (%X) for (%d, %d)", got, computed.Address, i, j))

From 22d95c7b5196fed5bd41e9731619c8e1bda30373 Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Sun, 5 Mar 2017 23:38:11 -0500
Subject: [PATCH 126/132] test: docker exec doesnt work on circle

---
 test/p2p/pex/test_addrbook.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/test/p2p/pex/test_addrbook.sh b/test/p2p/pex/test_addrbook.sh
index c472dff6..0fd02fa9 100644
--- a/test/p2p/pex/test_addrbook.sh
+++ b/test/p2p/pex/test_addrbook.sh
@@ -24,7 +24,9 @@ set -e
 bash test/p2p/peer.sh "$DOCKER_IMAGE" "$NETWORK_NAME" "$ID" "$PROXY_APP" "--pex"
 docker cp "/tmp/addrbook.json" "local_testnet_$ID:/go/src/github.com/tendermint/tendermint/test/p2p/data/mach1/core/addrbook.json"
 echo "with the following addrbook:"
-docker exec "local_testnet_$ID" cat "/go/src/github.com/tendermint/tendermint/test/p2p/data/mach1/core/addrbook.json"
+cat /tmp/addrbook.json
+# exec doesn't work on circle
+# docker exec "local_testnet_$ID" cat "/go/src/github.com/tendermint/tendermint/test/p2p/data/mach1/core/addrbook.json"
 echo ""
 
 # if the client runs forever, it means addrbook wasn't saved or was empty

From d58a666445b7d24d6ca95f8956267860ace0e010 Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Mon, 6 Mar 2017 00:32:15 -0500
Subject: [PATCH 127/132] test/persist: bump sleep to 5 for bound ports release

---
 test/persist/test_failure_indices.sh | 2 +-
 test/test.sh                         | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 mode change 100644 => 100755 test/test.sh

diff --git a/test/persist/test_failure_indices.sh b/test/persist/test_failure_indices.sh
index 0d0b3b2b..42a76cb8 100644
--- a/test/persist/test_failure_indices.sh
+++ b/test/persist/test_failure_indices.sh
@@ -49,7 +49,7 @@ function kill_procs(){
 	wait_for_port 46657
 	
 	# XXX: sometimes the port is still bound :(
-	sleep 2
+	sleep 5
 }
 
 # wait for port to be available
diff --git a/test/test.sh b/test/test.sh
old mode 100644
new mode 100755

From fc6d22db320c71427d3f45c20b6b5ef408d35740 Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Mon, 6 Mar 2017 01:04:55 -0500
Subject: [PATCH 128/132] test: better client naming

---
 test/p2p/pex/test_addrbook.sh        | 8 ++++++--
 test/p2p/pex/test_dial_seeds.sh      | 8 ++++++--
 test/p2p/test.sh                     | 4 ++--
 test/persist/test_failure_indices.sh | 2 +-
 4 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/test/p2p/pex/test_addrbook.sh b/test/p2p/pex/test_addrbook.sh
index 0fd02fa9..d63096c8 100644
--- a/test/p2p/pex/test_addrbook.sh
+++ b/test/p2p/pex/test_addrbook.sh
@@ -12,6 +12,8 @@ echo "----------------------------------------------------------------------"
 echo "Testing pex creates the addrbook and uses it if seeds are not provided"
 echo "(assuming peers are started with pex enabled)"
 
+CLIENT_NAME="pex_addrbook_$ID"
+
 echo "1. restart peer $ID"
 docker stop "local_testnet_$ID"
 # preserve addrbook.json
@@ -30,12 +32,14 @@ cat /tmp/addrbook.json
 echo ""
 
 # if the client runs forever, it means addrbook wasn't saved or was empty
-bash test/p2p/client.sh "$DOCKER_IMAGE" "$NETWORK_NAME" "pex_$ID" "test/p2p/pex/check_peer.sh $ID $N"
+bash test/p2p/client.sh "$DOCKER_IMAGE" "$NETWORK_NAME" "$CLIENT_NAME" "test/p2p/pex/check_peer.sh $ID $N"
 
 echo "----------------------------------------------------------------------"
 echo "Testing other peers connect to us if we have neither seeds nor the addrbook"
 echo "(assuming peers are started with pex enabled)"
 
+CLIENT_NAME="pex_no_addrbook_$ID"
+
 echo "1. restart peer $ID"
 docker stop "local_testnet_$ID"
 set +e #CIRCLE
@@ -46,7 +50,7 @@ set -e
 bash test/p2p/peer.sh "$DOCKER_IMAGE" "$NETWORK_NAME" "$ID" "$PROXY_APP" "--pex"
 
 # if the client runs forever, it means other peers have removed us from their books (which should not happen)
-bash test/p2p/client.sh "$DOCKER_IMAGE" "$NETWORK_NAME" "pex_$ID" "test/p2p/pex/check_peer.sh $ID $N"
+bash test/p2p/client.sh "$DOCKER_IMAGE" "$NETWORK_NAME" "$CLIENT_NAME" "test/p2p/pex/check_peer.sh $ID $N"
 
 echo ""
 echo "PASS"
diff --git a/test/p2p/pex/test_dial_seeds.sh b/test/p2p/pex/test_dial_seeds.sh
index 5b0bfffb..ea72004d 100644
--- a/test/p2p/pex/test_dial_seeds.sh
+++ b/test/p2p/pex/test_dial_seeds.sh
@@ -24,9 +24,13 @@ set -e
 SEEDS=""
 bash test/p2p/local_testnet_start.sh $DOCKER_IMAGE $NETWORK_NAME $N $PROXY_APP $SEEDS
 
+
+
 # dial seeds from one node
-bash test/p2p/client.sh $DOCKER_IMAGE $NETWORK_NAME dial_seeds "test/p2p/pex/dial_seeds.sh $N"
+CLIENT_NAME="dial_seeds"
+bash test/p2p/client.sh $DOCKER_IMAGE $NETWORK_NAME $CLIENT_NAME "test/p2p/pex/dial_seeds.sh $N"
 
 # test basic connectivity and consensus
 # start client container and check the num peers and height for all nodes
-bash test/p2p/client.sh $DOCKER_IMAGE $NETWORK_NAME basic "test/p2p/basic/test.sh $N"
+CLIENT_NAME="dial_seeds_basic"
+bash test/p2p/client.sh $DOCKER_IMAGE $NETWORK_NAME $CLIENT_NAME "test/p2p/basic/test.sh $N"
diff --git a/test/p2p/test.sh b/test/p2p/test.sh
index 0522a422..3e67dfbd 100644
--- a/test/p2p/test.sh
+++ b/test/p2p/test.sh
@@ -27,8 +27,8 @@ bash test/p2p/client.sh $DOCKER_IMAGE $NETWORK_NAME basic "test/p2p/basic/test.s
 # start client container and test sending a tx to each node
 bash test/p2p/client.sh $DOCKER_IMAGE $NETWORK_NAME ab "test/p2p/atomic_broadcast/test.sh $N"
 
-# test fast sync (from current state of network):
-# for each node, kill it and readd via fast sync
+ test fast sync (from current state of network):
+ for each node, kill it and readd via fast sync
 bash test/p2p/fast_sync/test.sh $DOCKER_IMAGE $NETWORK_NAME $N $PROXY_APP
 
 # test killing all peers 3 times
diff --git a/test/persist/test_failure_indices.sh b/test/persist/test_failure_indices.sh
index 42a76cb8..cb78297e 100644
--- a/test/persist/test_failure_indices.sh
+++ b/test/persist/test_failure_indices.sh
@@ -102,7 +102,7 @@ for failIndex in $(seq $failsStart $failsEnd); do
 		curl -s "$addr/status" > /dev/null
 		ERR=$?
 		i=$((i + 1))
-		if [[ $i == 10 ]]; then
+		if [[ $i == 20 ]]; then
 			echo "Timed out waiting for tendermint to start"
 			exit 1
 		fi

From caaafa192bfe41f7c45929d626b53e5599d2c720 Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Mon, 6 Mar 2017 02:42:00 -0500
Subject: [PATCH 129/132] test/persist: use unix socket for rpc

---
 test/docker/Dockerfile               |  5 ++++-
 test/persist/test_failure_indices.sh | 25 +++++++++++--------------
 2 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/test/docker/Dockerfile b/test/docker/Dockerfile
index 829f5e45..7c322976 100644
--- a/test/docker/Dockerfile
+++ b/test/docker/Dockerfile
@@ -1,10 +1,13 @@
 # Pull base image.
 FROM golang:1.7.4
 
+# Add testing deps for curl
+RUN echo 'deb http://httpredir.debian.org/debian testing main non-free contrib' >> /etc/apt/sources.list
+
 # Grab deps (jq, hexdump, xxd, killall)
 RUN apt-get update && \
   apt-get install -y --no-install-recommends \
-  jq bsdmainutils vim-common psmisc netcat
+  jq bsdmainutils vim-common psmisc netcat curl
 
 # Setup tendermint repo 
 ENV REPO $GOPATH/src/github.com/tendermint/tendermint
diff --git a/test/persist/test_failure_indices.sh b/test/persist/test_failure_indices.sh
index cb78297e..f47ab372 100644
--- a/test/persist/test_failure_indices.sh
+++ b/test/persist/test_failure_indices.sh
@@ -1,12 +1,14 @@
 #! /bin/bash
 
-
 export TMROOT=$HOME/.tendermint_persist
 
 rm -rf "$TMROOT"
 tendermint init
 
-TM_CMD="tendermint node --log_level=debug" # &> tendermint_${name}.log"
+# use a unix socket so we can remove it
+RPC_ADDR="$(pwd)/rpc.sock"
+
+TM_CMD="tendermint node --log_level=debug --rpc_laddr=unix://$RPC_ADDR" # &> tendermint_${name}.log"
 DUMMY_CMD="dummy --persist $TMROOT/dummy" # &> dummy_${name}.log"
 
 
@@ -20,6 +22,9 @@ function start_procs(){
 		$DUMMY_CMD &> "dummy_${name}.log" &
 	fi
 	PID_DUMMY=$!
+
+	# before starting tendermint, remove the rpc socket
+	rm $RPC_ADDR
 	if [[ "$indexToFail" == "" ]]; then
 		# run in background, dont fail
 		if [[ "$CIRCLECI" == true ]]; then
@@ -43,13 +48,6 @@ function kill_procs(){
 	kill -9 "$PID_DUMMY" "$PID_TENDERMINT"
 	wait "$PID_DUMMY"
 	wait "$PID_TENDERMINT"
-
-	# wait for the ports to be released 
-	wait_for_port 46656
-	wait_for_port 46657
-	
-	# XXX: sometimes the port is still bound :(
-	sleep 5
 }
 
 # wait for port to be available
@@ -93,13 +91,12 @@ for failIndex in $(seq $failsStart $failsEnd); do
 	start_procs 2
 
 	# wait for node to handshake and make a new block
-	addr="localhost:46657"
-	curl -s "$addr/status" > /dev/null
+	curl -s --unix-socket "$RPC_ADDR" http://localhost/status > /dev/null
 	ERR=$?
 	i=0
 	while [ "$ERR" != 0 ]; do
 		sleep 1	
-		curl -s "$addr/status" > /dev/null
+		curl -s --unix-socket "$RPC_ADDR" http://localhost/status > /dev/null
 		ERR=$?
 		i=$((i + 1))
 		if [[ $i == 20 ]]; then
@@ -109,11 +106,11 @@ for failIndex in $(seq $failsStart $failsEnd); do
 	done
 
 	# wait for a new block
-	h1=$(curl -s $addr/status | jq .result[1].latest_block_height)
+	h1=$(curl -s --unix-socket "$RPC_ADDR" http://localhost/status | jq .result[1].latest_block_height)
 	h2=$h1
 	while [ "$h2" == "$h1" ]; do
 		sleep 1
-		h2=$(curl -s $addr/status | jq .result[1].latest_block_height)
+		h2=$(curl -s --unix-socket "$RPC_ADDR" http://localhost/status | jq .result[1].latest_block_height)
 	done
 
 	kill_procs

From 097da55a2ccbb21d9c9fdbd391df70e737469629 Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Mon, 6 Mar 2017 03:08:04 -0500
Subject: [PATCH 130/132] test/p2p: shellcheck

---
 test/p2p/client.sh              |  8 ++++----
 test/p2p/local_testnet_start.sh |  8 ++++----
 test/p2p/local_testnet_stop.sh  |  8 ++++----
 test/p2p/peer.sh                | 16 ++++++++--------
 test/p2p/seeds.sh               |  6 +++---
 test/p2p/test.sh                | 20 ++++++++++----------
 6 files changed, 33 insertions(+), 33 deletions(-)

diff --git a/test/p2p/client.sh b/test/p2p/client.sh
index f3025ba5..fa11ce87 100644
--- a/test/p2p/client.sh
+++ b/test/p2p/client.sh
@@ -11,9 +11,9 @@ NAME=test_container_$ID
 echo "starting test client container with CMD=$CMD"
 # run the test container on the local network
 docker run -t --rm \
-	-v $GOPATH/src/github.com/tendermint/tendermint/test/p2p/:/go/src/github.com/tendermint/tendermint/test/p2p \
-	--net=$NETWORK_NAME \
+	-v "$GOPATH/src/github.com/tendermint/tendermint/test/p2p/:/go/src/github.com/tendermint/tendermint/test/p2p" \
+	--net="$NETWORK_NAME" \
 	--ip=$(test/p2p/ip.sh "-1") \
-	--name $NAME \
+	--name "$NAME" \
 	--entrypoint bash \
-	$DOCKER_IMAGE $CMD
+	"$DOCKER_IMAGE" $CMD
diff --git a/test/p2p/local_testnet_start.sh b/test/p2p/local_testnet_start.sh
index 9a280c59..b7cfb959 100644
--- a/test/p2p/local_testnet_start.sh
+++ b/test/p2p/local_testnet_start.sh
@@ -14,11 +14,11 @@ if [[ "$SEEDS" != "" ]]; then
 fi
 set -u
 
-cd $GOPATH/src/github.com/tendermint/tendermint
+cd "$GOPATH/src/github.com/tendermint/tendermint"
 
 # create docker network
-docker network create --driver bridge --subnet 172.57.0.0/16 $NETWORK_NAME
+docker network create --driver bridge --subnet 172.57.0.0/16 "$NETWORK_NAME"
 
-for i in `seq 1 $N`; do
-	bash test/p2p/peer.sh $DOCKER_IMAGE $NETWORK_NAME $i $APP_PROXY "$SEEDS --pex"
+for i in $(seq 1 "$N"); do
+	bash test/p2p/peer.sh "$DOCKER_IMAGE" "$NETWORK_NAME" "$i" "$APP_PROXY" "$SEEDS --pex"
 done
diff --git a/test/p2p/local_testnet_stop.sh b/test/p2p/local_testnet_stop.sh
index 8edd3eeb..1dace469 100644
--- a/test/p2p/local_testnet_stop.sh
+++ b/test/p2p/local_testnet_stop.sh
@@ -4,9 +4,9 @@ set -u
 NETWORK_NAME=$1
 N=$2
 
-for i in `seq 1 $N`; do
-  docker stop local_testnet_$i
-  docker rm -vf local_testnet_$i
+for i in $(seq 1 "$N"); do
+  docker stop "local_testnet_$i"
+  docker rm -vf "local_testnet_$i"
 done
 
-docker network rm $NETWORK_NAME
+docker network rm "$NETWORK_NAME"
diff --git a/test/p2p/peer.sh b/test/p2p/peer.sh
index adeb01a1..5299a3c6 100644
--- a/test/p2p/peer.sh
+++ b/test/p2p/peer.sh
@@ -17,23 +17,23 @@ echo "starting tendermint peer ID=$ID"
 if [[ "$CIRCLECI" == true ]]; then
 	set -u
 	docker run -d \
-	  --net=$NETWORK_NAME \
+	  --net="$NETWORK_NAME" \
 	  --ip=$(test/p2p/ip.sh $ID) \
-	  --name local_testnet_$ID \
+	  --name "local_testnet_$ID" \
 	  --entrypoint tendermint \
-	  -e TMROOT=/go/src/github.com/tendermint/tendermint/test/p2p/data/mach$ID/core \
+	  -e TMROOT="/go/src/github.com/tendermint/tendermint/test/p2p/data/mach$ID/core" \
 		--log-driver=syslog \
 		--log-opt syslog-address=udp://127.0.0.1:5514 \
 		--log-opt syslog-facility=daemon \
 		--log-opt tag="{{.Name}}" \
-  	  $DOCKER_IMAGE node $NODE_FLAGS --log_level=debug --proxy_app=$APP_PROXY
+  	  "$DOCKER_IMAGE" node $NODE_FLAGS --log_level=debug --proxy_app="$APP_PROXY"
 else
 	set -u
 	docker run -d \
-	  --net=$NETWORK_NAME \
+	  --net="$NETWORK_NAME" \
 	  --ip=$(test/p2p/ip.sh $ID) \
-	  --name local_testnet_$ID \
+	  --name "local_testnet_$ID" \
 	  --entrypoint tendermint \
-	  -e TMROOT=/go/src/github.com/tendermint/tendermint/test/p2p/data/mach$ID/core \
-  	  $DOCKER_IMAGE node $NODE_FLAGS --log_level=info --proxy_app=$APP_PROXY
+	  -e TMROOT="/go/src/github.com/tendermint/tendermint/test/p2p/data/mach$ID/core" \
+  	  "$DOCKER_IMAGE" node $NODE_FLAGS --log_level=info --proxy_app="$APP_PROXY"
 fi
diff --git a/test/p2p/seeds.sh b/test/p2p/seeds.sh
index 471476ad..4bf866cb 100644
--- a/test/p2p/seeds.sh
+++ b/test/p2p/seeds.sh
@@ -3,10 +3,10 @@ set -eu
 
 N=$1
 
-cd $GOPATH/src/github.com/tendermint/tendermint
+cd "$GOPATH/src/github.com/tendermint/tendermint"
 
 seeds="$(test/p2p/ip.sh 1):46656"
-for i in `seq 2 $N`; do
+for i in $(seq 2 $N); do
 	seeds="$seeds,$(test/p2p/ip.sh $i):46656"
 done
-echo $seeds
+echo "$seeds"
diff --git a/test/p2p/test.sh b/test/p2p/test.sh
index 3e67dfbd..6a5537b9 100644
--- a/test/p2p/test.sh
+++ b/test/p2p/test.sh
@@ -6,33 +6,33 @@ NETWORK_NAME=local_testnet
 N=4
 PROXY_APP=persistent_dummy
 
-cd $GOPATH/src/github.com/tendermint/tendermint
+cd "$GOPATH/src/github.com/tendermint/tendermint"
 
 # stop the existing testnet and remove local network
 set +e
-bash test/p2p/local_testnet_stop.sh $NETWORK_NAME $N
+bash test/p2p/local_testnet_stop.sh "$NETWORK_NAME" "$N"
 set -e
 
 SEEDS=$(bash test/p2p/seeds.sh $N)
 
 # start the testnet on a local network
 # NOTE we re-use the same network for all tests
-bash test/p2p/local_testnet_start.sh $DOCKER_IMAGE $NETWORK_NAME $N $PROXY_APP $SEEDS
+bash test/p2p/local_testnet_start.sh "$DOCKER_IMAGE" "$NETWORK_NAME" "$N" "$PROXY_APP" "$SEEDS"
 
 # test basic connectivity and consensus
 # start client container and check the num peers and height for all nodes
-bash test/p2p/client.sh $DOCKER_IMAGE $NETWORK_NAME basic "test/p2p/basic/test.sh $N"
+bash test/p2p/client.sh "$DOCKER_IMAGE" "$NETWORK_NAME" basic "test/p2p/basic/test.sh $N"
 
 # test atomic broadcast:
 # start client container and test sending a tx to each node
-bash test/p2p/client.sh $DOCKER_IMAGE $NETWORK_NAME ab "test/p2p/atomic_broadcast/test.sh $N"
+bash test/p2p/client.sh "$DOCKER_IMAGE" "$NETWORK_NAME" ab "test/p2p/atomic_broadcast/test.sh $N"
 
- test fast sync (from current state of network):
- for each node, kill it and readd via fast sync
-bash test/p2p/fast_sync/test.sh $DOCKER_IMAGE $NETWORK_NAME $N $PROXY_APP
+# test fast sync (from current state of network):
+# for each node, kill it and readd via fast sync
+bash test/p2p/fast_sync/test.sh "$DOCKER_IMAGE" "$NETWORK_NAME" "$N" "$PROXY_APP"
 
 # test killing all peers 3 times
-bash test/p2p/kill_all/test.sh $DOCKER_IMAGE $NETWORK_NAME $N 3
+bash test/p2p/kill_all/test.sh "$DOCKER_IMAGE" "$NETWORK_NAME" "$N" 3
 
 # test pex
-bash test/p2p/pex/test.sh $DOCKER_IMAGE $NETWORK_NAME $N $PROXY_APP
+bash test/p2p/pex/test.sh "$DOCKER_IMAGE" "$NETWORK_NAME" "$N" "$PROXY_APP"

From 07a9242dba29a03f4dc01358e930e31b8985295b Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Mon, 6 Mar 2017 04:02:09 -0500
Subject: [PATCH 131/132] update glide

---
 glide.lock | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/glide.lock b/glide.lock
index 216871f8..c3912163 100644
--- a/glide.lock
+++ b/glide.lock
@@ -1,5 +1,5 @@
 hash: 41f8fec708e98b7f8c4804be46008493199fa45e89b2d5dc237fd65fe431c62f
-updated: 2017-03-05T22:59:46.61454297-05:00
+updated: 2017-03-06T04:01:33.319604992-05:00
 imports:
 - name: github.com/btcsuite/btcd
   version: d06c0bb181529331be8f8d9350288c420d9e60e4
@@ -62,7 +62,7 @@ imports:
   - leveldb/table
   - leveldb/util
 - name: github.com/tendermint/abci
-  version: 0b1ee4b67318047d20a90bc6a8b0119a15c7c771
+  version: 1236e8fb6eee3a63909f4014a8e84385ead7933d
   subpackages:
   - client
   - example/counter
@@ -101,7 +101,7 @@ imports:
 - name: github.com/tendermint/go-merkle
   version: 714d4d04557fd068a7c2a1748241ce8428015a96
 - name: github.com/tendermint/go-p2p
-  version: beb3eda438fbbfc3b566ead4be5705474a12e50a
+  version: 97a5ed2d1a17eaee8717b8a32cfaf7a9a82a273d
   subpackages:
   - upnp
 - name: github.com/tendermint/go-rpc
@@ -117,7 +117,7 @@ imports:
   subpackages:
   - term
 - name: github.com/tendermint/merkleeyes
-  version: acd8e9c42e1d819c51e9e1cd3870ea4d94b167f5
+  version: 9fb76efa5aebe773a598f97e68e75fe53d520e70
   subpackages:
   - app
   - client

From 8449e9794a392e0bbf549a6f8d520c828e03cb08 Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Mon, 6 Mar 2017 04:25:55 -0500
Subject: [PATCH 132/132] CHANGELOG

---
 CHANGELOG.md                         | 220 +++++++++++++++++++++++++++
 test/persist/test_failure_indices.sh |   1 +
 2 files changed, 221 insertions(+)
 create mode 100644 CHANGELOG.md

diff --git a/CHANGELOG.md b/CHANGELOG.md
new file mode 100644
index 00000000..26c83b37
--- /dev/null
+++ b/CHANGELOG.md
@@ -0,0 +1,220 @@
+# Changelog
+
+## 0.9.0 (March 6, 2017)
+
+BREAKING CHANGES:
+
+- Update ABCI to v0.4.0, where Query is now `Query(RequestQuery) ResponseQuery`, enabling precise proofs at particular heights:
+
+```
+message RequestQuery{
+	bytes data = 1;
+	string path = 2;
+	uint64 height = 3;
+	bool prove = 4; 
+}
+
+message ResponseQuery{
+	CodeType          code        = 1;
+	int64             index       = 2;
+	bytes             key         = 3;
+	bytes             value       = 4;
+	bytes             proof       = 5;
+	uint64            height      = 6;
+	string            log         = 7;
+}
+```
+
+
+- `BlockMeta` data type unifies its Hash and PartSetHash under a `BlockID`:
+
+```
+type BlockMeta struct {
+	BlockID BlockID `json:"block_id"` // the block hash and partsethash
+	Header  *Header `json:"header"`   // The block's Header
+}
+```
+
+- `tendermint gen_validator` command output is now pure JSON
+- `ValidatorSet` data type:
+	- expose a `Proposer` field. Note this means the `Proposer` is persisted with the `State`.
+	- change `.Proposer()` to `.GetProposer()`
+
+FEATURES:
+
+- New RPC endpoint `/commit?height=X` returns header and commit for block at height `X`
+- Client API for each endpoint, including mocks for testing
+
+IMPROVEMENTS:
+
+- `Node` is now a `BaseService`
+- Simplified starting Tendermint in-process from another application
+- Better organized Makefile
+- Scripts for auto-building binaries across platforms
+- Docker image improved, slimmed down (using Alpine), and changed from tendermint/tmbase to tendermint/tendermint
+- New repo files: `CONTRIBUTING.md`, Github `ISSUE_TEMPLATE`, `CHANGELOG.md`
+- Improvements on CircleCI for managing build/test artifacts
+- Handshake replay is doen through the consensus package, possibly using a mockApp
+- Graceful shutdown of RPC listeners
+- Tests for the PEX reactor and DialSeeds
+
+BUG FIXES:
+
+- Check peer.Send for failure before updating PeerState in consensus
+- Fix panic in `/dial_seeds` with invalid addresses
+- Fix proposer selection logic in ValidatorSet by taking the address into account in the `accumComparable`
+- Fix inconcistencies with `ValidatorSet.Proposer` across restarts by persisting it in the `State`
+
+
+## 0.8.0 (January 13, 2017)
+
+BREAKING CHANGES:
+
+- New data type `BlockID` to represent blocks:
+
+```
+type BlockID struct {
+	Hash        []byte        `json:"hash"`
+	PartsHeader PartSetHeader `json:"parts"`
+}
+```
+
+- `Vote` data type now includes validator address and index: 
+
+```
+type Vote struct {
+	ValidatorAddress []byte           `json:"validator_address"`
+	ValidatorIndex   int              `json:"validator_index"`
+	Height           int              `json:"height"`
+	Round            int              `json:"round"`
+	Type             byte             `json:"type"`
+	BlockID          BlockID          `json:"block_id"` // zero if vote is nil.
+	Signature        crypto.Signature `json:"signature"`
+}
+```
+
+- Update TMSP to v0.3.0, where it is now called ABCI and AppendTx is DeliverTx
+- Hex strings in the RPC are now "0x" prefixed
+
+
+FEATURES:
+
+- New message type on the ConsensusReactor, `Maj23Msg`, for peers to alert others they've seen a Maj23, 
+in order to track and handle conflicting votes intelligently to prevent Byzantine faults from causing halts:
+
+```
+type VoteSetMaj23Message struct {
+	Height  int
+	Round   int
+	Type    byte
+	BlockID types.BlockID
+}
+```
+
+- Configurable block part set size
+- Validator set changes
+- Optionally skip TimeoutCommit if we have all the votes
+- Handshake between Tendermint and App on startup to sync latest state and ensure consistent recovery from crashes
+- GRPC server for BroadcastTx endpoint
+
+IMPROVEMENTS:
+
+- Less verbose logging
+- Better test coverage (37% -> 49%)
+- Canonical SignBytes for signable types
+- Write-Ahead Log for Mempool and Consensus via go-autofile 
+- Better in-process testing for the consensus reactor and byzantine faults
+- Better crash/restart testing for individual nodes at preset failure points, and of networks at arbitrary points
+- Better abstraction over timeout mechanics
+
+BUG FIXES:
+
+- Fix memory leak in mempool peer
+- Fix panic on POLRound=-1
+- Actually set the CommitTime
+- Actually send BeginBlock message
+- Fix a liveness issues caused by Byzantine proposals/votes. Uses the new `Maj23Msg`.
+
+
+## 0.7.4 (December 14, 2016)
+
+FEATURES:
+
+- Enable the Peer Exchange reactor with the `--pex` flag for more resilient gossip network (feature still in development, beware dragons)
+
+IMPROVEMENTS:
+
+- Remove restrictions on RPC endpoint `/dial_seeds` to enable manual network configuration
+
+## 0.7.3 (October 20, 2016)
+
+IMPROVEMENTS:
+
+- Type safe FireEvent
+- More WAL/replay tests
+- Cleanup some docs
+
+BUG FIXES:
+
+- Fix deadlock in mempool for synchronous apps
+- Replay handles non-empty blocks
+- Fix race condition in HeightVoteSet
+
+## 0.7.2 (September 11, 2016)
+
+BUG FIXES:
+
+- Set mustConnect=false so tendermint will retry connecting to the app
+
+## 0.7.1 (September 10, 2016)
+
+FEATURES:
+
+- New TMSP connection for Query/Info
+- New RPC endpoints:
+	- `tmsp_query`
+	- `tmsp_info`
+- Allow application to filter peers through Query (off by default)
+
+IMPROVEMENTS:
+
+- TMSP connection type enforced at compile time
+- All listen/client urls use a "tcp://" or "unix://" prefix
+
+BUG FIXES:
+
+- Save LastSignature/LastSignBytes to `priv_validator.json` for recovery
+- Fix event unsubscribe
+- Fix fastsync/blockchain reactor
+
+## 0.7.0 (August 7, 2016)
+
+BREAKING CHANGES:
+
+- Strict SemVer starting now!
+- Update to ABCI v0.2.0
+- Validation types now called Commit
+- NewBlock event only returns the block header
+
+
+FEATURES:
+
+- TMSP and RPC support TCP and UNIX sockets
+- Addition config options including block size and consensus parameters
+- New WAL mode `cswal_light`; logs only the validator's own votes
+- New RPC endpoints: 
+	- for starting/stopping profilers, and for updating config
+	- `/broadcast_tx_commit`, returns when tx is included in a block, else an error
+	- `/unsafe_flush_mempool`, empties the mempool
+
+
+IMPROVEMENTS:
+
+- Various optimizations
+- Remove bad or invalidated transactions from the mempool cache (allows later duplicates)
+- More elaborate testing using CircleCI including benchmarking throughput on 4 digitalocean droplets
+
+BUG FIXES:
+
+- Various fixes to WAL and replay logic
+- Various race conditions
diff --git a/test/persist/test_failure_indices.sh b/test/persist/test_failure_indices.sh
index f47ab372..8a708e50 100644
--- a/test/persist/test_failure_indices.sh
+++ b/test/persist/test_failure_indices.sh
@@ -91,6 +91,7 @@ for failIndex in $(seq $failsStart $failsEnd); do
 	start_procs 2
 
 	# wait for node to handshake and make a new block
+	# NOTE: --unix-socket is only available in curl v7.40+
 	curl -s --unix-socket "$RPC_ADDR" http://localhost/status > /dev/null
 	ERR=$?
 	i=0