testing: add CT (Custom Testing) structure; create uniq point for 'lnd process errors' and 'test panic/failed errors' handling

glide.lock

@@ -155,4 +155,7 @@ imports:
   - naming
   - transport
   - peer
+- name: github.com/go-errors/errors
+  version: a41850380601eeb43f4350f7d17c6bbd8944aaf8
+
 testImports: []

glide.yaml

@@ -60,3 +60,4 @@ import:
 - package: github.com/grpc-ecosystem/grpc-gateway
   version: ^1.1.0
 - package: github.com/aead/chacha20
+- package: github.com/go-errors/errors

lnd_test.go

@@ -3,42 +3,100 @@ package main
 import (
 	"bytes"
 	"fmt"
-	"runtime/debug"
-	"sync"
-	"testing"
-	"time"
-
 	"golang.org/x/net/context"
-
+	"sync"
+	"time"
 	"github.com/davecgh/go-spew/spew"
+	"github.com/go-errors/errors"
 	"github.com/lightningnetwork/lnd/lnrpc"
 	"github.com/roasbeef/btcd/rpctest"
 	"github.com/roasbeef/btcd/wire"
 	"github.com/roasbeef/btcrpcclient"
 	"github.com/roasbeef/btcutil"
+	"testing"
 )
 
-func assertTxInBlock(block *btcutil.Block, txid *wire.ShaHash, t *testing.T) {
+// CT is needed for:
+// - have uniform way of handling panic and fatal error from test cases.
+// - have ability to properly wrap errors in order to see stack trace.
+// - have nice and elegant way to handle lnd process errors in one
+// select structure with test cases.
+type CT struct {
+	*testing.T
+
+	// Channel for sending retransmitted panic errors and fatal error which
+	// happens in test case.
+	errChan chan error
+}
+
+func NewCT(t *testing.T) *CT {
+	return &CT{t, nil}
+}
+
+func (ct *CT) Error(err error) {
+	if ct.errChan != nil {
+		ct.errChan <- fmt.Errorf(errors.Wrap(err, 1).ErrorStack())
+		ct.FailNow()
+	} else {
+		ct.Fatal("can't sen error when test isn't running")
+	}
+}
+
+// Errorf create and send the description about the error in the error channel
+// and exit.
+func (ct *CT) Errorf(format string, a ...interface{}) {
+	if ct.errChan != nil {
+		description := fmt.Sprintf(format, a...)
+		ct.errChan <- fmt.Errorf(errors.Wrap(description, 1).ErrorStack())
+		ct.FailNow()
+	} else {
+		ct.Fatal("can't sen error when test isn't running")
+	}
+
+}
+
+// RunTest wraps test case function in goroutine and also redirects the panic
+// error from test case into error channel.
+func (ct *CT) RunTest(net *networkHarness, test testCase) chan error {
+	// a channel to signal that test was exited with error
+	ct.errChan = make(chan error)
+
+	go func() {
+		defer func() {
+			if err := recover(); err != nil {
+				// Retransmit test panic into main "process"
+				ct.errChan <- fmt.Errorf(err.(string))
+			}
+			close(ct.errChan)
+			ct.errChan = nil
+		}()
+		test(net, ct)
+	}()
+
+	return ct.errChan
+}
+
+func assertTxInBlock(ct *CT, block *btcutil.Block, txid *wire.ShaHash) {
 	for _, tx := range block.Transactions() {
 		if bytes.Equal(txid[:], tx.Sha()[:]) {
 			return
 		}
 	}
 
-	t.Fatalf("funding tx was not included in block")
+	ct.Errorf("funding tx was not included in block")
 }
 
 // openChannelAndAssert attempts to open a channel with the specified
 // parameters extended from Alice to Bob. Additionally, two items are asserted
-// after the channel is considered open: the funding transactino should be
+// after the channel is considered open: the funding transaction should be
 // found within a block, and that Alice can report the status of the new
 // channel.
-func openChannelAndAssert(t *testing.T, net *networkHarness, ctx context.Context,
+func openChannelAndAssert(ct *CT, net *networkHarness, ctx context.Context,
 	alice, bob *lightningNode, amount btcutil.Amount) *lnrpc.ChannelPoint {
 
 	chanOpenUpdate, err := net.OpenChannel(ctx, alice, bob, amount, 1)
 	if err != nil {
-		t.Fatalf("unable to open channel: %v", err)
+		ct.Errorf("unable to open channel: %v", err)
 	}
 
 	// Mine a block, then wait for Alice's node to notify us that the
@@ -46,21 +104,21 @@ func openChannelAndAssert(t *testing.T, net *networkHarness, ctx context.Context
 	// within the newly mined block.
 	blockHash, err := net.Miner.Node.Generate(1)
 	if err != nil {
-		t.Fatalf("unable to generate block: %v", err)
+		ct.Errorf("unable to generate block: %v", err)
 	}
 	block, err := net.Miner.Node.GetBlock(blockHash[0])
 	if err != nil {
-		t.Fatalf("unable to get block: %v", err)
+		ct.Errorf("unable to get block: %v", err)
 	}
 	fundingChanPoint, err := net.WaitForChannelOpen(ctx, chanOpenUpdate)
 	if err != nil {
-		t.Fatalf("error while waiting for channel open: %v", err)
+		ct.Errorf("error while waiting for channel open: %v", err)
 	}
 	fundingTxID, err := wire.NewShaHash(fundingChanPoint.FundingTxid)
 	if err != nil {
-		t.Fatalf("unable to create sha hash: %v", err)
+		ct.Errorf("unable to create sha hash: %v", err)
 	}
-	assertTxInBlock(block, fundingTxID, t)
+	assertTxInBlock(ct, block, fundingTxID)
 
 	// The channel should be listed in the peer information returned by
 	// both peers.
@@ -68,27 +126,25 @@ func openChannelAndAssert(t *testing.T, net *networkHarness, ctx context.Context
 		Hash:  *fundingTxID,
 		Index: fundingChanPoint.OutputIndex,
 	}
-	err = net.AssertChannelExists(ctx, alice, &chanPoint)
+	if err := net.AssertChannelExists(ctx, alice, &chanPoint); err != nil {
-	if err != nil {
+		ct.Errorf("unable to assert channel existence: %v", err)
-		t.Fatalf("unable to assert channel existence: %v", err)
 	}
 
 	return fundingChanPoint
 }
 
-// closeChannelAndAssert attemps to close a channel identified by the passed
+// closeChannelAndAssert attempts to close a channel identified by the passed
 // channel point owned by the passed lighting node. A fully blocking channel
 // closure is attempted, therefore the passed context should be a child derived
 // via timeout from a base parent. Additionally, once the channel has been
 // detected as closed, an assertion checks that the transaction is found within
 // a block.
-func closeChannelAndAssert(t *testing.T, net *networkHarness,
+func closeChannelAndAssert(ct *CT, net *networkHarness, ctx context.Context,
-	ctx context.Context, node *lightningNode,
+	node *lightningNode, fundingChanPoint *lnrpc.ChannelPoint) {
-	fundingChanPoint *lnrpc.ChannelPoint) {
 
 	closeUpdates, err := net.CloseChannel(ctx, node, fundingChanPoint, false)
 	if err != nil {
-		t.Fatalf("unable to close channel: %v", err)
+		ct.Errorf("unable to close channel: %v", err)
 	}
 
 	// Finally, generate a single block, wait for the final close status
@@ -96,19 +152,19 @@ func closeChannelAndAssert(t *testing.T, net *networkHarness,
 	// block.
 	blockHash, err := net.Miner.Node.Generate(1)
 	if err != nil {
-		t.Fatalf("unable to generate block: %v", err)
+		ct.Errorf("unable to generate block: %v", err)
 	}
 	block, err := net.Miner.Node.GetBlock(blockHash[0])
 	if err != nil {
-		t.Fatalf("unable to get block: %v", err)
+		ct.Errorf("unable to get block: %v", err)
 	}
 
 	closingTxid, err := net.WaitForChannelClose(ctx, closeUpdates)
 	if err != nil {
-		t.Fatalf("error while waiting for channel close: %v", err)
+		ct.Errorf("error while waiting for channel close: %v", err)
 	}
 
-	assertTxInBlock(block, closingTxid, t)
+	assertTxInBlock(ct, block, closingTxid)
 }
 
 // testBasicChannelFunding performs a test exercising expected behavior from a
@@ -116,7 +172,7 @@ func closeChannelAndAssert(t *testing.T, net *networkHarness,
 // Bob, then immediately closes the channel after asserting some expected post
 // conditions. Finally, the chain itself is checked to ensure the closing
 // transaction was mined.
-func testBasicChannelFunding(net *networkHarness, t *testing.T) {
+func testBasicChannelFunding(net *networkHarness, ct *CT) {
 	timeout := time.Duration(time.Second * 5)
 	ctxb := context.Background()
 
@@ -128,40 +184,44 @@ func testBasicChannelFunding(net *networkHarness, t *testing.T) {
 	// assertions will be executed to ensure the funding process completed
 	// successfully.
 	ctxt, _ := context.WithTimeout(ctxb, timeout)
-	chanPoint := openChannelAndAssert(t, net, ctxt, net.Alice, net.Bob, chanAmt)
+	chanPoint := openChannelAndAssert(ct, net, ctxt, net.Alice, net.Bob, chanAmt)
 
 	// Finally, immediately close the channel. This function will also
 	// block until the channel is closed and will additionally assert the
 	// relevant channel closing post conditions.
 	ctxt, _ = context.WithTimeout(ctxb, timeout)
-	closeChannelAndAssert(t, net, ctxt, net.Alice, chanPoint)
+	closeChannelAndAssert(ct, net, ctxt, net.Alice, chanPoint)
 }
 
 // testChannelBalance creates a new channel between Alice and  Bob, then
 // checks channel balance to be equal amount specified while creation of channel.
-func testChannelBalance(net *networkHarness, t *testing.T) {
+func testChannelBalance(net *networkHarness, ct *CT) {
 	timeout := time.Duration(time.Second * 5)
-	ctxb := context.Background()
-
-	// Creates a helper closure to be used below which asserts the proper
-	// response to a channel balance RPC.
-	checkChannelBalance := func(node lnrpc.LightningClient, amount btcutil.Amount) {
-		response, err := node.ChannelBalance(ctxb, &lnrpc.ChannelBalanceRequest{})
-		if err != nil {
-			t.Fatalf("unable to get channel balance: %v", err)
-		}
-
-		balance := btcutil.Amount(response.Balance)
-		if balance != amount {
-			t.Fatalf("channel balance wrong: %v != %v", balance, amount)
-		}
-	}
 
 	// Open a channel with 0.5 BTC between Alice and Bob, ensuring the
 	// channel has been opened properly.
 	amount := btcutil.Amount(btcutil.SatoshiPerBitcoin / 2)
-	ctxt, _ := context.WithTimeout(ctxb, timeout)
+	ctx, _ := context.WithTimeout(context.Background(), timeout)
-	chanPoint := openChannelAndAssert(t, net, ctxt, net.Alice, net.Bob, amount)
+
+	// Creates a helper closure to be used below which asserts the proper
+	// response to a channel balance RPC.
+	checkChannelBalance := func(node lnrpc.LightningClient,
+		amount btcutil.Amount) {
+
+		response, err := node.ChannelBalance(ctx, &lnrpc.ChannelBalanceRequest{})
+		if err != nil {
+			ct.Errorf("unable to get channel balance: %v", err)
+		}
+
+		balance := btcutil.Amount(response.Balance)
+		if balance != amount {
+			ct.Errorf("channel balance wrong: %v != %v", balance,
+				amount)
+		}
+	}
+
+	chanPoint := openChannelAndAssert(ct, net, ctx, net.Alice, net.Bob,
+		amount)
 
 	// As this is a single funder channel, Alice's balance should be
 	// exactly 0.5 BTC since now state transitions have taken place yet.
@@ -180,8 +240,8 @@ func testChannelBalance(net *networkHarness, t *testing.T) {
 
 	// Finally close the channel between Alice and Bob, asserting that the
 	// channel has been properly closed on-chain.
-	ctxt, _ = context.WithTimeout(ctxb, timeout)
+	ctx, _ = context.WithTimeout(context.Background(), timeout)
-	closeChannelAndAssert(t, net, ctxt, net.Alice, chanPoint)
+	closeChannelAndAssert(ct, net, ctx, net.Alice, chanPoint)
 }
 
 // testChannelForceClosure performs a test to exercise the behavior of "force"
@@ -193,7 +253,7 @@ func testChannelBalance(net *networkHarness, t *testing.T) {
 // once the output(s) become mature.
 //
 // TODO(roabeef): also add an unsettled HTLC before force closing.
-func testChannelForceClosure(net *networkHarness, t *testing.T) {
+func testChannelForceClosure(net *networkHarness, ct *CT) {
 	timeout := time.Duration(time.Second * 5)
 	ctxb := context.Background()
 
@@ -204,15 +264,17 @@ func testChannelForceClosure(net *networkHarness, t *testing.T) {
 	chanOpenUpdate, err := net.OpenChannel(ctxb, net.Alice, net.Bob,
 		chanAmt, numFundingConfs)
 	if err != nil {
-		t.Fatalf("unable to open channel: %v", err)
+		ct.Errorf("unable to open channel: %v", err)
 	}
+
 	if _, err := net.Miner.Node.Generate(numFundingConfs); err != nil {
-		t.Fatalf("unable to mine block: %v", err)
+		ct.Errorf("unable to mine block: %v", err)
 	}
+
 	ctxt, _ := context.WithTimeout(ctxb, timeout)
 	chanPoint, err := net.WaitForChannelOpen(ctxt, chanOpenUpdate)
 	if err != nil {
-		t.Fatalf("error while waiting for channel to open: %v", err)
+		ct.Errorf("error while waiting for channel to open: %v", err)
 	}
 
 	// Now that the channel is open, immediately execute a force closure of
@@ -221,18 +283,18 @@ func testChannelForceClosure(net *networkHarness, t *testing.T) {
 	// request.
 	closeUpdate, err := net.CloseChannel(ctxb, net.Alice, chanPoint, true)
 	if err != nil {
-		t.Fatalf("unable to execute force channel closure: %v", err)
+		ct.Errorf("unable to execute force channel closure: %v", err)
 	}
 
 	// Mine a block which should confirm the commitment transaction
 	// broadcast as a result of the force closure.
 	if _, err := net.Miner.Node.Generate(1); err != nil {
-		t.Fatalf("unable to generate block: %v", err)
+		ct.Errorf("unable to generate block: %v", err)
 	}
 	ctxt, _ = context.WithTimeout(ctxb, timeout)
 	closingTxID, err := net.WaitForChannelClose(ctxt, closeUpdate)
 	if err != nil {
-		t.Fatalf("error while waiting for channel close: %v", err)
+		ct.Errorf("error while waiting for channel close: %v", err)
 	}
 
 	// Currently within the codebase, the default CSV is 4 relative blocks.
@@ -241,7 +303,7 @@ func testChannelForceClosure(net *networkHarness, t *testing.T) {
 	// or make delay a param
 	const defaultCSV = 4
 	if _, err := net.Miner.Node.Generate(defaultCSV); err != nil {
-		t.Fatalf("unable to mine blocks: %v", err)
+		ct.Errorf("unable to mine blocks: %v", err)
 	}
 
 	// At this point, the sweeping transaction should now be broadcast. So
@@ -253,11 +315,11 @@ mempoolPoll:
 	for {
 		select {
 		case <-time.After(time.Second * 5):
-			t.Fatalf("sweep tx not found in mempool")
+			ct.Errorf("sweep tx not found in mempool")
 		default:
 			mempool, err = net.Miner.Node.GetRawMempool()
 			if err != nil {
-				t.Fatalf("unable to fetch node's mempool: %v", err)
+				ct.Errorf("unable to fetch node's mempool: %v", err)
 			}
 			if len(mempool) == 0 {
 				continue
@@ -271,7 +333,7 @@ mempoolPoll:
 	// TODO(roasbeef): assertion may not necessarily hold with concurrent
 	// test executions
 	if len(mempool) != 1 {
-		t.Fatalf("node's mempool is wrong size, expected 1 got %v",
+		ct.Errorf("node's mempool is wrong size, expected 1 got %v",
 			len(mempool))
 	}
 	sweepingTXID = mempool[0]
@@ -280,11 +342,11 @@ mempoolPoll:
 	// the commitment transaction which was broadcast on-chain.
 	sweepTx, err := net.Miner.Node.GetRawTransaction(sweepingTXID)
 	if err != nil {
-		t.Fatalf("unable to fetch sweep tx: %v", err)
+		ct.Errorf("unable to fetch sweep tx: %v", err)
 	}
 	for _, txIn := range sweepTx.MsgTx().TxIn {
 		if !closingTxID.IsEqual(&txIn.PreviousOutPoint.Hash) {
-			t.Fatalf("sweep transaction not spending from commit "+
+			ct.Errorf("sweep transaction not spending from commit "+
 				"tx %v, instead spending %v",
 				closingTxID, txIn.PreviousOutPoint)
 		}
@@ -295,16 +357,17 @@ mempoolPoll:
 	// inputs should be properly met.
 	blockHash, err := net.Miner.Node.Generate(1)
 	if err != nil {
-		t.Fatalf("unable to generate block: %v", err)
+		ct.Errorf("unable to generate block: %v", err)
 	}
 	block, err := net.Miner.Node.GetBlock(blockHash[0])
 	if err != nil {
-		t.Fatalf("unable to get block: %v", err)
+		ct.Errorf("unable to get block: %v", err)
 	}
-	assertTxInBlock(block, sweepTx.Sha(), t)
+
+	assertTxInBlock(ct, block, sweepTx.Sha())
 }
 
-func testSingleHopInvoice(net *networkHarness, t *testing.T) {
+func testSingleHopInvoice(net *networkHarness, ct *CT) {
 	ctxb := context.Background()
 	timeout := time.Duration(time.Second * 5)
 
@@ -312,7 +375,7 @@ func testSingleHopInvoice(net *networkHarness, t *testing.T) {
 	// the sole funder of the channel.
 	ctxt, _ := context.WithTimeout(ctxb, timeout)
 	chanAmt := btcutil.Amount(100000)
-	chanPoint := openChannelAndAssert(t, net, ctxt, net.Alice, net.Bob, chanAmt)
+	chanPoint := openChannelAndAssert(ct, net, ctxt, net.Alice, net.Bob, chanAmt)
 
 	// Now that the channel is open, create an invoice for Bob which
 	// expects a payment of 1000 satoshis from Alice paid via a particular
@@ -326,14 +389,14 @@ func testSingleHopInvoice(net *networkHarness, t *testing.T) {
 	}
 	invoiceResp, err := net.Bob.AddInvoice(ctxb, invoice)
 	if err != nil {
-		t.Fatalf("unable to add invoice: %v", err)
+		ct.Errorf("unable to add invoice: %v", err)
 	}
 
 	// With the invoice for Bob added, send a payment towards Alice paying
 	// to the above generated invoice.
 	sendStream, err := net.Alice.SendPayment(ctxb)
 	if err != nil {
-		t.Fatalf("unable to create alice payment stream: %v", err)
+		ct.Errorf("unable to create alice payment stream: %v", err)
 	}
 	sendReq := &lnrpc.SendRequest{
 		PaymentHash: invoiceResp.RHash,
@@ -341,10 +404,10 @@ func testSingleHopInvoice(net *networkHarness, t *testing.T) {
 		Amt:         paymentAmt,
 	}
 	if err := sendStream.Send(sendReq); err != nil {
-		t.Fatalf("unable to send payment: %v", err)
+		ct.Errorf("unable to send payment: %v", err)
 	}
 	if _, err := sendStream.Recv(); err != nil {
-		t.Fatalf("error when attempting recv: %v", err)
+		ct.Errorf("error when attempting recv: %v", err)
 	}
 
 	// Bob's invoice should now be found and marked as settled.
@@ -355,37 +418,38 @@ func testSingleHopInvoice(net *networkHarness, t *testing.T) {
 	}
 	dbInvoice, err := net.Bob.LookupInvoice(ctxb, payHash)
 	if err != nil {
-		t.Fatalf("unable to lookup invoice: %v", err)
+		ct.Errorf("unable to lookup invoice: %v", err)
 	}
 	if !dbInvoice.Settled {
-		t.Fatalf("bob's invoice should be marked as settled: %v",
+		ct.Errorf("bob's invoice should be marked as settled: %v",
 			spew.Sdump(dbInvoice))
 	}
 
 	// The balances of Alice and Bob should be updated accordingly.
 	aliceBalance, err := net.Alice.ChannelBalance(ctxb, &lnrpc.ChannelBalanceRequest{})
 	if err != nil {
-		t.Fatalf("unable to query for alice's balance: %v", err)
+		ct.Errorf("unable to query for alice's balance: %v", err)
 	}
 	bobBalance, err := net.Bob.ChannelBalance(ctxb, &lnrpc.ChannelBalanceRequest{})
 	if err != nil {
-		t.Fatalf("unable to query for bob's balance: %v", err)
+		ct.Errorf("unable to query for bob's balance: %v", err)
 	}
 
 	if aliceBalance.Balance != int64(chanAmt-paymentAmt) {
-		t.Fatalf("Alice's balance is incorrect got %v, expected %v",
+		ct.Errorf("Alice's balance is incorrect got %v, expected %v",
 			aliceBalance, int64(chanAmt-paymentAmt))
 	}
 	if bobBalance.Balance != paymentAmt {
-		t.Fatalf("Bob's balance is incorrect got %v, expected %v",
+		ct.Errorf("Bob's balance is incorrect got %v, expected %v",
 			bobBalance, paymentAmt)
 	}
 
 	ctxt, _ = context.WithTimeout(ctxb, timeout)
-	closeChannelAndAssert(t, net, ctxt, net.Alice, chanPoint)
+	closeChannelAndAssert(ct, net, ctxt, net.Alice, chanPoint)
 }
 
-func testMultiHopPayments(net *networkHarness, t *testing.T) {
+func testMultiHopPayments(net *networkHarness, ct *CT) {
+
 	const chanAmt = btcutil.Amount(100000)
 	ctxb := context.Background()
 	timeout := time.Duration(time.Second * 5)
@@ -393,11 +457,12 @@ func testMultiHopPayments(net *networkHarness, t *testing.T) {
 	// Open a channel with 100k satoshis between Alice and Bob with Alice
 	// being the sole funder of the channel.
 	ctxt, _ := context.WithTimeout(ctxb, timeout)
-	chanPointAlice := openChannelAndAssert(t, net, ctxt, net.Alice, net.Bob,
+	chanPointAlice := openChannelAndAssert(ct, net, ctxt, net.Alice,
-		chanAmt)
+		net.Bob, chanAmt)
+
 	aliceChanTXID, err := wire.NewShaHash(chanPointAlice.FundingTxid)
 	if err != nil {
-		t.Fatalf("unable to create sha hash: %v", err)
+		ct.Errorf("unable to create sha hash: %v", err)
 	}
 	aliceFundPoint := wire.OutPoint{
 		Hash:  *aliceChanTXID,
@@ -411,21 +476,22 @@ func testMultiHopPayments(net *networkHarness, t *testing.T) {
 	// The network topology should now look like: Carol -> Alice -> Bob
 	carol, err := net.NewNode(nil)
 	if err != nil {
-		t.Fatalf("unable to create new nodes: %v", err)
+		ct.Errorf("unable to create new nodes: %v", err)
 	}
 	if err := net.ConnectNodes(ctxb, carol, net.Alice); err != nil {
-		t.Fatalf("unable to connect carol to alice: %v", err)
+		ct.Errorf("unable to connect carol to alice: %v", err)
 	}
 	err = net.SendCoins(ctxb, btcutil.SatoshiPerBitcoin, carol)
 	if err != nil {
-		t.Fatalf("unable to send coins to carol: %v", err)
+		ct.Errorf("unable to send coins to carol: %v", err)
 	}
 	ctxt, _ = context.WithTimeout(ctxb, timeout)
-	chanPointCarol := openChannelAndAssert(t, net, ctxt, carol, net.Alice,
+	chanPointCarol := openChannelAndAssert(ct, net, ctxt, carol,
-		chanAmt)
+		net.Alice, chanAmt)
+
 	carolChanTXID, err := wire.NewShaHash(chanPointCarol.FundingTxid)
 	if err != nil {
-		t.Fatalf("unable to create sha hash: %v", err)
+		ct.Errorf("unable to create sha hash: %v", err)
 	}
 	carolFundPoint := wire.OutPoint{
 		Hash:  *carolChanTXID,
@@ -446,7 +512,7 @@ func testMultiHopPayments(net *networkHarness, t *testing.T) {
 		}
 		resp, err := net.Bob.AddInvoice(ctxb, invoice)
 		if err != nil {
-			t.Fatalf("unable to add invoice: %v", err)
+			ct.Errorf("unable to add invoice: %v", err)
 		}
 
 		rHashes[i] = resp.RHash
@@ -459,10 +525,10 @@ func testMultiHopPayments(net *networkHarness, t *testing.T) {
 	req := &lnrpc.ShowRoutingTableRequest{}
 	routingResp, err := carol.ShowRoutingTable(ctxb, req)
 	if err != nil {
-		t.Fatalf("unable to query for carol's routing table: %v", err)
+		ct.Errorf("unable to query for carol's routing table: %v", err)
 	}
 	if len(routingResp.Channels) != 2 {
-		t.Fatalf("only two channels should be seen as active in the "+
+		ct.Errorf("only two channels should be seen as active in the "+
 			"network, instead %v are", len(routingResp.Channels))
 	}
 	for _, link := range routingResp.Channels {
@@ -476,7 +542,7 @@ func testMultiHopPayments(net *networkHarness, t *testing.T) {
 				link.Id2 == net.Alice.PubKeyStr:
 				continue
 			default:
-				t.Fatalf("unkown link within routing "+
+				ct.Errorf("unkown link within routing "+
 					"table: %v", spew.Sdump(link))
 			}
 		case link.Outpoint == carolFundPoint.String():
@@ -488,11 +554,11 @@ func testMultiHopPayments(net *networkHarness, t *testing.T) {
 				link.Id2 == net.Alice.PubKeyStr:
 				continue
 			default:
-				t.Fatalf("unkown link within routing "+
+				ct.Errorf("unkown link within routing "+
 					"table: %v", spew.Sdump(link))
 			}
 		default:
-			t.Fatalf("unkown channel %v found in routing table, "+
+			ct.Errorf("unkown channel %v found in routing table, "+
 				"only %v and %v should exist", link.Outpoint,
 				aliceFundPoint, carolFundPoint)
 		}
@@ -501,7 +567,7 @@ func testMultiHopPayments(net *networkHarness, t *testing.T) {
 	// Using Carol as the source, pay to the 5 invoices from Bob created above.
 	carolPayStream, err := carol.SendPayment(ctxb)
 	if err != nil {
-		t.Fatalf("unable to create payment stream for carol: %v", err)
+		ct.Errorf("unable to create payment stream for carol: %v", err)
 	}
 
 	// Concurrently pay off all 5 of Bob's invoices. Each of the goroutines
@@ -518,10 +584,10 @@ func testMultiHopPayments(net *networkHarness, t *testing.T) {
 		wg.Add(1)
 		go func() {
 			if err := carolPayStream.Send(sendReq); err != nil {
-				t.Fatalf("unable to send payment: %v", err)
+				ct.Errorf("unable to send payment: %v", err)
 			}
 			if _, err := carolPayStream.Recv(); err != nil {
-				t.Fatalf("unable to recv pay resp: %v", err)
+				ct.Errorf("unable to recv pay resp: %v", err)
 			}
 			wg.Done()
 		}()
@@ -535,16 +601,18 @@ func testMultiHopPayments(net *networkHarness, t *testing.T) {
 
 	select {
 	case <-time.After(time.Second * 10):
-		t.Fatalf("HLTC's not cleared after 10 seconds")
+		ct.Errorf("HLTC's not cleared after 10 seconds")
 	case <-finClear:
 	}
 
 	assertAsymmetricBalance := func(node *lightningNode,
-		chanPoint *wire.OutPoint, localBalance, remoteBalance int64) {
+		chanPoint *wire.OutPoint, localBalance,
+		remoteBalance int64) {
+
 		listReq := &lnrpc.ListChannelsRequest{}
 		resp, err := node.ListChannels(ctxb, listReq)
 		if err != nil {
-			t.Fatalf("unable to for node's channels: %v", err)
+			ct.Errorf("unable to for node's channels: %v", err)
 		}
 		for _, channel := range resp.Channels {
 			if channel.ChannelPoint != chanPoint.String() {
@@ -553,12 +621,12 @@ func testMultiHopPayments(net *networkHarness, t *testing.T) {
 
 			if channel.LocalBalance != localBalance ||
 				channel.RemoteBalance != remoteBalance {
-				t.Fatalf("incorrect balances: %v",
+				ct.Errorf("incorrect balances: %v",
 					spew.Sdump(channel))
 			}
 			return
 		}
-		t.Fatalf("channel not found")
+		ct.Errorf("channel not found")
 	}
 
 	// At this point all the channels within our proto network should be
@@ -575,12 +643,12 @@ func testMultiHopPayments(net *networkHarness, t *testing.T) {
 	assertAsymmetricBalance(net.Bob, &aliceFundPoint, sinkBal, sourceBal)
 
 	ctxt, _ = context.WithTimeout(ctxb, timeout)
-	closeChannelAndAssert(t, net, ctxt, net.Alice, chanPointAlice)
+	closeChannelAndAssert(ct, net, ctxt, net.Alice, chanPointAlice)
 	ctxt, _ = context.WithTimeout(ctxb, timeout)
-	closeChannelAndAssert(t, net, ctxt, carol, chanPointCarol)
+	closeChannelAndAssert(ct, net, ctxt, carol, chanPointCarol)
 }
 
-func testInvoiceSubscriptions(net *networkHarness, t *testing.T) {
+func testInvoiceSubscriptions(net *networkHarness, ct *CT) {
 	const chanAmt = btcutil.Amount(500000)
 	ctxb := context.Background()
 	timeout := time.Duration(time.Second * 5)
@@ -588,7 +656,7 @@ func testInvoiceSubscriptions(net *networkHarness, t *testing.T) {
 	// Open a channel with 500k satoshis between Alice and Bob with Alice
 	// being the sole funder of the channel.
 	ctxt, _ := context.WithTimeout(ctxb, timeout)
-	chanPoint := openChannelAndAssert(t, net, ctxt, net.Alice, net.Bob,
+	chanPoint := openChannelAndAssert(ct, net, ctxt, net.Alice, net.Bob,
 		chanAmt)
 
 	// Next create a new invoice for Bob requesting 1k satoshis.
@@ -601,7 +669,7 @@ func testInvoiceSubscriptions(net *networkHarness, t *testing.T) {
 	}
 	invoiceResp, err := net.Bob.AddInvoice(ctxb, invoice)
 	if err != nil {
-		t.Fatalf("unable to add invoice: %v", err)
+		ct.Errorf("unable to add invoice: %v", err)
 	}
 
 	// Create a new invoice subscription client for Bob, the notification
@@ -609,23 +677,23 @@ func testInvoiceSubscriptions(net *networkHarness, t *testing.T) {
 	req := &lnrpc.InvoiceSubscription{}
 	bobInvoiceSubscription, err := net.Bob.SubscribeInvoices(ctxb, req)
 	if err != nil {
-		t.Fatalf("unable to subscribe to bob's invoice updates: %v", err)
+		ct.Errorf("unable to subscribe to bob's invoice updates: %v", err)
 	}
 
 	updateSent := make(chan struct{})
 	go func() {
 		invoiceUpdate, err := bobInvoiceSubscription.Recv()
 		if err != nil {
-			t.Fatalf("unable to recv invoice update: %v", err)
+			ct.Errorf("unable to recv invoice update: %v", err)
 		}
 
 		// The invoice update should exactly match the invoice created
 		// above, but should now be settled.
 		if !invoiceUpdate.Settled {
-			t.Fatalf("invoice not settled but shoudl be")
+			ct.Errorf("invoice not settled but shoudl be")
 		}
 		if !bytes.Equal(invoiceUpdate.RPreimage, invoice.RPreimage) {
-			t.Fatalf("payment preimages don't match: expected %v, got %v",
+			ct.Errorf("payment preimages don't match: expected %v, got %v",
 				invoice.RPreimage, invoiceUpdate.RPreimage)
 		}
 
@@ -636,7 +704,7 @@ func testInvoiceSubscriptions(net *networkHarness, t *testing.T) {
 	// which should finalize and settle the invoice.
 	sendStream, err := net.Alice.SendPayment(ctxb)
 	if err != nil {
-		t.Fatalf("unable to create alice payment stream: %v", err)
+		ct.Errorf("unable to create alice payment stream: %v", err)
 	}
 	sendReq := &lnrpc.SendRequest{
 		PaymentHash: invoiceResp.RHash,
@@ -644,24 +712,24 @@ func testInvoiceSubscriptions(net *networkHarness, t *testing.T) {
 		Amt:         paymentAmt,
 	}
 	if err := sendStream.Send(sendReq); err != nil {
-		t.Fatalf("unable to send payment: %v", err)
+		ct.Errorf("unable to send payment: %v", err)
 	}
 	if _, err := sendStream.Recv(); err != nil {
-		t.Fatalf("error when attempting recv: %v", err)
+		ct.Errorf("error when attempting recv: %v", err)
 	}
 
 	select {
 	case <-time.After(time.Second * 5):
-		t.Fatalf("update not sent after 5 seconds")
+		ct.Errorf("update not sent after 5 seconds")
 	case <-updateSent: // Fall through on success
 	}
 
 	ctxt, _ = context.WithTimeout(ctxb, timeout)
-	closeChannelAndAssert(t, net, ctxt, net.Alice, chanPoint)
+	closeChannelAndAssert(ct, net, ctxt, net.Alice, chanPoint)
 }
 
 // testBasicChannelCreation test multiple channel opening and closing.
-func testBasicChannelCreation(net *networkHarness, t *testing.T) {
+func testBasicChannelCreation(net *networkHarness, ct *CT) {
 	timeout := time.Duration(time.Second * 5)
 	ctx, _ := context.WithTimeout(context.Background(), timeout)
 
@@ -673,20 +741,20 @@ func testBasicChannelCreation(net *networkHarness, t *testing.T) {
 	// channel has been properly open on-chain.
 	chanPoints := make([]*lnrpc.ChannelPoint, num)
 	for i := 0; i < num; i++ {
-		chanPoints[i] = openChannelAndAssert(t, net, ctx, net.Alice,
+		chanPoints[i] = openChannelAndAssert(ct, net, ctx, net.Alice,
 			net.Bob, amount)
 	}
 
 	// Close the channel between Alice and Bob, asserting that the
 	// channel has been properly closed on-chain.
 	for _, chanPoint := range chanPoints {
-		closeChannelAndAssert(t, net, ctx, net.Alice, chanPoint)
+		closeChannelAndAssert(ct, net, ctx, net.Alice, chanPoint)
 	}
 }
 
-type lndTestCase func(net *networkHarness, t *testing.T)
+type testCase func(net *networkHarness, t *testing.T)
 
-var lndTestCases = map[string]lndTestCase{
+var testCases = map[string]testCase{
 	"basic funding flow":          testBasicChannelFunding,
 	"channel force closure":       testChannelForceClosure,
 	"channel balance":             testChannelBalance,
@@ -699,72 +767,69 @@ var lndTestCases = map[string]lndTestCase{
 // TestLightningNetworkDaemon performs a series of integration tests amongst a
 // programmatically driven network of lnd nodes.
 func TestLightningNetworkDaemon(t *testing.T) {
-	var (
+	ct := NewCT(t)
-		btcdHarness      *rpctest.Harness
-		lightningNetwork *networkHarness
-		currentTest      string
-		err              error
-	)
-
-	defer func() {
-		// If one of the integration tests caused a panic within the main
-		// goroutine, then tear down all the harnesses in order to avoid
-		// any leaked processes.
-		if r := recover(); r != nil {
-			fmt.Println("recovering from test panic: ", r)
-			if err := btcdHarness.TearDown(); err != nil {
-				fmt.Println("unable to tear btcd harnesses: ", err)
-			}
-			if err := lightningNetwork.TearDownAll(); err != nil {
-				fmt.Println("unable to tear lnd harnesses: ", err)
-			}
-			t.Fatalf("test %v panicked: %s", currentTest, debug.Stack())
-		}
-	}()
 
 	// First create the network harness to gain access to its
 	// 'OnTxAccepted' call back.
-	lightningNetwork, err = newNetworkHarness()
+	lndHarness, err := newNetworkHarness()
 	if err != nil {
-		t.Fatalf("unable to create lightning network harness: %v", err)
+		ct.Fatalf("unable to create lightning network harness: %v", err)
 	}
-	defer lightningNetwork.TearDownAll()
+	defer lndHarness.TearDownAll()
 
 	handlers := &btcrpcclient.NotificationHandlers{
-		OnTxAccepted: lightningNetwork.OnTxAccepted,
+		OnTxAccepted: lndHarness.OnTxAccepted,
 	}
 
 	// First create an instance of the btcd's rpctest.Harness. This will be
 	// used to fund the wallets of the nodes within the test network and to
 	// drive blockchain related events within the network.
-	btcdHarness, err = rpctest.New(harnessNetParams, handlers, nil)
+	btcdHarness, err := rpctest.New(harnessNetParams, handlers, nil)
 	if err != nil {
-		t.Fatalf("unable to create mining node: %v", err)
+		ct.Fatalf("unable to create mining node: %v", err)
 	}
 	defer btcdHarness.TearDown()
-	if err = btcdHarness.SetUp(true, 50); err != nil {
+	if err := btcdHarness.SetUp(true, 50); err != nil {
-		t.Fatalf("unable to set up mining node: %v", err)
+		ct.Fatalf("unable to set up mining node: %v", err)
 	}
 	if err := btcdHarness.Node.NotifyNewTransactions(false); err != nil {
-		t.Fatalf("unable to request transaction notifications: %v", err)
+		ct.Fatalf("unable to request transaction notifications: %v", err)
 	}
 
 	// With the btcd harness created, we can now complete the
 	// initialization of the network. args - list of lnd arguments,
 	// example: "--debuglevel=debug"
 	// TODO(roasbeef): create master balanced channel with all the monies?
-	if err := lightningNetwork.InitializeSeedNodes(btcdHarness, nil); err != nil {
+	if err := lndHarness.InitializeSeedNodes(btcdHarness, nil); err != nil {
-		t.Fatalf("unable to initialize seed nodes: %v", err)
+		ct.Fatalf("unable to initialize seed nodes: %v", err)
 	}
-	if err = lightningNetwork.SetUp(); err != nil {
+	if err = lndHarness.SetUp(); err != nil {
-		t.Fatalf("unable to set up test lightning network: %v", err)
+		ct.Fatalf("unable to set up test lightning network: %v", err)
 	}
 
-	t.Logf("Running %v integration tests", len(lndTestCases))
+	ct.Logf("Running %v integration tests", len(testCases))
-	for testName, lnTest := range lndTestCases {
-		t.Logf("Executing test %v", testName)
 
-		currentTest = testName
+	for name, test := range testCases {
-		lnTest(lightningNetwork, t)
+		errChan := ct.RunTest(lndHarness, test)
+
+		select {
+		// Receive both types of err - panic and fatal from
+		// one channel and raise the fatal in main goroutine.
+		case err := <-errChan:
+			if err != nil {
+				ct.Fatalf("Fail: (%v): exited with error: \n%v",
+					name, err)
+			}
+			ct.Logf("Successed: (%v)", name)
+
+		// In this case lightning node process finished with error
+		// status. It might be because of wrong flag, or it might
+		// be because of nil pointer access. Who knows!? Who knows...
+		// TODO(andrew.shvv) When two nodes are closing simultanisly
+		// it leads to panic - 'sending to the closed channel'. Fix it?
+		case err := <-lndHarness.lndErrorChan:
+			ct.Fatalf("Fail:  (%v): lnd finished with error "+
+				"(stderr): \n%v", name, err)
+		}
 	}
 }

networktest.go

@@ -19,6 +19,8 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/grpclog"
 
+	"bytes"
+	"github.com/go-errors/errors"
 	"github.com/lightningnetwork/lnd/lnrpc"
 	"github.com/roasbeef/btcd/chaincfg"
 	"github.com/roasbeef/btcd/rpctest"
@@ -159,14 +161,27 @@ func (l *lightningNode) genArgs() []string {
 // start launches a new process running lnd. Additionally, the PID of the
 // launched process is saved in order to possibly kill the process forcibly
 // later.
-func (l *lightningNode) start() error {
+func (l *lightningNode) start(lndError chan error) error {
 	args := l.genArgs()
 
 	l.cmd = exec.Command("lnd", args...)
+
+	// Redirect stderr output to buffer
+	var errb bytes.Buffer
+	l.cmd.Stderr = &errb
+
 	if err := l.cmd.Start(); err != nil {
 		return err
 	}
 
+	go func() {
+		// If lightning node process exited with error status
+		// then we should transmit stderr output in main process.
+		if err := l.cmd.Wait(); err != nil {
+			lndError <- errors.New(errb.String())
+		}
+	}()
+
 	pid, err := os.Create(filepath.Join(l.cfg.DataDir,
 		fmt.Sprintf("%s.pid", l.nodeId)))
 	if err != nil {
@@ -234,7 +249,14 @@ func (l *lightningNode) cleanup() error {
 
 // stop attempts to stop the active lnd process.
 func (l *lightningNode) stop() error {
-	if l.cmd == nil || l.cmd.Process == nil {
+
+	// We should skip node stop in case:
+	// - start of the node wasn't initiated
+	// - process wasn't spawned
+	// - process already finished
+	if l.cmd == nil ||
+		l.cmd.Process == nil ||
+		(l.cmd.ProcessState != nil && l.cmd.ProcessState.Exited()) {
 		return nil
 	}
 
@@ -276,6 +298,10 @@ type networkHarness struct {
 	seenTxns      chan wire.ShaHash
 	watchRequests chan *watchRequest
 
+	// Channel for transmitting stderr output from failed lightning node
+	// to main process.
+	lndErrorChan chan error
+
 	sync.Mutex
 }
 
@@ -288,6 +314,7 @@ func newNetworkHarness() (*networkHarness, error) {
 		activeNodes:   make(map[int]*lightningNode),
 		seenTxns:      make(chan wire.ShaHash),
 		watchRequests: make(chan *watchRequest),
+		lndErrorChan:  make(chan error),
 	}, nil
 }
 
@@ -345,7 +372,7 @@ func (n *networkHarness) SetUp() error {
 	go func() {
 		var err error
 		defer wg.Done()
-		if err = n.Alice.start(); err != nil {
+		if err = n.Alice.start(n.lndErrorChan); err != nil {
 			errChan <- err
 			return
 		}
@@ -353,7 +380,7 @@ func (n *networkHarness) SetUp() error {
 	go func() {
 		var err error
 		defer wg.Done()
-		if err = n.Bob.start(); err != nil {
+		if err = n.Bob.start(n.lndErrorChan); err != nil {
 			errChan <- err
 			return
 		}
@@ -462,7 +489,7 @@ func (n *networkHarness) NewNode(extraArgs []string) (*lightningNode, error) {
 		return nil, err
 	}
 
-	if err := node.start(); err != nil {
+	if err := node.start(n.lndErrorChan); err != nil {
 		return nil, err
 	}