Merge pull request #84 from getamis/feature/send-tx-in-load-testing

Load testing core

benchmark/kubernetes/validator/templates/service.yaml

@@ -0,0 +1,21 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "fullname" . }}
+  labels:
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    release: "{{ .Release.Name }}"
+    heritage: "{{ .Release.Service }}"
+    environment: {{ template "environment" . }}
+spec:
+  type: {{ .Values.service.type }}
+  {{- if .Values.service.staticIP }}
+  clusterIP: {{ .Values.service.staticIP | quote }}
+  {{- end }}
+  ports:
+  - port: {{ .Values.ethereum.port }}
+    targetPort: {{ .Values.ethereum.port }}
+    protocol: TCP
+    name: p2p
+  selector:
+    app: {{ template "fullname" . }}

charts/genesis.go

@@ -18,6 +18,7 @@ package charts
 
 import (
 	"fmt"
+	"math/big"
 	"os"
 	"path/filepath"
 	"strings"
@@ -51,6 +52,7 @@ func NewGenesisChart(addrs []common.Address, gasLimit uint64) *GenesisChart {
 			false,
 			genesis.Validators(addrs...),
 			genesis.GasLimit(gasLimit),
+			genesis.Alloc(addrs, new(big.Int).Exp(big.NewInt(10), big.NewInt(50), nil)),
 		),
 	}
 

common/transactions.go

@@ -27,15 +27,20 @@ import (
 	"github.com/getamis/istanbul-tools/client"
 )
 
+var (
+	DefaultGasPrice int64 = 20000000000
+	DefaultGasLimit int64 = 22000 // the gas of ether tx should be 21000
+)
+
 func SendEther(client *client.Client, from *ecdsa.PrivateKey, to common.Address, amount *big.Int, nonce uint64) error {
-	tx := types.NewTransaction(nonce, to, amount, nil, nil, []byte{})
-	signedTx, err := types.SignTx(tx, types.EIP155Signer{}, from)
+	tx := types.NewTransaction(nonce, to, amount, big.NewInt(DefaultGasLimit), big.NewInt(DefaultGasPrice), []byte{})
+	signedTx, err := types.SignTx(tx, types.NewEIP155Signer(big.NewInt(2017)), from)
 	if err != nil {
 		log.Error("Failed to sign transaction", "tx", tx, "err", err)
 		return err
 	}
 
-	err = client.SendRawTransaction(context.Background(), tx)
+	err = client.SendRawTransaction(context.Background(), signedTx)
 	if err != nil {
 		log.Error("Failed to send transaction", "tx", signedTx, "nonce", nonce, "err", err)
 		return err

genesis/genesis.go

@@ -46,6 +46,7 @@ func New(options ...Option) *core.Genesis {
 		Difficulty: big.NewInt(InitDifficulty),
 		Alloc:      make(core.GenesisAlloc),
 		Config: &params.ChainConfig{
+			ChainId:        big.NewInt(2017),
 			HomesteadBlock: big.NewInt(1),
 			EIP150Block:    big.NewInt(2),
 			EIP155Block:    big.NewInt(3),

infra/.gitignore

@@ -0,0 +1,2 @@
+*.tfstate*
+.terraform

infra/load-testing/azure.tf

@@ -0,0 +1,74 @@
+provider "azurerm" {
+  subscription_id = "${var.subscription_id}"
+  client_id       = "${var.client_id}"
+  client_secret   = "${var.client_secret}"
+  tenant_id       = "${var.tenant_id}"
+}
+
+resource "random_id" "namer" {
+  keepers = {
+    resource_group    = "res"
+    container_service = "cs"
+    dns_prefix        = "k8s"
+  }
+
+  byte_length = 8
+}
+
+variable "username" {
+  type = "string"
+  default = "amis"
+}
+
+resource "azurerm_resource_group" "test" {
+  name     = "istanbul-test-${random_id.namer.hex}"
+  location = "Southeast Asia"
+}
+
+resource "azurerm_container_service" "test" {
+  name                   = "istanbul-test-${random_id.namer.hex}"
+  location               = "${azurerm_resource_group.test.location}"
+  resource_group_name    = "${azurerm_resource_group.test.name}"
+  orchestration_platform = "Kubernetes"
+
+  master_profile {
+    count      = 1
+    dns_prefix = "istanbul-${random_id.namer.hex}"
+  }
+
+  linux_profile {
+    admin_username = "${var.username}"
+
+    ssh_key {
+      key_data = "${file("~/.ssh/id_rsa.pub")}"
+    }
+  }
+
+  agent_pool_profile {
+    name       = "default"
+    count      = "${length(var.svcs)}"
+    dns_prefix = "agent-${random_id.namer.hex}"
+    vm_size    = "Standard_DS2_v2"
+  }
+
+  service_principal {
+    client_id     = "${var.client_id}"
+    client_secret = "${var.client_secret}"
+  }
+
+  diagnostics_profile {
+    enabled = false
+  }
+
+  tags {
+    Environment = "Testing"
+  }
+}
+
+resource "null_resource" "kubeconfig" {
+  provisioner "local-exec" {
+    command = "sleep 10 && scp -o StrictHostKeyChecking=no ${var.username}@istanbul-${random_id.namer.hex}.${azurerm_resource_group.test.location}.cloudapp.azure.com:~/.kube/config ~/.kube/config"
+    interpreter = ["bash", "-c"]
+  }
+  depends_on = ["azurerm_container_service.test"]
+}

infra/load-testing/kubernetes.tf

@@ -0,0 +1,20 @@
+resource "kubernetes_service" "validator-svc" {
+  metadata {
+    name = "validator-svc-${count.index}"
+  }
+
+  spec {
+    selector {
+      app = "validator-${count.index}"
+    }
+    type = "LoadBalancer"
+    port {
+      port = 8546
+      target_port = 8546
+    }
+
+    type = "LoadBalancer"
+  }
+
+  count = "${length(var.svcs)}"
+}

infra/load-testing/variables.tf

@@ -0,0 +1,12 @@
+variable "count" {
+  default = 4
+}
+
+variable "svcs" {
+  default = [0, 1, 2, 3]
+}
+
+variable "subscription_id" {}
+variable "client_id" {}
+variable "client_secret" {}
+variable "tenant_id" {}

k8s/blockchain.go

@@ -94,10 +94,6 @@ func (bc *blockchain) Stop(force bool) error {
 }
 
 func (bc *blockchain) Finalize() {
-	for _, v := range bc.validators {
-		v.Stop()
-	}
-
 	bc.staticNodes.Uninstall()
 	bc.genesis.Uninstall()
 }
@@ -121,7 +117,9 @@ func (bc *blockchain) setupValidators(num int, nodekeys []string, ips []string,
 			opts...,
 		)
 
-		bc.validators = append(bc.validators, geth)
+		if geth != nil {
+			bc.validators = append(bc.validators, geth)
+		}
 	}
 }
 

k8s/ethereum.go

@@ -21,6 +21,7 @@ import (
 	"crypto/ecdsa"
 	"errors"
 	"math/big"
+	"strings"
 	"sync"
 	"time"
 
@@ -106,11 +107,18 @@ func (eth *ethereum) DockerBinds() []string {
 }
 
 func (eth *ethereum) NewClient() *client.Client {
-	client, err := client.Dial("ws://" + eth.Host() + ":8546")
-	if err != nil {
-		return nil
+	for i := 0; i < healthCheckRetryCount; i++ {
+		client, err := client.Dial("ws://" + eth.Host() + ":8546")
+		if err != nil {
+			log.Warn("Failed to create client", "err", err)
+			<-time.After(healthCheckRetryDelay)
+			continue
+		} else {
+			return client
+		}
 	}
-	return client
+
+	return nil
 }
 
 func (eth *ethereum) NodeAddress() string {
@@ -350,9 +358,16 @@ func (eth *ethereum) Accounts() []accounts.Account {
 // ----------------------------------------------------------------------------
 
 func (eth *ethereum) Host() string {
-	svc, err := eth.k8sClient.CoreV1().Services(defaultNamespace).Get(eth.chart.Name()+"-0", metav1.GetOptions{})
-	if err != nil {
+	index := strings.LastIndex(eth.chart.Name(), "-")
+	if index < 0 {
+		log.Error("Invalid validator pod name")
 		return ""
 	}
-	return svc.Spec.LoadBalancerIP
+	name := "validator-svc-" + eth.chart.Name()[index+1:]
+	svc, err := eth.k8sClient.CoreV1().Services(defaultNamespace).Get(name, metav1.GetOptions{})
+	if err != nil {
+		log.Error("Failed to find service", "svc", name, "err", err)
+		return ""
+	}
+	return svc.Status.LoadBalancer.Ingress[0].IP
 }

k8s/transactor.go

@@ -0,0 +1,56 @@
+// Copyright 2017 AMIS Technologies
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package k8s
+
+import (
+	"context"
+	"errors"
+	"math/big"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+
+	istcommon "github.com/getamis/istanbul-tools/common"
+)
+
+type Transactor interface {
+	SendTransactions(context.Context, common.Address, *big.Int, time.Duration) error
+}
+
+func (eth *ethereum) SendTransactions(ctx context.Context, to common.Address, amount *big.Int, duration time.Duration) error {
+	client := eth.NewClient()
+	if client == nil {
+		return errors.New("failed to retrieve client")
+	}
+
+	nonce, err := client.NonceAt(context.Background(), eth.Address(), nil)
+	if err != nil {
+		log.Error("Failed to get nonce", "addr", eth.Address(), "err", err)
+		return err
+	}
+
+	timeout := time.After(duration)
+	for {
+		select {
+		case <-timeout:
+			return nil
+		default:
+			_ = istcommon.SendEther(client, eth.key, to, amount, nonce)
+			nonce++
+		}
+	}
+}

tests/load/load_test.go

@@ -18,7 +18,6 @@ package load
 
 import (
 	"context"
-	"fmt"
 	"math/big"
 	"sync"
 	"testing"
@@ -28,35 +27,14 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/getamis/istanbul-tools/charts"
 	"github.com/getamis/istanbul-tools/container"
 	"github.com/getamis/istanbul-tools/k8s"
 	"github.com/getamis/istanbul-tools/tests"
 )
 
 var _ = Describe("TPS-01: Large amount of transactions", func() {
-
 	tests.CaseTable("with number of validators",
 		func(numberOfValidators int) {
-			var svcCharts []*charts.ValidatorServiceChart
-
-			BeforeSuite(func() {
-				for i := 0; i < numberOfValidators; i++ {
-					chart := charts.NewValidatorServiceChart(fmt.Sprintf("%d", i), nil)
-					svcCharts = append(svcCharts, chart)
-
-					if err := chart.Install(false); err != nil {
-						fmt.Println(err)
-					}
-				}
-			})
-
-			AfterSuite(func() {
-				for i := 0; i < numberOfValidators; i++ {
-					svcCharts[i].Uninstall()
-				}
-			})
-
 			tests.CaseTable("with gas limit",
 				func(gaslimit int) {
 					tests.CaseTable("with txpool size",
@@ -82,7 +60,10 @@ var _ = Describe("TPS-01: Large amount of transactions", func() {
 func runTests(numberOfValidators int, gaslimit int, txpoolSize int) {
 	Describe("", func() {
 		var (
-			blockchain container.Blockchain
+			blockchain     container.Blockchain
+			sendEtherAddrs map[common.Address]common.Address
+
+			duration = 10 * time.Minute
 		)
 
 		BeforeEach(func() {
@@ -91,38 +72,49 @@ func runTests(numberOfValidators int, gaslimit int, txpoolSize int) {
 				uint64(gaslimit),
 				k8s.ImageRepository("quay.io/amis/geth"),
 				k8s.ImageTag("istanbul_develop"),
-				k8s.ServiceType("LoadBalancer"),
 				k8s.Mine(),
 				k8s.TxPoolSize(txpoolSize),
 			)
+			Expect(blockchain).NotTo(BeNil())
 			Expect(blockchain.Start(true)).To(BeNil())
 
-			tests.WaitFor(blockchain.Validators(), func(geth container.Ethereum, wg *sync.WaitGroup) {
-				richman, ok := geth.(k8s.RichMan)
-				Expect(ok).To(BeTrue())
-
-				var addrs []common.Address
-				addr := common.HexToAddress("0x1a9afb711302c5f83b5902843d1c007a1a137632")
-				addrs = append(addrs, addr)
-
-				// Give ether to all accounts
-				err := richman.GiveEther(context.Background(), addrs, new(big.Int).Exp(big.NewInt(10), big.NewInt(24), nil))
-				Expect(err).NotTo(BeNil())
-
-				err = geth.WaitForBalances(addrs, 10*time.Second)
-				Expect(err).NotTo(BeNil())
-
-				wg.Done()
-			})
+			sendEtherAddrs = make(map[common.Address]common.Address)
+			num := len(blockchain.Validators())
+			for i, v := range blockchain.Validators() {
+				sendEtherAddrs[v.Address()] = blockchain.Validators()[(i+1)%num].Address()
+			}
 		})
 
 		AfterEach(func() {
+			Expect(blockchain).NotTo(BeNil())
 			Expect(blockchain.Stop(true)).To(BeNil())
 			blockchain.Finalize()
 		})
 
 		It("", func() {
+			By("Wait for p2p connection", func() {
+				tests.WaitFor(blockchain.Validators(), func(geth container.Ethereum, wg *sync.WaitGroup) {
+					Expect(geth.WaitForPeersConnected(numberOfValidators - 1)).To(BeNil())
+					wg.Done()
+				})
+			})
 
+			By("Send transactions", func() {
+				tests.WaitFor(blockchain.Validators(), func(geth container.Ethereum, wg *sync.WaitGroup) {
+					transactor, ok := geth.(k8s.Transactor)
+					Expect(ok).To(BeTrue())
+
+					Expect(
+						transactor.SendTransactions(
+							context.Background(),
+							sendEtherAddrs[geth.Address()],
+							new(big.Int).Exp(big.NewInt(10), big.NewInt(3), nil),
+							duration),
+					).To(BeNil())
+
+					wg.Done()
+				})
+			})
 		})
 	})
 }