From a51e1a53973ba2387d1f87677c814050e4fa7d5b Mon Sep 17 00:00:00 2001 From: Alan Chen Date: Thu, 21 Sep 2017 17:50:46 +0800 Subject: [PATCH 1/4] benchmark/kubernetes: fix consensus --- .../validator/templates/service.yaml | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100755 benchmark/kubernetes/validator/templates/service.yaml diff --git a/benchmark/kubernetes/validator/templates/service.yaml b/benchmark/kubernetes/validator/templates/service.yaml new file mode 100755 index 00000000..8d5dd4d9 --- /dev/null +++ b/benchmark/kubernetes/validator/templates/service.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "fullname" . }} + labels: + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + environment: {{ template "environment" . }} +spec: + type: {{ .Values.service.type }} + {{- if .Values.service.staticIP }} + clusterIP: {{ .Values.service.staticIP | quote }} + {{- end }} + ports: + - port: {{ .Values.ethereum.port }} + targetPort: {{ .Values.ethereum.port }} + protocol: TCP + name: p2p + selector: + app: {{ template "fullname" . }} From 0afa3f3bd7a0435c06d5da7e11297f91552f93f1 Mon Sep 17 00:00:00 2001 From: Alan Chen Date: Thu, 21 Sep 2017 17:52:47 +0800 Subject: [PATCH 2/4] charts, common, genesis, k8s: Fix bugs about ether sending --- charts/genesis.go | 2 ++ common/transactions.go | 11 ++++++++--- genesis/genesis.go | 1 + k8s/blockchain.go | 8 +++----- k8s/ethereum.go | 29 ++++++++++++++++++++++------- 5 files changed, 36 insertions(+), 15 deletions(-) diff --git a/charts/genesis.go b/charts/genesis.go index b9dbd947..822081ec 100644 --- a/charts/genesis.go +++ b/charts/genesis.go @@ -18,6 +18,7 @@ package charts import ( "fmt" + "math/big" "os" "path/filepath" "strings" @@ -51,6 +52,7 @@ func NewGenesisChart(addrs []common.Address, gasLimit uint64) *GenesisChart { false, genesis.Validators(addrs...), genesis.GasLimit(gasLimit), + genesis.Alloc(addrs, new(big.Int).Exp(big.NewInt(10), big.NewInt(50), nil)), ), } diff --git a/common/transactions.go b/common/transactions.go index 6306c0c6..1139af66 100644 --- a/common/transactions.go +++ b/common/transactions.go @@ -27,15 +27,20 @@ import ( "github.com/getamis/istanbul-tools/client" ) +var ( + DefaultGasPrice int64 = 20000000000 + DefaultGasLimit int64 = 22000 // the gas of ether tx should be 21000 +) + func SendEther(client *client.Client, from *ecdsa.PrivateKey, to common.Address, amount *big.Int, nonce uint64) error { - tx := types.NewTransaction(nonce, to, amount, nil, nil, []byte{}) - signedTx, err := types.SignTx(tx, types.EIP155Signer{}, from) + tx := types.NewTransaction(nonce, to, amount, big.NewInt(DefaultGasLimit), big.NewInt(DefaultGasPrice), []byte{}) + signedTx, err := types.SignTx(tx, types.NewEIP155Signer(big.NewInt(2017)), from) if err != nil { log.Error("Failed to sign transaction", "tx", tx, "err", err) return err } - err = client.SendRawTransaction(context.Background(), tx) + err = client.SendRawTransaction(context.Background(), signedTx) if err != nil { log.Error("Failed to send transaction", "tx", signedTx, "nonce", nonce, "err", err) return err diff --git a/genesis/genesis.go b/genesis/genesis.go index 0aba7135..0a2800ab 100644 --- a/genesis/genesis.go +++ b/genesis/genesis.go @@ -46,6 +46,7 @@ func New(options ...Option) *core.Genesis { Difficulty: big.NewInt(InitDifficulty), Alloc: make(core.GenesisAlloc), Config: ¶ms.ChainConfig{ + ChainId: big.NewInt(2017), HomesteadBlock: big.NewInt(1), EIP150Block: big.NewInt(2), EIP155Block: big.NewInt(3), diff --git a/k8s/blockchain.go b/k8s/blockchain.go index 9879d5c4..f3a6b879 100644 --- a/k8s/blockchain.go +++ b/k8s/blockchain.go @@ -94,10 +94,6 @@ func (bc *blockchain) Stop(force bool) error { } func (bc *blockchain) Finalize() { - for _, v := range bc.validators { - v.Stop() - } - bc.staticNodes.Uninstall() bc.genesis.Uninstall() } @@ -121,7 +117,9 @@ func (bc *blockchain) setupValidators(num int, nodekeys []string, ips []string, opts..., ) - bc.validators = append(bc.validators, geth) + if geth != nil { + bc.validators = append(bc.validators, geth) + } } } diff --git a/k8s/ethereum.go b/k8s/ethereum.go index 9570972c..79753146 100644 --- a/k8s/ethereum.go +++ b/k8s/ethereum.go @@ -21,6 +21,7 @@ import ( "crypto/ecdsa" "errors" "math/big" + "strings" "sync" "time" @@ -106,11 +107,18 @@ func (eth *ethereum) DockerBinds() []string { } func (eth *ethereum) NewClient() *client.Client { - client, err := client.Dial("ws://" + eth.Host() + ":8546") - if err != nil { - return nil + for i := 0; i < healthCheckRetryCount; i++ { + client, err := client.Dial("ws://" + eth.Host() + ":8546") + if err != nil { + log.Warn("Failed to create client", "err", err) + <-time.After(healthCheckRetryDelay) + continue + } else { + return client + } } - return client + + return nil } func (eth *ethereum) NodeAddress() string { @@ -350,9 +358,16 @@ func (eth *ethereum) Accounts() []accounts.Account { // ---------------------------------------------------------------------------- func (eth *ethereum) Host() string { - svc, err := eth.k8sClient.CoreV1().Services(defaultNamespace).Get(eth.chart.Name()+"-0", metav1.GetOptions{}) - if err != nil { + index := strings.LastIndex(eth.chart.Name(), "-") + if index < 0 { + log.Error("Invalid validator pod name") return "" } - return svc.Spec.LoadBalancerIP + name := "validator-service-" + eth.chart.Name()[index+1:] + svc, err := eth.k8sClient.CoreV1().Services(defaultNamespace).Get(name, metav1.GetOptions{}) + if err != nil { + log.Error("Failed to find service", "svc", name, "err", err) + return "" + } + return svc.Status.LoadBalancer.Ingress[0].IP } From 5eb5cf7e397d71ec88ca84423b6108fdf2bfb439 Mon Sep 17 00:00:00 2001 From: Alan Chen Date: Thu, 21 Sep 2017 17:53:35 +0800 Subject: [PATCH 3/4] k8s, tests/load: add ether sending to load testing --- k8s/ethereum.go | 2 +- k8s/transactor.go | 56 +++++++++++++++++++++++++++++++ tests/load/load_test.go | 74 ++++++++++++++++++----------------------- 3 files changed, 90 insertions(+), 42 deletions(-) create mode 100644 k8s/transactor.go diff --git a/k8s/ethereum.go b/k8s/ethereum.go index 79753146..1d43a9df 100644 --- a/k8s/ethereum.go +++ b/k8s/ethereum.go @@ -363,7 +363,7 @@ func (eth *ethereum) Host() string { log.Error("Invalid validator pod name") return "" } - name := "validator-service-" + eth.chart.Name()[index+1:] + name := "validator-svc-" + eth.chart.Name()[index+1:] svc, err := eth.k8sClient.CoreV1().Services(defaultNamespace).Get(name, metav1.GetOptions{}) if err != nil { log.Error("Failed to find service", "svc", name, "err", err) diff --git a/k8s/transactor.go b/k8s/transactor.go new file mode 100644 index 00000000..a3a532a9 --- /dev/null +++ b/k8s/transactor.go @@ -0,0 +1,56 @@ +// Copyright 2017 AMIS Technologies +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package k8s + +import ( + "context" + "errors" + "math/big" + "time" + + "github.com/ethereum/go-ethereum/common" + + istcommon "github.com/getamis/istanbul-tools/common" +) + +type Transactor interface { + SendTransactions(context.Context, common.Address, *big.Int, time.Duration) error +} + +func (eth *ethereum) SendTransactions(ctx context.Context, to common.Address, amount *big.Int, duration time.Duration) error { + client := eth.NewClient() + if client == nil { + return errors.New("failed to retrieve client") + } + + nonce, err := client.NonceAt(context.Background(), eth.Address(), nil) + if err != nil { + log.Error("Failed to get nonce", "addr", eth.Address(), "err", err) + return err + } + + timeout := time.After(duration) + for { + select { + case <-timeout: + return nil + default: + _ = istcommon.SendEther(client, eth.key, to, amount, nonce) + nonce++ + } + } +} diff --git a/tests/load/load_test.go b/tests/load/load_test.go index bb6f04f9..d17f4c8a 100644 --- a/tests/load/load_test.go +++ b/tests/load/load_test.go @@ -18,7 +18,6 @@ package load import ( "context" - "fmt" "math/big" "sync" "testing" @@ -28,35 +27,14 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/getamis/istanbul-tools/charts" "github.com/getamis/istanbul-tools/container" "github.com/getamis/istanbul-tools/k8s" "github.com/getamis/istanbul-tools/tests" ) var _ = Describe("TPS-01: Large amount of transactions", func() { - tests.CaseTable("with number of validators", func(numberOfValidators int) { - var svcCharts []*charts.ValidatorServiceChart - - BeforeSuite(func() { - for i := 0; i < numberOfValidators; i++ { - chart := charts.NewValidatorServiceChart(fmt.Sprintf("%d", i), nil) - svcCharts = append(svcCharts, chart) - - if err := chart.Install(false); err != nil { - fmt.Println(err) - } - } - }) - - AfterSuite(func() { - for i := 0; i < numberOfValidators; i++ { - svcCharts[i].Uninstall() - } - }) - tests.CaseTable("with gas limit", func(gaslimit int) { tests.CaseTable("with txpool size", @@ -82,7 +60,10 @@ var _ = Describe("TPS-01: Large amount of transactions", func() { func runTests(numberOfValidators int, gaslimit int, txpoolSize int) { Describe("", func() { var ( - blockchain container.Blockchain + blockchain container.Blockchain + sendEtherAddrs map[common.Address]common.Address + + duration = 10 * time.Minute ) BeforeEach(func() { @@ -91,38 +72,49 @@ func runTests(numberOfValidators int, gaslimit int, txpoolSize int) { uint64(gaslimit), k8s.ImageRepository("quay.io/amis/geth"), k8s.ImageTag("istanbul_develop"), - k8s.ServiceType("LoadBalancer"), k8s.Mine(), k8s.TxPoolSize(txpoolSize), ) + Expect(blockchain).NotTo(BeNil()) Expect(blockchain.Start(true)).To(BeNil()) - tests.WaitFor(blockchain.Validators(), func(geth container.Ethereum, wg *sync.WaitGroup) { - richman, ok := geth.(k8s.RichMan) - Expect(ok).To(BeTrue()) - - var addrs []common.Address - addr := common.HexToAddress("0x1a9afb711302c5f83b5902843d1c007a1a137632") - addrs = append(addrs, addr) - - // Give ether to all accounts - err := richman.GiveEther(context.Background(), addrs, new(big.Int).Exp(big.NewInt(10), big.NewInt(24), nil)) - Expect(err).NotTo(BeNil()) - - err = geth.WaitForBalances(addrs, 10*time.Second) - Expect(err).NotTo(BeNil()) - - wg.Done() - }) + sendEtherAddrs = make(map[common.Address]common.Address) + num := len(blockchain.Validators()) + for i, v := range blockchain.Validators() { + sendEtherAddrs[v.Address()] = blockchain.Validators()[(i+1)%num].Address() + } }) AfterEach(func() { + Expect(blockchain).NotTo(BeNil()) Expect(blockchain.Stop(true)).To(BeNil()) blockchain.Finalize() }) It("", func() { + By("Wait for p2p connection", func() { + tests.WaitFor(blockchain.Validators(), func(geth container.Ethereum, wg *sync.WaitGroup) { + Expect(geth.WaitForPeersConnected(numberOfValidators - 1)).To(BeNil()) + wg.Done() + }) + }) + By("Send transactions", func() { + tests.WaitFor(blockchain.Validators(), func(geth container.Ethereum, wg *sync.WaitGroup) { + transactor, ok := geth.(k8s.Transactor) + Expect(ok).To(BeTrue()) + + Expect( + transactor.SendTransactions( + context.Background(), + sendEtherAddrs[geth.Address()], + new(big.Int).Exp(big.NewInt(10), big.NewInt(3), nil), + duration), + ).To(BeNil()) + + wg.Done() + }) + }) }) }) } From cf2b999685fdd932b01fa3c306cc400de682a1c7 Mon Sep 17 00:00:00 2001 From: Alan Chen Date: Fri, 22 Sep 2017 17:05:30 +0800 Subject: [PATCH 4/4] infra: add load testing deployment configs for terraform --- infra/.gitignore | 2 + infra/load-testing/azure.tf | 74 ++++++++++++++++++++++++++++++++ infra/load-testing/kubernetes.tf | 20 +++++++++ infra/load-testing/variables.tf | 12 ++++++ 4 files changed, 108 insertions(+) create mode 100644 infra/.gitignore create mode 100644 infra/load-testing/azure.tf create mode 100644 infra/load-testing/kubernetes.tf create mode 100644 infra/load-testing/variables.tf diff --git a/infra/.gitignore b/infra/.gitignore new file mode 100644 index 00000000..84fcb582 --- /dev/null +++ b/infra/.gitignore @@ -0,0 +1,2 @@ +*.tfstate* +.terraform diff --git a/infra/load-testing/azure.tf b/infra/load-testing/azure.tf new file mode 100644 index 00000000..ba8b0888 --- /dev/null +++ b/infra/load-testing/azure.tf @@ -0,0 +1,74 @@ +provider "azurerm" { + subscription_id = "${var.subscription_id}" + client_id = "${var.client_id}" + client_secret = "${var.client_secret}" + tenant_id = "${var.tenant_id}" +} + +resource "random_id" "namer" { + keepers = { + resource_group = "res" + container_service = "cs" + dns_prefix = "k8s" + } + + byte_length = 8 +} + +variable "username" { + type = "string" + default = "amis" +} + +resource "azurerm_resource_group" "test" { + name = "istanbul-test-${random_id.namer.hex}" + location = "Southeast Asia" +} + +resource "azurerm_container_service" "test" { + name = "istanbul-test-${random_id.namer.hex}" + location = "${azurerm_resource_group.test.location}" + resource_group_name = "${azurerm_resource_group.test.name}" + orchestration_platform = "Kubernetes" + + master_profile { + count = 1 + dns_prefix = "istanbul-${random_id.namer.hex}" + } + + linux_profile { + admin_username = "${var.username}" + + ssh_key { + key_data = "${file("~/.ssh/id_rsa.pub")}" + } + } + + agent_pool_profile { + name = "default" + count = "${length(var.svcs)}" + dns_prefix = "agent-${random_id.namer.hex}" + vm_size = "Standard_DS2_v2" + } + + service_principal { + client_id = "${var.client_id}" + client_secret = "${var.client_secret}" + } + + diagnostics_profile { + enabled = false + } + + tags { + Environment = "Testing" + } +} + +resource "null_resource" "kubeconfig" { + provisioner "local-exec" { + command = "sleep 10 && scp -o StrictHostKeyChecking=no ${var.username}@istanbul-${random_id.namer.hex}.${azurerm_resource_group.test.location}.cloudapp.azure.com:~/.kube/config ~/.kube/config" + interpreter = ["bash", "-c"] + } + depends_on = ["azurerm_container_service.test"] +} diff --git a/infra/load-testing/kubernetes.tf b/infra/load-testing/kubernetes.tf new file mode 100644 index 00000000..3d61a398 --- /dev/null +++ b/infra/load-testing/kubernetes.tf @@ -0,0 +1,20 @@ +resource "kubernetes_service" "validator-svc" { + metadata { + name = "validator-svc-${count.index}" + } + + spec { + selector { + app = "validator-${count.index}" + } + type = "LoadBalancer" + port { + port = 8546 + target_port = 8546 + } + + type = "LoadBalancer" + } + + count = "${length(var.svcs)}" +} \ No newline at end of file diff --git a/infra/load-testing/variables.tf b/infra/load-testing/variables.tf new file mode 100644 index 00000000..80ec9d02 --- /dev/null +++ b/infra/load-testing/variables.tf @@ -0,0 +1,12 @@ +variable "count" { + default = 4 +} + +variable "svcs" { + default = [0, 1, 2, 3] +} + +variable "subscription_id" {} +variable "client_id" {} +variable "client_secret" {} +variable "tenant_id" {}