diff --git a/benchmark/kubernetes/delete-validator b/benchmark/kubernetes/delete-validator index b7b06553..ccf612e4 100755 --- a/benchmark/kubernetes/delete-validator +++ b/benchmark/kubernetes/delete-validator @@ -3,4 +3,5 @@ from subprocess import call for v in range(0, 26, 1): + call(["helm" ,"delete", "--purge", "validator-svc-{}".format(v)]) call(["helm" ,"delete", "--purge", "validator-{}".format(v)]) diff --git a/benchmark/kubernetes/deploy-validator b/benchmark/kubernetes/deploy-validator index d5c6864f..2af99c21 100755 --- a/benchmark/kubernetes/deploy-validator +++ b/benchmark/kubernetes/deploy-validator @@ -3,4 +3,5 @@ from subprocess import call for v in range(0, 26, 1): + call(["helm" ,"install", "-n", "validator-svc-{}".format(v), "-f", "values.validator-{}.yaml".format(v), "validator-service"]) call(["helm" ,"install", "-n", "validator-{}".format(v), "-f", "values.validator-{}.yaml".format(v), "validator"]) diff --git a/benchmark/kubernetes/validator-service/.helmignore b/benchmark/kubernetes/validator-service/.helmignore new file mode 100755 index 00000000..f0c13194 --- /dev/null +++ b/benchmark/kubernetes/validator-service/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/benchmark/kubernetes/validator-service/Chart.yaml b/benchmark/kubernetes/validator-service/Chart.yaml new file mode 100755 index 00000000..0e1c623f --- /dev/null +++ b/benchmark/kubernetes/validator-service/Chart.yaml @@ -0,0 +1,10 @@ +name: validator-service +home: https://github.com/maichain/mchain_service +apiVersion: v1 +description: A reference Helm chart for Ethereum validator +maintainers: + - name: Alan Chen + email: alan@am.is +version: 0.0.1 +sources: + - https://github.com/maichain/mchain_service/kubernetes/amis/generic/validator diff --git a/benchmark/kubernetes/validator-service/LICENSE b/benchmark/kubernetes/validator-service/LICENSE new file mode 100644 index 00000000..bf43db95 --- /dev/null +++ b/benchmark/kubernetes/validator-service/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 AMIS Technologies + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/benchmark/kubernetes/validator-service/README.md b/benchmark/kubernetes/validator-service/README.md new file mode 100644 index 00000000..fbe1e368 --- /dev/null +++ b/benchmark/kubernetes/validator-service/README.md @@ -0,0 +1,74 @@ +# Validator Helm Chart + +A validator is an Ethereum client to validate transactions and generate blocks. + +## Prerequisites +* [StatefulSets](https://kubernetes.io/docs/concepts/abstractions/controllers/statefulsets/) support +* Dynamically provisioned persistent volumes + +## Installing the Chart + +To install the chart: + +```bash +helm install \ + --name ${DEPLOYMENT_ENVIRONMENT}-transactor-${NAME} \ + kubernetes/amis/generic/validator +``` + +## Configuration + +The following tables lists the configurable parameters of the transactor chart and their default values. + +| Parameter | Description | Default | +| ------------------------------- | ------------------------------------------------- | ---------------------------------------------------------- | +| `service.type` | The Kubernetes service type | `ClusterIP` | +| `service.externalPeerPort` | Exposed Ethereum P2P port | `30303` | +| `service.ExternalRPCPort` | Exposed Ethereum RPC port | `8545` | +| `service.ExternalWSPort` | Exposed Ethereum WebSocket port | `8546` | +| `replicaCount` | Kubernetes statefulset replicas | `1` | +| `image.repository` | Docker image repository | `ethereum/client-go` | +| `image.tag` | Docker image tag | `alpine` | +| `image.pullPolicy` | Docker image pulling policy | `Always` | +| `ethereum.identity` | Custom node name | `$POD_NAME` | +| `ethereum.port` | Network listening port | `30303` | +| `ethereum.networkID` | Network identifier | `12345` | +| `ethereum.cache` | Megabytes of memory allocated to internal caching | `512` | +| `ethereum.rpc.enabled` | Enable the HTTP-RPC server | `false` | +| `ethereum.rpc.addr` | HTTP-RPC server listening interface | `localhost` | +| `ethereum.rpc.port` | HTTP-RPC server listening port | `8545` | +| `ethereum.rpc.api` | API's offered over the HTTP-RPC interface | `"eth,net,web3,personal"` | +| `ethereum.rpc.corsdomain` | Comma separated list of domains from which to accept cross origin requests | `*` | +| `ethereum.ws.enabled` | Enable the WS-RPC server | `false` | +| `ethereum.ws.addr` | WS-RPC server listening interface | `localhost` | +| `ethereum.ws.port` | WS-RPC server listening port | `8546` | +| `ethereum.ws.api` | API's offered over the WS-RPC interface | `"eth,net,web3,personal"` | +| `ethereum.ws.origins` | Origins from which to accept websockets requests | `*` | +| `ethereum.mining.enabled` | Enable mining | `true` | +| `ethereum.mining.threads` | Number of CPU threads to use for mining | `2` | +| `ethereum.mining.etherbase` | Public address for block mining rewards | `"1a9afb711302c5f83b5902843d1c007a1a137632"` | +| `ethereum.ethstats.enabled` | Enable ethstats reporting | `true` | +| `ethereum.ethstats.addr` | Ethstats service address | `ws://eth-netstats` | +| `ethereum.ethstats.port` | Ethstats service port | `3000` | +| `ethereum.ethstats.secret` | Ethstats service websocket secret | `bb98a0b6442386d0cdf8a31b267892c1` | +| `ethereum.nodekey.hex` | P2P node key as hex | `aaaaaaaaaaaaaa5302ccdd5b6ffa092e148ba497e352c2824f366ec399072068` | +| `ethereum.account.address` | Account address assigned to the validator | `1a9afb711302c5f83b5902843d1c007a1a137632` | +| `ethereum.account.data` | Full account data file as JSON string | `{"address":"1a9afb711302c5f83b5902843d1c007a1a137632","Crypto":{"cipher":"aes-128-ctr","ciphertext":"132b50d7c8944a115824de7c00911c40a90f84f27c614b7a3ef05ee8fd414312","cipherparams":{"iv":"0f745599d1b3303988ce210fb82b8c7f"},"kdf":"scrypt","kdfparams":{"dklen":32,"n":262144,"p":1,"r":8,"salt":"bce940bac232b4a9c5a2d50e5be51fde5cecfa7da9d49d8f650f91167bebf0de"},"mac":"36d515070b797aec58a574a3e04ea109498ee7674b15d7f952322cda7dcb68e3"},"id":"5d212b4c-3dd0-4c52-a32f-e42bf1b41133","version":3}` | +| `volumes.chaindir.size` | The maximum size usage of Ethereum data directory | `10Gi` | +| `volumes.chaindir.storageClass` | The Kubernetes storage class of Ethereum data | `$NAMESPACE-chaindata` | +| `global.computingSpec` | The computing spec level of node to schedule | `low` | +| `global.nodeType` | The type of node to schedule | `generic` | + + +Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. + +Alternatively, a YAML file that specifies the values for the parameters can be provided while installing the chart. For example, + +```bash +helm install \ + --name ${DEPLOYMENT_ENVIRONMENT}-validator-${NAME} \ + -f values.yaml \ + kubernetes/amis/generic/validator +``` + +> **Tip**: You can use the default [values.yaml](values.yaml) \ No newline at end of file diff --git a/benchmark/kubernetes/validator-service/templates/NOTES.txt b/benchmark/kubernetes/validator-service/templates/NOTES.txt new file mode 100755 index 00000000..8b137891 --- /dev/null +++ b/benchmark/kubernetes/validator-service/templates/NOTES.txt @@ -0,0 +1 @@ + diff --git a/benchmark/kubernetes/validator-service/templates/_helpers.tpl b/benchmark/kubernetes/validator-service/templates/_helpers.tpl new file mode 100755 index 00000000..54766765 --- /dev/null +++ b/benchmark/kubernetes/validator-service/templates/_helpers.tpl @@ -0,0 +1,23 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "name" -}} +{{- default .Chart.Name | trunc 60 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 60 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "fullname" -}} +{{- $name := .Values.nameOverride -}} +{{- printf "%s-%v" .Chart.Name $name | trunc 60 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Deployment environment +*/}} +{{- define "environment" -}} +{{- default .Release.Namespace -}} +{{- end -}} diff --git a/benchmark/kubernetes/validator/templates/service.yaml b/benchmark/kubernetes/validator-service/templates/service.yaml similarity index 82% rename from benchmark/kubernetes/validator/templates/service.yaml rename to benchmark/kubernetes/validator-service/templates/service.yaml index 6bc61d4b..bcb46133 100755 --- a/benchmark/kubernetes/validator/templates/service.yaml +++ b/benchmark/kubernetes/validator-service/templates/service.yaml @@ -13,10 +13,6 @@ spec: clusterIP: {{ .Values.service.staticIP | quote }} {{- end }} ports: - - port: {{ .Values.service.externalPeerPort }} - targetPort: {{ .Values.ethereum.port }} - protocol: TCP - name: peer {{- if .Values.ethereum.rpc.enabled }} - port: {{ .Values.service.externalRPCPort }} targetPort: {{ .Values.ethereum.rpc.port }} @@ -30,4 +26,4 @@ spec: name: ws {{- end }} selector: - app: {{ template "fullname" . }} + app: {{ .Values.app }} diff --git a/benchmark/kubernetes/validator-service/values.yaml b/benchmark/kubernetes/validator-service/values.yaml new file mode 100755 index 00000000..39096b97 --- /dev/null +++ b/benchmark/kubernetes/validator-service/values.yaml @@ -0,0 +1,21 @@ +# Default values for validator-service. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +nameOverride: + +# service.yaml +service: + type: ClusterIP + externalRPCPort: 8545 + externalWSPort: 8546 + +ethereum: + rpc: + enabled: false + port: 8545 + ws: + enabled: true + port: 8546 + +app: validator \ No newline at end of file diff --git a/charts/validator_svc.go b/charts/validator_svc.go new file mode 100644 index 00000000..87975b16 --- /dev/null +++ b/charts/validator_svc.go @@ -0,0 +1,65 @@ +// Copyright 2017 AMIS Technologies +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package charts + +import ( + "fmt" + "path/filepath" +) + +type ValidatorServiceChart struct { + name string + chartPath string + args []string +} + +func NewValidatorServiceChart(name string, args []string) *ValidatorServiceChart { + chartPath := filepath.Join(chartBasePath, "validator-service") + + chart := &ValidatorServiceChart{ + name: "validator-svc-" + name, + args: args, + chartPath: chartPath, + } + + chart.Override("nameOverride", name) + chart.Override("service.type", "LoadBalancer") + chart.Override("app", "validator-"+name) + + return chart +} + +func (chart *ValidatorServiceChart) Override(key, value string) { + chart.args = append(chart.args, fmt.Sprintf("%s=%s", key, value)) +} + +func (chart *ValidatorServiceChart) Install(debug bool) error { + return installRelease( + chart.name, + chart.args, + chart.chartPath, + debug, + ) +} + +func (chart *ValidatorServiceChart) Uninstall() error { + return uninstallRelease(chart.name) +} + +func (chart *ValidatorServiceChart) Name() string { + return chart.name +} diff --git a/common/transactions.go b/common/transactions.go new file mode 100644 index 00000000..6306c0c6 --- /dev/null +++ b/common/transactions.go @@ -0,0 +1,45 @@ +// Copyright 2017 AMIS Technologies +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package common + +import ( + "context" + "crypto/ecdsa" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + + "github.com/getamis/istanbul-tools/client" +) + +func SendEther(client *client.Client, from *ecdsa.PrivateKey, to common.Address, amount *big.Int, nonce uint64) error { + tx := types.NewTransaction(nonce, to, amount, nil, nil, []byte{}) + signedTx, err := types.SignTx(tx, types.EIP155Signer{}, from) + if err != nil { + log.Error("Failed to sign transaction", "tx", tx, "err", err) + return err + } + + err = client.SendRawTransaction(context.Background(), tx) + if err != nil { + log.Error("Failed to send transaction", "tx", signedTx, "nonce", nonce, "err", err) + return err + } + + return nil +} diff --git a/container/blockchain.go b/container/blockchain.go index e99f2e87..01c8673d 100644 --- a/container/blockchain.go +++ b/container/blockchain.go @@ -44,6 +44,10 @@ const ( defaultPassword = "" ) +type NodeIncubator interface { + CreateNodes(int, ...Option) ([]Ethereum, error) +} + type Blockchain interface { AddValidators(numOfValidators int) ([]Ethereum, error) RemoveValidators(candidates []Ethereum, t time.Duration) error @@ -52,7 +56,6 @@ type Blockchain interface { Stop(bool) error Validators() []Ethereum Finalize() - CreateNodes(int, ...Option) ([]Ethereum, error) } func NewBlockchain(network *DockerNetwork, numOfValidators int, options ...Option) (bc *blockchain) { diff --git a/container/ethereum.go b/container/ethereum.go index 6913e3ec..dfc04053 100644 --- a/container/ethereum.go +++ b/container/ethereum.go @@ -27,6 +27,7 @@ import ( "net" "os" "path/filepath" + "sync" "time" "github.com/docker/docker/api/types" @@ -53,8 +54,8 @@ const ( ) var ( - ErrNoBlock = errors.New("no block generated") - ErrConsensusTimeout = errors.New("consensus timeout") + ErrNoBlock = errors.New("no block generated") + ErrTimeout = errors.New("timeout") ) type Ethereum interface { @@ -77,6 +78,9 @@ type Ethereum interface { // Want for block for no more than the given number during the given time duration WaitForNoBlocks(int, time.Duration) error + // Wait for settling balances for the given accounts + WaitForBalances([]common.Address, ...time.Duration) error + AddPeer(string) error StartMining() error @@ -400,6 +404,7 @@ func (eth *ethereum) ConsensusMonitor(errCh chan<- error, quit chan struct{}) { defer sub.Unsubscribe() timer := time.NewTimer(10 * time.Second) + defer timer.Stop() blockNumber := uint64(0) for { select { @@ -411,7 +416,7 @@ func (eth *ethereum) ConsensusMonitor(errCh chan<- error, quit chan struct{}) { if blockNumber == 0 { errCh <- ErrNoBlock } else { - errCh <- ErrConsensusTimeout + errCh <- ErrTimeout } return case head := <-subCh: @@ -442,6 +447,7 @@ func (eth *ethereum) WaitForProposed(expectedAddress common.Address, timeout tim defer sub.Unsubscribe() timer := time.NewTimer(timeout) + defer timer.Stop() for { select { case err := <-sub.Err(): @@ -464,6 +470,7 @@ func (eth *ethereum) WaitForPeersConnected(expectedPeercount int) error { defer client.Close() ticker := time.NewTicker(time.Second * 1) + defer ticker.Stop() for _ = range ticker.C { infos, err := client.AdminPeers(context.Background()) if err != nil { @@ -472,7 +479,6 @@ func (eth *ethereum) WaitForPeersConnected(expectedPeercount int) error { if len(infos) < expectedPeercount { continue } else { - ticker.Stop() break } } @@ -498,6 +504,7 @@ func (eth *ethereum) WaitForBlocks(num int, waitingTime ...time.Duration) error timeout := time.After(t) ticker := time.NewTicker(time.Millisecond * 500) + defer ticker.Stop() for { select { case <-timeout: @@ -514,7 +521,6 @@ func (eth *ethereum) WaitForBlocks(num int, waitingTime ...time.Duration) error } // Check if new blocks are getting generated if new(big.Int).Sub(n, first).Int64() >= int64(num) { - ticker.Stop() return nil } } @@ -529,13 +535,13 @@ func (eth *ethereum) WaitForBlockHeight(num int) error { defer client.Close() ticker := time.NewTicker(time.Millisecond * 500) + defer ticker.Stop() for _ = range ticker.C { n, err := client.BlockNumber(context.Background()) if err != nil { return err } if n.Int64() >= int64(num) { - ticker.Stop() break } } @@ -552,12 +558,13 @@ func (eth *ethereum) WaitForNoBlocks(num int, duration time.Duration) error { } timeout := time.After(duration) - tick := time.Tick(time.Millisecond * 500) + ticker := time.NewTicker(time.Millisecond * 500) + defer ticker.Stop() for { select { case <-timeout: return nil - case <-tick: + case <-ticker.C: n, err := client.BlockNumber(context.Background()) if err != nil { return err @@ -574,6 +581,67 @@ func (eth *ethereum) WaitForNoBlocks(num int, duration time.Duration) error { } } +func (eth *ethereum) WaitForBalances(addrs []common.Address, duration ...time.Duration) error { + client := eth.NewClient() + if client == nil { + return errors.New("failed to retrieve client") + } + + var t time.Duration + if len(duration) > 0 { + t = duration[0] + } else { + t = 1 * time.Hour + } + + waitBalance := func(addr common.Address) error { + timeout := time.After(t) + ticker := time.NewTicker(time.Millisecond * 500) + defer ticker.Stop() + for { + select { + case <-timeout: + return ErrTimeout + case <-ticker.C: + n, err := client.BalanceAt(context.Background(), addr, nil) + if err != nil { + return err + } + + // Check if new blocks are getting generated + if n.Uint64() <= 0 { + continue + } else { + return nil + } + } + } + } + + var wg sync.WaitGroup + errc := make(chan error, len(addrs)) + wg.Add(len(addrs)) + + for _, addr := range addrs { + addr := addr + go func() { + defer wg.Done() + errc <- waitBalance(addr) + }() + } + // Wait for the first error, then terminate the others. + var err error + for i := 0; i < len(addrs); i++ { + if err = <-errc; err != nil { + break + } + } + wg.Wait() + return err +} + +// ---------------------------------------------------------------------------- + func (eth *ethereum) AddPeer(address string) error { client := eth.NewClient() if client == nil { diff --git a/k8s/blockchain.go b/k8s/blockchain.go index b692d562..9879d5c4 100644 --- a/k8s/blockchain.go +++ b/k8s/blockchain.go @@ -106,10 +106,6 @@ func (bc *blockchain) Validators() []container.Ethereum { return bc.validators } -func (bc *blockchain) CreateNodes(num int, options ...Option) (nodes []container.Ethereum, err error) { - return nil, errors.New("unsupported") -} - // ---------------------------------------------------------------------------- func (bc *blockchain) setupValidators(num int, nodekeys []string, ips []string, options ...Option) { diff --git a/k8s/ethereum.go b/k8s/ethereum.go index 5f5a0e6e..9570972c 100644 --- a/k8s/ethereum.go +++ b/k8s/ethereum.go @@ -17,6 +17,11 @@ package k8s import ( + "context" + "crypto/ecdsa" + "errors" + "math/big" + "sync" "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -24,10 +29,13 @@ import ( "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/common" + ethtypes "github.com/ethereum/go-ethereum/core/types" + "github.com/getamis/go-ethereum/crypto" "github.com/getamis/istanbul-tools/charts" "github.com/getamis/istanbul-tools/client" istcommon "github.com/getamis/istanbul-tools/common" + "github.com/getamis/istanbul-tools/container" ) func NewEthereum(options ...Option) *ethereum { @@ -39,6 +47,12 @@ func NewEthereum(options ...Option) *ethereum { opt(eth) } + var err error + eth.key, err = crypto.HexToECDSA(eth.nodekey) + if err != nil { + log.Error("Failed to create private key from nodekey", "nodekey", eth.nodekey) + return nil + } eth.chart = charts.NewValidatorChart(eth.name, eth.args) return eth @@ -49,6 +63,8 @@ type ethereum struct { name string args []string + nodekey string + key *ecdsa.PrivateKey k8sClient *kubernetes.Clientset } @@ -90,7 +106,7 @@ func (eth *ethereum) DockerBinds() []string { } func (eth *ethereum) NewClient() *client.Client { - client, err := client.Dial("ws://" + eth.Host() + ":8545") + client, err := client.Dial("ws://" + eth.Host() + ":8546") if err != nil { return nil } @@ -102,7 +118,7 @@ func (eth *ethereum) NodeAddress() string { } func (eth *ethereum) Address() common.Address { - return common.Address{} + return crypto.PubkeyToAddress(eth.key.PublicKey) } func (eth *ethereum) ConsensusMonitor(errCh chan<- error, quit chan struct{}) { @@ -110,25 +126,211 @@ func (eth *ethereum) ConsensusMonitor(errCh chan<- error, quit chan struct{}) { } func (eth *ethereum) WaitForProposed(expectedAddress common.Address, timeout time.Duration) error { - return nil + cli := eth.NewClient() + + subCh := make(chan *ethtypes.Header) + + sub, err := cli.SubscribeNewHead(context.Background(), subCh) + if err != nil { + return err + } + defer sub.Unsubscribe() + + timer := time.NewTimer(timeout) + defer timer.Stop() + for { + select { + case err := <-sub.Err(): + return err + case <-timer.C: // FIXME: this event may be missed + return errors.New("no result") + case head := <-subCh: + if container.GetProposer(head) == expectedAddress { + return nil + } + } + } } func (eth *ethereum) WaitForPeersConnected(expectedPeercount int) error { + client := eth.NewClient() + if client == nil { + return errors.New("failed to retrieve client") + } + defer client.Close() + + ticker := time.NewTicker(time.Second * 1) + defer ticker.Stop() + for _ = range ticker.C { + infos, err := client.AdminPeers(context.Background()) + if err != nil { + return err + } + if len(infos) < expectedPeercount { + continue + } else { + break + } + } + return nil } func (eth *ethereum) WaitForBlocks(num int, waitingTime ...time.Duration) error { - return nil + var first *big.Int + + client := eth.NewClient() + if client == nil { + return errors.New("failed to retrieve client") + } + defer client.Close() + + var t time.Duration + if len(waitingTime) > 0 { + t = waitingTime[0] + } else { + t = 1 * time.Hour + } + + timeout := time.After(t) + ticker := time.NewTicker(time.Millisecond * 500) + defer ticker.Stop() + for { + select { + case <-timeout: + return container.ErrNoBlock + case <-ticker.C: + n, err := client.BlockNumber(context.Background()) + if err != nil { + return err + } + if first == nil { + first = new(big.Int).Set(n) + continue + } + // Check if new blocks are getting generated + if new(big.Int).Sub(n, first).Int64() >= int64(num) { + return nil + } + } + } } func (eth *ethereum) WaitForBlockHeight(num int) error { + client := eth.NewClient() + if client == nil { + return errors.New("failed to retrieve client") + } + defer client.Close() + + ticker := time.NewTicker(time.Millisecond * 500) + defer ticker.Stop() + for _ = range ticker.C { + n, err := client.BlockNumber(context.Background()) + if err != nil { + return err + } + if n.Int64() >= int64(num) { + break + } + } + return nil } func (eth *ethereum) WaitForNoBlocks(num int, duration time.Duration) error { - return nil + var first *big.Int + + client := eth.NewClient() + if client == nil { + return errors.New("failed to retrieve client") + } + + timeout := time.After(duration) + ticker := time.NewTicker(time.Millisecond * 500) + defer ticker.Stop() + for { + select { + case <-timeout: + return nil + case <-ticker.C: + n, err := client.BlockNumber(context.Background()) + if err != nil { + return err + } + if first == nil { + first = new(big.Int).Set(n) + continue + } + // Check if new blocks are getting generated + if new(big.Int).Sub(n, first).Int64() > int64(num) { + return errors.New("generated more blocks than expected") + } + } + } } +func (eth *ethereum) WaitForBalances(addrs []common.Address, duration ...time.Duration) error { + client := eth.NewClient() + if client == nil { + return errors.New("failed to retrieve client") + } + + var t time.Duration + if len(duration) > 0 { + t = duration[0] + } else { + t = 1 * time.Hour + } + + waitBalance := func(addr common.Address) error { + timeout := time.After(t) + ticker := time.NewTicker(time.Millisecond * 500) + defer ticker.Stop() + for { + select { + case <-timeout: + return container.ErrTimeout + case <-ticker.C: + n, err := client.BalanceAt(context.Background(), addr, nil) + if err != nil { + return err + } + + // Check if new blocks are getting generated + if n.Uint64() <= 0 { + continue + } else { + return nil + } + } + } + } + + var wg sync.WaitGroup + errc := make(chan error, len(addrs)) + wg.Add(len(addrs)) + + for _, addr := range addrs { + addr := addr + go func() { + defer wg.Done() + errc <- waitBalance(addr) + }() + } + // Wait for the first error, then terminate the others. + var err error + for i := 0; i < len(addrs); i++ { + if err = <-errc; err != nil { + break + } + } + wg.Wait() + return err +} + +// ---------------------------------------------------------------------------- + func (eth *ethereum) AddPeer(address string) error { return nil } diff --git a/k8s/option.go b/k8s/option.go index 933b8992..96c0732f 100644 --- a/k8s/option.go +++ b/k8s/option.go @@ -67,10 +67,20 @@ func Mine() Option { func NodeKeyHex(hex string) Option { return func(eth *ethereum) { + eth.nodekey = hex eth.args = append(eth.args, fmt.Sprintf("ethereum.nodekey.hex=%s", hex)) } } +func TxPoolSize(size int) Option { + return func(eth *ethereum) { + eth.args = append(eth.args, fmt.Sprintf("benchmark.txpool.globalslots=%d", size)) + eth.args = append(eth.args, fmt.Sprintf("benchmark.txpool.accountslots=%d", size)) + eth.args = append(eth.args, fmt.Sprintf("benchmark.txpool.globalqueue=%d", size)) + eth.args = append(eth.args, fmt.Sprintf("benchmark.txpool.accountqueue=%d", size)) + } +} + func Verbosity(verbosity int) Option { return func(eth *ethereum) { eth.args = append(eth.args, fmt.Sprintf("ethereum.verbosity=%d", verbosity)) diff --git a/k8s/rich_man.go b/k8s/rich_man.go new file mode 100644 index 00000000..458866c2 --- /dev/null +++ b/k8s/rich_man.go @@ -0,0 +1,51 @@ +// Copyright 2017 AMIS Technologies +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package k8s + +import ( + "context" + "errors" + "math/big" + + "github.com/ethereum/go-ethereum/common" + + istcommon "github.com/getamis/istanbul-tools/common" +) + +type RichMan interface { + GiveEther(context.Context, []common.Address, *big.Int) error +} + +func (eth *ethereum) GiveEther(ctx context.Context, accounts []common.Address, amount *big.Int) error { + client := eth.NewClient() + if client == nil { + return errors.New("failed to retrieve client") + } + + nonce, err := client.NonceAt(context.Background(), eth.Address(), nil) + if err != nil { + log.Error("Failed to get nonce", "addr", eth.Address(), "err", err) + return err + } + + for _, account := range accounts { + _ = istcommon.SendEther(client, eth.key, account, amount, nonce) + nonce++ + } + + return nil +} diff --git a/k8s/utils.go b/k8s/utils.go index 0b94d739..ebbb09b9 100644 --- a/k8s/utils.go +++ b/k8s/utils.go @@ -78,7 +78,7 @@ func executeInParallel(fns ...func() error) error { var err error for i := 0; i < len(fns); i++ { if err = <-errc; err != nil { - return err + break } } wg.Wait() diff --git a/tests/functional/block_sync_test.go b/tests/functional/block_sync_test.go index 17065ea9..4e1c7f72 100644 --- a/tests/functional/block_sync_test.go +++ b/tests/functional/block_sync_test.go @@ -52,7 +52,11 @@ var _ = Describe("Block synchronization testing", func() { BeforeEach(func() { var err error - nodes, err = blockchain.CreateNodes(numberOfNodes, + + incubator, ok := blockchain.(container.NodeIncubator) + Expect(ok).To(BeTrue()) + + nodes, err = incubator.CreateNodes(numberOfNodes, container.ImageRepository("quay.io/amis/geth"), container.ImageTag("istanbul_develop"), container.DataDir("/data"), diff --git a/tests/load/load_test.go b/tests/load/load_test.go index 4399e629..bb6f04f9 100644 --- a/tests/load/load_test.go +++ b/tests/load/load_test.go @@ -17,19 +17,46 @@ package load import ( + "context" + "fmt" + "math/big" + "sync" "testing" + "time" - "github.com/getamis/istanbul-tools/charts" - "github.com/getamis/istanbul-tools/common" - - "github.com/getamis/istanbul-tools/tests" + "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + + "github.com/getamis/istanbul-tools/charts" + "github.com/getamis/istanbul-tools/container" + "github.com/getamis/istanbul-tools/k8s" + "github.com/getamis/istanbul-tools/tests" ) var _ = Describe("TPS-01: Large amount of transactions", func() { + tests.CaseTable("with number of validators", func(numberOfValidators int) { + var svcCharts []*charts.ValidatorServiceChart + + BeforeSuite(func() { + for i := 0; i < numberOfValidators; i++ { + chart := charts.NewValidatorServiceChart(fmt.Sprintf("%d", i), nil) + svcCharts = append(svcCharts, chart) + + if err := chart.Install(false); err != nil { + fmt.Println(err) + } + } + }) + + AfterSuite(func() { + for i := 0; i < numberOfValidators; i++ { + svcCharts[i].Uninstall() + } + }) + tests.CaseTable("with gas limit", func(gaslimit int) { tests.CaseTable("with txpool size", @@ -45,41 +72,62 @@ var _ = Describe("TPS-01: Large amount of transactions", func() { tests.Case("21000*1000", 21000*1000), tests.Case("21000*3000", 21000*3000), ) + }, tests.Case("4 validators", 4), - tests.Case("7 validators", 7), - tests.Case("10 validators", 10), ) }) func runTests(numberOfValidators int, gaslimit int, txpoolSize int) { Describe("", func() { var ( - genesisChart tests.ChartInstaller - staticNodesChart tests.ChartInstaller + blockchain container.Blockchain ) BeforeEach(func() { - _, nodekeys, addrs := common.GenerateKeys(numberOfValidators) - genesisChart = charts.NewGenesisChart(addrs, uint64(gaslimit)) - Expect(genesisChart.Install(false)).To(BeNil()) + blockchain = k8s.NewBlockchain( + numberOfValidators, + uint64(gaslimit), + k8s.ImageRepository("quay.io/amis/geth"), + k8s.ImageTag("istanbul_develop"), + k8s.ServiceType("LoadBalancer"), + k8s.Mine(), + k8s.TxPoolSize(txpoolSize), + ) + Expect(blockchain.Start(true)).To(BeNil()) - staticNodesChart = charts.NewStaticNodesChart(nodekeys, common.GenerateIPs(len(nodekeys))) - Expect(staticNodesChart.Install(false)).To(BeNil()) + tests.WaitFor(blockchain.Validators(), func(geth container.Ethereum, wg *sync.WaitGroup) { + richman, ok := geth.(k8s.RichMan) + Expect(ok).To(BeTrue()) + + var addrs []common.Address + addr := common.HexToAddress("0x1a9afb711302c5f83b5902843d1c007a1a137632") + addrs = append(addrs, addr) + + // Give ether to all accounts + err := richman.GiveEther(context.Background(), addrs, new(big.Int).Exp(big.NewInt(10), big.NewInt(24), nil)) + Expect(err).NotTo(BeNil()) + + err = geth.WaitForBalances(addrs, 10*time.Second) + Expect(err).NotTo(BeNil()) + + wg.Done() + }) }) AfterEach(func() { - Expect(genesisChart.Uninstall()).To(BeNil()) - Expect(staticNodesChart.Uninstall()).To(BeNil()) + Expect(blockchain.Stop(true)).To(BeNil()) + blockchain.Finalize() }) It("", func() { + }) }) } -func IstanbulLoadTest(t *testing.T) { +func TestIstanbulLoadTesting(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "Istanbul Load Test Suite") } diff --git a/tests/quorum/functional/block_sync_test.go b/tests/quorum/functional/block_sync_test.go index bf714e19..b11376fc 100644 --- a/tests/quorum/functional/block_sync_test.go +++ b/tests/quorum/functional/block_sync_test.go @@ -57,7 +57,11 @@ var _ = Describe("Block synchronization testing", func() { BeforeEach(func() { var err error - nodes, err = blockchain.CreateNodes(numberOfNodes, + + incubator, ok := blockchain.(container.NodeIncubator) + Expect(ok).To(BeTrue()) + + nodes, err = incubator.CreateNodes(numberOfNodes, container.ImageRepository("quay.io/amis/geth"), container.ImageTag("istanbul_develop"), container.DataDir("/data"),