From 14681bf9133c73dbffcbca1b0c734699c5f51570 Mon Sep 17 00:00:00 2001 From: Miya Chen Date: Fri, 1 Sep 2017 16:52:08 +0800 Subject: [PATCH] container: run geth container with given IP --- container/blockchain.go | 87 +++++++++++--- container/blockchain_test.go | 1 + container/ethereum.go | 38 ++++-- container/network.go | 115 +++++++++++++++---- container/options.go | 13 +++ tests/funtional/block_sync_test.go | 2 +- tests/funtional/byzantine_faulty_test.go | 4 +- tests/funtional/dynamic_test.go | 2 +- tests/funtional/general_consensus_test.go | 2 +- tests/funtional/gossip_network_test.go | 2 +- tests/funtional/integration_test.go | 15 +++ tests/funtional/non_byzantine_faulty_test.go | 2 +- tests/funtional/recoverability_test.go | 2 +- 13 files changed, 234 insertions(+), 51 deletions(-) diff --git a/container/blockchain.go b/container/blockchain.go index 24d64dd3..418d4093 100644 --- a/container/blockchain.go +++ b/container/blockchain.go @@ -20,6 +20,7 @@ import ( "context" "crypto/ecdsa" "log" + "net" "os" "path/filepath" "time" @@ -42,7 +43,7 @@ type Blockchain interface { CreateNodes(int, ...Option) ([]Ethereum, error) } -func NewBlockchain(numOfValidators int, options ...Option) (bc *blockchain) { +func NewBlockchain(network *DockerNetwork, numOfValidators int, options ...Option) (bc *blockchain) { bc = &blockchain{opts: options} var err error @@ -51,12 +52,25 @@ func NewBlockchain(numOfValidators int, options ...Option) (bc *blockchain) { log.Fatalf("Cannot connect to Docker daemon, err: %v", err) } + if network == nil { + bc.defaultNetwork, err = NewDockerNetwork() + if err != nil { + log.Fatalf("Cannot create Docker network, err: %v", err) + } + network = bc.defaultNetwork + } + + bc.dockerNetworkName = network.Name() + bc.getFreeIPAddrs = network.GetFreeIPAddrs + bc.opts = append(bc.opts, DockerNetworkName(bc.dockerNetworkName)) + bc.addValidators(numOfValidators) return bc } -func NewDefaultBlockchain(numOfValidators int) (bc *blockchain) { - return NewBlockchain(numOfValidators, +func NewDefaultBlockchain(network *DockerNetwork, numOfValidators int) (bc *blockchain) { + return NewBlockchain(network, + numOfValidators, ImageRepository("quay.io/amis/geth"), ImageTag("istanbul_develop"), DataDir("/data"), @@ -73,7 +87,7 @@ func NewDefaultBlockchain(numOfValidators int) (bc *blockchain) { ) } -func NewDefaultBlockchainWithFaulty(numOfNormal int, numOfFaulty int) (bc *blockchain) { +func NewDefaultBlockchainWithFaulty(network *DockerNetwork, numOfNormal int, numOfFaulty int) (bc *blockchain) { commonOpts := [...]Option{ DataDir("/data"), WebSocket(), @@ -101,24 +115,48 @@ func NewDefaultBlockchainWithFaulty(numOfNormal int, numOfFaulty int) (bc *block log.Fatalf("Cannot connect to Docker daemon, err: %v", err) } - keys, addrs := generateKeys(numOfNormal + numOfFaulty) + if network == nil { + bc.defaultNetwork, err = NewDockerNetwork() + if err != nil { + log.Fatalf("Cannot create Docker network, err: %v", err) + } + network = bc.defaultNetwork + } + + bc.dockerNetworkName = network.Name() + bc.getFreeIPAddrs = network.GetFreeIPAddrs + + normalOpts = append(normalOpts, DockerNetworkName(bc.dockerNetworkName)) + faultyOpts = append(faultyOpts, DockerNetworkName(bc.dockerNetworkName)) + + totalNodes := numOfNormal + numOfFaulty + + ips, err := bc.getFreeIPAddrs(totalNodes) + if err != nil { + log.Fatalf("Failed to get free ip addresses, err: %v", err) + } + + keys, addrs := generateKeys(totalNodes) bc.setupGenesis(addrs) // Create normal validators bc.opts = normalOpts - bc.setupValidators(keys[:numOfNormal], bc.opts...) + bc.setupValidators(ips[:numOfNormal], keys[:numOfNormal], bc.opts...) // Create faulty validators bc.opts = faultyOpts - bc.setupValidators(keys[numOfNormal:], bc.opts...) + bc.setupValidators(ips[numOfNormal:], keys[numOfNormal:], bc.opts...) return bc } // ---------------------------------------------------------------------------- type blockchain struct { - dockerClient *client.Client - genesisFile string - validators []Ethereum - opts []Option + dockerClient *client.Client + defaultNetwork *DockerNetwork + dockerNetworkName string + getFreeIPAddrs func(int) ([]net.IP, error) + genesisFile string + validators []Ethereum + opts []Option } func (bc *blockchain) AddValidators(numOfValidators int) ([]Ethereum, error) { @@ -202,7 +240,16 @@ func (bc *blockchain) Start(strong bool) error { } func (bc *blockchain) Stop(force bool) error { - return bc.stop(bc.validators, force) + if err := bc.stop(bc.validators, force); err != nil { + return err + } + + if bc.defaultNetwork != nil { + err := bc.defaultNetwork.Remove() + bc.defaultNetwork = nil + return err + } + return nil } func (bc *blockchain) Finalize() { @@ -214,6 +261,11 @@ func (bc *blockchain) Validators() []Ethereum { } func (bc *blockchain) CreateNodes(num int, options ...Option) (nodes []Ethereum, err error) { + ips, err := bc.getFreeIPAddrs(num) + if err != nil { + return nil, err + } + for i := 0; i < num; i++ { var opts []Option opts = append(opts, options...) @@ -226,6 +278,8 @@ func (bc *blockchain) CreateNodes(num int, options ...Option) (nodes []Ethereum, } opts = append(opts, HostDataDir(dataDir)) opts = append(opts, HostWebSocketPort(freeport.GetPort())) + opts = append(opts, HostIP(ips[i])) + opts = append(opts, DockerNetworkName(bc.dockerNetworkName)) geth := NewEthereum( bc.dockerClient, @@ -247,9 +301,13 @@ func (bc *blockchain) CreateNodes(num int, options ...Option) (nodes []Ethereum, // ---------------------------------------------------------------------------- func (bc *blockchain) addValidators(numOfValidators int) error { + ips, err := bc.getFreeIPAddrs(numOfValidators) + if err != nil { + return err + } keys, addrs := generateKeys(numOfValidators) bc.setupGenesis(addrs) - bc.setupValidators(keys, bc.opts...) + bc.setupValidators(ips, keys, bc.opts...) return nil } @@ -282,7 +340,7 @@ func (bc *blockchain) setupGenesis(addrs []common.Address) { } } -func (bc *blockchain) setupValidators(keys []*ecdsa.PrivateKey, options ...Option) { +func (bc *blockchain) setupValidators(ips []net.IP, keys []*ecdsa.PrivateKey, options ...Option) { for i := 0; i < len(keys); i++ { var opts []Option opts = append(opts, options...) @@ -295,6 +353,7 @@ func (bc *blockchain) setupValidators(keys []*ecdsa.PrivateKey, options ...Optio opts = append(opts, HostDataDir(dataDir)) opts = append(opts, HostWebSocketPort(freeport.GetPort())) opts = append(opts, Key(keys[i])) + opts = append(opts, HostIP(ips[i])) geth := NewEthereum( bc.dockerClient, diff --git a/container/blockchain_test.go b/container/blockchain_test.go index fe84828d..e385db78 100644 --- a/container/blockchain_test.go +++ b/container/blockchain_test.go @@ -23,6 +23,7 @@ import ( func TestEthereumBlockchain(t *testing.T) { chain := NewBlockchain( + nil, 4, ImageRepository("quay.io/amis/geth"), ImageTag("istanbul_develop"), diff --git a/container/ethereum.go b/container/ethereum.go index b128d8db..be25da07 100644 --- a/container/ethereum.go +++ b/container/ethereum.go @@ -33,6 +33,7 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/filters" + "github.com/docker/docker/api/types/network" "github.com/docker/docker/client" "github.com/docker/go-connections/nat" "github.com/ethereum/go-ethereum/cmd/utils" @@ -119,6 +120,7 @@ type ethereum struct { ok bool flags []string dataDir string + ip string port string rpcPort string wsPort string @@ -126,8 +128,9 @@ type ethereum struct { containerID string node *discover.Node - imageRepository string - imageTag string + imageRepository string + imageTag string + dockerNetworkName string key *ecdsa.PrivateKey logging bool @@ -224,6 +227,19 @@ func (eth *ethereum) Start() error { binds = append(binds, eth.dataDir+":"+utils.DataDirFlag.Value.Value) } + var networkingConfig *network.NetworkingConfig + if eth.ip != "" && eth.dockerNetworkName != "" { + endpointsConfig := make(map[string]*network.EndpointSettings) + endpointsConfig[eth.dockerNetworkName] = &network.EndpointSettings{ + IPAMConfig: &network.EndpointIPAMConfig{ + IPv4Address: eth.ip, + }, + } + networkingConfig = &network.NetworkingConfig{ + EndpointsConfig: endpointsConfig, + } + } + resp, err := eth.client.ContainerCreate(context.Background(), &container.Config{ Hostname: "geth-" + eth.hostName, @@ -234,7 +250,7 @@ func (eth *ethereum) Start() error { &container.HostConfig{ Binds: binds, PortBindings: portBindings, - }, nil, "") + }, networkingConfig, "") if err != nil { log.Printf("Failed to create container, err: %v", err) return err @@ -244,7 +260,7 @@ func (eth *ethereum) Start() error { err = eth.client.ContainerStart(context.Background(), eth.containerID, types.ContainerStartOptions{}) if err != nil { - log.Printf("Failed to start container, err: %v", err) + log.Printf("Failed to start container, err: %v, ip:%v", err, eth.ip) return err } @@ -268,16 +284,20 @@ func (eth *ethereum) Start() error { return errors.New("Failed to start geth") } - containerJSON, err := eth.client.ContainerInspect(context.Background(), eth.containerID) - if err != nil { - log.Print("Failed to inspect container,", err) - return err + containerIP := eth.ip + if containerIP == "" { + containerJSON, err := eth.client.ContainerInspect(context.Background(), eth.containerID) + if err != nil { + log.Print("Failed to inspect container,", err) + return err + } + containerIP = containerJSON.NetworkSettings.IPAddress } if eth.key != nil { eth.node = discover.NewNode( discover.PubkeyID(ð.key.PublicKey), - net.ParseIP(containerJSON.NetworkSettings.IPAddress), + net.ParseIP(containerIP), 0, uint16(utils.ListenPortFlag.Value)) } diff --git a/container/network.go b/container/network.go index 06eb96fc..e068712f 100644 --- a/container/network.go +++ b/container/network.go @@ -18,47 +18,122 @@ package container import ( "context" + "errors" "fmt" + "net" + "sync" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/network" "github.com/docker/docker/client" ) -type networkManager struct { - client *client.Client - field1 uint8 - field2 uint8 +const ( + FirstOctet = 172 + SecondOctet = 19 + NetworkName = "testnet" +) + +type DockerNetwork struct { + client *client.Client + id string + name string + ipv4Net *net.IPNet + + mutex sync.Mutex + ipIndex net.IP } -func NewNetworkManager(c *client.Client, field1 uint8, field2 uint8) *networkManager { - n := &networkManager{ - client: c, - field1: field1, - field2: field2, +func NewDockerNetwork() (*DockerNetwork, error) { + // IP xxx.xxx.0.1 is reserved for docker network gateway + ipv4Addr, ipv4Net, err := net.ParseCIDR(fmt.Sprintf("%d.%d.0.1/16", FirstOctet, SecondOctet)) + if err != nil { + return nil, err } - return n + c, err := client.NewEnvClient() + if err != nil { + return nil, err + } + + network := &DockerNetwork{ + client: c, + name: NetworkName, + ipv4Net: ipv4Net, + ipIndex: ipv4Addr, + } + + if err := network.create(); err != nil { + return nil, err + } + + return network, nil } -// CreateNetwork returns network id -func (n *networkManager) CreateNetwork(name string) (string, error) { +// create creates a docker network with given subnet +func (n *DockerNetwork) create() error { ipamConfig := network.IPAMConfig{ - Subnet: fmt.Sprintf("%d.%d.0.0/16", n.field1, n.field2), + Subnet: n.ipv4Net.String(), } ipam := &network.IPAM{ Config: []network.IPAMConfig{ipamConfig}, } - r, e := n.client.NetworkCreate(context.Background(), name, types.NetworkCreate{ + + r, err := n.client.NetworkCreate(context.Background(), n.name, types.NetworkCreate{ IPAM: ipam, }) - if e != nil { - return "", e - } else { - return r.ID, nil + if err != nil { + return err } + n.id = r.ID + return nil } -func (n *networkManager) RemoveNetwork(id string) error { - return n.client.NetworkRemove(context.Background(), id) +func (n *DockerNetwork) ID() string { + return n.id +} + +func (n *DockerNetwork) Name() string { + return n.name +} + +func (n *DockerNetwork) Remove() error { + return n.client.NetworkRemove(context.Background(), n.id) +} + +func (n *DockerNetwork) GetFreeIPAddrs(num int) ([]net.IP, error) { + n.mutex.Lock() + defer n.mutex.Unlock() + + ips := make([]net.IP, 0) + for i := 0; i < num; i++ { + ip := dupIP(n.ipIndex) + for j := len(ip) - 1; j >= 0; j-- { + ip[j]++ + if ip[j] > 0 { + break + } + } + + if !n.ipv4Net.Contains(ip) { + break + } + ips = append(ips, ip) + n.ipIndex = ip + } + + if len(ips) != num { + return nil, errors.New("Insufficient IP address.") + } + return ips, nil +} + +func dupIP(ip net.IP) net.IP { + // To save space, try and only use 4 bytes + if x := ip.To4(); x != nil { + ip = x + } + dup := make(net.IP, len(ip)) + copy(dup, ip) + return dup } diff --git a/container/options.go b/container/options.go index 51f50639..aa7fde84 100644 --- a/container/options.go +++ b/container/options.go @@ -19,6 +19,7 @@ package container import ( "crypto/ecdsa" "fmt" + "net" "github.com/ethereum/go-ethereum/cmd/utils" ) @@ -37,6 +38,12 @@ func ImageTag(tag string) Option { } } +func DockerNetworkName(dockerNetworkName string) Option { + return func(eth *ethereum) { + eth.dockerNetworkName = dockerNetworkName + } +} + func HostName(hostName string) Option { return func(eth *ethereum) { eth.hostName = hostName @@ -49,6 +56,12 @@ func HostDataDir(path string) Option { } } +func HostIP(ip net.IP) Option { + return func(eth *ethereum) { + eth.ip = ip.String() + } +} + func HostPort(port int) Option { return func(eth *ethereum) { eth.port = fmt.Sprintf("%d", port) diff --git a/tests/funtional/block_sync_test.go b/tests/funtional/block_sync_test.go index c7c7ed0d..c642ef9e 100644 --- a/tests/funtional/block_sync_test.go +++ b/tests/funtional/block_sync_test.go @@ -37,7 +37,7 @@ var _ = Describe("Block synchronization testing", func() { ) BeforeEach(func() { - blockchain = container.NewDefaultBlockchain(numberOfValidators) + blockchain = container.NewDefaultBlockchain(dockerNetwork, numberOfValidators) Expect(blockchain.Start(true)).To(BeNil()) }) diff --git a/tests/funtional/byzantine_faulty_test.go b/tests/funtional/byzantine_faulty_test.go index 395ea9e0..4fe8dc83 100644 --- a/tests/funtional/byzantine_faulty_test.go +++ b/tests/funtional/byzantine_faulty_test.go @@ -38,7 +38,7 @@ var _ = Describe("TFS-05: Byzantine Faulty", func() { blockchain container.Blockchain ) BeforeEach(func() { - blockchain = container.NewDefaultBlockchainWithFaulty(numberOfNormal, numberOfFaulty) + blockchain = container.NewDefaultBlockchainWithFaulty(dockerNetwork, numberOfNormal, numberOfFaulty) Expect(blockchain.Start(true)).To(BeNil()) }) @@ -77,7 +77,7 @@ var _ = Describe("TFS-05: Byzantine Faulty", func() { blockchain container.Blockchain ) BeforeEach(func() { - blockchain = container.NewDefaultBlockchainWithFaulty(numberOfNormal, numberOfFaulty) + blockchain = container.NewDefaultBlockchainWithFaulty(dockerNetwork, numberOfNormal, numberOfFaulty) Expect(blockchain.Start(true)).To(BeNil()) }) diff --git a/tests/funtional/dynamic_test.go b/tests/funtional/dynamic_test.go index 8bd61e4f..b451ac39 100644 --- a/tests/funtional/dynamic_test.go +++ b/tests/funtional/dynamic_test.go @@ -38,7 +38,7 @@ var _ = Describe("Dynamic validators addition/removal testing", func() { ) BeforeEach(func() { - blockchain = container.NewDefaultBlockchain(numberOfValidators) + blockchain = container.NewDefaultBlockchain(dockerNetwork, numberOfValidators) Expect(blockchain.Start(true)).To(BeNil()) }) diff --git a/tests/funtional/general_consensus_test.go b/tests/funtional/general_consensus_test.go index b781c850..f349e58a 100644 --- a/tests/funtional/general_consensus_test.go +++ b/tests/funtional/general_consensus_test.go @@ -42,7 +42,7 @@ var _ = Describe("TFS-01: General consensus", func() { ) BeforeEach(func() { - blockchain = container.NewDefaultBlockchain(numberOfValidators) + blockchain = container.NewDefaultBlockchain(dockerNetwork, numberOfValidators) Expect(blockchain.Start(true)).To(BeNil()) }) diff --git a/tests/funtional/gossip_network_test.go b/tests/funtional/gossip_network_test.go index 500e2b3b..a0e41d85 100644 --- a/tests/funtional/gossip_network_test.go +++ b/tests/funtional/gossip_network_test.go @@ -36,7 +36,7 @@ var _ = Describe("TFS-07: Gossip Network", func() { ) BeforeEach(func() { - blockchain = container.NewDefaultBlockchain(numberOfValidators) + blockchain = container.NewDefaultBlockchain(dockerNetwork, numberOfValidators) Expect(blockchain.Start(false)).To(BeNil()) }) diff --git a/tests/funtional/integration_test.go b/tests/funtional/integration_test.go index 02003c86..c1c3264f 100644 --- a/tests/funtional/integration_test.go +++ b/tests/funtional/integration_test.go @@ -21,6 +21,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + + "github.com/getamis/istanbul-tools/container" ) // Example @@ -71,7 +73,20 @@ import ( // }) // +var dockerNetwork *container.DockerNetwork + func TestIstanbul(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "Istanbul Test Suite") } + +var _ = BeforeSuite(func() { + var err error + dockerNetwork, err = container.NewDockerNetwork() + Expect(err).To(BeNil()) +}) + +var _ = AfterSuite(func() { + err := dockerNetwork.Remove() + Expect(err).To(BeNil()) +}) diff --git a/tests/funtional/non_byzantine_faulty_test.go b/tests/funtional/non_byzantine_faulty_test.go index 150b6744..ef0c3a0f 100644 --- a/tests/funtional/non_byzantine_faulty_test.go +++ b/tests/funtional/non_byzantine_faulty_test.go @@ -36,7 +36,7 @@ var _ = Describe("TFS-04: Non-Byzantine Faulty", func() { ) BeforeEach(func() { - blockchain = container.NewDefaultBlockchain(numberOfValidators) + blockchain = container.NewDefaultBlockchain(dockerNetwork, numberOfValidators) Expect(blockchain.Start(true)).To(BeNil()) }) diff --git a/tests/funtional/recoverability_test.go b/tests/funtional/recoverability_test.go index 49348ab0..1857716b 100644 --- a/tests/funtional/recoverability_test.go +++ b/tests/funtional/recoverability_test.go @@ -36,7 +36,7 @@ var _ = Describe("TFS-03: Recoverability testing", func() { ) BeforeEach(func() { - blockchain = container.NewDefaultBlockchain(numberOfValidators) + blockchain = container.NewDefaultBlockchain(dockerNetwork, numberOfValidators) Expect(blockchain.Start(true)).To(BeNil()) })