container: run geth container with given IP

container/blockchain.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"crypto/ecdsa"
 	"log"
+	"net"
 	"os"
 	"path/filepath"
 	"time"
@@ -42,7 +43,7 @@ type Blockchain interface {
 	CreateNodes(int, ...Option) ([]Ethereum, error)
 }
 
-func NewBlockchain(numOfValidators int, options ...Option) (bc *blockchain) {
+func NewBlockchain(network *DockerNetwork, numOfValidators int, options ...Option) (bc *blockchain) {
 	bc = &blockchain{opts: options}
 
 	var err error
@@ -51,12 +52,25 @@ func NewBlockchain(numOfValidators int, options ...Option) (bc *blockchain) {
 		log.Fatalf("Cannot connect to Docker daemon, err: %v", err)
 	}
 
+	if network == nil {
+		bc.defaultNetwork, err = NewDockerNetwork()
+		if err != nil {
+			log.Fatalf("Cannot create Docker network, err: %v", err)
+		}
+		network = bc.defaultNetwork
+	}
+
+	bc.dockerNetworkName = network.Name()
+	bc.getFreeIPAddrs = network.GetFreeIPAddrs
+	bc.opts = append(bc.opts, DockerNetworkName(bc.dockerNetworkName))
+
 	bc.addValidators(numOfValidators)
 	return bc
 }
 
-func NewDefaultBlockchain(numOfValidators int) (bc *blockchain) {
+func NewDefaultBlockchain(network *DockerNetwork, numOfValidators int) (bc *blockchain) {
-	return NewBlockchain(numOfValidators,
+	return NewBlockchain(network,
+		numOfValidators,
 		ImageRepository("quay.io/amis/geth"),
 		ImageTag("istanbul_develop"),
 		DataDir("/data"),
@@ -73,7 +87,7 @@ func NewDefaultBlockchain(numOfValidators int) (bc *blockchain) {
 	)
 }
 
-func NewDefaultBlockchainWithFaulty(numOfNormal int, numOfFaulty int) (bc *blockchain) {
+func NewDefaultBlockchainWithFaulty(network *DockerNetwork, numOfNormal int, numOfFaulty int) (bc *blockchain) {
 	commonOpts := [...]Option{
 		DataDir("/data"),
 		WebSocket(),
@@ -101,24 +115,48 @@ func NewDefaultBlockchainWithFaulty(numOfNormal int, numOfFaulty int) (bc *block
 		log.Fatalf("Cannot connect to Docker daemon, err: %v", err)
 	}
 
-	keys, addrs := generateKeys(numOfNormal + numOfFaulty)
+	if network == nil {
+		bc.defaultNetwork, err = NewDockerNetwork()
+		if err != nil {
+			log.Fatalf("Cannot create Docker network, err: %v", err)
+		}
+		network = bc.defaultNetwork
+	}
+
+	bc.dockerNetworkName = network.Name()
+	bc.getFreeIPAddrs = network.GetFreeIPAddrs
+
+	normalOpts = append(normalOpts, DockerNetworkName(bc.dockerNetworkName))
+	faultyOpts = append(faultyOpts, DockerNetworkName(bc.dockerNetworkName))
+
+	totalNodes := numOfNormal + numOfFaulty
+
+	ips, err := bc.getFreeIPAddrs(totalNodes)
+	if err != nil {
+		log.Fatalf("Failed to get free ip addresses, err: %v", err)
+	}
+
+	keys, addrs := generateKeys(totalNodes)
 	bc.setupGenesis(addrs)
 	// Create normal validators
 	bc.opts = normalOpts
-	bc.setupValidators(keys[:numOfNormal], bc.opts...)
+	bc.setupValidators(ips[:numOfNormal], keys[:numOfNormal], bc.opts...)
 	// Create faulty validators
 	bc.opts = faultyOpts
-	bc.setupValidators(keys[numOfNormal:], bc.opts...)
+	bc.setupValidators(ips[numOfNormal:], keys[numOfNormal:], bc.opts...)
 	return bc
 }
 
 // ----------------------------------------------------------------------------
 
 type blockchain struct {
-	dockerClient *client.Client
+	dockerClient      *client.Client
-	genesisFile  string
+	defaultNetwork    *DockerNetwork
-	validators   []Ethereum
+	dockerNetworkName string
-	opts         []Option
+	getFreeIPAddrs    func(int) ([]net.IP, error)
+	genesisFile       string
+	validators        []Ethereum
+	opts              []Option
 }
 
 func (bc *blockchain) AddValidators(numOfValidators int) ([]Ethereum, error) {
@@ -202,7 +240,16 @@ func (bc *blockchain) Start(strong bool) error {
 }
 
 func (bc *blockchain) Stop(force bool) error {
-	return bc.stop(bc.validators, force)
+	if err := bc.stop(bc.validators, force); err != nil {
+		return err
+	}
+
+	if bc.defaultNetwork != nil {
+		err := bc.defaultNetwork.Remove()
+		bc.defaultNetwork = nil
+		return err
+	}
+	return nil
 }
 
 func (bc *blockchain) Finalize() {
@@ -214,6 +261,11 @@ func (bc *blockchain) Validators() []Ethereum {
 }
 
 func (bc *blockchain) CreateNodes(num int, options ...Option) (nodes []Ethereum, err error) {
+	ips, err := bc.getFreeIPAddrs(num)
+	if err != nil {
+		return nil, err
+	}
+
 	for i := 0; i < num; i++ {
 		var opts []Option
 		opts = append(opts, options...)
@@ -226,6 +278,8 @@ func (bc *blockchain) CreateNodes(num int, options ...Option) (nodes []Ethereum,
 		}
 		opts = append(opts, HostDataDir(dataDir))
 		opts = append(opts, HostWebSocketPort(freeport.GetPort()))
+		opts = append(opts, HostIP(ips[i]))
+		opts = append(opts, DockerNetworkName(bc.dockerNetworkName))
 
 		geth := NewEthereum(
 			bc.dockerClient,
@@ -247,9 +301,13 @@ func (bc *blockchain) CreateNodes(num int, options ...Option) (nodes []Ethereum,
 // ----------------------------------------------------------------------------
 
 func (bc *blockchain) addValidators(numOfValidators int) error {
+	ips, err := bc.getFreeIPAddrs(numOfValidators)
+	if err != nil {
+		return err
+	}
 	keys, addrs := generateKeys(numOfValidators)
 	bc.setupGenesis(addrs)
-	bc.setupValidators(keys, bc.opts...)
+	bc.setupValidators(ips, keys, bc.opts...)
 
 	return nil
 }
@@ -282,7 +340,7 @@ func (bc *blockchain) setupGenesis(addrs []common.Address) {
 	}
 }
 
-func (bc *blockchain) setupValidators(keys []*ecdsa.PrivateKey, options ...Option) {
+func (bc *blockchain) setupValidators(ips []net.IP, keys []*ecdsa.PrivateKey, options ...Option) {
 	for i := 0; i < len(keys); i++ {
 		var opts []Option
 		opts = append(opts, options...)
@@ -295,6 +353,7 @@ func (bc *blockchain) setupValidators(keys []*ecdsa.PrivateKey, options ...Optio
 		opts = append(opts, HostDataDir(dataDir))
 		opts = append(opts, HostWebSocketPort(freeport.GetPort()))
 		opts = append(opts, Key(keys[i]))
+		opts = append(opts, HostIP(ips[i]))
 
 		geth := NewEthereum(
 			bc.dockerClient,

container/blockchain_test.go

@@ -23,6 +23,7 @@ import (
 
 func TestEthereumBlockchain(t *testing.T) {
 	chain := NewBlockchain(
+		nil,
 		4,
 		ImageRepository("quay.io/amis/geth"),
 		ImageTag("istanbul_develop"),

container/ethereum.go

@@ -33,6 +33,7 @@ import (
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/api/types/network"
 	"github.com/docker/docker/client"
 	"github.com/docker/go-connections/nat"
 	"github.com/ethereum/go-ethereum/cmd/utils"
@@ -119,6 +120,7 @@ type ethereum struct {
 	ok          bool
 	flags       []string
 	dataDir     string
+	ip          string
 	port        string
 	rpcPort     string
 	wsPort      string
@@ -126,8 +128,9 @@ type ethereum struct {
 	containerID string
 	node        *discover.Node
 
-	imageRepository string
+	imageRepository   string
-	imageTag        string
+	imageTag          string
+	dockerNetworkName string
 
 	key     *ecdsa.PrivateKey
 	logging bool
@@ -224,6 +227,19 @@ func (eth *ethereum) Start() error {
 		binds = append(binds, eth.dataDir+":"+utils.DataDirFlag.Value.Value)
 	}
 
+	var networkingConfig *network.NetworkingConfig
+	if eth.ip != "" && eth.dockerNetworkName != "" {
+		endpointsConfig := make(map[string]*network.EndpointSettings)
+		endpointsConfig[eth.dockerNetworkName] = &network.EndpointSettings{
+			IPAMConfig: &network.EndpointIPAMConfig{
+				IPv4Address: eth.ip,
+			},
+		}
+		networkingConfig = &network.NetworkingConfig{
+			EndpointsConfig: endpointsConfig,
+		}
+	}
+
 	resp, err := eth.client.ContainerCreate(context.Background(),
 		&container.Config{
 			Hostname:     "geth-" + eth.hostName,
@@ -234,7 +250,7 @@ func (eth *ethereum) Start() error {
 		&container.HostConfig{
 			Binds:        binds,
 			PortBindings: portBindings,
-		}, nil, "")
+		}, networkingConfig, "")
 	if err != nil {
 		log.Printf("Failed to create container, err: %v", err)
 		return err
@@ -244,7 +260,7 @@ func (eth *ethereum) Start() error {
 
 	err = eth.client.ContainerStart(context.Background(), eth.containerID, types.ContainerStartOptions{})
 	if err != nil {
-		log.Printf("Failed to start container, err: %v", err)
+		log.Printf("Failed to start container, err: %v, ip:%v", err, eth.ip)
 		return err
 	}
 
@@ -268,16 +284,20 @@ func (eth *ethereum) Start() error {
 		return errors.New("Failed to start geth")
 	}
 
-	containerJSON, err := eth.client.ContainerInspect(context.Background(), eth.containerID)
+	containerIP := eth.ip
-	if err != nil {
+	if containerIP == "" {
-		log.Print("Failed to inspect container,", err)
+		containerJSON, err := eth.client.ContainerInspect(context.Background(), eth.containerID)
-		return err
+		if err != nil {
+			log.Print("Failed to inspect container,", err)
+			return err
+		}
+		containerIP = containerJSON.NetworkSettings.IPAddress
 	}
 
 	if eth.key != nil {
 		eth.node = discover.NewNode(
 			discover.PubkeyID(&eth.key.PublicKey),
-			net.ParseIP(containerJSON.NetworkSettings.IPAddress),
+			net.ParseIP(containerIP),
 			0,
 			uint16(utils.ListenPortFlag.Value))
 	}

container/network.go

@@ -18,47 +18,122 @@ package container
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"net"
+	"sync"
 
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/network"
 	"github.com/docker/docker/client"
 )
 
-type networkManager struct {
+const (
-	client *client.Client
+	FirstOctet  = 172
-	field1 uint8
+	SecondOctet = 19
-	field2 uint8
+	NetworkName = "testnet"
+)
+
+type DockerNetwork struct {
+	client  *client.Client
+	id      string
+	name    string
+	ipv4Net *net.IPNet
+
+	mutex   sync.Mutex
+	ipIndex net.IP
 }
 
-func NewNetworkManager(c *client.Client, field1 uint8, field2 uint8) *networkManager {
+func NewDockerNetwork() (*DockerNetwork, error) {
-	n := &networkManager{
+	// IP xxx.xxx.0.1 is reserved for docker network gateway
-		client: c,
+	ipv4Addr, ipv4Net, err := net.ParseCIDR(fmt.Sprintf("%d.%d.0.1/16", FirstOctet, SecondOctet))
-		field1: field1,
+	if err != nil {
-		field2: field2,
+		return nil, err
 	}
 
-	return n
+	c, err := client.NewEnvClient()
+	if err != nil {
+		return nil, err
+	}
+
+	network := &DockerNetwork{
+		client:  c,
+		name:    NetworkName,
+		ipv4Net: ipv4Net,
+		ipIndex: ipv4Addr,
+	}
+
+	if err := network.create(); err != nil {
+		return nil, err
+	}
+
+	return network, nil
 }
 
-// CreateNetwork returns network id
+// create creates a docker network with given subnet
-func (n *networkManager) CreateNetwork(name string) (string, error) {
+func (n *DockerNetwork) create() error {
 	ipamConfig := network.IPAMConfig{
-		Subnet: fmt.Sprintf("%d.%d.0.0/16", n.field1, n.field2),
+		Subnet: n.ipv4Net.String(),
 	}
 	ipam := &network.IPAM{
 		Config: []network.IPAMConfig{ipamConfig},
 	}
-	r, e := n.client.NetworkCreate(context.Background(), name, types.NetworkCreate{
+
+	r, err := n.client.NetworkCreate(context.Background(), n.name, types.NetworkCreate{
 		IPAM: ipam,
 	})
-	if e != nil {
+	if err != nil {
-		return "", e
+		return err
-	} else {
-		return r.ID, nil
 	}
+	n.id = r.ID
+	return nil
 }
 
-func (n *networkManager) RemoveNetwork(id string) error {
+func (n *DockerNetwork) ID() string {
-	return n.client.NetworkRemove(context.Background(), id)
+	return n.id
+}
+
+func (n *DockerNetwork) Name() string {
+	return n.name
+}
+
+func (n *DockerNetwork) Remove() error {
+	return n.client.NetworkRemove(context.Background(), n.id)
+}
+
+func (n *DockerNetwork) GetFreeIPAddrs(num int) ([]net.IP, error) {
+	n.mutex.Lock()
+	defer n.mutex.Unlock()
+
+	ips := make([]net.IP, 0)
+	for i := 0; i < num; i++ {
+		ip := dupIP(n.ipIndex)
+		for j := len(ip) - 1; j >= 0; j-- {
+			ip[j]++
+			if ip[j] > 0 {
+				break
+			}
+		}
+
+		if !n.ipv4Net.Contains(ip) {
+			break
+		}
+		ips = append(ips, ip)
+		n.ipIndex = ip
+	}
+
+	if len(ips) != num {
+		return nil, errors.New("Insufficient IP address.")
+	}
+	return ips, nil
+}
+
+func dupIP(ip net.IP) net.IP {
+	// To save space, try and only use 4 bytes
+	if x := ip.To4(); x != nil {
+		ip = x
+	}
+	dup := make(net.IP, len(ip))
+	copy(dup, ip)
+	return dup
 }

container/options.go

@@ -19,6 +19,7 @@ package container
 import (
 	"crypto/ecdsa"
 	"fmt"
+	"net"
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
 )
@@ -37,6 +38,12 @@ func ImageTag(tag string) Option {
 	}
 }
 
+func DockerNetworkName(dockerNetworkName string) Option {
+	return func(eth *ethereum) {
+		eth.dockerNetworkName = dockerNetworkName
+	}
+}
+
 func HostName(hostName string) Option {
 	return func(eth *ethereum) {
 		eth.hostName = hostName
@@ -49,6 +56,12 @@ func HostDataDir(path string) Option {
 	}
 }
 
+func HostIP(ip net.IP) Option {
+	return func(eth *ethereum) {
+		eth.ip = ip.String()
+	}
+}
+
 func HostPort(port int) Option {
 	return func(eth *ethereum) {
 		eth.port = fmt.Sprintf("%d", port)

tests/funtional/block_sync_test.go

@@ -37,7 +37,7 @@ var _ = Describe("Block synchronization testing", func() {
 	)
 
 	BeforeEach(func() {
-		blockchain = container.NewDefaultBlockchain(numberOfValidators)
+		blockchain = container.NewDefaultBlockchain(dockerNetwork, numberOfValidators)
 		Expect(blockchain.Start(true)).To(BeNil())
 	})
 

tests/funtional/byzantine_faulty_test.go

@@ -38,7 +38,7 @@ var _ = Describe("TFS-05: Byzantine Faulty", func() {
 			blockchain container.Blockchain
 		)
 		BeforeEach(func() {
-			blockchain = container.NewDefaultBlockchainWithFaulty(numberOfNormal, numberOfFaulty)
+			blockchain = container.NewDefaultBlockchainWithFaulty(dockerNetwork, numberOfNormal, numberOfFaulty)
 			Expect(blockchain.Start(true)).To(BeNil())
 		})
 
@@ -77,7 +77,7 @@ var _ = Describe("TFS-05: Byzantine Faulty", func() {
 			blockchain container.Blockchain
 		)
 		BeforeEach(func() {
-			blockchain = container.NewDefaultBlockchainWithFaulty(numberOfNormal, numberOfFaulty)
+			blockchain = container.NewDefaultBlockchainWithFaulty(dockerNetwork, numberOfNormal, numberOfFaulty)
 			Expect(blockchain.Start(true)).To(BeNil())
 		})
 

tests/funtional/dynamic_test.go

@@ -38,7 +38,7 @@ var _ = Describe("Dynamic validators addition/removal testing", func() {
 	)
 
 	BeforeEach(func() {
-		blockchain = container.NewDefaultBlockchain(numberOfValidators)
+		blockchain = container.NewDefaultBlockchain(dockerNetwork, numberOfValidators)
 		Expect(blockchain.Start(true)).To(BeNil())
 	})
 

tests/funtional/general_consensus_test.go

@@ -42,7 +42,7 @@ var _ = Describe("TFS-01: General consensus", func() {
 	)
 
 	BeforeEach(func() {
-		blockchain = container.NewDefaultBlockchain(numberOfValidators)
+		blockchain = container.NewDefaultBlockchain(dockerNetwork, numberOfValidators)
 		Expect(blockchain.Start(true)).To(BeNil())
 	})
 

tests/funtional/gossip_network_test.go

@@ -36,7 +36,7 @@ var _ = Describe("TFS-07: Gossip Network", func() {
 	)
 
 	BeforeEach(func() {
-		blockchain = container.NewDefaultBlockchain(numberOfValidators)
+		blockchain = container.NewDefaultBlockchain(dockerNetwork, numberOfValidators)
 		Expect(blockchain.Start(false)).To(BeNil())
 	})
 

tests/funtional/integration_test.go

@@ -21,6 +21,8 @@ import (
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+
+	"github.com/getamis/istanbul-tools/container"
 )
 
 // Example
@@ -71,7 +73,20 @@ import (
 // })
 //
 
+var dockerNetwork *container.DockerNetwork
+
 func TestIstanbul(t *testing.T) {
 	RegisterFailHandler(Fail)
 	RunSpecs(t, "Istanbul Test Suite")
 }
+
+var _ = BeforeSuite(func() {
+	var err error
+	dockerNetwork, err = container.NewDockerNetwork()
+	Expect(err).To(BeNil())
+})
+
+var _ = AfterSuite(func() {
+	err := dockerNetwork.Remove()
+	Expect(err).To(BeNil())
+})

tests/funtional/non_byzantine_faulty_test.go

@@ -36,7 +36,7 @@ var _ = Describe("TFS-04: Non-Byzantine Faulty", func() {
 	)
 
 	BeforeEach(func() {
-		blockchain = container.NewDefaultBlockchain(numberOfValidators)
+		blockchain = container.NewDefaultBlockchain(dockerNetwork, numberOfValidators)
 		Expect(blockchain.Start(true)).To(BeNil())
 	})
 

tests/funtional/recoverability_test.go

@@ -36,7 +36,7 @@ var _ = Describe("TFS-03: Recoverability testing", func() {
 	)
 
 	BeforeEach(func() {
-		blockchain = container.NewDefaultBlockchain(numberOfValidators)
+		blockchain = container.NewDefaultBlockchain(dockerNetwork, numberOfValidators)
 		Expect(blockchain.Start(true)).To(BeNil())
 	})