container: run geth container with given IP
This commit is contained in:
parent
c4bddfe6fb
commit
14681bf913
|
@ -20,6 +20,7 @@ import (
|
|||
"context"
|
||||
"crypto/ecdsa"
|
||||
"log"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
@ -42,7 +43,7 @@ type Blockchain interface {
|
|||
CreateNodes(int, ...Option) ([]Ethereum, error)
|
||||
}
|
||||
|
||||
func NewBlockchain(numOfValidators int, options ...Option) (bc *blockchain) {
|
||||
func NewBlockchain(network *DockerNetwork, numOfValidators int, options ...Option) (bc *blockchain) {
|
||||
bc = &blockchain{opts: options}
|
||||
|
||||
var err error
|
||||
|
@ -51,12 +52,25 @@ func NewBlockchain(numOfValidators int, options ...Option) (bc *blockchain) {
|
|||
log.Fatalf("Cannot connect to Docker daemon, err: %v", err)
|
||||
}
|
||||
|
||||
if network == nil {
|
||||
bc.defaultNetwork, err = NewDockerNetwork()
|
||||
if err != nil {
|
||||
log.Fatalf("Cannot create Docker network, err: %v", err)
|
||||
}
|
||||
network = bc.defaultNetwork
|
||||
}
|
||||
|
||||
bc.dockerNetworkName = network.Name()
|
||||
bc.getFreeIPAddrs = network.GetFreeIPAddrs
|
||||
bc.opts = append(bc.opts, DockerNetworkName(bc.dockerNetworkName))
|
||||
|
||||
bc.addValidators(numOfValidators)
|
||||
return bc
|
||||
}
|
||||
|
||||
func NewDefaultBlockchain(numOfValidators int) (bc *blockchain) {
|
||||
return NewBlockchain(numOfValidators,
|
||||
func NewDefaultBlockchain(network *DockerNetwork, numOfValidators int) (bc *blockchain) {
|
||||
return NewBlockchain(network,
|
||||
numOfValidators,
|
||||
ImageRepository("quay.io/amis/geth"),
|
||||
ImageTag("istanbul_develop"),
|
||||
DataDir("/data"),
|
||||
|
@ -73,7 +87,7 @@ func NewDefaultBlockchain(numOfValidators int) (bc *blockchain) {
|
|||
)
|
||||
}
|
||||
|
||||
func NewDefaultBlockchainWithFaulty(numOfNormal int, numOfFaulty int) (bc *blockchain) {
|
||||
func NewDefaultBlockchainWithFaulty(network *DockerNetwork, numOfNormal int, numOfFaulty int) (bc *blockchain) {
|
||||
commonOpts := [...]Option{
|
||||
DataDir("/data"),
|
||||
WebSocket(),
|
||||
|
@ -101,24 +115,48 @@ func NewDefaultBlockchainWithFaulty(numOfNormal int, numOfFaulty int) (bc *block
|
|||
log.Fatalf("Cannot connect to Docker daemon, err: %v", err)
|
||||
}
|
||||
|
||||
keys, addrs := generateKeys(numOfNormal + numOfFaulty)
|
||||
if network == nil {
|
||||
bc.defaultNetwork, err = NewDockerNetwork()
|
||||
if err != nil {
|
||||
log.Fatalf("Cannot create Docker network, err: %v", err)
|
||||
}
|
||||
network = bc.defaultNetwork
|
||||
}
|
||||
|
||||
bc.dockerNetworkName = network.Name()
|
||||
bc.getFreeIPAddrs = network.GetFreeIPAddrs
|
||||
|
||||
normalOpts = append(normalOpts, DockerNetworkName(bc.dockerNetworkName))
|
||||
faultyOpts = append(faultyOpts, DockerNetworkName(bc.dockerNetworkName))
|
||||
|
||||
totalNodes := numOfNormal + numOfFaulty
|
||||
|
||||
ips, err := bc.getFreeIPAddrs(totalNodes)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to get free ip addresses, err: %v", err)
|
||||
}
|
||||
|
||||
keys, addrs := generateKeys(totalNodes)
|
||||
bc.setupGenesis(addrs)
|
||||
// Create normal validators
|
||||
bc.opts = normalOpts
|
||||
bc.setupValidators(keys[:numOfNormal], bc.opts...)
|
||||
bc.setupValidators(ips[:numOfNormal], keys[:numOfNormal], bc.opts...)
|
||||
// Create faulty validators
|
||||
bc.opts = faultyOpts
|
||||
bc.setupValidators(keys[numOfNormal:], bc.opts...)
|
||||
bc.setupValidators(ips[numOfNormal:], keys[numOfNormal:], bc.opts...)
|
||||
return bc
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
|
||||
type blockchain struct {
|
||||
dockerClient *client.Client
|
||||
genesisFile string
|
||||
validators []Ethereum
|
||||
opts []Option
|
||||
dockerClient *client.Client
|
||||
defaultNetwork *DockerNetwork
|
||||
dockerNetworkName string
|
||||
getFreeIPAddrs func(int) ([]net.IP, error)
|
||||
genesisFile string
|
||||
validators []Ethereum
|
||||
opts []Option
|
||||
}
|
||||
|
||||
func (bc *blockchain) AddValidators(numOfValidators int) ([]Ethereum, error) {
|
||||
|
@ -202,7 +240,16 @@ func (bc *blockchain) Start(strong bool) error {
|
|||
}
|
||||
|
||||
func (bc *blockchain) Stop(force bool) error {
|
||||
return bc.stop(bc.validators, force)
|
||||
if err := bc.stop(bc.validators, force); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if bc.defaultNetwork != nil {
|
||||
err := bc.defaultNetwork.Remove()
|
||||
bc.defaultNetwork = nil
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (bc *blockchain) Finalize() {
|
||||
|
@ -214,6 +261,11 @@ func (bc *blockchain) Validators() []Ethereum {
|
|||
}
|
||||
|
||||
func (bc *blockchain) CreateNodes(num int, options ...Option) (nodes []Ethereum, err error) {
|
||||
ips, err := bc.getFreeIPAddrs(num)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for i := 0; i < num; i++ {
|
||||
var opts []Option
|
||||
opts = append(opts, options...)
|
||||
|
@ -226,6 +278,8 @@ func (bc *blockchain) CreateNodes(num int, options ...Option) (nodes []Ethereum,
|
|||
}
|
||||
opts = append(opts, HostDataDir(dataDir))
|
||||
opts = append(opts, HostWebSocketPort(freeport.GetPort()))
|
||||
opts = append(opts, HostIP(ips[i]))
|
||||
opts = append(opts, DockerNetworkName(bc.dockerNetworkName))
|
||||
|
||||
geth := NewEthereum(
|
||||
bc.dockerClient,
|
||||
|
@ -247,9 +301,13 @@ func (bc *blockchain) CreateNodes(num int, options ...Option) (nodes []Ethereum,
|
|||
// ----------------------------------------------------------------------------
|
||||
|
||||
func (bc *blockchain) addValidators(numOfValidators int) error {
|
||||
ips, err := bc.getFreeIPAddrs(numOfValidators)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
keys, addrs := generateKeys(numOfValidators)
|
||||
bc.setupGenesis(addrs)
|
||||
bc.setupValidators(keys, bc.opts...)
|
||||
bc.setupValidators(ips, keys, bc.opts...)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
@ -282,7 +340,7 @@ func (bc *blockchain) setupGenesis(addrs []common.Address) {
|
|||
}
|
||||
}
|
||||
|
||||
func (bc *blockchain) setupValidators(keys []*ecdsa.PrivateKey, options ...Option) {
|
||||
func (bc *blockchain) setupValidators(ips []net.IP, keys []*ecdsa.PrivateKey, options ...Option) {
|
||||
for i := 0; i < len(keys); i++ {
|
||||
var opts []Option
|
||||
opts = append(opts, options...)
|
||||
|
@ -295,6 +353,7 @@ func (bc *blockchain) setupValidators(keys []*ecdsa.PrivateKey, options ...Optio
|
|||
opts = append(opts, HostDataDir(dataDir))
|
||||
opts = append(opts, HostWebSocketPort(freeport.GetPort()))
|
||||
opts = append(opts, Key(keys[i]))
|
||||
opts = append(opts, HostIP(ips[i]))
|
||||
|
||||
geth := NewEthereum(
|
||||
bc.dockerClient,
|
||||
|
|
|
@ -23,6 +23,7 @@ import (
|
|||
|
||||
func TestEthereumBlockchain(t *testing.T) {
|
||||
chain := NewBlockchain(
|
||||
nil,
|
||||
4,
|
||||
ImageRepository("quay.io/amis/geth"),
|
||||
ImageTag("istanbul_develop"),
|
||||
|
|
|
@ -33,6 +33,7 @@ import (
|
|||
"github.com/docker/docker/api/types"
|
||||
"github.com/docker/docker/api/types/container"
|
||||
"github.com/docker/docker/api/types/filters"
|
||||
"github.com/docker/docker/api/types/network"
|
||||
"github.com/docker/docker/client"
|
||||
"github.com/docker/go-connections/nat"
|
||||
"github.com/ethereum/go-ethereum/cmd/utils"
|
||||
|
@ -119,6 +120,7 @@ type ethereum struct {
|
|||
ok bool
|
||||
flags []string
|
||||
dataDir string
|
||||
ip string
|
||||
port string
|
||||
rpcPort string
|
||||
wsPort string
|
||||
|
@ -126,8 +128,9 @@ type ethereum struct {
|
|||
containerID string
|
||||
node *discover.Node
|
||||
|
||||
imageRepository string
|
||||
imageTag string
|
||||
imageRepository string
|
||||
imageTag string
|
||||
dockerNetworkName string
|
||||
|
||||
key *ecdsa.PrivateKey
|
||||
logging bool
|
||||
|
@ -224,6 +227,19 @@ func (eth *ethereum) Start() error {
|
|||
binds = append(binds, eth.dataDir+":"+utils.DataDirFlag.Value.Value)
|
||||
}
|
||||
|
||||
var networkingConfig *network.NetworkingConfig
|
||||
if eth.ip != "" && eth.dockerNetworkName != "" {
|
||||
endpointsConfig := make(map[string]*network.EndpointSettings)
|
||||
endpointsConfig[eth.dockerNetworkName] = &network.EndpointSettings{
|
||||
IPAMConfig: &network.EndpointIPAMConfig{
|
||||
IPv4Address: eth.ip,
|
||||
},
|
||||
}
|
||||
networkingConfig = &network.NetworkingConfig{
|
||||
EndpointsConfig: endpointsConfig,
|
||||
}
|
||||
}
|
||||
|
||||
resp, err := eth.client.ContainerCreate(context.Background(),
|
||||
&container.Config{
|
||||
Hostname: "geth-" + eth.hostName,
|
||||
|
@ -234,7 +250,7 @@ func (eth *ethereum) Start() error {
|
|||
&container.HostConfig{
|
||||
Binds: binds,
|
||||
PortBindings: portBindings,
|
||||
}, nil, "")
|
||||
}, networkingConfig, "")
|
||||
if err != nil {
|
||||
log.Printf("Failed to create container, err: %v", err)
|
||||
return err
|
||||
|
@ -244,7 +260,7 @@ func (eth *ethereum) Start() error {
|
|||
|
||||
err = eth.client.ContainerStart(context.Background(), eth.containerID, types.ContainerStartOptions{})
|
||||
if err != nil {
|
||||
log.Printf("Failed to start container, err: %v", err)
|
||||
log.Printf("Failed to start container, err: %v, ip:%v", err, eth.ip)
|
||||
return err
|
||||
}
|
||||
|
||||
|
@ -268,16 +284,20 @@ func (eth *ethereum) Start() error {
|
|||
return errors.New("Failed to start geth")
|
||||
}
|
||||
|
||||
containerJSON, err := eth.client.ContainerInspect(context.Background(), eth.containerID)
|
||||
if err != nil {
|
||||
log.Print("Failed to inspect container,", err)
|
||||
return err
|
||||
containerIP := eth.ip
|
||||
if containerIP == "" {
|
||||
containerJSON, err := eth.client.ContainerInspect(context.Background(), eth.containerID)
|
||||
if err != nil {
|
||||
log.Print("Failed to inspect container,", err)
|
||||
return err
|
||||
}
|
||||
containerIP = containerJSON.NetworkSettings.IPAddress
|
||||
}
|
||||
|
||||
if eth.key != nil {
|
||||
eth.node = discover.NewNode(
|
||||
discover.PubkeyID(ð.key.PublicKey),
|
||||
net.ParseIP(containerJSON.NetworkSettings.IPAddress),
|
||||
net.ParseIP(containerIP),
|
||||
0,
|
||||
uint16(utils.ListenPortFlag.Value))
|
||||
}
|
||||
|
|
|
@ -18,47 +18,122 @@ package container
|
|||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"sync"
|
||||
|
||||
"github.com/docker/docker/api/types"
|
||||
"github.com/docker/docker/api/types/network"
|
||||
"github.com/docker/docker/client"
|
||||
)
|
||||
|
||||
type networkManager struct {
|
||||
client *client.Client
|
||||
field1 uint8
|
||||
field2 uint8
|
||||
const (
|
||||
FirstOctet = 172
|
||||
SecondOctet = 19
|
||||
NetworkName = "testnet"
|
||||
)
|
||||
|
||||
type DockerNetwork struct {
|
||||
client *client.Client
|
||||
id string
|
||||
name string
|
||||
ipv4Net *net.IPNet
|
||||
|
||||
mutex sync.Mutex
|
||||
ipIndex net.IP
|
||||
}
|
||||
|
||||
func NewNetworkManager(c *client.Client, field1 uint8, field2 uint8) *networkManager {
|
||||
n := &networkManager{
|
||||
client: c,
|
||||
field1: field1,
|
||||
field2: field2,
|
||||
func NewDockerNetwork() (*DockerNetwork, error) {
|
||||
// IP xxx.xxx.0.1 is reserved for docker network gateway
|
||||
ipv4Addr, ipv4Net, err := net.ParseCIDR(fmt.Sprintf("%d.%d.0.1/16", FirstOctet, SecondOctet))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return n
|
||||
c, err := client.NewEnvClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
network := &DockerNetwork{
|
||||
client: c,
|
||||
name: NetworkName,
|
||||
ipv4Net: ipv4Net,
|
||||
ipIndex: ipv4Addr,
|
||||
}
|
||||
|
||||
if err := network.create(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return network, nil
|
||||
}
|
||||
|
||||
// CreateNetwork returns network id
|
||||
func (n *networkManager) CreateNetwork(name string) (string, error) {
|
||||
// create creates a docker network with given subnet
|
||||
func (n *DockerNetwork) create() error {
|
||||
ipamConfig := network.IPAMConfig{
|
||||
Subnet: fmt.Sprintf("%d.%d.0.0/16", n.field1, n.field2),
|
||||
Subnet: n.ipv4Net.String(),
|
||||
}
|
||||
ipam := &network.IPAM{
|
||||
Config: []network.IPAMConfig{ipamConfig},
|
||||
}
|
||||
r, e := n.client.NetworkCreate(context.Background(), name, types.NetworkCreate{
|
||||
|
||||
r, err := n.client.NetworkCreate(context.Background(), n.name, types.NetworkCreate{
|
||||
IPAM: ipam,
|
||||
})
|
||||
if e != nil {
|
||||
return "", e
|
||||
} else {
|
||||
return r.ID, nil
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
n.id = r.ID
|
||||
return nil
|
||||
}
|
||||
|
||||
func (n *networkManager) RemoveNetwork(id string) error {
|
||||
return n.client.NetworkRemove(context.Background(), id)
|
||||
func (n *DockerNetwork) ID() string {
|
||||
return n.id
|
||||
}
|
||||
|
||||
func (n *DockerNetwork) Name() string {
|
||||
return n.name
|
||||
}
|
||||
|
||||
func (n *DockerNetwork) Remove() error {
|
||||
return n.client.NetworkRemove(context.Background(), n.id)
|
||||
}
|
||||
|
||||
func (n *DockerNetwork) GetFreeIPAddrs(num int) ([]net.IP, error) {
|
||||
n.mutex.Lock()
|
||||
defer n.mutex.Unlock()
|
||||
|
||||
ips := make([]net.IP, 0)
|
||||
for i := 0; i < num; i++ {
|
||||
ip := dupIP(n.ipIndex)
|
||||
for j := len(ip) - 1; j >= 0; j-- {
|
||||
ip[j]++
|
||||
if ip[j] > 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !n.ipv4Net.Contains(ip) {
|
||||
break
|
||||
}
|
||||
ips = append(ips, ip)
|
||||
n.ipIndex = ip
|
||||
}
|
||||
|
||||
if len(ips) != num {
|
||||
return nil, errors.New("Insufficient IP address.")
|
||||
}
|
||||
return ips, nil
|
||||
}
|
||||
|
||||
func dupIP(ip net.IP) net.IP {
|
||||
// To save space, try and only use 4 bytes
|
||||
if x := ip.To4(); x != nil {
|
||||
ip = x
|
||||
}
|
||||
dup := make(net.IP, len(ip))
|
||||
copy(dup, ip)
|
||||
return dup
|
||||
}
|
||||
|
|
|
@ -19,6 +19,7 @@ package container
|
|||
import (
|
||||
"crypto/ecdsa"
|
||||
"fmt"
|
||||
"net"
|
||||
|
||||
"github.com/ethereum/go-ethereum/cmd/utils"
|
||||
)
|
||||
|
@ -37,6 +38,12 @@ func ImageTag(tag string) Option {
|
|||
}
|
||||
}
|
||||
|
||||
func DockerNetworkName(dockerNetworkName string) Option {
|
||||
return func(eth *ethereum) {
|
||||
eth.dockerNetworkName = dockerNetworkName
|
||||
}
|
||||
}
|
||||
|
||||
func HostName(hostName string) Option {
|
||||
return func(eth *ethereum) {
|
||||
eth.hostName = hostName
|
||||
|
@ -49,6 +56,12 @@ func HostDataDir(path string) Option {
|
|||
}
|
||||
}
|
||||
|
||||
func HostIP(ip net.IP) Option {
|
||||
return func(eth *ethereum) {
|
||||
eth.ip = ip.String()
|
||||
}
|
||||
}
|
||||
|
||||
func HostPort(port int) Option {
|
||||
return func(eth *ethereum) {
|
||||
eth.port = fmt.Sprintf("%d", port)
|
||||
|
|
|
@ -37,7 +37,7 @@ var _ = Describe("Block synchronization testing", func() {
|
|||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
blockchain = container.NewDefaultBlockchain(numberOfValidators)
|
||||
blockchain = container.NewDefaultBlockchain(dockerNetwork, numberOfValidators)
|
||||
Expect(blockchain.Start(true)).To(BeNil())
|
||||
})
|
||||
|
||||
|
|
|
@ -38,7 +38,7 @@ var _ = Describe("TFS-05: Byzantine Faulty", func() {
|
|||
blockchain container.Blockchain
|
||||
)
|
||||
BeforeEach(func() {
|
||||
blockchain = container.NewDefaultBlockchainWithFaulty(numberOfNormal, numberOfFaulty)
|
||||
blockchain = container.NewDefaultBlockchainWithFaulty(dockerNetwork, numberOfNormal, numberOfFaulty)
|
||||
Expect(blockchain.Start(true)).To(BeNil())
|
||||
})
|
||||
|
||||
|
@ -77,7 +77,7 @@ var _ = Describe("TFS-05: Byzantine Faulty", func() {
|
|||
blockchain container.Blockchain
|
||||
)
|
||||
BeforeEach(func() {
|
||||
blockchain = container.NewDefaultBlockchainWithFaulty(numberOfNormal, numberOfFaulty)
|
||||
blockchain = container.NewDefaultBlockchainWithFaulty(dockerNetwork, numberOfNormal, numberOfFaulty)
|
||||
Expect(blockchain.Start(true)).To(BeNil())
|
||||
})
|
||||
|
||||
|
|
|
@ -38,7 +38,7 @@ var _ = Describe("Dynamic validators addition/removal testing", func() {
|
|||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
blockchain = container.NewDefaultBlockchain(numberOfValidators)
|
||||
blockchain = container.NewDefaultBlockchain(dockerNetwork, numberOfValidators)
|
||||
Expect(blockchain.Start(true)).To(BeNil())
|
||||
})
|
||||
|
||||
|
|
|
@ -42,7 +42,7 @@ var _ = Describe("TFS-01: General consensus", func() {
|
|||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
blockchain = container.NewDefaultBlockchain(numberOfValidators)
|
||||
blockchain = container.NewDefaultBlockchain(dockerNetwork, numberOfValidators)
|
||||
Expect(blockchain.Start(true)).To(BeNil())
|
||||
})
|
||||
|
||||
|
|
|
@ -36,7 +36,7 @@ var _ = Describe("TFS-07: Gossip Network", func() {
|
|||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
blockchain = container.NewDefaultBlockchain(numberOfValidators)
|
||||
blockchain = container.NewDefaultBlockchain(dockerNetwork, numberOfValidators)
|
||||
Expect(blockchain.Start(false)).To(BeNil())
|
||||
})
|
||||
|
||||
|
|
|
@ -21,6 +21,8 @@ import (
|
|||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/getamis/istanbul-tools/container"
|
||||
)
|
||||
|
||||
// Example
|
||||
|
@ -71,7 +73,20 @@ import (
|
|||
// })
|
||||
//
|
||||
|
||||
var dockerNetwork *container.DockerNetwork
|
||||
|
||||
func TestIstanbul(t *testing.T) {
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecs(t, "Istanbul Test Suite")
|
||||
}
|
||||
|
||||
var _ = BeforeSuite(func() {
|
||||
var err error
|
||||
dockerNetwork, err = container.NewDockerNetwork()
|
||||
Expect(err).To(BeNil())
|
||||
})
|
||||
|
||||
var _ = AfterSuite(func() {
|
||||
err := dockerNetwork.Remove()
|
||||
Expect(err).To(BeNil())
|
||||
})
|
||||
|
|
|
@ -36,7 +36,7 @@ var _ = Describe("TFS-04: Non-Byzantine Faulty", func() {
|
|||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
blockchain = container.NewDefaultBlockchain(numberOfValidators)
|
||||
blockchain = container.NewDefaultBlockchain(dockerNetwork, numberOfValidators)
|
||||
Expect(blockchain.Start(true)).To(BeNil())
|
||||
})
|
||||
|
||||
|
|
|
@ -36,7 +36,7 @@ var _ = Describe("TFS-03: Recoverability testing", func() {
|
|||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
blockchain = container.NewDefaultBlockchain(numberOfValidators)
|
||||
blockchain = container.NewDefaultBlockchain(dockerNetwork, numberOfValidators)
|
||||
Expect(blockchain.Start(true)).To(BeNil())
|
||||
})
|
||||
|
||||
|
|
Loading…
Reference in New Issue