poanetwork
/
quorum
mirror of https://github.com/poanetwork/quorum.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge remote-tracking branch 'remotes/goeth/release/1.8' into geth-upgrade-1.8.15_new 

# Conflicts:
#	.travis.yml
#	README.md
#	cmd/utils/flags.go
#	consensus/consensus.go
#	consensus/ethash/consensus.go
#	consensus/ethash/ethash.go
#	core/blockchain.go
#	core/genesis_test.go
#	core/state/state_object.go
#	core/state_processor.go
#	core/vm/errors.go
#	core/vm/evm.go
#	core/vm/gas_table.go
#	core/vm/instructions_test.go
#	core/vm/interface.go
#	eth/api_tracer.go
#	eth/backend.go
#	eth/config.go
#	eth/filters/filter.go
#	les/backend.go
#	les/helper_test.go
#	miner/worker.go
#	params/config.go
#	params/version.go
This commit is contained in:
amalraj.manigmail.com 2018-10-04 16:19:02 +08:00
parent d35f6e2c5e 477eb0933b
commit 47571689ee
731 changed files with 76030 additions and 29582 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/no-response.yml vendored
View File

@ -7,5 +7,5 @@ closeComment: >
This issue has been automatically closed because there has been no response
to our request for more information from the original author. With only the
information that is currently in the issue, we don't have enough information
to take action. Please reach out if you have or find the answers we need so
that we can investigate further.
to take action. Please reach out if you have more relevant information or
answers to our questions so that we can investigate further.

29
.travis.yml
View File

@ -22,11 +22,22 @@ matrix:
- go run build/ci.go install
- go run build/ci.go test -coverage
# These are the latest Go versions.
- os: linux
dist: trusty
sudo: required
go: 1.10.x
script:
- sudo modprobe fuse
- sudo chmod 666 /dev/fuse
- sudo chown root:$USER /etc/fuse.conf
- go run build/ci.go install
- go run build/ci.go test -coverage $TEST_PACKAGES
# These are the latest Go versions.
- os: linux
dist: trusty
sudo: required
go: 1.11.x
script:
- sudo -E apt-add-repository -y "ppa:chris-lea/libsodium"
- sudo apt-get update
@ -41,7 +52,7 @@ matrix:
- go run build/ci.go test -coverage
- os: osx
go: 1.10.x
go: 1.11.x
script:
- unset -f cd # workaround for https://github.com/travis-ci/travis-ci/issues/8703
- brew update
@ -54,7 +65,7 @@ matrix:
# This builder only tests code linters on latest version of Go
- os: linux
dist: trusty
go: 1.10.x
go: 1.11.x
env:
- lint
git:
@ -65,7 +76,7 @@ matrix:
# This builder does the Ubuntu PPA upload
- os: linux
dist: trusty
go: 1.10.x
go: 1.11.x
env:
- ubuntu-ppa
git:
@ -84,7 +95,7 @@ matrix:
- os: linux
dist: trusty
sudo: required
go: 1.10.x
go: 1.11.x
env:
- azure-linux
git:
@ -118,7 +129,7 @@ matrix:
dist: trusty
services:
- docker
go: 1.10.x
go: 1.11.x
env:
- azure-linux-mips
git:
@ -162,7 +173,7 @@ matrix:
git:
submodules: false # avoid cloning ethereum/tests
before_install:
- curl https://storage.googleapis.com/golang/go1.10.3.linux-amd64.tar.gz | tar -xz
- curl https://storage.googleapis.com/golang/go1.11.linux-amd64.tar.gz | tar -xz
- export PATH=`pwd`/go/bin:$PATH
- export GOROOT=`pwd`/go
- export GOPATH=$HOME/go
@ -179,7 +190,7 @@ matrix:
# This builder does the OSX Azure, iOS CocoaPods and iOS Azure uploads
- os: osx
go: 1.10.x
go: 1.11.x
env:
- azure-osx
- azure-ios
@ -208,7 +219,7 @@ matrix:
# This builder does the Azure archive purges to avoid accumulating junk
- os: linux
dist: trusty
go: 1.10.x
go: 1.11.x
env:
- azure-purge
git:

1
AUTHORS
View File

@ -171,3 +171,4 @@ xiekeyang <xiekeyang@users.noreply.github.com>
yoza <yoza.is12s@gmail.com>
ΞTHΞЯSPHΞЯΞ <{viktor.tron,nagydani,zsfelfoldi}@gmail.com>
Максим Чусовлянов <mchusovlianov@gmail.com>
Ralph Caraveo <deckarep@gmail.com>

2
Dockerfile
View File

@ -1,5 +1,5 @@
# Build Geth in a stock Go builder container
FROM golang:1.10-alpine as builder
FROM golang:1.11-alpine as builder
RUN apk add --no-cache make gcc musl-dev linux-headers

2
Dockerfile.alltools
View File

@ -1,5 +1,5 @@
# Build Geth in a stock Go builder container
FROM golang:1.10-alpine as builder
FROM golang:1.11-alpine as builder
RUN apk add --no-cache make gcc musl-dev linux-headers

1
Makefile
View File

@ -45,6 +45,7 @@ lint: ## Run linters.
build/env.sh go run build/ci.go lint
clean:
./build/clean_go_build_cache.sh
rm -fr build/_workspace/pkg/ $(GOBIN)/*
# The devtools target installs tools required for 'go generate'.

1
VERSION
View File

@ -1 +0,0 @@
1.8.12

38
accounts/abi/bind/backends/simulated.go
View File

@ -65,11 +65,11 @@ type SimulatedBackend struct {
// NewSimulatedBackend creates a new binding backend using a simulated blockchain
// for testing purposes.
func NewSimulatedBackend(alloc core.GenesisAlloc) *SimulatedBackend {
func NewSimulatedBackend(alloc core.GenesisAlloc, gasLimit uint64) *SimulatedBackend {
database := ethdb.NewMemDatabase()
genesis := core.Genesis{Config: params.AllEthashProtocolChanges, Alloc: alloc}
genesis := core.Genesis{Config: params.AllEthashProtocolChanges, GasLimit: gasLimit, Alloc: alloc}
genesis.MustCommit(database)
blockchain, _ := core.NewBlockChain(database, nil, genesis.Config, ethash.NewFaker(), vm.Config{})
blockchain, _ := core.NewBlockChain(database, nil, genesis.Config, ethash.NewFaker(), vm.Config{}, nil)
backend := &SimulatedBackend{
database: database,
@ -324,18 +324,24 @@ func (b *SimulatedBackend) SendTransaction(ctx context.Context, tx *types.Transa
//
// TODO(karalabe): Deprecate when the subscription one can return past data too.
func (b *SimulatedBackend) FilterLogs(ctx context.Context, query ethereum.FilterQuery) ([]types.Log, error) {
// Initialize unset filter boundaried to run from genesis to chain head
from := int64(0)
if query.FromBlock != nil {
from = query.FromBlock.Int64()
var filter *filters.Filter
if query.BlockHash != nil {
// Block filter requested, construct a single-shot filter
filter = filters.NewBlockFilter(&filterBackend{b.database, b.blockchain}, *query.BlockHash, query.Addresses, query.Topics)
} else {
// Initialize unset filter boundaried to run from genesis to chain head
from := int64(0)
if query.FromBlock != nil {
from = query.FromBlock.Int64()
}
to := int64(-1)
if query.ToBlock != nil {
to = query.ToBlock.Int64()
}
// Construct the range filter
filter = filters.NewRangeFilter(&filterBackend{b.database, b.blockchain}, from, to, query.Addresses, query.Topics)
}
to := int64(-1)
if query.ToBlock != nil {
to = query.ToBlock.Int64()
}
// Construct and execute the filter
filter := filters.New(&filterBackend{b.database, b.blockchain}, from, to, query.Addresses, query.Topics)
// Run the filter and return all the logs
logs, err := filter.Logs(ctx)
if err != nil {
return nil, err
@ -430,6 +436,10 @@ func (fb *filterBackend) HeaderByNumber(ctx context.Context, block rpc.BlockNumb
return fb.bc.GetHeaderByNumber(uint64(block.Int64())), nil
}
func (fb *filterBackend) HeaderByHash(ctx context.Context, hash common.Hash) (*types.Header, error) {
return fb.bc.GetHeaderByHash(hash), nil
}
func (fb *filterBackend) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) {
number := rawdb.ReadHeaderNumber(fb.db, hash)
if number == nil {

4
accounts/abi/bind/bind.go
View File

@ -207,7 +207,7 @@ func bindTypeGo(kind abi.Type) string {
// The inner function of bindTypeGo, this finds the inner type of stringKind.
// (Or just the type itself if it is not an array or slice)
// The length of the matched part is returned, with the the translated type.
// The length of the matched part is returned, with the translated type.
func bindUnnestedTypeGo(stringKind string) (int, string) {
switch {
@ -255,7 +255,7 @@ func bindTypeJava(kind abi.Type) string {
// The inner function of bindTypeJava, this finds the inner type of stringKind.
// (Or just the type itself if it is not an array or slice)
// The length of the matched part is returned, with the the translated type.
// The length of the matched part is returned, with the translated type.
func bindUnnestedTypeJava(stringKind string) (int, string) {
switch {

24
accounts/abi/bind/bind_test.go
View File

@ -229,7 +229,7 @@ var bindTests = []struct {
// Generate a new random account and a funded simulator
key, _ := crypto.GenerateKey()
auth := bind.NewKeyedTransactor(key)
sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}})
sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000)
// Deploy an interaction tester contract and call a transaction on it
_, _, interactor, err := DeployInteractor(auth, sim, "Deploy string")
@ -270,7 +270,7 @@ var bindTests = []struct {
// Generate a new random account and a funded simulator
key, _ := crypto.GenerateKey()
auth := bind.NewKeyedTransactor(key)
sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}})
sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000)
// Deploy a tuple tester contract and execute a structured call on it
_, _, getter, err := DeployGetter(auth, sim)
@ -302,7 +302,7 @@ var bindTests = []struct {
// Generate a new random account and a funded simulator
key, _ := crypto.GenerateKey()
auth := bind.NewKeyedTransactor(key)
sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}})
sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000)
// Deploy a tuple tester contract and execute a structured call on it
_, _, tupler, err := DeployTupler(auth, sim)
@ -344,7 +344,7 @@ var bindTests = []struct {
// Generate a new random account and a funded simulator
key, _ := crypto.GenerateKey()
auth := bind.NewKeyedTransactor(key)
sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}})
sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000)
// Deploy a slice tester contract and execute a n array call on it
_, _, slicer, err := DeploySlicer(auth, sim)
@ -378,7 +378,7 @@ var bindTests = []struct {
// Generate a new random account and a funded simulator
key, _ := crypto.GenerateKey()
auth := bind.NewKeyedTransactor(key)
sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}})
sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000)
// Deploy a default method invoker contract and execute its default method
_, _, defaulter, err := DeployDefaulter(auth, sim)
@ -411,7 +411,7 @@ var bindTests = []struct {
`[{"constant":true,"inputs":[],"name":"String","outputs":[{"name":"","type":"string"}],"type":"function"}]`,
`
// Create a simulator and wrap a non-deployed contract
sim := backends.NewSimulatedBackend(nil)
sim := backends.NewSimulatedBackend(nil, uint64(10000000000))
nonexistent, err := NewNonExistent(common.Address{}, sim)
if err != nil {
@ -447,7 +447,7 @@ var bindTests = []struct {
// Generate a new random account and a funded simulator
key, _ := crypto.GenerateKey()
auth := bind.NewKeyedTransactor(key)
sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}})
sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000)
// Deploy a funky gas pattern contract
_, _, limiter, err := DeployFunkyGasPattern(auth, sim)
@ -482,7 +482,7 @@ var bindTests = []struct {
// Generate a new random account and a funded simulator
key, _ := crypto.GenerateKey()
auth := bind.NewKeyedTransactor(key)
sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}})
sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000)
// Deploy a sender tester contract and execute a structured call on it
_, _, callfrom, err := DeployCallFrom(auth, sim)
@ -542,7 +542,7 @@ var bindTests = []struct {
// Generate a new random account and a funded simulator
key, _ := crypto.GenerateKey()
auth := bind.NewKeyedTransactor(key)
sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}})
sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000)
// Deploy a underscorer tester contract and execute a structured call on it
_, _, underscorer, err := DeployUnderscorer(auth, sim)
@ -612,7 +612,7 @@ var bindTests = []struct {
// Generate a new random account and a funded simulator
key, _ := crypto.GenerateKey()
auth := bind.NewKeyedTransactor(key)
sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}})
sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000)
// Deploy an eventer contract
_, _, eventer, err := DeployEventer(auth, sim)
@ -761,7 +761,7 @@ var bindTests = []struct {
// Generate a new random account and a funded simulator
key, _ := crypto.GenerateKey()
auth := bind.NewKeyedTransactor(key)
sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}})
sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000)
//deploy the test contract
_, _, testContract, err := DeployDeeplyNestedArray(auth, sim)
@ -820,7 +820,7 @@ func TestBindings(t *testing.T) {
t.Skip("go sdk not found for testing")
}
// Skip the test if the go-ethereum sources are symlinked (https://github.com/golang/go/issues/14845)
linkTestCode := fmt.Sprintf("package linktest\nfunc CheckSymlinks(){\nfmt.Println(backends.NewSimulatedBackend(nil))\n}")
linkTestCode := fmt.Sprintf("package linktest\nfunc CheckSymlinks(){\nfmt.Println(backends.NewSimulatedBackend(nil,uint64(10000000000)))\n}")
linkTestDeps, err := imports.Process(os.TempDir(), []byte(linkTestCode), nil)
if err != nil {
t.Fatalf("failed check for goimports symlink bug: %v", err)

8
accounts/abi/bind/util_test.go
View File

@ -53,9 +53,11 @@ var waitDeployedTests = map[string]struct {
func TestWaitDeployed(t *testing.T) {
for name, test := range waitDeployedTests {
backend := backends.NewSimulatedBackend(core.GenesisAlloc{
crypto.PubkeyToAddress(testKey.PublicKey): {Balance: big.NewInt(10000000000)},
})
backend := backends.NewSimulatedBackend(
core.GenesisAlloc{
crypto.PubkeyToAddress(testKey.PublicKey): {Balance: big.NewInt(10000000000)},
}, 10000000,
)
// Create the transaction.
tx := types.NewContractCreation(0, big.NewInt(0), test.gas, big.NewInt(1), common.FromHex(test.code))

4
accounts/abi/method.go
View File

@ -47,10 +47,8 @@ type Method struct {
// Please note that "int" is substitute for its canonical representation "int256"
func (method Method) Sig() string {
types := make([]string, len(method.Inputs))
i := 0
for _, input := range method.Inputs {
for i, input := range method.Inputs {
types[i] = input.Type.String()
i++
}
return fmt.Sprintf("%v(%v)", method.Name, strings.Join(types, ","))
}

7
accounts/abi/type.go
View File

@ -103,7 +103,12 @@ func NewType(t string) (typ Type, err error) {
return typ, err
}
// parse the type and size of the abi-type.
parsedType := typeRegex.FindAllStringSubmatch(t, -1)[0]
matches := typeRegex.FindAllStringSubmatch(t, -1)
if len(matches) == 0 {
return Type{}, fmt.Errorf("invalid type '%v'", t)
}
parsedType := matches[0]
// varSize is the size of the variable
var varSize int
if len(parsedType[3]) > 0 {

28
accounts/abi/unpack.go
View File

@ -25,8 +25,17 @@ import (
"github.com/ethereum/go-ethereum/common"
)
var (
maxUint256 = big.NewInt(0).Add(
big.NewInt(0).Exp(big.NewInt(2), big.NewInt(256), nil),
big.NewInt(-1))
maxInt256 = big.NewInt(0).Add(
big.NewInt(0).Exp(big.NewInt(2), big.NewInt(255), nil),
big.NewInt(-1))
)
// reads the integer based on its kind
func readInteger(kind reflect.Kind, b []byte) interface{} {
func readInteger(typ byte, kind reflect.Kind, b []byte) interface{} {
switch kind {
case reflect.Uint8:
return b[len(b)-1]
@ -45,7 +54,20 @@ func readInteger(kind reflect.Kind, b []byte) interface{} {
case reflect.Int64:
return int64(binary.BigEndian.Uint64(b[len(b)-8:]))
default:
return new(big.Int).SetBytes(b)
// the only case lefts for integer is int256/uint256.
// big.SetBytes can't tell if a number is negative, positive on itself.
// On EVM, if the returned number > max int256, it is negative.
ret := new(big.Int).SetBytes(b)
if typ == UintTy {
return ret
}
if ret.Cmp(maxInt256) > 0 {
ret.Add(maxUint256, big.NewInt(0).Neg(ret))
ret.Add(ret, big.NewInt(1))
ret.Neg(ret)
}
return ret
}
}
@ -179,7 +201,7 @@ func toGoType(index int, t Type, output []byte) (interface{}, error) {
case StringTy: // variable arrays are written at the end of the return bytes
return string(output[begin : begin+end]), nil
case IntTy, UintTy:
return readInteger(t.Kind, returnOutput), nil
return readInteger(t.T, t.Kind, returnOutput), nil
case BoolTy:
return readBool(returnOutput)
case AddressTy:

5
accounts/abi/unpack_test.go
View File

@ -117,6 +117,11 @@ var unpackTests = []unpackTest{
enc: "0000000000000000000000000000000000000000000000000000000000000001",
want: big.NewInt(1),
},
{
def: `[{"type": "int256"}]`,
enc: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff",
want: big.NewInt(-1),
},
{
def: `[{"type": "address"}]`,
enc: "0000000000000000000000000100000000000000000000000000000000000000",

2
accounts/accounts.go
View File

@ -106,7 +106,7 @@ type Wallet interface {
// or optionally with the aid of any location metadata from the embedded URL field.
//
// If the wallet requires additional authentication to sign the request (e.g.
// a password to decrypt the account, or a PIN code o verify the transaction),
// a password to decrypt the account, or a PIN code to verify the transaction),
// an AuthNeededError instance will be returned, containing infos for the user
// about which fields or actions are needed. The user may retry by providing
// the needed details via SignTxWithPassphrase, or by other means (e.g. unlock

12
accounts/keystore/account_cache.go
View File

@ -27,10 +27,10 @@ import (
"sync"
"time"
mapset "github.com/deckarep/golang-set"
"github.com/ethereum/go-ethereum/accounts"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/log"
"gopkg.in/fatih/set.v0"
)
// Minimum amount of time between cache reloads. This limit applies if the platform does
@ -79,7 +79,7 @@ func newAccountCache(keydir string) (*accountCache, chan struct{}) {
keydir: keydir,
byAddr: make(map[common.Address][]accounts.Account),
notify: make(chan struct{}, 1),
fileC: fileCache{all: set.NewNonTS()},
fileC: fileCache{all: mapset.NewThreadUnsafeSet()},
}
ac.watcher = newWatcher(ac)
return ac, ac.notify
@ -237,7 +237,7 @@ func (ac *accountCache) scanAccounts() error {
log.Debug("Failed to reload keystore contents", "err", err)
return err
}
if creates.Size() == 0 && deletes.Size() == 0 && updates.Size() == 0 {
if creates.Cardinality() == 0 && deletes.Cardinality() == 0 && updates.Cardinality() == 0 {
return nil
}
// Create a helper method to scan the contents of the key files
@ -272,15 +272,15 @@ func (ac *accountCache) scanAccounts() error {
// Process all the file diffs
start := time.Now()
for _, p := range creates.List() {
for _, p := range creates.ToSlice() {
if a := readAccount(p.(string)); a != nil {
ac.add(*a)
}
}
for _, p := range deletes.List() {
for _, p := range deletes.ToSlice() {
ac.deleteByFile(p.(string))
}
for _, p := range updates.List() {
for _, p := range updates.ToSlice() {
path := p.(string)
ac.deleteByFile(path)
if a := readAccount(path); a != nil {

26
accounts/keystore/file_cache.go
View File

@ -24,20 +24,20 @@ import (
"sync"
"time"
mapset "github.com/deckarep/golang-set"
"github.com/ethereum/go-ethereum/log"
set "gopkg.in/fatih/set.v0"
)
// fileCache is a cache of files seen during scan of keystore.
type fileCache struct {
all *set.SetNonTS // Set of all files from the keystore folder
lastMod time.Time // Last time instance when a file was modified
all mapset.Set // Set of all files from the keystore folder
lastMod time.Time // Last time instance when a file was modified
mu sync.RWMutex
}
// scan performs a new scan on the given directory, compares against the already
// cached filenames, and returns file sets: creates, deletes, updates.
func (fc *fileCache) scan(keyDir string) (set.Interface, set.Interface, set.Interface, error) {
func (fc *fileCache) scan(keyDir string) (mapset.Set, mapset.Set, mapset.Set, error) {
t0 := time.Now()
// List all the failes from the keystore folder
@ -51,14 +51,14 @@ func (fc *fileCache) scan(keyDir string) (set.Interface, set.Interface, set.Inte
defer fc.mu.Unlock()
// Iterate all the files and gather their metadata
all := set.NewNonTS()
mods := set.NewNonTS()
all := mapset.NewThreadUnsafeSet()
mods := mapset.NewThreadUnsafeSet()
var newLastMod time.Time
for _, fi := range files {
// Skip any non-key files from the folder
path := filepath.Join(keyDir, fi.Name())
if skipKeyFile(fi) {
// Skip any non-key files from the folder
if nonKeyFile(fi) {
log.Trace("Ignoring file on account scan", "path", path)
continue
}
@ -76,9 +76,9 @@ func (fc *fileCache) scan(keyDir string) (set.Interface, set.Interface, set.Inte
t2 := time.Now()
// Update the tracked files and return the three sets
deletes := set.Difference(fc.all, all) // Deletes = previous - current
creates := set.Difference(all, fc.all) // Creates = current - previous
updates := set.Difference(mods, creates) // Updates = modified - creates
deletes := fc.all.Difference(all) // Deletes = previous - current
creates := all.Difference(fc.all) // Creates = current - previous
updates := mods.Difference(creates) // Updates = modified - creates
fc.all, fc.lastMod = all, newLastMod
t3 := time.Now()
@ -88,8 +88,8 @@ func (fc *fileCache) scan(keyDir string) (set.Interface, set.Interface, set.Inte
return creates, deletes, updates, nil
}
// skipKeyFile ignores editor backups, hidden files and folders/symlinks.
func skipKeyFile(fi os.FileInfo) bool {
// nonKeyFile ignores editor backups, hidden files and folders/symlinks.
func nonKeyFile(fi os.FileInfo) bool {
// Skip editor backups and UNIX-style hidden files.
if strings.HasSuffix(fi.Name(), "~") || strings.HasPrefix(fi.Name(), ".") {
return true

18
accounts/keystore/key.go
View File

@ -179,26 +179,34 @@ func storeNewKey(ks keyStore, rand io.Reader, auth string) (*Key, accounts.Accou
return key, a, err
}
func writeKeyFile(file string, content []byte) error {
func writeTemporaryKeyFile(file string, content []byte) (string, error) {
// Create the keystore directory with appropriate permissions
// in case it is not present yet.
const dirPerm = 0700
if err := os.MkdirAll(filepath.Dir(file), dirPerm); err != nil {
return err
return "", err
}
// Atomic write: create a temporary hidden file first
// then move it into place. TempFile assigns mode 0600.
f, err := ioutil.TempFile(filepath.Dir(file), "."+filepath.Base(file)+".tmp")
if err != nil {
return err
return "", err
}
if _, err := f.Write(content); err != nil {
f.Close()
os.Remove(f.Name())
return err
return "", err
}
f.Close()
return os.Rename(f.Name(), file)
return f.Name(), nil
}
func writeKeyFile(file string, content []byte) error {
name, err := writeTemporaryKeyFile(file, content)
if err != nil {
return err
}
return os.Rename(name, file)
}
// keyFileName implements the naming convention for keyfiles:

2
accounts/keystore/keystore.go
View File

@ -78,7 +78,7 @@ type unlocked struct {
// NewKeyStore creates a keystore for the given directory.
func NewKeyStore(keydir string, scryptN, scryptP int) *KeyStore {
keydir, _ = filepath.Abs(keydir)
ks := &KeyStore{storage: &keyStorePassphrase{keydir, scryptN, scryptP}}
ks := &KeyStore{storage: &keyStorePassphrase{keydir, scryptN, scryptP, false}}
ks.init(keydir)
return ks
}

42
accounts/keystore/keystore_passphrase.go
View File

@ -28,18 +28,19 @@ package keystore
import (
"bytes"
"crypto/aes"
crand "crypto/rand"
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/math"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/crypto/randentropy"
"github.com/pborman/uuid"
"golang.org/x/crypto/pbkdf2"
"golang.org/x/crypto/scrypt"
@ -72,6 +73,10 @@ type keyStorePassphrase struct {
keysDirPath string
scryptN int
scryptP int
// skipKeyFileVerification disables the security-feature which does
// reads and decrypts any newly created keyfiles. This should be 'false' in all
// cases except tests -- setting this to 'true' is not recommended.
skipKeyFileVerification bool
}
func (ks keyStorePassphrase) GetKey(addr common.Address, filename, auth string) (*Key, error) {
@ -93,7 +98,7 @@ func (ks keyStorePassphrase) GetKey(addr common.Address, filename, auth string)
// StoreKey generates a key, encrypts with 'auth' and stores in the given directory
func StoreKey(dir, auth string, scryptN, scryptP int) (common.Address, error) {
_, a, err := storeNewKey(&keyStorePassphrase{dir, scryptN, scryptP}, crand.Reader, auth)
_, a, err := storeNewKey(&keyStorePassphrase{dir, scryptN, scryptP, false}, rand.Reader, auth)
return a.Address, err
}
@ -102,7 +107,25 @@ func (ks keyStorePassphrase) StoreKey(filename string, key *Key, auth string) er
if err != nil {
return err
}
return writeKeyFile(filename, keyjson)
// Write into temporary file
tmpName, err := writeTemporaryKeyFile(filename, keyjson)
if err != nil {
return err
}
if !ks.skipKeyFileVerification {
// Verify that we can decrypt the file with the given password.
_, err = ks.GetKey(key.Address, tmpName, auth)
if err != nil {
msg := "An error was encountered when saving and verifying the keystore file. \n" +
"This indicates that the keystore is corrupted. \n" +
"The corrupted file is stored at \n%v\n" +
"Please file a ticket at:\n\n" +
"https://github.com/ethereum/go-ethereum/issues." +
"The error was : %s"
return fmt.Errorf(msg, tmpName, err)
}
}
return os.Rename(tmpName, filename)
}
func (ks keyStorePassphrase) JoinPath(filename string) string {
@ -116,7 +139,11 @@ func (ks keyStorePassphrase) JoinPath(filename string) string {
// blob that can be decrypted later on.
func EncryptKey(key *Key, auth string, scryptN, scryptP int) ([]byte, error) {
authArray := []byte(auth)
salt := randentropy.GetEntropyCSPRNG(32)
salt := make([]byte, 32)
if _, err := io.ReadFull(rand.Reader, salt); err != nil {
panic("reading from crypto/rand failed: " + err.Error())
}
derivedKey, err := scrypt.Key(authArray, salt, scryptN, scryptR, scryptP, scryptDKLen)
if err != nil {
return nil, err
@ -124,7 +151,10 @@ func EncryptKey(key *Key, auth string, scryptN, scryptP int) ([]byte, error) {
encryptKey := derivedKey[:16]
keyBytes := math.PaddedBigBytes(key.PrivateKey.D, 32)
iv := randentropy.GetEntropyCSPRNG(aes.BlockSize) // 16
iv := make([]byte, aes.BlockSize) // 16
if _, err := io.ReadFull(rand.Reader, iv); err != nil {
panic("reading from crypto/rand failed: " + err.Error())
}
cipherText, err := aesCTRXOR(encryptKey, keyBytes, iv)
if err != nil {
return nil, err

4
accounts/keystore/keystore_plain_test.go
View File

@ -37,7 +37,7 @@ func tmpKeyStoreIface(t *testing.T, encrypted bool) (dir string, ks keyStore) {
t.Fatal(err)
}
if encrypted {
ks = &keyStorePassphrase{d, veryLightScryptN, veryLightScryptP}
ks = &keyStorePassphrase{d, veryLightScryptN, veryLightScryptP, true}
} else {
ks = &keyStorePlain{d}
}
@ -191,7 +191,7 @@ func TestV1_1(t *testing.T) {
func TestV1_2(t *testing.T) {
t.Parallel()
ks := &keyStorePassphrase{"testdata/v1", LightScryptN, LightScryptP}
ks := &keyStorePassphrase{"testdata/v1", LightScryptN, LightScryptP, true}
addr := common.HexToAddress("cb61d5a9c4896fb9658090b597ef0e7be6f7b67e")
file := "testdata/v1/cb61d5a9c4896fb9658090b597ef0e7be6f7b67e/cb61d5a9c4896fb9658090b597ef0e7be6f7b67e"
k, err := ks.GetKey(addr, file, "g")

6
accounts/url.go
View File

@ -76,12 +76,12 @@ func (u URL) MarshalJSON() ([]byte, error) {
// UnmarshalJSON parses url.
func (u *URL) UnmarshalJSON(input []byte) error {
var textUrl string
err := json.Unmarshal(input, &textUrl)
var textURL string
err := json.Unmarshal(input, &textURL)
if err != nil {
return err
}
url, err := parseURL(textUrl)
url, err := parseURL(textURL)
if err != nil {
return err
}

96
accounts/url_test.go Normal file
View File

@ -0,0 +1,96 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package accounts
import (
"testing"
)
func TestURLParsing(t *testing.T) {
url, err := parseURL("https://ethereum.org")
if err != nil {
t.Errorf("unexpected error: %v", err)
}
if url.Scheme != "https" {
t.Errorf("expected: %v, got: %v", "https", url.Scheme)
}
if url.Path != "ethereum.org" {
t.Errorf("expected: %v, got: %v", "ethereum.org", url.Path)
}
_, err = parseURL("ethereum.org")
if err == nil {
t.Error("expected err, got: nil")
}
}
func TestURLString(t *testing.T) {
url := URL{Scheme: "https", Path: "ethereum.org"}
if url.String() != "https://ethereum.org" {
t.Errorf("expected: %v, got: %v", "https://ethereum.org", url.String())
}
url = URL{Scheme: "", Path: "ethereum.org"}
if url.String() != "ethereum.org" {
t.Errorf("expected: %v, got: %v", "ethereum.org", url.String())
}
}
func TestURLMarshalJSON(t *testing.T) {
url := URL{Scheme: "https", Path: "ethereum.org"}
json, err := url.MarshalJSON()
if err != nil {
t.Errorf("unexpcted error: %v", err)
}
if string(json) != "\"https://ethereum.org\"" {
t.Errorf("expected: %v, got: %v", "\"https://ethereum.org\"", string(json))
}
}
func TestURLUnmarshalJSON(t *testing.T) {
url := &URL{}
err := url.UnmarshalJSON([]byte("\"https://ethereum.org\""))
if err != nil {
t.Errorf("unexpcted error: %v", err)
}
if url.Scheme != "https" {
t.Errorf("expected: %v, got: %v", "https", url.Scheme)
}
if url.Path != "ethereum.org" {
t.Errorf("expected: %v, got: %v", "https", url.Path)
}
}
func TestURLComparison(t *testing.T) {
tests := []struct {
urlA URL
urlB URL
expect int
}{
{URL{"https", "ethereum.org"}, URL{"https", "ethereum.org"}, 0},
{URL{"http", "ethereum.org"}, URL{"https", "ethereum.org"}, -1},
{URL{"https", "ethereum.org/a"}, URL{"https", "ethereum.org"}, 1},
{URL{"https", "abc.org"}, URL{"https", "ethereum.org"}, -1},
}
for i, tt := range tests {
result := tt.urlA.Cmp(tt.urlB)
if result != tt.expect {
t.Errorf("test %d: cmp mismatch: expected: %d, got: %d", i, tt.expect, result)
}
}
}

4
appveyor.yml
View File

@ -23,8 +23,8 @@ environment:
install:
- git submodule update --init
- rmdir C:\go /s /q
- appveyor DownloadFile https://storage.googleapis.com/golang/go1.10.3.windows-%GETH_ARCH%.zip
- 7z x go1.10.3.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
- appveyor DownloadFile https://storage.googleapis.com/golang/go1.11.windows-%GETH_ARCH%.zip
- 7z x go1.11.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
- go version
- gcc --version

204
build/ci.go
View File

@ -26,7 +26,7 @@ Available commands are:
install [ -arch architecture ] [ -cc compiler ] [ packages... ] -- builds packages and executables
test [ -coverage ] [ packages... ] -- runs the tests
lint -- runs certain pre-selected linters
archive [ -arch architecture ] [ -type zip|tar ] [ -signer key-envvar ] [ -upload dest ] -- archives build artefacts
archive [ -arch architecture ] [ -type zip|tar ] [ -signer key-envvar ] [ -upload dest ] -- archives build artifacts
importkeys -- imports signing keys from env
debsrc [ -signer key-id ] [ -upload dest ] -- creates a debian source package
nsis -- creates a Windows NSIS installer
@ -59,6 +59,8 @@ import (
"time"
"github.com/ethereum/go-ethereum/internal/build"
"github.com/ethereum/go-ethereum/params"
sv "github.com/ethereum/go-ethereum/swarm/version"
)
var (
@ -77,52 +79,84 @@ var (
executablePath("geth"),
executablePath("puppeth"),
executablePath("rlpdump"),
executablePath("swarm"),
executablePath("wnode"),
}
// Files that end up in the swarm*.zip archive.
swarmArchiveFiles = []string{
"COPYING",
executablePath("swarm"),
}
// A debian package is created for all executables listed here.
debExecutables = []debExecutable{
{
Name: "abigen",
BinaryName: "abigen",
Description: "Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages.",
},
{
Name: "bootnode",
BinaryName: "bootnode",
Description: "Ethereum bootnode.",
},
{
Name: "evm",
BinaryName: "evm",
Description: "Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode.",
},
{
Name: "geth",
BinaryName: "geth",
Description: "Ethereum CLI client.",
},
{
Name: "puppeth",
BinaryName: "puppeth",
Description: "Ethereum private network manager.",
},
{
Name: "rlpdump",
BinaryName: "rlpdump",
Description: "Developer utility tool that prints RLP structures.",
},
{
Name: "swarm",
Description: "Ethereum Swarm daemon and tools",
},
{
Name: "wnode",
BinaryName: "wnode",
Description: "Ethereum Whisper diagnostic tool",
},
}
// A debian package is created for all executables listed here.
debSwarmExecutables = []debExecutable{
{
BinaryName: "swarm",
PackageName: "ethereum-swarm",
Description: "Ethereum Swarm daemon and tools",
},
}
debEthereum = debPackage{
Name: "ethereum",
Version: params.Version,
Executables: debExecutables,
}
debSwarm = debPackage{
Name: "ethereum-swarm",
Version: sv.Version,
Executables: debSwarmExecutables,
}
// Debian meta packages to build and push to Ubuntu PPA
debPackages = []debPackage{
debSwarm,
debEthereum,
}
// Packages to be cross-compiled by the xgo command
allCrossCompiledArchiveFiles = append(allToolsArchiveFiles, swarmArchiveFiles...)
// Distros for which packages are created.
// Note: vivid is unsupported because there is no golang-1.6 package for it.
// Note: wily is unsupported because it was officially deprecated on lanchpad.
// Note: yakkety is unsupported because it was officially deprecated on lanchpad.
// Note: zesty is unsupported because it was officially deprecated on lanchpad.
debDistros = []string{"trusty", "xenial", "artful", "bionic"}
// Note: artful is unsupported because it was officially deprecated on lanchpad.
debDistros = []string{"trusty", "xenial", "bionic", "cosmic"}
)
var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin"))
@ -350,7 +384,6 @@ func doLint(cmdline []string) {
}
// Release Packaging
func doArchive(cmdline []string) {
var (
arch = flag.String("arch", runtime.GOARCH, "Architecture cross packaging")
@ -370,10 +403,14 @@ func doArchive(cmdline []string) {
}
var (
env = build.Env()
base = archiveBasename(*arch, env)
geth = "geth-" + base + ext
alltools = "geth-alltools-" + base + ext
env = build.Env()
basegeth = archiveBasename(*arch, params.ArchiveVersion(env.Commit))
geth = "geth-" + basegeth + ext
alltools = "geth-alltools-" + basegeth + ext
baseswarm = archiveBasename(*arch, sv.ArchiveVersion(env.Commit))
swarm = "swarm-" + baseswarm + ext
)
maybeSkipArchive(env)
if err := build.WriteArchive(geth, gethArchiveFiles); err != nil {
@ -382,14 +419,17 @@ func doArchive(cmdline []string) {
if err := build.WriteArchive(alltools, allToolsArchiveFiles); err != nil {
log.Fatal(err)
}
for _, archive := range []string{geth, alltools} {
if err := build.WriteArchive(swarm, swarmArchiveFiles); err != nil {
log.Fatal(err)
}
for _, archive := range []string{geth, alltools, swarm} {
if err := archiveUpload(archive, *upload, *signer); err != nil {
log.Fatal(err)
}
}
}
func archiveBasename(arch string, env build.Environment) string {
func archiveBasename(arch string, archiveVersion string) string {
platform := runtime.GOOS + "-" + arch
if arch == "arm" {
platform += os.Getenv("GOARM")
@ -400,18 +440,7 @@ func archiveBasename(arch string, env build.Environment) string {
if arch == "ios" {
platform = "ios-all"
}
return platform + "-" + archiveVersion(env)
}
func archiveVersion(env build.Environment) string {
version := build.VERSION()
if isUnstableBuild(env) {
version += "-unstable"
}
if env.Commit != "" {
version += "-" + env.Commit[:8]
}
return version
return platform + "-" + archiveVersion
}
func archiveUpload(archive string, blobstore string, signer string) error {
@ -461,7 +490,6 @@ func maybeSkipArchive(env build.Environment) {
}
// Debian Packaging
func doDebianSource(cmdline []string) {
var (
signer = flag.String("signer", "", `Signing key name, also used as package author`)
@ -485,21 +513,23 @@ func doDebianSource(cmdline []string) {
build.MustRun(gpg)
}
// Create the packages.
for _, distro := range debDistros {
meta := newDebMetadata(distro, *signer, env, now)
pkgdir := stageDebianSource(*workdir, meta)
debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc")
debuild.Dir = pkgdir
build.MustRun(debuild)
// Create Debian packages and upload them
for _, pkg := range debPackages {
for _, distro := range debDistros {
meta := newDebMetadata(distro, *signer, env, now, pkg.Name, pkg.Version, pkg.Executables)
pkgdir := stageDebianSource(*workdir, meta)
debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc")
debuild.Dir = pkgdir
build.MustRun(debuild)
changes := fmt.Sprintf("%s_%s_source.changes", meta.Name(), meta.VersionString())
changes = filepath.Join(*workdir, changes)
if *signer != "" {
build.MustRunCommand("debsign", changes)
}
if *upload != "" {
build.MustRunCommand("dput", *upload, changes)
changes := fmt.Sprintf("%s_%s_source.changes", meta.Name(), meta.VersionString())
changes = filepath.Join(*workdir, changes)
if *signer != "" {
build.MustRunCommand("debsign", changes)
}
if *upload != "" {
build.MustRunCommand("dput", *upload, changes)
}
}
}
}
@ -524,9 +554,17 @@ func isUnstableBuild(env build.Environment) bool {
return true
}
type debPackage struct {
Name string // the name of the Debian package to produce, e.g. "ethereum", or "ethereum-swarm"
Version string // the clean version of the debPackage, e.g. 1.8.12 or 0.3.0, without any metadata
Executables []debExecutable // executables to be included in the package
}
type debMetadata struct {
Env build.Environment
PackageName string
// go-ethereum version being built. Note that this
// is not the debian package version. The package version
// is constructed by VersionString.
@ -538,21 +576,33 @@ type debMetadata struct {
}
type debExecutable struct {
Name, Description string
PackageName string
BinaryName string
Description string
}
func newDebMetadata(distro, author string, env build.Environment, t time.Time) debMetadata {
// Package returns the name of the package if present, or
// fallbacks to BinaryName
func (d debExecutable) Package() string {
if d.PackageName != "" {
return d.PackageName
}
return d.BinaryName
}
func newDebMetadata(distro, author string, env build.Environment, t time.Time, name string, version string, exes []debExecutable) debMetadata {
if author == "" {
// No signing key, use default author.
author = "Ethereum Builds <fjl@ethereum.org>"
}
return debMetadata{
PackageName: name,
Env: env,
Author: author,
Distro: distro,
Version: build.VERSION(),
Version: version,
Time: t.Format(time.RFC1123Z),
Executables: debExecutables,
Executables: exes,
}
}
@ -560,9 +610,9 @@ func newDebMetadata(distro, author string, env build.Environment, t time.Time) d
// on all executable packages.
func (meta debMetadata) Name() string {
if isUnstableBuild(meta.Env) {
return "ethereum-unstable"
return meta.PackageName + "-unstable"
}
return "ethereum"
return meta.PackageName
}
// VersionString returns the debian version of the packages.
@ -589,9 +639,9 @@ func (meta debMetadata) ExeList() string {
// ExeName returns the package name of an executable package.
func (meta debMetadata) ExeName(exe debExecutable) string {
if isUnstableBuild(meta.Env) {
return exe.Name + "-unstable"
return exe.Package() + "-unstable"
}
return exe.Name
return exe.Package()
}
// ExeConflicts returns the content of the Conflicts field
@ -606,7 +656,7 @@ func (meta debMetadata) ExeConflicts(exe debExecutable) string {
// be preferred and the conflicting files should be handled via
// alternates. We might do this eventually but using a conflict is
// easier now.
return "ethereum, " + exe.Name
return "ethereum, " + exe.Package()
}
return ""
}
@ -623,24 +673,23 @@ func stageDebianSource(tmpdir string, meta debMetadata) (pkgdir string) {
// Put the debian build files in place.
debian := filepath.Join(pkgdir, "debian")
build.Render("build/deb.rules", filepath.Join(debian, "rules"), 0755, meta)
build.Render("build/deb.changelog", filepath.Join(debian, "changelog"), 0644, meta)
build.Render("build/deb.control", filepath.Join(debian, "control"), 0644, meta)
build.Render("build/deb.copyright", filepath.Join(debian, "copyright"), 0644, meta)
build.Render("build/deb/"+meta.PackageName+"/deb.rules", filepath.Join(debian, "rules"), 0755, meta)
build.Render("build/deb/"+meta.PackageName+"/deb.changelog", filepath.Join(debian, "changelog"), 0644, meta)
build.Render("build/deb/"+meta.PackageName+"/deb.control", filepath.Join(debian, "control"), 0644, meta)
build.Render("build/deb/"+meta.PackageName+"/deb.copyright", filepath.Join(debian, "copyright"), 0644, meta)
build.RenderString("8\n", filepath.Join(debian, "compat"), 0644, meta)
build.RenderString("3.0 (native)\n", filepath.Join(debian, "source/format"), 0644, meta)
for _, exe := range meta.Executables {
install := filepath.Join(debian, meta.ExeName(exe)+".install")
docs := filepath.Join(debian, meta.ExeName(exe)+".docs")
build.Render("build/deb.install", install, 0644, exe)
build.Render("build/deb.docs", docs, 0644, exe)
build.Render("build/deb/"+meta.PackageName+"/deb.install", install, 0644, exe)
build.Render("build/deb/"+meta.PackageName+"/deb.docs", docs, 0644, exe)
}
return pkgdir
}
// Windows installer
func doWindowsInstaller(cmdline []string) {
// Parse the flags and make skip installer generation on PRs
var (
@ -690,11 +739,11 @@ func doWindowsInstaller(cmdline []string) {
// Build the installer. This assumes that all the needed files have been previously
// built (don't mix building and packaging to keep cross compilation complexity to a
// minimum).
version := strings.Split(build.VERSION(), ".")
version := strings.Split(params.Version, ".")
if env.Commit != "" {
version[2] += "-" + env.Commit[:8]
}
installer, _ := filepath.Abs("geth-" + archiveBasename(*arch, env) + ".exe")
installer, _ := filepath.Abs("geth-" + archiveBasename(*arch, params.ArchiveVersion(env.Commit)) + ".exe")
build.MustRunCommand("makensis.exe",
"/DOUTPUTFILE="+installer,
"/DMAJORVERSION="+version[0],
@ -746,7 +795,7 @@ func doAndroidArchive(cmdline []string) {
maybeSkipArchive(env)
// Sign and upload the archive to Azure
archive := "geth-" + archiveBasename("android", env) + ".aar"
archive := "geth-" + archiveBasename("android", params.ArchiveVersion(env.Commit)) + ".aar"
os.Rename("geth.aar", archive)
if err := archiveUpload(archive, *upload, *signer); err != nil {
@ -831,7 +880,7 @@ func newMavenMetadata(env build.Environment) mavenMetadata {
}
}
// Render the version and package strings
version := build.VERSION()
version := params.Version
if isUnstableBuild(env) {
version += "-SNAPSHOT"
}
@ -866,7 +915,7 @@ func doXCodeFramework(cmdline []string) {
build.MustRun(bind)
return
}
archive := "geth-" + archiveBasename("ios", env)
archive := "geth-" + archiveBasename("ios", params.ArchiveVersion(env.Commit))
if err := os.Mkdir(archive, os.ModePerm); err != nil {
log.Fatal(err)
}
@ -922,7 +971,7 @@ func newPodMetadata(env build.Environment, archive string) podMetadata {
}
}
}
version := build.VERSION()
version := params.Version
if isUnstableBuild(env) {
version += "-unstable." + env.Buildnum
}
@ -952,7 +1001,7 @@ func doXgo(cmdline []string) {
if *alltools {
args = append(args, []string{"--dest", GOBIN}...)
for _, res := range allToolsArchiveFiles {
for _, res := range allCrossCompiledArchiveFiles {
if strings.HasPrefix(res, GOBIN) {
// Binary tool found, cross build it explicitly
args = append(args, "./"+filepath.Join("cmd", filepath.Base(res)))
@ -1018,23 +1067,14 @@ func doPurge(cmdline []string) {
}
for i := 0; i < len(blobs); i++ {
for j := i + 1; j < len(blobs); j++ {
iTime, err := time.Parse(time.RFC1123, blobs[i].Properties.LastModified)
if err != nil {
log.Fatal(err)
}
jTime, err := time.Parse(time.RFC1123, blobs[j].Properties.LastModified)
if err != nil {
log.Fatal(err)
}
if iTime.After(jTime) {
if blobs[i].Properties.LastModified.After(blobs[j].Properties.LastModified) {
blobs[i], blobs[j] = blobs[j], blobs[i]
}
}
}
// Filter out all archives more recent that the given threshold
for i, blob := range blobs {
timestamp, _ := time.Parse(time.RFC1123, blob.Properties.LastModified)
if time.Since(timestamp) < time.Duration(*limit)*24*time.Hour {
if time.Since(blob.Properties.LastModified) < time.Duration(*limit)*24*time.Hour {
blobs = blobs[:i]
break
}

19
build/clean_go_build_cache.sh Executable file
View File

@ -0,0 +1,19 @@
#!/bin/sh
# Cleaning the Go cache only makes sense if we actually have Go installed... or
# if Go is actually callable. This does not hold true during deb packaging, so
# we need an explicit check to avoid build failures.
if ! command -v go > /dev/null; then
exit
fi
version_gt() {
test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1"
}
golang_version=$(go version |cut -d' ' -f3 |sed 's/go//')
# Clean go build cache when go version is greater than or equal to 1.10
if !(version_gt 1.10 $golang_version); then
go clean -cache
fi

1
build/deb.install
View File

@ -1 +0,0 @@
build/bin/{{.Name}} usr/bin

0
build/deb.changelog → build/deb/ethereum-swarm/deb.changelog
View File

19
build/deb/ethereum-swarm/deb.control Normal file
View File

@ -0,0 +1,19 @@
Source: {{.Name}}
Section: science
Priority: extra
Maintainer: {{.Author}}
Build-Depends: debhelper (>= 8.0.0), golang-1.10
Standards-Version: 3.9.5
Homepage: https://ethereum.org
Vcs-Git: git://github.com/ethereum/go-ethereum.git
Vcs-Browser: https://github.com/ethereum/go-ethereum
{{range .Executables}}
Package: {{$.ExeName .}}
Conflicts: {{$.ExeConflicts .}}
Architecture: any
Depends: ${shlibs:Depends}, ${misc:Depends}
Built-Using: ${misc:Built-Using}
Description: {{.Description}}
{{.Description}}
{{end}}

2
build/deb.copyright → build/deb/ethereum-swarm/deb.copyright
View File

@ -1,4 +1,4 @@
Copyright 2016 The go-ethereum Authors
Copyright 2018 The go-ethereum Authors
go-ethereum is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by

0
build/deb.docs → build/deb/ethereum-swarm/deb.docs
View File

1
build/deb/ethereum-swarm/deb.install Normal file
View File

@ -0,0 +1 @@
build/bin/{{.BinaryName}} usr/bin

0
build/deb.rules → build/deb/ethereum-swarm/deb.rules
View File

5
build/deb/ethereum/deb.changelog Normal file
View File

@ -0,0 +1,5 @@
{{.Name}} ({{.VersionString}}) {{.Distro}}; urgency=low
* git build of {{.Env.Commit}}
-- {{.Author}} {{.Time}}

4
build/deb.control → build/deb/ethereum/deb.control
View File

@ -11,8 +11,8 @@ Vcs-Browser: https://github.com/ethereum/go-ethereum
Package: {{.Name}}
Architecture: any
Depends: ${misc:Depends}, {{.ExeList}}
Description: Meta-package to install geth and other tools
Meta-package to install geth and other tools
Description: Meta-package to install geth, swarm, and other tools
Meta-package to install geth, swarm and other tools
{{range .Executables}}
Package: {{$.ExeName .}}

14
build/deb/ethereum/deb.copyright Normal file
View File

@ -0,0 +1,14 @@
Copyright 2018 The go-ethereum Authors
go-ethereum is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
go-ethereum is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.

1
build/deb/ethereum/deb.docs Normal file
View File

@ -0,0 +1 @@
AUTHORS

1
build/deb/ethereum/deb.install Normal file
View File

@ -0,0 +1 @@
build/bin/{{.BinaryName}} usr/bin

13
build/deb/ethereum/deb.rules Normal file
View File

@ -0,0 +1,13 @@
#!/usr/bin/make -f
# -*- makefile -*-
# Uncomment this to turn on verbose mode.
#export DH_VERBOSE=1
override_dh_auto_build:
build/env.sh /usr/lib/go-1.10/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}}
override_dh_auto_test:
%:
dh $@

8
cmd/clef/main.go
View File

@ -225,7 +225,7 @@ func initializeSecrets(c *cli.Context) error {
if _, err := os.Stat(location); err == nil {
return fmt.Errorf("file %v already exists, will not overwrite", location)
}
err = ioutil.WriteFile(location, masterSeed, 0700)
err = ioutil.WriteFile(location, masterSeed, 0400)
if err != nil {
return err
}
@ -415,7 +415,7 @@ func signer(c *cli.Context) error {
// start http server
httpEndpoint := fmt.Sprintf("%s:%d", c.String(utils.RPCListenAddrFlag.Name), c.Int(rpcPortFlag.Name))
listener, _, err := rpc.StartHTTPEndpoint(httpEndpoint, rpcAPI, []string{"account"}, cors, vhosts)
listener, _, err := rpc.StartHTTPEndpoint(httpEndpoint, rpcAPI, []string{"account"}, cors, vhosts, rpc.DefaultHTTPTimeouts)
if err != nil {
utils.Fatalf("Could not start RPC api: %v", err)
}
@ -540,14 +540,14 @@ func readMasterKey(ctx *cli.Context) ([]byte, error) {
// checkFile is a convenience function to check if a file
// * exists
// * is mode 0600
// * is mode 0400
func checkFile(filename string) error {
info, err := os.Stat(filename)
if err != nil {
return fmt.Errorf("failed stat on %s: %v", filename, err)
}
// Check the unix permission bits
if info.Mode().Perm()&077 != 0 {
if info.Mode().Perm()&0377 != 0 {
return fmt.Errorf("file (%v) has insecure file permissions (%v)", filename, info.Mode().String())
}
return nil

18
cmd/ethkey/README.md
View File

@ -21,21 +21,33 @@ Private key information can be printed by using the `--private` flag;
make sure to use this feature with great caution!
### `ethkey sign <keyfile> <message/file>`
### `ethkey signmessage <keyfile> <message/file>`
Sign the message with a keyfile.
It is possible to refer to a file containing the message.
To sign a message contained in a file, use the `--msgfile` flag.
### `ethkey verify <address> <signature> <message/file>`
### `ethkey verifymessage <address> <signature> <message/file>`
Verify the signature of the message.
It is possible to refer to a file containing the message.
To sign a message contained in a file, use the --msgfile flag.
### `ethkey changepassphrase <keyfile>`
Change the passphrase of a keyfile.
use the `--newpasswordfile` to point to the new password file.
## Passphrases
For every command that uses a keyfile, you will be prompted to provide the
passphrase for decrypting the keyfile. To avoid this message, it is possible
to pass the passphrase by using the `--passphrase` flag pointing to a file that
to pass the passphrase by using the `--passwordfile` flag pointing to a file that
contains the passphrase.
## JSON
In case you need to output the result in a JSON format, you shall by using the `--json` flag.

2
cmd/evm/disasm.go
View File

@ -44,7 +44,7 @@ func disasmCmd(ctx *cli.Context) error {
return err
}
code := strings.TrimSpace(string(in[:]))
code := strings.TrimSpace(string(in))
fmt.Printf("%v\n", code)
return asm.PrintDisassembled(code)
}

25
cmd/evm/runner.go
View File

@ -80,13 +80,13 @@ func runCmd(ctx *cli.Context) error {
}
var (
tracer vm.Tracer
debugLogger *vm.StructLogger
statedb *state.StateDB
chainConfig *params.ChainConfig
sender = common.BytesToAddress([]byte("sender"))
receiver = common.BytesToAddress([]byte("receiver"))
blockNumber uint64
tracer vm.Tracer
debugLogger *vm.StructLogger
statedb *state.StateDB
chainConfig *params.ChainConfig
sender = common.BytesToAddress([]byte("sender"))
receiver = common.BytesToAddress([]byte("receiver"))
genesisConfig *core.Genesis
)
if ctx.GlobalBool(MachineFlag.Name) {
tracer = NewJSONLogger(logconfig, os.Stdout)
@ -98,13 +98,14 @@ func runCmd(ctx *cli.Context) error {
}
if ctx.GlobalString(GenesisFlag.Name) != "" {
gen := readGenesis(ctx.GlobalString(GenesisFlag.Name))
genesisConfig = gen
db := ethdb.NewMemDatabase()
genesis := gen.ToBlock(db)
statedb, _ = state.New(genesis.Root(), state.NewDatabase(db))
chainConfig = gen.Config
blockNumber = gen.Number
} else {
statedb, _ = state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase()))
genesisConfig = new(core.Genesis)
}
if ctx.GlobalString(SenderFlag.Name) != "" {
sender = common.HexToAddress(ctx.GlobalString(SenderFlag.Name))
@ -156,13 +157,19 @@ func runCmd(ctx *cli.Context) error {
}
initialGas := ctx.GlobalUint64(GasFlag.Name)
if genesisConfig.GasLimit != 0 {
initialGas = genesisConfig.GasLimit
}
runtimeConfig := runtime.Config{
Origin: sender,
State: statedb,
GasLimit: initialGas,
GasPrice: utils.GlobalBig(ctx, PriceFlag.Name),
Value: utils.GlobalBig(ctx, ValueFlag.Name),
BlockNumber: new(big.Int).SetUint64(blockNumber),
Difficulty: genesisConfig.Difficulty,
Time: new(big.Int).SetUint64(genesisConfig.Timestamp),
Coinbase: genesisConfig.Coinbase,
BlockNumber: new(big.Int).SetUint64(genesisConfig.Number),
EVMConfig: vm.Config{
Tracer: tracer,
Debug: ctx.GlobalBool(DebugFlag.Name) || ctx.GlobalBool(MachineFlag.Name),

119
cmd/faucet/faucet.go
View File

@ -157,7 +157,8 @@ func main() {
if blob, err = ioutil.ReadFile(*accPassFlag); err != nil {
log.Crit("Failed to read account password contents", "file", *accPassFlag, "err", err)
}
pass := string(blob)
// Delete trailing newline in password
pass := strings.TrimSuffix(string(blob), "\n")
ks := keystore.NewKeyStore(filepath.Join(os.Getenv("HOME"), ".faucet", "keys"), keystore.StandardScryptN, keystore.StandardScryptP)
if blob, err = ioutil.ReadFile(*accJSONFlag); err != nil {
@ -198,6 +199,8 @@ type faucet struct {
keystore *keystore.KeyStore // Keystore containing the single signer
account accounts.Account // Account funding user faucet requests
head *types.Header // Current head header of the faucet
balance *big.Int // Current balance of the faucet
nonce uint64 // Current pending nonce of the faucet
price *big.Int // Current gas price to issue funds with
@ -213,7 +216,7 @@ func newFaucet(genesis *core.Genesis, port int, enodes []*discv5.Node, network u
// Assemble the raw devp2p protocol stack
stack, err := node.New(&node.Config{
Name: "geth",
Version: params.Version,
Version: params.VersionWithMeta,
DataDir: filepath.Join(os.Getenv("HOME"), ".faucet"),
P2P: p2p.Config{
NAT: nat.Any(),
@ -323,33 +326,30 @@ func (f *faucet) apiHandler(conn *websocket.Conn) {
nonce uint64
err error
)
for {
// Attempt to retrieve the stats, may error on no faucet connectivity
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
head, err = f.client.HeaderByNumber(ctx, nil)
if err == nil {
balance, err = f.client.BalanceAt(ctx, f.account.Address, head.Number)
if err == nil {
nonce, err = f.client.NonceAt(ctx, f.account.Address, nil)
}
for head == nil || balance == nil {
// Retrieve the current stats cached by the faucet
f.lock.RLock()
if f.head != nil {
head = types.CopyHeader(f.head)
}
cancel()
if f.balance != nil {
balance = new(big.Int).Set(f.balance)
}
nonce = f.nonce
f.lock.RUnlock()
// If stats retrieval failed, wait a bit and retry
if err != nil {
if err = sendError(conn, errors.New("Faucet offline: "+err.Error())); err != nil {
if head == nil || balance == nil {
// Report the faucet offline until initial stats are ready
if err = sendError(conn, errors.New("Faucet offline")); err != nil {
log.Warn("Failed to send faucet error to client", "err", err)
return
}
time.Sleep(3 * time.Second)
continue
}
// Initial stats reported successfully, proceed with user interaction
break
}
// Send over the initial stats and the latest header
if err = send(conn, map[string]interface{}{
"funds": balance.Div(balance, ether),
"funds": new(big.Int).Div(balance, ether),
"funded": nonce,
"peers": f.stack.Server().PeerCount(),
"requests": f.reqs,
@ -519,6 +519,47 @@ func (f *faucet) apiHandler(conn *websocket.Conn) {
}
}
// refresh attempts to retrieve the latest header from the chain and extract the
// associated faucet balance and nonce for connectivity caching.
func (f *faucet) refresh(head *types.Header) error {
// Ensure a state update does not run for too long
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
// If no header was specified, use the current chain head
var err error
if head == nil {
if head, err = f.client.HeaderByNumber(ctx, nil); err != nil {
return err
}
}
// Retrieve the balance, nonce and gas price from the current head
var (
balance *big.Int
nonce uint64
price *big.Int
)
if balance, err = f.client.BalanceAt(ctx, f.account.Address, head.Number); err != nil {
return err
}
if nonce, err = f.client.NonceAt(ctx, f.account.Address, head.Number); err != nil {
return err
}
if price, err = f.client.SuggestGasPrice(ctx); err != nil {
return err
}
// Everything succeeded, update the cached stats and eject old requests
f.lock.Lock()
f.head, f.balance = head, balance
f.price, f.nonce = price, nonce
for len(f.reqs) > 0 && f.reqs[0].Tx.Nonce() < f.nonce {
f.reqs = f.reqs[1:]
}
f.lock.Unlock()
return nil
}
// loop keeps waiting for interesting events and pushes them out to connected
// websockets.
func (f *faucet) loop() {
@ -536,45 +577,27 @@ func (f *faucet) loop() {
go func() {
for head := range update {
// New chain head arrived, query the current stats and stream to clients
var (
balance *big.Int
nonce uint64
price *big.Int
err error
)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
balance, err = f.client.BalanceAt(ctx, f.account.Address, head.Number)
if err == nil {
nonce, err = f.client.NonceAt(ctx, f.account.Address, nil)
if err == nil {
price, err = f.client.SuggestGasPrice(ctx)
}
timestamp := time.Unix(head.Time.Int64(), 0)
if time.Since(timestamp) > time.Hour {
log.Warn("Skipping faucet refresh, head too old", "number", head.Number, "hash", head.Hash(), "age", common.PrettyAge(timestamp))
continue
}
cancel()
// If querying the data failed, try for the next block
if err != nil {
if err := f.refresh(head); err != nil {
log.Warn("Failed to update faucet state", "block", head.Number, "hash", head.Hash(), "err", err)
continue
} else {
log.Info("Updated faucet state", "block", head.Number, "hash", head.Hash(), "balance", balance, "nonce", nonce, "price", price)
}
// Faucet state retrieved, update locally and send to clients
balance = new(big.Int).Div(balance, ether)
f.lock.Lock()
f.price, f.nonce = price, nonce
for len(f.reqs) > 0 && f.reqs[0].Tx.Nonce() < f.nonce {
f.reqs = f.reqs[1:]
}
f.lock.Unlock()
f.lock.RLock()
log.Info("Updated faucet state", "number", head.Number, "hash", head.Hash(), "age", common.PrettyAge(timestamp), "balance", f.balance, "nonce", f.nonce, "price", f.price)
balance := new(big.Int).Div(f.balance, ether)
peers := f.stack.Server().PeerCount()
for _, conn := range f.conns {
if err := send(conn, map[string]interface{}{
"funds": balance,
"funded": f.nonce,
"peers": f.stack.Server().PeerCount(),
"peers": peers,
"requests": f.reqs,
}, time.Second); err != nil {
log.Warn("Failed to send stats to client", "err", err)

2
cmd/geth/bugcmd.go
View File

@ -51,7 +51,7 @@ func reportBug(ctx *cli.Context) error {
fmt.Fprintln(&buff, "#### System information")
fmt.Fprintln(&buff)
fmt.Fprintln(&buff, "Version:", params.Version)
fmt.Fprintln(&buff, "Version:", params.VersionWithMeta)
fmt.Fprintln(&buff, "Go Version:", runtime.Version())
fmt.Fprintln(&buff, "OS:", runtime.GOOS)
printOSDetails(&buff)

19
cmd/geth/chaincmd.go
View File

@ -49,7 +49,6 @@ var (
ArgsUsage: "<genesisPath>",
Flags: []cli.Flag{
utils.DataDirFlag,
utils.LightModeFlag,
},
Category: "BLOCKCHAIN COMMANDS",
Description: `
@ -67,7 +66,7 @@ It expects the genesis file as argument.`,
Flags: []cli.Flag{
utils.DataDirFlag,
utils.CacheFlag,
utils.LightModeFlag,
utils.SyncModeFlag,
utils.GCModeFlag,
utils.CacheDatabaseFlag,
utils.CacheGCFlag,
@ -88,14 +87,15 @@ processing will proceed even if an individual RLP-file import failure occurs.`,
Flags: []cli.Flag{
utils.DataDirFlag,
utils.CacheFlag,
utils.LightModeFlag,
utils.SyncModeFlag,
},
Category: "BLOCKCHAIN COMMANDS",
Description: `
Requires a first argument of the file to write to.
Optional second and third arguments control the first and
last block to write. In this mode, the file will be appended
if already existing.`,
if already existing. If the file ends with .gz, the output will
be gzipped.`,
}
importPreimagesCommand = cli.Command{
Action: utils.MigrateFlags(importPreimages),
@ -105,7 +105,7 @@ if already existing.`,
Flags: []cli.Flag{
utils.DataDirFlag,
utils.CacheFlag,
utils.LightModeFlag,
utils.SyncModeFlag,
},
Category: "BLOCKCHAIN COMMANDS",
Description: `
@ -119,7 +119,7 @@ if already existing.`,
Flags: []cli.Flag{
utils.DataDirFlag,
utils.CacheFlag,
utils.LightModeFlag,
utils.SyncModeFlag,
},
Category: "BLOCKCHAIN COMMANDS",
Description: `
@ -149,7 +149,6 @@ The first argument must be the directory containing the blockchain to download f
ArgsUsage: " ",
Flags: []cli.Flag{
utils.DataDirFlag,
utils.LightModeFlag,
},
Category: "BLOCKCHAIN COMMANDS",
Description: `
@ -163,7 +162,7 @@ Remove blockchain and state databases`,
Flags: []cli.Flag{
utils.DataDirFlag,
utils.CacheFlag,
utils.LightModeFlag,
utils.SyncModeFlag,
},
Category: "BLOCKCHAIN COMMANDS",
Description: `
@ -364,9 +363,9 @@ func importPreimages(ctx *cli.Context) error {
start := time.Now()
if err := utils.ImportPreimages(diskdb, ctx.Args().First()); err != nil {
utils.Fatalf("Export error: %v\n", err)
utils.Fatalf("Import error: %v\n", err)
}
fmt.Printf("Export done in %v\n", time.Since(start))
fmt.Printf("Import done in %v\n", time.Since(start))
return nil
}

3
cmd/geth/config.go
View File

@ -177,6 +177,9 @@ func makeFullNode(ctx *cli.Context) *node.Node {
if ctx.GlobalIsSet(utils.WhisperMinPOWFlag.Name) {
cfg.Shh.MinimumAcceptedPOW = ctx.Float64(utils.WhisperMinPOWFlag.Name)
}
if ctx.GlobalIsSet(utils.WhisperRestrictConnectionBetweenLightClientsFlag.Name) {
cfg.Shh.RestrictConnectionBetweenLightClients = true
}
utils.RegisterShhService(stack, &cfg.Shh)
}

6
cmd/geth/consolecmd_test.go
View File

@ -31,7 +31,7 @@ import (
)
const (
ipcAPIs = "admin:1.0 debug:1.0 eth:1.0 miner:1.0 net:1.0 personal:1.0 rpc:1.0 shh:1.0 txpool:1.0 web3:1.0"
ipcAPIs = "admin:1.0 debug:1.0 eth:1.0 ethash:1.0 miner:1.0 net:1.0 personal:1.0 rpc:1.0 shh:1.0 txpool:1.0 web3:1.0"
httpAPIs = "eth:1.0 net:1.0 rpc:1.0 web3:1.0"
)
@ -50,7 +50,7 @@ func TestConsoleWelcome(t *testing.T) {
geth.SetTemplateFunc("goos", func() string { return runtime.GOOS })
geth.SetTemplateFunc("goarch", func() string { return runtime.GOARCH })
geth.SetTemplateFunc("gover", runtime.Version)
geth.SetTemplateFunc("gethver", func() string { return params.Version })
geth.SetTemplateFunc("gethver", func() string { return params.VersionWithMeta })
geth.SetTemplateFunc("niltime", func() string { return time.Unix(0, 0).Format(time.RFC1123) })
geth.SetTemplateFunc("apis", func() string { return ipcAPIs })
@ -133,7 +133,7 @@ func testAttachWelcome(t *testing.T, geth *testgeth, endpoint, apis string) {
attach.SetTemplateFunc("goos", func() string { return runtime.GOOS })
attach.SetTemplateFunc("goarch", func() string { return runtime.GOARCH })
attach.SetTemplateFunc("gover", runtime.Version)
attach.SetTemplateFunc("gethver", func() string { return params.Version })
attach.SetTemplateFunc("gethver", func() string { return params.VersionWithMeta })
attach.SetTemplateFunc("etherbase", func() string { return geth.Etherbase })
attach.SetTemplateFunc("niltime", func() string { return time.Unix(0, 0).Format(time.RFC1123) })
attach.SetTemplateFunc("ipc", func() bool { return strings.HasPrefix(endpoint, "ipc") })

68
cmd/geth/main.go
View File

@ -72,6 +72,7 @@ var (
utils.EthashDatasetDirFlag,
utils.EthashDatasetsInMemoryFlag,
utils.EthashDatasetsOnDiskFlag,
utils.TxPoolLocalsFlag,
utils.TxPoolNoLocalsFlag,
utils.TxPoolJournalFlag,
utils.TxPoolRejournalFlag,
@ -82,8 +83,6 @@ var (
utils.TxPoolAccountQueueFlag,
utils.TxPoolGlobalQueueFlag,
utils.TxPoolLifetimeFlag,
utils.FastSyncFlag,
utils.LightModeFlag,
utils.SyncModeFlag,
utils.GCModeFlag,
utils.LightServFlag,
@ -96,11 +95,21 @@ var (
utils.ListenPortFlag,
utils.MaxPeersFlag,
utils.MaxPendingPeersFlag,
utils.EtherbaseFlag,
utils.GasPriceFlag,
utils.MinerThreadsFlag,
utils.MiningEnabledFlag,
utils.TargetGasLimitFlag,
utils.MinerThreadsFlag,
utils.MinerLegacyThreadsFlag,
utils.MinerNotifyFlag,
utils.MinerGasTargetFlag,
utils.MinerLegacyGasTargetFlag,
utils.MinerGasLimitFlag,
utils.MinerGasPriceFlag,
utils.MinerLegacyGasPriceFlag,
utils.MinerEtherbaseFlag,
utils.MinerLegacyEtherbaseFlag,
utils.MinerExtraDataFlag,
utils.MinerLegacyExtraDataFlag,
utils.MinerRecommitIntervalFlag,
utils.MinerNoVerfiyFlag,
utils.NATFlag,
utils.NoDiscoverFlag,
utils.DiscoveryV5Flag,
@ -122,7 +131,8 @@ var (
utils.NoCompactionFlag,
utils.GpoBlocksFlag,
utils.GpoPercentileFlag,
utils.ExtraDataFlag,
utils.EWASMInterpreterFlag,
utils.EVMInterpreterFlag,
configFileFlag,
utils.EnableNodePermissionFlag,
utils.RaftModeFlag,
@ -152,6 +162,7 @@ var (
utils.WhisperEnabledFlag,
utils.WhisperMaxMessageSizeFlag,
utils.WhisperMinPOWFlag,
utils.WhisperRestrictConnectionBetweenLightClientsFlag,
}
metricsFlags = []cli.Flag{
@ -208,10 +219,15 @@ func init() {
app.Before = func(ctx *cli.Context) error {
runtime.GOMAXPROCS(runtime.NumCPU())
if err := debug.Setup(ctx); err != nil {
logdir := ""
if ctx.GlobalBool(utils.DashboardEnabledFlag.Name) {
logdir = (&node.Config{DataDir: utils.MakeDataDir(ctx)}).ResolvePath("logs")
}
if err := debug.Setup(ctx, logdir); err != nil {
return err
}
// Cap the cache allowance and tune the garbage colelctor
// Cap the cache allowance and tune the garbage collector
var mem gosigar.Mem
if err := mem.Get(); err == nil {
allowance := int(mem.Total / 1024 / 1024 / 3)
@ -233,7 +249,6 @@ func init() {
// Start system runtime metrics collection
go metrics.CollectProcessMetrics(3 * time.Second)
utils.SetupNetwork(ctx)
return nil
}
@ -255,6 +270,9 @@ func main() {
// It creates a default node based on the command line arguments and runs it in
// blocking mode, waiting for it to be shut down.
func geth(ctx *cli.Context) error {
if args := ctx.Args(); len(args) > 0 {
return fmt.Errorf("invalid command: %q", args[0])
}
node := makeFullNode(ctx)
startNode(ctx, node)
node.Wait()
@ -310,11 +328,11 @@ func startNode(ctx *cli.Context, stack *node.Node) {
status, _ := event.Wallet.Status()
log.Info("New wallet appeared", "url", event.Wallet.URL(), "status", status)
derivationPath := accounts.DefaultBaseDerivationPath
if event.Wallet.URL().Scheme == "ledger" {
event.Wallet.SelfDerive(accounts.DefaultLedgerBaseDerivationPath, stateReader)
} else {
event.Wallet.SelfDerive(accounts.DefaultBaseDerivationPath, stateReader)
derivationPath = accounts.DefaultLedgerBaseDerivationPath
}
event.Wallet.SelfDerive(derivationPath, stateReader)
case accounts.WalletDropped:
log.Info("Old wallet dropped", "url", event.Wallet.URL())
@ -325,25 +343,25 @@ func startNode(ctx *cli.Context, stack *node.Node) {
// Start auxiliary services if enabled
if ctx.GlobalBool(utils.MiningEnabledFlag.Name) || ctx.GlobalBool(utils.DeveloperFlag.Name) {
// Mining only makes sense if a full Ethereum node is running
if ctx.GlobalBool(utils.LightModeFlag.Name) || ctx.GlobalString(utils.SyncModeFlag.Name) == "light" {
if ctx.GlobalString(utils.SyncModeFlag.Name) == "light" {
utils.Fatalf("Light clients do not support mining")
}
var ethereum *eth.Ethereum
if err := stack.Service(&ethereum); err != nil {
utils.Fatalf("Ethereum service not running: %v", err)
}
// Use a reduced number of threads if requested
if threads := ctx.GlobalInt(utils.MinerThreadsFlag.Name); threads > 0 {
type threaded interface {
SetThreads(threads int)
}
if th, ok := ethereum.Engine().(threaded); ok {
th.SetThreads(threads)
}
}
// Set the gas price to the limits from the CLI and start mining
ethereum.TxPool().SetGasPrice(utils.GlobalBig(ctx, utils.GasPriceFlag.Name))
if err := ethereum.StartMining(true); err != nil {
gasprice := utils.GlobalBig(ctx, utils.MinerLegacyGasPriceFlag.Name)
if ctx.IsSet(utils.MinerGasPriceFlag.Name) {
gasprice = utils.GlobalBig(ctx, utils.MinerGasPriceFlag.Name)
}
ethereum.TxPool().SetGasPrice(gasprice)
threads := ctx.GlobalInt(utils.MinerLegacyThreadsFlag.Name)
if ctx.GlobalIsSet(utils.MinerThreadsFlag.Name) {
threads = ctx.GlobalInt(utils.MinerThreadsFlag.Name)
}
if err := ethereum.StartMining(threads); err != nil {
utils.Fatalf("Failed to start mining: %v", err)
}
}

2
cmd/geth/misccmd.go
View File

@ -108,7 +108,7 @@ func makedag(ctx *cli.Context) error {
func version(ctx *cli.Context) error {
fmt.Println(strings.Title(clientIdentifier))
fmt.Println("Version:", params.Version)
fmt.Println("Version:", params.VersionWithMeta)
if gitCommit != "" {
fmt.Println("Git Commit:", gitCommit)
}

6
cmd/geth/monitorcmd.go
View File

@ -185,12 +185,12 @@ func resolveMetric(metrics map[string]interface{}, pattern string, path string)
parts := strings.SplitN(pattern, "/", 2)
if len(parts) > 1 {
for _, variation := range strings.Split(parts[0], ",") {
if submetrics, ok := metrics[variation].(map[string]interface{}); !ok {
submetrics, ok := metrics[variation].(map[string]interface{})
if !ok {
utils.Fatalf("Failed to retrieve system metrics: %s", path+variation)
return nil
} else {
results = append(results, resolveMetric(submetrics, parts[1], path+variation+"/")...)
}
results = append(results, resolveMetric(submetrics, parts[1], path+variation+"/")...)
}
return results
}

25
cmd/geth/usage.go
View File

@ -84,7 +84,8 @@ var AppHelpFlagGroups = []flagGroup{
utils.LightKDFFlag,
},
},
{Name: "DEVELOPER CHAIN",
{
Name: "DEVELOPER CHAIN",
Flags: []cli.Flag{
utils.DeveloperFlag,
utils.DeveloperPeriodFlag,
@ -114,6 +115,7 @@ var AppHelpFlagGroups = []flagGroup{
{
Name: "TRANSACTION POOL",
Flags: []cli.Flag{
utils.TxPoolLocalsFlag,
utils.TxPoolNoLocalsFlag,
utils.TxPoolJournalFlag,
utils.TxPoolRejournalFlag,
@ -211,10 +213,14 @@ var AppHelpFlagGroups = []flagGroup{
Flags: []cli.Flag{
utils.MiningEnabledFlag,
utils.MinerThreadsFlag,
utils.EtherbaseFlag,
utils.TargetGasLimitFlag,
utils.GasPriceFlag,
utils.ExtraDataFlag,
utils.MinerNotifyFlag,
utils.MinerGasPriceFlag,
utils.MinerGasTargetFlag,
utils.MinerGasLimitFlag,
utils.MinerEtherbaseFlag,
utils.MinerExtraDataFlag,
utils.MinerRecommitIntervalFlag,
utils.MinerNoVerfiyFlag,
},
},
{
@ -228,6 +234,8 @@ var AppHelpFlagGroups = []flagGroup{
Name: "VIRTUAL MACHINE",
Flags: []cli.Flag{
utils.VMEnableDebugFlag,
utils.EVMInterpreterFlag,
utils.EWASMInterpreterFlag,
},
},
{
@ -256,8 +264,11 @@ var AppHelpFlagGroups = []flagGroup{
{
Name: "DEPRECATED",
Flags: []cli.Flag{
utils.FastSyncFlag,
utils.LightModeFlag,
utils.MinerLegacyThreadsFlag,
utils.MinerLegacyGasTargetFlag,
utils.MinerLegacyGasPriceFlag,
utils.MinerLegacyEtherbaseFlag,
utils.MinerLegacyExtraDataFlag,
},
},
{

4
cmd/puppeth/module_dashboard.go
View File

@ -678,9 +678,9 @@ func deployDashboard(client *sshClient, network string, conf *config, config *da
// Build and deploy the dashboard service
if nocache {
return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate", workdir, network, network))
return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate --timeout 60", workdir, network, network))
}
return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate", workdir, network))
return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate --timeout 60", workdir, network))
}
// dashboardInfos is returned from a dashboard status check to allow reporting

6
cmd/puppeth/module_ethstats.go
View File

@ -100,9 +100,9 @@ func deployEthstats(client *sshClient, network string, port int, secret string,
// Build and deploy the ethstats service
if nocache {
return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate", workdir, network, network))
return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate --timeout 60", workdir, network, network))
}
return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate", workdir, network))
return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate --timeout 60", workdir, network))
}
// ethstatsInfos is returned from an ethstats status check to allow reporting
@ -122,7 +122,7 @@ func (info *ethstatsInfos) Report() map[string]string {
"Website address": info.host,
"Website listener port": strconv.Itoa(info.port),
"Login secret": info.secret,
"Banned addresses": fmt.Sprintf("%v", info.banned),
"Banned addresses": strings.Join(info.banned, "\n"),
}
}

6
cmd/puppeth/module_explorer.go
View File

@ -38,7 +38,7 @@ ADD chain.json /chain.json
RUN \
echo '(cd ../eth-net-intelligence-api && pm2 start /ethstats.json)' > explorer.sh && \
echo '(cd ../etherchain-light && npm start &)' >> explorer.sh && \
echo '/parity/parity --chain=/chain.json --port={{.NodePort}} --tracing=on --fat-db=on --pruning=archive' >> explorer.sh
echo 'exec /parity/parity --chain=/chain.json --port={{.NodePort}} --tracing=on --fat-db=on --pruning=archive' >> explorer.sh
ENTRYPOINT ["/bin/sh", "explorer.sh"]
`
@ -140,9 +140,9 @@ func deployExplorer(client *sshClient, network string, chainspec []byte, config
// Build and deploy the boot or seal node service
if nocache {
return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate", workdir, network, network))
return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate --timeout 60", workdir, network, network))
}
return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate", workdir, network))
return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate --timeout 60", workdir, network))
}
// explorerInfos is returned from a block explorer status check to allow reporting

4
cmd/puppeth/module_faucet.go
View File

@ -133,9 +133,9 @@ func deployFaucet(client *sshClient, network string, bootnodes []string, config
// Build and deploy the faucet service
if nocache {
return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate", workdir, network, network))
return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate --timeout 60", workdir, network, network))
}
return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate", workdir, network))
return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate --timeout 60", workdir, network))
}
// faucetInfos is returned from a faucet status check to allow reporting various

4
cmd/puppeth/module_nginx.go
View File

@ -81,9 +81,9 @@ func deployNginx(client *sshClient, network string, port int, nocache bool) ([]b
// Build and deploy the reverse-proxy service
if nocache {
return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate", workdir, network, network))
return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate --timeout 60", workdir, network, network))
}
return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate", workdir, network))
return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate --timeout 60", workdir, network))
}
// nginxInfos is returned from an nginx reverse-proxy status check to allow

17
cmd/puppeth/module_node.go
View File

@ -42,7 +42,7 @@ ADD genesis.json /genesis.json
RUN \
echo 'geth --cache 512 init /genesis.json' > geth.sh && \{{if .Unlock}}
echo 'mkdir -p /root/.ethereum/keystore/ && cp /signer.json /root/.ethereum/keystore/' >> geth.sh && \{{end}}
echo $'geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .Bootnodes}}--bootnodes {{.Bootnodes}}{{end}} {{if .Etherbase}}--etherbase {{.Etherbase}} --mine --minerthreads 1{{end}} {{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --targetgaslimit {{.GasTarget}} --gasprice {{.GasPrice}}' >> geth.sh
echo $'exec geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .Bootnodes}}--bootnodes {{.Bootnodes}}{{end}} {{if .Etherbase}}--miner.etherbase {{.Etherbase}} --mine --miner.threads 1{{end}} {{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --miner.gastarget {{.GasTarget}} --miner.gaslimit {{.GasLimit}} --miner.gasprice {{.GasPrice}}' >> geth.sh
ENTRYPOINT ["/bin/sh", "geth.sh"]
`
@ -68,6 +68,7 @@ services:
- STATS_NAME={{.Ethstats}}
- MINER_NAME={{.Etherbase}}
- GAS_TARGET={{.GasTarget}}
- GAS_LIMIT={{.GasLimit}}
- GAS_PRICE={{.GasPrice}}
logging:
driver: "json-file"
@ -104,6 +105,7 @@ func deployNode(client *sshClient, network string, bootnodes []string, config *n
"Ethstats": config.ethstats,
"Etherbase": config.etherbase,
"GasTarget": uint64(1000000 * config.gasTarget),
"GasLimit": uint64(1000000 * config.gasLimit),
"GasPrice": uint64(1000000000 * config.gasPrice),
"Unlock": config.keyJSON != "",
})
@ -122,6 +124,7 @@ func deployNode(client *sshClient, network string, bootnodes []string, config *n
"Ethstats": config.ethstats[:strings.Index(config.ethstats, ":")],
"Etherbase": config.etherbase,
"GasTarget": config.gasTarget,
"GasLimit": config.gasLimit,
"GasPrice": config.gasPrice,
})
files[filepath.Join(workdir, "docker-compose.yaml")] = composefile.Bytes()
@ -139,9 +142,9 @@ func deployNode(client *sshClient, network string, bootnodes []string, config *n
// Build and deploy the boot or seal node service
if nocache {
return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate", workdir, network, network))
return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate --timeout 60", workdir, network, network))
}
return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate", workdir, network))
return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate --timeout 60", workdir, network))
}
// nodeInfos is returned from a boot or seal node status check to allow reporting
@ -160,6 +163,7 @@ type nodeInfos struct {
keyJSON string
keyPass string
gasTarget float64
gasLimit float64
gasPrice float64
}
@ -175,8 +179,9 @@ func (info *nodeInfos) Report() map[string]string {
}
if info.gasTarget > 0 {
// Miner or signer node
report["Gas limit (baseline target)"] = fmt.Sprintf("%0.3f MGas", info.gasTarget)
report["Gas price (minimum accepted)"] = fmt.Sprintf("%0.3f GWei", info.gasPrice)
report["Gas floor (baseline target)"] = fmt.Sprintf("%0.3f MGas", info.gasTarget)
report["Gas ceil (target maximum)"] = fmt.Sprintf("%0.3f MGas", info.gasLimit)
if info.etherbase != "" {
// Ethash proof-of-work miner
@ -217,11 +222,12 @@ func checkNode(client *sshClient, network string, boot bool) (*nodeInfos, error)
totalPeers, _ := strconv.Atoi(infos.envvars["TOTAL_PEERS"])
lightPeers, _ := strconv.Atoi(infos.envvars["LIGHT_PEERS"])
gasTarget, _ := strconv.ParseFloat(infos.envvars["GAS_TARGET"], 64)
gasLimit, _ := strconv.ParseFloat(infos.envvars["GAS_LIMIT"], 64)
gasPrice, _ := strconv.ParseFloat(infos.envvars["GAS_PRICE"], 64)
// Container available, retrieve its node ID and its genesis json
var out []byte
if out, err = client.Run(fmt.Sprintf("docker exec %s_%s_1 geth --exec admin.nodeInfo.id attach", network, kind)); err != nil {
if out, err = client.Run(fmt.Sprintf("docker exec %s_%s_1 geth --exec admin.nodeInfo.id --cache=16 attach", network, kind)); err != nil {
return nil, ErrServiceUnreachable
}
id := bytes.Trim(bytes.TrimSpace(out), "\"")
@ -256,6 +262,7 @@ func checkNode(client *sshClient, network string, boot bool) (*nodeInfos, error)
keyJSON: keyJSON,
keyPass: keyPass,
gasTarget: gasTarget,
gasLimit: gasLimit,
gasPrice: gasPrice,
}
stats.enode = fmt.Sprintf("enode://%s@%s:%d", id, client.address, stats.port)

6
cmd/puppeth/module_wallet.go
View File

@ -37,7 +37,7 @@ ADD genesis.json /genesis.json
RUN \
echo 'node server.js &' > wallet.sh && \
echo 'geth --cache 512 init /genesis.json' >> wallet.sh && \
echo $'geth --networkid {{.NetworkID}} --port {{.NodePort}} --bootnodes {{.Bootnodes}} --ethstats \'{{.Ethstats}}\' --cache=512 --rpc --rpcaddr=0.0.0.0 --rpccorsdomain "*" --rpcvhosts "*"' >> wallet.sh
echo $'exec geth --networkid {{.NetworkID}} --port {{.NodePort}} --bootnodes {{.Bootnodes}} --ethstats \'{{.Ethstats}}\' --cache=512 --rpc --rpcaddr=0.0.0.0 --rpccorsdomain "*" --rpcvhosts "*"' >> wallet.sh
RUN \
sed -i 's/PuppethNetworkID/{{.NetworkID}}/g' dist/js/etherwallet-master.js && \
@ -120,9 +120,9 @@ func deployWallet(client *sshClient, network string, bootnodes []string, config
// Build and deploy the boot or seal node service
if nocache {
return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate", workdir, network, network))
return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate --timeout 60", workdir, network, network))
}
return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate", workdir, network))
return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate --timeout 60", workdir, network))
}
// walletInfos is returned from a web wallet status check to allow reporting

52
cmd/puppeth/ssh.go
View File

@ -45,33 +45,44 @@ type sshClient struct {
// dial establishes an SSH connection to a remote node using the current user and
// the user's configured private RSA key. If that fails, password authentication
// is fallen back to. The caller may override the login user via user@server:port.
// is fallen back to. server can be a string like user:identity@server:port.
func dial(server string, pubkey []byte) (*sshClient, error) {
// Figure out a label for the server and a logger
label := server
if strings.Contains(label, ":") {
label = label[:strings.Index(label, ":")]
}
login := ""
// Figure out username, identity, hostname and port
hostname := ""
hostport := server
username := ""
identity := "id_rsa" // default
if strings.Contains(server, "@") {
login = label[:strings.Index(label, "@")]
label = label[strings.Index(label, "@")+1:]
server = server[strings.Index(server, "@")+1:]
prefix := server[:strings.Index(server, "@")]
if strings.Contains(prefix, ":") {
username = prefix[:strings.Index(prefix, ":")]
identity = prefix[strings.Index(prefix, ":")+1:]
} else {
username = prefix
}
hostport = server[strings.Index(server, "@")+1:]
}
logger := log.New("server", label)
if strings.Contains(hostport, ":") {
hostname = hostport[:strings.Index(hostport, ":")]
} else {
hostname = hostport
hostport += ":22"
}
logger := log.New("server", server)
logger.Debug("Attempting to establish SSH connection")
user, err := user.Current()
if err != nil {
return nil, err
}
if login == "" {
login = user.Username
if username == "" {
username = user.Username
}
// Configure the supported authentication methods (private key and password)
var auths []ssh.AuthMethod
path := filepath.Join(user.HomeDir, ".ssh", "id_rsa")
path := filepath.Join(user.HomeDir, ".ssh", identity)
if buf, err := ioutil.ReadFile(path); err != nil {
log.Warn("No SSH key, falling back to passwords", "path", path, "err", err)
} else {
@ -94,14 +105,14 @@ func dial(server string, pubkey []byte) (*sshClient, error) {
}
}
auths = append(auths, ssh.PasswordCallback(func() (string, error) {
fmt.Printf("What's the login password for %s at %s? (won't be echoed)\n> ", login, server)
fmt.Printf("What's the login password for %s at %s? (won't be echoed)\n> ", username, server)
blob, err := terminal.ReadPassword(int(os.Stdin.Fd()))
fmt.Println()
return string(blob), err
}))
// Resolve the IP address of the remote server
addr, err := net.LookupHost(label)
addr, err := net.LookupHost(hostname)
if err != nil {
return nil, err
}
@ -109,10 +120,7 @@ func dial(server string, pubkey []byte) (*sshClient, error) {
return nil, errors.New("no IPs associated with domain")
}
// Try to dial in to the remote server
logger.Trace("Dialing remote SSH server", "user", login)
if !strings.Contains(server, ":") {
server += ":22"
}
logger.Trace("Dialing remote SSH server", "user", username)
keycheck := func(hostname string, remote net.Addr, key ssh.PublicKey) error {
// If no public key is known for SSH, ask the user to confirm
if pubkey == nil {
@ -139,13 +147,13 @@ func dial(server string, pubkey []byte) (*sshClient, error) {
// We have a mismatch, forbid connecting
return errors.New("ssh key mismatch, readd the machine to update")
}
client, err := ssh.Dial("tcp", server, &ssh.ClientConfig{User: login, Auth: auths, HostKeyCallback: keycheck})
client, err := ssh.Dial("tcp", hostport, &ssh.ClientConfig{User: username, Auth: auths, HostKeyCallback: keycheck})
if err != nil {
return nil, err
}
// Connection established, return our utility wrapper
c := &sshClient{
server: label,
server: hostname,
address: addr[0],
pubkey: pubkey,
client: client,

2
cmd/puppeth/wizard_dashboard.go
View File

@ -92,7 +92,7 @@ func (w *wizard) deployDashboard() {
pages = append(pages, page)
}
}
// Promt the user to chose one, enter manually or simply not list this service
// Prompt the user to chose one, enter manually or simply not list this service
defLabel, defChoice := "don't list", len(pages)+2
if len(pages) > 0 {
defLabel, defChoice = pages[0], 1

38
cmd/puppeth/wizard_netstats.go
View File

@ -82,7 +82,6 @@ func (w *wizard) gatherStats(server string, pubkey []byte, client *sshClient) *s
logger.Info("Starting remote server health-check")
stat := &serverStat{
address: client.address,
services: make(map[string]map[string]string),
}
if client == nil {
@ -94,6 +93,8 @@ func (w *wizard) gatherStats(server string, pubkey []byte, client *sshClient) *s
}
client = conn
}
stat.address = client.address
// Client connected one way or another, run health-checks
logger.Debug("Checking for nginx availability")
if infos, err := checkNginx(client, w.network); err != nil {
@ -203,7 +204,7 @@ func (stats serverStats) render() {
table.SetHeader([]string{"Server", "Address", "Service", "Config", "Value"})
table.SetAlignment(tablewriter.ALIGN_LEFT)
table.SetColWidth(100)
table.SetColWidth(40)
// Find the longest lines for all columns for the hacked separator
separator := make([]string, 5)
@ -214,6 +215,9 @@ func (stats serverStats) render() {
if len(stat.address) > len(separator[1]) {
separator[1] = strings.Repeat("-", len(stat.address))
}
if len(stat.failure) > len(separator[1]) {
separator[1] = strings.Repeat("-", len(stat.failure))
}
for service, configs := range stat.services {
if len(service) > len(separator[2]) {
separator[2] = strings.Repeat("-", len(service))
@ -222,8 +226,10 @@ func (stats serverStats) render() {
if len(config) > len(separator[3]) {
separator[3] = strings.Repeat("-", len(config))
}
if len(value) > len(separator[4]) {
separator[4] = strings.Repeat("-", len(value))
for _, val := range strings.Split(value, "\n") {
if len(val) > len(separator[4]) {
separator[4] = strings.Repeat("-", len(val))
}
}
}
}
@ -248,7 +254,11 @@ func (stats serverStats) render() {
sort.Strings(services)
if len(services) == 0 {
table.Append([]string{server, stats[server].address, "", "", ""})
if stats[server].failure != "" {
table.Append([]string{server, stats[server].failure, "", "", ""})
} else {
table.Append([]string{server, stats[server].address, "", "", ""})
}
}
for j, service := range services {
// Add an empty line between all services
@ -263,13 +273,17 @@ func (stats serverStats) render() {
sort.Strings(configs)
for k, config := range configs {
switch {
case j == 0 && k == 0:
table.Append([]string{server, stats[server].address, service, config, stats[server].services[service][config]})
case k == 0:
table.Append([]string{"", "", service, config, stats[server].services[service][config]})
default:
table.Append([]string{"", "", "", config, stats[server].services[service][config]})
for l, value := range strings.Split(stats[server].services[service][config], "\n") {
switch {
case j == 0 && k == 0 && l == 0:
table.Append([]string{server, stats[server].address, service, config, value})
case k == 0 && l == 0:
table.Append([]string{"", "", service, config, value})
case l == 0:
table.Append([]string{"", "", "", config, value})
default:
table.Append([]string{"", "", "", "", value})
}
}
}
}

12
cmd/puppeth/wizard_network.go
View File

@ -62,14 +62,14 @@ func (w *wizard) manageServers() {
}
}
// makeServer reads a single line from stdin and interprets it as a hostname to
// connect to. It tries to establish a new SSH session and also executing some
// baseline validations.
// makeServer reads a single line from stdin and interprets it as
// username:identity@hostname to connect to. It tries to establish a
// new SSH session and also executing some baseline validations.
//
// If connection succeeds, the server is added to the wizards configs!
func (w *wizard) makeServer() string {
fmt.Println()
fmt.Println("Please enter remote server's address:")
fmt.Println("What is the remote server's address ([username[:identity]@]hostname[:port])?")
// Read and dial the server to ensure docker is present
input := w.readString()
@ -87,7 +87,7 @@ func (w *wizard) makeServer() string {
return input
}
// selectServer lists the user all the currnetly known servers to choose from,
// selectServer lists the user all the currently known servers to choose from,
// also granting the option to add a new one.
func (w *wizard) selectServer() string {
// List the available server to the user and wait for a choice
@ -115,7 +115,7 @@ func (w *wizard) selectServer() string {
// manageComponents displays a list of network components the user can tear down
// and an option
func (w *wizard) manageComponents() {
// List all the componens we can tear down, along with an entry to deploy a new one
// List all the components we can tear down, along with an entry to deploy a new one
fmt.Println()
var serviceHosts, serviceNames []string

6
cmd/puppeth/wizard_node.go
View File

@ -50,7 +50,7 @@ func (w *wizard) deployNode(boot bool) {
if boot {
infos = &nodeInfos{port: 30303, peersTotal: 512, peersLight: 256}
} else {
infos = &nodeInfos{port: 30303, peersTotal: 50, peersLight: 0, gasTarget: 4.7, gasPrice: 18}
infos = &nodeInfos{port: 30303, peersTotal: 50, peersLight: 0, gasTarget: 7.5, gasLimit: 10, gasPrice: 1}
}
}
existed := err == nil
@ -152,6 +152,10 @@ func (w *wizard) deployNode(boot bool) {
fmt.Printf("What gas limit should empty blocks target (MGas)? (default = %0.3f)\n", infos.gasTarget)
infos.gasTarget = w.readDefaultFloat(infos.gasTarget)
fmt.Println()
fmt.Printf("What gas limit should full blocks target (MGas)? (default = %0.3f)\n", infos.gasLimit)
infos.gasLimit = w.readDefaultFloat(infos.gasLimit)
fmt.Println()
fmt.Printf("What gas price should the signer require (GWei)? (default = %0.3f)\n", infos.gasPrice)
infos.gasPrice = w.readDefaultFloat(infos.gasPrice)

233
cmd/swarm/access.go Normal file
View File

@ -0,0 +1,233 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"crypto/rand"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"strings"
"github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/swarm/api"
"github.com/ethereum/go-ethereum/swarm/api/client"
"gopkg.in/urfave/cli.v1"
)
var salt = make([]byte, 32)
func init() {
if _, err := io.ReadFull(rand.Reader, salt); err != nil {
panic("reading from crypto/rand failed: " + err.Error())
}
}
func accessNewPass(ctx *cli.Context) {
args := ctx.Args()
if len(args) != 1 {
utils.Fatalf("Expected 1 argument - the ref")
}
var (
ae *api.AccessEntry
accessKey []byte
err error
ref = args[0]
password = getPassPhrase("", 0, makePasswordList(ctx))
dryRun = ctx.Bool(SwarmDryRunFlag.Name)
)
accessKey, ae, err = api.DoPassword(ctx, password, salt)
if err != nil {
utils.Fatalf("error getting session key: %v", err)
}
m, err := api.GenerateAccessControlManifest(ctx, ref, accessKey, ae)
if dryRun {
err = printManifests(m, nil)
if err != nil {
utils.Fatalf("had an error printing the manifests: %v", err)
}
} else {
err = uploadManifests(ctx, m, nil)
if err != nil {
utils.Fatalf("had an error uploading the manifests: %v", err)
}
}
}
func accessNewPK(ctx *cli.Context) {
args := ctx.Args()
if len(args) != 1 {
utils.Fatalf("Expected 1 argument - the ref")
}
var (
ae *api.AccessEntry
sessionKey []byte
err error
ref = args[0]
privateKey = getPrivKey(ctx)
granteePublicKey = ctx.String(SwarmAccessGrantKeyFlag.Name)
dryRun = ctx.Bool(SwarmDryRunFlag.Name)
)
sessionKey, ae, err = api.DoPK(ctx, privateKey, granteePublicKey, salt)
if err != nil {
utils.Fatalf("error getting session key: %v", err)
}
m, err := api.GenerateAccessControlManifest(ctx, ref, sessionKey, ae)
if dryRun {
err = printManifests(m, nil)
if err != nil {
utils.Fatalf("had an error printing the manifests: %v", err)
}
} else {
err = uploadManifests(ctx, m, nil)
if err != nil {
utils.Fatalf("had an error uploading the manifests: %v", err)
}
}
}
func accessNewACT(ctx *cli.Context) {
args := ctx.Args()
if len(args) != 1 {
utils.Fatalf("Expected 1 argument - the ref")
}
var (
ae *api.AccessEntry
actManifest *api.Manifest
accessKey []byte
err error
ref = args[0]
pkGrantees = []string{}
passGrantees = []string{}
pkGranteesFilename = ctx.String(SwarmAccessGrantKeysFlag.Name)
passGranteesFilename = ctx.String(utils.PasswordFileFlag.Name)
privateKey = getPrivKey(ctx)
dryRun = ctx.Bool(SwarmDryRunFlag.Name)
)
if pkGranteesFilename == "" && passGranteesFilename == "" {
utils.Fatalf("you have to provide either a grantee public-keys file or an encryption passwords file (or both)")
}
if pkGranteesFilename != "" {
bytes, err := ioutil.ReadFile(pkGranteesFilename)
if err != nil {
utils.Fatalf("had an error reading the grantee public key list")
}
pkGrantees = strings.Split(string(bytes), "\n")
}
if passGranteesFilename != "" {
bytes, err := ioutil.ReadFile(passGranteesFilename)
if err != nil {
utils.Fatalf("could not read password filename: %v", err)
}
passGrantees = strings.Split(string(bytes), "\n")
}
accessKey, ae, actManifest, err = api.DoACT(ctx, privateKey, salt, pkGrantees, passGrantees)
if err != nil {
utils.Fatalf("error generating ACT manifest: %v", err)
}
if err != nil {
utils.Fatalf("error getting session key: %v", err)
}
m, err := api.GenerateAccessControlManifest(ctx, ref, accessKey, ae)
if err != nil {
utils.Fatalf("error generating root access manifest: %v", err)
}
if dryRun {
err = printManifests(m, actManifest)
if err != nil {
utils.Fatalf("had an error printing the manifests: %v", err)
}
} else {
err = uploadManifests(ctx, m, actManifest)
if err != nil {
utils.Fatalf("had an error uploading the manifests: %v", err)
}
}
}
func printManifests(rootAccessManifest, actManifest *api.Manifest) error {
js, err := json.Marshal(rootAccessManifest)
if err != nil {
return err
}
fmt.Println(string(js))
if actManifest != nil {
js, err := json.Marshal(actManifest)
if err != nil {
return err
}
fmt.Println(string(js))
}
return nil
}
func uploadManifests(ctx *cli.Context, rootAccessManifest, actManifest *api.Manifest) error {
bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client := client.NewClient(bzzapi)
var (
key string
err error
)
if actManifest != nil {
key, err = client.UploadManifest(actManifest, false)
if err != nil {
return err
}
rootAccessManifest.Entries[0].Access.Act = key
}
key, err = client.UploadManifest(rootAccessManifest, false)
if err != nil {
return err
}
fmt.Println(key)
return nil
}
// makePasswordList reads password lines from the file specified by the global --password flag
// and also by the same subcommand --password flag.
// This function ia a fork of utils.MakePasswordList to lookup cli context for subcommand.
// Function ctx.SetGlobal is not setting the global flag value that can be accessed
// by ctx.GlobalString using the current version of cli package.
func makePasswordList(ctx *cli.Context) []string {
path := ctx.GlobalString(utils.PasswordFileFlag.Name)
if path == "" {
path = ctx.String(utils.PasswordFileFlag.Name)
if path == "" {
return nil
}
}
text, err := ioutil.ReadFile(path)
if err != nil {
utils.Fatalf("Failed to read password file: %v", err)
}
lines := strings.Split(string(text), "\n")
// Sanitise DOS line endings.
for i := range lines {
lines[i] = strings.TrimRight(lines[i], "\r")
}
return lines
}

605
cmd/swarm/access_test.go Normal file
View File

@ -0,0 +1,605 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
// +build !windows
package main
import (
"bytes"
"crypto/rand"
"encoding/hex"
"encoding/json"
"io"
"io/ioutil"
gorand "math/rand"
"net/http"
"os"
"strings"
"testing"
"time"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/crypto/ecies"
"github.com/ethereum/go-ethereum/crypto/sha3"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/swarm/api"
swarm "github.com/ethereum/go-ethereum/swarm/api/client"
"github.com/ethereum/go-ethereum/swarm/testutil"
)
const (
hashRegexp = `[a-f\d]{128}`
data = "notsorandomdata"
)
var DefaultCurve = crypto.S256()
// TestAccessPassword tests for the correct creation of an ACT manifest protected by a password.
// The test creates bogus content, uploads it encrypted, then creates the wrapping manifest with the Access entry
// The parties participating - node (publisher), uploads to second node then disappears. Content which was uploaded
// is then fetched through 2nd node. since the tested code is not key-aware - we can just
// fetch from the 2nd node using HTTP BasicAuth
func TestAccessPassword(t *testing.T) {
cluster := newTestCluster(t, 1)
defer cluster.Shutdown()
proxyNode := cluster.Nodes[0]
dataFilename := testutil.TempFileWithContent(t, data)
defer os.RemoveAll(dataFilename)
// upload the file with 'swarm up' and expect a hash
up := runSwarm(t,
"--bzzapi",
proxyNode.URL, //it doesn't matter through which node we upload content
"up",
"--encrypt",
dataFilename)
_, matches := up.ExpectRegexp(hashRegexp)
up.ExpectExit()
if len(matches) < 1 {
t.Fatal("no matches found")
}
ref := matches[0]
tmp, err := ioutil.TempDir("", "swarm-test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmp)
password := "smth"
passwordFilename := testutil.TempFileWithContent(t, "smth")
defer os.RemoveAll(passwordFilename)
up = runSwarm(t,
"access",
"new",
"pass",
"--dry-run",
"--password",
passwordFilename,
ref,
)
_, matches = up.ExpectRegexp(".+")
up.ExpectExit()
if len(matches) == 0 {
t.Fatalf("stdout not matched")
}
var m api.Manifest
err = json.Unmarshal([]byte(matches[0]), &m)
if err != nil {
t.Fatalf("unmarshal manifest: %v", err)
}
if len(m.Entries) != 1 {
t.Fatalf("expected one manifest entry, got %v", len(m.Entries))
}
e := m.Entries[0]
ct := "application/bzz-manifest+json"
if e.ContentType != ct {
t.Errorf("expected %q content type, got %q", ct, e.ContentType)
}
if e.Access == nil {
t.Fatal("manifest access is nil")
}
a := e.Access
if a.Type != "pass" {
t.Errorf(`got access type %q, expected "pass"`, a.Type)
}
if len(a.Salt) < 32 {
t.Errorf(`got salt with length %v, expected not less the 32 bytes`, len(a.Salt))
}
if a.KdfParams == nil {
t.Fatal("manifest access kdf params is nil")
}
if a.Publisher != "" {
t.Fatal("should be empty")
}
client := swarm.NewClient(cluster.Nodes[0].URL)
hash, err := client.UploadManifest(&m, false)
if err != nil {
t.Fatal(err)
}
httpClient := &http.Client{}
url := cluster.Nodes[0].URL + "/" + "bzz:/" + hash
response, err := httpClient.Get(url)
if err != nil {
t.Fatal(err)
}
if response.StatusCode != http.StatusUnauthorized {
t.Fatal("should be a 401")
}
authHeader := response.Header.Get("WWW-Authenticate")
if authHeader == "" {
t.Fatal("should be something here")
}
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
t.Fatal(err)
}
req.SetBasicAuth("", password)
response, err = http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
t.Errorf("expected status %v, got %v", http.StatusOK, response.StatusCode)
}
d, err := ioutil.ReadAll(response.Body)
if err != nil {
t.Fatal(err)
}
if string(d) != data {
t.Errorf("expected decrypted data %q, got %q", data, string(d))
}
wrongPasswordFilename := testutil.TempFileWithContent(t, "just wr0ng")
defer os.RemoveAll(wrongPasswordFilename)
//download file with 'swarm down' with wrong password
up = runSwarm(t,
"--bzzapi",
proxyNode.URL,
"down",
"bzz:/"+hash,
tmp,
"--password",
wrongPasswordFilename)
_, matches = up.ExpectRegexp("unauthorized")
if len(matches) != 1 && matches[0] != "unauthorized" {
t.Fatal(`"unauthorized" not found in output"`)
}
up.ExpectExit()
}
// TestAccessPK tests for the correct creation of an ACT manifest between two parties (publisher and grantee).
// The test creates bogus content, uploads it encrypted, then creates the wrapping manifest with the Access entry
// The parties participating - node (publisher), uploads to second node (which is also the grantee) then disappears.
// Content which was uploaded is then fetched through the grantee's http proxy. Since the tested code is private-key aware,
// the test will fail if the proxy's given private key is not granted on the ACT.
func TestAccessPK(t *testing.T) {
// Setup Swarm and upload a test file to it
cluster := newTestCluster(t, 2)
defer cluster.Shutdown()
dataFilename := testutil.TempFileWithContent(t, data)
defer os.RemoveAll(dataFilename)
// upload the file with 'swarm up' and expect a hash
up := runSwarm(t,
"--bzzapi",
cluster.Nodes[0].URL,
"up",
"--encrypt",
dataFilename)
_, matches := up.ExpectRegexp(hashRegexp)
up.ExpectExit()
if len(matches) < 1 {
t.Fatal("no matches found")
}
ref := matches[0]
pk := cluster.Nodes[0].PrivateKey
granteePubKey := crypto.CompressPubkey(&pk.PublicKey)
publisherDir, err := ioutil.TempDir("", "swarm-account-dir-temp")
if err != nil {
t.Fatal(err)
}
passwordFilename := testutil.TempFileWithContent(t, testPassphrase)
defer os.RemoveAll(passwordFilename)
_, publisherAccount := getTestAccount(t, publisherDir)
up = runSwarm(t,
"--bzzaccount",
publisherAccount.Address.String(),
"--password",
passwordFilename,
"--datadir",
publisherDir,
"--bzzapi",
cluster.Nodes[0].URL,
"access",
"new",
"pk",
"--dry-run",
"--grant-key",
hex.EncodeToString(granteePubKey),
ref,
)
_, matches = up.ExpectRegexp(".+")
up.ExpectExit()
if len(matches) == 0 {
t.Fatalf("stdout not matched")
}
//get the public key from the publisher directory
publicKeyFromDataDir := runSwarm(t,
"--bzzaccount",
publisherAccount.Address.String(),
"--password",
passwordFilename,
"--datadir",
publisherDir,
"print-keys",
"--compressed",
)
_, publicKeyString := publicKeyFromDataDir.ExpectRegexp(".+")
publicKeyFromDataDir.ExpectExit()
pkComp := strings.Split(publicKeyString[0], "=")[1]
var m api.Manifest
err = json.Unmarshal([]byte(matches[0]), &m)
if err != nil {
t.Fatalf("unmarshal manifest: %v", err)
}
if len(m.Entries) != 1 {
t.Fatalf("expected one manifest entry, got %v", len(m.Entries))
}
e := m.Entries[0]
ct := "application/bzz-manifest+json"
if e.ContentType != ct {
t.Errorf("expected %q content type, got %q", ct, e.ContentType)
}
if e.Access == nil {
t.Fatal("manifest access is nil")
}
a := e.Access
if a.Type != "pk" {
t.Errorf(`got access type %q, expected "pk"`, a.Type)
}
if len(a.Salt) < 32 {
t.Errorf(`got salt with length %v, expected not less the 32 bytes`, len(a.Salt))
}
if a.KdfParams != nil {
t.Fatal("manifest access kdf params should be nil")
}
if a.Publisher != pkComp {
t.Fatal("publisher key did not match")
}
client := swarm.NewClient(cluster.Nodes[0].URL)
hash, err := client.UploadManifest(&m, false)
if err != nil {
t.Fatal(err)
}
httpClient := &http.Client{}
url := cluster.Nodes[0].URL + "/" + "bzz:/" + hash
response, err := httpClient.Get(url)
if err != nil {
t.Fatal(err)
}
if response.StatusCode != http.StatusOK {
t.Fatal("should be a 200")
}
d, err := ioutil.ReadAll(response.Body)
if err != nil {
t.Fatal(err)
}
if string(d) != data {
t.Errorf("expected decrypted data %q, got %q", data, string(d))
}
}
// TestAccessACT tests the creation of the ACT manifest end-to-end, without any bogus entries (i.e. default scenario = 3 nodes 1 unauthorized)
func TestAccessACT(t *testing.T) {
testAccessACT(t, 0)
}
// TestAccessACTScale tests the creation of the ACT manifest end-to-end, with 1000 bogus entries (i.e. 1000 EC keys + default scenario = 3 nodes 1 unauthorized = 1003 keys in the ACT manifest)
func TestAccessACTScale(t *testing.T) {
testAccessACT(t, 1000)
}
// TestAccessACT tests the e2e creation, uploading and downloading of an ACT access control with both EC keys AND password protection
// the test fires up a 3 node cluster, then randomly picks 2 nodes which will be acting as grantees to the data
// set and also protects the ACT with a password. the third node should fail decoding the reference as it will not be granted access.
// the third node then then tries to download using a correct password (and succeeds) then uses a wrong password and fails.
// the publisher uploads through one of the nodes then disappears.
func testAccessACT(t *testing.T, bogusEntries int) {
// Setup Swarm and upload a test file to it
const clusterSize = 3
cluster := newTestCluster(t, clusterSize)
defer cluster.Shutdown()
var uploadThroughNode = cluster.Nodes[0]
client := swarm.NewClient(uploadThroughNode.URL)
r1 := gorand.New(gorand.NewSource(time.Now().UnixNano()))
nodeToSkip := r1.Intn(clusterSize) // a number between 0 and 2 (node indices in `cluster`)
dataFilename := testutil.TempFileWithContent(t, data)
defer os.RemoveAll(dataFilename)
// upload the file with 'swarm up' and expect a hash
up := runSwarm(t,
"--bzzapi",
cluster.Nodes[0].URL,
"up",
"--encrypt",
dataFilename)
_, matches := up.ExpectRegexp(hashRegexp)
up.ExpectExit()
if len(matches) < 1 {
t.Fatal("no matches found")
}
ref := matches[0]
grantees := []string{}
for i, v := range cluster.Nodes {
if i == nodeToSkip {
continue
}
pk := v.PrivateKey
granteePubKey := crypto.CompressPubkey(&pk.PublicKey)
grantees = append(grantees, hex.EncodeToString(granteePubKey))
}
if bogusEntries > 0 {
bogusGrantees := []string{}
for i := 0; i < bogusEntries; i++ {
prv, err := ecies.GenerateKey(rand.Reader, DefaultCurve, nil)
if err != nil {
t.Fatal(err)
}
bogusGrantees = append(bogusGrantees, hex.EncodeToString(crypto.CompressPubkey(&prv.ExportECDSA().PublicKey)))
}
r2 := gorand.New(gorand.NewSource(time.Now().UnixNano()))
for i := 0; i < len(grantees); i++ {
insertAtIdx := r2.Intn(len(bogusGrantees))
bogusGrantees = append(bogusGrantees[:insertAtIdx], append([]string{grantees[i]}, bogusGrantees[insertAtIdx:]...)...)
}
grantees = bogusGrantees
}
granteesPubkeyListFile := testutil.TempFileWithContent(t, strings.Join(grantees, "\n"))
defer os.RemoveAll(granteesPubkeyListFile)
publisherDir, err := ioutil.TempDir("", "swarm-account-dir-temp")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(publisherDir)
passwordFilename := testutil.TempFileWithContent(t, testPassphrase)
defer os.RemoveAll(passwordFilename)
actPasswordFilename := testutil.TempFileWithContent(t, "smth")
defer os.RemoveAll(actPasswordFilename)
_, publisherAccount := getTestAccount(t, publisherDir)
up = runSwarm(t,
"--bzzaccount",
publisherAccount.Address.String(),
"--password",
passwordFilename,
"--datadir",
publisherDir,
"--bzzapi",
cluster.Nodes[0].URL,
"access",
"new",
"act",
"--grant-keys",
granteesPubkeyListFile,
"--password",
actPasswordFilename,
ref,
)
_, matches = up.ExpectRegexp(`[a-f\d]{64}`)
up.ExpectExit()
if len(matches) == 0 {
t.Fatalf("stdout not matched")
}
//get the public key from the publisher directory
publicKeyFromDataDir := runSwarm(t,
"--bzzaccount",
publisherAccount.Address.String(),
"--password",
passwordFilename,
"--datadir",
publisherDir,
"print-keys",
"--compressed",
)
_, publicKeyString := publicKeyFromDataDir.ExpectRegexp(".+")
publicKeyFromDataDir.ExpectExit()
pkComp := strings.Split(publicKeyString[0], "=")[1]
hash := matches[0]
m, _, err := client.DownloadManifest(hash)
if err != nil {
t.Fatalf("unmarshal manifest: %v", err)
}
if len(m.Entries) != 1 {
t.Fatalf("expected one manifest entry, got %v", len(m.Entries))
}
e := m.Entries[0]
ct := "application/bzz-manifest+json"
if e.ContentType != ct {
t.Errorf("expected %q content type, got %q", ct, e.ContentType)
}
if e.Access == nil {
t.Fatal("manifest access is nil")
}
a := e.Access
if a.Type != "act" {
t.Fatalf(`got access type %q, expected "act"`, a.Type)
}
if len(a.Salt) < 32 {
t.Fatalf(`got salt with length %v, expected not less the 32 bytes`, len(a.Salt))
}
if a.Publisher != pkComp {
t.Fatal("publisher key did not match")
}
httpClient := &http.Client{}
// all nodes except the skipped node should be able to decrypt the content
for i, node := range cluster.Nodes {
log.Debug("trying to fetch from node", "node index", i)
url := node.URL + "/" + "bzz:/" + hash
response, err := httpClient.Get(url)
if err != nil {
t.Fatal(err)
}
log.Debug("got response from node", "response code", response.StatusCode)
if i == nodeToSkip {
log.Debug("reached node to skip", "status code", response.StatusCode)
if response.StatusCode != http.StatusUnauthorized {
t.Fatalf("should be a 401")
}
// try downloading using a password instead, using the unauthorized node
passwordUrl := strings.Replace(url, "http://", "http://:smth@", -1)
response, err = httpClient.Get(passwordUrl)
if err != nil {
t.Fatal(err)
}
if response.StatusCode != http.StatusOK {
t.Fatal("should be a 200")
}
// now try with the wrong password, expect 401
passwordUrl = strings.Replace(url, "http://", "http://:smthWrong@", -1)
response, err = httpClient.Get(passwordUrl)
if err != nil {
t.Fatal(err)
}
if response.StatusCode != http.StatusUnauthorized {
t.Fatal("should be a 401")
}
continue
}
if response.StatusCode != http.StatusOK {
t.Fatal("should be a 200")
}
d, err := ioutil.ReadAll(response.Body)
if err != nil {
t.Fatal(err)
}
if string(d) != data {
t.Errorf("expected decrypted data %q, got %q", data, string(d))
}
}
}
// TestKeypairSanity is a sanity test for the crypto scheme for ACT. it asserts the correct shared secret according to
// the specs at https://github.com/ethersphere/swarm-docs/blob/eb857afda906c6e7bb90d37f3f334ccce5eef230/act.md
func TestKeypairSanity(t *testing.T) {
salt := make([]byte, 32)
if _, err := io.ReadFull(rand.Reader, salt); err != nil {
t.Fatalf("reading from crypto/rand failed: %v", err.Error())
}
sharedSecret := "a85586744a1ddd56a7ed9f33fa24f40dd745b3a941be296a0d60e329dbdb896d"
for i, v := range []struct {
publisherPriv string
granteePub string
}{
{
publisherPriv: "ec5541555f3bc6376788425e9d1a62f55a82901683fd7062c5eddcc373a73459",
granteePub: "0226f213613e843a413ad35b40f193910d26eb35f00154afcde9ded57479a6224a",
},
{
publisherPriv: "70c7a73011aa56584a0009ab874794ee7e5652fd0c6911cd02f8b6267dd82d2d",
granteePub: "02e6f8d5e28faaa899744972bb847b6eb805a160494690c9ee7197ae9f619181db",
},
} {
b, _ := hex.DecodeString(v.granteePub)
granteePub, _ := crypto.DecompressPubkey(b)
publisherPrivate, _ := crypto.HexToECDSA(v.publisherPriv)
ssKey, err := api.NewSessionKeyPK(publisherPrivate, granteePub, salt)
if err != nil {
t.Fatal(err)
}
hasher := sha3.NewKeccak256()
hasher.Write(salt)
shared, err := hex.DecodeString(sharedSecret)
if err != nil {
t.Fatal(err)
}
hasher.Write(shared)
sum := hasher.Sum(nil)
if !bytes.Equal(ssKey, sum) {
t.Fatalf("%d: got a session key mismatch", i)
}
}
}

77
cmd/swarm/bootnodes.go Normal file
View File

@ -0,0 +1,77 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
var SwarmBootnodes = []string{
// Foundation Swarm Gateway Cluster
"enode://e5c6f9215c919a5450a7b8c14c22535607b69f2c8e1e7f6f430cb25d7a2c27cd1df4c4f18ad7c1d7e5162e271ffcd3f20b1a1467fb6e790e7d727f3b2193de97@52.232.7.187:30399",
"enode://9b2fe07e69ccc7db5fef15793dab7d7d2e697ed92132d6e9548218e68a34613a8671ad03a6658d862b468ed693cae8a0f8f8d37274e4a657ffb59ca84676e45b@52.232.7.187:30400",
"enode://76c1059162c93ef9df0f01097c824d17c492634df211ef4c806935b349082233b63b90c23970254b3b7138d630400f7cf9b71e80355a446a8b733296cb04169a@52.232.7.187:30401",
"enode://ce46bbe2a8263145d65252d52da06e000ad350ed09c876a71ea9544efa42f63c1e1b6cc56307373aaad8f9dd069c90d0ed2dd1530106200e16f4ca681dd8ae2d@52.232.7.187:30402",
"enode://f431e0d6008a6c35c6e670373d828390c8323e53da8158e7bfc43cf07e632cc9e472188be8df01decadea2d4a068f1428caba769b632554a8fb0607bc296988f@52.232.7.187:30403",
"enode://174720abfff83d7392f121108ae50ea54e04889afe020df883655c0f6cb95414db945a0228d8982fe000d86fc9f4b7669161adc89cd7cd56f78f01489ab2b99b@52.232.7.187:30404",
"enode://2ae89be4be61a689b6f9ecee4360a59e185e010ab750f14b63b4ae43d4180e872e18e3437d4386ce44875dc7cc6eb761acba06412fe3178f3dac1dab3b65703e@52.232.7.187:30405",
"enode://24abebe1c0e6d75d6052ce3219a87be8573fd6397b4cb51f0773b83abba9b3d872bfb273cdc07389715b87adfac02f5235f5241442c5089802cbd8d42e310fce@52.232.7.187:30406",
"enode://d08dfa46bfbbdbcaafbb6e34abee4786610f6c91e0b76d7881f0334ac10dda41d8c1f2b6eedffb4493293c335c0ad46776443b2208d1fbbb9e1a90b25ee4eef2@52.232.7.187:30407",
"enode://8d95eb0f837d27581a43668ed3b8783d69dc4e84aa3edd7a0897e026155c8f59c8702fdc0375ee7bac15757c9c78e1315d9b73e4ce59c936db52ea4ae2f501c7@52.232.7.187:30408",
"enode://a5967cc804aebd422baaaba9f06f27c9e695ccab335b61088130f8cbe64e3cdf78793868c7051dfc06eecfe844fad54bc7f6dfaed9db3c7ecef279cb829c25fb@52.232.7.187:30409",
"enode://5f00134d81a8f2ebcc46f8766f627f492893eda48138f811b7de2168308171968f01710bca6da05764e74f14bae41652f554e6321f1aed85fa3461e89d075dbf@52.232.7.187:30410",
"enode://b2142b79b01a5aa66a5e23cc35e78219a8e97bc2412a6698cee24ae02e87078b725d71730711bd62e25ff1aa8658c6633778af8ac14c63814a337c3dd0ebda9f@52.232.7.187:30411",
"enode://1ffa7651094867d6486ce3ef46d27a052c2cb968b618346c6df7040322c7efc3337547ba85d4cbba32e8b31c42c867202554735c06d4c664b9afada2ed0c4b3c@52.232.7.187:30412",
"enode://129e0c3d5f5df12273754f6f703d2424409fa4baa599e0b758c55600169313887855e75b082028d2302ec034b303898cd697cc7ae8256ba924ce927510da2c8d@52.232.7.187:30413",
"enode://419e2dc0d2f5b022cf16b0e28842658284909fa027a0fbbb5e2b755e7f846ea02a8f0b66a7534981edf6a7bcf8a14855344c6668e2cd4476ccd35a11537c9144@52.232.7.187:30414",
"enode://23d55ad900583231b91f2f62e3f72eb498b342afd58b682be3af052eed62b5651094471065981de33d8786f075f05e3cca499503b0ac8ae84b2a06e99f5b0723@52.232.7.187:30415",
"enode://bc56e4158c00e9f616d7ea533def20a89bef959df4e62a768ff238ff4e1e9223f57ecff969941c20921bad98749baae311c0fbebce53bf7bbb9d3dc903640990@52.232.7.187:30416",
"enode://433ce15199c409875e7e72fffd69fdafe746f17b20f0d5555281722a65fde6c80328fab600d37d8624509adc072c445ce0dad4a1c01cff6acf3132c11d429d4d@52.232.7.187:30417",
"enode://632ee95b8f0eac51ef89ceb29313fef3a60050181d66a6b125583b1a225a7694b252edc016efb58aa3b251da756cb73280842a022c658ed405223b2f58626343@52.232.7.187:30418",
"enode://4a0f9bcff7a4b9ee453fb298d0fb222592efe121512e30cd72fef631beb8c6a15153a1456eb073ee18551c0e003c569651a101892dc4124e90b933733a498bb5@52.232.7.187:30419",
"enode://f0d80fbc72d16df30e19aac3051eb56a7aff0c8367686702e01ea132d8b0b3ee00cadd6a859d2cca98ec68d3d574f8a8a87dba2347ec1e2818dc84bc3fa34fae@52.232.7.187:30420",
"enode://a199146906e4f9f2b94b195a8308d9a59a3564b92efaab898a4243fe4c2ad918b7a8e4853d9d901d94fad878270a2669d644591299c3d43de1b298c00b92b4a7@52.232.7.187:30421",
"enode://052036ea8736b37adbfb684d90ce43e11b3591b51f31489d7c726b03618dea4f73b1e659deb928e6bf40564edcdcf08351643f42db3d4ca1c2b5db95dad59e94@52.232.7.187:30422",
"enode://460e2b8c6da8f12fac96c836e7d108f4b7ec55a1c64631bb8992339e117e1c28328fee83af863196e20af1487a655d13e5ceba90e980e92502d5bac5834c1f71@52.232.7.187:30423",
"enode://6d2cdd13741b2e72e9031e1b93c6d9a4e68de2844aa4e939f6a8a8498a7c1d7e2ee4c64217e92a6df08c9a32c6764d173552810ef1bd2ecb356532d389dd2136@52.232.7.187:30424",
"enode://62105fc25ce2cd5b299647f47eaa9211502dc76f0e9f461df915782df7242ac3223e3db04356ae6ed2977ccac20f0b16864406e9ca514a40a004cb6a5d0402aa@52.232.7.187:30425",
"enode://e0e388fc520fd493c33f0ce16685e6f98fb6aec28f2edc14ee6b179594ee519a896425b0025bb6f0e182dd3e468443f19c70885fbc66560d000093a668a86aa8@52.232.7.187:30426",
"enode://63f3353a72521ea10022127a4fe6b4acbef197c3fe668fd9f4805542d8a6fcf79f6335fbab62d180a35e19b739483e740858b113fdd7c13a26ad7b4e318a5aef@52.232.7.187:30427",
"enode://33a42b927085678d4aefd4e70b861cfca6ef5f6c143696c4f755973fd29e64c9e658cad57a66a687a7a156da1e3688b1fbdd17bececff2ee009fff038fa5666b@52.232.7.187:30428",
"enode://259ab5ab5c1daee3eab7e3819ab3177b82d25c29e6c2444fdd3f956e356afae79a72840ccf2d0665fe82c81ebc3b3734da1178ac9fd5d62c67e674b69f86b6be@52.232.7.187:30429",
"enode://558bccad7445ce3fd8db116ed6ab4aed1324fdbdac2348417340c1764dc46d46bffe0728e5b7d5c36f12e794c289f18f57f08f085d2c65c9910a5c7a65b6a66a@52.232.7.187:30430",
"enode://abe60937a0657ffded718e3f84a32987286983be257bdd6004775c4b525747c2b598f4fac49c8de324de5ce75b22673fa541a7ce2d555fb7f8ca325744ae3577@52.232.7.187:30431",
"enode://bce6f0aaa5b230742680084df71d4f026b3eff7f564265599216a1b06b765303fdc9325de30ffd5dfdaf302ce4b14322891d2faea50ce2ca298d7409f5858339@52.232.7.187:30432",
"enode://21b957c4e03277d42be6660730ec1b93f540764f26c6abdb54d006611139c7081248486206dfbf64fcaffd62589e9c6b8ea77a5297e4b21a605f1bcf49483ed0@52.232.7.187:30433",
"enode://ff104e30e64f24c3d7328acee8b13354e5551bc8d60bb25ecbd9632d955c7e34bb2d969482d173355baad91c8282f8b592624eb3929151090da3b4448d4d58fb@52.232.7.187:30434",
"enode://c76e2b5f81a521bceaec1518926a21380a345df9cf463461562c6845795512497fb67679e155fc96a74350f8b78de8f4c135dd52b106dbbb9795452021d09ea5@52.232.7.187:30435",
"enode://3288fd860105164f3e9b69934c4eb18f7146cfab31b5a671f994e21a36e9287766e5f9f075aefbc404538c77f7c2eb2a4495020a7633a1c3970d94e9fa770aeb@52.232.7.187:30436",
"enode://6cea859c7396d46b20cfcaa80f9a11cd112f8684f2f782f7b4c0e1e0af9212113429522075101923b9b957603e6c32095a6a07b5e5e35183c521952ee108dfaf@52.232.7.187:30437",
"enode://f628ec56e4ca8317cc24cc4ac9b27b95edcce7b96e1c7f3b53e30de4a8580fe44f2f0694a513bdb0a431acaf2824074d6ace4690247bbc34c14f426af8c056ea@52.232.7.187:30438",
"enode://055ec8b26fc105c4f97970a1cce9773a5e34c03f511b839db742198a1c571e292c54aa799e9afb991cc8a560529b8cdf3e0c344bc6c282aff2f68eec59361ddf@52.232.7.187:30439",
"enode://48cb0d430c328974226aa33a931d8446cd5a8d40f3ead8f4ce7ad60faa1278192eb6d58bed91258d63e81f255fc107eec2425ce2ae8b22350dd556076e160610@52.232.7.187:30440",
"enode://3fadb7af7f770d5ffc6b073b8d42834bebb18ce1fe8a4fe270d2b799e7051327093960dc61d9a18870db288f7746a0e6ea2a013cd6ab0e5f97ca08199473aace@52.232.7.187:30441",
"enode://a5d7168024c9992769cf380ffa559a64b4f39a29d468f579559863814eb0ae0ed689ac0871a3a2b4c78b03297485ec322d578281131ef5d5c09a4beb6200a97a@52.232.7.187:30442",
"enode://9c57744c5b2c2d71abcbe80512652f9234d4ab041b768a2a886ab390fe6f184860f40e113290698652d7e20a8ac74d27ac8671db23eb475b6c5e6253e4693bf8@52.232.7.187:30443",
"enode://daca9ff0c3176045a0e0ed228dee00ec86bc0939b135dc6b1caa23745d20fd0332e1ee74ad04020e89df56c7146d831a91b89d15ca3df05ba7618769fefab376@52.232.7.187:30444",
"enode://a3f6af59428cb4b9acb198db15ef5554fa43c2b0c18e468a269722d64a27218963a2975eaf82750b6262e42192b5e3669ea51337b4cda62b33987981bc5e0c1a@52.232.7.187:30445",
"enode://fe571422fa4651c3354c85dac61911a6a6520dd3c0332967a49d4133ca30e16a8a4946fa73ca2cb5de77917ea701a905e1c3015b2f4defcd53132b61cc84127a@52.232.7.187:30446",
// Mainframe
"enode://ee9a5a571ea6c8a59f9a8bb2c569c865e922b41c91d09b942e8c1d4dd2e1725bd2c26149da14de1f6321a2c6fdf1e07c503c3e093fb61696daebf74d6acd916b@54.186.219.160:30399",
"enode://a03f0562ecb8a992ad5242345535e73483cdc18ab934d36bf24b567d43447c2cea68f89f1d51d504dd13acc30f24ebce5a150bea2ccb1b722122ce4271dc199d@52.67.248.147:30399",
"enode://e2cbf9eafd85903d3b1c56743035284320695e0072bc8d7396e0542aa5e1c321b236f67eab66b79c2f15d4447fa4bbe74dd67d0467da23e7eb829f60ec8a812b@13.58.169.1:30399",
"enode://8b8c6bda6047f1cad9fab2db4d3d02b7aa26279902c32879f7bcd4a7d189fee77fdc36ee151ce6b84279b4792e72578fd529d2274d014132465758fbfee51cee@13.209.13.15:30399",
"enode://63f6a8818927e429585287cf2ca0cb9b11fa990b7b9b331c2962cdc6f21807a2473b26e8256225c26caff70d7218e59586d704d49061452c6852e382c885d03c@35.154.106.174:30399",
"enode://ed4bd3b794ed73f18e6dcc70c6624dfec63b5654f6ab54e8f40b16eff8afbd342d4230e099ddea40e84423f81b2d2ea79799dc345257b1fec6f6c422c9d008f7@52.213.20.99:30399",
}

27
cmd/swarm/config.go
View File

@ -38,8 +38,6 @@ import (
bzzapi "github.com/ethereum/go-ethereum/swarm/api"
)
const SWARM_VERSION = "0.3"
var (
//flag definition for the dumpconfig command
DumpConfigCommand = cli.Command{
@ -70,6 +68,7 @@ const (
SWARM_ENV_SWAP_API = "SWARM_SWAP_API"
SWARM_ENV_SYNC_DISABLE = "SWARM_SYNC_DISABLE"
SWARM_ENV_SYNC_UPDATE_DELAY = "SWARM_ENV_SYNC_UPDATE_DELAY"
SWARM_ENV_LIGHT_NODE_ENABLE = "SWARM_LIGHT_NODE_ENABLE"
SWARM_ENV_DELIVERY_SKIP_CHECK = "SWARM_DELIVERY_SKIP_CHECK"
SWARM_ENV_ENS_API = "SWARM_ENS_API"
SWARM_ENV_ENS_ADDR = "SWARM_ENS_ADDR"
@ -79,6 +78,7 @@ const (
SWARM_ENV_STORE_PATH = "SWARM_STORE_PATH"
SWARM_ENV_STORE_CAPACITY = "SWARM_STORE_CAPACITY"
SWARM_ENV_STORE_CACHE_CAPACITY = "SWARM_STORE_CACHE_CAPACITY"
SWARM_ACCESS_PASSWORD = "SWARM_ACCESS_PASSWORD"
GETH_ENV_DATADIR = "GETH_DATADIR"
)
@ -133,7 +133,7 @@ func initSwarmNode(config *bzzapi.Config, stack *node.Node, ctx *cli.Context) {
log.Debug(printConfig(config))
}
//override the current config with whatever is in the config file, if a config file has been provided
//configFileOverride overrides the current config with the config file, if a config file has been provided
func configFileOverride(config *bzzapi.Config, ctx *cli.Context) (*bzzapi.Config, error) {
var err error
@ -143,7 +143,8 @@ func configFileOverride(config *bzzapi.Config, ctx *cli.Context) (*bzzapi.Config
if filepath = ctx.GlobalString(SwarmTomlConfigPathFlag.Name); filepath == "" {
utils.Fatalf("Config file flag provided with invalid file path")
}
f, err := os.Open(filepath)
var f *os.File
f, err = os.Open(filepath)
if err != nil {
return nil, err
}
@ -206,6 +207,10 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con
currentConfig.SyncUpdateDelay = d
}
if ctx.GlobalIsSet(SwarmLightNodeEnabled.Name) {
currentConfig.LightNodeEnabled = true
}
if ctx.GlobalIsSet(SwarmDeliverySkipCheckFlag.Name) {
currentConfig.DeliverySkipCheck = true
}
@ -228,10 +233,6 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con
currentConfig.Cors = cors
}
if ctx.GlobalIsSet(utils.BootnodesFlag.Name) {
currentConfig.BootNodes = ctx.GlobalString(utils.BootnodesFlag.Name)
}
if storePath := ctx.GlobalString(SwarmStorePath.Name); storePath != "" {
currentConfig.LocalStoreParams.ChunkDbPath = storePath
}
@ -303,6 +304,12 @@ func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) {
}
}
if lne := os.Getenv(SWARM_ENV_LIGHT_NODE_ENABLE); lne != "" {
if lightnode, err := strconv.ParseBool(lne); err != nil {
currentConfig.LightNodeEnabled = lightnode
}
}
if swapapi := os.Getenv(SWARM_ENV_SWAP_API); swapapi != "" {
currentConfig.SwapAPI = swapapi
}
@ -323,10 +330,6 @@ func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) {
currentConfig.Cors = cors
}
if bootnodes := os.Getenv(SWARM_ENV_BOOTNODES); bootnodes != "" {
currentConfig.BootNodes = bootnodes
}
return currentConfig
}

14
cmd/swarm/config_test.go
View File

@ -20,6 +20,7 @@ import (
"fmt"
"io"
"io/ioutil"
"net"
"os"
"os/exec"
"testing"
@ -559,3 +560,16 @@ func TestValidateConfig(t *testing.T) {
}
}
}
func assignTCPPort() (string, error) {
l, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
return "", err
}
l.Close()
_, port, err := net.SplitHostPort(l.Addr().String())
if err != nil {
return "", err
}
return port, nil
}

40
cmd/swarm/download.go
View File

@ -68,18 +68,36 @@ func download(ctx *cli.Context) {
utils.Fatalf("could not parse uri argument: %v", err)
}
// assume behaviour according to --recursive switch
if isRecursive {
if err := client.DownloadDirectory(uri.Addr, uri.Path, dest); err != nil {
utils.Fatalf("encoutered an error while downloading directory: %v", err)
}
} else {
// we are downloading a file
log.Debug(fmt.Sprintf("downloading file/path from a manifest. hash: %s, path:%s", uri.Addr, uri.Path))
dl := func(credentials string) error {
// assume behaviour according to --recursive switch
if isRecursive {
if err := client.DownloadDirectory(uri.Addr, uri.Path, dest, credentials); err != nil {
if err == swarm.ErrUnauthorized {
return err
}
return fmt.Errorf("directory %s: %v", uri.Path, err)
}
} else {
// we are downloading a file
log.Debug("downloading file/path from a manifest", "uri.Addr", uri.Addr, "uri.Path", uri.Path)
err := client.DownloadFile(uri.Addr, uri.Path, dest)
if err != nil {
utils.Fatalf("could not download %s from given address: %s. error: %v", uri.Path, uri.Addr, err)
err := client.DownloadFile(uri.Addr, uri.Path, dest, credentials)
if err != nil {
if err == swarm.ErrUnauthorized {
return err
}
return fmt.Errorf("file %s from address: %s: %v", uri.Path, uri.Addr, err)
}
}
return nil
}
if passwords := makePasswordList(ctx); passwords != nil {
password := getPassPhrase(fmt.Sprintf("Downloading %s is restricted", uri), 0, passwords)
err = dl(password)
} else {
err = dl("")
}
if err != nil {
utils.Fatalf("download: %v", err)
}
}

2
cmd/swarm/fs.go
View File

@ -92,7 +92,7 @@ func listMounts(cliContext *cli.Context) {
mf := []fuse.MountInfo{}
err = client.CallContext(ctx, &mf, "swarmfs_listmounts")
if err != nil {
utils.Fatalf("encountered an error calling the RPC endpoint while unmounting: %v", err)
utils.Fatalf("encountered an error calling the RPC endpoint while listing mounts: %v", err)
}
if len(mf) == 0 {
fmt.Print("Could not found any swarmfs mounts. Please make sure you've specified the correct RPC endpoint\n")

7
cmd/swarm/fs_test.go
View File

@ -14,6 +14,8 @@
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
// +build linux freebsd
package main
import (
@ -41,6 +43,11 @@ type testFile struct {
}
// TestCLISwarmFs is a high-level test of swarmfs
//
// This test fails on travis for macOS as this executable exits with code 1
// and without any log messages in the log:
// /Library/Filesystems/osxfuse.fs/Contents/Resources/load_osxfuse.
// This is the reason for this file not being built on darwin architecture.
func TestCLISwarmFs(t *testing.T) {
cluster := newTestCluster(t, 3)
defer cluster.Shutdown()

5
cmd/swarm/hash.go
View File

@ -18,6 +18,7 @@
package main
import (
"context"
"fmt"
"os"
@ -38,8 +39,8 @@ func hash(ctx *cli.Context) {
defer f.Close()
stat, _ := f.Stat()
fileStore := storage.NewFileStore(storage.NewMapChunkStore(), storage.NewFileStoreParams())
addr, _, err := fileStore.Store(f, stat.Size(), false)
fileStore := storage.NewFileStore(&storage.FakeChunkStore{}, storage.NewFileStoreParams())
addr, _, err := fileStore.Store(context.TODO(), f, stat.Size(), false)
if err != nil {
utils.Fatalf("%v\n", err)
} else {

2
cmd/swarm/list.go
View File

@ -44,7 +44,7 @@ func list(ctx *cli.Context) {
bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client := swarm.NewClient(bzzapi)
list, err := client.List(manifest, prefix)
list, err := client.List(manifest, prefix, "")
if err != nil {
utils.Fatalf("Failed to generate file and directory list: %s", err)
}

289
cmd/swarm/main.go
View File

@ -18,6 +18,7 @@ package main
import (
"crypto/ecdsa"
"encoding/hex"
"fmt"
"io/ioutil"
"os"
@ -37,12 +38,12 @@ import (
"github.com/ethereum/go-ethereum/internal/debug"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/node"
"github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/p2p/discover"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/swarm"
bzzapi "github.com/ethereum/go-ethereum/swarm/api"
swarmmetrics "github.com/ethereum/go-ethereum/swarm/metrics"
"github.com/ethereum/go-ethereum/swarm/tracing"
sv "github.com/ethereum/go-ethereum/swarm/version"
"gopkg.in/urfave/cli.v1"
)
@ -66,14 +67,7 @@ OPTIONS:
`
var (
gitCommit string // Git SHA1 commit hash of the release (set via linker flags)
testbetBootNodes = []string{
"enode://ec8ae764f7cb0417bdfb009b9d0f18ab3818a3a4e8e7c67dd5f18971a93510a2e6f43cd0b69a27e439a9629457ea804104f37c85e41eed057d3faabbf7744cdf@13.74.157.139:30429",
"enode://c2e1fceb3bf3be19dff71eec6cccf19f2dbf7567ee017d130240c670be8594bc9163353ca55dd8df7a4f161dd94b36d0615c17418b5a3cdcbb4e9d99dfa4de37@13.74.157.139:30430",
"enode://fe29b82319b734ce1ec68b84657d57145fee237387e63273989d354486731e59f78858e452ef800a020559da22dcca759536e6aa5517c53930d29ce0b1029286@13.74.157.139:30431",
"enode://1d7187e7bde45cf0bee489ce9852dd6d1a0d9aa67a33a6b8e6db8a4fbc6fcfa6f0f1a5419343671521b863b187d1c73bad3603bae66421d157ffef357669ddb8@13.74.157.139:30432",
"enode://0e4cba800f7b1ee73673afa6a4acead4018f0149d2e3216be3f133318fd165b324cd71b81fbe1e80deac8dbf56e57a49db7be67f8b9bc81bd2b7ee496434fb5d@13.74.157.139:30433",
}
gitCommit string // Git SHA1 commit hash of the release (set via linker flags)
)
var (
@ -122,6 +116,11 @@ var (
Usage: "Duration for sync subscriptions update after no new peers are added (default 15s)",
EnvVar: SWARM_ENV_SYNC_UPDATE_DELAY,
}
SwarmLightNodeEnabled = cli.BoolFlag{
Name: "lightnode",
Usage: "Enable Swarm LightNode (default false)",
EnvVar: SWARM_ENV_LIGHT_NODE_ENABLE,
}
SwarmDeliverySkipCheckFlag = cli.BoolFlag{
Name: "delivery-skip-check",
Usage: "Skip chunk delivery check (default false)",
@ -143,24 +142,41 @@ var (
}
SwarmWantManifestFlag = cli.BoolTFlag{
Name: "manifest",
Usage: "Automatic manifest upload",
Usage: "Automatic manifest upload (default true)",
}
SwarmUploadDefaultPath = cli.StringFlag{
Name: "defaultpath",
Usage: "path to file served for empty url path (none)",
}
SwarmAccessGrantKeyFlag = cli.StringFlag{
Name: "grant-key",
Usage: "grants a given public key access to an ACT",
}
SwarmAccessGrantKeysFlag = cli.StringFlag{
Name: "grant-keys",
Usage: "grants a given list of public keys in the following file (separated by line breaks) access to an ACT",
}
SwarmUpFromStdinFlag = cli.BoolFlag{
Name: "stdin",
Usage: "reads data to be uploaded from stdin",
}
SwarmUploadMimeType = cli.StringFlag{
Name: "mime",
Usage: "force mime type",
Usage: "Manually specify MIME type",
}
SwarmEncryptedFlag = cli.BoolFlag{
Name: "encrypt",
Usage: "use encrypted upload",
}
SwarmAccessPasswordFlag = cli.StringFlag{
Name: "password",
Usage: "Password",
EnvVar: SWARM_ACCESS_PASSWORD,
}
SwarmDryRunFlag = cli.BoolFlag{
Name: "dry-run",
Usage: "dry-run",
}
CorsStringFlag = cli.StringFlag{
Name: "corsdomain",
Usage: "Domain on which to send Access-Control-Allow-Origin header (multiple domains can be supplied separated by a ',')",
@ -181,6 +197,22 @@ var (
Usage: "Number of recent chunks cached in memory (default 5000)",
EnvVar: SWARM_ENV_STORE_CACHE_CAPACITY,
}
SwarmResourceMultihashFlag = cli.BoolFlag{
Name: "multihash",
Usage: "Determines how to interpret data for a resource update. If not present, data will be interpreted as raw, literal data that will be included in the resource",
}
SwarmResourceNameFlag = cli.StringFlag{
Name: "name",
Usage: "User-defined name for the new resource",
}
SwarmResourceDataOnCreateFlag = cli.StringFlag{
Name: "data",
Usage: "Initializes the resource with the given hex-encoded data. Data must be prefixed by 0x",
}
SwarmCompressedFlag = cli.BoolFlag{
Name: "compressed",
Usage: "Prints encryption keys in compressed form",
}
)
//declare a few constant error messages, useful for later error check comparisons in test
@ -189,12 +221,21 @@ var (
SWARM_ERR_SWAP_SET_NO_API = "SWAP is enabled but --swap-api is not set"
)
// this help command gets added to any subcommand that does not define it explicitly
var defaultSubcommandHelp = cli.Command{
Action: func(ctx *cli.Context) { cli.ShowCommandHelpAndExit(ctx, "", 1) },
CustomHelpTemplate: helpTemplate,
Name: "help",
Usage: "shows this help",
Hidden: true,
}
var defaultNodeConfig = node.DefaultConfig
// This init function sets defaults so cmd/swarm can run alongside geth.
func init() {
defaultNodeConfig.Name = clientIdentifier
defaultNodeConfig.Version = params.VersionWithCommit(gitCommit)
defaultNodeConfig.Version = sv.VersionWithCommit(gitCommit)
defaultNodeConfig.P2P.ListenAddr = ":30399"
defaultNodeConfig.IPCPath = "bzzd.ipc"
// Set flag defaults for --help display.
@ -216,6 +257,14 @@ func init() {
Usage: "Print version numbers",
Description: "The output of this command is supposed to be machine-readable",
},
{
Action: keys,
CustomHelpTemplate: helpTemplate,
Name: "print-keys",
Flags: []cli.Flag{SwarmCompressedFlag},
Usage: "Print public key information",
Description: "The output of this command is supposed to be machine-readable",
},
{
Action: upload,
CustomHelpTemplate: helpTemplate,
@ -225,6 +274,97 @@ func init() {
Flags: []cli.Flag{SwarmEncryptedFlag},
Description: "uploads a file or directory to swarm using the HTTP API and prints the root hash",
},
{
CustomHelpTemplate: helpTemplate,
Name: "access",
Usage: "encrypts a reference and embeds it into a root manifest",
ArgsUsage: "<ref>",
Description: "encrypts a reference and embeds it into a root manifest",
Subcommands: []cli.Command{
{
CustomHelpTemplate: helpTemplate,
Name: "new",
Usage: "encrypts a reference and embeds it into a root manifest",
ArgsUsage: "<ref>",
Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
Subcommands: []cli.Command{
{
Action: accessNewPass,
CustomHelpTemplate: helpTemplate,
Flags: []cli.Flag{
utils.PasswordFileFlag,
SwarmDryRunFlag,
},
Name: "pass",
Usage: "encrypts a reference with a password and embeds it into a root manifest",
ArgsUsage: "<ref>",
Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
},
{
Action: accessNewPK,
CustomHelpTemplate: helpTemplate,
Flags: []cli.Flag{
utils.PasswordFileFlag,
SwarmDryRunFlag,
SwarmAccessGrantKeyFlag,
},
Name: "pk",
Usage: "encrypts a reference with the node's private key and a given grantee's public key and embeds it into a root manifest",
ArgsUsage: "<ref>",
Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
},
{
Action: accessNewACT,
CustomHelpTemplate: helpTemplate,
Flags: []cli.Flag{
SwarmAccessGrantKeysFlag,
SwarmDryRunFlag,
utils.PasswordFileFlag,
},
Name: "act",
Usage: "encrypts a reference with the node's private key and a given grantee's public key and embeds it into a root manifest",
ArgsUsage: "<ref>",
Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
},
},
},
},
},
{
CustomHelpTemplate: helpTemplate,
Name: "resource",
Usage: "(Advanced) Create and update Mutable Resources",
ArgsUsage: "<create|update|info>",
Description: "Works with Mutable Resource Updates",
Subcommands: []cli.Command{
{
Action: resourceCreate,
CustomHelpTemplate: helpTemplate,
Name: "create",
Usage: "creates a new Mutable Resource",
ArgsUsage: "<frequency>",
Description: "creates a new Mutable Resource",
Flags: []cli.Flag{SwarmResourceNameFlag, SwarmResourceDataOnCreateFlag, SwarmResourceMultihashFlag},
},
{
Action: resourceUpdate,
CustomHelpTemplate: helpTemplate,
Name: "update",
Usage: "updates the content of an existing Mutable Resource",
ArgsUsage: "<Manifest Address or ENS domain> <0x Hex data>",
Description: "updates the content of an existing Mutable Resource",
Flags: []cli.Flag{SwarmResourceMultihashFlag},
},
{
Action: resourceInfo,
CustomHelpTemplate: helpTemplate,
Name: "info",
Usage: "obtains information about an existing Mutable Resource",
ArgsUsage: "<Manifest Address or ENS domain>",
Description: "obtains information about an existing Mutable Resource",
},
},
},
{
Action: list,
CustomHelpTemplate: helpTemplate,
@ -242,16 +382,13 @@ func init() {
Description: "Prints the swarm hash of file or directory",
},
{
Action: download,
Name: "down",
Flags: []cli.Flag{SwarmRecursiveFlag},
Usage: "downloads a swarm manifest or a file inside a manifest",
ArgsUsage: " <uri> [<dir>]",
Description: `
Downloads a swarm bzz uri to the given dir. When no dir is provided, working directory is assumed. --recursive flag is expected when downloading a manifest with multiple entries.
`,
Action: download,
Name: "down",
Flags: []cli.Flag{SwarmRecursiveFlag, SwarmAccessPasswordFlag},
Usage: "downloads a swarm manifest or a file inside a manifest",
ArgsUsage: " <uri> [<dir>]",
Description: `Downloads a swarm bzz uri to the given dir. When no dir is provided, working directory is assumed. --recursive flag is expected when downloading a manifest with multiple entries.`,
},
{
Name: "manifest",
CustomHelpTemplate: helpTemplate,
@ -260,23 +397,23 @@ Downloads a swarm bzz uri to the given dir. When no dir is provided, working dir
Description: "Updates a MANIFEST by adding/removing/updating the hash of a path.\nCOMMAND could be: add, update, remove",
Subcommands: []cli.Command{
{
Action: add,
Action: manifestAdd,
CustomHelpTemplate: helpTemplate,
Name: "add",
Usage: "add a new path to the manifest",
ArgsUsage: "<MANIFEST> <path> <hash> [<content-type>]",
ArgsUsage: "<MANIFEST> <path> <hash>",
Description: "Adds a new path to the manifest",
},
{
Action: update,
Action: manifestUpdate,
CustomHelpTemplate: helpTemplate,
Name: "update",
Usage: "update the hash for an already existing path in the manifest",
ArgsUsage: "<MANIFEST> <path> <newhash> [<newcontent-type>]",
ArgsUsage: "<MANIFEST> <path> <newhash>",
Description: "Update the hash for an already existing path in the manifest",
},
{
Action: remove,
Action: manifestRemove,
CustomHelpTemplate: helpTemplate,
Name: "remove",
Usage: "removes a path from the manifest",
@ -351,16 +488,14 @@ pv(1) tool to get a progress bar:
Name: "import",
Usage: "import chunks from a tar archive into a local chunk database (use - to read from stdin)",
ArgsUsage: "<chunkdb> <file>",
Description: `
Import chunks from a tar archive into a local chunk database (use - to read from stdin).
Description: `Import chunks from a tar archive into a local chunk database (use - to read from stdin).
swarm db import ~/.ethereum/swarm/bzz-KEY/chunks chunks.tar
The import may be quite large, consider piping the input through the Unix
pv(1) tool to get a progress bar:
pv chunks.tar | swarm db import ~/.ethereum/swarm/bzz-KEY/chunks -
`,
pv chunks.tar | swarm db import ~/.ethereum/swarm/bzz-KEY/chunks -`,
},
{
Action: dbClean,
@ -376,6 +511,11 @@ pv(1) tool to get a progress bar:
// See config.go
DumpConfigCommand,
}
// append a hidden help subcommand to all commands that have subcommands
// if a help command was already defined above, that one will take precedence.
addDefaultHelpSubcommands(app.Commands)
sort.Sort(cli.CommandsByName(app.Commands))
app.Flags = []cli.Flag{
@ -402,6 +542,7 @@ pv(1) tool to get a progress bar:
SwarmSwapAPIFlag,
SwarmSyncDisabledFlag,
SwarmSyncUpdateDelay,
SwarmLightNodeEnabled,
SwarmDeliverySkipCheckFlag,
SwarmListenAddrFlag,
SwarmPortFlag,
@ -430,12 +571,14 @@ pv(1) tool to get a progress bar:
app.Flags = append(app.Flags, rpcFlags...)
app.Flags = append(app.Flags, debug.Flags...)
app.Flags = append(app.Flags, swarmmetrics.Flags...)
app.Flags = append(app.Flags, tracing.Flags...)
app.Before = func(ctx *cli.Context) error {
runtime.GOMAXPROCS(runtime.NumCPU())
if err := debug.Setup(ctx); err != nil {
if err := debug.Setup(ctx, ""); err != nil {
return err
}
swarmmetrics.Setup(ctx)
tracing.Setup(ctx)
return nil
}
app.After = func(ctx *cli.Context) error {
@ -451,8 +594,20 @@ func main() {
}
}
func keys(ctx *cli.Context) error {
privateKey := getPrivKey(ctx)
pub := hex.EncodeToString(crypto.FromECDSAPub(&privateKey.PublicKey))
pubCompressed := hex.EncodeToString(crypto.CompressPubkey(&privateKey.PublicKey))
if !ctx.Bool(SwarmCompressedFlag.Name) {
fmt.Println(fmt.Sprintf("publicKey=%s", pub))
}
fmt.Println(fmt.Sprintf("publicKeyCompressed=%s", pubCompressed))
return nil
}
func version(ctx *cli.Context) error {
fmt.Println("Version:", SWARM_VERSION)
fmt.Println(strings.Title(clientIdentifier))
fmt.Println("Version:", sv.VersionWithMeta)
if gitCommit != "" {
fmt.Println("Git Commit:", gitCommit)
}
@ -464,6 +619,7 @@ func version(ctx *cli.Context) error {
func bzzd(ctx *cli.Context) error {
//build a valid bzzapi.Config from all available sources:
//default config, file config, command line and env vars
bzzconfig, err := buildConfig(ctx)
if err != nil {
utils.Fatalf("unable to configure swarm: %v", err)
@ -480,12 +636,16 @@ func bzzd(ctx *cli.Context) error {
if _, err := os.Stat(bzzconfig.Path); err == nil {
cfg.DataDir = bzzconfig.Path
}
//optionally set the bootnodes before configuring the node
setSwarmBootstrapNodes(ctx, &cfg)
//setup the ethereum node
utils.SetNodeConfig(ctx, &cfg)
stack, err := node.New(&cfg)
if err != nil {
utils.Fatalf("can't create node: %v", err)
}
//a few steps need to be done after the config phase is completed,
//due to overriding behavior
initSwarmNode(bzzconfig, stack, ctx)
@ -503,16 +663,6 @@ func bzzd(ctx *cli.Context) error {
stack.Stop()
}()
// Add bootnodes as initial peers.
if bzzconfig.BootNodes != "" {
bootnodes := strings.Split(bzzconfig.BootNodes, ",")
injectBootnodes(stack.Server(), bootnodes)
} else {
if bzzconfig.NetworkID == 3 {
injectBootnodes(stack.Server(), testbetBootNodes)
}
}
stack.Wait()
return nil
}
@ -546,6 +696,26 @@ func getAccount(bzzaccount string, ctx *cli.Context, stack *node.Node) *ecdsa.Pr
return decryptStoreAccount(ks, bzzaccount, utils.MakePasswordList(ctx))
}
// getPrivKey returns the private key of the specified bzzaccount
// Used only by client commands, such as `resource`
func getPrivKey(ctx *cli.Context) *ecdsa.PrivateKey {
// booting up the swarm node just as we do in bzzd action
bzzconfig, err := buildConfig(ctx)
if err != nil {
utils.Fatalf("unable to configure swarm: %v", err)
}
cfg := defaultNodeConfig
if _, err := os.Stat(bzzconfig.Path); err == nil {
cfg.DataDir = bzzconfig.Path
}
utils.SetNodeConfig(ctx, &cfg)
stack, err := node.New(&cfg)
if err != nil {
utils.Fatalf("can't create node: %v", err)
}
return getAccount(bzzconfig.BzzAccount, ctx, stack)
}
func decryptStoreAccount(ks *keystore.KeyStore, account string, passwords []string) *ecdsa.PrivateKey {
var a accounts.Account
var err error
@ -600,13 +770,32 @@ func getPassPhrase(prompt string, i int, passwords []string) string {
return password
}
func injectBootnodes(srv *p2p.Server, nodes []string) {
for _, url := range nodes {
n, err := discover.ParseNode(url)
if err != nil {
log.Error("Invalid swarm bootnode", "err", err)
continue
// addDefaultHelpSubcommand scans through defined CLI commands and adds
// a basic help subcommand to each
// if a help command is already defined, it will take precedence over the default.
func addDefaultHelpSubcommands(commands []cli.Command) {
for i := range commands {
cmd := &commands[i]
if cmd.Subcommands != nil {
cmd.Subcommands = append(cmd.Subcommands, defaultSubcommandHelp)
addDefaultHelpSubcommands(cmd.Subcommands)
}
srv.AddPeer(n)
}
}
func setSwarmBootstrapNodes(ctx *cli.Context, cfg *node.Config) {
if ctx.GlobalIsSet(utils.BootnodesFlag.Name) || ctx.GlobalIsSet(utils.BootnodesV4Flag.Name) {
return
}
cfg.P2P.BootstrapNodes = []*discover.Node{}
for _, url := range SwarmBootnodes {
node, err := discover.ParseNode(url)
if err != nil {
log.Error("Bootstrap URL invalid", "enode", url, "err", err)
}
cfg.P2P.BootstrapNodes = append(cfg.P2P.BootstrapNodes, node)
}
log.Debug("added default swarm bootnodes", "length", len(cfg.P2P.BootstrapNodes))
}

234
cmd/swarm/manifest.go
View File

@ -18,10 +18,8 @@
package main
import (
"encoding/json"
"fmt"
"mime"
"path/filepath"
"os"
"strings"
"github.com/ethereum/go-ethereum/cmd/utils"
@ -30,127 +28,118 @@ import (
"gopkg.in/urfave/cli.v1"
)
const bzzManifestJSON = "application/bzz-manifest+json"
func add(ctx *cli.Context) {
// manifestAdd adds a new entry to the manifest at the given path.
// New entry hash, the last argument, must be the hash of a manifest
// with only one entry, which meta-data will be added to the original manifest.
// On success, this function will print new (updated) manifest's hash.
func manifestAdd(ctx *cli.Context) {
args := ctx.Args()
if len(args) < 3 {
utils.Fatalf("Need at least three arguments <MHASH> <path> <HASH> [<content-type>]")
if len(args) != 3 {
utils.Fatalf("Need exactly three arguments <MHASH> <path> <HASH>")
}
var (
mhash = args[0]
path = args[1]
hash = args[2]
ctype string
wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
mroot api.Manifest
)
if len(args) > 3 {
ctype = args[3]
} else {
ctype = mime.TypeByExtension(filepath.Ext(path))
bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client := swarm.NewClient(bzzapi)
m, _, err := client.DownloadManifest(hash)
if err != nil {
utils.Fatalf("Error downloading manifest to add: %v", err)
}
l := len(m.Entries)
if l == 0 {
utils.Fatalf("No entries in manifest %s", hash)
} else if l > 1 {
utils.Fatalf("Too many entries in manifest %s", hash)
}
newManifest := addEntryToManifest(ctx, mhash, path, hash, ctype)
newManifest := addEntryToManifest(client, mhash, path, m.Entries[0])
fmt.Println(newManifest)
if !wantManifest {
// Print the manifest. This is the only output to stdout.
mrootJSON, _ := json.MarshalIndent(mroot, "", " ")
fmt.Println(string(mrootJSON))
return
}
}
func update(ctx *cli.Context) {
// manifestUpdate replaces an existing entry of the manifest at the given path.
// New entry hash, the last argument, must be the hash of a manifest
// with only one entry, which meta-data will be added to the original manifest.
// On success, this function will print hash of the updated manifest.
func manifestUpdate(ctx *cli.Context) {
args := ctx.Args()
if len(args) < 3 {
utils.Fatalf("Need at least three arguments <MHASH> <path> <HASH>")
if len(args) != 3 {
utils.Fatalf("Need exactly three arguments <MHASH> <path> <HASH>")
}
var (
mhash = args[0]
path = args[1]
hash = args[2]
ctype string
wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
mroot api.Manifest
)
if len(args) > 3 {
ctype = args[3]
} else {
ctype = mime.TypeByExtension(filepath.Ext(path))
bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client := swarm.NewClient(bzzapi)
m, _, err := client.DownloadManifest(hash)
if err != nil {
utils.Fatalf("Error downloading manifest to update: %v", err)
}
l := len(m.Entries)
if l == 0 {
utils.Fatalf("No entries in manifest %s", hash)
} else if l > 1 {
utils.Fatalf("Too many entries in manifest %s", hash)
}
newManifest := updateEntryInManifest(ctx, mhash, path, hash, ctype)
newManifest, _, defaultEntryUpdated := updateEntryInManifest(client, mhash, path, m.Entries[0], true)
if defaultEntryUpdated {
// Print informational message to stderr
// allowing the user to get the new manifest hash from stdout
// without the need to parse the complete output.
fmt.Fprintln(os.Stderr, "Manifest default entry is updated, too")
}
fmt.Println(newManifest)
if !wantManifest {
// Print the manifest. This is the only output to stdout.
mrootJSON, _ := json.MarshalIndent(mroot, "", " ")
fmt.Println(string(mrootJSON))
return
}
}
func remove(ctx *cli.Context) {
// manifestRemove removes an existing entry of the manifest at the given path.
// On success, this function will print hash of the manifest which does not
// contain the path.
func manifestRemove(ctx *cli.Context) {
args := ctx.Args()
if len(args) < 2 {
utils.Fatalf("Need at least two arguments <MHASH> <path>")
if len(args) != 2 {
utils.Fatalf("Need exactly two arguments <MHASH> <path>")
}
var (
mhash = args[0]
path = args[1]
wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
mroot api.Manifest
)
newManifest := removeEntryFromManifest(ctx, mhash, path)
fmt.Println(newManifest)
bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client := swarm.NewClient(bzzapi)
if !wantManifest {
// Print the manifest. This is the only output to stdout.
mrootJSON, _ := json.MarshalIndent(mroot, "", " ")
fmt.Println(string(mrootJSON))
return
}
newManifest := removeEntryFromManifest(client, mhash, path)
fmt.Println(newManifest)
}
func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) string {
var (
bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client = swarm.NewClient(bzzapi)
longestPathEntry = api.ManifestEntry{}
)
func addEntryToManifest(client *swarm.Client, mhash, path string, entry api.ManifestEntry) string {
var longestPathEntry = api.ManifestEntry{}
mroot, isEncrypted, err := client.DownloadManifest(mhash)
if err != nil {
utils.Fatalf("Manifest download failed: %v", err)
}
//TODO: check if the "hash" to add is valid and present in swarm
_, _, err = client.DownloadManifest(hash)
if err != nil {
utils.Fatalf("Hash to add is not present: %v", err)
}
// See if we path is in this Manifest or do we have to dig deeper
for _, entry := range mroot.Entries {
if path == entry.Path {
for _, e := range mroot.Entries {
if path == e.Path {
utils.Fatalf("Path %s already present, not adding anything", path)
} else {
if entry.ContentType == bzzManifestJSON {
prfxlen := strings.HasPrefix(path, entry.Path)
if e.ContentType == api.ManifestType {
prfxlen := strings.HasPrefix(path, e.Path)
if prfxlen && len(path) > len(longestPathEntry.Path) {
longestPathEntry = entry
longestPathEntry = e
}
}
}
@ -159,25 +148,21 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
if longestPathEntry.Path != "" {
// Load the child Manifest add the entry there
newPath := path[len(longestPathEntry.Path):]
newHash := addEntryToManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype)
newHash := addEntryToManifest(client, longestPathEntry.Hash, newPath, entry)
// Replace the hash for parent Manifests
newMRoot := &api.Manifest{}
for _, entry := range mroot.Entries {
if longestPathEntry.Path == entry.Path {
entry.Hash = newHash
for _, e := range mroot.Entries {
if longestPathEntry.Path == e.Path {
e.Hash = newHash
}
newMRoot.Entries = append(newMRoot.Entries, entry)
newMRoot.Entries = append(newMRoot.Entries, e)
}
mroot = newMRoot
} else {
// Add the entry in the leaf Manifest
newEntry := api.ManifestEntry{
Hash: hash,
Path: path,
ContentType: ctype,
}
mroot.Entries = append(mroot.Entries, newEntry)
entry.Path = path
mroot.Entries = append(mroot.Entries, entry)
}
newManifestHash, err := client.UploadManifest(mroot, isEncrypted)
@ -185,14 +170,16 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
utils.Fatalf("Manifest upload failed: %v", err)
}
return newManifestHash
}
func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) string {
// updateEntryInManifest updates an existing entry o path with a new one in the manifest with provided mhash
// finding the path recursively through all nested manifests. Argument isRoot is used for default
// entry update detection. If the updated entry has the same hash as the default entry, then the
// default entry in root manifest will be updated too.
// Returned values are the new manifest hash, hash of the entry that was replaced by the new entry and
// a a bool that is true if default entry is updated.
func updateEntryInManifest(client *swarm.Client, mhash, path string, entry api.ManifestEntry, isRoot bool) (newManifestHash, oldHash string, defaultEntryUpdated bool) {
var (
bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client = swarm.NewClient(bzzapi)
newEntry = api.ManifestEntry{}
longestPathEntry = api.ManifestEntry{}
)
@ -202,17 +189,18 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
utils.Fatalf("Manifest download failed: %v", err)
}
//TODO: check if the "hash" with which to update is valid and present in swarm
// See if we path is in this Manifest or do we have to dig deeper
for _, entry := range mroot.Entries {
if path == entry.Path {
newEntry = entry
for _, e := range mroot.Entries {
if path == e.Path {
newEntry = e
// keep the reference of the hash of the entry that should be replaced
// for default entry detection
oldHash = e.Hash
} else {
if entry.ContentType == bzzManifestJSON {
prfxlen := strings.HasPrefix(path, entry.Path)
if e.ContentType == api.ManifestType {
prfxlen := strings.HasPrefix(path, e.Path)
if prfxlen && len(path) > len(longestPathEntry.Path) {
longestPathEntry = entry
longestPathEntry = e
}
}
}
@ -225,50 +213,50 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
if longestPathEntry.Path != "" {
// Load the child Manifest add the entry there
newPath := path[len(longestPathEntry.Path):]
newHash := updateEntryInManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype)
var newHash string
newHash, oldHash, _ = updateEntryInManifest(client, longestPathEntry.Hash, newPath, entry, false)
// Replace the hash for parent Manifests
newMRoot := &api.Manifest{}
for _, entry := range mroot.Entries {
if longestPathEntry.Path == entry.Path {
entry.Hash = newHash
for _, e := range mroot.Entries {
if longestPathEntry.Path == e.Path {
e.Hash = newHash
}
newMRoot.Entries = append(newMRoot.Entries, entry)
newMRoot.Entries = append(newMRoot.Entries, e)
}
mroot = newMRoot
}
if newEntry.Path != "" {
// update the manifest if the new entry is found and
// check if default entry should be updated
if newEntry.Path != "" || isRoot {
// Replace the hash for leaf Manifest
newMRoot := &api.Manifest{}
for _, entry := range mroot.Entries {
if newEntry.Path == entry.Path {
myEntry := api.ManifestEntry{
Hash: hash,
Path: entry.Path,
ContentType: ctype,
}
newMRoot.Entries = append(newMRoot.Entries, myEntry)
} else {
for _, e := range mroot.Entries {
if newEntry.Path == e.Path {
entry.Path = e.Path
newMRoot.Entries = append(newMRoot.Entries, entry)
} else if isRoot && e.Path == "" && e.Hash == oldHash {
entry.Path = e.Path
newMRoot.Entries = append(newMRoot.Entries, entry)
defaultEntryUpdated = true
} else {
newMRoot.Entries = append(newMRoot.Entries, e)
}
}
mroot = newMRoot
}
newManifestHash, err := client.UploadManifest(mroot, isEncrypted)
newManifestHash, err = client.UploadManifest(mroot, isEncrypted)
if err != nil {
utils.Fatalf("Manifest upload failed: %v", err)
}
return newManifestHash
return newManifestHash, oldHash, defaultEntryUpdated
}
func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
func removeEntryFromManifest(client *swarm.Client, mhash, path string) string {
var (
bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client = swarm.NewClient(bzzapi)
entryToRemove = api.ManifestEntry{}
longestPathEntry = api.ManifestEntry{}
)
@ -283,7 +271,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
if path == entry.Path {
entryToRemove = entry
} else {
if entry.ContentType == bzzManifestJSON {
if entry.ContentType == api.ManifestType {
prfxlen := strings.HasPrefix(path, entry.Path)
if prfxlen && len(path) > len(longestPathEntry.Path) {
longestPathEntry = entry
@ -299,7 +287,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
if longestPathEntry.Path != "" {
// Load the child Manifest remove the entry there
newPath := path[len(longestPathEntry.Path):]
newHash := removeEntryFromManifest(ctx, longestPathEntry.Hash, newPath)
newHash := removeEntryFromManifest(client, longestPathEntry.Hash, newPath)
// Replace the hash for parent Manifests
newMRoot := &api.Manifest{}

579
cmd/swarm/manifest_test.go Normal file
View File

@ -0,0 +1,579 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"bytes"
"io/ioutil"
"os"
"path/filepath"
"testing"
"github.com/ethereum/go-ethereum/swarm/api"
swarm "github.com/ethereum/go-ethereum/swarm/api/client"
)
// TestManifestChange tests manifest add, update and remove
// cli commands without encryption.
func TestManifestChange(t *testing.T) {
testManifestChange(t, false)
}
// TestManifestChange tests manifest add, update and remove
// cli commands with encryption enabled.
func TestManifestChangeEncrypted(t *testing.T) {
testManifestChange(t, true)
}
// testManifestChange performs cli commands:
// - manifest add
// - manifest update
// - manifest remove
// on a manifest, testing the functionality of this
// comands on paths that are in root manifest or a nested one.
// Argument encrypt controls whether to use encryption or not.
func testManifestChange(t *testing.T, encrypt bool) {
t.Parallel()
cluster := newTestCluster(t, 1)
defer cluster.Shutdown()
tmp, err := ioutil.TempDir("", "swarm-manifest-test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmp)
origDir := filepath.Join(tmp, "orig")
if err := os.Mkdir(origDir, 0777); err != nil {
t.Fatal(err)
}
indexDataFilename := filepath.Join(origDir, "index.html")
err = ioutil.WriteFile(indexDataFilename, []byte("<h1>Test</h1>"), 0666)
if err != nil {
t.Fatal(err)
}
// Files paths robots.txt and robots.html share the same prefix "robots."
// which will result a manifest with a nested manifest under path "robots.".
// This will allow testing manifest changes on both root and nested manifest.
err = ioutil.WriteFile(filepath.Join(origDir, "robots.txt"), []byte("Disallow: /"), 0666)
if err != nil {
t.Fatal(err)
}
err = ioutil.WriteFile(filepath.Join(origDir, "robots.html"), []byte("<strong>No Robots Allowed</strong>"), 0666)
if err != nil {
t.Fatal(err)
}
err = ioutil.WriteFile(filepath.Join(origDir, "mutants.txt"), []byte("Frank\nMarcus"), 0666)
if err != nil {
t.Fatal(err)
}
args := []string{
"--bzzapi",
cluster.Nodes[0].URL,
"--recursive",
"--defaultpath",
indexDataFilename,
"up",
origDir,
}
if encrypt {
args = append(args, "--encrypt")
}
origManifestHash := runSwarmExpectHash(t, args...)
checkHashLength(t, origManifestHash, encrypt)
client := swarm.NewClient(cluster.Nodes[0].URL)
// upload a new file and use its manifest to add it the original manifest.
t.Run("add", func(t *testing.T) {
humansData := []byte("Ann\nBob")
humansDataFilename := filepath.Join(tmp, "humans.txt")
err = ioutil.WriteFile(humansDataFilename, humansData, 0666)
if err != nil {
t.Fatal(err)
}
humansManifestHash := runSwarmExpectHash(t,
"--bzzapi",
cluster.Nodes[0].URL,
"up",
humansDataFilename,
)
newManifestHash := runSwarmExpectHash(t,
"--bzzapi",
cluster.Nodes[0].URL,
"manifest",
"add",
origManifestHash,
"humans.txt",
humansManifestHash,
)
checkHashLength(t, newManifestHash, encrypt)
newManifest := downloadManifest(t, client, newManifestHash, encrypt)
var found bool
for _, e := range newManifest.Entries {
if e.Path == "humans.txt" {
found = true
if e.Size != int64(len(humansData)) {
t.Errorf("expected humans.txt size %v, got %v", len(humansData), e.Size)
}
if e.ModTime.IsZero() {
t.Errorf("got zero mod time for humans.txt")
}
ct := "text/plain; charset=utf-8"
if e.ContentType != ct {
t.Errorf("expected content type %q, got %q", ct, e.ContentType)
}
break
}
}
if !found {
t.Fatal("no humans.txt in new manifest")
}
checkFile(t, client, newManifestHash, "humans.txt", humansData)
})
// upload a new file and use its manifest to add it the original manifest,
// but ensure that the file will be in the nested manifest of the original one.
t.Run("add nested", func(t *testing.T) {
robotsData := []byte(`{"disallow": "/"}`)
robotsDataFilename := filepath.Join(tmp, "robots.json")
err = ioutil.WriteFile(robotsDataFilename, robotsData, 0666)
if err != nil {
t.Fatal(err)
}
robotsManifestHash := runSwarmExpectHash(t,
"--bzzapi",
cluster.Nodes[0].URL,
"up",
robotsDataFilename,
)
newManifestHash := runSwarmExpectHash(t,
"--bzzapi",
cluster.Nodes[0].URL,
"manifest",
"add",
origManifestHash,
"robots.json",
robotsManifestHash,
)
checkHashLength(t, newManifestHash, encrypt)
newManifest := downloadManifest(t, client, newManifestHash, encrypt)
var found bool
loop:
for _, e := range newManifest.Entries {
if e.Path == "robots." {
nestedManifest := downloadManifest(t, client, e.Hash, encrypt)
for _, e := range nestedManifest.Entries {
if e.Path == "json" {
found = true
if e.Size != int64(len(robotsData)) {
t.Errorf("expected robots.json size %v, got %v", len(robotsData), e.Size)
}
if e.ModTime.IsZero() {
t.Errorf("got zero mod time for robots.json")
}
ct := "application/json"
if e.ContentType != ct {
t.Errorf("expected content type %q, got %q", ct, e.ContentType)
}
break loop
}
}
}
}
if !found {
t.Fatal("no robots.json in new manifest")
}
checkFile(t, client, newManifestHash, "robots.json", robotsData)
})
// upload a new file and use its manifest to change the file it the original manifest.
t.Run("update", func(t *testing.T) {
indexData := []byte("<h1>Ethereum Swarm</h1>")
indexDataFilename := filepath.Join(tmp, "index.html")
err = ioutil.WriteFile(indexDataFilename, indexData, 0666)
if err != nil {
t.Fatal(err)
}
indexManifestHash := runSwarmExpectHash(t,
"--bzzapi",
cluster.Nodes[0].URL,
"up",
indexDataFilename,
)
newManifestHash := runSwarmExpectHash(t,
"--bzzapi",
cluster.Nodes[0].URL,
"manifest",
"update",
origManifestHash,
"index.html",
indexManifestHash,
)
checkHashLength(t, newManifestHash, encrypt)
newManifest := downloadManifest(t, client, newManifestHash, encrypt)
var found bool
for _, e := range newManifest.Entries {
if e.Path == "index.html" {
found = true
if e.Size != int64(len(indexData)) {
t.Errorf("expected index.html size %v, got %v", len(indexData), e.Size)
}
if e.ModTime.IsZero() {
t.Errorf("got zero mod time for index.html")
}
ct := "text/html; charset=utf-8"
if e.ContentType != ct {
t.Errorf("expected content type %q, got %q", ct, e.ContentType)
}
break
}
}
if !found {
t.Fatal("no index.html in new manifest")
}
checkFile(t, client, newManifestHash, "index.html", indexData)
// check default entry change
checkFile(t, client, newManifestHash, "", indexData)
})
// upload a new file and use its manifest to change the file it the original manifest,
// but ensure that the file is in the nested manifest of the original one.
t.Run("update nested", func(t *testing.T) {
robotsData := []byte(`<string>Only humans allowed!!!</strong>`)
robotsDataFilename := filepath.Join(tmp, "robots.html")
err = ioutil.WriteFile(robotsDataFilename, robotsData, 0666)
if err != nil {
t.Fatal(err)
}
humansManifestHash := runSwarmExpectHash(t,
"--bzzapi",
cluster.Nodes[0].URL,
"up",
robotsDataFilename,
)
newManifestHash := runSwarmExpectHash(t,
"--bzzapi",
cluster.Nodes[0].URL,
"manifest",
"update",
origManifestHash,
"robots.html",
humansManifestHash,
)
checkHashLength(t, newManifestHash, encrypt)
newManifest := downloadManifest(t, client, newManifestHash, encrypt)
var found bool
loop:
for _, e := range newManifest.Entries {
if e.Path == "robots." {
nestedManifest := downloadManifest(t, client, e.Hash, encrypt)
for _, e := range nestedManifest.Entries {
if e.Path == "html" {
found = true
if e.Size != int64(len(robotsData)) {
t.Errorf("expected robots.html size %v, got %v", len(robotsData), e.Size)
}
if e.ModTime.IsZero() {
t.Errorf("got zero mod time for robots.html")
}
ct := "text/html; charset=utf-8"
if e.ContentType != ct {
t.Errorf("expected content type %q, got %q", ct, e.ContentType)
}
break loop
}
}
}
}
if !found {
t.Fatal("no robots.html in new manifest")
}
checkFile(t, client, newManifestHash, "robots.html", robotsData)
})
// remove a file from the manifest.
t.Run("remove", func(t *testing.T) {
newManifestHash := runSwarmExpectHash(t,
"--bzzapi",
cluster.Nodes[0].URL,
"manifest",
"remove",
origManifestHash,
"mutants.txt",
)
checkHashLength(t, newManifestHash, encrypt)
newManifest := downloadManifest(t, client, newManifestHash, encrypt)
var found bool
for _, e := range newManifest.Entries {
if e.Path == "mutants.txt" {
found = true
break
}
}
if found {
t.Fatal("mutants.txt is not removed")
}
})
// remove a file from the manifest, but ensure that the file is in
// the nested manifest of the original one.
t.Run("remove nested", func(t *testing.T) {
newManifestHash := runSwarmExpectHash(t,
"--bzzapi",
cluster.Nodes[0].URL,
"manifest",
"remove",
origManifestHash,
"robots.html",
)
checkHashLength(t, newManifestHash, encrypt)
newManifest := downloadManifest(t, client, newManifestHash, encrypt)
var found bool
loop:
for _, e := range newManifest.Entries {
if e.Path == "robots." {
nestedManifest := downloadManifest(t, client, e.Hash, encrypt)
for _, e := range nestedManifest.Entries {
if e.Path == "html" {
found = true
break loop
}
}
}
}
if found {
t.Fatal("robots.html in not removed")
}
})
}
// TestNestedDefaultEntryUpdate tests if the default entry is updated
// if the file in nested manifest used for it is also updated.
func TestNestedDefaultEntryUpdate(t *testing.T) {
testNestedDefaultEntryUpdate(t, false)
}
// TestNestedDefaultEntryUpdateEncrypted tests if the default entry
// of encrypted upload is updated if the file in nested manifest
// used for it is also updated.
func TestNestedDefaultEntryUpdateEncrypted(t *testing.T) {
testNestedDefaultEntryUpdate(t, true)
}
func testNestedDefaultEntryUpdate(t *testing.T, encrypt bool) {
t.Parallel()
cluster := newTestCluster(t, 1)
defer cluster.Shutdown()
tmp, err := ioutil.TempDir("", "swarm-manifest-test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmp)
origDir := filepath.Join(tmp, "orig")
if err := os.Mkdir(origDir, 0777); err != nil {
t.Fatal(err)
}
indexData := []byte("<h1>Test</h1>")
indexDataFilename := filepath.Join(origDir, "index.html")
err = ioutil.WriteFile(indexDataFilename, indexData, 0666)
if err != nil {
t.Fatal(err)
}
// Add another file with common prefix as the default entry to test updates of
// default entry with nested manifests.
err = ioutil.WriteFile(filepath.Join(origDir, "index.txt"), []byte("Test"), 0666)
if err != nil {
t.Fatal(err)
}
args := []string{
"--bzzapi",
cluster.Nodes[0].URL,
"--recursive",
"--defaultpath",
indexDataFilename,
"up",
origDir,
}
if encrypt {
args = append(args, "--encrypt")
}
origManifestHash := runSwarmExpectHash(t, args...)
checkHashLength(t, origManifestHash, encrypt)
client := swarm.NewClient(cluster.Nodes[0].URL)
newIndexData := []byte("<h1>Ethereum Swarm</h1>")
newIndexDataFilename := filepath.Join(tmp, "index.html")
err = ioutil.WriteFile(newIndexDataFilename, newIndexData, 0666)
if err != nil {
t.Fatal(err)
}
newIndexManifestHash := runSwarmExpectHash(t,
"--bzzapi",
cluster.Nodes[0].URL,
"up",
newIndexDataFilename,
)
newManifestHash := runSwarmExpectHash(t,
"--bzzapi",
cluster.Nodes[0].URL,
"manifest",
"update",
origManifestHash,
"index.html",
newIndexManifestHash,
)
checkHashLength(t, newManifestHash, encrypt)
newManifest := downloadManifest(t, client, newManifestHash, encrypt)
var found bool
for _, e := range newManifest.Entries {
if e.Path == "index." {
found = true
newManifest = downloadManifest(t, client, e.Hash, encrypt)
break
}
}
if !found {
t.Fatal("no index. path in new manifest")
}
found = false
for _, e := range newManifest.Entries {
if e.Path == "html" {
found = true
if e.Size != int64(len(newIndexData)) {
t.Errorf("expected index.html size %v, got %v", len(newIndexData), e.Size)
}
if e.ModTime.IsZero() {
t.Errorf("got zero mod time for index.html")
}
ct := "text/html; charset=utf-8"
if e.ContentType != ct {
t.Errorf("expected content type %q, got %q", ct, e.ContentType)
}
break
}
}
if !found {
t.Fatal("no html in new manifest")
}
checkFile(t, client, newManifestHash, "index.html", newIndexData)
// check default entry change
checkFile(t, client, newManifestHash, "", newIndexData)
}
func runSwarmExpectHash(t *testing.T, args ...string) (hash string) {
t.Helper()
hashRegexp := `[a-f\d]{64,128}`
up := runSwarm(t, args...)
_, matches := up.ExpectRegexp(hashRegexp)
up.ExpectExit()
if len(matches) < 1 {
t.Fatal("no matches found")
}
return matches[0]
}
func checkHashLength(t *testing.T, hash string, encrypted bool) {
t.Helper()
l := len(hash)
if encrypted && l != 128 {
t.Errorf("expected hash length 128, got %v", l)
}
if !encrypted && l != 64 {
t.Errorf("expected hash length 64, got %v", l)
}
}
func downloadManifest(t *testing.T, client *swarm.Client, hash string, encrypted bool) (manifest *api.Manifest) {
t.Helper()
m, isEncrypted, err := client.DownloadManifest(hash)
if err != nil {
t.Fatal(err)
}
if encrypted != isEncrypted {
t.Error("new manifest encryption flag is not correct")
}
return m
}
func checkFile(t *testing.T, client *swarm.Client, hash, path string, expected []byte) {
t.Helper()
f, err := client.Download(hash, path)
if err != nil {
t.Fatal(err)
}
got, err := ioutil.ReadAll(f)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(got, expected) {
t.Errorf("expected file content %q, got %q", expected, got)
}
}

169
cmd/swarm/mru.go Normal file
View File

@ -0,0 +1,169 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
// Command resource allows the user to create and update signed mutable resource updates
package main
import (
"fmt"
"strconv"
"strings"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/cmd/utils"
swarm "github.com/ethereum/go-ethereum/swarm/api/client"
"github.com/ethereum/go-ethereum/swarm/storage/mru"
"gopkg.in/urfave/cli.v1"
)
func NewGenericSigner(ctx *cli.Context) mru.Signer {
return mru.NewGenericSigner(getPrivKey(ctx))
}
// swarm resource create <frequency> [--name <name>] [--data <0x Hexdata> [--multihash=false]]
// swarm resource update <Manifest Address or ENS domain> <0x Hexdata> [--multihash=false]
// swarm resource info <Manifest Address or ENS domain>
func resourceCreate(ctx *cli.Context) {
args := ctx.Args()
var (
bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client = swarm.NewClient(bzzapi)
multihash = ctx.Bool(SwarmResourceMultihashFlag.Name)
initialData = ctx.String(SwarmResourceDataOnCreateFlag.Name)
name = ctx.String(SwarmResourceNameFlag.Name)
)
if len(args) < 1 {
fmt.Println("Incorrect number of arguments")
cli.ShowCommandHelpAndExit(ctx, "create", 1)
return
}
signer := NewGenericSigner(ctx)
frequency, err := strconv.ParseUint(args[0], 10, 64)
if err != nil {
fmt.Printf("Frequency formatting error: %s\n", err.Error())
cli.ShowCommandHelpAndExit(ctx, "create", 1)
return
}
metadata := mru.ResourceMetadata{
Name: name,
Frequency: frequency,
Owner: signer.Address(),
}
var newResourceRequest *mru.Request
if initialData != "" {
initialDataBytes, err := hexutil.Decode(initialData)
if err != nil {
fmt.Printf("Error parsing data: %s\n", err.Error())
cli.ShowCommandHelpAndExit(ctx, "create", 1)
return
}
newResourceRequest, err = mru.NewCreateUpdateRequest(&metadata)
if err != nil {
utils.Fatalf("Error creating new resource request: %s", err)
}
newResourceRequest.SetData(initialDataBytes, multihash)
if err = newResourceRequest.Sign(signer); err != nil {
utils.Fatalf("Error signing resource update: %s", err.Error())
}
} else {
newResourceRequest, err = mru.NewCreateRequest(&metadata)
if err != nil {
utils.Fatalf("Error creating new resource request: %s", err)
}
}
manifestAddress, err := client.CreateResource(newResourceRequest)
if err != nil {
utils.Fatalf("Error creating resource: %s", err.Error())
return
}
fmt.Println(manifestAddress) // output manifest address to the user in a single line (useful for other commands to pick up)
}
func resourceUpdate(ctx *cli.Context) {
args := ctx.Args()
var (
bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client = swarm.NewClient(bzzapi)
multihash = ctx.Bool(SwarmResourceMultihashFlag.Name)
)
if len(args) < 2 {
fmt.Println("Incorrect number of arguments")
cli.ShowCommandHelpAndExit(ctx, "update", 1)
return
}
signer := NewGenericSigner(ctx)
manifestAddressOrDomain := args[0]
data, err := hexutil.Decode(args[1])
if err != nil {
utils.Fatalf("Error parsing data: %s", err.Error())
return
}
// Retrieve resource status and metadata out of the manifest
updateRequest, err := client.GetResourceMetadata(manifestAddressOrDomain)
if err != nil {
utils.Fatalf("Error retrieving resource status: %s", err.Error())
}
// set the new data
updateRequest.SetData(data, multihash)
// sign update
if err = updateRequest.Sign(signer); err != nil {
utils.Fatalf("Error signing resource update: %s", err.Error())
}
// post update
err = client.UpdateResource(updateRequest)
if err != nil {
utils.Fatalf("Error updating resource: %s", err.Error())
return
}
}
func resourceInfo(ctx *cli.Context) {
var (
bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client = swarm.NewClient(bzzapi)
)
args := ctx.Args()
if len(args) < 1 {
fmt.Println("Incorrect number of arguments.")
cli.ShowCommandHelpAndExit(ctx, "info", 1)
return
}
manifestAddressOrDomain := args[0]
metadata, err := client.GetResourceMetadata(manifestAddressOrDomain)
if err != nil {
utils.Fatalf("Error retrieving resource metadata: %s", err.Error())
return
}
encodedMetadata, err := metadata.MarshalJSON()
if err != nil {
utils.Fatalf("Error encoding metadata to JSON for display:%s", err)
}
fmt.Println(string(encodedMetadata))
}

163
cmd/swarm/run_test.go
View File

@ -17,12 +17,17 @@
package main
import (
"context"
"crypto/ecdsa"
"fmt"
"io/ioutil"
"net"
"os"
"path"
"path/filepath"
"runtime"
"sync"
"syscall"
"testing"
"time"
@ -172,14 +177,15 @@ func (c *testCluster) Cleanup() {
}
type testNode struct {
Name string
Addr string
URL string
Enode string
Dir string
IpcPath string
Client *rpc.Client
Cmd *cmdtest.TestCmd
Name string
Addr string
URL string
Enode string
Dir string
IpcPath string
PrivateKey *ecdsa.PrivateKey
Client *rpc.Client
Cmd *cmdtest.TestCmd
}
const testPassphrase = "swarm-test-passphrase"
@ -218,14 +224,12 @@ func existingTestNode(t *testing.T, dir string, bzzaccount string) *testNode {
}
// assign ports
httpPort, err := assignTCPPort()
if err != nil {
t.Fatal(err)
}
p2pPort, err := assignTCPPort()
ports, err := getAvailableTCPPorts(2)
if err != nil {
t.Fatal(err)
}
p2pPort := ports[0]
httpPort := ports[1]
// start the node
node.Cmd = runSwarm(t,
@ -246,6 +250,17 @@ func existingTestNode(t *testing.T, dir string, bzzaccount string) *testNode {
}
}()
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
// ensure that all ports have active listeners
// so that the next node will not get the same
// when calling getAvailableTCPPorts
err = waitTCPPorts(ctx, ports...)
if err != nil {
t.Fatal(err)
}
// wait for the node to start
for start := time.Now(); time.Since(start) < 10*time.Second; time.Sleep(50 * time.Millisecond) {
node.Client, err = rpc.Dial(conf.IPCEndpoint())
@ -277,17 +292,19 @@ func existingTestNode(t *testing.T, dir string, bzzaccount string) *testNode {
func newTestNode(t *testing.T, dir string) *testNode {
conf, account := getTestAccount(t, dir)
node := &testNode{Dir: dir}
ks := keystore.NewKeyStore(path.Join(dir, "keystore"), 1<<18, 1)
pk := decryptStoreAccount(ks, account.Address.Hex(), []string{testPassphrase})
node := &testNode{Dir: dir, PrivateKey: pk}
// assign ports
httpPort, err := assignTCPPort()
if err != nil {
t.Fatal(err)
}
p2pPort, err := assignTCPPort()
ports, err := getAvailableTCPPorts(2)
if err != nil {
t.Fatal(err)
}
p2pPort := ports[0]
httpPort := ports[1]
// start the node
node.Cmd = runSwarm(t,
@ -308,6 +325,17 @@ func newTestNode(t *testing.T, dir string) *testNode {
}
}()
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
// ensure that all ports have active listeners
// so that the next node will not get the same
// when calling getAvailableTCPPorts
err = waitTCPPorts(ctx, ports...)
if err != nil {
t.Fatal(err)
}
// wait for the node to start
for start := time.Now(); time.Since(start) < 10*time.Second; time.Sleep(50 * time.Millisecond) {
node.Client, err = rpc.Dial(conf.IPCEndpoint())
@ -343,15 +371,92 @@ func (n *testNode) Shutdown() {
}
}
func assignTCPPort() (string, error) {
l, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
return "", err
// getAvailableTCPPorts returns a set of ports that
// nothing is listening on at the time.
//
// Function assignTCPPort cannot be called in sequence
// and guardantee that the same port will be returned in
// different calls as the listener is closed within the function,
// not after all listeners are started and selected unique
// available ports.
func getAvailableTCPPorts(count int) (ports []string, err error) {
for i := 0; i < count; i++ {
l, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
return nil, err
}
// defer close in the loop to be sure the same port will not
// be selected in the next iteration
defer l.Close()
_, port, err := net.SplitHostPort(l.Addr().String())
if err != nil {
return nil, err
}
ports = append(ports, port)
}
return ports, nil
}
// waitTCPPorts blocks until tcp connections can be
// established on all provided ports. It runs all
// ports dialers in parallel, and returns the first
// encountered error.
// See waitTCPPort also.
func waitTCPPorts(ctx context.Context, ports ...string) error {
var err error
// mu locks err variable that is assigned in
// other goroutines
var mu sync.Mutex
// cancel is canceling all goroutines
// when the firs error is returned
// to prevent unnecessary waiting
ctx, cancel := context.WithCancel(ctx)
defer cancel()
var wg sync.WaitGroup
for _, port := range ports {
wg.Add(1)
go func(port string) {
defer wg.Done()
e := waitTCPPort(ctx, port)
mu.Lock()
defer mu.Unlock()
if e != nil && err == nil {
err = e
cancel()
}
}(port)
}
wg.Wait()
return err
}
// waitTCPPort blocks until tcp connection can be established
// ona provided port. It has a 3 minute timeout as maximum,
// to prevent long waiting, but it can be shortened with
// a provided context instance. Dialer has a 10 second timeout
// in every iteration, and connection refused error will be
// retried in 100 milliseconds periods.
func waitTCPPort(ctx context.Context, port string) error {
ctx, cancel := context.WithTimeout(ctx, 3*time.Minute)
defer cancel()
for {
c, err := (&net.Dialer{Timeout: 10 * time.Second}).DialContext(ctx, "tcp", "127.0.0.1:"+port)
if err != nil {
if operr, ok := err.(*net.OpError); ok {
if syserr, ok := operr.Err.(*os.SyscallError); ok && syserr.Err == syscall.ECONNREFUSED {
time.Sleep(100 * time.Millisecond)
continue
}
}
return err
}
return c.Close()
}
l.Close()
_, port, err := net.SplitHostPort(l.Addr().String())
if err != nil {
return "", err
}
return port, nil
}

6
cmd/swarm/swarm-smoke/main.go
View File

@ -48,7 +48,7 @@ func main() {
cli.StringFlag{
Name: "cluster-endpoint",
Value: "testing",
Usage: "cluster to point to (open, or testing)",
Usage: "cluster to point to (local, open or testing)",
Destination: &cluster,
},
cli.IntFlag{
@ -76,8 +76,8 @@ func main() {
},
cli.IntFlag{
Name: "filesize",
Value: 1,
Usage: "file size for generated random file in MB",
Value: 1024,
Usage: "file size for generated random file in KB",
Destination: &filesize,
},
}

25
cmd/swarm/swarm-smoke/upload_and_sync.go
View File

@ -37,8 +37,19 @@ import (
)
func generateEndpoints(scheme string, cluster string, from int, to int) {
if cluster == "prod" {
cluster = ""
} else if cluster == "local" {
for port := from; port <= to; port++ {
endpoints = append(endpoints, fmt.Sprintf("%s://localhost:%v", scheme, port))
}
return
} else {
cluster = cluster + "."
}
for port := from; port <= to; port++ {
endpoints = append(endpoints, fmt.Sprintf("%s://%v.%s.swarm-gateways.net", scheme, port, cluster))
endpoints = append(endpoints, fmt.Sprintf("%s://%v.%sswarm-gateways.net", scheme, port, cluster))
}
if includeLocalhost {
@ -47,13 +58,13 @@ func generateEndpoints(scheme string, cluster string, from int, to int) {
}
func cliUploadAndSync(c *cli.Context) error {
defer func(now time.Time) { log.Info("total time", "time", time.Since(now), "size", filesize) }(time.Now())
defer func(now time.Time) { log.Info("total time", "time", time.Since(now), "size (kb)", filesize) }(time.Now())
generateEndpoints(scheme, cluster, from, to)
log.Info("uploading to " + endpoints[0] + " and syncing")
f, cleanup := generateRandomFile(filesize * 1000000)
f, cleanup := generateRandomFile(filesize * 1000)
defer cleanup()
hash, err := upload(f, endpoints[0])
@ -70,11 +81,7 @@ func cliUploadAndSync(c *cli.Context) error {
log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fhash))
if filesize < 10 {
time.Sleep(15 * time.Second)
} else {
time.Sleep(2 * time.Duration(filesize) * time.Second)
}
time.Sleep(3 * time.Second)
wg := sync.WaitGroup{}
for _, endpoint := range endpoints {
@ -102,7 +109,7 @@ func cliUploadAndSync(c *cli.Context) error {
// fetch is getting the requested `hash` from the `endpoint` and compares it with the `original` file
func fetch(hash string, endpoint string, original []byte, ruid string) error {
log.Trace("sleeping", "ruid", ruid)
time.Sleep(1 * time.Second)
time.Sleep(3 * time.Second)
log.Trace("http get request", "ruid", ruid, "api", endpoint, "hash", hash)
res, err := http.Get(endpoint + "/bzz:/" + hash + "/")

11
cmd/swarm/upload.go
View File

@ -98,6 +98,17 @@ func upload(ctx *cli.Context) {
if !recursive {
return "", errors.New("Argument is a directory and recursive upload is disabled")
}
if defaultPath != "" {
// construct absolute default path
absDefaultPath, _ := filepath.Abs(defaultPath)
absFile, _ := filepath.Abs(file)
// make sure absolute directory ends with only one "/"
// to trim it from absolute default path and get relative default path
absFile = strings.TrimRight(absFile, "/") + "/"
if absDefaultPath != "" && absFile != "" && strings.HasPrefix(absDefaultPath, absFile) {
defaultPath = strings.TrimPrefix(absDefaultPath, absFile)
}
}
return client.UploadDirectory(file, defaultPath, "", toEncrypt)
}
} else {

81
cmd/swarm/upload_test.go
View File

@ -273,3 +273,84 @@ func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) {
}
}
}
// TestCLISwarmUpDefaultPath tests swarm recursive upload with relative and absolute
// default paths and with encryption.
func TestCLISwarmUpDefaultPath(t *testing.T) {
testCLISwarmUpDefaultPath(false, false, t)
testCLISwarmUpDefaultPath(false, true, t)
testCLISwarmUpDefaultPath(true, false, t)
testCLISwarmUpDefaultPath(true, true, t)
}
func testCLISwarmUpDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T) {
cluster := newTestCluster(t, 1)
defer cluster.Shutdown()
tmp, err := ioutil.TempDir("", "swarm-defaultpath-test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmp)
err = ioutil.WriteFile(filepath.Join(tmp, "index.html"), []byte("<h1>Test</h1>"), 0666)
if err != nil {
t.Fatal(err)
}
err = ioutil.WriteFile(filepath.Join(tmp, "robots.txt"), []byte("Disallow: /"), 0666)
if err != nil {
t.Fatal(err)
}
defaultPath := "index.html"
if absDefaultPath {
defaultPath = filepath.Join(tmp, defaultPath)
}
args := []string{
"--bzzapi",
cluster.Nodes[0].URL,
"--recursive",
"--defaultpath",
defaultPath,
"up",
tmp,
}
if toEncrypt {
args = append(args, "--encrypt")
}
up := runSwarm(t, args...)
hashRegexp := `[a-f\d]{64,128}`
_, matches := up.ExpectRegexp(hashRegexp)
up.ExpectExit()
hash := matches[0]
client := swarm.NewClient(cluster.Nodes[0].URL)
m, isEncrypted, err := client.DownloadManifest(hash)
if err != nil {
t.Fatal(err)
}
if toEncrypt != isEncrypted {
t.Error("downloaded manifest is not encrypted")
}
var found bool
var entriesCount int
for _, e := range m.Entries {
entriesCount++
if e.Path == "" {
found = true
}
}
if !found {
t.Error("manifest default entry was not found")
}
if entriesCount != 3 {
t.Errorf("manifest contains %v entries, expected %v", entriesCount, 3)
}
}

210
cmd/utils/flags.go
View File

@ -24,7 +24,6 @@ import (
"math/big"
"os"
"path/filepath"
"runtime"
"strconv"
"strings"
@ -98,7 +97,7 @@ func NewApp(gitCommit, usage string) *cli.App {
app.Author = ""
//app.Authors = nil
app.Email = ""
app.Version = params.Version
app.Version = params.VersionWithMeta
if len(gitCommit) >= 8 {
app.Version += "-" + gitCommit[:8]
}
@ -162,14 +161,6 @@ var (
Usage: "Document Root for HTTPClient file scheme",
Value: DirectoryString{homeDir()},
}
FastSyncFlag = cli.BoolFlag{
Name: "fast",
Usage: "Enable fast syncing through state downloads (replaced by --syncmode)",
}
LightModeFlag = cli.BoolFlag{
Name: "light",
Usage: "Enable light client mode (replaced by --syncmode)",
}
defaultSyncMode = eth.DefaultConfig.SyncMode
SyncModeFlag = TextMarshalerFlag{
Name: "syncmode",
@ -197,7 +188,7 @@ var (
}
// Dashboard settings
DashboardEnabledFlag = cli.BoolFlag{
Name: "dashboard",
Name: metrics.DashboardEnabledFlag,
Usage: "Enable the dashboard",
}
DashboardAddrFlag = cli.StringFlag{
@ -246,6 +237,10 @@ var (
Value: eth.DefaultConfig.Ethash.DatasetsOnDisk,
}
// Transaction pool settings
TxPoolLocalsFlag = cli.StringFlag{
Name: "txpool.locals",
Usage: "Comma separated accounts to treat as locals (no flush, priority inclusion)",
}
TxPoolNoLocalsFlag = cli.BoolFlag{
Name: "txpool.nolocals",
Usage: "Disables price exemptions for locally submitted transactions",
@ -322,29 +317,71 @@ var (
Usage: "Enable mining",
}
MinerThreadsFlag = cli.IntFlag{
Name: "minerthreads",
Name: "miner.threads",
Usage: "Number of CPU threads to use for mining",
Value: runtime.NumCPU(),
Value: 0,
}
TargetGasLimitFlag = cli.Uint64Flag{
MinerLegacyThreadsFlag = cli.IntFlag{
Name: "minerthreads",
Usage: "Number of CPU threads to use for mining (deprecated, use --miner.threads)",
Value: 0,
}
MinerNotifyFlag = cli.StringFlag{
Name: "miner.notify",
Usage: "Comma separated HTTP URL list to notify of new work packages",
}
MinerGasTargetFlag = cli.Uint64Flag{
Name: "miner.gastarget",
Usage: "Target gas floor for mined blocks",
Value: eth.DefaultConfig.MinerGasFloor,
}
MinerLegacyGasTargetFlag = cli.Uint64Flag{
Name: "targetgaslimit",
Usage: "Target gas limit sets the artificial target gas floor for the blocks to mine",
Value: params.GenesisGasLimit,
Usage: "Target gas floor for mined blocks (deprecated, use --miner.gastarget)",
Value: eth.DefaultConfig.MinerGasFloor,
}
EtherbaseFlag = cli.StringFlag{
Name: "etherbase",
Usage: "Public address for block mining rewards (default = first account created)",
MinerGasLimitFlag = cli.Uint64Flag{
Name: "miner.gaslimit",
Usage: "Target gas ceiling for mined blocks",
Value: eth.DefaultConfig.MinerGasCeil,
}
MinerGasPriceFlag = BigFlag{
Name: "miner.gasprice",
Usage: "Minimum gas price for mining a transaction",
Value: eth.DefaultConfig.MinerGasPrice,
}
MinerLegacyGasPriceFlag = BigFlag{
Name: "gasprice",
Usage: "Minimum gas price for mining a transaction (deprecated, use --miner.gasprice)",
Value: eth.DefaultConfig.MinerGasPrice,
}
MinerEtherbaseFlag = cli.StringFlag{
Name: "miner.etherbase",
Usage: "Public address for block mining rewards (default = first account)",
Value: "0",
}
GasPriceFlag = BigFlag{
Name: "gasprice",
Usage: "Minimal gas price to accept for mining a transactions",
Value: eth.DefaultConfig.GasPrice,
MinerLegacyEtherbaseFlag = cli.StringFlag{
Name: "etherbase",
Usage: "Public address for block mining rewards (default = first account, deprecated, use --miner.etherbase)",
Value: "0",
}
ExtraDataFlag = cli.StringFlag{
Name: "extradata",
MinerExtraDataFlag = cli.StringFlag{
Name: "miner.extradata",
Usage: "Block extra data set by the miner (default = client version)",
}
MinerLegacyExtraDataFlag = cli.StringFlag{
Name: "extradata",
Usage: "Block extra data set by the miner (default = client version, deprecated, use --miner.extradata)",
}
MinerRecommitIntervalFlag = cli.DurationFlag{
Name: "miner.recommit",
Usage: "Time interval to recreate the block being mined",
Value: eth.DefaultConfig.MinerRecommit,
}
MinerNoVerfiyFlag = cli.BoolFlag{
Name: "miner.noverify",
Usage: "Disable remote sealing verification",
}
// Account settings
UnlockedAccountFlag = cli.StringFlag{
Name: "unlock",
@ -534,6 +571,10 @@ var (
Usage: "Minimum POW accepted",
Value: whisper.DefaultMinimumPoW,
}
WhisperRestrictConnectionBetweenLightClientsFlag = cli.BoolFlag{
Name: "shh.restrict-light",
Usage: "Restrict connection between two whisper light clients",
}
// Raft flags
RaftModeFlag = cli.BoolFlag{
@ -616,6 +657,17 @@ var (
Usage: "InfluxDB `host` tag attached to all measurements",
Value: "localhost",
}
EWASMInterpreterFlag = cli.StringFlag{
Name: "vm.ewasm",
Usage: "External ewasm configuration (default = built-in interpreter)",
Value: "",
}
EVMInterpreterFlag = cli.StringFlag{
Name: "vm.evm",
Usage: "External EVM configuration (default = built-in interpreter)",
Value: "",
}
)
// MakeDataDir retrieves the currently requested data directory, terminating
@ -696,8 +748,7 @@ func setBootstrapNodes(ctx *cli.Context, cfg *p2p.Config) {
for _, url := range urls {
node, err := discover.ParseNode(url)
if err != nil {
log.Error("Bootstrap URL invalid", "enode", url, "err", err)
continue
log.Crit("Bootstrap URL invalid", "enode", url, "err", err)
}
cfg.BootstrapNodes = append(cfg.BootstrapNodes, node)
}
@ -863,10 +914,19 @@ func MakeAddress(ks *keystore.KeyStore, account string) (accounts.Account, error
// setEtherbase retrieves the etherbase either from the directly specified
// command line flags or from the keystore if CLI indexed.
func setEtherbase(ctx *cli.Context, ks *keystore.KeyStore, cfg *eth.Config) {
if ctx.GlobalIsSet(EtherbaseFlag.Name) {
account, err := MakeAddress(ks, ctx.GlobalString(EtherbaseFlag.Name))
// Extract the current etherbase, new flag overriding legacy one
var etherbase string
if ctx.GlobalIsSet(MinerLegacyEtherbaseFlag.Name) {
etherbase = ctx.GlobalString(MinerLegacyEtherbaseFlag.Name)
}
if ctx.GlobalIsSet(MinerEtherbaseFlag.Name) {
etherbase = ctx.GlobalString(MinerEtherbaseFlag.Name)
}
// Convert the etherbase into an address and configure it
if etherbase != "" {
account, err := MakeAddress(ks, etherbase)
if err != nil {
Fatalf("Option %q: %v", EtherbaseFlag.Name, err)
Fatalf("Invalid miner etherbase: %v", err)
}
cfg.Etherbase = account.Address
}
@ -897,7 +957,7 @@ func SetP2PConfig(ctx *cli.Context, cfg *p2p.Config) {
setBootstrapNodes(ctx, cfg)
setBootstrapNodesV5(ctx, cfg)
lightClient := ctx.GlobalBool(LightModeFlag.Name) || ctx.GlobalString(SyncModeFlag.Name) == "light"
lightClient := ctx.GlobalString(SyncModeFlag.Name) == "light"
lightServer := ctx.GlobalInt(LightServFlag.Name) != 0
lightPeers := ctx.GlobalInt(LightPeersFlag.Name)
@ -1001,6 +1061,16 @@ func setGPO(ctx *cli.Context, cfg *gasprice.Config) {
}
func setTxPool(ctx *cli.Context, cfg *core.TxPoolConfig) {
if ctx.GlobalIsSet(TxPoolLocalsFlag.Name) {
locals := strings.Split(ctx.GlobalString(TxPoolLocalsFlag.Name), ",")
for _, account := range locals {
if trimmed := strings.TrimSpace(account); !common.IsHexAddress(trimmed) {
Fatalf("Invalid account in --txpool.locals: %s", trimmed)
} else {
cfg.Locals = append(cfg.Locals, common.HexToAddress(account))
}
}
}
if ctx.GlobalIsSet(TxPoolNoLocalsFlag.Name) {
cfg.NoLocals = ctx.GlobalBool(TxPoolNoLocalsFlag.Name)
}
@ -1063,6 +1133,9 @@ func setIstanbul(ctx *cli.Context, cfg *eth.Config) {
}
}
// checkExclusive verifies that only a single instance of the provided flags was
// set by the user. Each flag might optionally be followed by a string type to
// specialize it further.
func checkExclusive(ctx *cli.Context, args ...interface{}) {
set := make([]string, 0, 1)
for i := 0; i < len(args); i++ {
@ -1106,14 +1179,15 @@ func SetShhConfig(ctx *cli.Context, stack *node.Node, cfg *whisper.Config) {
if ctx.GlobalIsSet(WhisperMinPOWFlag.Name) {
cfg.MinimumAcceptedPOW = ctx.GlobalFloat64(WhisperMinPOWFlag.Name)
}
if ctx.GlobalIsSet(WhisperRestrictConnectionBetweenLightClientsFlag.Name) {
cfg.RestrictConnectionBetweenLightClients = true
}
}
// SetEthConfig applies eth-related command line flags to the config.
func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
// Avoid conflicting network flags
checkExclusive(ctx, DeveloperFlag, TestnetFlag, RinkebyFlag, OttomanFlag)
checkExclusive(ctx, FastSyncFlag, LightModeFlag, SyncModeFlag)
checkExclusive(ctx, LightServFlag, LightModeFlag)
checkExclusive(ctx, DeveloperFlag, TestnetFlag, RinkebyFlag)
checkExclusive(ctx, LightServFlag, SyncModeFlag, "light")
ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
@ -1123,13 +1197,8 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
setEthash(ctx, cfg)
setIstanbul(ctx, cfg)
switch {
case ctx.GlobalIsSet(SyncModeFlag.Name):
if ctx.GlobalIsSet(SyncModeFlag.Name) {
cfg.SyncMode = *GlobalTextMarshaler(ctx, SyncModeFlag.Name).(*downloader.SyncMode)
case ctx.GlobalBool(FastSyncFlag.Name):
cfg.SyncMode = downloader.FastSync
case ctx.GlobalBool(LightModeFlag.Name):
cfg.SyncMode = downloader.LightSync
}
if ctx.GlobalIsSet(LightServFlag.Name) {
cfg.LightServ = ctx.GlobalInt(LightServFlag.Name)
@ -1154,23 +1223,52 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheGCFlag.Name) {
cfg.TrieCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100
}
if ctx.GlobalIsSet(MinerThreadsFlag.Name) {
cfg.MinerThreads = ctx.GlobalInt(MinerThreadsFlag.Name)
if ctx.GlobalIsSet(MinerNotifyFlag.Name) {
cfg.MinerNotify = strings.Split(ctx.GlobalString(MinerNotifyFlag.Name), ",")
}
if ctx.GlobalIsSet(DocRootFlag.Name) {
cfg.DocRoot = ctx.GlobalString(DocRootFlag.Name)
}
if ctx.GlobalIsSet(ExtraDataFlag.Name) {
cfg.ExtraData = []byte(ctx.GlobalString(ExtraDataFlag.Name))
if ctx.GlobalIsSet(MinerLegacyExtraDataFlag.Name) {
cfg.MinerExtraData = []byte(ctx.GlobalString(MinerLegacyExtraDataFlag.Name))
}
if ctx.GlobalIsSet(GasPriceFlag.Name) {
cfg.GasPrice = GlobalBig(ctx, GasPriceFlag.Name)
if ctx.GlobalIsSet(MinerExtraDataFlag.Name) {
cfg.MinerExtraData = []byte(ctx.GlobalString(MinerExtraDataFlag.Name))
}
if ctx.GlobalIsSet(MinerLegacyGasTargetFlag.Name) {
cfg.MinerGasFloor = ctx.GlobalUint64(MinerLegacyGasTargetFlag.Name)
}
if ctx.GlobalIsSet(MinerGasTargetFlag.Name) {
cfg.MinerGasFloor = ctx.GlobalUint64(MinerGasTargetFlag.Name)
}
if ctx.GlobalIsSet(MinerGasLimitFlag.Name) {
cfg.MinerGasCeil = ctx.GlobalUint64(MinerGasLimitFlag.Name)
}
if ctx.GlobalIsSet(MinerLegacyGasPriceFlag.Name) {
cfg.MinerGasPrice = GlobalBig(ctx, MinerLegacyGasPriceFlag.Name)
}
if ctx.GlobalIsSet(MinerGasPriceFlag.Name) {
cfg.MinerGasPrice = GlobalBig(ctx, MinerGasPriceFlag.Name)
}
if ctx.GlobalIsSet(MinerRecommitIntervalFlag.Name) {
cfg.MinerRecommit = ctx.Duration(MinerRecommitIntervalFlag.Name)
}
if ctx.GlobalIsSet(MinerNoVerfiyFlag.Name) {
cfg.MinerNoverify = ctx.Bool(MinerNoVerfiyFlag.Name)
}
if ctx.GlobalIsSet(VMEnableDebugFlag.Name) {
// TODO(fjl): force-enable this in --dev mode
cfg.EnablePreimageRecording = ctx.GlobalBool(VMEnableDebugFlag.Name)
}
if ctx.GlobalIsSet(EWASMInterpreterFlag.Name) {
cfg.EWASMInterpreter = ctx.GlobalString(EWASMInterpreterFlag.Name)
}
if ctx.GlobalIsSet(EVMInterpreterFlag.Name) {
cfg.EVMInterpreter = ctx.GlobalString(EVMInterpreterFlag.Name)
}
// Override any default configs for hard coded networks.
switch {
case ctx.GlobalBool(TestnetFlag.Name):
@ -1211,8 +1309,8 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
log.Info("Using developer account", "address", developer.Address)
cfg.Genesis = core.DeveloperGenesisBlock(uint64(ctx.GlobalInt(DeveloperPeriodFlag.Name)), developer.Address)
if !ctx.GlobalIsSet(GasPriceFlag.Name) {
cfg.GasPrice = big.NewInt(1)
if !ctx.GlobalIsSet(MinerGasPriceFlag.Name) && !ctx.GlobalIsSet(MinerLegacyGasPriceFlag.Name) {
cfg.MinerGasPrice = big.NewInt(1)
}
}
// TODO(fjl): move trie cache generations into config
@ -1257,7 +1355,7 @@ func RegisterEthService(stack *node.Node, cfg *eth.Config) <-chan *eth.Ethereum
// RegisterDashboardService adds a dashboard to the stack.
func RegisterDashboardService(stack *node.Node, cfg *dashboard.Config, commit string) {
stack.Register(func(ctx *node.ServiceContext) (node.Service, error) {
return dashboard.New(cfg, commit)
return dashboard.New(cfg, commit, ctx.ResolvePath("logs")), nil
})
}
@ -1271,7 +1369,7 @@ func RegisterShhService(stack *node.Node, cfg *whisper.Config) {
}
// RegisterEthStatsService configures the Ethereum Stats daemon and adds it to
// th egiven node.
// the given node.
func RegisterEthStatsService(stack *node.Node, url string) {
if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) {
// Retrieve both eth and les services
@ -1287,12 +1385,6 @@ func RegisterEthStatsService(stack *node.Node, url string) {
}
}
// SetupNetwork configures the system for either the main net or some test network.
func SetupNetwork(ctx *cli.Context) {
// TODO(fjl): move target gas limit into config
params.TargetGasLimit = ctx.GlobalUint64(TargetGasLimitFlag.Name)
}
func SetupMetrics(ctx *cli.Context) {
if metrics.Enabled {
log.Info("Enabling metrics collection")
@ -1321,7 +1413,7 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node) ethdb.Database {
handles = makeDatabaseHandles()
)
name := "chaindata"
if ctx.GlobalBool(LightModeFlag.Name) {
if ctx.GlobalString(SyncModeFlag.Name) == "light" {
name = "lightchaindata"
}
chainDb, err := stack.OpenDatabase(name, cache, handles)
@ -1368,7 +1460,7 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai
DatasetDir: stack.ResolvePath(eth.DefaultConfig.Ethash.DatasetDir),
DatasetsInMem: eth.DefaultConfig.Ethash.DatasetsInMem,
DatasetsOnDisk: eth.DefaultConfig.Ethash.DatasetsOnDisk,
})
}, nil, false)
}
}
if gcmode := ctx.GlobalString(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" {
@ -1383,7 +1475,7 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai
cache.TrieNodeLimit = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100
}
vmcfg := vm.Config{EnablePreimageRecording: ctx.GlobalBool(VMEnableDebugFlag.Name)}
chain, err = core.NewBlockChain(chainDb, cache, config, engine, vmcfg)
chain, err = core.NewBlockChain(chainDb, cache, config, engine, vmcfg, nil)
if err != nil {
Fatalf("Can't create BlockChain: %v", err)
}

2
cmd/wnode/main.go
View File

@ -754,7 +754,7 @@ func extractIDFromEnode(s string) []byte {
return n.ID[:]
}
// obfuscateBloom adds 16 random bits to the the bloom
// obfuscateBloom adds 16 random bits to the bloom
// filter, in order to obfuscate the containing topics.
// it does so deterministically within every session.
// despite additional bits, it will match on average

4
common/bitutil/compress_test.go
View File

@ -117,7 +117,7 @@ func TestDecodingCycle(t *testing.T) {
// TestCompression tests that compression works by returning either the bitset
// encoded input, or the actual input if the bitset version is longer.
func TestCompression(t *testing.T) {
// Check the the compression returns the bitset encoding is shorter
// Check the compression returns the bitset encoding is shorter
in := hexutil.MustDecode("0x4912385c0e7b64000000")
out := hexutil.MustDecode("0x80fe4912385c0e7b64")
@ -127,7 +127,7 @@ func TestCompression(t *testing.T) {
if data, err := DecompressBytes(out, len(in)); err != nil || !bytes.Equal(data, in) {
t.Errorf("decoding mismatch for sparse data: have %x, want %x, error %v", data, in, err)
}
// Check the the compression returns the input if the bitset encoding is longer
// Check the compression returns the input if the bitset encoding is longer
in = hexutil.MustDecode("0xdf7070533534333636313639343638373532313536346c1bc33339343837313070706336343035336336346c65fefb3930393233383838ac2f65fefb")
out = hexutil.MustDecode("0xdf7070533534333636313639343638373532313536346c1bc33339343837313070706336343035336336346c65fefb3930393233383838ac2f65fefb")

2
common/bytes.go
View File

@ -100,7 +100,7 @@ func Hex2BytesFixed(str string, flen int) []byte {
return h[len(h)-flen:]
}
hh := make([]byte, flen)
copy(hh[flen-len(h):flen], h[:])
copy(hh[flen-len(h):flen], h)
return hh
}

42
common/format.go
View File

@ -38,3 +38,45 @@ func (d PrettyDuration) String() string {
}
return label
}
// PrettyAge is a pretty printed version of a time.Duration value that rounds
// the values up to a single most significant unit, days/weeks/years included.
type PrettyAge time.Time
// ageUnits is a list of units the age pretty printing uses.
var ageUnits = []struct {
Size time.Duration
Symbol string
}{
{12 * 30 * 24 * time.Hour, "y"},
{30 * 24 * time.Hour, "mo"},
{7 * 24 * time.Hour, "w"},
{24 * time.Hour, "d"},
{time.Hour, "h"},
{time.Minute, "m"},
{time.Second, "s"},
}
// String implements the Stringer interface, allowing pretty printing of duration
// values rounded to the most significant time unit.
func (t PrettyAge) String() string {
// Calculate the time difference and handle the 0 cornercase
diff := time.Since(time.Time(t))
if diff < time.Second {
return "0"
}
// Accumulate a precision of 3 components before returning
result, prec := "", 0
for _, unit := range ageUnits {
if diff > unit.Size {
result = fmt.Sprintf("%s%d%s", result, diff/unit.Size, unit.Symbol)
diff %= unit.Size
if prec += 1; prec >= 3 {
break
}
}
}
return result
}

31
common/mclock/mclock.go
View File

@ -30,3 +30,34 @@ type AbsTime time.Duration
func Now() AbsTime {
return AbsTime(monotime.Now())
}
// Add returns t + d.
func (t AbsTime) Add(d time.Duration) AbsTime {
return t + AbsTime(d)
}
// Clock interface makes it possible to replace the monotonic system clock with
// a simulated clock.
type Clock interface {
Now() AbsTime
Sleep(time.Duration)
After(time.Duration) <-chan time.Time
}
// System implements Clock using the system clock.
type System struct{}
// Now implements Clock.
func (System) Now() AbsTime {
return AbsTime(monotime.Now())
}
// Sleep implements Clock.
func (System) Sleep(d time.Duration) {
time.Sleep(d)
}
// After implements Clock.
func (System) After(d time.Duration) <-chan time.Time {
return time.After(d)
}

129
common/mclock/simclock.go Normal file
View File

@ -0,0 +1,129 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package mclock
import (
"sync"
"time"
)
// Simulated implements a virtual Clock for reproducible time-sensitive tests. It
// simulates a scheduler on a virtual timescale where actual processing takes zero time.
//
// The virtual clock doesn't advance on its own, call Run to advance it and execute timers.
// Since there is no way to influence the Go scheduler, testing timeout behaviour involving
// goroutines needs special care. A good way to test such timeouts is as follows: First
// perform the action that is supposed to time out. Ensure that the timer you want to test
// is created. Then run the clock until after the timeout. Finally observe the effect of
// the timeout using a channel or semaphore.
type Simulated struct {
now AbsTime
scheduled []event
mu sync.RWMutex
cond *sync.Cond
}
type event struct {
do func()
at AbsTime
}
// Run moves the clock by the given duration, executing all timers before that duration.
func (s *Simulated) Run(d time.Duration) {
s.mu.Lock()
defer s.mu.Unlock()
s.init()
end := s.now + AbsTime(d)
for len(s.scheduled) > 0 {
ev := s.scheduled[0]
if ev.at > end {
break
}
s.now = ev.at
ev.do()
s.scheduled = s.scheduled[1:]
}
s.now = end
}
func (s *Simulated) ActiveTimers() int {
s.mu.RLock()
defer s.mu.RUnlock()
return len(s.scheduled)
}
func (s *Simulated) WaitForTimers(n int) {
s.mu.Lock()
defer s.mu.Unlock()
s.init()
for len(s.scheduled) < n {
s.cond.Wait()
}
}
// Now implements Clock.
func (s *Simulated) Now() AbsTime {
s.mu.RLock()
defer s.mu.RUnlock()
return s.now
}
// Sleep implements Clock.
func (s *Simulated) Sleep(d time.Duration) {
<-s.After(d)
}
// After implements Clock.
func (s *Simulated) After(d time.Duration) <-chan time.Time {
after := make(chan time.Time, 1)
s.insert(d, func() {
after <- (time.Time{}).Add(time.Duration(s.now))
})
return after
}
func (s *Simulated) insert(d time.Duration, do func()) {
s.mu.Lock()
defer s.mu.Unlock()
s.init()
at := s.now + AbsTime(d)
l, h := 0, len(s.scheduled)
ll := h
for l != h {
m := (l + h) / 2
if at < s.scheduled[m].at {
h = m
} else {
l = m + 1
}
}
s.scheduled = append(s.scheduled, event{})
copy(s.scheduled[l+1:], s.scheduled[l:ll])
s.scheduled[l] = event{do: do, at: at}
s.cond.Broadcast()
}
func (s *Simulated) init() {
if s.cond == nil {
s.cond = sync.NewCond(&s.mu)
}
}

57
common/prque/prque.go Executable file
View File

@ -0,0 +1,57 @@
// This is a duplicated and slightly modified version of "gopkg.in/karalabe/cookiejar.v2/collections/prque".
package prque
import (
"container/heap"
)
// Priority queue data structure.
type Prque struct {
cont *sstack
}
// Creates a new priority queue.
func New(setIndex setIndexCallback) *Prque {
return &Prque{newSstack(setIndex)}
}
// Pushes a value with a given priority into the queue, expanding if necessary.
func (p *Prque) Push(data interface{}, priority int64) {
heap.Push(p.cont, &item{data, priority})
}
// Pops the value with the greates priority off the stack and returns it.
// Currently no shrinking is done.
func (p *Prque) Pop() (interface{}, int64) {
item := heap.Pop(p.cont).(*item)
return item.value, item.priority
}
// Pops only the item from the queue, dropping the associated priority value.
func (p *Prque) PopItem() interface{} {
return heap.Pop(p.cont).(*item).value
}
// Remove removes the element with the given index.
func (p *Prque) Remove(i int) interface{} {
if i < 0 {
return nil
}
return heap.Remove(p.cont, i)
}
// Checks whether the priority queue is empty.
func (p *Prque) Empty() bool {
return p.cont.Len() == 0
}
// Returns the number of element in the priority queue.
func (p *Prque) Size() int {
return p.cont.Len()
}
// Clears the contents of the priority queue.
func (p *Prque) Reset() {
*p = *New(p.cont.setIndex)
}

39
vendor/gopkg.in/karalabe/cookiejar.v2/collections/prque/sstack.go → common/prque/sstack.go
View File

@ -1,10 +1,4 @@
// CookieJar - A contestant's algorithm toolbox
// Copyright (c) 2013 Peter Szilagyi. All rights reserved.
//
// CookieJar is dual licensed: use of this source code is governed by a BSD
// license that can be found in the LICENSE file. Alternatively, the CookieJar
// toolbox may be used in accordance with the terms and conditions contained
// in a signed written agreement between you and the author(s).
// This is a duplicated and slightly modified version of "gopkg.in/karalabe/cookiejar.v2/collections/prque".
package prque
@ -12,15 +6,24 @@ package prque
const blockSize = 4096
// A prioritized item in the sorted stack.
//
// Note: priorities can "wrap around" the int64 range, a comes before b if (a.priority - b.priority) > 0.
// The difference between the lowest and highest priorities in the queue at any point should be less than 2^63.
type item struct {
value interface{}
priority float32
priority int64
}
// setIndexCallback is called when the element is moved to a new index.
// Providing setIndexCallback is optional, it is needed only if the application needs
// to delete elements other than the top one.
type setIndexCallback func(a interface{}, i int)
// Internal sortable stack data structure. Implements the Push and Pop ops for
// the stack (heap) functionality and the Len, Less and Swap methods for the
// sortability requirements of the heaps.
type sstack struct {
setIndex setIndexCallback
size int
capacity int
offset int
@ -30,8 +33,9 @@ type sstack struct {
}
// Creates a new, empty stack.
func newSstack() *sstack {
func newSstack(setIndex setIndexCallback) *sstack {
result := new(sstack)
result.setIndex = setIndex
result.active = make([]*item, blockSize)
result.blocks = [][]*item{result.active}
result.capacity = blockSize
@ -50,6 +54,9 @@ func (s *sstack) Push(data interface{}) {
s.active = s.blocks[s.size/blockSize]
s.offset = 0
}
if s.setIndex != nil {
s.setIndex(data.(*item).value, s.size)
}
s.active[s.offset] = data.(*item)
s.offset++
s.size++
@ -65,6 +72,9 @@ func (s *sstack) Pop() (res interface{}) {
s.active = s.blocks[s.size/blockSize]
}
res, s.active[s.offset] = s.active[s.offset], nil
if s.setIndex != nil {
s.setIndex(res.(*item).value, -1)
}
return
}
@ -76,16 +86,21 @@ func (s *sstack) Len() int {
// Compares the priority of two elements of the stack (higher is first).
// Required by sort.Interface.
func (s *sstack) Less(i, j int) bool {
return s.blocks[i/blockSize][i%blockSize].priority > s.blocks[j/blockSize][j%blockSize].priority
return (s.blocks[i/blockSize][i%blockSize].priority - s.blocks[j/blockSize][j%blockSize].priority) > 0
}
// Swaps two elements in the stack. Required by sort.Interface.
func (s *sstack) Swap(i, j int) {
ib, io, jb, jo := i/blockSize, i%blockSize, j/blockSize, j%blockSize
s.blocks[ib][io], s.blocks[jb][jo] = s.blocks[jb][jo], s.blocks[ib][io]
a, b := s.blocks[jb][jo], s.blocks[ib][io]
if s.setIndex != nil {
s.setIndex(a.value, i)
s.setIndex(b.value, j)
}
s.blocks[ib][io], s.blocks[jb][jo] = a, b
}
// Resets the stack, effectively clearing its contents.
func (s *sstack) Reset() {
*s = *newSstack()
*s = *newSstack(s.setIndex)
}

41
common/types.go
View File

@ -17,6 +17,7 @@
package common
import (
"database/sql/driver"
"encoding/hex"
"encoding/json"
"fmt"
@ -31,7 +32,9 @@ import (
// Lengths of hashes and addresses in bytes.
const (
HashLength = 32
// HashLength is the expected length of the hash
HashLength = 32
// AddressLength is the expected length of the address
AddressLength = 20
)
@ -126,6 +129,24 @@ func (h Hash) Generate(rand *rand.Rand, size int) reflect.Value {
return reflect.ValueOf(h)
}
// Scan implements Scanner for database/sql.
func (h *Hash) Scan(src interface{}) error {
srcB, ok := src.([]byte)
if !ok {
return fmt.Errorf("can't scan %T into Hash", src)
}
if len(srcB) != HashLength {
return fmt.Errorf("can't scan []byte of len %d into Hash, want %d", len(srcB), HashLength)
}
copy(h[:], srcB)
return nil
}
// Value implements valuer for database/sql.
func (h Hash) Value() (driver.Value, error) {
return h[:], nil
}
// UnprefixedHash allows marshaling a Hash without 0x prefix.
type UnprefixedHash Hash
@ -238,6 +259,24 @@ func (a *Address) UnmarshalJSON(input []byte) error {
return hexutil.UnmarshalFixedJSON(addressT, input, a[:])
}
// Scan implements Scanner for database/sql.
func (a *Address) Scan(src interface{}) error {
srcB, ok := src.([]byte)
if !ok {
return fmt.Errorf("can't scan %T into Address", src)
}
if len(srcB) != AddressLength {
return fmt.Errorf("can't scan []byte of len %d into Address, want %d", len(srcB), AddressLength)
}
copy(a[:], srcB)
return nil
}
// Value implements valuer for database/sql.
func (a Address) Value() (driver.Value, error) {
return a[:], nil
}
// UnprefixedAddress allows marshaling an Address without 0x prefix.
type UnprefixedAddress Address

180
common/types_test.go
View File

@ -17,9 +17,10 @@
package common
import (
"database/sql/driver"
"encoding/json"
"math/big"
"reflect"
"strings"
"testing"
)
@ -193,3 +194,180 @@ func TestMixedcaseAccount_Address(t *testing.T) {
}
}
func TestHash_Scan(t *testing.T) {
type args struct {
src interface{}
}
tests := []struct {
name string
args args
wantErr bool
}{
{
name: "working scan",
args: args{src: []byte{
0xb2, 0x6f, 0x2b, 0x34, 0x2a, 0xab, 0x24, 0xbc, 0xf6, 0x3e,
0xa2, 0x18, 0xc6, 0xa9, 0x27, 0x4d, 0x30, 0xab, 0x9a, 0x15,
0xa2, 0x18, 0xc6, 0xa9, 0x27, 0x4d, 0x30, 0xab, 0x9a, 0x15,
0x10, 0x00,
}},
wantErr: false,
},
{
name: "non working scan",
args: args{src: int64(1234567890)},
wantErr: true,
},
{
name: "invalid length scan",
args: args{src: []byte{
0xb2, 0x6f, 0x2b, 0x34, 0x2a, 0xab, 0x24, 0xbc, 0xf6, 0x3e,
0xa2, 0x18, 0xc6, 0xa9, 0x27, 0x4d, 0x30, 0xab, 0x9a, 0x15,
0xa2, 0x18, 0xc6, 0xa9, 0x27, 0x4d, 0x30, 0xab, 0x9a, 0x15,
}},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
h := &Hash{}
if err := h.Scan(tt.args.src); (err != nil) != tt.wantErr {
t.Errorf("Hash.Scan() error = %v, wantErr %v", err, tt.wantErr)
}
if !tt.wantErr {
for i := range h {
if h[i] != tt.args.src.([]byte)[i] {
t.Errorf(
"Hash.Scan() didn't scan the %d src correctly (have %X, want %X)",
i, h[i], tt.args.src.([]byte)[i],
)
}
}
}
})
}
}
func TestHash_Value(t *testing.T) {
b := []byte{
0xb2, 0x6f, 0x2b, 0x34, 0x2a, 0xab, 0x24, 0xbc, 0xf6, 0x3e,
0xa2, 0x18, 0xc6, 0xa9, 0x27, 0x4d, 0x30, 0xab, 0x9a, 0x15,
0xa2, 0x18, 0xc6, 0xa9, 0x27, 0x4d, 0x30, 0xab, 0x9a, 0x15,
0x10, 0x00,
}
var usedH Hash
usedH.SetBytes(b)
tests := []struct {
name string
h Hash
want driver.Value
wantErr bool
}{
{
name: "Working value",
h: usedH,
want: b,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.h.Value()
if (err != nil) != tt.wantErr {
t.Errorf("Hash.Value() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("Hash.Value() = %v, want %v", got, tt.want)
}
})
}
}
func TestAddress_Scan(t *testing.T) {
type args struct {
src interface{}
}
tests := []struct {
name string
args args
wantErr bool
}{
{
name: "working scan",
args: args{src: []byte{
0xb2, 0x6f, 0x2b, 0x34, 0x2a, 0xab, 0x24, 0xbc, 0xf6, 0x3e,
0xa2, 0x18, 0xc6, 0xa9, 0x27, 0x4d, 0x30, 0xab, 0x9a, 0x15,
}},
wantErr: false,
},
{
name: "non working scan",
args: args{src: int64(1234567890)},
wantErr: true,
},
{
name: "invalid length scan",
args: args{src: []byte{
0xb2, 0x6f, 0x2b, 0x34, 0x2a, 0xab, 0x24, 0xbc, 0xf6, 0x3e,
0xa2, 0x18, 0xc6, 0xa9, 0x27, 0x4d, 0x30, 0xab, 0x9a,
}},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
a := &Address{}
if err := a.Scan(tt.args.src); (err != nil) != tt.wantErr {
t.Errorf("Address.Scan() error = %v, wantErr %v", err, tt.wantErr)
}
if !tt.wantErr {
for i := range a {
if a[i] != tt.args.src.([]byte)[i] {
t.Errorf(
"Address.Scan() didn't scan the %d src correctly (have %X, want %X)",
i, a[i], tt.args.src.([]byte)[i],
)
}
}
}
})
}
}
func TestAddress_Value(t *testing.T) {
b := []byte{
0xb2, 0x6f, 0x2b, 0x34, 0x2a, 0xab, 0x24, 0xbc, 0xf6, 0x3e,
0xa2, 0x18, 0xc6, 0xa9, 0x27, 0x4d, 0x30, 0xab, 0x9a, 0x15,
}
var usedA Address
usedA.SetBytes(b)
tests := []struct {
name string
a Address
want driver.Value
wantErr bool
}{
{
name: "Working value",
a: usedA,
want: b,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.a.Value()
if (err != nil) != tt.wantErr {
t.Errorf("Address.Value() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("Address.Value() = %v, want %v", got, tt.want)
}
})
}
}

2
consensus/clique/api.go
View File

@ -75,7 +75,7 @@ func (api *API) GetSigners(number *rpc.BlockNumber) ([]common.Address, error) {
return snap.signers(), nil
}
// GetSignersAtHash retrieves the state snapshot at a given block.
// GetSignersAtHash retrieves the list of authorized signers at the specified block.
func (api *API) GetSignersAtHash(hash common.Hash) ([]common.Address, error) {
header := api.chain.GetHeaderByHash(hash)
if header == nil {

132
consensus/clique/clique.go
View File

@ -53,7 +53,6 @@ const (
// Clique proof-of-authority protocol constants.
var (
epochLength = uint64(30000) // Default number of blocks after which to checkpoint and reset the pending votes
blockPeriod = uint64(15) // Default minimum difference between two consecutive block's timestamps
extraVanity = 32 // Fixed number of extra-data prefix bytes reserved for signer vanity
extraSeal = 65 // Fixed number of extra-data suffix bytes reserved for signer seal
@ -94,27 +93,33 @@ var (
// errMissingSignature is returned if a block's extra-data section doesn't seem
// to contain a 65 byte secp256k1 signature.
errMissingSignature = errors.New("extra-data 65 byte suffix signature missing")
errMissingSignature = errors.New("extra-data 65 byte signature suffix missing")
// errExtraSigners is returned if non-checkpoint block contain signer data in
// their extra-data fields.
errExtraSigners = errors.New("non-checkpoint block contains extra signer list")
// errInvalidCheckpointSigners is returned if a checkpoint block contains an
// invalid list of signers (i.e. non divisible by 20 bytes, or not the correct
// ones).
// invalid list of signers (i.e. non divisible by 20 bytes).
errInvalidCheckpointSigners = errors.New("invalid signer list on checkpoint block")
// errMismatchingCheckpointSigners is returned if a checkpoint block contains a
// list of signers different than the one the local node calculated.
errMismatchingCheckpointSigners = errors.New("mismatching signer list on checkpoint block")
// errInvalidMixDigest is returned if a block's mix digest is non-zero.
errInvalidMixDigest = errors.New("non-zero mix digest")
// errInvalidUncleHash is returned if a block contains an non-empty uncle list.
errInvalidUncleHash = errors.New("non empty uncle hash")
// errInvalidDifficulty is returned if the difficulty of a block is not either
// of 1 or 2, or if the value does not match the turn of the signer.
// errInvalidDifficulty is returned if the difficulty of a block neither 1 or 2.
errInvalidDifficulty = errors.New("invalid difficulty")
// errWrongDifficulty is returned if the difficulty of a block doesn't match the
// turn of the signer.
errWrongDifficulty = errors.New("wrong difficulty")
// ErrInvalidTimestamp is returned if the timestamp of a block is lower than
// the previous block's timestamp + the minimum block period.
ErrInvalidTimestamp = errors.New("invalid timestamp")
@ -123,13 +128,12 @@ var (
// be modified via out-of-range or non-contiguous headers.
errInvalidVotingChain = errors.New("invalid voting chain")
// errUnauthorized is returned if a header is signed by a non-authorized entity.
errUnauthorized = errors.New("unauthorized")
// errUnauthorizedSigner is returned if a header is signed by a non-authorized entity.
errUnauthorizedSigner = errors.New("unauthorized signer")
// errWaitTransactions is returned if an empty block is attempted to be sealed
// on an instant chain (0 second period). It's important to refuse these as the
// block reward is zero, so an empty block just bloats the chain... fast.
errWaitTransactions = errors.New("waiting for transactions")
// errRecentlySigned is returned if a header is signed by an authorized entity
// that already signed a header recently, thus is temporarily not allowed to.
errRecentlySigned = errors.New("recently signed")
)
// SignerFn is a signer callback function to request a hash to be signed by a
@ -206,6 +210,9 @@ type Clique struct {
signer common.Address // Ethereum address of the signing key
signFn SignerFn // Signer function to authorize hashes with
lock sync.RWMutex // Protects the signer fields
// The fields below are for testing only
fakeDiff bool // Skip difficulty verifications
}
// New creates a Clique proof-of-authority consensus engine with the initial
@ -360,7 +367,7 @@ func (c *Clique) verifyCascadingFields(chain consensus.ChainReader, header *type
}
extraSuffix := len(header.Extra) - extraSeal
if !bytes.Equal(header.Extra[extraVanity:extraSuffix], signers) {
return errInvalidCheckpointSigners
return errMismatchingCheckpointSigners
}
}
// All basic checks passed, verify the seal and return
@ -388,22 +395,23 @@ func (c *Clique) snapshot(chain consensus.ChainReader, number uint64, hash commo
break
}
}
// If we're at block zero, make a snapshot
if number == 0 {
genesis := chain.GetHeaderByNumber(0)
if err := c.VerifyHeader(chain, genesis, false); err != nil {
return nil, err
// If we're at an checkpoint block, make a snapshot if it's known
if number == 0 || (number%c.config.Epoch == 0 && chain.GetHeaderByNumber(number-1) == nil) {
checkpoint := chain.GetHeaderByNumber(number)
if checkpoint != nil {
hash := checkpoint.Hash()
signers := make([]common.Address, (len(checkpoint.Extra)-extraVanity-extraSeal)/common.AddressLength)
for i := 0; i < len(signers); i++ {
copy(signers[i][:], checkpoint.Extra[extraVanity+i*common.AddressLength:])
}
snap = newSnapshot(c.config, c.signatures, number, hash, signers)
if err := snap.store(c.db); err != nil {
return nil, err
}
log.Info("Stored checkpoint snapshot to disk", "number", number, "hash", hash)
break
}
signers := make([]common.Address, (len(genesis.Extra)-extraVanity-extraSeal)/common.AddressLength)
for i := 0; i < len(signers); i++ {
copy(signers[i][:], genesis.Extra[extraVanity+i*common.AddressLength:])
}
snap = newSnapshot(c.config, c.signatures, 0, genesis.Hash(), signers)
if err := snap.store(c.db); err != nil {
return nil, err
}
log.Trace("Stored genesis voting snapshot to disk")
break
}
// No snapshot for this header, gather the header and move backward
var header *types.Header
@ -481,23 +489,25 @@ func (c *Clique) verifySeal(chain consensus.ChainReader, header *types.Header, p
return err
}
if _, ok := snap.Signers[signer]; !ok {
return errUnauthorized
return errUnauthorizedSigner
}
for seen, recent := range snap.Recents {
if recent == signer {
// Signer is among recents, only fail if the current block doesn't shift it out
if limit := uint64(len(snap.Signers)/2 + 1); seen > number-limit {
return errUnauthorized
return errRecentlySigned
}
}
}
// Ensure that the difficulty corresponds to the turn-ness of the signer
inturn := snap.inturn(header.Number.Uint64(), signer)
if inturn && header.Difficulty.Cmp(diffInTurn) != 0 {
return errInvalidDifficulty
}
if !inturn && header.Difficulty.Cmp(diffNoTurn) != 0 {
return errInvalidDifficulty
if !c.fakeDiff {
inturn := snap.inturn(header.Number.Uint64(), signer)
if inturn && header.Difficulty.Cmp(diffInTurn) != 0 {
return errWrongDifficulty
}
if !inturn && header.Difficulty.Cmp(diffNoTurn) != 0 {
return errWrongDifficulty
}
}
return nil
}
@ -590,17 +600,18 @@ func (c *Clique) Authorize(signer common.Address, signFn SignerFn) {
// Seal implements consensus.Engine, attempting to create a sealed block using
// the local signing credentials.
func (c *Clique) Seal(chain consensus.ChainReader, block *types.Block, stop <-chan struct{}) (*types.Block, error) {
func (c *Clique) Seal(chain consensus.ChainReader, block *types.Block, results chan<- *types.Block, stop <-chan struct{}) error {
header := block.Header()
// Sealing the genesis block is not supported
number := header.Number.Uint64()
if number == 0 {
return nil, errUnknownBlock
return errUnknownBlock
}
// For 0-period chains, refuse to seal empty blocks (no reward but would spin sealing)
if c.config.Period == 0 && len(block.Transactions()) == 0 {
return nil, errWaitTransactions
log.Info("Sealing paused, waiting for transactions")
return nil
}
// Don't hold the signer fields for the entire sealing procedure
c.lock.RLock()
@ -610,10 +621,10 @@ func (c *Clique) Seal(chain consensus.ChainReader, block *types.Block, stop <-ch
// Bail out if we're unauthorized to sign a block
snap, err := c.snapshot(chain, number-1, header.ParentHash, nil)
if err != nil {
return nil, err
return err
}
if _, authorized := snap.Signers[signer]; !authorized {
return nil, errUnauthorized
return errUnauthorizedSigner
}
// If we're amongst the recent signers, wait for the next block
for seen, recent := range snap.Recents {
@ -621,8 +632,7 @@ func (c *Clique) Seal(chain consensus.ChainReader, block *types.Block, stop <-ch
// Signer is among recents, only wait if the current block doesn't shift it out
if limit := uint64(len(snap.Signers)/2 + 1); number < limit || seen > number-limit {
log.Info("Signed recently, must wait for others")
<-stop
return nil, nil
return nil
}
}
}
@ -635,21 +645,29 @@ func (c *Clique) Seal(chain consensus.ChainReader, block *types.Block, stop <-ch
log.Trace("Out-of-turn signing requested", "wiggle", common.PrettyDuration(wiggle))
}
log.Trace("Waiting for slot to sign and propagate", "delay", common.PrettyDuration(delay))
select {
case <-stop:
return nil, nil
case <-time.After(delay):
}
// Sign all the things!
sighash, err := signFn(accounts.Account{Address: signer}, sigHash(header).Bytes())
if err != nil {
return nil, err
return err
}
copy(header.Extra[len(header.Extra)-extraSeal:], sighash)
// Wait until sealing is terminated or delay timeout.
log.Trace("Waiting for slot to sign and propagate", "delay", common.PrettyDuration(delay))
go func() {
select {
case <-stop:
return
case <-time.After(delay):
}
return block.WithSeal(header), nil
select {
case results <- block.WithSeal(header):
default:
log.Warn("Sealing result is not read by miner", "sealhash", c.SealHash(header))
}
}()
return nil
}
// CalcDifficulty is the difficulty adjustment algorithm. It returns the difficulty
@ -673,6 +691,16 @@ func CalcDifficulty(snap *Snapshot, signer common.Address) *big.Int {
return new(big.Int).Set(diffNoTurn)
}
// SealHash returns the hash of a block prior to it being sealed.
func (c *Clique) SealHash(header *types.Header) common.Hash {
return sigHash(header)
}
// Close implements consensus.Engine. It's a noop for clique as there is are no background threads.
func (c *Clique) Close() error {
return nil
}
// APIs implements consensus.Engine, returning the user facing RPC API to allow
// controlling the signer voting.
func (c *Clique) APIs(chain consensus.ChainReader) []rpc.API {

28
consensus/clique/snapshot.go
View File

@ -19,6 +19,7 @@ package clique
import (
"bytes"
"encoding/json"
"sort"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
@ -56,6 +57,13 @@ type Snapshot struct {
Tally map[common.Address]Tally `json:"tally"` // Current vote tally to avoid recalculating
}
// signersAscending implements the sort interface to allow sorting a list of addresses
type signersAscending []common.Address
func (s signersAscending) Len() int { return len(s) }
func (s signersAscending) Less(i, j int) bool { return bytes.Compare(s[i][:], s[j][:]) < 0 }
func (s signersAscending) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
// newSnapshot creates a new snapshot with the specified startup parameters. This
// method does not initialize the set of recent signers, so only ever use if for
// the genesis block.
@ -206,11 +214,11 @@ func (s *Snapshot) apply(headers []*types.Header) (*Snapshot, error) {
return nil, err
}
if _, ok := snap.Signers[signer]; !ok {
return nil, errUnauthorized
return nil, errUnauthorizedSigner
}
for _, recent := range snap.Recents {
if recent == signer {
return nil, errUnauthorized
return nil, errRecentlySigned
}
}
snap.Recents[number] = signer
@ -286,18 +294,12 @@ func (s *Snapshot) apply(headers []*types.Header) (*Snapshot, error) {
// signers retrieves the list of authorized signers in ascending order.
func (s *Snapshot) signers() []common.Address {
signers := make([]common.Address, 0, len(s.Signers))
for signer := range s.Signers {
signers = append(signers, signer)
sigs := make([]common.Address, 0, len(s.Signers))
for sig := range s.Signers {
sigs = append(sigs, sig)
}
for i := 0; i < len(signers); i++ {
for j := i + 1; j < len(signers); j++ {
if bytes.Compare(signers[i][:], signers[j][:]) > 0 {
signers[i], signers[j] = signers[j], signers[i]
}
}
}
return signers
sort.Sort(signersAscending(sigs))
return sigs
}
// inturn returns if a signer at a given block height is in-turn or not.

194
consensus/clique/snapshot_test.go
View File

@ -19,24 +19,18 @@ package clique
import (
"bytes"
"crypto/ecdsa"
"math/big"
"sort"
"testing"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/params"
)
type testerVote struct {
signer string
voted string
auth bool
}
// testerAccountPool is a pool to maintain currently active tester accounts,
// mapped from textual names used in the tests below to actual Ethereum private
// keys capable of signing transactions.
@ -50,17 +44,26 @@ func newTesterAccountPool() *testerAccountPool {
}
}
func (ap *testerAccountPool) sign(header *types.Header, signer string) {
// Ensure we have a persistent key for the signer
if ap.accounts[signer] == nil {
ap.accounts[signer], _ = crypto.GenerateKey()
// checkpoint creates a Clique checkpoint signer section from the provided list
// of authorized signers and embeds it into the provided header.
func (ap *testerAccountPool) checkpoint(header *types.Header, signers []string) {
auths := make([]common.Address, len(signers))
for i, signer := range signers {
auths[i] = ap.address(signer)
}
sort.Sort(signersAscending(auths))
for i, auth := range auths {
copy(header.Extra[extraVanity+i*common.AddressLength:], auth.Bytes())
}
// Sign the header and embed the signature in extra data
sig, _ := crypto.Sign(sigHash(header).Bytes(), ap.accounts[signer])
copy(header.Extra[len(header.Extra)-65:], sig)
}
// address retrieves the Ethereum address of a tester account by label, creating
// a new account if no previous one exists yet.
func (ap *testerAccountPool) address(account string) common.Address {
// Return the zero account for non-addresses
if account == "" {
return common.Address{}
}
// Ensure we have a persistent key for the account
if ap.accounts[account] == nil {
ap.accounts[account], _ = crypto.GenerateKey()
@ -69,32 +72,38 @@ func (ap *testerAccountPool) address(account string) common.Address {
return crypto.PubkeyToAddress(ap.accounts[account].PublicKey)
}
// testerChainReader implements consensus.ChainReader to access the genesis
// block. All other methods and requests will panic.
type testerChainReader struct {
db ethdb.Database
}
func (r *testerChainReader) Config() *params.ChainConfig { return params.AllCliqueProtocolChanges }
func (r *testerChainReader) CurrentHeader() *types.Header { panic("not supported") }
func (r *testerChainReader) GetHeader(common.Hash, uint64) *types.Header { panic("not supported") }
func (r *testerChainReader) GetBlock(common.Hash, uint64) *types.Block { panic("not supported") }
func (r *testerChainReader) GetHeaderByHash(common.Hash) *types.Header { panic("not supported") }
func (r *testerChainReader) GetHeaderByNumber(number uint64) *types.Header {
if number == 0 {
return rawdb.ReadHeader(r.db, rawdb.ReadCanonicalHash(r.db, 0), 0)
// sign calculates a Clique digital signature for the given block and embeds it
// back into the header.
func (ap *testerAccountPool) sign(header *types.Header, signer string) {
// Ensure we have a persistent key for the signer
if ap.accounts[signer] == nil {
ap.accounts[signer], _ = crypto.GenerateKey()
}
panic("not supported")
// Sign the header and embed the signature in extra data
sig, _ := crypto.Sign(sigHash(header).Bytes(), ap.accounts[signer])
copy(header.Extra[len(header.Extra)-extraSeal:], sig)
}
// Tests that voting is evaluated correctly for various simple and complex scenarios.
func TestVoting(t *testing.T) {
// testerVote represents a single block signed by a parcitular account, where
// the account may or may not have cast a Clique vote.
type testerVote struct {
signer string
voted string
auth bool
checkpoint []string
newbatch bool
}
// Tests that Clique signer voting is evaluated correctly for various simple and
// complex scenarios, as well as that a few special corner cases fail correctly.
func TestClique(t *testing.T) {
// Define the various voting scenarios to test
tests := []struct {
epoch uint64
signers []string
votes []testerVote
results []string
failure error
}{
{
// Single signer, no votes cast
@ -322,10 +331,49 @@ func TestVoting(t *testing.T) {
votes: []testerVote{
{signer: "A", voted: "C", auth: true},
{signer: "B"},
{signer: "A"}, // Checkpoint block, (don't vote here, it's validated outside of snapshots)
{signer: "A", checkpoint: []string{"A", "B"}},
{signer: "B", voted: "C", auth: true},
},
results: []string{"A", "B"},
}, {
// An unauthorized signer should not be able to sign blocks
signers: []string{"A"},
votes: []testerVote{
{signer: "B"},
},
failure: errUnauthorizedSigner,
}, {
// An authorized signer that signed recenty should not be able to sign again
signers: []string{"A", "B"},
votes: []testerVote{
{signer: "A"},
{signer: "A"},
},
failure: errRecentlySigned,
}, {
// Recent signatures should not reset on checkpoint blocks imported in a batch
epoch: 3,
signers: []string{"A", "B", "C"},
votes: []testerVote{
{signer: "A"},
{signer: "B"},
{signer: "A", checkpoint: []string{"A", "B", "C"}},
{signer: "A"},
},
failure: errRecentlySigned,
}, {
// Recent signatures should not reset on checkpoint blocks imported in a new
// batch (https://github.com/ethereum/go-ethereum/issues/17593). Whilst this
// seems overly specific and weird, it was a Rinkeby consensus split.
epoch: 3,
signers: []string{"A", "B", "C"},
votes: []testerVote{
{signer: "A"},
{signer: "B"},
{signer: "A", checkpoint: []string{"A", "B", "C"}},
{signer: "A", newbatch: true},
},
failure: errRecentlySigned,
},
}
// Run through the scenarios and test them
@ -356,28 +404,78 @@ func TestVoting(t *testing.T) {
genesis.Commit(db)
// Assemble a chain of headers from the cast votes
headers := make([]*types.Header, len(tt.votes))
for j, vote := range tt.votes {
headers[j] = &types.Header{
Number: big.NewInt(int64(j) + 1),
Time: big.NewInt(int64(j) * int64(blockPeriod)),
Coinbase: accounts.address(vote.voted),
Extra: make([]byte, extraVanity+extraSeal),
config := *params.TestChainConfig
config.Clique = &params.CliqueConfig{
Period: 1,
Epoch: tt.epoch,
}
engine := New(config.Clique, db)
engine.fakeDiff = true
blocks, _ := core.GenerateChain(&config, genesis.ToBlock(db), engine, db, len(tt.votes), func(j int, gen *core.BlockGen) {
// Cast the vote contained in this block
gen.SetCoinbase(accounts.address(tt.votes[j].voted))
if tt.votes[j].auth {
var nonce types.BlockNonce
copy(nonce[:], nonceAuthVote)
gen.SetNonce(nonce)
}
})
// Iterate through the blocks and seal them individually
for j, block := range blocks {
// Geth the header and prepare it for signing
header := block.Header()
if j > 0 {
headers[j].ParentHash = headers[j-1].Hash()
header.ParentHash = blocks[j-1].Hash()
}
if vote.auth {
copy(headers[j].Nonce[:], nonceAuthVote)
header.Extra = make([]byte, extraVanity+extraSeal)
if auths := tt.votes[j].checkpoint; auths != nil {
header.Extra = make([]byte, extraVanity+len(auths)*common.AddressLength+extraSeal)
accounts.checkpoint(header, auths)
}
accounts.sign(headers[j], vote.signer)
header.Difficulty = diffInTurn // Ignored, we just need a valid number
// Generate the signature, embed it into the header and the block
accounts.sign(header, tt.votes[j].signer)
blocks[j] = block.WithSeal(header)
}
// Split the blocks up into individual import batches (cornercase testing)
batches := [][]*types.Block{nil}
for j, block := range blocks {
if tt.votes[j].newbatch {
batches = append(batches, nil)
}
batches[len(batches)-1] = append(batches[len(batches)-1], block)
}
// Pass all the headers through clique and ensure tallying succeeds
head := headers[len(headers)-1]
snap, err := New(&params.CliqueConfig{Epoch: tt.epoch}, db).snapshot(&testerChainReader{db: db}, head.Number.Uint64(), head.Hash(), headers)
chain, err := core.NewBlockChain(db, nil, &config, engine, vm.Config{}, nil)
if err != nil {
t.Errorf("test %d: failed to create voting snapshot: %v", i, err)
t.Errorf("test %d: failed to create test chain: %v", i, err)
continue
}
failed := false
for j := 0; j < len(batches)-1; j++ {
if k, err := chain.InsertChain(batches[j]); err != nil {
t.Errorf("test %d: failed to import batch %d, block %d: %v", i, j, k, err)
failed = true
break
}
}
if failed {
continue
}
if _, err = chain.InsertChain(batches[len(batches)-1]); err != tt.failure {
t.Errorf("test %d: failure mismatch: have %v, want %v", i, err, tt.failure)
}
if tt.failure != nil {
continue
}
// No failure was produced or requested, generate the final voting snapshot
head := blocks[len(blocks)-1]
snap, err := engine.snapshot(chain, head.NumberU64(), head.Hash(), nil)
if err != nil {
t.Errorf("test %d: failed to retrieve voting snapshot: %v", i, err)
continue
}
// Verify the final list of signers against the expected ones

15
consensus/consensus.go
View File

@ -87,9 +87,15 @@ type Engine interface {
Finalize(chain ChainReader, header *types.Header, state *state.StateDB, txs []*types.Transaction,
uncles []*types.Header, receipts []*types.Receipt) (*types.Block, error)
// Seal generates a new block for the given input block with the local miner's
// seal place on top.
Seal(chain ChainReader, block *types.Block, stop <-chan struct{}) (*types.Block, error)
// Seal generates a new sealing request for the given input block and pushes
// the result into the given channel.
//
// Note, the method returns immediately and will send the result async. More
// than one result may also be returned depending on the consensus algorithm.
Seal(chain ChainReader, block *types.Block, results chan<- *types.Block, stop <-chan struct{}) error
// SealHash returns the hash of a block prior to it being sealed.
SealHash(header *types.Header) common.Hash
// CalcDifficulty is the difficulty adjustment algorithm. It returns the difficulty
// that a new block should have.
@ -100,6 +106,9 @@ type Engine interface {
// Protocol returns the protocol for this consensus
Protocol() Protocol
// Close terminates any background threads maintained by the consensus engine.
Close() error
}
// Handler should be implemented is the consensus needs to handle and send peer's message

Some files were not shown because too many files have changed in this diff Show More