From e7bdb0070058cb03a1ff09c7e7ab8893f54a9d7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 23 Feb 2017 16:24:32 +0200 Subject: [PATCH 1/3] cmd/gethrpctest: ethereum/rpc-tests is deprecated Only ethereum/rpc-tests used this command, which hasn't been maintained for over a year now, a lot of tests failing. What's left of it was moved underneath hive, which can run the entire test against a black-box geth without special commands. Also a new RPC test suite is being added which is also based on black box tests, not needing special commands any more. --- cmd/gethrpctest/main.go | 161 ---------------------------------------- 1 file changed, 161 deletions(-) delete mode 100644 cmd/gethrpctest/main.go diff --git a/cmd/gethrpctest/main.go b/cmd/gethrpctest/main.go deleted file mode 100644 index 69a6074e7..000000000 --- a/cmd/gethrpctest/main.go +++ /dev/null @@ -1,161 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of go-ethereum. -// -// go-ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// go-ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with go-ethereum. If not, see . - -// gethrpctest is a command to run the external RPC tests. -package main - -import ( - "flag" - "fmt" - "os" - "os/signal" - - "github.com/ethereum/go-ethereum/accounts/keystore" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/eth" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/node" - "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/tests" - whisper "github.com/ethereum/go-ethereum/whisper/whisperv2" -) - -const defaultTestKey = "b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291" - -var ( - testFile = flag.String("json", "", "Path to the .json test file to load") - testName = flag.String("test", "", "Name of the test from the .json file to run") - testKey = flag.String("key", defaultTestKey, "Private key of a test account to inject") -) - -func main() { - flag.Parse() - - // Enable logging errors, we really do want to see those - log.Root().SetHandler(log.LvlFilterHandler(log.LvlError, log.StreamHandler(os.Stderr, log.TerminalFormat()))) - - // Load the test suite to run the RPC against - tests, err := tests.LoadBlockTests(*testFile) - if err != nil { - log.Crit(fmt.Sprintf("Failed to load test suite: %v", err)) - } - test, found := tests[*testName] - if !found { - log.Crit(fmt.Sprintf("Requested test (%s) not found within suite", *testName)) - } - - stack, err := MakeSystemNode(*testKey, test) - if err != nil { - log.Crit(fmt.Sprintf("Failed to assemble test stack: %v", err)) - } - if err := stack.Start(); err != nil { - log.Crit(fmt.Sprintf("Failed to start test node: %v", err)) - } - defer stack.Stop() - - log.Info("Test node started...") - - // Make sure the tests contained within the suite pass - if err := RunTest(stack, test); err != nil { - log.Crit(fmt.Sprintf("Failed to run the pre-configured test: %v", err)) - } - log.Info("Initial test suite passed...") - - quit := make(chan os.Signal, 1) - signal.Notify(quit, os.Interrupt) - <-quit -} - -// MakeSystemNode configures a protocol stack for the RPC tests based on a given -// keystore path and initial pre-state. -func MakeSystemNode(privkey string, test *tests.BlockTest) (*node.Node, error) { - // Create a networkless protocol stack - stack, err := node.New(&node.Config{ - UseLightweightKDF: true, - IPCPath: node.DefaultIPCEndpoint(""), - HTTPHost: node.DefaultHTTPHost, - HTTPPort: node.DefaultHTTPPort, - HTTPModules: []string{"admin", "db", "eth", "debug", "miner", "net", "shh", "txpool", "personal", "web3"}, - WSHost: node.DefaultWSHost, - WSPort: node.DefaultWSPort, - WSModules: []string{"admin", "db", "eth", "debug", "miner", "net", "shh", "txpool", "personal", "web3"}, - NoDiscovery: true, - }) - if err != nil { - return nil, err - } - // Create the keystore and inject an unlocked account if requested - ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore) - - if len(privkey) > 0 { - key, err := crypto.HexToECDSA(privkey) - if err != nil { - return nil, err - } - a, err := ks.ImportECDSA(key, "") - if err != nil { - return nil, err - } - if err := ks.Unlock(a, ""); err != nil { - return nil, err - } - } - // Initialize and register the Ethereum protocol - db, _ := ethdb.NewMemDatabase() - if _, err := test.InsertPreState(db); err != nil { - return nil, err - } - ethConf := ð.Config{ - TestGenesisState: db, - TestGenesisBlock: test.Genesis, - ChainConfig: ¶ms.ChainConfig{HomesteadBlock: params.MainNetHomesteadBlock}, - } - if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) { return eth.New(ctx, ethConf) }); err != nil { - return nil, err - } - // Initialize and register the Whisper protocol - if err := stack.Register(func(*node.ServiceContext) (node.Service, error) { return whisper.New(), nil }); err != nil { - return nil, err - } - return stack, nil -} - -// RunTest executes the specified test against an already pre-configured protocol -// stack to ensure basic checks pass before running RPC tests. -func RunTest(stack *node.Node, test *tests.BlockTest) error { - var ethereum *eth.Ethereum - stack.Service(ðereum) - blockchain := ethereum.BlockChain() - - // Process the blocks and verify the imported headers - blocks, err := test.TryBlocksInsert(blockchain) - if err != nil { - return err - } - if err := test.ValidateImportedHeaders(blockchain, blocks); err != nil { - return err - } - // Retrieve the assembled state and validate it - stateDb, err := blockchain.State() - if err != nil { - return err - } - if err := test.ValidatePostState(stateDb); err != nil { - return err - } - return nil -} From 5ee00209d2a916125d89227f1773917ce1fb0861 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 23 Feb 2017 16:27:36 +0200 Subject: [PATCH 2/3] cmd/ethtest: drop the manual test tool in favor of hive All the state and block tests are ran as part of our CU builds internally, as well as have been added to hive black-box tests. As such, there is no reason for maintaining an extra standalone tool. --- cmd/ethtest/main.go | 224 -------------------------------------------- 1 file changed, 224 deletions(-) delete mode 100644 cmd/ethtest/main.go diff --git a/cmd/ethtest/main.go b/cmd/ethtest/main.go deleted file mode 100644 index fcae668c6..000000000 --- a/cmd/ethtest/main.go +++ /dev/null @@ -1,224 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of go-ethereum. -// -// go-ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// go-ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with go-ethereum. If not, see . - -// ethtest executes Ethereum JSON tests. -package main - -import ( - "fmt" - "io" - "io/ioutil" - "os" - "path/filepath" - "strings" - - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/tests" - "gopkg.in/urfave/cli.v1" -) - -var ( - continueOnError = false - testExtension = ".json" - defaultTest = "all" - defaultDir = "." - allTests = []string{"BlockTests", "StateTests", "TransactionTests", "VMTests", "RLPTests"} - testDirMapping = map[string]string{"BlockTests": "BlockchainTests"} - skipTests = []string{} - - TestFlag = cli.StringFlag{ - Name: "test", - Usage: "Test type (string): VMTests, TransactionTests, StateTests, BlockTests", - Value: defaultTest, - } - FileFlag = cli.StringFlag{ - Name: "file", - Usage: "Test file or directory. Directories are searched for .json files 1 level deep", - Value: defaultDir, - EnvVar: "ETHEREUM_TEST_PATH", - } - ContinueOnErrorFlag = cli.BoolFlag{ - Name: "continue", - Usage: "Continue running tests on error (true) or [default] exit immediately (false)", - } - ReadStdInFlag = cli.BoolFlag{ - Name: "stdin", - Usage: "Accept input from stdin instead of reading from file", - } - SkipTestsFlag = cli.StringFlag{ - Name: "skip", - Usage: "Tests names to skip", - } - TraceFlag = cli.BoolFlag{ - Name: "trace", - Usage: "Enable VM tracing", - } -) - -func runTestWithReader(test string, r io.Reader) error { - log.Info("Running test", "test", test) - - var err error - switch strings.ToLower(test) { - case "bk", "block", "blocktest", "blockchaintest", "blocktests", "blockchaintests": - err = tests.RunBlockTestWithReader(params.MainNetHomesteadBlock, params.MainNetDAOForkBlock, params.MainNetHomesteadGasRepriceBlock, r, skipTests) - case "st", "state", "statetest", "statetests": - rs := ¶ms.ChainConfig{HomesteadBlock: params.MainNetHomesteadBlock, DAOForkBlock: params.MainNetDAOForkBlock, DAOForkSupport: true, EIP150Block: params.MainNetHomesteadGasRepriceBlock} - err = tests.RunStateTestWithReader(rs, r, skipTests) - case "tx", "transactiontest", "transactiontests": - rs := ¶ms.ChainConfig{HomesteadBlock: params.MainNetHomesteadBlock, DAOForkBlock: params.MainNetDAOForkBlock, DAOForkSupport: true, EIP150Block: params.MainNetHomesteadGasRepriceBlock} - err = tests.RunTransactionTestsWithReader(rs, r, skipTests) - case "vm", "vmtest", "vmtests": - err = tests.RunVmTestWithReader(r, skipTests) - case "rlp", "rlptest", "rlptests": - err = tests.RunRLPTestWithReader(r, skipTests) - default: - err = fmt.Errorf("Invalid test type specified: %v", test) - } - return err -} - -func getFiles(path string) ([]string, error) { - log.Info("Listing files", "path", path) - - var files []string - f, err := os.Open(path) - if err != nil { - return nil, err - } - defer f.Close() - - fi, err := f.Stat() - if err != nil { - return nil, err - } - - switch mode := fi.Mode(); { - case mode.IsDir(): - fi, _ := ioutil.ReadDir(path) - files = make([]string, len(fi)) - for i, v := range fi { - // only go 1 depth and leave directory entires blank - if !v.IsDir() && v.Name()[len(v.Name())-len(testExtension):len(v.Name())] == testExtension { - files[i] = filepath.Join(path, v.Name()) - log.Info("Found test file", "file", files[i]) - } - } - case mode.IsRegular(): - files = make([]string, 1) - files[0] = path - } - - return files, nil -} - -func runSuite(test, file string) { - var tests []string - - if test == defaultTest { - tests = allTests - } else { - tests = []string{test} - } - - for _, curTest := range tests { - suiteLogger := log.New("suite", file, "test", curTest) - suiteLogger.Info("Running test suite") - - var err error - var files []string - if test == defaultTest { - // check if we have an explicit directory mapping for the test - if _, ok := testDirMapping[curTest]; ok { - files, err = getFiles(filepath.Join(file, testDirMapping[curTest])) - } else { - // otherwise assume test name - files, err = getFiles(filepath.Join(file, curTest)) - } - } else { - files, err = getFiles(file) - } - if err != nil { - suiteLogger.Crit("Failed to gather files", "error", err) - } - - if len(files) == 0 { - suiteLogger.Warn("No files matched path") - } - for _, curFile := range files { - // Skip blank entries - if len(curFile) == 0 { - continue - } - testLogger := suiteLogger.New("file", curFile) - - r, err := os.Open(curFile) - if err != nil { - testLogger.Crit("Failed to open file") - } - defer r.Close() - - err = runTestWithReader(curTest, r) - if err != nil { - if continueOnError { - testLogger.Error("Test failed, continuing", "error", err) - } else { - testLogger.Crit("Test failed, aborting", "error", err) - } - } - } - } -} - -func setupApp(c *cli.Context) error { - flagTest := c.GlobalString(TestFlag.Name) - flagFile := c.GlobalString(FileFlag.Name) - continueOnError = c.GlobalBool(ContinueOnErrorFlag.Name) - useStdIn := c.GlobalBool(ReadStdInFlag.Name) - skipTests = strings.Split(c.GlobalString(SkipTestsFlag.Name), " ") - - if !useStdIn { - runSuite(flagTest, flagFile) - } else { - return runTestWithReader(flagTest, os.Stdin) - } - return nil -} - -func main() { - log.Root().SetHandler(log.StreamHandler(os.Stderr, log.TerminalFormat())) - - app := cli.NewApp() - app.Name = "ethtest" - app.Usage = "go-ethereum test interface" - app.Action = setupApp - app.Version = "0.2.0" - app.Author = "go-ethereum team" - - app.Flags = []cli.Flag{ - TestFlag, - FileFlag, - ContinueOnErrorFlag, - ReadStdInFlag, - SkipTestsFlag, - TraceFlag, - } - - if err := app.Run(os.Args); err != nil { - log.Crit("Failed to run the tester", "error", err) - } -} From aca066f3378514bf99c56ff9723e9e7e86452326 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 23 Feb 2017 16:35:16 +0200 Subject: [PATCH 3/3] cmd/geth: drop upgradedb subcommand since it's unfeasible This command was meant as a hackish way to upgrade our chain database way back when nobody cared for live updates and the size of the database along with its import times was small. With the current database weighing hundreds of GBs and processing times of many days, this command is just ludicrous. --- cmd/geth/chaincmd.go | 60 -------------------------------------------- cmd/geth/main.go | 1 - 2 files changed, 61 deletions(-) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 74950697c..1c21c4ded 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -19,7 +19,6 @@ package main import ( "fmt" "os" - "path/filepath" "runtime" "strconv" "sync/atomic" @@ -72,16 +71,6 @@ Requires a first argument of the file to write to. Optional second and third arguments control the first and last block to write. In this mode, the file will be appended if already existing. -`, - } - upgradedbCommand = cli.Command{ - Action: upgradeDB, - Name: "upgradedb", - Usage: "Upgrade chainblock database", - ArgsUsage: " ", - Category: "BLOCKCHAIN COMMANDS", - Description: ` -TODO: Please write this `, } removedbCommand = cli.Command{ @@ -255,49 +244,6 @@ func removeDB(ctx *cli.Context) error { return nil } -func upgradeDB(ctx *cli.Context) error { - log.Info(fmt.Sprint("Upgrading blockchain database")) - - stack := utils.MakeNode(ctx, clientIdentifier, gitCommit) - chain, chainDb := utils.MakeChain(ctx, stack) - bcVersion := core.GetBlockChainVersion(chainDb) - if bcVersion == 0 { - bcVersion = core.BlockChainVersion - } - - // Export the current chain. - filename := fmt.Sprintf("blockchain_%d_%s.chain", bcVersion, time.Now().Format("20060102_150405")) - exportFile := filepath.Join(ctx.GlobalString(utils.DataDirFlag.Name), filename) - if err := utils.ExportChain(chain, exportFile); err != nil { - utils.Fatalf("Unable to export chain for reimport %s", err) - } - chainDb.Close() - if dir := dbDirectory(chainDb); dir != "" { - os.RemoveAll(dir) - } - - // Import the chain file. - chain, chainDb = utils.MakeChain(ctx, stack) - core.WriteBlockChainVersion(chainDb, core.BlockChainVersion) - err := utils.ImportChain(chain, exportFile) - chainDb.Close() - if err != nil { - utils.Fatalf("Import error %v (a backup is made in %s, use the import command to import it)", err, exportFile) - } else { - os.Remove(exportFile) - log.Info(fmt.Sprint("Import finished")) - } - return nil -} - -func dbDirectory(db ethdb.Database) string { - ldb, ok := db.(*ethdb.LDBDatabase) - if !ok { - return "" - } - return ldb.Path() -} - func dump(ctx *cli.Context) error { stack := makeFullNode(ctx) chain, chainDb := utils.MakeChain(ctx, stack) @@ -329,9 +275,3 @@ func hashish(x string) bool { _, err := strconv.Atoi(x) return err != nil } - -func closeAll(dbs ...ethdb.Database) { - for _, db := range dbs { - db.Close() - } -} diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 803a618f1..fa61f7386 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -65,7 +65,6 @@ func init() { initCommand, importCommand, exportCommand, - upgradedbCommand, removedbCommand, dumpCommand, // See monitorcmd.go: