From a23478c0be94e1e727a64d20341b8d6f98d7f0a0 Mon Sep 17 00:00:00 2001 From: Jeffrey Wilcke Date: Thu, 6 Aug 2015 19:57:39 +0200 Subject: [PATCH] core, eth, trie, xeth: merged state, chain, extra databases in one --- cmd/geth/chaincmd.go | 26 ++--- cmd/utils/flags.go | 18 ++-- core/bench_test.go | 4 +- core/block_processor.go | 16 ++- core/block_processor_test.go | 8 +- core/chain_makers.go | 6 +- core/chain_makers_test.go | 4 +- core/chain_manager.go | 48 ++++----- core/chain_manager_test.go | 18 ++-- core/genesis.go | 18 ++-- core/manager.go | 5 +- eth/backend.go | 192 ++++++++++++++++++++++------------- eth/protocol_test.go | 4 +- ethdb/database.go | 5 +- miner/worker.go | 10 +- rpc/api/debug.go | 2 +- tests/block_test_util.go | 2 +- trie/cache.go | 6 +- trie/trie.go | 2 + xeth/state.go | 2 +- xeth/xeth.go | 16 +-- 21 files changed, 224 insertions(+), 188 deletions(-) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 2d8eb15c2..876b8c6ba 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -74,10 +74,10 @@ func importChain(ctx *cli.Context) { if len(ctx.Args()) != 1 { utils.Fatalf("This command requires an argument.") } - chain, blockDB, stateDB, extraDB := utils.MakeChain(ctx) + chain, chainDb := utils.MakeChain(ctx) start := time.Now() err := utils.ImportChain(chain, ctx.Args().First()) - closeAll(blockDB, stateDB, extraDB) + chainDb.Close() if err != nil { utils.Fatalf("Import error: %v", err) } @@ -88,7 +88,7 @@ func exportChain(ctx *cli.Context) { if len(ctx.Args()) < 1 { utils.Fatalf("This command requires an argument.") } - chain, _, _, _ := utils.MakeChain(ctx) + chain, _ := utils.MakeChain(ctx) start := time.Now() var err error @@ -136,8 +136,8 @@ func removeDB(ctx *cli.Context) { func upgradeDB(ctx *cli.Context) { glog.Infoln("Upgrading blockchain database") - chain, blockDB, stateDB, extraDB := utils.MakeChain(ctx) - v, _ := blockDB.Get([]byte("BlockchainVersion")) + chain, chainDb := utils.MakeChain(ctx) + v, _ := chainDb.Get([]byte("BlockchainVersion")) bcVersion := int(common.NewValue(v).Uint()) if bcVersion == 0 { bcVersion = core.BlockChainVersion @@ -149,15 +149,14 @@ func upgradeDB(ctx *cli.Context) { if err := utils.ExportChain(chain, exportFile); err != nil { utils.Fatalf("Unable to export chain for reimport %s", err) } - closeAll(blockDB, stateDB, extraDB) - os.RemoveAll(filepath.Join(ctx.GlobalString(utils.DataDirFlag.Name), "blockchain")) - os.RemoveAll(filepath.Join(ctx.GlobalString(utils.DataDirFlag.Name), "state")) + chainDb.Close() + os.RemoveAll(filepath.Join(ctx.GlobalString(utils.DataDirFlag.Name), "chaindata")) // Import the chain file. - chain, blockDB, stateDB, extraDB = utils.MakeChain(ctx) - blockDB.Put([]byte("BlockchainVersion"), common.NewValue(core.BlockChainVersion).Bytes()) + chain, chainDb = utils.MakeChain(ctx) + chainDb.Put([]byte("BlockchainVersion"), common.NewValue(core.BlockChainVersion).Bytes()) err := utils.ImportChain(chain, exportFile) - closeAll(blockDB, stateDB, extraDB) + chainDb.Close() if err != nil { utils.Fatalf("Import error %v (a backup is made in %s, use the import command to import it)", err, exportFile) } else { @@ -167,7 +166,7 @@ func upgradeDB(ctx *cli.Context) { } func dump(ctx *cli.Context) { - chain, _, stateDB, _ := utils.MakeChain(ctx) + chain, chainDb := utils.MakeChain(ctx) for _, arg := range ctx.Args() { var block *types.Block if hashish(arg) { @@ -180,10 +179,11 @@ func dump(ctx *cli.Context) { fmt.Println("{}") utils.Fatalf("block not found") } else { - state := state.New(block.Root(), stateDB) + state := state.New(block.Root(), chainDb) fmt.Printf("%s\n", state.Dump()) } } + chainDb.Close() } // hashish returns true for strings that look like hashes. diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index cf969805d..e0ea7116d 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -435,23 +435,17 @@ func SetupLogger(ctx *cli.Context) { } // MakeChain creates a chain manager from set command line flags. -func MakeChain(ctx *cli.Context) (chain *core.ChainManager, blockDB, stateDB, extraDB common.Database) { +func MakeChain(ctx *cli.Context) (chain *core.ChainManager, chainDb common.Database) { datadir := ctx.GlobalString(DataDirFlag.Name) cache := ctx.GlobalInt(CacheFlag.Name) var err error - if blockDB, err = ethdb.NewLDBDatabase(filepath.Join(datadir, "blockchain"), cache); err != nil { - Fatalf("Could not open database: %v", err) - } - if stateDB, err = ethdb.NewLDBDatabase(filepath.Join(datadir, "state"), cache); err != nil { - Fatalf("Could not open database: %v", err) - } - if extraDB, err = ethdb.NewLDBDatabase(filepath.Join(datadir, "extra"), cache); err != nil { + if chainDb, err = ethdb.NewLDBDatabase(filepath.Join(datadir, "chaindata"), cache); err != nil { Fatalf("Could not open database: %v", err) } if ctx.GlobalBool(OlympicFlag.Name) { InitOlympic() - _, err := core.WriteTestNetGenesisBlock(stateDB, blockDB, 42) + _, err := core.WriteTestNetGenesisBlock(chainDb, 42) if err != nil { glog.Fatalln(err) } @@ -460,14 +454,14 @@ func MakeChain(ctx *cli.Context) (chain *core.ChainManager, blockDB, stateDB, ex eventMux := new(event.TypeMux) pow := ethash.New() //genesis := core.GenesisBlock(uint64(ctx.GlobalInt(GenesisNonceFlag.Name)), blockDB) - chain, err = core.NewChainManager(blockDB, stateDB, extraDB, pow, eventMux) + chain, err = core.NewChainManager(chainDb, pow, eventMux) if err != nil { Fatalf("Could not start chainmanager: %v", err) } - proc := core.NewBlockProcessor(stateDB, extraDB, pow, chain, eventMux) + proc := core.NewBlockProcessor(chainDb, pow, chain, eventMux) chain.SetProcessor(proc) - return chain, blockDB, stateDB, extraDB + return chain, chainDb } // MakeChain creates an account manager from set command line flags. diff --git a/core/bench_test.go b/core/bench_test.go index 67ba15970..baae8a7a5 100644 --- a/core/bench_test.go +++ b/core/bench_test.go @@ -168,8 +168,8 @@ func benchInsertChain(b *testing.B, disk bool, gen func(int, *BlockGen)) { // Time the insertion of the new chain. // State and blocks are stored in the same DB. evmux := new(event.TypeMux) - chainman, _ := NewChainManager(db, db, db, FakePow{}, evmux) - chainman.SetProcessor(NewBlockProcessor(db, db, FakePow{}, chainman, evmux)) + chainman, _ := NewChainManager(db, FakePow{}, evmux) + chainman.SetProcessor(NewBlockProcessor(db, FakePow{}, chainman, evmux)) defer chainman.Stop() b.ReportAllocs() b.ResetTimer() diff --git a/core/block_processor.go b/core/block_processor.go index 5a2ad8377..38bf772fb 100644 --- a/core/block_processor.go +++ b/core/block_processor.go @@ -41,8 +41,7 @@ const ( ) type BlockProcessor struct { - db common.Database - extraDb common.Database + chainDb common.Database // Mutex for locking the block processor. Blocks can only be handled one at a time mutex sync.Mutex // Canonical block chain @@ -57,10 +56,9 @@ type BlockProcessor struct { eventMux *event.TypeMux } -func NewBlockProcessor(db, extra common.Database, pow pow.PoW, chainManager *ChainManager, eventMux *event.TypeMux) *BlockProcessor { +func NewBlockProcessor(db common.Database, pow pow.PoW, chainManager *ChainManager, eventMux *event.TypeMux) *BlockProcessor { sm := &BlockProcessor{ - db: db, - extraDb: extra, + chainDb: db, mem: make(map[string]*big.Int), Pow: pow, bc: chainManager, @@ -201,7 +199,7 @@ func (sm *BlockProcessor) Process(block *types.Block) (logs state.Logs, receipts func (sm *BlockProcessor) processWithParent(block, parent *types.Block) (logs state.Logs, receipts types.Receipts, err error) { // Create a new state based on the parent's root (e.g., create copy) - state := state.New(parent.Root(), sm.db) + state := state.New(parent.Root(), sm.chainDb) header := block.Header() uncles := block.Uncles() txs := block.Transactions() @@ -342,7 +340,7 @@ func (sm *BlockProcessor) VerifyUncles(statedb *state.StateDB, block, parent *ty // GetBlockReceipts returns the receipts beloniging to the block hash func (sm *BlockProcessor) GetBlockReceipts(bhash common.Hash) types.Receipts { if block := sm.ChainManager().GetBlock(bhash); block != nil { - return GetBlockReceipts(sm.extraDb, block.Hash()) + return GetBlockReceipts(sm.chainDb, block.Hash()) } return nil @@ -352,7 +350,7 @@ func (sm *BlockProcessor) GetBlockReceipts(bhash common.Hash) types.Receipts { // where it tries to get it from the (updated) method which gets them from the receipts or // the depricated way by re-processing the block. func (sm *BlockProcessor) GetLogs(block *types.Block) (logs state.Logs, err error) { - receipts := GetBlockReceipts(sm.extraDb, block.Hash()) + receipts := GetBlockReceipts(sm.chainDb, block.Hash()) if len(receipts) > 0 { // coalesce logs for _, receipt := range receipts { @@ -364,7 +362,7 @@ func (sm *BlockProcessor) GetLogs(block *types.Block) (logs state.Logs, err erro // TODO: remove backward compatibility var ( parent = sm.bc.GetBlock(block.ParentHash()) - state = state.New(parent.Root(), sm.db) + state = state.New(parent.Root(), sm.chainDb) ) sm.TransitionState(state, parent, block, true) diff --git a/core/block_processor_test.go b/core/block_processor_test.go index f48ce9607..4525f417b 100644 --- a/core/block_processor_test.go +++ b/core/block_processor_test.go @@ -33,19 +33,19 @@ func proc() (*BlockProcessor, *ChainManager) { db, _ := ethdb.NewMemDatabase() var mux event.TypeMux - WriteTestNetGenesisBlock(db, db, 0) - chainMan, err := NewChainManager(db, db, db, thePow(), &mux) + WriteTestNetGenesisBlock(db, 0) + chainMan, err := NewChainManager(db, thePow(), &mux) if err != nil { fmt.Println(err) } - return NewBlockProcessor(db, db, ezp.New(), chainMan, &mux), chainMan + return NewBlockProcessor(db, ezp.New(), chainMan, &mux), chainMan } func TestNumber(t *testing.T) { pow := ezp.New() _, chain := proc() - statedb := state.New(chain.Genesis().Root(), chain.stateDb) + statedb := state.New(chain.Genesis().Root(), chain.chainDb) header := makeHeader(chain.Genesis(), statedb) header.Number = big.NewInt(3) err := ValidateHeader(pow, header, chain.Genesis(), false) diff --git a/core/chain_makers.go b/core/chain_makers.go index 85a6175dc..0bb1df95a 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -184,9 +184,9 @@ func makeHeader(parent *types.Block, state *state.StateDB) *types.Header { func newCanonical(n int, db common.Database) (*BlockProcessor, error) { evmux := &event.TypeMux{} - WriteTestNetGenesisBlock(db, db, 0) - chainman, _ := NewChainManager(db, db, db, FakePow{}, evmux) - bman := NewBlockProcessor(db, db, FakePow{}, chainman, evmux) + WriteTestNetGenesisBlock(db, 0) + chainman, _ := NewChainManager(db, FakePow{}, evmux) + bman := NewBlockProcessor(db, FakePow{}, chainman, evmux) bman.bc.SetProcessor(bman) parent := bman.bc.CurrentBlock() if n == 0 { diff --git a/core/chain_makers_test.go b/core/chain_makers_test.go index 98a585f9b..1c868624d 100644 --- a/core/chain_makers_test.go +++ b/core/chain_makers_test.go @@ -77,8 +77,8 @@ func ExampleGenerateChain() { // Import the chain. This runs all block validation rules. evmux := &event.TypeMux{} - chainman, _ := NewChainManager(db, db, db, FakePow{}, evmux) - chainman.SetProcessor(NewBlockProcessor(db, db, FakePow{}, chainman, evmux)) + chainman, _ := NewChainManager(db, FakePow{}, evmux) + chainman.SetProcessor(NewBlockProcessor(db, FakePow{}, chainman, evmux)) if i, err := chainman.InsertChain(chain); err != nil { fmt.Printf("insert error (block %d): %v\n", i, err) return diff --git a/core/chain_manager.go b/core/chain_manager.go index fc1d1304f..87353b944 100644 --- a/core/chain_manager.go +++ b/core/chain_manager.go @@ -56,9 +56,7 @@ const ( type ChainManager struct { //eth EthManager - blockDb common.Database - stateDb common.Database - extraDb common.Database + chainDb common.Database processor types.BlockProcessor eventMux *event.TypeMux genesisBlock *types.Block @@ -85,12 +83,10 @@ type ChainManager struct { pow pow.PoW } -func NewChainManager(blockDb, stateDb, extraDb common.Database, pow pow.PoW, mux *event.TypeMux) (*ChainManager, error) { +func NewChainManager(chainDb common.Database, pow pow.PoW, mux *event.TypeMux) (*ChainManager, error) { cache, _ := lru.New(blockCacheLimit) bc := &ChainManager{ - blockDb: blockDb, - stateDb: stateDb, - extraDb: extraDb, + chainDb: chainDb, eventMux: mux, quit: make(chan struct{}), cache: cache, @@ -103,7 +99,7 @@ func NewChainManager(blockDb, stateDb, extraDb common.Database, pow pow.PoW, mux if err != nil { return nil, err } - bc.genesisBlock, err = WriteGenesisBlock(stateDb, blockDb, reader) + bc.genesisBlock, err = WriteGenesisBlock(chainDb, reader) if err != nil { return nil, err } @@ -195,15 +191,15 @@ func (self *ChainManager) SetProcessor(proc types.BlockProcessor) { } func (self *ChainManager) State() *state.StateDB { - return state.New(self.CurrentBlock().Root(), self.stateDb) + return state.New(self.CurrentBlock().Root(), self.chainDb) } func (bc *ChainManager) recover() bool { - data, _ := bc.blockDb.Get([]byte("checkpoint")) + data, _ := bc.chainDb.Get([]byte("checkpoint")) if len(data) != 0 { block := bc.GetBlock(common.BytesToHash(data)) if block != nil { - err := bc.blockDb.Put([]byte("LastBlock"), block.Hash().Bytes()) + err := bc.chainDb.Put([]byte("LastBlock"), block.Hash().Bytes()) if err != nil { glog.Fatalln("db write err:", err) } @@ -217,7 +213,7 @@ func (bc *ChainManager) recover() bool { } func (bc *ChainManager) setLastState() error { - data, _ := bc.blockDb.Get([]byte("LastBlock")) + data, _ := bc.chainDb.Get([]byte("LastBlock")) if len(data) != 0 { block := bc.GetBlock(common.BytesToHash(data)) if block != nil { @@ -264,7 +260,7 @@ func (bc *ChainManager) Reset() { bc.cache, _ = lru.New(blockCacheLimit) // Prepare the genesis block - err := WriteBlock(bc.blockDb, bc.genesisBlock) + err := WriteBlock(bc.chainDb, bc.genesisBlock) if err != nil { glog.Fatalln("db err:", err) } @@ -277,7 +273,7 @@ func (bc *ChainManager) Reset() { } func (bc *ChainManager) removeBlock(block *types.Block) { - bc.blockDb.Delete(append(blockHashPre, block.Hash().Bytes()...)) + bc.chainDb.Delete(append(blockHashPre, block.Hash().Bytes()...)) } func (bc *ChainManager) ResetWithGenesisBlock(gb *types.Block) { @@ -292,7 +288,7 @@ func (bc *ChainManager) ResetWithGenesisBlock(gb *types.Block) { gb.Td = gb.Difficulty() bc.genesisBlock = gb - err := WriteBlock(bc.blockDb, bc.genesisBlock) + err := WriteBlock(bc.chainDb, bc.genesisBlock) if err != nil { glog.Fatalln("db err:", err) } @@ -339,14 +335,14 @@ func (self *ChainManager) ExportN(w io.Writer, first uint64, last uint64) error // insert injects a block into the current chain block chain. Note, this function // assumes that the `mu` mutex is held! func (bc *ChainManager) insert(block *types.Block) { - err := WriteHead(bc.blockDb, block) + err := WriteHead(bc.chainDb, block) if err != nil { glog.Fatal("db write fail:", err) } bc.checkpoint++ if bc.checkpoint > checkpointLimit { - err = bc.blockDb.Put([]byte("checkpoint"), block.Hash().Bytes()) + err = bc.chainDb.Put([]byte("checkpoint"), block.Hash().Bytes()) if err != nil { glog.Fatal("db write fail:", err) } @@ -369,7 +365,7 @@ func (bc *ChainManager) HasBlock(hash common.Hash) bool { return true } - data, _ := bc.blockDb.Get(append(blockHashPre, hash[:]...)) + data, _ := bc.chainDb.Get(append(blockHashPre, hash[:]...)) return len(data) != 0 } @@ -399,7 +395,7 @@ func (self *ChainManager) GetBlock(hash common.Hash) *types.Block { return block.(*types.Block) } - block := GetBlockByHash(self.blockDb, hash) + block := GetBlockByHash(self.chainDb, hash) if block == nil { return nil } @@ -433,7 +429,7 @@ func (self *ChainManager) GetBlocksFromHash(hash common.Hash, n int) (blocks []* // non blocking version func (self *ChainManager) getBlockByNumber(num uint64) *types.Block { - return GetBlockByNumber(self.blockDb, num) + return GetBlockByNumber(self.chainDb, num) } func (self *ChainManager) GetUnclesInChain(block *types.Block, length int) (uncles []*types.Header) { @@ -521,7 +517,7 @@ func (self *ChainManager) WriteBlock(block *types.Block, queued bool) (status wr status = SideStatTy } - err = WriteBlock(self.blockDb, block) + err = WriteBlock(self.chainDb, block) if err != nil { glog.Fatalln("db err:", err) } @@ -638,9 +634,9 @@ func (self *ChainManager) InsertChain(chain types.Blocks) (int, error) { queueEvent.canonicalCount++ // This puts transactions in a extra db for rpc - PutTransactions(self.extraDb, block, block.Transactions()) + PutTransactions(self.chainDb, block, block.Transactions()) // store the receipts - PutReceipts(self.extraDb, receipts) + PutReceipts(self.chainDb, receipts) case SideStatTy: if glog.V(logger.Detail) { glog.Infof("inserted forked block #%d (TD=%v) (%d TXs %d UNCs) (%x...). Took %v\n", block.Number(), block.Difficulty(), len(block.Transactions()), len(block.Uncles()), block.Hash().Bytes()[0:4], time.Since(bstart)) @@ -651,7 +647,7 @@ func (self *ChainManager) InsertChain(chain types.Blocks) (int, error) { queue[i] = ChainSplitEvent{block, logs} queueEvent.splitCount++ } - PutBlockReceipts(self.extraDb, block, receipts) + PutBlockReceipts(self.chainDb, block, receipts) stats.processed++ } @@ -733,8 +729,8 @@ func (self *ChainManager) merge(oldBlock, newBlock *types.Block) error { // insert the block in the canonical way, re-writing history self.insert(block) // write canonical receipts and transactions - PutTransactions(self.extraDb, block, block.Transactions()) - PutReceipts(self.extraDb, GetBlockReceipts(self.extraDb, block.Hash())) + PutTransactions(self.chainDb, block, block.Transactions()) + PutReceipts(self.chainDb, GetBlockReceipts(self.chainDb, block.Hash())) } self.mu.Unlock() diff --git a/core/chain_manager_test.go b/core/chain_manager_test.go index f0c097df6..002dcbe44 100644 --- a/core/chain_manager_test.go +++ b/core/chain_manager_test.go @@ -48,14 +48,14 @@ func thePow() pow.PoW { func theChainManager(db common.Database, t *testing.T) *ChainManager { var eventMux event.TypeMux - WriteTestNetGenesisBlock(db, db, 0) - chainMan, err := NewChainManager(db, db, db, thePow(), &eventMux) + WriteTestNetGenesisBlock(db, 0) + chainMan, err := NewChainManager(db, thePow(), &eventMux) if err != nil { t.Error("failed creating chainmanager:", err) t.FailNow() return nil } - blockMan := NewBlockProcessor(db, db, nil, chainMan, &eventMux) + blockMan := NewBlockProcessor(db, nil, chainMan, &eventMux) chainMan.SetProcessor(blockMan) return chainMan @@ -125,7 +125,7 @@ func testChain(chainB types.Blocks, bman *BlockProcessor) (*big.Int, error) { bman.bc.mu.Lock() { - WriteBlock(bman.bc.blockDb, block) + WriteBlock(bman.bc.chainDb, block) } bman.bc.mu.Unlock() } @@ -387,7 +387,7 @@ func makeChainWithDiff(genesis *types.Block, d []int, seed byte) []*types.Block func chm(genesis *types.Block, db common.Database) *ChainManager { var eventMux event.TypeMux - bc := &ChainManager{extraDb: db, blockDb: db, stateDb: db, genesisBlock: genesis, eventMux: &eventMux, pow: FakePow{}} + bc := &ChainManager{chainDb: db, genesisBlock: genesis, eventMux: &eventMux, pow: FakePow{}} bc.cache, _ = lru.New(100) bc.futureBlocks, _ = lru.New(100) bc.processor = bproc{} @@ -399,7 +399,7 @@ func chm(genesis *types.Block, db common.Database) *ChainManager { func TestReorgLongest(t *testing.T) { db, _ := ethdb.NewMemDatabase() - genesis, err := WriteTestNetGenesisBlock(db, db, 0) + genesis, err := WriteTestNetGenesisBlock(db, 0) if err != nil { t.Error(err) t.FailNow() @@ -422,7 +422,7 @@ func TestReorgLongest(t *testing.T) { func TestReorgShortest(t *testing.T) { db, _ := ethdb.NewMemDatabase() - genesis, err := WriteTestNetGenesisBlock(db, db, 0) + genesis, err := WriteTestNetGenesisBlock(db, 0) if err != nil { t.Error(err) t.FailNow() @@ -446,13 +446,13 @@ func TestReorgShortest(t *testing.T) { func TestInsertNonceError(t *testing.T) { for i := 1; i < 25 && !t.Failed(); i++ { db, _ := ethdb.NewMemDatabase() - genesis, err := WriteTestNetGenesisBlock(db, db, 0) + genesis, err := WriteTestNetGenesisBlock(db, 0) if err != nil { t.Error(err) t.FailNow() } bc := chm(genesis, db) - bc.processor = NewBlockProcessor(db, db, bc.pow, bc, bc.eventMux) + bc.processor = NewBlockProcessor(db, bc.pow, bc, bc.eventMux) blocks := makeChain(bc.currentBlock, i, db, 0) fail := rand.Int() % len(blocks) diff --git a/core/genesis.go b/core/genesis.go index 4c0323c17..97afb3a4a 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -33,7 +33,7 @@ import ( ) // WriteGenesisBlock writes the genesis block to the database as block number 0 -func WriteGenesisBlock(stateDb, blockDb common.Database, reader io.Reader) (*types.Block, error) { +func WriteGenesisBlock(chainDb common.Database, reader io.Reader) (*types.Block, error) { contents, err := ioutil.ReadAll(reader) if err != nil { return nil, err @@ -59,7 +59,7 @@ func WriteGenesisBlock(stateDb, blockDb common.Database, reader io.Reader) (*typ return nil, err } - statedb := state.New(common.Hash{}, stateDb) + statedb := state.New(common.Hash{}, chainDb) for addr, account := range genesis.Alloc { address := common.HexToAddress(addr) statedb.AddBalance(address, common.String2Big(account.Balance)) @@ -84,9 +84,9 @@ func WriteGenesisBlock(stateDb, blockDb common.Database, reader io.Reader) (*typ }, nil, nil, nil) block.Td = difficulty - if block := GetBlockByHash(blockDb, block.Hash()); block != nil { + if block := GetBlockByHash(chainDb, block.Hash()); block != nil { glog.V(logger.Info).Infoln("Genesis block already in chain. Writing canonical number") - err := WriteCanonNumber(blockDb, block) + err := WriteCanonNumber(chainDb, block) if err != nil { return nil, err } @@ -95,11 +95,11 @@ func WriteGenesisBlock(stateDb, blockDb common.Database, reader io.Reader) (*typ statedb.Sync() - err = WriteBlock(blockDb, block) + err = WriteBlock(chainDb, block) if err != nil { return nil, err } - err = WriteHead(blockDb, block) + err = WriteHead(chainDb, block) if err != nil { return nil, err } @@ -133,11 +133,11 @@ func WriteGenesisBlockForTesting(db common.Database, addr common.Address, balanc "0x%x":{"balance":"0x%x"} } }`, types.EncodeNonce(0), params.GenesisGasLimit.Bytes(), params.GenesisDifficulty.Bytes(), addr, balance.Bytes()) - block, _ := WriteGenesisBlock(db, db, strings.NewReader(testGenesis)) + block, _ := WriteGenesisBlock(db, strings.NewReader(testGenesis)) return block } -func WriteTestNetGenesisBlock(stateDb, blockDb common.Database, nonce uint64) (*types.Block, error) { +func WriteTestNetGenesisBlock(chainDb common.Database, nonce uint64) (*types.Block, error) { testGenesis := fmt.Sprintf(`{ "nonce":"0x%x", "gasLimit":"0x%x", @@ -157,5 +157,5 @@ func WriteTestNetGenesisBlock(stateDb, blockDb common.Database, nonce uint64) (* "1a26338f0d905e295fccb71fa9ea849ffa12aaf4": {"balance": "1606938044258990275541962092341162602522202993782792835301376"} } }`, types.EncodeNonce(nonce), params.GenesisGasLimit.Bytes(), params.GenesisDifficulty.Bytes()) - return WriteGenesisBlock(stateDb, blockDb, strings.NewReader(testGenesis)) + return WriteGenesisBlock(chainDb, strings.NewReader(testGenesis)) } diff --git a/core/manager.go b/core/manager.go index a07c32659..8b0401b03 100644 --- a/core/manager.go +++ b/core/manager.go @@ -28,8 +28,7 @@ type Backend interface { BlockProcessor() *BlockProcessor ChainManager() *ChainManager TxPool() *TxPool - BlockDb() common.Database - StateDb() common.Database - ExtraDb() common.Database + ChainDb() common.Database + DappDb() common.Database EventMux() *event.TypeMux } diff --git a/eth/backend.go b/eth/backend.go index ed46a4ab3..c9b71803f 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -45,6 +45,7 @@ import ( "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/p2p/nat" + "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/whisper" ) @@ -206,9 +207,8 @@ type Ethereum struct { shutdownChan chan bool // DB interfaces - blockDb common.Database // Block chain database - stateDb common.Database // State changes database - extraDb common.Database // Extra database (txs, etc) + chainDb common.Database // Block chain databe + dappDb common.Database // Dapp database // Closed when databases are flushed and closed databasesClosed chan bool @@ -266,27 +266,27 @@ func New(config *Config) (*Ethereum, error) { if newdb == nil { newdb = func(path string) (common.Database, error) { return ethdb.NewLDBDatabase(path, config.DatabaseCache) } } - blockDb, err := newdb(filepath.Join(config.DataDir, "blockchain")) + + // attempt to merge database together, upgrading from an old version + if err := mergeDatabases(config.DataDir, newdb); err != nil { + return nil, err + } + + chainDb, err := newdb(filepath.Join(config.DataDir, "chaindata")) if err != nil { return nil, fmt.Errorf("blockchain db err: %v", err) } - if db, ok := blockDb.(*ethdb.LDBDatabase); ok { - db.Meter("eth/db/block/") + if db, ok := chainDb.(*ethdb.LDBDatabase); ok { + db.Meter("eth/db/chaindata/") } - stateDb, err := newdb(filepath.Join(config.DataDir, "state")) + dappDb, err := newdb(filepath.Join(config.DataDir, "dapp")) if err != nil { - return nil, fmt.Errorf("state db err: %v", err) + return nil, fmt.Errorf("dapp db err: %v", err) } - if db, ok := stateDb.(*ethdb.LDBDatabase); ok { - db.Meter("eth/db/state/") - } - extraDb, err := newdb(filepath.Join(config.DataDir, "extra")) - if err != nil { - return nil, fmt.Errorf("extra db err: %v", err) - } - if db, ok := extraDb.(*ethdb.LDBDatabase); ok { - db.Meter("eth/db/extra/") + if db, ok := dappDb.(*ethdb.LDBDatabase); ok { + db.Meter("eth/db/dapp/") } + nodeDb := filepath.Join(config.DataDir, "nodes") glog.V(logger.Info).Infof("Protocol Versions: %v, Network Id: %v", ProtocolVersions, config.NetworkId) @@ -296,7 +296,7 @@ func New(config *Config) (*Ethereum, error) { return nil, err } - block, err := core.WriteGenesisBlock(stateDb, blockDb, fr) + block, err := core.WriteGenesisBlock(chainDb, fr) if err != nil { return nil, err } @@ -304,7 +304,7 @@ func New(config *Config) (*Ethereum, error) { } if config.Olympic { - _, err := core.WriteTestNetGenesisBlock(stateDb, blockDb, 42) + _, err := core.WriteTestNetGenesisBlock(chainDb, 42) if err != nil { return nil, err } @@ -313,26 +313,25 @@ func New(config *Config) (*Ethereum, error) { // This is for testing only. if config.GenesisBlock != nil { - core.WriteBlock(blockDb, config.GenesisBlock) - core.WriteHead(blockDb, config.GenesisBlock) + core.WriteBlock(chainDb, config.GenesisBlock) + core.WriteHead(chainDb, config.GenesisBlock) } if !config.SkipBcVersionCheck { - b, _ := blockDb.Get([]byte("BlockchainVersion")) + b, _ := chainDb.Get([]byte("BlockchainVersion")) bcVersion := int(common.NewValue(b).Uint()) if bcVersion != config.BlockChainVersion && bcVersion != 0 { return nil, fmt.Errorf("Blockchain DB version mismatch (%d / %d). Run geth upgradedb.\n", bcVersion, config.BlockChainVersion) } - saveBlockchainVersion(blockDb, config.BlockChainVersion) + saveBlockchainVersion(chainDb, config.BlockChainVersion) } glog.V(logger.Info).Infof("Blockchain DB Version: %d", config.BlockChainVersion) eth := &Ethereum{ shutdownChan: make(chan bool), databasesClosed: make(chan bool), - blockDb: blockDb, - stateDb: stateDb, - extraDb: extraDb, + chainDb: chainDb, + dappDb: dappDb, eventMux: &event.TypeMux{}, accountManager: config.AccountManager, DataDir: config.DataDir, @@ -362,7 +361,7 @@ func New(config *Config) (*Ethereum, error) { eth.pow = ethash.New() } //genesis := core.GenesisBlock(uint64(config.GenesisNonce), stateDb) - eth.chainManager, err = core.NewChainManager(blockDb, stateDb, extraDb, eth.pow, eth.EventMux()) + eth.chainManager, err = core.NewChainManager(chainDb, eth.pow, eth.EventMux()) if err != nil { if err == core.ErrNoGenesis { return nil, fmt.Errorf(`Genesis block not found. Please supply a genesis block with the "--genesis /path/to/file" argument`) @@ -372,7 +371,7 @@ func New(config *Config) (*Ethereum, error) { } eth.txPool = core.NewTxPool(eth.EventMux(), eth.chainManager.State, eth.chainManager.GasLimit) - eth.blockProcessor = core.NewBlockProcessor(stateDb, extraDb, eth.pow, eth.chainManager, eth.EventMux()) + eth.blockProcessor = core.NewBlockProcessor(chainDb, eth.pow, eth.chainManager, eth.EventMux()) eth.chainManager.SetProcessor(eth.blockProcessor) eth.protocolManager = NewProtocolManager(config.NetworkId, eth.eventMux, eth.txPool, eth.pow, eth.chainManager) @@ -520,9 +519,8 @@ func (s *Ethereum) BlockProcessor() *core.BlockProcessor { return s.blockProcess func (s *Ethereum) TxPool() *core.TxPool { return s.txPool } func (s *Ethereum) Whisper() *whisper.Whisper { return s.whisper } func (s *Ethereum) EventMux() *event.TypeMux { return s.eventMux } -func (s *Ethereum) BlockDb() common.Database { return s.blockDb } -func (s *Ethereum) StateDb() common.Database { return s.stateDb } -func (s *Ethereum) ExtraDb() common.Database { return s.extraDb } +func (s *Ethereum) ChainDb() common.Database { return s.chainDb } +func (s *Ethereum) DappDb() common.Database { return s.dappDb } func (s *Ethereum) IsListening() bool { return true } // Always listening func (s *Ethereum) PeerCount() int { return s.net.PeerCount() } func (s *Ethereum) Peers() []*p2p.Peer { return s.net.Peers() } @@ -569,23 +567,19 @@ done: select { case <-ticker.C: // don't change the order of database flushes - if err := s.extraDb.Flush(); err != nil { - glog.Fatalf("fatal error: flush extraDb: %v (Restart your node. We are aware of this issue)\n", err) + if err := s.dappDb.Flush(); err != nil { + glog.Fatalf("fatal error: flush dappDb: %v (Restart your node. We are aware of this issue)\n", err) } - if err := s.stateDb.Flush(); err != nil { - glog.Fatalf("fatal error: flush stateDb: %v (Restart your node. We are aware of this issue)\n", err) - } - if err := s.blockDb.Flush(); err != nil { - glog.Fatalf("fatal error: flush blockDb: %v (Restart your node. We are aware of this issue)\n", err) + if err := s.chainDb.Flush(); err != nil { + glog.Fatalf("fatal error: flush chainDb: %v (Restart your node. We are aware of this issue)\n", err) } case <-s.shutdownChan: break done } } - s.blockDb.Close() - s.stateDb.Close() - s.extraDb.Close() + s.chainDb.Close() + s.dappDb.Close() close(s.databasesClosed) } @@ -683,14 +677,6 @@ func (self *Ethereum) StartAutoDAG() { }() } -// dagFiles(epoch) returns the two alternative DAG filenames (not a path) -// 1) - 2) full-R- -func dagFiles(epoch uint64) (string, string) { - seedHash, _ := ethash.GetSeedHash(epoch * epochLength) - dag := fmt.Sprintf("full-R%d-%x", ethashRevision, seedHash[:8]) - return dag, "full-R" + dag -} - // stopAutoDAG stops automatic DAG pregeneration by quitting the loop func (self *Ethereum) StopAutoDAG() { if self.autodagquit != nil { @@ -700,30 +686,6 @@ func (self *Ethereum) StopAutoDAG() { glog.V(logger.Info).Infof("Automatic pregeneration of ethash DAG OFF (ethash dir: %s)", ethash.DefaultDir) } -/* - // The databases were previously tied to protocol versions. Currently we - // are moving away from this decision as approaching Frontier. The below - // code was left in for now but should eventually be just dropped. - - func saveProtocolVersion(db common.Database, protov int) { - d, _ := db.Get([]byte("ProtocolVersion")) - protocolVersion := common.NewValue(d).Uint() - - if protocolVersion == 0 { - db.Put([]byte("ProtocolVersion"), common.NewValue(protov).Bytes()) - } - } -*/ - -func saveBlockchainVersion(db common.Database, bcVersion int) { - d, _ := db.Get([]byte("BlockchainVersion")) - blockchainVersion := common.NewValue(d).Uint() - - if blockchainVersion == 0 { - db.Put([]byte("BlockchainVersion"), common.NewValue(bcVersion).Bytes()) - } -} - func (self *Ethereum) Solc() (*compiler.Solidity, error) { var err error if self.solc == nil { @@ -738,3 +700,87 @@ func (self *Ethereum) SetSolc(solcPath string) (*compiler.Solidity, error) { self.solc = nil return self.Solc() } + +// dagFiles(epoch) returns the two alternative DAG filenames (not a path) +// 1) - 2) full-R- +func dagFiles(epoch uint64) (string, string) { + seedHash, _ := ethash.GetSeedHash(epoch * epochLength) + dag := fmt.Sprintf("full-R%d-%x", ethashRevision, seedHash[:8]) + return dag, "full-R" + dag +} + +func saveBlockchainVersion(db common.Database, bcVersion int) { + d, _ := db.Get([]byte("BlockchainVersion")) + blockchainVersion := common.NewValue(d).Uint() + + if blockchainVersion == 0 { + db.Put([]byte("BlockchainVersion"), common.NewValue(bcVersion).Bytes()) + } +} + +// mergeDatabases when required merge old database layout to one single database +func mergeDatabases(datadir string, newdb func(path string) (common.Database, error)) error { + // Check if already upgraded + data := filepath.Join(datadir, "chaindata") + if _, err := os.Stat(data); !os.IsNotExist(err) { + return nil + } + // make sure it's not just a clean path + chainPath := filepath.Join(datadir, "blockchain") + if _, err := os.Stat(chainPath); os.IsNotExist(err) { + return nil + } + glog.Infoln("Database upgrade required. Upgrading...") + + database, err := newdb(data) + if err != nil { + return fmt.Errorf("creating data db err: %v", err) + } + defer database.Close() + + glog.Infoln("Merging blockchain database...") + chainDb, err := newdb(chainPath) + if err != nil { + return fmt.Errorf("state db err: %v", err) + } + defer chainDb.Close() + + if db, ok := chainDb.(*ethdb.LDBDatabase); ok { + it := db.NewIterator() + for it.Next() { + database.Put(it.Key(), it.Value()) + } + } + + glog.Infoln("Merging state database...") + state := filepath.Join(datadir, "state") + stateDb, err := newdb(state) + if err != nil { + return fmt.Errorf("state db err: %v", err) + } + defer stateDb.Close() + + if db, ok := chainDb.(*ethdb.LDBDatabase); ok { + it := db.NewIterator() + for it.Next() { + database.Put(append(trie.StatePre, it.Key()...), it.Value()) + } + } + + glog.Infoln("Merging transaction database...") + extra := filepath.Join(datadir, "extra") + extraDb, err := newdb(extra) + if err != nil { + return fmt.Errorf("state db err: %v", err) + } + defer extraDb.Close() + + if db, ok := chainDb.(*ethdb.LDBDatabase); ok { + it := db.NewIterator() + for it.Next() { + database.Put(it.Key(), it.Value()) + } + } + + return nil +} diff --git a/eth/protocol_test.go b/eth/protocol_test.go index a24d98f69..08c9b6a06 100644 --- a/eth/protocol_test.go +++ b/eth/protocol_test.go @@ -179,10 +179,10 @@ type testPeer struct { func newProtocolManagerForTesting(txAdded chan<- []*types.Transaction) *ProtocolManager { db, _ := ethdb.NewMemDatabase() - core.WriteTestNetGenesisBlock(db, db, 0) + core.WriteTestNetGenesisBlock(db, 0) var ( em = new(event.TypeMux) - chain, _ = core.NewChainManager(db, db, db, core.FakePow{}, em) + chain, _ = core.NewChainManager(db, core.FakePow{}, em) txpool = &fakeTxPool{added: txAdded} pm = NewProtocolManager(NetworkId, em, txpool, core.FakePow{}, chain) ) diff --git a/ethdb/database.go b/ethdb/database.go index ace56c6c7..9e80e5409 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -39,9 +39,8 @@ var OpenFileLimit = 64 // cacheRatio specifies how the total alloted cache is distributed between the // various system databases. var cacheRatio = map[string]float64{ - "blockchain": 1.0 / 13.0, - "extra": 2.0 / 13.0, - "state": 10.0 / 13.0, + "dapp": 2.0 / 13.0, + "chaindata": 11.0 / 13.0, } type LDBDatabase struct { diff --git a/miner/worker.go b/miner/worker.go index 535ce5144..df3681470 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -100,7 +100,7 @@ type worker struct { eth core.Backend chain *core.ChainManager proc *core.BlockProcessor - extraDb common.Database + chainDb common.Database coinbase common.Address gasPrice *big.Int @@ -126,7 +126,7 @@ func newWorker(coinbase common.Address, eth core.Backend) *worker { worker := &worker{ eth: eth, mux: eth.EventMux(), - extraDb: eth.ExtraDb(), + chainDb: eth.ChainDb(), recv: make(chan *Result, resultQueueSize), gasPrice: new(big.Int), chain: eth.ChainManager(), @@ -291,9 +291,9 @@ func (self *worker) wait() { // check if canon block and write transactions if stat == core.CanonStatTy { // This puts transactions in a extra db for rpc - core.PutTransactions(self.extraDb, block, block.Transactions()) + core.PutTransactions(self.chainDb, block, block.Transactions()) // store the receipts - core.PutReceipts(self.extraDb, work.receipts) + core.PutReceipts(self.chainDb, work.receipts) } // broadcast before waiting for validation @@ -344,7 +344,7 @@ func (self *worker) push(work *Work) { // makeCurrent creates a new environment for the current cycle. func (self *worker) makeCurrent(parent *types.Block, header *types.Header) { - state := state.New(parent.Root(), self.eth.StateDb()) + state := state.New(parent.Root(), self.eth.ChainDb()) work := &Work{ state: state, ancestors: set.New(), diff --git a/rpc/api/debug.go b/rpc/api/debug.go index cdacd6c62..d325b1720 100644 --- a/rpc/api/debug.go +++ b/rpc/api/debug.go @@ -119,7 +119,7 @@ func (self *debugApi) DumpBlock(req *shared.Request) (interface{}, error) { return nil, fmt.Errorf("block #%d not found", args.BlockNumber) } - stateDb := state.New(block.Root(), self.ethereum.StateDb()) + stateDb := state.New(block.Root(), self.ethereum.ChainDb()) if stateDb == nil { return nil, nil } diff --git a/tests/block_test_util.go b/tests/block_test_util.go index 42e4383d1..8cb7b7882 100644 --- a/tests/block_test_util.go +++ b/tests/block_test_util.go @@ -204,7 +204,7 @@ func (test *BlockTest) makeEthConfig() *eth.Config { // InsertPreState populates the given database with the genesis // accounts defined by the test. func (t *BlockTest) InsertPreState(ethereum *eth.Ethereum) (*state.StateDB, error) { - db := ethereum.StateDb() + db := ethereum.ChainDb() statedb := state.New(common.Hash{}, db) for addrString, acct := range t.preAccounts { addr, err := hex.DecodeString(addrString) diff --git a/trie/cache.go b/trie/cache.go index 2705b0e45..99d8033a6 100644 --- a/trie/cache.go +++ b/trie/cache.go @@ -38,6 +38,8 @@ func NewCache(backend Backend) *Cache { } func (self *Cache) Get(key []byte) []byte { + key = append(StatePre, key...) + data := self.store[string(key)] if data == nil { data, _ = self.backend.Get(key) @@ -47,8 +49,8 @@ func (self *Cache) Get(key []byte) []byte { } func (self *Cache) Put(key []byte, data []byte) { - // write the data to the ldb batch - //self.batch.Put(key, rle.Compress(data)) + key = append(StatePre, key...) + self.batch.Put(key, data) self.store[string(key)] = data } diff --git a/trie/trie.go b/trie/trie.go index abf48a850..2970bc185 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -27,6 +27,8 @@ import ( "github.com/ethereum/go-ethereum/crypto" ) +var StatePre = []byte("state-") + func ParanoiaCheck(t1 *Trie, backend Backend) (bool, *Trie) { t2 := New(nil, backend) diff --git a/xeth/state.go b/xeth/state.go index 16bfb523d..981fe63b7 100644 --- a/xeth/state.go +++ b/xeth/state.go @@ -45,7 +45,7 @@ func (self *State) SafeGet(addr string) *Object { func (self *State) safeGet(addr string) *state.StateObject { object := self.state.GetStateObject(common.HexToAddress(addr)) if object == nil { - object = state.NewStateObject(common.HexToAddress(addr), self.xeth.backend.StateDb()) + object = state.NewStateObject(common.HexToAddress(addr), self.xeth.backend.ChainDb()) } return object diff --git a/xeth/xeth.go b/xeth/xeth.go index 372068c14..b80b62155 100644 --- a/xeth/xeth.go +++ b/xeth/xeth.go @@ -213,9 +213,9 @@ func (self *XEth) AtStateNum(num int64) *XEth { st = self.backend.Miner().PendingState().Copy() default: if block := self.getBlockByHeight(num); block != nil { - st = state.New(block.Root(), self.backend.StateDb()) + st = state.New(block.Root(), self.backend.ChainDb()) } else { - st = state.New(self.backend.ChainManager().GetBlockByNumber(0).Root(), self.backend.StateDb()) + st = state.New(self.backend.ChainManager().GetBlockByNumber(0).Root(), self.backend.ChainDb()) } } @@ -259,7 +259,7 @@ func (self *XEth) UpdateState() (wait chan *big.Int) { wait <- n n = nil } - statedb := state.New(ev.Block.Root(), self.backend.StateDb()) + statedb := state.New(ev.Block.Root(), self.backend.ChainDb()) self.state = NewState(self, statedb) } case n, ok = <-wait: @@ -311,7 +311,7 @@ func (self *XEth) EthBlockByHash(strHash string) *types.Block { func (self *XEth) EthTransactionByHash(hash string) (tx *types.Transaction, blhash common.Hash, blnum *big.Int, txi uint64) { // Due to increasing return params and need to determine if this is from transaction pool or // some chain, this probably needs to be refactored for more expressiveness - data, _ := self.backend.ExtraDb().Get(common.FromHex(hash)) + data, _ := self.backend.ChainDb().Get(common.FromHex(hash)) if len(data) != 0 { dtx := new(types.Transaction) if err := rlp.DecodeBytes(data, dtx); err != nil { @@ -330,7 +330,7 @@ func (self *XEth) EthTransactionByHash(hash string) (tx *types.Transaction, blha Index uint64 } - v, dberr := self.backend.ExtraDb().Get(append(common.FromHex(hash), 0x0001)) + v, dberr := self.backend.ChainDb().Get(append(common.FromHex(hash), 0x0001)) // TODO check specifically for ErrNotFound if dberr != nil { return @@ -365,7 +365,7 @@ func (self *XEth) GetBlockReceipts(bhash common.Hash) types.Receipts { } func (self *XEth) GetTxReceipt(txhash common.Hash) *types.Receipt { - return core.GetReceipt(self.backend.ExtraDb(), txhash) + return core.GetReceipt(self.backend.ChainDb(), txhash) } func (self *XEth) GasLimit() *big.Int { @@ -408,13 +408,13 @@ func (self *XEth) SetSolc(solcPath string) (*compiler.Solidity, error) { // store DApp value in extra database func (self *XEth) DbPut(key, val []byte) bool { - self.backend.ExtraDb().Put(append(dappStorePre, key...), val) + self.backend.DappDb().Put(append(dappStorePre, key...), val) return true } // retrieve DApp value from extra database func (self *XEth) DbGet(key []byte) ([]byte, error) { - val, err := self.backend.ExtraDb().Get(append(dappStorePre, key...)) + val, err := self.backend.DappDb().Get(append(dappStorePre, key...)) return val, err }