Merge pull request #1508 from karalabe/database-caching

cmd, core, eth, ethdb: cache flag to allocate memory for db internal use
This commit is contained in:
Jeffrey Wilcke 2015-07-23 01:17:18 -07:00
commit b403b9e4c3
6 changed files with 43 additions and 15 deletions

View File

@ -281,6 +281,7 @@ JavaScript API. See https://github.com/ethereum/go-ethereum/wiki/Javascipt-Conso
utils.BootnodesFlag,
utils.DataDirFlag,
utils.BlockchainVersionFlag,
utils.CacheFlag,
utils.JSpathFlag,
utils.ListenPortFlag,
utils.MaxPeersFlag,
@ -501,7 +502,7 @@ func blockRecovery(ctx *cli.Context) {
cfg := utils.MakeEthConfig(ClientIdentifier, nodeNameVersion, ctx)
utils.CheckLegalese(cfg.DataDir)
blockDb, err := ethdb.NewLDBDatabase(filepath.Join(cfg.DataDir, "blockchain"))
blockDb, err := ethdb.NewLDBDatabase(filepath.Join(cfg.DataDir, "blockchain"), cfg.DatabaseCache)
if err != nil {
glog.Fatalln("could not open db:", err)
}

View File

@ -126,6 +126,11 @@ var (
Name: "natspec",
Usage: "Enable NatSpec confirmation notice",
}
CacheFlag = cli.IntFlag{
Name: "cache",
Usage: "Megabytes of memory allocated to internal caching",
Value: 0,
}
// miner settings
MinerThreadsFlag = cli.IntFlag{
@ -384,6 +389,7 @@ func MakeEthConfig(clientID, version string, ctx *cli.Context) *eth.Config {
GenesisNonce: ctx.GlobalInt(GenesisNonceFlag.Name),
GenesisFile: ctx.GlobalString(GenesisFileFlag.Name),
BlockChainVersion: ctx.GlobalInt(BlockchainVersionFlag.Name),
DatabaseCache: ctx.GlobalInt(CacheFlag.Name),
SkipBcVersionCheck: false,
NetworkId: ctx.GlobalInt(NetworkIdFlag.Name),
LogFile: ctx.GlobalString(LogFileFlag.Name),
@ -425,15 +431,17 @@ func SetupLogger(ctx *cli.Context) {
// MakeChain creates a chain manager from set command line flags.
func MakeChain(ctx *cli.Context) (chain *core.ChainManager, blockDB, stateDB, extraDB common.Database) {
dd := ctx.GlobalString(DataDirFlag.Name)
datadir := ctx.GlobalString(DataDirFlag.Name)
cache := ctx.GlobalInt(CacheFlag.Name)
var err error
if blockDB, err = ethdb.NewLDBDatabase(filepath.Join(dd, "blockchain")); err != nil {
if blockDB, err = ethdb.NewLDBDatabase(filepath.Join(datadir, "blockchain"), cache); err != nil {
Fatalf("Could not open database: %v", err)
}
if stateDB, err = ethdb.NewLDBDatabase(filepath.Join(dd, "state")); err != nil {
if stateDB, err = ethdb.NewLDBDatabase(filepath.Join(datadir, "state"), cache); err != nil {
Fatalf("Could not open database: %v", err)
}
if extraDB, err = ethdb.NewLDBDatabase(filepath.Join(dd, "extra")); err != nil {
if extraDB, err = ethdb.NewLDBDatabase(filepath.Join(datadir, "extra"), cache); err != nil {
Fatalf("Could not open database: %v", err)
}

View File

@ -153,7 +153,7 @@ func benchInsertChain(b *testing.B, disk bool, gen func(int, *BlockGen)) {
b.Fatalf("cannot create temporary directory: %v", err)
}
defer os.RemoveAll(dir)
db, err = ethdb.NewLDBDatabase(dir)
db, err = ethdb.NewLDBDatabase(dir, 0)
if err != nil {
b.Fatalf("cannot create temporary database: %v", err)
}

View File

@ -80,6 +80,7 @@ type Config struct {
BlockChainVersion int
SkipBcVersionCheck bool // e.g. blockchain export
DatabaseCache int
DataDir string
LogFile string
@ -261,7 +262,7 @@ func New(config *Config) (*Ethereum, error) {
newdb := config.NewDB
if newdb == nil {
newdb = func(path string) (common.Database, error) { return ethdb.NewLDBDatabase(path) }
newdb = func(path string) (common.Database, error) { return ethdb.NewLDBDatabase(path, config.DatabaseCache) }
}
blockDb, err := newdb(filepath.Join(config.DataDir, "blockchain"))
if err != nil {

View File

@ -17,6 +17,7 @@
package ethdb
import (
"path/filepath"
"strconv"
"strings"
"sync"
@ -36,6 +37,14 @@ import (
var OpenFileLimit = 64
// cacheRatio specifies how the total alloted cache is distributed between the
// various system databases.
var cacheRatio = map[string]float64{
"blockchain": 1.0 / 13.0,
"extra": 2.0 / 13.0,
"state": 10.0 / 13.0,
}
type LDBDatabase struct {
fn string // filename for reporting
db *leveldb.DB // LevelDB instance
@ -57,14 +66,24 @@ type LDBDatabase struct {
// NewLDBDatabase returns a LevelDB wrapped object. LDBDatabase does not persist data by
// it self but requires a background poller which syncs every X. `Flush` should be called
// when data needs to be stored and written to disk.
func NewLDBDatabase(file string) (*LDBDatabase, error) {
// Open the db
db, err := leveldb.OpenFile(file, &opt.Options{OpenFilesCacheCapacity: OpenFileLimit})
// check for corruption and attempt to recover
if _, iscorrupted := err.(*errors.ErrCorrupted); iscorrupted {
func NewLDBDatabase(file string, cache int) (*LDBDatabase, error) {
// Calculate the cache allowance for this particular database
cache = int(float64(cache) * cacheRatio[filepath.Base(file)])
if cache < 16 {
cache = 16
}
glog.V(logger.Info).Infof("Alloted %dMB cache to %s", cache, file)
// Open the db and recover any potential corruptions
db, err := leveldb.OpenFile(file, &opt.Options{
OpenFilesCacheCapacity: OpenFileLimit,
BlockCacheCapacity: cache / 2 * opt.MiB,
WriteBuffer: cache / 4 * opt.MiB, // Two of these are used internally
})
if _, corrupted := err.(*errors.ErrCorrupted); corrupted {
db, err = leveldb.RecoverFile(file, nil)
}
// (re) check for errors and abort if opening of the db failed
// (Re)check for errors and abort if opening of the db failed
if err != nil {
return nil, err
}

View File

@ -28,8 +28,7 @@ func newDb() *LDBDatabase {
if common.FileExist(file) {
os.RemoveAll(file)
}
db, _ := NewLDBDatabase(file)
db, _ := NewLDBDatabase(file, 0)
return db
}