db: add options to configure block index database
There was a previous assumption that blockindex would be quite small. With addressindex and spentindex enabled the blockindex is much larger and the amount of cache allocated for it should also increase. Furthermore, enabling compression should decrease the amount of disk space required and less data to write/read. The default leveldb max_open_files is set to 1000, for the blockindex the default is set to 1000 with compression. The 64 value that is current is kept for the utxo database and does not enable compression. Two additional options are added here to be able to configure the values for leveldb and the block index: - `-dbmaxopenfiles` A number of files for leveldb to keep open - `-dbcompression` Boolean 0 or 1 to enable snappy leveldb compression
This commit is contained in:
parent
f402ac3736
commit
14ac450af3
|
@ -14,14 +14,14 @@
|
|||
#include <memenv.h>
|
||||
#include <stdint.h>
|
||||
|
||||
static leveldb::Options GetOptions(size_t nCacheSize)
|
||||
static leveldb::Options GetOptions(size_t nCacheSize, bool compression, int maxOpenFiles)
|
||||
{
|
||||
leveldb::Options options;
|
||||
options.block_cache = leveldb::NewLRUCache(nCacheSize / 2);
|
||||
options.write_buffer_size = nCacheSize / 4; // up to two write buffers may be held in memory simultaneously
|
||||
options.filter_policy = leveldb::NewBloomFilterPolicy(10);
|
||||
options.compression = leveldb::kNoCompression;
|
||||
options.max_open_files = 64;
|
||||
options.compression = compression ? leveldb::kSnappyCompression : leveldb::kNoCompression;
|
||||
options.max_open_files = maxOpenFiles;
|
||||
if (leveldb::kMajorVersion > 1 || (leveldb::kMajorVersion == 1 && leveldb::kMinorVersion >= 16)) {
|
||||
// LevelDB versions before 1.16 consider short writes to be corruption. Only trigger error
|
||||
// on corruption in later versions.
|
||||
|
@ -30,14 +30,14 @@ static leveldb::Options GetOptions(size_t nCacheSize)
|
|||
return options;
|
||||
}
|
||||
|
||||
CDBWrapper::CDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory, bool fWipe)
|
||||
CDBWrapper::CDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory, bool fWipe, bool compression, int maxOpenFiles)
|
||||
{
|
||||
penv = NULL;
|
||||
readoptions.verify_checksums = true;
|
||||
iteroptions.verify_checksums = true;
|
||||
iteroptions.fill_cache = false;
|
||||
syncoptions.sync = true;
|
||||
options = GetOptions(nCacheSize);
|
||||
options = GetOptions(nCacheSize, compression, maxOpenFiles);
|
||||
options.create_if_missing = true;
|
||||
if (fMemory) {
|
||||
penv = leveldb::NewMemEnv(leveldb::Env::Default());
|
||||
|
|
|
@ -173,7 +173,7 @@ public:
|
|||
* @param[in] fMemory If true, use leveldb's memory environment.
|
||||
* @param[in] fWipe If true, remove all existing data.
|
||||
*/
|
||||
CDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false);
|
||||
CDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false, bool compression = false, int maxOpenFiles = 64);
|
||||
~CDBWrapper();
|
||||
|
||||
template <typename K, typename V>
|
||||
|
|
21
src/init.cpp
21
src/init.cpp
|
@ -1443,18 +1443,33 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler)
|
|||
}
|
||||
}
|
||||
|
||||
// block tree db settings
|
||||
int dbMaxOpenFiles = GetArg("-dbmaxopenfiles", DEFAULT_DB_MAX_OPEN_FILES);
|
||||
bool dbCompression = GetBoolArg("-dbcompression", DEFAULT_DB_COMPRESSION);
|
||||
|
||||
LogPrintf("Block index database configuration:\n");
|
||||
LogPrintf("* Using %d max open files\n", dbMaxOpenFiles);
|
||||
LogPrintf("* Compression is %s\n", dbCompression ? "enabled" : "disabled");
|
||||
|
||||
// cache size calculations
|
||||
int64_t nTotalCache = (GetArg("-dbcache", nDefaultDbCache) << 20);
|
||||
nTotalCache = std::max(nTotalCache, nMinDbCache << 20); // total cache cannot be less than nMinDbCache
|
||||
nTotalCache = std::min(nTotalCache, nMaxDbCache << 20); // total cache cannot be greated than nMaxDbcache
|
||||
int64_t nBlockTreeDBCache = nTotalCache / 8;
|
||||
if (nBlockTreeDBCache > (1 << 21) && !GetBoolArg("-txindex", false))
|
||||
nBlockTreeDBCache = (1 << 21); // block tree db cache shouldn't be larger than 2 MiB
|
||||
if (GetBoolArg("-addressindex", DEFAULT_ADDRESSINDEX) || GetBoolArg("-spentindex", DEFAULT_SPENTINDEX)) {
|
||||
// enable 3/4 of the cache if addressindex and/or spentindex is enabled
|
||||
nBlockTreeDBCache = nTotalCache * 3 / 4;
|
||||
} else {
|
||||
if (nBlockTreeDBCache > (1 << 21) && !GetBoolArg("-txindex", false)) {
|
||||
nBlockTreeDBCache = (1 << 21); // block tree db cache shouldn't be larger than 2 MiB
|
||||
}
|
||||
}
|
||||
nTotalCache -= nBlockTreeDBCache;
|
||||
int64_t nCoinDBCache = std::min(nTotalCache / 2, (nTotalCache / 4) + (1 << 23)); // use 25%-50% of the remainder for disk cache
|
||||
nTotalCache -= nCoinDBCache;
|
||||
nCoinCacheUsage = nTotalCache; // the rest goes to in-memory cache
|
||||
LogPrintf("Cache configuration:\n");
|
||||
LogPrintf("* Max cache setting possible %.1fMiB\n", nMaxDbCache);
|
||||
LogPrintf("* Using %.1fMiB for block index database\n", nBlockTreeDBCache * (1.0 / 1024 / 1024));
|
||||
LogPrintf("* Using %.1fMiB for chain state database\n", nCoinDBCache * (1.0 / 1024 / 1024));
|
||||
LogPrintf("* Using %.1fMiB for in-memory UTXO set\n", nCoinCacheUsage * (1.0 / 1024 / 1024));
|
||||
|
@ -1477,7 +1492,7 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler)
|
|||
delete pcoinscatcher;
|
||||
delete pblocktree;
|
||||
|
||||
pblocktree = new CBlockTreeDB(nBlockTreeDBCache, false, fReindex);
|
||||
pblocktree = new CBlockTreeDB(nBlockTreeDBCache, false, fReindex, dbCompression, dbMaxOpenFiles);
|
||||
pcoinsdbview = new CCoinsViewDB(nCoinDBCache, false, fReindex);
|
||||
pcoinscatcher = new CCoinsViewErrorCatcher(pcoinsdbview);
|
||||
pcoinsTip = new CCoinsViewCache(pcoinscatcher);
|
||||
|
|
|
@ -105,6 +105,8 @@ static const int64_t DEFAULT_MAX_TIP_AGE = 24 * 60 * 60;
|
|||
static const bool DEFAULT_ADDRESSINDEX = false;
|
||||
static const bool DEFAULT_TIMESTAMPINDEX = false;
|
||||
static const bool DEFAULT_SPENTINDEX = false;
|
||||
static const unsigned int DEFAULT_DB_MAX_OPEN_FILES = 1000;
|
||||
static const bool DEFAULT_DB_COMPRESSION = true;
|
||||
|
||||
// Sanity check the magic numbers when we change them
|
||||
BOOST_STATIC_ASSERT(DEFAULT_BLOCK_MAX_SIZE <= MAX_BLOCK_SIZE);
|
||||
|
|
|
@ -40,11 +40,10 @@ static const char DB_REINDEX_FLAG = 'R';
|
|||
static const char DB_LAST_BLOCK = 'l';
|
||||
|
||||
|
||||
CCoinsViewDB::CCoinsViewDB(std::string dbName, size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / dbName, nCacheSize, fMemory, fWipe) {
|
||||
CCoinsViewDB::CCoinsViewDB(std::string dbName, size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / dbName, nCacheSize, fMemory, fWipe, false, 64) {
|
||||
}
|
||||
|
||||
CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "chainstate", nCacheSize, fMemory, fWipe)
|
||||
{
|
||||
CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "chainstate", nCacheSize, fMemory, fWipe, false, 64) {
|
||||
}
|
||||
|
||||
|
||||
|
@ -197,7 +196,7 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins,
|
|||
return db.WriteBatch(batch);
|
||||
}
|
||||
|
||||
CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe) : CDBWrapper(GetDataDir() / "blocks" / "index", nCacheSize, fMemory, fWipe) {
|
||||
CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe, bool compression, int maxOpenFiles) : CDBWrapper(GetDataDir() / "blocks" / "index", nCacheSize, fMemory, fWipe, compression, maxOpenFiles) {
|
||||
}
|
||||
|
||||
bool CBlockTreeDB::ReadBlockFileInfo(int nFile, CBlockFileInfo &info) {
|
||||
|
|
|
@ -66,7 +66,7 @@ public:
|
|||
class CBlockTreeDB : public CDBWrapper
|
||||
{
|
||||
public:
|
||||
CBlockTreeDB(size_t nCacheSize, bool fMemory = false, bool fWipe = false);
|
||||
CBlockTreeDB(size_t nCacheSize, bool fMemory = false, bool fWipe = false, bool compression = true, int maxOpenFiles = 1000);
|
||||
private:
|
||||
CBlockTreeDB(const CBlockTreeDB&);
|
||||
void operator=(const CBlockTreeDB&);
|
||||
|
|
Loading…
Reference in New Issue