diff --git a/src/db.cpp b/src/db.cpp index cef395c44..53be48cb0 100644 --- a/src/db.cpp +++ b/src/db.cpp @@ -546,6 +546,22 @@ bool CTxDB::WriteBlockIndex(const CDiskBlockIndex& blockindex) return Write(make_pair(string("blockindex"), blockindex.GetBlockHash()), blockindex); } +bool CTxDB::WriteBlockFileInfo(int nFile, const CBlockFileInfo &info) { + return Write(make_pair(string("blockfile"), nFile), info); +} + +bool CTxDB::ReadBlockFileInfo(int nFile, CBlockFileInfo &info) { + return Read(make_pair(string("blockfile"), nFile), info); +} + +bool CTxDB::WriteLastBlockFile(int nFile) { + return Write(string("lastblockfile"), nFile); +} + +bool CTxDB::ReadLastBlockFile(int &nFile) { + return Read(string("lastblockfile"), nFile); +} + bool CTxDB::ReadHashBestChain(uint256& hashBestChain) { return Read(string("hashBestChain"), hashBestChain); @@ -609,6 +625,12 @@ bool CTxDB::LoadBlockIndex() pindex->bnChainWork = (pindex->pprev ? pindex->pprev->bnChainWork : 0) + pindex->GetBlockWork(); } + // Load block file info + ReadLastBlockFile(nLastBlockFile); + printf("LoadBlockIndex(): last block file = %i\n", nLastBlockFile); + if (ReadBlockFileInfo(nLastBlockFile, infoLastBlockFile)) + printf("LoadBlockIndex(): last block file: %s\n", infoLastBlockFile.ToString().c_str()); + // Load hashBestChain pointer to end of best chain if (!ReadHashBestChain(hashBestChain)) { @@ -788,7 +810,8 @@ bool CTxDB::LoadBlockIndexGuts() pindexNew->pprev = InsertBlockIndex(diskindex.hashPrev); pindexNew->pnext = InsertBlockIndex(diskindex.hashNext); pindexNew->nHeight = diskindex.nHeight; - pindexNew->nAlternative = diskindex.nAlternative; + pindexNew->pos = diskindex.pos; + pindexNew->nUndoPos = diskindex.nUndoPos; pindexNew->nVersion = diskindex.nVersion; pindexNew->hashMerkleRoot = diskindex.hashMerkleRoot; pindexNew->nTime = diskindex.nTime; diff --git a/src/db.h b/src/db.h index 798913645..61b59060a 100644 --- a/src/db.h +++ b/src/db.h @@ -339,6 +339,10 @@ public: bool WriteHashBestChain(uint256 hashBestChain); bool ReadBestInvalidWork(CBigNum& bnBestInvalidWork); bool WriteBestInvalidWork(CBigNum bnBestInvalidWork); + bool ReadBlockFileInfo(int nFile, CBlockFileInfo &fileinfo); + bool WriteBlockFileInfo(int nFile, const CBlockFileInfo &fileinfo); + bool ReadLastBlockFile(int &nFile); + bool WriteLastBlockFile(int nFile); bool LoadBlockIndex(); private: bool LoadBlockIndexGuts(); diff --git a/src/main.cpp b/src/main.cpp index 8b1532e48..15a233137 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1399,6 +1399,8 @@ bool CBlock::DisconnectBlock(CTxDB& txdb, CBlockIndex* pindex) return true; } +bool FindUndoPos(CTxDB &txdb, int nFile, CDiskBlockPos &pos, unsigned int nAddSize); + bool CBlock::ConnectBlock(CTxDB& txdb, CBlockIndex* pindex, bool fJustCheck) { // Check it again in case a previous version let a bad block in @@ -1430,7 +1432,7 @@ bool CBlock::ConnectBlock(CTxDB& txdb, CBlockIndex* pindex, bool fJustCheck) // Since we're just checking the block and not actually connecting it, it might not (and probably shouldn't) be on the disk to get the transaction from nTxPos = 1; else - nTxPos = ::GetSerializeSize(CBlock(), SER_DISK, CLIENT_VERSION) - 1 + GetSizeOfCompactSize(vtx.size()); + nTxPos = pindex->pos.nPos + ::GetSerializeSize(CBlock(), SER_DISK, CLIENT_VERSION) - 1 + GetSizeOfCompactSize(vtx.size()); CBlockUndo blockundo; @@ -1507,6 +1509,17 @@ bool CBlock::ConnectBlock(CTxDB& txdb, CBlockIndex* pindex, bool fJustCheck) return error("ConnectBlock() : UpdateTxIndex failed"); } + // Write undo information to disk + if (pindex->GetUndoPos().IsNull() && pindex->nHeight > Checkpoints::GetTotalBlocksEstimate()) + { + CDiskBlockPos pos; + if (!FindUndoPos(txdb, pindex->pos.nFile, pos, ::GetSerializeSize(blockundo, SER_DISK, CLIENT_VERSION) + 8)) + return error("ConnectBlock() : FindUndoPos failed"); + if (!blockundo.WriteToDisk(pos)) + return error("ConnectBlock() : CBlockUndo::WriteToDisk failed"); + pindex->nUndoPos = pos.nPos; + } + // Update block index on disk without changing it in memory. // The memory index structure will be changed after the db commits. if (pindex->pprev) @@ -1517,13 +1530,6 @@ bool CBlock::ConnectBlock(CTxDB& txdb, CBlockIndex* pindex, bool fJustCheck) return error("ConnectBlock() : WriteBlockIndex failed"); } - // Write undo information to disk - if (pindex->nHeight > Checkpoints::GetTotalBlocksEstimate()) - { - CAutoFile fileUndo(fopen(pindex->GetBlockPos().GetUndoFile(GetDataDir()).string().c_str(), "wb"), SER_DISK, CLIENT_VERSION); - fileUndo << blockundo; - } - // Watch for transactions paying to me BOOST_FOREACH(CTransaction& tx, vtx) SyncWithWallets(tx, this, true); @@ -1788,8 +1794,8 @@ bool CBlock::AddToBlockIndex(const CDiskBlockPos &pos) pindexNew->nHeight = pindexNew->pprev->nHeight + 1; } pindexNew->bnChainWork = (pindexNew->pprev ? pindexNew->pprev->bnChainWork : 0) + pindexNew->GetBlockWork(); - assert(pos.nHeight == pindexNew->nHeight); - pindexNew->nAlternative = pos.nAlternative; + pindexNew->pos = pos; + pindexNew->nUndoPos = 0; CTxDB txdb; if (!txdb.TxnBegin()) @@ -1819,6 +1825,57 @@ bool CBlock::AddToBlockIndex(const CDiskBlockPos &pos) +bool FindBlockPos(CTxDB &txdb, CDiskBlockPos &pos, unsigned int nAddSize, unsigned int nHeight, uint64 nTime) +{ + bool fUpdatedLast = false; + + LOCK(cs_LastBlockFile); + + while (infoLastBlockFile.nSize + nAddSize >= MAX_BLOCKFILE_SIZE) { + printf("Leaving block file %i: %s\n", nLastBlockFile, infoLastBlockFile.ToString().c_str()); + nLastBlockFile++; + infoLastBlockFile.SetNull(); + txdb.ReadBlockFileInfo(nLastBlockFile, infoLastBlockFile); // check whether data for the new file somehow already exist; can fail just fine + fUpdatedLast = true; + } + + pos.nFile = nLastBlockFile; + pos.nPos = infoLastBlockFile.nSize; + infoLastBlockFile.nSize += nAddSize; + infoLastBlockFile.AddBlock(nHeight, nTime); + + if (!txdb.WriteBlockFileInfo(nLastBlockFile, infoLastBlockFile)) + return error("FindBlockPos() : cannot write updated block info"); + if (fUpdatedLast) + txdb.WriteLastBlockFile(nLastBlockFile); + + return true; +} + +bool FindUndoPos(CTxDB &txdb, int nFile, CDiskBlockPos &pos, unsigned int nAddSize) +{ + pos.nFile = nFile; + + LOCK(cs_LastBlockFile); + + if (nFile == nLastBlockFile) { + pos.nPos = infoLastBlockFile.nUndoSize; + infoLastBlockFile.nUndoSize += nAddSize; + if (!txdb.WriteBlockFileInfo(nLastBlockFile, infoLastBlockFile)) + return error("FindUndoPos() : cannot write updated block info"); + return true; + } + + CBlockFileInfo info; + if (!txdb.ReadBlockFileInfo(nFile, info)) + return error("FindUndoPos() : cannot read block info"); + pos.nPos = info.nUndoSize; + info.nUndoSize += nAddSize; + if (!txdb.WriteBlockFileInfo(nFile, info)) + return error("FindUndoPos() : cannot write updated block info"); + return true; +} + bool CBlock::CheckBlock(bool fCheckPOW, bool fCheckMerkleRoot) const { @@ -1928,9 +1985,15 @@ bool CBlock::AcceptBlock() } // Write block to history file - CDiskBlockPos blockPos = CDiskBlockPos(nHeight); + unsigned int nBlockSize = ::GetSerializeSize(*this, SER_DISK, CLIENT_VERSION); if (!CheckDiskSpace(::GetSerializeSize(*this, SER_DISK, CLIENT_VERSION))) return error("AcceptBlock() : out of disk space"); + CDiskBlockPos blockPos; + { + CTxDB txdb; + if (!FindBlockPos(txdb, blockPos, nBlockSize+8, nHeight, nTime)) + return error("AcceptBlock() : FindBlockPos failed"); + } if (!WriteToDisk(blockPos)) return error("AcceptBlock() : WriteToDisk failed"); if (!AddToBlockIndex(blockPos)) @@ -2067,18 +2130,39 @@ bool CheckDiskSpace(uint64 nAdditionalBytes) return true; } -FILE* OpenBlockFile(const CDiskBlockPos &pos, const char* pszMode) +CCriticalSection cs_LastBlockFile; +CBlockFileInfo infoLastBlockFile; +int nLastBlockFile = 0; + +FILE* OpenDiskFile(const CDiskBlockPos &pos, const char *prefix, bool fReadOnly) { - boost::filesystem::path path = pos.GetFileName(GetDataDir()); - boost::filesystem::create_directories(path.parent_path()); if (pos.IsNull() || pos.IsMemPool()) return NULL; - FILE* file = fopen(path.string().c_str(), pszMode); + boost::filesystem::path path = GetDataDir() / "blocks" / strprintf("%s%05u.dat", prefix, pos.nFile); + boost::filesystem::create_directories(path.parent_path()); + FILE* file = fopen(path.string().c_str(), "rb+"); + if (!file && !fReadOnly) + file = fopen(path.string().c_str(), "wb+"); if (!file) return NULL; + if (pos.nPos) { + if (fseek(file, pos.nPos, SEEK_SET)) { + printf("Unable to seek to position %u of %s\n", pos.nPos, path.string().c_str()); + fclose(file); + return NULL; + } + } return file; } +FILE* OpenBlockFile(const CDiskBlockPos &pos, bool fReadOnly) { + return OpenDiskFile(pos, "blk", fReadOnly); +} + +FILE *OpenUndoFile(const CDiskBlockPos &pos, bool fReadOnly) { + return OpenDiskFile(pos, "rev", fReadOnly); +} + bool LoadBlockIndex(bool fAllowNew) { if (fTestNet) @@ -2146,7 +2230,13 @@ bool LoadBlockIndex(bool fAllowNew) assert(hash == hashGenesisBlock); // Start new block file - CDiskBlockPos blockPos(0); + unsigned int nBlockSize = ::GetSerializeSize(block, SER_DISK, CLIENT_VERSION); + CDiskBlockPos blockPos; + { + CTxDB txdb; + if (!FindBlockPos(txdb, blockPos, nBlockSize+8, 0, block.nTime)) + return error("AcceptBlock() : FindBlockPos failed"); + } if (!block.WriteToDisk(blockPos)) return error("LoadBlockIndex() : writing genesis block to disk failed"); if (!block.AddToBlockIndex(blockPos)) @@ -2203,9 +2293,9 @@ void PrintBlockTree() // print item CBlock block; block.ReadFromDisk(pindex); - printf("%d (%s) %s tx %"PRIszu"", + printf("%d (blk%05u.dat:0x%lx) %s tx %"PRIszu"", pindex->nHeight, - pindex->GetBlockPos().GetFileName("").string().c_str(), + pindex->GetBlockPos().nFile, pindex->GetBlockPos().nPos, DateTimeStrFormat("%x %H:%M:%S", block.GetBlockTime()).c_str(), block.vtx.size()); diff --git a/src/main.h b/src/main.h index 9257b53f3..449cc47df 100644 --- a/src/main.h +++ b/src/main.h @@ -28,6 +28,7 @@ static const unsigned int MAX_BLOCK_SIZE_GEN = MAX_BLOCK_SIZE/2; static const unsigned int MAX_BLOCK_SIGOPS = MAX_BLOCK_SIZE/50; static const unsigned int MAX_ORPHAN_TRANSACTIONS = MAX_BLOCK_SIZE/100; static const unsigned int MAX_INV_SZ = 50000; +static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB static const int64 MIN_TX_FEE = 50000; static const int64 MIN_RELAY_TX_FEE = 10000; static const int64 MAX_MONEY = 21000000 * COIN; @@ -87,7 +88,8 @@ void UnregisterWallet(CWallet* pwalletIn); void SyncWithWallets(const CTransaction& tx, const CBlock* pblock = NULL, bool fUpdate = false); bool ProcessBlock(CNode* pfrom, CBlock* pblock); bool CheckDiskSpace(uint64 nAdditionalBytes=0); -FILE* OpenBlockFile(const CDiskBlockPos &pos, const char* pszMode="rb"); +FILE* OpenBlockFile(const CDiskBlockPos &pos, bool fReadOnly = false); +FILE* OpenUndoFile(const CDiskBlockPos &pos, bool fReadOnly = false); bool LoadBlockIndex(bool fAllowNew=true); void PrintBlockTree(); CBlockIndex* FindBlockByHeight(int nHeight); @@ -121,86 +123,27 @@ bool GetWalletFile(CWallet* pwallet, std::string &strWalletFileOut); class CDiskBlockPos { public: - int nHeight; - int nAlternative; + int nFile; + unsigned int nPos; - CDiskBlockPos() { - SetNull(); - } - - CDiskBlockPos(int nHeightIn, int nAlternativeIn = 0) { - nHeight = nHeightIn; - nAlternative = nAlternativeIn; - } - - std::string GetAlternative() const { - char c[9]={0,0,0,0,0,0,0,0,0}; - char *cp = &c[8]; - unsigned int n = nAlternative; - while (n > 0 && cp>c) { - n--; - *(--cp) = 'a' + (n % 26); - n /= 26; - } - return std::string(cp); - } - - boost::filesystem::path GetDirectory(const boost::filesystem::path &base) const { - assert(nHeight != -1); - return base / strprintf("era%02u", nHeight / 210000) / - strprintf("cycle%04u", nHeight / 2016); - } - - boost::filesystem::path GetFileName(const boost::filesystem::path &base) const { - return GetDirectory(base) / strprintf("%08u%s.blk", nHeight, GetAlternative().c_str()); - } - - boost::filesystem::path GetUndoFile(const boost::filesystem::path &base) const { - return GetDirectory(base) / strprintf("%08u%s.und", nHeight, GetAlternative().c_str()); - } - - // TODO: make thread-safe (lockfile, atomic file creation, ...?) - void MakeUnique(const boost::filesystem::path &base) { - while (boost::filesystem::exists(GetFileName(base))) - nAlternative++; - } - - IMPLEMENT_SERIALIZE(({ - CDiskBlockPos *me = const_cast(this); - if (!fRead) { - unsigned int nCode = (nHeight + 1) * 2 + (nAlternative > 0); - READWRITE(VARINT(nCode)); - if (nAlternative > 0) { - unsigned int nAlt = nAlternative - 1; - READWRITE(VARINT(nAlt)); - } - } else { - unsigned int nCode = 0; - READWRITE(VARINT(nCode)); - me->nHeight = (nCode / 2) - 1; - if (nCode & 1) { - unsigned int nAlt = 0; - READWRITE(VARINT(nAlt)); - me->nAlternative = 1 + nAlt; - } else { - me->nAlternative = 0; - } - } - });) + IMPLEMENT_SERIALIZE( + READWRITE(VARINT(nFile)); + READWRITE(VARINT(nPos)); + ) friend bool operator==(const CDiskBlockPos &a, const CDiskBlockPos &b) { - return ((a.nHeight == b.nHeight) && (a.nAlternative == b.nAlternative)); + return (a.nFile == b.nFile && a.nPos == b.nPos); } friend bool operator!=(const CDiskBlockPos &a, const CDiskBlockPos &b) { return !(a == b); } - void SetNull() { nHeight = -1; nAlternative = 0; } - bool IsNull() const { return ((nHeight == -1) && (nAlternative == 0)); } + void SetNull() { nFile = -1; nPos = 0; } + bool IsNull() const { return (nFile == -1); } - void SetMemPool() { nHeight = -1; nAlternative = -1; } - bool IsMemPool() const { return ((nHeight == -1) && (nAlternative == -1)); } + void SetMemPool() { nFile = -2; nPos = 0; } + bool IsMemPool() const { return (nFile == -2); } }; /** Position on disk for a particular transaction. */ @@ -225,7 +168,7 @@ public: ) void SetNull() { blockPos.SetNull(); nTxPos = 0; } - bool IsNull() const { return blockPos.IsNull(); } + bool IsNull() const { return (nTxPos == 0); } bool IsMemPool() const { return blockPos.IsMemPool(); } friend bool operator==(const CDiskTxPos& a, const CDiskTxPos& b) @@ -246,7 +189,7 @@ public: else if (blockPos.IsMemPool()) return "mempool"; else - return strprintf("(%s, nTxPos=%u)", blockPos.GetFileName("").string().c_str(), nTxPos); + return strprintf("\"blk%05i.dat:0x%x\"", blockPos.nFile, nTxPos); } void print() const @@ -632,7 +575,7 @@ public: bool ReadFromDisk(CDiskTxPos pos, FILE** pfileRet=NULL) { - CAutoFile filein = CAutoFile(OpenBlockFile(pos.blockPos, pfileRet ? "rb+" : "rb"), SER_DISK, CLIENT_VERSION); + CAutoFile filein = CAutoFile(OpenBlockFile(pos.blockPos, pfileRet==NULL), SER_DISK, CLIENT_VERSION); if (!filein) return error("CTransaction::ReadFromDisk() : OpenBlockFile failed"); @@ -823,6 +766,33 @@ public: IMPLEMENT_SERIALIZE( READWRITE(vtxundo); ) + + bool WriteToDisk(CDiskBlockPos &pos) + { + // Open history file to append + CAutoFile fileout = CAutoFile(OpenUndoFile(pos), SER_DISK, CLIENT_VERSION); + if (!fileout) + return error("CBlockUndo::WriteToDisk() : OpenUndoFile failed"); + + // Write index header + unsigned int nSize = fileout.GetSerializeSize(*this); + fileout << FLATDATA(pchMessageStart) << nSize; + + // Write undo data + long fileOutPos = ftell(fileout); + if (fileOutPos < 0) + return error("CBlock::WriteToDisk() : ftell failed"); + pos.nPos = (unsigned int)fileOutPos; + fileout << *this; + + // Flush stdio buffers and commit to disk before returning + fflush(fileout); + if (!IsInitialBlockDownload() || (nBestHeight+1) % 500 == 0) + FileCommit(fileout); + + return true; + } + }; /** pruned version of CTransaction: only retains metadata and unspent transaction outputs @@ -1316,12 +1286,19 @@ public: bool WriteToDisk(CDiskBlockPos &pos) { // Open history file to append - pos.MakeUnique(GetDataDir()); - CAutoFile fileout = CAutoFile(OpenBlockFile(pos, "ab"), SER_DISK, CLIENT_VERSION); + CAutoFile fileout = CAutoFile(OpenBlockFile(pos), SER_DISK, CLIENT_VERSION); if (!fileout) - return error("CBlock::WriteToDisk() : AppendBlockFile failed"); + return error("CBlock::WriteToDisk() : OpenBlockFile failed"); + + // Write index header + unsigned int nSize = fileout.GetSerializeSize(*this); + fileout << FLATDATA(pchMessageStart) << nSize; // Write block + long fileOutPos = ftell(fileout); + if (fileOutPos < 0) + return error("CBlock::WriteToDisk() : ftell failed"); + pos.nPos = (unsigned int)fileOutPos; fileout << *this; // Flush stdio buffers and commit to disk before returning @@ -1337,7 +1314,7 @@ public: SetNull(); // Open history file to read - CAutoFile filein = CAutoFile(OpenBlockFile(pos, "rb"), SER_DISK, CLIENT_VERSION); + CAutoFile filein = CAutoFile(OpenBlockFile(pos, true), SER_DISK, CLIENT_VERSION); if (!filein) return error("CBlock::ReadFromDisk() : OpenBlockFile failed"); if (!fReadTransactions) @@ -1397,6 +1374,62 @@ private: +class CBlockFileInfo +{ +public: + unsigned int nBlocks; // number of blocks stored in file + unsigned int nSize; // number of used bytes of block file + unsigned int nUndoSize; // number of used bytes in the undo file + unsigned int nHeightFirst; // lowest height of block in file + unsigned int nHeightLast; // highest height of block in file + uint64 nTimeFirst; // earliest time of block in file + uint64 nTimeLast; // latest time of block in file + + IMPLEMENT_SERIALIZE( + READWRITE(VARINT(nBlocks)); + READWRITE(VARINT(nSize)); + READWRITE(VARINT(nUndoSize)); + READWRITE(VARINT(nHeightFirst)); + READWRITE(VARINT(nHeightLast)); + READWRITE(VARINT(nTimeFirst)); + READWRITE(VARINT(nTimeLast)); + ) + + void SetNull() { + nBlocks = 0; + nSize = 0; + nUndoSize = 0; + nHeightFirst = 0; + nHeightLast = 0; + nTimeFirst = 0; + nTimeLast = 0; + } + + CBlockFileInfo() { + SetNull(); + } + + std::string ToString() const { + return strprintf("CBlockFileInfo(blocks=%u, size=%lu, heights=%u..%u, time=%s..%s)", nBlocks, nSize, nHeightFirst, nHeightLast, DateTimeStrFormat("%Y-%m-%d", nTimeFirst).c_str(), DateTimeStrFormat("%Y-%m-%d", nTimeLast).c_str()); + } + + // update statistics (does not update nSize) + void AddBlock(unsigned int nHeightIn, uint64 nTimeIn) { + if (nBlocks==0 || nHeightFirst > nHeightIn) + nHeightFirst = nHeightIn; + if (nBlocks==0 || nTimeFirst > nTimeIn) + nTimeFirst = nTimeIn; + nBlocks++; + if (nHeightIn > nHeightFirst) + nHeightLast = nHeightIn; + if (nTimeIn > nTimeLast) + nTimeLast = nTimeIn; + } +}; + +extern CCriticalSection cs_LastBlockFile; +extern CBlockFileInfo infoLastBlockFile; +extern int nLastBlockFile; /** The block chain is a tree shaped structure starting with the * genesis block at the root, with each block potentially having multiple @@ -1412,7 +1445,8 @@ public: CBlockIndex* pprev; CBlockIndex* pnext; int nHeight; - unsigned int nAlternative; + CDiskBlockPos pos; + unsigned int nUndoPos; CBigNum bnChainWork; // block header @@ -1429,8 +1463,9 @@ public: pprev = NULL; pnext = NULL; nHeight = 0; + pos.SetNull(); + nUndoPos = (unsigned int)(-1); bnChainWork = 0; - nAlternative = 0; nVersion = 0; hashMerkleRoot = 0; @@ -1445,8 +1480,9 @@ public: pprev = NULL; pnext = NULL; nHeight = 0; + pos.SetNull(); + nUndoPos = 0; bnChainWork = 0; - nAlternative = 0; nVersion = block.nVersion; hashMerkleRoot = block.hashMerkleRoot; @@ -1456,7 +1492,16 @@ public: } CDiskBlockPos GetBlockPos() const { - return CDiskBlockPos(nHeight, nAlternative); + return pos; + } + + CDiskBlockPos GetUndoPos() const { + CDiskBlockPos ret = pos; + if (nUndoPos == (unsigned int)(-1)) + ret.SetNull(); + else + ret.nPos = nUndoPos; + return ret; } CBlock GetBlockHeader() const @@ -1578,7 +1623,8 @@ public: READWRITE(hashNext); READWRITE(nHeight); - READWRITE(nAlternative); + READWRITE(pos); + READWRITE(nUndoPos); // block header READWRITE(this->nVersion);