contrib/linearize: split output files based on new-timestamp-year or max-file-size

This commit is contained in:
Jeff Garzik 2014-08-23 22:59:16 -04:00
parent 476eb7eb53
commit b4a72a75b4
3 changed files with 39 additions and 9 deletions

View File

@ -26,4 +26,7 @@ output.
Optional config file setting for linearize-data: Optional config file setting for linearize-data:
* "netmagic": network magic number * "netmagic": network magic number
* "max_out_sz": maximum output file size (default 1000*1000*1000)
* "split_year": Split files when a new year is first seen, in addition to
reaching a maximum file size.

View File

@ -13,4 +13,5 @@ netmagic=f9beb4d9
input=/home/example/.bitcoin/blocks input=/home/example/.bitcoin/blocks
output_file=/home/example/Downloads/bootstrap.dat output_file=/home/example/Downloads/bootstrap.dat
hashlist=hashlist.txt hashlist=hashlist.txt
split_year=1

View File

@ -14,8 +14,7 @@ import base64
import httplib import httplib
import sys import sys
import hashlib import hashlib
import datetime
MAX_OUT_SZ = 128 * 1024 * 1024
settings = {} settings = {}
@ -41,9 +40,7 @@ def wordreverse(in_buf):
out_words.reverse() out_words.reverse()
return ''.join(out_words) return ''.join(out_words)
def calc_hdr_hash(rawblock): def calc_hdr_hash(blk_hdr):
blk_hdr = rawblock[:80]
hash1 = hashlib.sha256() hash1 = hashlib.sha256()
hash1.update(blk_hdr) hash1.update(blk_hdr)
hash1_o = hash1.digest() hash1_o = hash1.digest()
@ -54,13 +51,18 @@ def calc_hdr_hash(rawblock):
return hash2_o return hash2_o
def calc_hash_str(rawblock): def calc_hash_str(blk_hdr):
hash = calc_hdr_hash(rawblock) hash = calc_hdr_hash(blk_hdr)
hash = bufreverse(hash) hash = bufreverse(hash)
hash = wordreverse(hash) hash = wordreverse(hash)
hash_str = hash.encode('hex') hash_str = hash.encode('hex')
return hash_str return hash_str
def get_blk_year(blk_hdr):
members = struct.unpack("<I", blk_hdr[68:68+4])
dt = datetime.datetime.fromtimestamp(members[0])
return dt.year
def get_block_hashes(settings): def get_block_hashes(settings):
blkindex = [] blkindex = []
f = open(settings['hashlist'], "r") f = open(settings['hashlist'], "r")
@ -86,9 +88,14 @@ def copydata(settings, blkindex, blkset):
outF = None outF = None
blkCount = 0 blkCount = 0
lastYear = 0
splitYear = False
fileOutput = True fileOutput = True
maxOutSz = settings['max_out_sz']
if 'output' in settings: if 'output' in settings:
fileOutput = False fileOutput = False
if settings['split_year'] != 0:
splitYear = True
while True: while True:
if not inF: if not inF:
@ -111,17 +118,30 @@ def copydata(settings, blkindex, blkset):
su = struct.unpack("<I", inLenLE) su = struct.unpack("<I", inLenLE)
inLen = su[0] inLen = su[0]
rawblock = inF.read(inLen) rawblock = inF.read(inLen)
blk_hdr = rawblock[:80]
hash_str = calc_hash_str(rawblock) hash_str = calc_hash_str(blk_hdr)
if not hash_str in blkset: if not hash_str in blkset:
print("Skipping unknown block " + hash_str) print("Skipping unknown block " + hash_str)
continue continue
if not fileOutput and ((outsz + inLen) > MAX_OUT_SZ): if not fileOutput and ((outsz + inLen) > maxOutSz):
outF.close() outF.close()
outF = None outF = None
outFn = outFn + 1 outFn = outFn + 1
outsz = 0 outsz = 0
if splitYear:
blkYear = get_blk_year(blk_hdr)
if blkYear > lastYear:
print("New year " + str(blkYear) + " @ " + hash_str)
lastYear = blkYear
if outF:
outF.close()
outF = None
outFn = outFn + 1
outsz = 0
if not outF: if not outF:
if fileOutput: if fileOutput:
fname = settings['output_file'] fname = settings['output_file']
@ -164,7 +184,13 @@ if __name__ == '__main__':
settings['input'] = 'input' settings['input'] = 'input'
if 'hashlist' not in settings: if 'hashlist' not in settings:
settings['hashlist'] = 'hashlist.txt' settings['hashlist'] = 'hashlist.txt'
if 'split_year' not in settings:
settings['split_year'] = 0
if 'max_out_sz' not in settings:
settings['max_out_sz'] = 1000L * 1000 * 1000
settings['max_out_sz'] = long(settings['max_out_sz'])
settings['split_year'] = int(settings['split_year'])
settings['netmagic'] = settings['netmagic'].decode('hex') settings['netmagic'] = settings['netmagic'].decode('hex')
if 'output_file' not in settings and 'output' not in settings: if 'output_file' not in settings and 'output' not in settings: