Remove additional sources of determinism from benchmark archive
The archive has also been moved from .tar.gz to .tar.xz for a 33% reduction in size. Closes #2388.
This commit is contained in:
parent
8214ebc61c
commit
08dc7889d2
|
@ -1,17 +1,21 @@
|
||||||
import binascii
|
import binascii
|
||||||
|
import calendar
|
||||||
import json
|
import json
|
||||||
import plyvel
|
import plyvel
|
||||||
import progressbar
|
import progressbar
|
||||||
import os
|
import os
|
||||||
|
import stat
|
||||||
import struct
|
import struct
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
|
import tarfile
|
||||||
|
import time
|
||||||
|
|
||||||
ZCASH_CLI = './src/zcash-cli'
|
ZCASH_CLI = './src/zcash-cli'
|
||||||
USAGE = """
|
USAGE = """
|
||||||
Requirements:
|
Requirements:
|
||||||
- faketime
|
- find
|
||||||
- tar
|
- xz
|
||||||
- %s (edit ZCASH_CLI in this script to alter the path)
|
- %s (edit ZCASH_CLI in this script to alter the path)
|
||||||
- A running mainnet zcashd using the default datadir with -txindex=1
|
- A running mainnet zcashd using the default datadir with -txindex=1
|
||||||
|
|
||||||
|
@ -26,7 +30,7 @@ LD_LIBRARY_PATH=src/leveldb python qa/zcash/create_benchmark_archive.py
|
||||||
""" % ZCASH_CLI
|
""" % ZCASH_CLI
|
||||||
|
|
||||||
def check_deps():
|
def check_deps():
|
||||||
if subprocess.call(['which', 'faketime', 'tar', ZCASH_CLI], stdout=subprocess.PIPE):
|
if subprocess.call(['which', 'find', 'xz', ZCASH_CLI], stdout=subprocess.PIPE):
|
||||||
print USAGE
|
print USAGE
|
||||||
sys.exit()
|
sys.exit()
|
||||||
|
|
||||||
|
@ -132,6 +136,24 @@ def compress_script(script):
|
||||||
result.extend(script)
|
result.extend(script)
|
||||||
return bytes(result)
|
return bytes(result)
|
||||||
|
|
||||||
|
def deterministic_filter(tarinfo):
|
||||||
|
tarinfo.uid = tarinfo.gid = 0
|
||||||
|
tarinfo.uname = tarinfo.gname = "root"
|
||||||
|
tarinfo.mtime = calendar.timegm(time.strptime('2017-05-17', '%Y-%m-%d'))
|
||||||
|
tarinfo.mode |= stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP
|
||||||
|
tarinfo.mode &= ~stat.S_IWGRP
|
||||||
|
if tarinfo.isdir():
|
||||||
|
tarinfo.mode |= \
|
||||||
|
stat.S_IXUSR | \
|
||||||
|
stat.S_IXGRP | \
|
||||||
|
stat.S_IXOTH
|
||||||
|
else:
|
||||||
|
tarinfo.mode &= \
|
||||||
|
~stat.S_IXUSR & \
|
||||||
|
~stat.S_IXGRP & \
|
||||||
|
~stat.S_IXOTH
|
||||||
|
return tarinfo
|
||||||
|
|
||||||
def create_benchmark_archive(blk_hash):
|
def create_benchmark_archive(blk_hash):
|
||||||
blk = json.loads(subprocess.check_output([ZCASH_CLI, 'getblock', blk_hash]))
|
blk = json.loads(subprocess.check_output([ZCASH_CLI, 'getblock', blk_hash]))
|
||||||
print 'Height: %d' % blk['height']
|
print 'Height: %d' % blk['height']
|
||||||
|
@ -226,9 +248,14 @@ def create_benchmark_archive(blk_hash):
|
||||||
|
|
||||||
# Make reproducible archive
|
# Make reproducible archive
|
||||||
os.remove('%s/LOG' % db_path)
|
os.remove('%s/LOG' % db_path)
|
||||||
archive_name = 'block-%d.tar.gz' % blk['height']
|
files = subprocess.check_output(['find', 'benchmark']).strip().split('\n')
|
||||||
subprocess.check_call(['faketime', '2017-05-17T00:00:00Z', 'tar', 'czf', archive_name, '--mtime=2017-05-17T00:00:00Z', 'benchmark'])
|
archive_name = 'block-%d.tar' % blk['height']
|
||||||
print 'Created archive %s' % archive_name
|
tar = tarfile.open(archive_name, 'w')
|
||||||
|
for name in sorted(files):
|
||||||
|
tar.add(name, recursive=False, filter=deterministic_filter)
|
||||||
|
tar.close()
|
||||||
|
subprocess.check_call(['xz', '-6', archive_name])
|
||||||
|
print 'Created archive %s.xz' % archive_name
|
||||||
subprocess.call(['rm', '-r', 'benchmark'])
|
subprocess.call(['rm', '-r', 'benchmark'])
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
|
|
|
@ -66,14 +66,14 @@ function zcashd_valgrind_stop {
|
||||||
}
|
}
|
||||||
|
|
||||||
function extract_benchmark_data {
|
function extract_benchmark_data {
|
||||||
if [ -f "block-107134.tar.gz" ]; then
|
if [ -f "block-107134.tar.xz" ]; then
|
||||||
# Check the hash of the archive:
|
# Check the hash of the archive:
|
||||||
"$SHA256CMD" $SHA256ARGS -c <<EOF
|
"$SHA256CMD" $SHA256ARGS -c <<EOF
|
||||||
299a36b3445a9a0631eb9eb0b9e76c3e9e7493a98d6621ffd6dc362d3d86cbe8 block-107134.tar.gz
|
4bd5ad1149714394e8895fa536725ed5d6c32c99812b962bfa73f03b5ffad4bb block-107134.tar.xz
|
||||||
EOF
|
EOF
|
||||||
ARCHIVE_RESULT=$?
|
ARCHIVE_RESULT=$?
|
||||||
else
|
else
|
||||||
echo "block-107134.tar.gz not found."
|
echo "block-107134.tar.xz not found."
|
||||||
ARCHIVE_RESULT=1
|
ARCHIVE_RESULT=1
|
||||||
fi
|
fi
|
||||||
if [ $ARCHIVE_RESULT -ne 0 ]; then
|
if [ $ARCHIVE_RESULT -ne 0 ]; then
|
||||||
|
@ -84,7 +84,7 @@ EOF
|
||||||
echo "Usage details are inside the Python script."
|
echo "Usage details are inside the Python script."
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
tar xzf block-107134.tar.gz -C "$DATADIR/regtest"
|
xzcat block-107134.tar.xz | tar x -C "$DATADIR/regtest"
|
||||||
}
|
}
|
||||||
|
|
||||||
# Precomputation
|
# Precomputation
|
||||||
|
|
Loading…
Reference in New Issue