[tests] nits in dbcrash.py

This commit is contained in:
John Newbery 2017-06-29 14:52:13 +01:00
parent 2935b469ae
commit 27c63dc059
2 changed files with 65 additions and 49 deletions

View File

@ -2,21 +2,7 @@
# Copyright (c) 2017 The Bitcoin Core developers # Copyright (c) 2017 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying # Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php. # file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Test recovery from a crash during chainstate writing.""" """Test recovery from a crash during chainstate writing.
from test_framework.test_framework import BitcoinTestFramework
from test_framework.util import *
from test_framework.script import *
from test_framework.mininode import *
import random
try:
import http.client as httplib
except ImportError:
import httplib
import errno
'''
Test structure:
- 4 nodes - 4 nodes
* node0, node1, and node2 will have different dbcrash ratios, and different * node0, node1, and node2 will have different dbcrash ratios, and different
@ -37,11 +23,26 @@ Test structure:
* submit block to node * submit block to node
* if node crashed on/after submitting: * if node crashed on/after submitting:
- restart until recovery succeeds - restart until recovery succeeds
- check that utxo matches node3 using gettxoutsetinfo - check that utxo matches node3 using gettxoutsetinfo"""
'''
import errno
import http.client
import random
import sys
import time
from test_framework.mininode import *
from test_framework.script import *
from test_framework.test_framework import BitcoinTestFramework
from test_framework.util import *
HTTP_DISCONNECT_ERRORS = [http.client.CannotSendRequest]
try:
HTTP_DISCONNECT_ERRORS.append(http.client.RemoteDisconnected)
except AttributeError:
pass
class ChainstateWriteCrashTest(BitcoinTestFramework): class ChainstateWriteCrashTest(BitcoinTestFramework):
def __init__(self): def __init__(self):
super().__init__() super().__init__()
self.num_nodes = 4 self.num_nodes = 4
@ -50,32 +51,28 @@ class ChainstateWriteCrashTest(BitcoinTestFramework):
# Set -maxmempool=0 to turn off mempool memory sharing with dbcache # Set -maxmempool=0 to turn off mempool memory sharing with dbcache
# Set -rpcservertimeout=900 to reduce socket disconnects in this # Set -rpcservertimeout=900 to reduce socket disconnects in this
# long-running test # long-running test
self.base_args = ["-limitdescendantsize=0", "-maxmempool=0", "-rpcservertimeout=900"] self.base_args = ["-limitdescendantsize=0", "-maxmempool=0", "-rpcservertimeout=900", "-dbbatchsize=200000"]
# Set different crash ratios and cache sizes. Note that not all of # Set different crash ratios and cache sizes. Note that not all of
# -dbcache goes to pcoinsTip. # -dbcache goes to pcoinsTip.
self.node0_args = ["-dbcrashratio=8", "-dbcache=4", "-dbbatchsize=200000"] + self.base_args self.node0_args = ["-dbcrashratio=8", "-dbcache=4"] + self.base_args
self.node1_args = ["-dbcrashratio=16", "-dbcache=8", "-dbbatchsize=200000"] + self.base_args self.node1_args = ["-dbcrashratio=16", "-dbcache=8"] + self.base_args
self.node2_args = ["-dbcrashratio=24", "-dbcache=16", "-dbbatchsize=200000"] + self.base_args self.node2_args = ["-dbcrashratio=24", "-dbcache=16"] + self.base_args
# Node3 is a normal node with default args, except will mine full blocks # Node3 is a normal node with default args, except will mine full blocks
self.node3_args = ["-blockmaxweight=4000000"] self.node3_args = ["-blockmaxweight=4000000"]
self.extra_args = [self.node0_args, self.node1_args, self.node2_args, self.node3_args] self.extra_args = [self.node0_args, self.node1_args, self.node2_args, self.node3_args]
# We'll track some test coverage statistics
self.restart_counts = [0, 0, 0] # Track the restarts for nodes 0-2
self.crashed_on_restart = 0 # Track count of crashes during recovery
def setup_network(self): def setup_network(self):
self.setup_nodes() self.setup_nodes()
# Leave them unconnected, we'll use submitblock directly in this test # Leave them unconnected, we'll use submitblock directly in this test
# Starts up a given node id, waits for the tip to reach the given block
# hash, and calculates the utxo hash. Exceptions on startup should
# indicate node crash (due to -dbcrashratio), in which case we try again.
# Give up after 60 seconds.
# Returns the utxo hash of the given node.
def restart_node(self, node_index, expected_tip): def restart_node(self, node_index, expected_tip):
"""Start up a given node id, wait for the tip to reach the given block hash, and calculate the utxo hash.
Exceptions on startup should indicate node crash (due to -dbcrashratio), in which case we try again. Give up
after 60 seconds. Returns the utxo hash of the given node."""
time_start = time.time() time_start = time.time()
while time.time() - time_start < 60: while time.time() - time_start < 60:
try: try:
@ -99,14 +96,23 @@ class ChainstateWriteCrashTest(BitcoinTestFramework):
# and make sure that recovery happens. # and make sure that recovery happens.
raise AssertionError("Unable to successfully restart node %d in allotted time", node_index) raise AssertionError("Unable to successfully restart node %d in allotted time", node_index)
# Try submitting a block to the given node.
# Catch any exceptions that indicate the node has crashed.
# Returns true if the block was submitted successfully; false otherwise.
def submit_block_catch_error(self, node_index, block): def submit_block_catch_error(self, node_index, block):
"""Try submitting a block to the given node.
Catch any exceptions that indicate the node has crashed.
Returns true if the block was submitted successfully; false otherwise."""
try: try:
self.nodes[node_index].submitblock(block) self.nodes[node_index].submitblock(block)
return True return True
except (httplib.CannotSendRequest, httplib.RemoteDisconnected) as e: except http.client.BadStatusLine as e:
# Prior to 3.5 BadStatusLine('') was raised for a remote disconnect error.
if sys.version_info[0] == 3 and sys.version_info[1] < 5 and e.line == "''":
self.log.debug("node %d submitblock raised exception: %s", node_index, e)
return False
else:
raise
except tuple(HTTP_DISCONNECT_ERRORS) as e:
self.log.debug("node %d submitblock raised exception: %s", node_index, e) self.log.debug("node %d submitblock raised exception: %s", node_index, e)
return False return False
except OSError as e: except OSError as e:
@ -118,11 +124,13 @@ class ChainstateWriteCrashTest(BitcoinTestFramework):
# Unexpected exception, raise # Unexpected exception, raise
raise raise
# Use submitblock to sync node3's chain with the other nodes
# If submitblock fails, restart the node and get the new utxo hash.
def sync_node3blocks(self, block_hashes): def sync_node3blocks(self, block_hashes):
# If any nodes crash while updating, we'll compare utxo hashes to """Use submitblock to sync node3's chain with the other nodes
# ensure recovery was successful.
If submitblock fails, restart the node and get the new utxo hash.
If any nodes crash while updating, we'll compare utxo hashes to
ensure recovery was successful."""
node3_utxo_hash = self.nodes[3].gettxoutsetinfo()['hash_serialized_2'] node3_utxo_hash = self.nodes[3].gettxoutsetinfo()['hash_serialized_2']
# Retrieve all the blocks from node3 # Retrieve all the blocks from node3
@ -161,9 +169,10 @@ class ChainstateWriteCrashTest(BitcoinTestFramework):
self.log.debug("Checking txoutsetinfo matches for node %d", i) self.log.debug("Checking txoutsetinfo matches for node %d", i)
assert_equal(nodei_utxo_hash, node3_utxo_hash) assert_equal(nodei_utxo_hash, node3_utxo_hash)
# Verify that the utxo hash of each node matches node3.
# Restart any nodes that crash while querying.
def verify_utxo_hash(self): def verify_utxo_hash(self):
"""Verify that the utxo hash of each node matches node3.
Restart any nodes that crash while querying."""
node3_utxo_hash = self.nodes[3].gettxoutsetinfo()['hash_serialized_2'] node3_utxo_hash = self.nodes[3].gettxoutsetinfo()['hash_serialized_2']
self.log.info("Verifying utxo hash matches for all nodes") self.log.info("Verifying utxo hash matches for all nodes")
@ -175,7 +184,6 @@ class ChainstateWriteCrashTest(BitcoinTestFramework):
nodei_utxo_hash = self.restart_node(i, self.nodes[3].getbestblockhash()) nodei_utxo_hash = self.restart_node(i, self.nodes[3].getbestblockhash())
assert_equal(nodei_utxo_hash, node3_utxo_hash) assert_equal(nodei_utxo_hash, node3_utxo_hash)
def generate_small_transactions(self, node, count, utxo_list): def generate_small_transactions(self, node, count, utxo_list):
FEE = 1000 # TODO: replace this with node relay fee based calculation FEE = 1000 # TODO: replace this with node relay fee based calculation
num_transactions = 0 num_transactions = 0
@ -202,6 +210,9 @@ class ChainstateWriteCrashTest(BitcoinTestFramework):
num_transactions += 1 num_transactions += 1
def run_test(self): def run_test(self):
# Track test coverage statistics
self.restart_counts = [0, 0, 0] # Track the restarts for nodes 0-2
self.crashed_on_restart = 0 # Track count of crashes during recovery
# Start by creating a lot of utxos on node3 # Start by creating a lot of utxos on node3
initial_height = self.nodes[3].getblockcount() initial_height = self.nodes[3].getblockcount()
@ -239,7 +250,9 @@ class ChainstateWriteCrashTest(BitcoinTestFramework):
# Now generate new blocks until we pass the old tip height # Now generate new blocks until we pass the old tip height
self.log.debug("Mining longer tip") self.log.debug("Mining longer tip")
block_hashes = self.nodes[3].generate(current_height+1-self.nodes[3].getblockcount()) block_hashes = []
while current_height + 1 > self.nodes[3].getblockcount():
block_hashes.extend(self.nodes[3].generate(min(10, current_height + 1 - self.nodes[3].getblockcount())))
self.log.debug("Syncing %d new blocks...", len(block_hashes)) self.log.debug("Syncing %d new blocks...", len(block_hashes))
self.sync_node3blocks(block_hashes) self.sync_node3blocks(block_hashes)
utxo_list = self.nodes[3].listunspent() utxo_list = self.nodes[3].listunspent()

View File

@ -412,7 +412,10 @@ def random_transaction(nodes, amount, min_fee, fee_increment, fee_variants):
# Helper to create at least "count" utxos # Helper to create at least "count" utxos
# Pass in a fee that is sufficient for relay and mining new transactions. # Pass in a fee that is sufficient for relay and mining new transactions.
def create_confirmed_utxos(fee, node, count): def create_confirmed_utxos(fee, node, count):
node.generate(int(0.5 * count) + 101) to_generate = int(0.5 * count) + 101
while to_generate > 0:
node.generate(min(25, to_generate))
to_generate -= 25
utxos = node.listunspent() utxos = node.listunspent()
iterations = count - len(utxos) iterations = count - len(utxos)
addr1 = node.getnewaddress() addr1 = node.getnewaddress()