separate blockchain verifier from transaction verifier

This commit is contained in:
ThomasV 2013-09-01 18:16:15 +02:00
parent d99a381d83
commit d47892b690
6 changed files with 365 additions and 283 deletions

View File

@ -133,10 +133,14 @@ if __name__ == '__main__':
interface.start(wait = False)
interface.send([('server.peers.subscribe',[])])
gui = gui.ElectrumGui(config,interface)
blockchain = BlockchainVerifier(interface, config)
blockchain.start()
gui = gui.ElectrumGui(config, interface, blockchain)
gui.main(url)
interface.stop()
blockchain.stop()
# we use daemon threads, their termination is enforced.
# this sleep command gives them time to terminate cleanly.

View File

@ -42,7 +42,7 @@ except:
from electrum.wallet import format_satoshis
from electrum.bitcoin import Transaction, is_valid
from electrum import mnemonic
from electrum import util, bitcoin, commands, Interface, Wallet, WalletVerifier, WalletSynchronizer
from electrum import util, bitcoin, commands, Interface, Wallet, TxVerifier, WalletSynchronizer
from electrum import SimpleConfig, Wallet, WalletSynchronizer, WalletStorage
@ -307,7 +307,7 @@ class ElectrumWindow(QMainWindow):
self.wallet.interface.register_callback('disconnected', lambda: self.emit(QtCore.SIGNAL('update_status')))
self.wallet.interface.register_callback('disconnecting', lambda: self.emit(QtCore.SIGNAL('update_status')))
self.wallet.interface.register_callback('new_transaction', lambda: self.emit(QtCore.SIGNAL('transaction_signal')))
title = 'Electrum ' + self.wallet.electrum_version + ' - ' #+ self.config.path
title = 'Electrum ' + self.wallet.electrum_version + ' - ' + self.wallet.storage.path
if not self.wallet.seed: title += ' [%s]' % (_('seedless'))
self.setWindowTitle( title )
self.update_wallet()
@ -350,13 +350,19 @@ class ElectrumWindow(QMainWindow):
return
interface = self.wallet.interface
verifier = self.wallet.verifier
blockchain = self.wallet.verifier.blockchain
self.wallet.verifier.stop()
self.wallet.synchronizer.stop()
# create wallet
wallet = Wallet(storage)
wallet.interface = interface
wallet.verifier = verifier
verifier = TxVerifier(interface, blockchain, storage)
verifier.start()
wallet.set_verifier(verifier)
synchronizer = WalletSynchronizer(wallet)
synchronizer.start()
@ -2208,9 +2214,10 @@ class OpenFileEventFilter(QObject):
class ElectrumGui:
def __init__(self, config, interface, app=None):
def __init__(self, config, interface, blockchain, app=None):
self.interface = interface
self.config = config
self.blockchain = blockchain
self.windows = []
self.efilter = OpenFileEventFilter(self.windows)
if app is None:
@ -2232,9 +2239,10 @@ class ElectrumGui:
wallet.interface = self.interface
verifier = WalletVerifier(self.interface, storage)
verifier = TxVerifier(self.interface, self.blockchain, storage)
verifier.start()
wallet.set_verifier(verifier)
synchronizer = WalletSynchronizer(wallet)
synchronizer.start()

View File

@ -2,7 +2,8 @@ from version import ELECTRUM_VERSION
from util import format_satoshis, print_msg, print_json, print_error, set_verbosity
from wallet import WalletSynchronizer, WalletStorage
from wallet_factory import WalletFactory as Wallet
from verifier import WalletVerifier
from verifier import TxVerifier
from blockchain import BlockchainVerifier
from interface import Interface, pick_random_server, DEFAULT_SERVERS
from simple_config import SimpleConfig
import bitcoin

330
lib/blockchain.py Normal file
View File

@ -0,0 +1,330 @@
#!/usr/bin/env python
#
# Electrum - lightweight Bitcoin client
# Copyright (C) 2012 thomasv@ecdsa.org
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import threading, time, Queue, os, sys, shutil
from util import user_dir, appdata_dir, print_error
from bitcoin import *
class BlockchainVerifier(threading.Thread):
""" Simple Payment Verification """
def __init__(self, interface, config):
threading.Thread.__init__(self)
self.daemon = True
self.config = config
self.interface = interface
self.interface.register_channel('verifier')
self.lock = threading.Lock()
self.pending_headers = [] # headers that have not been verified
self.height = 0
self.local_height = 0
self.running = False
self.headers_url = 'http://headers.electrum.org/blockchain_headers'
def stop(self):
with self.lock: self.running = False
self.interface.poke('verifier')
def is_running(self):
with self.lock: return self.running
def run(self):
self.init_headers_file()
self.set_local_height()
with self.lock:
self.running = True
requested_merkle = []
requested_chunks = []
requested_headers = []
all_chunks = False
# subscribe to block headers
self.interface.send([ ('blockchain.headers.subscribe',[])], 'verifier')
while self.is_running():
# request missing chunks
if not all_chunks and self.height and not requested_chunks:
if self.local_height + 50 < self.height:
min_index = (self.local_height + 1)/2016
max_index = (self.height + 1)/2016
for i in range(min_index, max_index + 1):
print_error( "requesting chunk", i )
self.interface.send([ ('blockchain.block.get_chunk',[i])], 'verifier')
requested_chunks.append(i)
break
else:
all_chunks = True
print_error("downloaded all chunks")
# process pending headers
if self.pending_headers and all_chunks:
done = []
for header in self.pending_headers:
if self.verify_header(header):
done.append(header)
else:
# request previous header
i = header.get('block_height') - 1
if i not in requested_headers:
print_error("requesting header %d"%i)
self.interface.send([ ('blockchain.block.get_header',[i])], 'verifier')
requested_headers.append(i)
# no point continuing
break
if done:
self.interface.trigger_callback('updated')
for header in done:
self.pending_headers.remove(header)
try:
r = self.interface.get_response('verifier',timeout=1)
except Queue.Empty:
continue
if not r: continue
if r.get('error'):
print_error('Verifier received an error:', r)
continue
# 3. handle response
method = r['method']
params = r['params']
result = r['result']
if method == 'blockchain.block.get_chunk':
index = params[0]
self.verify_chunk(index, result)
requested_chunks.remove(index)
elif method in ['blockchain.headers.subscribe', 'blockchain.block.get_header']:
self.pending_headers.append(result)
if method == 'blockchain.block.get_header':
requested_headers.remove(result.get('block_height'))
else:
self.height = result.get('block_height')
self.interface.poke('synchronizer')
self.pending_headers.sort(key=lambda x: x.get('block_height'))
# print "pending headers", map(lambda x: x.get('block_height'), self.pending_headers)
def verify_chunk(self, index, hexdata):
data = hexdata.decode('hex')
height = index*2016
num = len(data)/80
print_error("validating headers %d"%height)
if index == 0:
previous_hash = ("0"*64)
else:
prev_header = self.read_header(index*2016-1)
if prev_header is None: raise
previous_hash = self.hash_header(prev_header)
bits, target = self.get_target(index)
for i in range(num):
height = index*2016 + i
raw_header = data[i*80:(i+1)*80]
header = self.header_from_string(raw_header)
_hash = self.hash_header(header)
assert previous_hash == header.get('prev_block_hash')
assert bits == header.get('bits')
assert eval('0x'+_hash) < target
previous_header = header
previous_hash = _hash
self.save_chunk(index, data)
def verify_header(self, header):
# add header to the blockchain file
# if there is a reorg, push it in a stack
height = header.get('block_height')
prev_header = self.read_header(height -1)
if not prev_header:
# return False to request previous header
return False
prev_hash = self.hash_header(prev_header)
bits, target = self.get_target(height/2016)
_hash = self.hash_header(header)
try:
assert prev_hash == header.get('prev_block_hash')
assert bits == header.get('bits')
assert eval('0x'+_hash) < target
except:
# this can be caused by a reorg.
print_error("verify header failed"+ repr(header))
# undo verifications
with self.lock:
items = self.verified_tx.items()[:]
for tx_hash, item in items:
tx_height, timestamp, pos = item
if tx_height >= height:
print_error("redoing", tx_hash)
with self.lock:
self.verified_tx.pop(tx_hash)
if tx_hash in self.merkle_roots:
self.merkle_roots.pop(tx_hash)
# return False to request previous header.
return False
self.save_header(header)
print_error("verify header:", _hash, height)
return True
def header_to_string(self, res):
s = int_to_hex(res.get('version'),4) \
+ rev_hex(res.get('prev_block_hash')) \
+ rev_hex(res.get('merkle_root')) \
+ int_to_hex(int(res.get('timestamp')),4) \
+ int_to_hex(int(res.get('bits')),4) \
+ int_to_hex(int(res.get('nonce')),4)
return s
def header_from_string(self, s):
hex_to_int = lambda s: eval('0x' + s[::-1].encode('hex'))
h = {}
h['version'] = hex_to_int(s[0:4])
h['prev_block_hash'] = hash_encode(s[4:36])
h['merkle_root'] = hash_encode(s[36:68])
h['timestamp'] = hex_to_int(s[68:72])
h['bits'] = hex_to_int(s[72:76])
h['nonce'] = hex_to_int(s[76:80])
return h
def hash_header(self, header):
return rev_hex(Hash(self.header_to_string(header).decode('hex')).encode('hex'))
def path(self):
wdir = self.config.get('blockchain_headers_path', user_dir())
if wdir and not os.path.exists( wdir ): os.mkdir(wdir)
return os.path.join( wdir, 'blockchain_headers')
def init_headers_file(self):
filename = self.path()
if os.path.exists(filename):
return
try:
import urllib, socket
socket.setdefaulttimeout(30)
print_error("downloading ", self.headers_url )
urllib.urlretrieve(self.headers_url, filename)
except:
print_error( "download failed. creating file", filename )
open(filename,'wb+').close()
def save_chunk(self, index, chunk):
filename = self.path()
f = open(filename,'rb+')
f.seek(index*2016*80)
h = f.write(chunk)
f.close()
self.set_local_height()
def save_header(self, header):
data = self.header_to_string(header).decode('hex')
assert len(data) == 80
height = header.get('block_height')
filename = self.path()
f = open(filename,'rb+')
f.seek(height*80)
h = f.write(data)
f.close()
self.set_local_height()
def set_local_height(self):
name = self.path()
if os.path.exists(name):
h = os.path.getsize(name)/80 - 1
if self.local_height != h:
self.local_height = h
def read_header(self, block_height):
name = self.path()
if os.path.exists(name):
f = open(name,'rb')
f.seek(block_height*80)
h = f.read(80)
f.close()
if len(h) == 80:
h = self.header_from_string(h)
return h
def get_target(self, index):
max_target = 0x00000000FFFF0000000000000000000000000000000000000000000000000000
if index == 0: return 0x1d00ffff, max_target
first = self.read_header((index-1)*2016)
last = self.read_header(index*2016-1)
nActualTimespan = last.get('timestamp') - first.get('timestamp')
nTargetTimespan = 14*24*60*60
nActualTimespan = max(nActualTimespan, nTargetTimespan/4)
nActualTimespan = min(nActualTimespan, nTargetTimespan*4)
bits = last.get('bits')
# convert to bignum
MM = 256*256*256
a = bits%MM
if a < 0x8000:
a *= 256
target = (a) * pow(2, 8 * (bits/MM - 3))
# new target
new_target = min( max_target, (target * nActualTimespan)/nTargetTimespan )
# convert it to bits
c = ("%064X"%new_target)[2:]
i = 31
while c[0:2]=="00":
c = c[2:]
i -= 1
c = eval('0x'+c[0:6])
if c > 0x800000:
c /= 256
i += 1
new_bits = c + MM * i
return new_bits, new_target

View File

@ -24,34 +24,29 @@ from bitcoin import *
class WalletVerifier(threading.Thread):
class TxVerifier(threading.Thread):
""" Simple Payment Verification """
def __init__(self, interface, config):
def __init__(self, interface, blockchain, storage):
threading.Thread.__init__(self)
self.daemon = True
self.config = config
self.storage = storage
self.blockchain = blockchain
self.interface = interface
self.transactions = {} # requested verifications (with height sent by the requestor)
self.interface.register_channel('verifier')
self.verified_tx = config.get('verified_tx3',{}) # height, timestamp of verified transactions
self.merkle_roots = config.get('merkle_roots',{}) # hashed by me
self.targets = config.get('targets',{}) # compute targets
self.interface.register_channel('txverifier')
self.verified_tx = storage.get('verified_tx3',{}) # height, timestamp of verified transactions
self.merkle_roots = storage.get('merkle_roots',{}) # hashed by me
self.lock = threading.Lock()
self.pending_headers = [] # headers that have not been verified
self.height = 0
self.local_height = 0
self.running = False
self.headers_url = 'http://headers.electrum.org/blockchain_headers'
def get_confirmations(self, tx):
""" return the number of confirmations of a monitored transaction. """
with self.lock:
if tx in self.verified_tx:
height, timestamp, pos = self.verified_tx[tx]
conf = (self.local_height - height + 1)
conf = (self.blockchain.local_height - height + 1)
if conf <= 0: timestamp = None
elif tx in self.transactions:
@ -102,66 +97,18 @@ class WalletVerifier(threading.Thread):
def run(self):
self.init_headers_file()
self.set_local_height()
with self.lock:
self.running = True
requested_merkle = []
requested_chunks = []
requested_headers = []
all_chunks = False
# subscribe to block headers
self.interface.send([ ('blockchain.headers.subscribe',[])], 'verifier')
while self.is_running():
# request missing chunks
if not all_chunks and self.height and not requested_chunks:
if self.local_height + 50 < self.height:
min_index = (self.local_height + 1)/2016
max_index = (self.height + 1)/2016
for i in range(min_index, max_index + 1):
print_error( "requesting chunk", i )
self.interface.send([ ('blockchain.block.get_chunk',[i])], 'verifier')
requested_chunks.append(i)
break
else:
all_chunks = True
print_error("downloaded all chunks")
# request missing tx
if all_chunks:
for tx_hash, tx_height in self.transactions.items():
if tx_hash not in self.verified_tx:
if self.merkle_roots.get(tx_hash) is None and tx_hash not in requested_merkle:
print_error('requesting merkle', tx_hash)
self.interface.send([ ('blockchain.transaction.get_merkle',[tx_hash, tx_height]) ], 'verifier')
self.interface.send([ ('blockchain.transaction.get_merkle',[tx_hash, tx_height]) ], 'txverifier')
requested_merkle.append(tx_hash)
# process pending headers
if self.pending_headers and all_chunks:
done = []
for header in self.pending_headers:
if self.verify_header(header):
done.append(header)
else:
# request previous header
i = header.get('block_height') - 1
if i not in requested_headers:
print_error("requesting header %d"%i)
self.interface.send([ ('blockchain.block.get_header',[i])], 'verifier')
requested_headers.append(i)
# no point continuing
break
if done:
self.interface.trigger_callback('updated')
for header in done:
self.pending_headers.remove(header)
try:
r = self.interface.get_response('verifier',timeout=1)
r = self.interface.get_response('txverifier',timeout=1)
except Queue.Empty:
continue
if not r: continue
@ -180,30 +127,12 @@ class WalletVerifier(threading.Thread):
self.verify_merkle(tx_hash, result)
requested_merkle.remove(tx_hash)
elif method == 'blockchain.block.get_chunk':
index = params[0]
self.verify_chunk(index, result)
requested_chunks.remove(index)
elif method in ['blockchain.headers.subscribe', 'blockchain.block.get_header']:
self.pending_headers.append(result)
if method == 'blockchain.block.get_header':
requested_headers.remove(result.get('block_height'))
else:
self.height = result.get('block_height')
self.interface.poke('synchronizer')
self.pending_headers.sort(key=lambda x: x.get('block_height'))
# print "pending headers", map(lambda x: x.get('block_height'), self.pending_headers)
def verify_merkle(self, tx_hash, result):
tx_height = result.get('block_height')
pos = result.get('pos')
self.merkle_roots[tx_hash] = self.hash_merkle_root(result['merkle'], tx_hash, pos)
header = self.read_header(tx_height)
header = self.blockchain.read_header(tx_height)
if not header: return
assert header.get('merkle_root') == self.merkle_roots[tx_hash]
# we passed all the tests
@ -212,106 +141,10 @@ class WalletVerifier(threading.Thread):
with self.lock:
self.verified_tx[tx_hash] = (tx_height, timestamp, pos)
print_error("verified %s"%tx_hash)
self.config.set_key('verified_tx3', self.verified_tx, True)
self.storage.set_key('verified_tx3', self.verified_tx, True)
self.interface.trigger_callback('updated')
def verify_chunk(self, index, hexdata):
data = hexdata.decode('hex')
height = index*2016
num = len(data)/80
print_error("validating headers %d"%height)
if index == 0:
previous_hash = ("0"*64)
else:
prev_header = self.read_header(index*2016-1)
if prev_header is None: raise
previous_hash = self.hash_header(prev_header)
bits, target = self.get_target(index)
for i in range(num):
height = index*2016 + i
raw_header = data[i*80:(i+1)*80]
header = self.header_from_string(raw_header)
_hash = self.hash_header(header)
assert previous_hash == header.get('prev_block_hash')
assert bits == header.get('bits')
assert eval('0x'+_hash) < target
previous_header = header
previous_hash = _hash
self.save_chunk(index, data)
def verify_header(self, header):
# add header to the blockchain file
# if there is a reorg, push it in a stack
height = header.get('block_height')
prev_header = self.read_header(height -1)
if not prev_header:
# return False to request previous header
return False
prev_hash = self.hash_header(prev_header)
bits, target = self.get_target(height/2016)
_hash = self.hash_header(header)
try:
assert prev_hash == header.get('prev_block_hash')
assert bits == header.get('bits')
assert eval('0x'+_hash) < target
except:
# this can be caused by a reorg.
print_error("verify header failed"+ repr(header))
# undo verifications
with self.lock:
items = self.verified_tx.items()[:]
for tx_hash, item in items:
tx_height, timestamp, pos = item
if tx_height >= height:
print_error("redoing", tx_hash)
with self.lock:
self.verified_tx.pop(tx_hash)
if tx_hash in self.merkle_roots:
self.merkle_roots.pop(tx_hash)
# return False to request previous header.
return False
self.save_header(header)
print_error("verify header:", _hash, height)
return True
def header_to_string(self, res):
s = int_to_hex(res.get('version'),4) \
+ rev_hex(res.get('prev_block_hash')) \
+ rev_hex(res.get('merkle_root')) \
+ int_to_hex(int(res.get('timestamp')),4) \
+ int_to_hex(int(res.get('bits')),4) \
+ int_to_hex(int(res.get('nonce')),4)
return s
def header_from_string(self, s):
hex_to_int = lambda s: eval('0x' + s[::-1].encode('hex'))
h = {}
h['version'] = hex_to_int(s[0:4])
h['prev_block_hash'] = hash_encode(s[4:36])
h['merkle_root'] = hash_encode(s[36:68])
h['timestamp'] = hex_to_int(s[68:72])
h['bits'] = hex_to_int(s[72:76])
h['nonce'] = hex_to_int(s[76:80])
return h
def hash_header(self, header):
return rev_hex(Hash(self.header_to_string(header).decode('hex')).encode('hex'))
def hash_merkle_root(self, merkle_s, target_hash, pos):
h = hash_decode(target_hash)
for i in range(len(merkle_s)):
@ -319,101 +152,6 @@ class WalletVerifier(threading.Thread):
h = Hash( hash_decode(item) + h ) if ((pos >> i) & 1) else Hash( h + hash_decode(item) )
return hash_encode(h)
def path(self):
wdir = self.config.get('blockchain_headers_path', user_dir())
if wdir and not os.path.exists( wdir ): os.mkdir(wdir)
return os.path.join( wdir, 'blockchain_headers')
def init_headers_file(self):
filename = self.path()
if os.path.exists(filename):
return
try:
import urllib, socket
socket.setdefaulttimeout(30)
print_error("downloading ", self.headers_url )
urllib.urlretrieve(self.headers_url, filename)
except:
print_error( "download failed. creating file", filename )
open(filename,'wb+').close()
def save_chunk(self, index, chunk):
filename = self.path()
f = open(filename,'rb+')
f.seek(index*2016*80)
h = f.write(chunk)
f.close()
self.set_local_height()
def save_header(self, header):
data = self.header_to_string(header).decode('hex')
assert len(data) == 80
height = header.get('block_height')
filename = self.path()
f = open(filename,'rb+')
f.seek(height*80)
h = f.write(data)
f.close()
self.set_local_height()
def set_local_height(self):
name = self.path()
if os.path.exists(name):
h = os.path.getsize(name)/80 - 1
if self.local_height != h:
self.local_height = h
def read_header(self, block_height):
name = self.path()
if os.path.exists(name):
f = open(name,'rb')
f.seek(block_height*80)
h = f.read(80)
f.close()
if len(h) == 80:
h = self.header_from_string(h)
return h
def get_target(self, index):
max_target = 0x00000000FFFF0000000000000000000000000000000000000000000000000000
if index == 0: return 0x1d00ffff, max_target
first = self.read_header((index-1)*2016)
last = self.read_header(index*2016-1)
nActualTimespan = last.get('timestamp') - first.get('timestamp')
nTargetTimespan = 14*24*60*60
nActualTimespan = max(nActualTimespan, nTargetTimespan/4)
nActualTimespan = min(nActualTimespan, nTargetTimespan*4)
bits = last.get('bits')
# convert to bignum
MM = 256*256*256
a = bits%MM
if a < 0x8000:
a *= 256
target = (a) * pow(2, 8 * (bits/MM - 3))
# new target
new_target = min( max_target, (target * nActualTimespan)/nTargetTimespan )
# convert it to bits
c = ("%064X"%new_target)[2:]
i = 31
while c[0:2]=="00":
c = c[2:]
i -= 1
c = eval('0x'+c[0:6])
if c > 0x800000:
c /= 256
i += 1
new_bits = c + MM * i
return new_bits, new_target

View File

@ -58,6 +58,7 @@ setup(name = "Electrum",
'electrum.wallet_bitkey',
'electrum.wallet_factory',
'electrum.interface',
'electrum.blockchain',
'electrum.commands',
'electrum.mnemonic',
'electrum.simple_config',