Rewrite accounts and transactions: store pubkeys instead of addresses in order to avoid unnecessary derivations.

This commit is contained in:
ThomasV 2014-07-06 21:10:41 +02:00
parent cd4c8335b0
commit a89abee969
5 changed files with 175 additions and 130 deletions

View File

@ -2063,7 +2063,7 @@ class ElectrumWindow(QMainWindow):
if is_hex:
try:
return Transaction(txt)
return Transaction.deserialize(txt)
except:
traceback.print_exc(file=sys.stdout)
QMessageBox.critical(None, _("Unable to parse transaction"), _("Electrum was unable to parse your transaction"))
@ -2072,7 +2072,7 @@ class ElectrumWindow(QMainWindow):
try:
tx_dict = json.loads(str(txt))
assert "hex" in tx_dict.keys()
tx = Transaction(tx_dict["hex"])
tx = Transaction.deserialize(tx_dict["hex"])
#if tx_dict.has_key("input_info"):
# input_info = json.loads(tx_dict['input_info'])
# tx.add_input_info(input_info)
@ -2123,7 +2123,7 @@ class ElectrumWindow(QMainWindow):
if ok and txid:
r = self.network.synchronous_get([ ('blockchain.transaction.get',[str(txid)]) ])[0]
if r:
tx = transaction.Transaction(r)
tx = transaction.Transaction.deserialize(r)
if tx:
self.show_transaction(tx)
else:

View File

@ -25,28 +25,46 @@ from util import print_msg
class Account(object):
def __init__(self, v):
self.addresses = v.get('0', [])
self.change = v.get('1', [])
self.receiving_pubkeys = v.get('receiving', [])
self.change_pubkeys = v.get('change', [])
# addresses will not be stored on disk
self.receiving_addresses = map(self.pubkeys_to_address, self.receiving_pubkeys)
self.change_addresses = map(self.pubkeys_to_address, self.change_pubkeys)
def dump(self):
return {'0':self.addresses, '1':self.change}
return {'receiving':self.receiving_pubkeys, 'change':self.change_pubkeys}
def get_pubkey(self, for_change, n):
pubkeys_list = self.change_pubkeys if for_change else self.receiving_pubkeys
return pubkeys_list[n]
def get_address(self, for_change, n):
addr_list = self.change_addresses if for_change else self.receiving_addresses
return addr_list[n]
def get_pubkeys(self, for_change, n):
return [ self.get_pubkey(for_change, n)]
def get_addresses(self, for_change):
return self.change[:] if for_change else self.addresses[:]
addr_list = self.change_addresses if for_change else self.receiving_addresses
return addr_list[:]
def derive_pubkeys(self, for_change, n):
pass
def create_new_address(self, for_change):
addresses = self.change if for_change else self.addresses
n = len(addresses)
address = self.get_address( for_change, n)
addresses.append(address)
pubkeys_list = self.change_pubkeys if for_change else self.receiving_pubkeys
addr_list = self.change_addresses if for_change else self.receiving_addresses
n = len(pubkeys_list)
pubkeys = self.derive_pubkeys(for_change, n)
address = self.pubkeys_to_address(pubkeys)
pubkeys_list.append(pubkeys)
addr_list.append(address)
print_msg(address)
return address
def get_address(self, for_change, n):
pass
def get_pubkeys(self, sequence):
return [ self.get_pubkey( *sequence )]
def pubkeys_to_address(self, pubkey):
return public_key_to_bc_address(pubkey.decode('hex'))
def has_change(self):
return True
@ -63,14 +81,19 @@ class Account(object):
class PendingAccount(Account):
def __init__(self, v):
self.addresses = [ v['pending'] ]
self.change = []
try:
self.pending_pubkey = v['pending_pubkey']
except:
pass
def get_addresses(self, is_change):
return []
def has_change(self):
return False
def dump(self):
return {'pending':self.addresses[0]}
return {} #{'pending_pubkey':self.pending_pubkey }
def get_name(self, k):
return _('Pending account')
@ -91,8 +114,8 @@ class ImportedAccount(Account):
addr = self.get_addresses(0)[i]
return self.keypairs[addr][0]
def get_xpubkeys(self, *sequence):
return self.get_pubkeys(*sequence)
def get_xpubkeys(self, for_change, n):
return self.get_pubkeys(for_change, n)
def get_private_key(self, sequence, wallet, password):
from wallet import pw_decode
@ -133,12 +156,9 @@ class OldAccount(Account):
""" Privatekey(type,n) = Master_private_key + H(n|S|type) """
def __init__(self, v):
self.addresses = v.get(0, [])
self.change = v.get(1, [])
Account.__init__(self, v)
self.mpk = v['mpk'].decode('hex')
def dump(self):
return {0:self.addresses, 1:self.change}
@classmethod
def mpk_from_seed(klass, seed):
@ -173,7 +193,7 @@ class OldAccount(Account):
public_key2 = ecdsa.VerifyingKey.from_public_point( pubkey_point, curve = SECP256k1 )
return '04' + public_key2.to_string().encode('hex')
def get_pubkey(self, for_change, n):
def derive_pubkeys(self, for_change, n):
return self.get_pubkey_from_mpk(self.mpk, for_change, n)
def get_private_key_from_stretched_exponent(self, for_change, n, secexp):
@ -212,10 +232,6 @@ class OldAccount(Account):
def get_type(self):
return _('Old Electrum format')
def get_keyID(self, sequence):
a, b = sequence
return 'old(%s,%d,%d)'%(self.mpk.encode('hex'),a,b)
def get_xpubkeys(self, sequence):
s = ''.join(map(lambda x: bitcoin.int_to_hex(x,2), sequence))
mpk = self.mpk.encode('hex')
@ -248,11 +264,6 @@ class BIP32_Account(Account):
d['xpub'] = self.xpub
return d
def get_address(self, for_change, n):
pubkey = self.get_pubkey(for_change, n)
address = public_key_to_bc_address( pubkey.decode('hex') )
return address
def first_address(self):
return self.get_address(0,0)
@ -260,17 +271,20 @@ class BIP32_Account(Account):
return [self.xpub]
@classmethod
def get_pubkey_from_x(self, xpub, for_change, n):
def derive_pubkey_from_xpub(self, xpub, for_change, n):
_, _, _, c, cK = deserialize_xkey(xpub)
for i in [for_change, n]:
cK, c = CKD_pub(cK, c, i)
return cK.encode('hex')
def get_pubkeys(self, sequence):
return sorted(map(lambda x: self.get_pubkey_from_x(x, *sequence), self.get_master_pubkeys()))
def get_pubkey_from_xpub(self, xpub, for_change, n):
xpubs = self.get_master_pubkeys()
i = xpubs.index(xpub)
pubkeys = self.get_pubkeys(sequence, n)
return pubkeys[i]
def get_pubkey(self, for_change, n):
return self.get_pubkeys((for_change, n))[0]
def derive_pubkeys(self, for_change, n):
return self.derive_pubkey_from_xpub(self.xpub, for_change, n)
def get_private_key(self, sequence, wallet, password):
@ -284,27 +298,19 @@ class BIP32_Account(Account):
_, _, _, c, k = deserialize_xkey(xpriv)
pk = bip32_private_key( sequence, k, c )
out.append(pk)
return out
def redeem_script(self, sequence):
return None
def get_type(self):
return _('Standard 1 of 1')
def get_xpubkeys(self, sequence):
s = ''.join(map(lambda x: bitcoin.int_to_hex(x,2), sequence))
mpks = self.get_master_pubkeys()
out = []
for xpub in mpks:
pubkey = self.get_pubkey_from_x(xpub, *sequence)
x_pubkey = 'ff' + bitcoin.DecodeBase58Check(xpub).encode('hex') + s
out.append( (pubkey, x_pubkey ) )
# sort it, so that x_pubkeys are in the same order as pubkeys
out.sort()
return map(lambda x:x[1], out )
def get_xpubkeys(self, for_change, n):
# unsorted
s = ''.join(map(lambda x: bitcoin.int_to_hex(x,2), (for_change,n)))
xpubs = self.get_master_pubkeys()
return map(lambda xpub: 'ff' + bitcoin.DecodeBase58Check(xpub).encode('hex') + s, xpubs)
@classmethod
def parse_xpubkey(self, pubkey):
@ -347,13 +353,23 @@ class BIP32_Account_2of2(BIP32_Account):
d['xpub2'] = self.xpub2
return d
def redeem_script(self, sequence):
pubkeys = self.get_pubkeys(sequence)
return Transaction.multisig_script(pubkeys, 2)
def get_pubkeys(self, for_change, n):
return self.get_pubkey(for_change, n)
def derive_pubkeys(self, for_change, n):
return map(lambda x: self.derive_pubkey_from_xpub(x, for_change, n), self.get_master_pubkeys())
def redeem_script(self, for_change, n):
pubkeys = self.get_pubkeys(for_change, n)
return Transaction.multisig_script(sorted(pubkeys), 2)
def pubkeys_to_address(self, pubkeys):
redeem_script = Transaction.multisig_script(sorted(pubkeys), 2)
address = hash_160_to_bc_address(hash_160(redeem_script.decode('hex')), 5)
return address
def get_address(self, for_change, n):
address = hash_160_to_bc_address(hash_160(self.redeem_script((for_change, n)).decode('hex')), 5)
return address
return self.pubkeys_to_address(self.get_pubkeys(for_change, n))
def get_master_pubkeys(self):
return [self.xpub, self.xpub2]

View File

@ -183,7 +183,7 @@ class WalletSynchronizer(threading.Thread):
tx_hash = params[0]
tx_height = params[1]
assert tx_hash == bitcoin.hash_encode(bitcoin.Hash(result.decode('hex')))
tx = Transaction(result)
tx = Transaction.deserialize(result)
self.wallet.receive_tx_callback(tx_hash, tx, tx_height)
self.was_updated = True
requested_tx.remove( (tx_hash, tx_height) )

View File

@ -324,7 +324,7 @@ def parse_xpub(x_pubkey):
if x_pubkey[0:2] == 'ff':
from account import BIP32_Account
xpub, s = BIP32_Account.parse_xpubkey(x_pubkey)
pubkey = BIP32_Account.get_pubkey_from_x(xpub, s[0], s[1])
pubkey = BIP32_Account.derive_pubkey_from_xpub(xpub, s[0], s[1])
elif x_pubkey[0:2] == 'fe':
from account import OldAccount
mpk, s = OldAccount.parse_xpubkey(x_pubkey)
@ -345,6 +345,12 @@ def parse_scriptSig(d, bytes):
# payto_pubkey
match = [ opcodes.OP_PUSHDATA4 ]
if match_decoded(decoded, match):
sig = decoded[0][1].encode('hex')
d['address'] = "(pubkey)"
d['signatures'] = [sig]
d['num_sig'] = 1
d['x_pubkeys'] = ["(pubkey)"]
d['pubkeys'] = ["(pubkey)"]
return
# non-generated TxIn transactions push a signature
@ -396,7 +402,6 @@ def parse_scriptSig(d, bytes):
d['x_pubkeys'] = x_pubkeys
pubkeys = map(parse_xpub, x_pubkeys)
d['pubkeys'] = pubkeys
redeemScript = Transaction.multisig_script(pubkeys,2)
d['redeemScript'] = redeemScript
d['address'] = hash_160_to_bc_address(hash_160(redeemScript.decode('hex')), 5)
@ -411,20 +416,20 @@ def get_address_from_output_script(bytes):
# 65 BYTES:... CHECKSIG
match = [ opcodes.OP_PUSHDATA4, opcodes.OP_CHECKSIG ]
if match_decoded(decoded, match):
return True, public_key_to_bc_address(decoded[0][1])
return "pubkey:" + decoded[0][1].encode('hex')
# Pay-by-Bitcoin-address TxOuts look like:
# DUP HASH160 20 BYTES:... EQUALVERIFY CHECKSIG
match = [ opcodes.OP_DUP, opcodes.OP_HASH160, opcodes.OP_PUSHDATA4, opcodes.OP_EQUALVERIFY, opcodes.OP_CHECKSIG ]
if match_decoded(decoded, match):
return False, hash_160_to_bc_address(decoded[2][1])
return hash_160_to_bc_address(decoded[2][1])
# p2sh
match = [ opcodes.OP_HASH160, opcodes.OP_PUSHDATA4, opcodes.OP_EQUAL ]
if match_decoded(decoded, match):
return False, hash_160_to_bc_address(decoded[1][1],5)
return hash_160_to_bc_address(decoded[1][1],5)
return False, "(None)"
return "(None)"
@ -432,24 +437,17 @@ push_script = lambda x: op_push(len(x)/2) + x
class Transaction:
def __init__(self, raw):
self.raw = raw
self.deserialize()
self.inputs = self.d['inputs']
self.outputs = self.d['outputs']
self.outputs = map(lambda x: (x['address'],x['value']), self.outputs)
self.locktime = self.d['lockTime']
def __str__(self):
if self.raw is None:
self.raw = self.serialize(self.inputs, self.outputs, for_sig = None) # for_sig=-1 means do not sign
return self.raw
@classmethod
def from_io(klass, inputs, outputs):
raw = klass.serialize(inputs, outputs, for_sig = None) # for_sig=-1 means do not sign
self = klass(raw)
def __init__(self, inputs, outputs):
self.inputs = inputs
self.outputs = outputs
return self
self.locktime = 0
self.raw = None
@classmethod
def sweep(klass, privkeys, network, to_address, fee):
@ -472,7 +470,7 @@ class Transaction:
total = sum( map(lambda x:int(x.get('value')), inputs) ) - fee
outputs = [(to_address, total)]
self = klass.from_io(inputs, outputs)
self = klass(inputs, outputs)
self.sign({ pubkey:privkey })
return self
@ -524,7 +522,7 @@ class Transaction:
@classmethod
def serialize( klass, inputs, outputs, for_sig = None ):
def serialize(klass, inputs, outputs, for_sig = None ):
s = int_to_hex(1,4) # version
s += var_int( len(inputs) ) # number of inputs
@ -680,25 +678,32 @@ class Transaction:
def deserialize(self):
@classmethod
def deserialize(klass, raw):
vds = BCDataStream()
vds.write(self.raw.decode('hex'))
vds.write(raw.decode('hex'))
d = {}
start = vds.read_cursor
d['version'] = vds.read_int32()
n_vin = vds.read_compact_size()
d['inputs'] = []
for i in xrange(n_vin):
d['inputs'].append(self.parse_input(vds))
d['inputs'].append(klass.parse_input(vds))
n_vout = vds.read_compact_size()
d['outputs'] = []
for i in xrange(n_vout):
d['outputs'].append(self.parse_output(vds, i))
d['outputs'].append(klass.parse_output(vds, i))
d['lockTime'] = vds.read_uint32()
self.d = d
return self.d
inputs = d['inputs']
outputs = map(lambda x: (x['address'], x['value']), d['outputs'])
self = klass(inputs, outputs)
self.raw = raw
self.locktime = d['lockTime']
return self
@classmethod
def parse_input(self, vds):
d = {}
prevout_hash = hash_encode(vds.read_bytes(32))
@ -713,7 +718,6 @@ class Transaction:
d['prevout_hash'] = prevout_hash
d['prevout_n'] = prevout_n
d['sequence'] = sequence
d['pubkeys'] = []
d['signatures'] = {}
d['address'] = None
@ -721,39 +725,54 @@ class Transaction:
parse_scriptSig(d, scriptSig)
return d
@classmethod
def parse_output(self, vds, i):
d = {}
d['value'] = vds.read_int64()
scriptPubKey = vds.read_bytes(vds.read_compact_size())
is_pubkey, address = get_address_from_output_script(scriptPubKey)
d['is_pubkey'] = is_pubkey
address = get_address_from_output_script(scriptPubKey)
d['address'] = address
d['scriptPubKey'] = scriptPubKey.encode('hex')
d['prevout_n'] = i
return d
def add_extra_addresses(self, txlist):
def add_pubkey_addresses(self, txlist):
for i in self.inputs:
if i.get("address") == "(pubkey)":
prev_tx = txlist.get(i.get('prevout_hash'))
if prev_tx:
address, value = prev_tx.outputs[i.get('prevout_n')]
address, value = prev_tx.get_outputs()[i.get('prevout_n')]
print_error("found pay-to-pubkey address:", address)
i["address"] = address
def get_outputs(self):
"""convert pubkeys to addresses"""
o = []
for x, v in self.outputs:
if bitcoin.is_address(x):
addr = x
elif x.startswith('pubkey:'):
addr = public_key_to_bc_address(x[7:].decode('hex'))
else:
addr = "(None)"
o.append((addr,v))
return o
def get_output_addresses(self):
return map(lambda x:x[0], self.get_outputs())
def has_address(self, addr):
found = False
for txin in self.inputs:
if addr == txin.get('address'):
found = True
break
for txout in self.outputs:
if addr == txout[0]:
found = True
break
if addr in self.get_output_addresses():
found = True
return found
@ -781,8 +800,7 @@ class Transaction:
if not is_send: is_partial = False
for item in self.outputs:
addr, value = item
for addr, value in self.get_outputs():
v_out += value
if addr in addresses:
v_out_mine += value

View File

@ -164,14 +164,14 @@ class Abstract_Wallet:
self.transactions = {}
tx_list = self.storage.get('transactions',{})
for k,v in tx_list.items():
for k, raw in tx_list.items():
try:
tx = Transaction(v)
tx = Transaction.deserialize(raw)
except Exception:
print_msg("Warning: Cannot deserialize transactions. skipping")
continue
self.add_extra_addresses(tx)
self.add_pubkey_addresses(tx)
self.transactions[k] = tx
for h,tx in self.transactions.items():
@ -180,6 +180,7 @@ class Abstract_Wallet:
self.transactions.pop(h)
# not saved
self.prevout_values = {} # my own transaction outputs
self.spent_outputs = []
@ -198,14 +199,18 @@ class Abstract_Wallet:
for tx_hash, tx in self.transactions.items():
self.update_tx_outputs(tx_hash)
def add_extra_addresses(self, tx):
h = tx.hash()
def add_pubkey_addresses(self, tx):
# find the address corresponding to pay-to-pubkey inputs
tx.add_extra_addresses(self.transactions)
for o in tx.d.get('outputs'):
if o.get('is_pubkey'):
h = tx.hash()
# inputs
tx.add_pubkey_addresses(self.transactions)
# outputs of tx: inputs of tx2
for x, v in tx.outputs:
if x.startswith('pubkey:'):
for tx2 in self.transactions.values():
tx2.add_extra_addresses({h:tx})
tx2.add_pubkey_addresses({h:tx})
def get_action(self):
pass
@ -381,6 +386,7 @@ class Abstract_Wallet:
mpk = [ self.master_public_keys[k] for k in self.master_private_keys.keys() ]
for xpub, sequence in xpub_list:
if xpub in mpk:
print "can sign", xpub
return True
return False
@ -463,7 +469,7 @@ class Abstract_Wallet:
for tx_hash, tx in self.transactions.items():
is_relevant, is_send, _, _ = self.get_tx_value(tx)
if is_send:
for addr, v in tx.outputs:
for addr in tx.get_output_addresses():
if not self.is_mine(addr) and addr not in self.addressbook:
self.addressbook.append(addr)
# redo labels
@ -472,7 +478,7 @@ class Abstract_Wallet:
def get_num_tx(self, address):
n = 0
for tx in self.transactions.values():
if address in map(lambda x:x[0], tx.outputs): n += 1
if address in tx.get_output_addresses(): n += 1
return n
def get_address_flags(self, addr):
@ -487,7 +493,7 @@ class Abstract_Wallet:
def update_tx_outputs(self, tx_hash):
tx = self.transactions.get(tx_hash)
for i, (addr, value) in enumerate(tx.outputs):
for i, (addr, value) in enumerate(tx.get_outputs()):
key = tx_hash+ ':%d'%i
self.prevout_values[key] = value
@ -507,7 +513,7 @@ class Abstract_Wallet:
tx = self.transactions.get(tx_hash)
if not tx: continue
for i, (addr, value) in enumerate(tx.outputs):
for i, (addr, value) in enumerate(tx.get_outputs()):
if addr == address:
key = tx_hash + ':%d'%i
received_coins.append(key)
@ -525,7 +531,7 @@ class Abstract_Wallet:
if key in received_coins:
v -= value
for i, (addr, value) in enumerate(tx.outputs):
for i, (addr, value) in enumerate(tx.get_outputs()):
key = tx_hash + ':%d'%i
if addr == address:
v += value
@ -579,10 +585,10 @@ class Abstract_Wallet:
tx = self.transactions.get(tx_hash)
if tx is None: raise Exception("Wallet not synchronized")
is_coinbase = tx.inputs[0].get('prevout_hash') == '0'*64
for o in tx.d.get('outputs'):
output = o.copy()
if output.get('address') != addr: continue
key = tx_hash + ":%d" % output.get('prevout_n')
for i, (address, value) in enumerate(tx.get_outputs()):
output = {'address':address, 'value':value, 'prevout_n':i}
if address != addr: continue
key = tx_hash + ":%d"%i
if key in self.spent_outputs: continue
output['prevout_hash'] = tx_hash
output['height'] = tx_height
@ -669,7 +675,7 @@ class Abstract_Wallet:
def receive_tx_callback(self, tx_hash, tx, tx_height):
with self.transaction_lock:
self.add_extra_addresses(tx)
self.add_pubkey_addresses(tx)
if not self.check_new_tx(tx_hash, tx):
# may happen due to pruning
print_error("received transaction that is no longer referenced in history", tx_hash)
@ -746,8 +752,7 @@ class Abstract_Wallet:
if tx:
is_relevant, is_mine, _, _ = self.get_tx_value(tx)
if is_mine:
for o in tx.outputs:
o_addr, _ = o
for o_addr in tx.get_output_addresses():
if not self.is_mine(o_addr):
try:
default_label = self.labels[o_addr]
@ -757,13 +762,11 @@ class Abstract_Wallet:
else:
default_label = '(internal)'
else:
for o in tx.outputs:
o_addr, _ = o
for o_addr in tx.get_output_addresses():
if self.is_mine(o_addr) and not self.is_change(o_addr):
break
else:
for o in tx.outputs:
o_addr, _ = o
for o_addr in tx.get_output_addresses():
if self.is_mine(o_addr):
break
else:
@ -789,7 +792,7 @@ class Abstract_Wallet:
for txin in inputs:
self.add_input_info(txin)
outputs = self.add_tx_change(inputs, outputs, amount, fee, total, change_addr)
return Transaction.from_io(inputs, outputs)
return Transaction(inputs, outputs)
def mktx(self, outputs, password, fee=None, change_addr=None, domain= None, coins = None ):
tx = self.make_unsigned_transaction(outputs, fee, change_addr, domain, coins)
@ -803,9 +806,13 @@ class Abstract_Wallet:
address = txin['address']
account_id, sequence = self.get_address_index(address)
account = self.accounts[account_id]
redeemScript = account.redeem_script(sequence)
txin['x_pubkeys'] = account.get_xpubkeys(sequence)
txin['pubkeys'] = pubkeys = account.get_pubkeys(sequence)
redeemScript = account.redeem_script(*sequence)
pubkeys = account.get_pubkeys(*sequence)
x_pubkeys = account.get_xpubkeys(*sequence)
# sort pubkeys and x_pubkeys, using the order of pubkeys
pubkeys, x_pubkeys = zip( *sorted(zip(pubkeys, x_pubkeys)))
txin['pubkeys'] = list(pubkeys)
txin['x_pubkeys'] = list(x_pubkeys)
txin['signatures'] = [None] * len(pubkeys)
if redeemScript:
@ -934,7 +941,7 @@ class Abstract_Wallet:
print_error("new history is orphaning transaction:", tx_hash)
# check that all outputs are not mine, request histories
ext_requests = []
for _addr, _v in tx.outputs:
for _addr in tx.get_output_addresses():
# assert not self.is_mine(_addr)
ext_requests.append( ('blockchain.address.get_history', [_addr]) )
@ -1279,7 +1286,11 @@ class NewWallet(Deterministic_Wallet):
return 'm/' in self.master_private_keys.keys()
def get_master_public_key(self):
return self.master_public_keys["m/"]
if self.is_watching_only():
return self.master_public_keys["m/0'"]
else:
return self.master_public_keys["m/"]
def get_master_public_keys(self):
out = {}