add lib changes from electrum 3.1.3

- Add electrum 3.1.3 changes to lib, lib/tests.
 - Get new electrum 3.1.3 lib/constants.py, add Zcash changes.
 - Get original electrum 3.1.3 lib/blockchain.py (will be
 - modified in next commits).
 - Remove debian/ lib-util patch (fixed in 3.1.3 code).
 - Remove lib/www (as in 3.1.3).

lib/base_wizard.py

@@ -24,11 +24,21 @@
 # SOFTWARE.
 
 import os
+import sys
+import traceback
+
 from . import bitcoin
 from . import keystore
 from .keystore import bip44_derivation
 from .wallet import Imported_Wallet, Standard_Wallet, Multisig_Wallet, wallet_types
+from .storage import STO_EV_USER_PW, STO_EV_XPUB_PW, get_derivation_used_for_hw_device_encryption
 from .i18n import _
+from .util import UserCancelled, InvalidPassword
+
+# hardware device setup purpose
+HWD_SETUP_NEW_WALLET, HWD_SETUP_DECRYPT_WALLET = range(0, 2)
+
+class ScriptTypeNotSupported(Exception): pass
 
 
 class BaseWizard(object):
@@ -59,7 +69,7 @@ class BaseWizard(object):
             f = getattr(self, action)
             f(*args)
         else:
-            raise BaseException("unknown action", action)
+            raise Exception("unknown action", action)
 
     def can_go_back(self):
         return len(self.stack)>1
@@ -112,7 +122,7 @@ class BaseWizard(object):
             choices = [
                 ('choose_seed_type', _('Create a new seed')),
                 ('restore_from_seed', _('I already have a seed')),
-                ('restore_from_key', _('Use public or private keys')),
+                ('restore_from_key', _('Use a master key')),
             ]
             if not self.is_kivy:
                 choices.append(('choose_hw_device',  _('Use a hardware device')))
@@ -131,20 +141,26 @@ class BaseWizard(object):
         v = lambda x: keystore.is_address_list(x) or keystore.is_private_key_list(x)
         title = _("Import Zcash Addresses")
         message = _("Enter a list of Zcash addresses (this will create a watching-only wallet), or a list of private keys.")
-        self.add_xpub_dialog(title=title, message=message, run_next=self.on_import, is_valid=v)
+        self.add_xpub_dialog(title=title, message=message, run_next=self.on_import,
+                             is_valid=v, allow_multi=True)
 
     def on_import(self, text):
+        # create a temporary wallet and exploit that modifications
+        # will be reflected on self.storage
         if keystore.is_address_list(text):
-            self.wallet = Imported_Wallet(self.storage)
+            w = Imported_Wallet(self.storage)
             for x in text.split():
-                self.wallet.import_address(x)
+                w.import_address(x)
         elif keystore.is_private_key_list(text):
             k = keystore.Imported_KeyStore({})
             self.storage.put('keystore', k.dump())
-            self.wallet = Imported_Wallet(self.storage)
-            for x in text.split():
-                self.wallet.import_private_key(x, None)
-        self.terminate()
+            w = Imported_Wallet(self.storage)
+            for x in keystore.get_private_keys(text):
+                w.import_private_key(x, None)
+            self.keystores.append(w.keystore)
+        else:
+            return self.terminate()
+        return self.run('create_wallet')
 
     def restore_from_key(self):
         if self.wallet_type == 'standard':
@@ -163,7 +179,7 @@ class BaseWizard(object):
         k = keystore.from_master_key(text)
         self.on_keystore(k)
 
-    def choose_hw_device(self):
+    def choose_hw_device(self, purpose=HWD_SETUP_NEW_WALLET):
         title = _('Hardware Keystore')
         # check available plugins
         support = self.plugins.get_hardware_support()
@@ -172,68 +188,121 @@ class BaseWizard(object):
                 _('No hardware wallet support found on your system.'),
                 _('Please install the relevant libraries (eg python-trezor for Trezor).'),
             ])
-            self.confirm_dialog(title=title, message=msg, run_next= lambda x: self.choose_hw_device())
+            self.confirm_dialog(title=title, message=msg, run_next= lambda x: self.choose_hw_device(purpose))
             return
         # scan devices
         devices = []
         devmgr = self.plugins.device_manager
-        for name, description, plugin in support:
-            try:
-                # FIXME: side-effect: unpaired_device_info sets client.handler
-                u = devmgr.unpaired_device_infos(None, plugin)
-            except:
-                devmgr.print_error("error", name)
-                continue
-            devices += list(map(lambda x: (name, x), u))
+        try:
+            scanned_devices = devmgr.scan_devices()
+        except BaseException as e:
+            devmgr.print_error('error scanning devices: {}'.format(e))
+            debug_msg = '  {}:\n    {}'.format(_('Error scanning devices'), e)
+        else:
+            debug_msg = ''
+            for name, description, plugin in support:
+                try:
+                    # FIXME: side-effect: unpaired_device_info sets client.handler
+                    u = devmgr.unpaired_device_infos(None, plugin, devices=scanned_devices)
+                except BaseException as e:
+                    devmgr.print_error('error getting device infos for {}: {}'.format(name, e))
+                    debug_msg += '  {}:\n    {}\n'.format(plugin.name, e)
+                    continue
+                devices += list(map(lambda x: (name, x), u))
+        if not debug_msg:
+            debug_msg = '  {}'.format(_('No exceptions encountered.'))
         if not devices:
             msg = ''.join([
                 _('No hardware device detected.') + '\n',
                 _('To trigger a rescan, press \'Next\'.') + '\n\n',
                 _('If your device is not detected on Windows, go to "Settings", "Devices", "Connected devices", and do "Remove device". Then, plug your device again.') + ' ',
-                _('On Linux, you might have to add a new permission to your udev rules.'),
+                _('On Linux, you might have to add a new permission to your udev rules.') + '\n\n',
+                _('Debug message') + '\n',
+                debug_msg
             ])
-            self.confirm_dialog(title=title, message=msg, run_next= lambda x: self.choose_hw_device())
+            self.confirm_dialog(title=title, message=msg, run_next= lambda x: self.choose_hw_device(purpose))
             return
         # select device
         self.devices = devices
         choices = []
         for name, info in devices:
             state = _("initialized") if info.initialized else _("wiped")
-            label = info.label or _("An unnamed %s")%name
+            label = info.label or _("An unnamed {}").format(name)
             descr = "%s [%s, %s]" % (label, name, state)
             choices.append(((name, info), descr))
         msg = _('Select a device') + ':'
-        self.choice_dialog(title=title, message=msg, choices=choices, run_next=self.on_device)
+        self.choice_dialog(title=title, message=msg, choices=choices, run_next= lambda *args: self.on_device(*args, purpose=purpose))
 
-    def on_device(self, name, device_info):
+    def on_device(self, name, device_info, *, purpose):
         self.plugin = self.plugins.get_plugin(name)
         try:
-            self.plugin.setup_device(device_info, self)
+            self.plugin.setup_device(device_info, self, purpose)
+        except OSError as e:
+            self.show_error(_('We encountered an error while connecting to your device:')
+                            + '\n' + str(e) + '\n'
+                            + _('To try to fix this, we will now re-pair with your device.') + '\n'
+                            + _('Please try again.'))
+            devmgr = self.plugins.device_manager
+            devmgr.unpair_id(device_info.device.id_)
+            self.choose_hw_device(purpose)
+            return
+        except UserCancelled:
+            self.choose_hw_device(purpose)
+            return
         except BaseException as e:
             self.show_error(str(e))
-            self.choose_hw_device()
+            self.choose_hw_device(purpose)
             return
-        if self.wallet_type=='multisig':
-            # There is no general standard for HD multisig.
-            # This is partially compatible with BIP45; assumes index=0
-            self.on_hw_derivation(name, device_info, "m/45'/0")
+        if purpose == HWD_SETUP_NEW_WALLET:
+            if self.wallet_type=='multisig':
+                # There is no general standard for HD multisig.
+                # This is partially compatible with BIP45; assumes index=0
+                self.on_hw_derivation(name, device_info, "m/45'/0")
+            else:
+                f = lambda x: self.run('on_hw_derivation', name, device_info, str(x))
+                self.derivation_dialog(f)
+        elif purpose == HWD_SETUP_DECRYPT_WALLET:
+            derivation = get_derivation_used_for_hw_device_encryption()
+            xpub = self.plugin.get_xpub(device_info.device.id_, derivation, 'standard', self)
+            password = keystore.Xpub.get_pubkey_from_xpub(xpub, ())
+            try:
+                self.storage.decrypt(password)
+            except InvalidPassword:
+                # try to clear session so that user can type another passphrase
+                devmgr = self.plugins.device_manager
+                client = devmgr.client_by_id(device_info.device.id_)
+                if hasattr(client, 'clear_session'):  # FIXME not all hw wallet plugins have this
+                    client.clear_session()
+                raise
         else:
-            f = lambda x: self.run('on_hw_derivation', name, device_info, str(x))
-            self.derivation_dialog(f)
+            raise Exception('unknown purpose: %s' % purpose)
 
     def derivation_dialog(self, f):
-        default = bip44_derivation(0, False)
+        default = bip44_derivation(0, bip43_purpose=44)
         message = '\n'.join([
             _('Enter your wallet derivation here.'),
             _('If you are not sure what this is, leave this field unchanged.')
         ])
-        self.line_dialog(run_next=f, title=_('Derivation'), message=message, default=default, test=bitcoin.is_bip32_derivation)
+        presets = (
+            ('legacy BIP44', bip44_derivation(0, bip43_purpose=44)),
+        )
+        while True:
+            try:
+                self.line_dialog(run_next=f, title=_('Derivation'), message=message,
+                                 default=default, test=bitcoin.is_bip32_derivation,
+                                 presets=presets)
+                return
+            except ScriptTypeNotSupported as e:
+                self.show_error(e)
+                # let the user choose again
 
     def on_hw_derivation(self, name, device_info, derivation):
         from .keystore import hardware_keystore
-        xtype = 'standard'
+        xtype = keystore.xtype_from_derivation(derivation)
         try:
             xpub = self.plugin.get_xpub(device_info.device.id_, derivation, xtype, self)
+        except ScriptTypeNotSupported:
+            raise  # this is handled in derivation_dialog
         except BaseException as e:
             self.show_error(e)
             return
@@ -277,7 +346,7 @@ class BaseWizard(object):
         elif self.seed_type == 'old':
             self.run('create_keystore', seed, '')
         else:
-            raise BaseException('Unknown seed type', self.seed_type)
+            raise Exception('Unknown seed type', self.seed_type)
 
     def on_restore_bip39(self, seed, passphrase):
         f = lambda x: self.run('on_bip43', seed, passphrase, str(x))
@@ -330,13 +399,45 @@ class BaseWizard(object):
                 self.run('create_wallet')
 
     def create_wallet(self):
-        if any(k.may_have_password() for k in self.keystores):
-            self.request_password(run_next=self.on_password)
+        encrypt_keystore = any(k.may_have_password() for k in self.keystores)
+        # note: the following condition ("if") is duplicated logic from
+        # wallet.get_available_storage_encryption_version()
+        if self.wallet_type == 'standard' and isinstance(self.keystores[0], keystore.Hardware_KeyStore):
+            # offer encrypting with a pw derived from the hw device
+            k = self.keystores[0]
+            try:
+                k.handler = self.plugin.create_handler(self)
+                password = k.get_password_for_storage_encryption()
+            except UserCancelled:
+                devmgr = self.plugins.device_manager
+                devmgr.unpair_xpub(k.xpub)
+                self.choose_hw_device()
+                return
+            except BaseException as e:
+                traceback.print_exc(file=sys.stderr)
+                self.show_error(str(e))
+                return
+            self.request_storage_encryption(
+                run_next=lambda encrypt_storage: self.on_password(
+                    password,
+                    encrypt_storage=encrypt_storage,
+                    storage_enc_version=STO_EV_XPUB_PW,
+                    encrypt_keystore=False))
         else:
-            self.on_password(None, False)
+            # prompt the user to set an arbitrary password
+            self.request_password(
+                run_next=lambda password, encrypt_storage: self.on_password(
+                    password,
+                    encrypt_storage=encrypt_storage,
+                    storage_enc_version=STO_EV_USER_PW,
+                    encrypt_keystore=encrypt_keystore),
+                force_disable_encrypt_cb=not encrypt_keystore)
 
-    def on_password(self, password, encrypt):
-        self.storage.set_password(password, encrypt)
+    def on_password(self, password, *, encrypt_storage,
+                    storage_enc_version=STO_EV_USER_PW, encrypt_keystore):
+        self.storage.set_keystore_encryption(bool(password) and encrypt_keystore)
+        if encrypt_storage:
+            self.storage.set_password(password, enc_version=storage_enc_version)
         for k in self.keystores:
             if k.may_have_password():
                 k.update_password(None, password)
@@ -352,18 +453,21 @@ class BaseWizard(object):
             self.storage.write()
             self.wallet = Multisig_Wallet(self.storage)
             self.run('create_addresses')
+        elif self.wallet_type == 'imported':
+            if len(self.keystores) > 0:
+                keys = self.keystores[0].dump()
+                self.storage.put('keystore', keys)
+            self.wallet = Imported_Wallet(self.storage)
+            self.wallet.storage.write()
+            self.terminate()
 
     def show_xpub_and_add_cosigners(self, xpub):
         self.show_xpub_dialog(xpub=xpub, run_next=lambda x: self.run('choose_keystore'))
 
-    def on_cosigner(self, text, password, i):
-        k = keystore.from_master_key(text, password)
-        self.on_keystore(k)
-
     def choose_seed_type(self):
         title = _('Choose Seed type')
         message = ' '.join([
-            "The type of addresses used by your wallet will depend on your seed.",
+            _("The type of addresses used by your wallet will depend on your seed."),
         ])
         choices = [
             ('create_standard_seed', _('Standard')),
@@ -408,5 +512,5 @@ class BaseWizard(object):
             self.wallet.synchronize()
             self.wallet.storage.write()
             self.terminate()
-        msg = _("Electrum-Zcash is generating your addresses, please wait.")
+        msg = _("Electrum-Zcash is generating your addresses, please wait...")
         self.waiting_dialog(task, msg)

lib/bitcoin.py

@@ -32,72 +32,18 @@ import json
 import ecdsa
 import pyaes
 
-from .util import bfh, bh2u, to_string
+from .util import bfh, bh2u, to_string, BitcoinException
 from . import version
 from .util import print_error, InvalidPassword, assert_bytes, to_bytes, inv_dict
+from . import constants
 
-def read_json_dict(filename):
-    path = os.path.join(os.path.dirname(__file__), filename)
-    try:
-        with open(path, 'r') as f:
-            r = json.loads(f.read())
-    except:
-        r = {}
-    return r
-
-
-
-
-# Version numbers for BIP32 extended keys
-# standard: xprv, xpub
-XPRV_HEADERS = {
-    'standard': 0x0488ade4,
-}
-XPUB_HEADERS = {
-    'standard': 0x0488b21e,
-}
-
-
-class NetworkConstants:
-
-    @classmethod
-    def set_mainnet(cls):
-        cls.TESTNET = False
-        cls.WIF_PREFIX = 0x80
-        cls.ADDRTYPE_P2PKH = bytes.fromhex('1CB8')
-        cls.ADDRTYPE_P2SH = bytes.fromhex('1CBD')
-        cls.HEADERS_URL = 'https://github.com/zebra-lucky/electrum-zcash/releases/download/3.0.6/blockchain_headers'
-        cls.GENESIS = '00040fe8ec8471911baa1db1266ea15dd06b4a8a5c453883c000b031973dce08'
-        cls.DEFAULT_PORTS = {'t': '50001', 's': '50002'}
-        cls.DEFAULT_SERVERS = read_json_dict('servers.json')
-        XPRV_HEADERS['standard'] = 0x0488ade4
-        XPUB_HEADERS['standard'] = 0x0488b21e
-
-    @classmethod
-    def set_testnet(cls):
-        cls.TESTNET = True
-        cls.WIF_PREFIX = 0xEF
-        cls.ADDRTYPE_P2PKH = bytes.fromhex('1D25')
-        cls.ADDRTYPE_P2SH = bytes.fromhex('1CBA')
-        cls.HEADERS_URL = 'https://github.com/zebra-lucky/electrum-zcash/releases/download/3.0.6/blockchain_headers_testnet'
-        cls.GENESIS = '05a60a92d99d85997cce3b87616c089f6124d7342af37106edc76126334a2c38'
-        cls.DEFAULT_PORTS = {'t':'51001', 's':'51002'}
-        cls.DEFAULT_SERVERS = read_json_dict('servers_testnet.json')
-        XPRV_HEADERS['standard'] = 0x04358394
-        XPUB_HEADERS['standard'] = 0x043587CF
-
-
-NetworkConstants.set_mainnet()
 
 ################################## transactions
 
-MAX_FEE_RATE = 10000
-FEE_TARGETS = [25, 10, 5, 2]
-
 COINBASE_MATURITY = 100
 COIN = 100000000
 
-# supported types of transction outputs
+# supported types of transaction outputs
 TYPE_ADDRESS = 0
 TYPE_PUBKEY  = 1
 TYPE_SCRIPT  = 2
@@ -196,7 +142,11 @@ def rev_hex(s):
 
 
 def int_to_hex(i, length=1):
-    assert isinstance(i, int)
+    if not isinstance(i, int):
+        raise TypeError('{} instead of int'.format(i))
+    if i < 0:
+        # two's complement
+        i = pow(256, length) + i
     s = hex(i)[2:].rstrip('L')
     s = "0"*(2*length - len(s)) + s
     return rev_hex(s)
@@ -215,11 +165,11 @@ def var_int(i):
 
 
 def op_push(i):
-    if i<0x4c:
+    if i<0x4c:  # OP_PUSHDATA1
         return int_to_hex(i)
-    elif i<0xff:
+    elif i<=0xff:
         return '4c' + int_to_hex(i)
-    elif i<0xffff:
+    elif i<=0xffff:
         return '4d' + int_to_hex(i,2)
     else:
         return '4e' + int_to_hex(i,4)
@@ -322,11 +272,15 @@ def b58_address_to_hash160(addr):
     return _bytes[0:2], _bytes[2:22]
 
 
-def hash160_to_p2pkh(h160):
-    return hash160_to_b58_address(h160, NetworkConstants.ADDRTYPE_P2PKH)
+def hash160_to_p2pkh(h160, *, net=None):
+    if net is None:
+        net = constants.net
+    return hash160_to_b58_address(h160, net.ADDRTYPE_P2PKH)
 
-def hash160_to_p2sh(h160):
-    return hash160_to_b58_address(h160, NetworkConstants.ADDRTYPE_P2SH)
+def hash160_to_p2sh(h160, *, net=None):
+    if net is None:
+        net = constants.net
+    return hash160_to_b58_address(h160, net.ADDRTYPE_P2SH)
 
 def public_key_to_p2pkh(public_key):
     return hash160_to_p2pkh(hash_160(public_key))
@@ -344,24 +298,26 @@ def redeem_script_to_address(txin_type, redeem_script):
         raise NotImplementedError(txin_type)
 
 
-def script_to_address(script):
+def script_to_address(script, *, net=None):
     from .transaction import get_address_from_output_script
-    t, addr = get_address_from_output_script(bfh(script))
+    t, addr = get_address_from_output_script(bfh(script), net=net)
     assert t == TYPE_ADDRESS
     return addr
 
-def address_to_script(addr):
+def address_to_script(addr, *, net=None):
+    if net is None:
+        net = constants.net
     addrtype, hash_160 = b58_address_to_hash160(addr)
-    if addrtype == NetworkConstants.ADDRTYPE_P2PKH:
+    if addrtype == net.ADDRTYPE_P2PKH:
         script = '76a9'                                      # op_dup, op_hash_160
         script += push_script(bh2u(hash_160))
         script += '88ac'                                     # op_equalverify, op_checksig
-    elif addrtype == NetworkConstants.ADDRTYPE_P2SH:
+    elif addrtype == net.ADDRTYPE_P2SH:
         script = 'a9'                                        # op_hash_160
         script += push_script(bh2u(hash_160))
         script += '87'                                       # op_equal
     else:
-        raise BaseException('unknown address type')
+        raise BitcoinException('unknown address type: {}'.format(addrtype))
     return script
 
 def address_to_scripthash(addr):
@@ -387,7 +343,8 @@ assert len(__b43chars) == 43
 def base_encode(v, base):
     """ encode v, which is a string of bytes, to base58."""
     assert_bytes(v)
-    assert base in (58, 43)
+    if base not in (58, 43):
+        raise ValueError('not supported base: {}'.format(base))
     chars = __b58chars
     if base == 43:
         chars = __b43chars
@@ -417,13 +374,17 @@ def base_decode(v, length, base):
     """ decode v into a string of len bytes."""
     # assert_bytes(v)
     v = to_bytes(v, 'ascii')
-    assert base in (58, 43)
+    if base not in (58, 43):
+        raise ValueError('not supported base: {}'.format(base))
     chars = __b58chars
     if base == 43:
         chars = __b43chars
     long_value = 0
     for (i, c) in enumerate(v[::-1]):
-        long_value += chars.find(bytes([c])) * (base**i)
+        digit = chars.find(bytes([c]))
+        if digit == -1:
+            raise ValueError('Forbidden character {} for base {}'.format(c, base))
+        long_value += digit * (base**i)
     result = bytearray()
     while long_value >= 256:
         div, mod = divmod(long_value, 256)
@@ -443,6 +404,10 @@ def base_decode(v, length, base):
     return bytes(result)
 
 
+class InvalidChecksum(Exception):
+    pass
+
+
 def EncodeBase58Check(vchIn):
     hash = Hash(vchIn)
     return base_encode(vchIn + hash[0:4], base=58)
@@ -455,37 +420,64 @@ def DecodeBase58Check(psz):
     hash = Hash(key)
     cs32 = hash[0:4]
     if cs32 != csum:
-        return None
+        raise InvalidChecksum('expected {}, actual {}'.format(bh2u(cs32), bh2u(csum)))
     else:
         return key
 
 
 
+# backwards compat
+# extended WIF for segwit (used in 3.0.x; but still used internally)
+# the keys in this dict should be a superset of what Imported Wallets can import
 SCRIPT_TYPES = {
     'p2pkh':0,
     'p2sh':5,
 }
 
 
-def serialize_privkey(secret, compressed, txin_type):
-    prefix = bytes([(SCRIPT_TYPES[txin_type]+NetworkConstants.WIF_PREFIX)&255])
+def serialize_privkey(secret, compressed, txin_type, internal_use=False):
+    if internal_use:
+        prefix = bytes([(SCRIPT_TYPES[txin_type] + constants.net.WIF_PREFIX) & 255])
+    else:
+        prefix = bytes([constants.net.WIF_PREFIX])
     suffix = b'\01' if compressed else b''
     vchIn = prefix + secret + suffix
-    return EncodeBase58Check(vchIn)
+    base58_wif = EncodeBase58Check(vchIn)
+    if internal_use:
+        return base58_wif
+    else:
+        return '{}:{}'.format(txin_type, base58_wif)
 
 
 def deserialize_privkey(key):
-    # whether the pubkey is compressed should be visible from the keystore
-    vch = DecodeBase58Check(key)
     if is_minikey(key):
         return 'p2pkh', minikey_to_private_key(key), True
-    elif vch:
-        txin_type = inv_dict(SCRIPT_TYPES)[vch[0] - NetworkConstants.WIF_PREFIX]
-        assert len(vch) in [33, 34]
-        compressed = len(vch) == 34
-        return txin_type, vch[1:33], compressed
+
+    txin_type = None
+    if ':' in key:
+        txin_type, key = key.split(sep=':', maxsplit=1)
+        if txin_type not in SCRIPT_TYPES:
+            raise BitcoinException('unknown script type: {}'.format(txin_type))
+    try:
+        vch = DecodeBase58Check(key)
+    except BaseException:
+        neutered_privkey = str(key)[:3] + '..' + str(key)[-2:]
+        raise BitcoinException("cannot deserialize privkey {}"
+                               .format(neutered_privkey))
+
+    if txin_type is None:
+        # keys exported in version 3.0.x encoded script type in first byte
+        txin_type = inv_dict(SCRIPT_TYPES)[vch[0] - constants.net.WIF_PREFIX]
     else:
-        raise BaseException("cannot deserialize", key)
+        # all other keys must have a fixed first byte
+        if vch[0] != constants.net.WIF_PREFIX:
+            raise BitcoinException('invalid prefix ({}) for WIF key'.format(vch[0]))
+
+    if len(vch) not in [33, 34]:
+        raise BitcoinException('invalid vch len for WIF key: {}'.format(len(vch)))
+    compressed = len(vch) == 34
+    return txin_type, vch[1:33], compressed
+
 
 def regenerate_key(pk):
     assert len(pk) == 32
@@ -519,7 +511,7 @@ def is_b58_address(addr):
         addrtype, h = b58_address_to_hash160(addr)
     except Exception as e:
         return False
-    if addrtype not in [NetworkConstants.ADDRTYPE_P2PKH, NetworkConstants.ADDRTYPE_P2SH]:
+    if addrtype not in [constants.net.ADDRTYPE_P2PKH, constants.net.ADDRTYPE_P2SH]:
         return False
     return addr == hash160_to_b58_address(h, addrtype)
 
@@ -582,8 +574,8 @@ def verify_message(address, sig, message):
         return False
 
 
-def encrypt_message(message, pubkey):
-    return EC_KEY.encrypt_message(message, bfh(pubkey))
+def encrypt_message(message, pubkey, magic=b'BIE1'):
+    return EC_KEY.encrypt_message(message, bfh(pubkey), magic)
 
 
 def chunks(l, n):
@@ -728,7 +720,7 @@ class EC_KEY(object):
     # ECIES encryption/decryption methods; AES-128-CBC with PKCS7 is used as the cipher; hmac-sha256 is used as the mac
 
     @classmethod
-    def encrypt_message(self, message, pubkey):
+    def encrypt_message(self, message, pubkey, magic=b'BIE1'):
         assert_bytes(message)
 
         pk = ser_to_point(pubkey)
@@ -742,20 +734,20 @@ class EC_KEY(object):
         iv, key_e, key_m = key[0:16], key[16:32], key[32:]
         ciphertext = aes_encrypt_with_iv(key_e, iv, message)
         ephemeral_pubkey = bfh(ephemeral.get_public_key(compressed=True))
-        encrypted = b'BIE1' + ephemeral_pubkey + ciphertext
+        encrypted = magic + ephemeral_pubkey + ciphertext
         mac = hmac.new(key_m, encrypted, hashlib.sha256).digest()
 
         return base64.b64encode(encrypted + mac)
 
-    def decrypt_message(self, encrypted):
+    def decrypt_message(self, encrypted, magic=b'BIE1'):
         encrypted = base64.b64decode(encrypted)
         if len(encrypted) < 85:
             raise Exception('invalid ciphertext: length')
-        magic = encrypted[:4]
+        magic_found = encrypted[:4]
         ephemeral_pubkey = encrypted[4:37]
         ciphertext = encrypted[37:-32]
         mac = encrypted[-32:]
-        if magic != b'BIE1':
+        if magic_found != magic:
             raise Exception('invalid ciphertext: invalid magic bytes')
         try:
             ephemeral_pubkey = ser_to_point(ephemeral_pubkey)
@@ -831,47 +823,60 @@ def _CKD_pub(cK, c, s):
     return cK_n, c_n
 
 
-def xprv_header(xtype):
-    return bfh("%08x" % XPRV_HEADERS[xtype])
+def xprv_header(xtype, *, net=None):
+    if net is None:
+        net = constants.net
+    return bfh("%08x" % net.XPRV_HEADERS[xtype])
 
 
-def xpub_header(xtype):
-    return bfh("%08x" % XPUB_HEADERS[xtype])
+def xpub_header(xtype, *, net=None):
+    if net is None:
+        net = constants.net
+    return bfh("%08x" % net.XPUB_HEADERS[xtype])
 
 
-def serialize_xprv(xtype, c, k, depth=0, fingerprint=b'\x00'*4, child_number=b'\x00'*4):
-    xprv = xprv_header(xtype) + bytes([depth]) + fingerprint + child_number + c + bytes([0]) + k
+def serialize_xprv(xtype, c, k, depth=0, fingerprint=b'\x00'*4,
+                   child_number=b'\x00'*4, *, net=None):
+    xprv = xprv_header(xtype, net=net) \
+           + bytes([depth]) + fingerprint + child_number + c + bytes([0]) + k
     return EncodeBase58Check(xprv)
 
 
-def serialize_xpub(xtype, c, cK, depth=0, fingerprint=b'\x00'*4, child_number=b'\x00'*4):
-    xpub = xpub_header(xtype) + bytes([depth]) + fingerprint + child_number + c + cK
+def serialize_xpub(xtype, c, cK, depth=0, fingerprint=b'\x00'*4,
+                   child_number=b'\x00'*4, *, net=None):
+    xpub = xpub_header(xtype, net=net) \
+           + bytes([depth]) + fingerprint + child_number + c + cK
     return EncodeBase58Check(xpub)
 
 
-def deserialize_xkey(xkey, prv):
+def deserialize_xkey(xkey, prv, *, net=None):
+    if net is None:
+        net = constants.net
     xkey = DecodeBase58Check(xkey)
     if len(xkey) != 78:
-        raise BaseException('Invalid length')
+        raise BitcoinException('Invalid length for extended key: {}'
+                               .format(len(xkey)))
     depth = xkey[4]
     fingerprint = xkey[5:9]
     child_number = xkey[9:13]
     c = xkey[13:13+32]
     header = int('0x' + bh2u(xkey[0:4]), 16)
-    headers = XPRV_HEADERS if prv else XPUB_HEADERS
+    headers = net.XPRV_HEADERS if prv else net.XPUB_HEADERS
     if header not in headers.values():
-        raise BaseException('Invalid xpub format', hex(header))
+        raise BitcoinException('Invalid extended key format: {}'
+                               .format(hex(header)))
     xtype = list(headers.keys())[list(headers.values()).index(header)]
     n = 33 if prv else 32
     K_or_k = xkey[13+n:]
     return xtype, depth, fingerprint, child_number, c, K_or_k
 
 
-def deserialize_xpub(xkey):
-    return deserialize_xkey(xkey, False)
 
-def deserialize_xprv(xkey):
-    return deserialize_xkey(xkey, True)
+def deserialize_xpub(xkey, *, net=None):
+    return deserialize_xkey(xkey, False, net=net)
+
+def deserialize_xprv(xkey, *, net=None):
+    return deserialize_xkey(xkey, True, net=net)
 
 def xpub_type(x):
     return deserialize_xpub(x)[0]
@@ -915,7 +920,8 @@ def xpub_from_pubkey(xtype, cK):
 
 
 def bip32_derivation(s):
-    assert s.startswith('m/')
+    if not s.startswith('m/'):
+        raise ValueError('invalid bip32 derivation path: {}'.format(s))
     s = s[2:]
     for n in s.split('/'):
         if n == '': continue
@@ -930,7 +936,9 @@ def is_bip32_derivation(x):
         return False
 
 def bip32_private_derivation(xprv, branch, sequence):
-    assert sequence.startswith(branch)
+    if not sequence.startswith(branch):
+        raise ValueError('incompatible branch ({}) and sequence ({})'
+                         .format(branch, sequence))
     if branch == sequence:
         return xprv, xpub_from_xprv(xprv)
     xtype, depth, fingerprint, child_number, c, k = deserialize_xprv(xprv)
@@ -952,7 +960,9 @@ def bip32_private_derivation(xprv, branch, sequence):
 
 def bip32_public_derivation(xpub, branch, sequence):
     xtype, depth, fingerprint, child_number, c, cK = deserialize_xpub(xpub)
-    assert sequence.startswith(branch)
+    if not sequence.startswith(branch):
+        raise ValueError('incompatible branch ({}) and sequence ({})'
+                         .format(branch, sequence))
     sequence = sequence[len(branch):]
     for n in sequence.split('/'):
         if n == '': continue

lib/blockchain.py

@@ -25,79 +25,33 @@ import threading
 
 from . import util
 from . import bitcoin
+from . import constants
 from .bitcoin import *
 
-
-HDR_LEN = 1487
-CHUNK_LEN = 100
-
-POW_LIMIT = 0x0007FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
-POW_AVERAGING_WINDOW = 17
-POW_MEDIAN_BLOCK_SPAN = 11
-POW_MAX_ADJUST_DOWN = 32
-POW_MAX_ADJUST_UP = 16
-POW_DAMPING_FACTOR = 4
-POW_TARGET_SPACING = 150
-
-AVERAGING_WINDOW_TIMESPAN = POW_AVERAGING_WINDOW * POW_TARGET_SPACING
-
-MIN_ACTUAL_TIMESPAN = AVERAGING_WINDOW_TIMESPAN * \
-    (100 - POW_MAX_ADJUST_UP) // 100
-
-MAX_ACTUAL_TIMESPAN = AVERAGING_WINDOW_TIMESPAN * \
-    (100 + POW_MAX_ADJUST_DOWN) // 100
-
-
-def bits_to_target(bits):
-    """Convert a compact representation to a hex target."""
-    MM = 256 * 256 * 256
-    a = bits % MM
-    if a < 0x8000:
-        a *= 256
-    target = a * pow(2, 8 * (bits // MM - 3))
-    return target
-
-def target_to_bits(target):
-    """Convert a target to compact representation."""
-    MM = 256 * 256 * 256
-    c = ('%064X' % target)[2:]
-    i = 31
-    while c[0:2] == '00':
-        c = c[2:]
-        i -= 1
-
-    c = int('0x%s' % c[0:6], 16)
-    if c >= 0x800000:
-        c //= 256
-        i += 1
-
-    new_bits = c + MM * i
-    return new_bits
+MAX_TARGET = 0x00000000FFFF0000000000000000000000000000000000000000000000000000
 
 def serialize_header(res):
     s = int_to_hex(res.get('version'), 4) \
         + rev_hex(res.get('prev_block_hash')) \
         + rev_hex(res.get('merkle_root')) \
-        + rev_hex(res.get('reserved_hash')) \
         + int_to_hex(int(res.get('timestamp')), 4) \
         + int_to_hex(int(res.get('bits')), 4) \
-        + rev_hex(res.get('nonce')) \
-        + rev_hex(res.get('sol_size')) \
-        + rev_hex(res.get('solution'))
+        + int_to_hex(int(res.get('nonce')), 4)
     return s
 
 def deserialize_header(s, height):
+    if not s:
+        raise Exception('Invalid header: {}'.format(s))
+    if len(s) != 80:
+        raise Exception('Invalid header length: {}'.format(len(s)))
     hex_to_int = lambda s: int('0x' + bh2u(s[::-1]), 16)
     h = {}
     h['version'] = hex_to_int(s[0:4])
     h['prev_block_hash'] = hash_encode(s[4:36])
     h['merkle_root'] = hash_encode(s[36:68])
-    h['reserved_hash'] = hash_encode(s[68:100])
-    h['timestamp'] = hex_to_int(s[100:104])
-    h['bits'] = hex_to_int(s[104:108])
-    h['nonce'] = hash_encode(s[108:140])
-    h['sol_size'] = hash_encode(s[140:143])
-    h['solution'] = hash_encode(s[143:1487])
+    h['timestamp'] = hex_to_int(s[68:72])
+    h['bits'] = hex_to_int(s[72:76])
+    h['nonce'] = hex_to_int(s[76:80])
     h['block_height'] = height
     return h
 
@@ -122,7 +76,11 @@ def read_blockchains(config):
         checkpoint = int(filename.split('_')[2])
         parent_id = int(filename.split('_')[1])
         b = Blockchain(config, checkpoint, parent_id)
-        blockchains[b.checkpoint] = b
+        h = b.read_header(b.checkpoint)
+        if b.parent().can_connect(h, check_height=False):
+            blockchains[b.checkpoint] = b
+        else:
+            util.print_error("cannot connect", filename)
     return blockchains
 
 def check_header(header):
@@ -149,6 +107,7 @@ class Blockchain(util.PrintError):
         self.config = config
         self.catch_up = None # interface catching up
         self.checkpoint = checkpoint
+        self.checkpoints = constants.net.CHECKPOINTS
         self.parent_id = parent_id
         self.lock = threading.Lock()
         with self.lock:
@@ -192,35 +151,29 @@ class Blockchain(util.PrintError):
 
     def update_size(self):
         p = self.path()
-        self._size = os.path.getsize(p)//HDR_LEN if os.path.exists(p) else 0
+        self._size = os.path.getsize(p)//80 if os.path.exists(p) else 0
 
-    def verify_header(self, header, prev_header, bits, target):
-        prev_hash = hash_header(prev_header)
+    def verify_header(self, header, prev_hash, target):
         _hash = hash_header(header)
         if prev_hash != header.get('prev_block_hash'):
-            raise BaseException("prev hash mismatch: %s vs %s" % (prev_hash, header.get('prev_block_hash')))
-        if bitcoin.NetworkConstants.TESTNET:
+            raise Exception("prev hash mismatch: %s vs %s" % (prev_hash, header.get('prev_block_hash')))
+        if constants.net.TESTNET:
             return
+        bits = self.target_to_bits(target)
         if bits != header.get('bits'):
-            raise BaseException("bits mismatch: %s vs %s" % (bits, header.get('bits')))
+            raise Exception("bits mismatch: %s vs %s" % (bits, header.get('bits')))
         if int('0x' + _hash, 16) > target:
-            raise BaseException("insufficient proof of work: %s vs target %s" % (int('0x' + _hash, 16), target))
+            raise Exception("insufficient proof of work: %s vs target %s" % (int('0x' + _hash, 16), target))
 
     def verify_chunk(self, index, data):
-        num = len(data) // HDR_LEN
-        prev_header = None
-        if index != 0:
-            prev_header = self.read_header(index * CHUNK_LEN - 1)
-        chain = []
+        num = len(data) // 80
+        prev_hash = self.get_hash(index * 2016 - 1)
+        target = self.get_target(index-1)
         for i in range(num):
-            raw_header = data[i*HDR_LEN:(i+1) * HDR_LEN]
-            header = deserialize_header(raw_header, index*CHUNK_LEN + i)
-            height = index * CHUNK_LEN + i
-            header['block_height'] = height
-            chain.append(header)
-            bits, target = self.get_target(height, chain)
-            self.verify_header(header, prev_header, bits, target)
-            prev_header = header
+            raw_header = data[i*80:(i+1) * 80]
+            header = deserialize_header(raw_header, index*2016 + i)
+            self.verify_header(header, prev_hash, target)
+            prev_hash = hash_header(header)
 
     def path(self):
         d = util.get_headers_dir(self.config)
@@ -229,11 +182,12 @@ class Blockchain(util.PrintError):
 
     def save_chunk(self, index, chunk):
         filename = self.path()
-        d = (index * CHUNK_LEN - self.checkpoint) * HDR_LEN
+        d = (index * 2016 - self.checkpoint) * 80
         if d < 0:
             chunk = chunk[-d:]
             d = 0
-        self.write(chunk, d)
+        truncate = index >= len(self.checkpoints)
+        self.write(chunk, d, truncate)
         self.swap_with_parent()
 
     def swap_with_parent(self):
@@ -249,10 +203,10 @@ class Blockchain(util.PrintError):
         with open(self.path(), 'rb') as f:
             my_data = f.read()
         with open(parent.path(), 'rb') as f:
-            f.seek((checkpoint - parent.checkpoint)*HDR_LEN)
-            parent_data = f.read(parent_branch_size*HDR_LEN)
+            f.seek((checkpoint - parent.checkpoint)*80)
+            parent_data = f.read(parent_branch_size*80)
         self.write(parent_data, 0)
-        parent.write(my_data, (checkpoint - parent.checkpoint)*HDR_LEN)
+        parent.write(my_data, (checkpoint - parent.checkpoint)*80)
         # store file path
         for b in blockchains.values():
             b.old_path = b.path()
@@ -270,11 +224,11 @@ class Blockchain(util.PrintError):
         blockchains[self.checkpoint] = self
         blockchains[parent.checkpoint] = parent
 
-    def write(self, data, offset):
+    def write(self, data, offset, truncate=True):
         filename = self.path()
         with self.lock:
             with open(filename, 'rb+') as f:
-                if offset != self._size*HDR_LEN:
+                if truncate and offset != self._size*80:
                     f.seek(offset)
                     f.truncate()
                 f.seek(offset)
@@ -287,8 +241,8 @@ class Blockchain(util.PrintError):
         delta = header.get('block_height') - self.checkpoint
         data = bfh(serialize_header(header))
         assert delta == self.size()
-        assert len(data) == HDR_LEN
-        self.write(data, delta*HDR_LEN)
+        assert len(data) == 80
+        self.write(data, delta*80)
         self.swap_with_parent()
 
     def read_header(self, height):
@@ -303,88 +257,90 @@ class Blockchain(util.PrintError):
         name = self.path()
         if os.path.exists(name):
             with open(name, 'rb') as f:
-                f.seek(delta * HDR_LEN)
-                h = f.read(HDR_LEN)
+                f.seek(delta * 80)
+                h = f.read(80)
+                if len(h) < 80:
+                    raise Exception('Expected to read a full header. This was only {} bytes'.format(len(h)))
+        elif not os.path.exists(util.get_headers_dir(self.config)):
+            raise Exception('Electrum datadir does not exist. Was it deleted while running?')
+        else:
+            raise Exception('Cannot find headers file but datadir is there. Should be at {}'.format(name))
+        if h == bytes([0])*80:
+            return None
         return deserialize_header(h, height)
 
     def get_hash(self, height):
-        return hash_header(self.read_header(height))
+        if height == -1:
+            return '0000000000000000000000000000000000000000000000000000000000000000'
+        elif height == 0:
+            return constants.net.GENESIS
+        elif height < len(self.checkpoints) * 2016:
+            assert (height+1) % 2016 == 0, height
+            index = height // 2016
+            h, t = self.checkpoints[index]
+            return h
+        else:
+            return hash_header(self.read_header(height))
 
-    def get_median_time(self, height, chain=None):
-        if chain is None:
-            chain = []
+    def get_target(self, index):
+        # compute target from chunk x, used in chunk x+1
+        if constants.net.TESTNET:
+            return 0
+        if index == -1:
+            return MAX_TARGET
+        if index < len(self.checkpoints):
+            h, t = self.checkpoints[index]
+            return t
+        # new target
+        first = self.read_header(index * 2016)
+        last = self.read_header(index * 2016 + 2015)
+        bits = last.get('bits')
+        target = self.bits_to_target(bits)
+        nActualTimespan = last.get('timestamp') - first.get('timestamp')
+        nTargetTimespan = 14 * 24 * 60 * 60
+        nActualTimespan = max(nActualTimespan, nTargetTimespan // 4)
+        nActualTimespan = min(nActualTimespan, nTargetTimespan * 4)
+        new_target = min(MAX_TARGET, (target * nActualTimespan) // nTargetTimespan)
+        return new_target
 
-        height_range = range(max(0, height - POW_MEDIAN_BLOCK_SPAN),
-                             max(1, height))
-        median = []
-        for h in height_range:
-            header = self.read_header(h)
-            if not header:
-                for header in chain:
-                    if header.get('block_height') == h:
-                        break
-            assert header and header.get('block_height') == h
-            median.append(header.get('timestamp'))
+    def bits_to_target(self, bits):
+        bitsN = (bits >> 24) & 0xff
+        if not (bitsN >= 0x03 and bitsN <= 0x1d):
+            raise Exception("First part of bits should be in [0x03, 0x1d]")
+        bitsBase = bits & 0xffffff
+        if not (bitsBase >= 0x8000 and bitsBase <= 0x7fffff):
+            raise Exception("Second part of bits should be in [0x8000, 0x7fffff]")
+        return bitsBase << (8 * (bitsN-3))
 
-        median.sort()
-        return median[len(median)//2];
-
-    def get_target(self, height, chain=None):
-        if chain is None:
-            chain = []
-
-        if bitcoin.NetworkConstants.TESTNET:
-            return 0, 0
-
-        if height <= POW_AVERAGING_WINDOW:
-            return target_to_bits(POW_LIMIT), POW_LIMIT
-
-        height_range = range(max(0, height - POW_AVERAGING_WINDOW),
-                             max(1, height))
-        mean_target = 0
-        for h in height_range:
-            header = self.read_header(h)
-            if not header:
-                for header in chain:
-                    if header.get('block_height') == h:
-                        break
-            assert header and header.get('block_height') == h
-            mean_target += bits_to_target(header.get('bits'))
-        mean_target //= POW_AVERAGING_WINDOW
-
-        actual_timespan = self.get_median_time(height, chain) - \
-            self.get_median_time(height - POW_AVERAGING_WINDOW, chain)
-        actual_timespan = AVERAGING_WINDOW_TIMESPAN + \
-            int((actual_timespan - AVERAGING_WINDOW_TIMESPAN) / \
-                POW_DAMPING_FACTOR)
-        if actual_timespan < MIN_ACTUAL_TIMESPAN:
-            actual_timespan = MIN_ACTUAL_TIMESPAN
-        elif actual_timespan > MAX_ACTUAL_TIMESPAN:
-            actual_timespan = MAX_ACTUAL_TIMESPAN
-
-        next_target = mean_target // AVERAGING_WINDOW_TIMESPAN * actual_timespan
-
-        if next_target > POW_LIMIT:
-            next_target = POW_LIMIT
-
-        return target_to_bits(next_target), next_target
+    def target_to_bits(self, target):
+        c = ("%064x" % target)[2:]
+        while c[:2] == '00' and len(c) > 6:
+            c = c[2:]
+        bitsN, bitsBase = len(c) // 2, int('0x' + c[:6], 16)
+        if bitsBase >= 0x800000:
+            bitsN += 1
+            bitsBase >>= 8
+        return bitsN << 24 | bitsBase
 
     def can_connect(self, header, check_height=True):
+        if header is None:
+            return False
         height = header['block_height']
         if check_height and self.height() != height - 1:
+            #self.print_error("cannot connect at height", height)
             return False
         if height == 0:
-            return hash_header(header) == bitcoin.NetworkConstants.GENESIS
-        previous_header = self.read_header(height -1)
-        if not previous_header:
+            return hash_header(header) == constants.net.GENESIS
+        try:
+            prev_hash = self.get_hash(height - 1)
+        except:
             return False
-        prev_hash = hash_header(previous_header)
         if prev_hash != header.get('prev_block_hash'):
             return False
-        bits, target = self.get_target(height)
+        target = self.get_target(height // 2016 - 1)
         try:
-            self.verify_header(header, previous_header, bits, target)
-        except:
+            self.verify_header(header, prev_hash, target)
+        except BaseException as e:
             return False
         return True
 
@@ -396,5 +352,15 @@ class Blockchain(util.PrintError):
             self.save_chunk(idx, data)
             return True
         except BaseException as e:
-            self.print_error('verify_chunk failed', str(e))
+            self.print_error('verify_chunk %d failed'%idx, str(e))
             return False
+
+    def get_checkpoints(self):
+        # for each chunk, store the hash of the last block and the target after the chunk
+        cp = []
+        n = self.height() // 2016
+        for index in range(n):
+            h = self.get_hash((index+1) * 2016 -1)
+            target = self.get_target(index)
+            cp.append((h, target))
+        return cp

lib/coinchooser.py

@@ -25,7 +25,7 @@
 from collections import defaultdict, namedtuple
 from math import floor, log10
 
-from .bitcoin import sha256, COIN, TYPE_ADDRESS
+from .bitcoin import sha256, COIN, TYPE_ADDRESS, is_address
 from .transaction import Transaction
 from .util import NotEnoughFunds, PrintError
 
@@ -68,7 +68,12 @@ class PRNG:
             x[i], x[j] = x[j], x[i]
 
 
-Bucket = namedtuple('Bucket', ['desc', 'size', 'value', 'coins'])
+Bucket = namedtuple('Bucket',
+                    ['desc',
+                     'weight',      # as in BIP-141
+                     'value',       # in satoshis
+                     'coins',       # UTXOs
+                     'min_height']) # min block height where a coin was confirmed
 
 def strip_unneeded(bkts, sufficient_funds):
     '''Remove buckets that are unnecessary in achieving the spend amount'''
@@ -81,6 +86,8 @@ def strip_unneeded(bkts, sufficient_funds):
 
 class CoinChooserBase(PrintError):
 
+    enable_output_value_rounding = False
+
     def keys(self, coins):
         raise NotImplementedError
 
@@ -92,10 +99,10 @@ class CoinChooserBase(PrintError):
 
         def make_Bucket(desc, coins):
             weight = sum(Transaction.estimated_input_weight(coin)
-                       for coin in coins)
-            size = Transaction.virtual_size_from_weight(weight)
+                         for coin in coins)
             value = sum(coin['value'] for coin in coins)
-            return Bucket(desc, size, value, coins)
+            min_height = min(coin['height'] for coin in coins)
+            return Bucket(desc, weight, value, coins, min_height)
 
         return list(map(make_Bucket, buckets.keys(), buckets.values()))
 
@@ -126,7 +133,13 @@ class CoinChooserBase(PrintError):
         zeroes = [trailing_zeroes(i) for i in output_amounts]
         min_zeroes = min(zeroes)
         max_zeroes = max(zeroes)
-        zeroes = range(max(0, min_zeroes - 1), (max_zeroes + 1) + 1)
+
+        if n > 1:
+            zeroes = range(max(0, min_zeroes - 1), (max_zeroes + 1) + 1)
+        else:
+            # if there is only one change output, this will ensure that we aim
+            # to have one that is exactly as precise as the most precise output
+            zeroes = [min_zeroes]
 
         # Calculate change; randomize it a bit if using more than 1 output
         remaining = change_amount
@@ -141,8 +154,10 @@ class CoinChooserBase(PrintError):
             n -= 1
 
         # Last change output.  Round down to maximum precision but lose
-        # no more than 100 satoshis to fees (2dp)
-        N = pow(10, min(2, zeroes[0]))
+        # no more than 10**max_dp_to_round_for_privacy
+        # e.g. a max of 2 decimal places means losing 100 satoshis to fees
+        max_dp_to_round_for_privacy = 2 if self.enable_output_value_rounding else 0
+        N = pow(10, min(max_dp_to_round_for_privacy, zeroes[0]))
         amount = (remaining // N) * N
         amounts.append(amount)
 
@@ -168,27 +183,40 @@ class CoinChooserBase(PrintError):
 
     def make_tx(self, coins, outputs, change_addrs, fee_estimator,
                 dust_threshold):
-        '''Select unspent coins to spend to pay outputs.  If the change is
+        """Select unspent coins to spend to pay outputs.  If the change is
         greater than dust_threshold (after adding the change output to
         the transaction) it is kept, otherwise none is sent and it is
-        added to the transaction fee.'''
+        added to the transaction fee.
+
+        Note: fee_estimator expects virtual bytes
+        """
 
         # Deterministic randomness from coins
         utxos = [c['prevout_hash'] + str(c['prevout_n']) for c in coins]
         self.p = PRNG(''.join(sorted(utxos)))
 
-        # Copy the ouputs so when adding change we don't modify "outputs"
+        # Copy the outputs so when adding change we don't modify "outputs"
         tx = Transaction.from_io([], outputs[:])
-        # Size of the transaction with no inputs and no change
-        base_size = tx.estimated_size()
+        # Weight of the transaction with no inputs and no change
+        # Note: this will use legacy tx serialization. The only side effect
+        # should be that the marker and flag are excluded, which is
+        # compensated in get_tx_weight()
+        base_weight = tx.estimated_weight()
         spent_amount = tx.output_value()
 
+        def fee_estimator_w(weight):
+            return fee_estimator(Transaction.virtual_size_from_weight(weight))
+
+        def get_tx_weight(buckets):
+            total_weight = base_weight + sum(bucket.weight for bucket in buckets)
+            return total_weight
+
         def sufficient_funds(buckets):
             '''Given a list of buckets, return True if it has enough
             value to pay for the transaction'''
             total_input = sum(bucket.value for bucket in buckets)
-            total_size = sum(bucket.size for bucket in buckets) + base_size
-            return total_input >= spent_amount + fee_estimator(total_size)
+            total_weight = get_tx_weight(buckets)
+            return total_input >= spent_amount + fee_estimator_w(total_weight)
 
         # Collect the coins into buckets, choose a subset of the buckets
         buckets = self.bucketize_coins(coins)
@@ -196,11 +224,18 @@ class CoinChooserBase(PrintError):
                                       self.penalty_func(tx))
 
         tx.add_inputs([coin for b in buckets for coin in b.coins])
-        tx_size = base_size + sum(bucket.size for bucket in buckets)
+        tx_weight = get_tx_weight(buckets)
 
-        # This takes a count of change outputs and returns a tx fee;
-        # each pay-to-bitcoin-address output serializes as 34 bytes
-        fee = lambda count: fee_estimator(tx_size + count * 34)
+        # change is sent back to sending address unless specified
+        if not change_addrs:
+            change_addrs = [tx.inputs()[0]['address']]
+            # note: this is not necessarily the final "first input address"
+            # because the inputs had not been sorted at this point
+            assert is_address(change_addrs[0])
+
+        # This takes a count of change outputs and returns a tx fee
+        output_weight = 4 * Transaction.estimated_output_size(change_addrs[0])
+        fee = lambda count: fee_estimator_w(tx_weight + count * output_weight)
         change = self.change_outputs(tx, change_addrs, fee, dust_threshold)
         tx.add_outputs(change)
 
@@ -212,35 +247,14 @@ class CoinChooserBase(PrintError):
     def choose_buckets(self, buckets, sufficient_funds, penalty_func):
         raise NotImplemented('To be subclassed')
 
-class CoinChooserOldestFirst(CoinChooserBase):
-    '''Maximize transaction priority. Select the oldest unspent
-    transaction outputs in your wallet, that are sufficient to cover
-    the spent amount. Then, remove any unneeded inputs, starting with
-    the smallest in value.
-    '''
-
-    def keys(self, coins):
-        return [coin['prevout_hash'] + ':' + str(coin['prevout_n'])
-                for coin in coins]
-
-    def choose_buckets(self, buckets, sufficient_funds, penalty_func):
-        '''Spend the oldest buckets first.'''
-        # Unconfirmed coins are young, not old
-        adj_height = lambda height: 99999999 if height <= 0 else height
-        buckets.sort(key = lambda b: max(adj_height(coin['height'])
-                                         for coin in b.coins))
-        selected = []
-        for bucket in buckets:
-            selected.append(bucket)
-            if sufficient_funds(selected):
-                return strip_unneeded(selected, sufficient_funds)
-        else:
-            raise NotEnoughFunds()
 
 class CoinChooserRandom(CoinChooserBase):
 
-    def bucket_candidates(self, buckets, sufficient_funds):
+    def bucket_candidates_any(self, buckets, sufficient_funds):
         '''Returns a list of bucket sets.'''
+        if not buckets:
+            raise NotEnoughFunds()
+
         candidates = set()
 
         # Add all singletons
@@ -262,13 +276,49 @@ class CoinChooserRandom(CoinChooserBase):
                     candidates.add(tuple(sorted(permutation[:count + 1])))
                     break
             else:
+                # FIXME this assumes that the effective value of any bkt is >= 0
+                # we should make sure not to choose buckets with <= 0 eff. val.
                 raise NotEnoughFunds()
 
         candidates = [[buckets[n] for n in c] for c in candidates]
         return [strip_unneeded(c, sufficient_funds) for c in candidates]
 
+    def bucket_candidates_prefer_confirmed(self, buckets, sufficient_funds):
+        """Returns a list of bucket sets preferring confirmed coins.
+
+        Any bucket can be:
+        1. "confirmed" if it only contains confirmed coins; else
+        2. "unconfirmed" if it does not contain coins with unconfirmed parents
+        3. other: e.g. "unconfirmed parent" or "local"
+
+        This method tries to only use buckets of type 1, and if the coins there
+        are not enough, tries to use the next type but while also selecting
+        all buckets of all previous types.
+        """
+        conf_buckets = [bkt for bkt in buckets if bkt.min_height > 0]
+        unconf_buckets = [bkt for bkt in buckets if bkt.min_height == 0]
+        other_buckets = [bkt for bkt in buckets if bkt.min_height < 0]
+
+        bucket_sets = [conf_buckets, unconf_buckets, other_buckets]
+        already_selected_buckets = []
+
+        for bkts_choose_from in bucket_sets:
+            try:
+                def sfunds(bkts):
+                    return sufficient_funds(already_selected_buckets + bkts)
+
+                candidates = self.bucket_candidates_any(bkts_choose_from, sfunds)
+                break
+            except NotEnoughFunds:
+                already_selected_buckets += bkts_choose_from
+        else:
+            raise NotEnoughFunds()
+
+        candidates = [(already_selected_buckets + c) for c in candidates]
+        return [strip_unneeded(c, sufficient_funds) for c in candidates]
+
     def choose_buckets(self, buckets, sufficient_funds, penalty_func):
-        candidates = self.bucket_candidates(buckets, sufficient_funds)
+        candidates = self.bucket_candidates_prefer_confirmed(buckets, sufficient_funds)
         penalties = [penalty_func(cand) for cand in candidates]
         winner = candidates[penalties.index(min(penalties))]
         self.print_error("Bucket sets:", len(buckets))
@@ -276,14 +326,15 @@ class CoinChooserRandom(CoinChooserBase):
         return winner
 
 class CoinChooserPrivacy(CoinChooserRandom):
-    '''Attempts to better preserve user privacy.  First, if any coin is
-    spent from a user address, all coins are.  Compared to spending
-    from other addresses to make up an amount, this reduces
+    """Attempts to better preserve user privacy.
+    First, if any coin is spent from a user address, all coins are.
+    Compared to spending from other addresses to make up an amount, this reduces
     information leakage about sender holdings.  It also helps to
     reduce blockchain UTXO bloat, and reduce future privacy loss that
-    would come from reusing that address' remaining UTXOs.  Second, it
-    penalizes change that is quite different to the sent amount.
-    Third, it penalizes change that is too big.'''
+    would come from reusing that address' remaining UTXOs.
+    Second, it penalizes change that is quite different to the sent amount.
+    Third, it penalizes change that is too big.
+    """
 
     def keys(self, coins):
         return [coin['address'] for coin in coins]
@@ -296,6 +347,7 @@ class CoinChooserPrivacy(CoinChooserRandom):
         def penalty(buckets):
             badness = len(buckets) - 1
             total_input = sum(bucket.value for bucket in buckets)
+            # FIXME "change" here also includes fees
             change = float(total_input - spent_amount)
             # Penalize change not roughly in output range
             if change < min_change:
@@ -309,15 +361,18 @@ class CoinChooserPrivacy(CoinChooserRandom):
         return penalty
 
 
-COIN_CHOOSERS = {'Priority': CoinChooserOldestFirst,
-                 'Privacy': CoinChooserPrivacy}
+COIN_CHOOSERS = {
+    'Privacy': CoinChooserPrivacy,
+}
 
 def get_name(config):
     kind = config.get('coin_chooser')
     if not kind in COIN_CHOOSERS:
-        kind = 'Priority'
+        kind = 'Privacy'
     return kind
 
 def get_coin_chooser(config):
     klass = COIN_CHOOSERS[get_name(config)]
-    return klass()
+    coinchooser = klass()
+    coinchooser.enable_output_value_rounding = config.get('coin_chooser_output_rounding', False)
+    return coinchooser

lib/commands.py

@@ -34,7 +34,7 @@ from functools import wraps
 from decimal import Decimal
 
 from .import util
-from .util import bfh, bh2u, format_satoshis, json_decode
+from .util import bfh, bh2u, format_satoshis, json_decode, print_error, json_encode
 from .import bitcoin
 from .bitcoin import is_address,  hash_160, COIN, TYPE_ADDRESS
 from .i18n import _
@@ -81,8 +81,8 @@ def command(s):
             wallet = args[0].wallet
             password = kwargs.get('password')
             if c.requires_wallet and wallet is None:
-                raise BaseException("wallet not loaded. Use 'electrum-zcash daemon load_wallet'")
-            if c.requires_password and password is None and wallet.storage.get('use_encryption'):
+                raise Exception("wallet not loaded. Use 'electrum-zcash daemon load_wallet'")
+            if c.requires_password and password is None and wallet.has_password():
                 return {'error': 'Password required' }
             return func(*args, **kwargs)
         return func_wrapper
@@ -125,7 +125,7 @@ class Commands:
     @command('')
     def create(self):
         """Create a new wallet"""
-        raise BaseException('Not a JSON-RPC command')
+        raise Exception('Not a JSON-RPC command')
 
     @command('wn')
     def restore(self, text):
@@ -133,11 +133,13 @@ class Commands:
         public key, a master private key, a list of Zcash addresses
         or Zcash private keys. If you want to be prompted for your
         seed, type '?' or ':' (concealed) """
-        raise BaseException('Not a JSON-RPC command')
+        raise Exception('Not a JSON-RPC command')
 
     @command('wp')
     def password(self, password=None, new_password=None):
         """Change wallet password. """
+        if self.wallet.storage.is_encrypted_with_hw_device() and new_password:
+            raise Exception("Can't change the password of a wallet encrypted with a hw device.")
         b = self.wallet.storage.is_encrypted()
         self.wallet.update_password(password, new_password, b)
         self.wallet.storage.write()
@@ -148,34 +150,38 @@ class Commands:
         """Return a configuration variable. """
         return self.config.get(key)
 
+    @classmethod
+    def _setconfig_normalize_value(cls, key, value):
+        if key not in ('rpcuser', 'rpcpassword'):
+            value = json_decode(value)
+            try:
+                value = ast.literal_eval(value)
+            except:
+                pass
+        return value
+
     @command('')
     def setconfig(self, key, value):
         """Set a configuration variable. 'value' may be a string or a Python expression."""
-        if key not in ('rpcuser', 'rpcpassword'):
-            value = json_decode(value)
+        value = self._setconfig_normalize_value(key, value)
         self.config.set_key(key, value)
         return True
 
     @command('')
-    def make_seed(self, nbits=132, entropy=1, language=None):
+    def make_seed(self, nbits=132, language=None):
         """Create a seed"""
         from .mnemonic import Mnemonic
         t = 'standard'
-        s = Mnemonic(language).make_seed(t, nbits, custom_entropy=entropy)
+        s = Mnemonic(language).make_seed(t, nbits)
         return s
 
-    @command('')
-    def check_seed(self, seed, entropy=1, language=None):
-        """Check that a seed was generated with given entropy"""
-        from .mnemonic import Mnemonic
-        return Mnemonic(language).check_seed(seed, entropy)
-
     @command('n')
     def getaddresshistory(self, address):
         """Return the transaction history of any address. Note: This is a
         walletless server query, results are not checked by SPV.
         """
-        return self.network.synchronous_get(('blockchain.address.get_history', [address]))
+        sh = bitcoin.address_to_scripthash(address)
+        return self.network.synchronous_get(('blockchain.scripthash.get_history', [sh]))
 
     @command('w')
     def listunspent(self):
@@ -192,7 +198,8 @@ class Commands:
         """Returns the UTXO list of any address. Note: This
         is a walletless server query, results are not checked by SPV.
         """
-        return self.network.synchronous_get(('blockchain.address.listunspent', [address]))
+        sh = bitcoin.address_to_scripthash(address)
+        return self.network.synchronous_get(('blockchain.scripthash.listunspent', [sh]))
 
     @command('')
     def serialize(self, jsontx):
@@ -203,7 +210,7 @@ class Commands:
         keypairs = {}
         inputs = jsontx.get('inputs')
         outputs = jsontx.get('outputs')
-        locktime = jsontx.get('locktime', 0)
+        locktime = jsontx.get('lockTime', 0)
         for txin in inputs:
             if txin.get('output'):
                 prevout_hash, prevout_n = txin['output'].split(':')
@@ -314,20 +321,12 @@ class Commands:
         """Return the balance of any address. Note: This is a walletless
         server query, results are not checked by SPV.
         """
-        out = self.network.synchronous_get(('blockchain.address.get_balance', [address]))
+        sh = bitcoin.address_to_scripthash(address)
+        out = self.network.synchronous_get(('blockchain.scripthash.get_balance', [sh]))
         out["confirmed"] =  str(Decimal(out["confirmed"])/COIN)
         out["unconfirmed"] =  str(Decimal(out["unconfirmed"])/COIN)
         return out
 
-    @command('n')
-    def getproof(self, address):
-        """Get Merkle branch of an address in the UTXO set"""
-        p = self.network.synchronous_get(('blockchain.address.get_proof', [address]))
-        out = []
-        for i,s in p:
-            out.append(i)
-        return out
-
     @command('n')
     def getmerkle(self, txid, height):
         """Get Merkle branch of a transaction included in a block. Electrum-Zcash
@@ -341,7 +340,7 @@ class Commands:
 
     @command('')
     def version(self):
-        """Return the version of electrum-zcash."""
+        """Return the version of Electrum-Zcash."""
         from .version import ELECTRUM_VERSION
         return ELECTRUM_VERSION
 
@@ -378,7 +377,7 @@ class Commands:
             return None
         out = self.wallet.contacts.resolve(x)
         if out.get('type') == 'openalias' and self.nocheck is False and out.get('validated') is False:
-            raise BaseException('cannot verify alias', x)
+            raise Exception('cannot verify alias', x)
         return out['address']
 
     @command('n')
@@ -444,46 +443,20 @@ class Commands:
         return tx.as_dict()
 
     @command('w')
-    def history(self):
+    def history(self, year=None, show_addresses=False, show_fiat=False):
         """Wallet history. Returns the transaction history of your wallet."""
-        balance = 0
-        out = []
-        for item in self.wallet.get_history():
-            tx_hash, height, conf, timestamp, value, balance = item
-            if timestamp:
-                date = datetime.datetime.fromtimestamp(timestamp).isoformat(' ')[:-3]
-            else:
-                date = "----"
-            label = self.wallet.get_label(tx_hash)
-            tx = self.wallet.transactions.get(tx_hash)
-            tx.deserialize()
-            input_addresses = []
-            output_addresses = []
-            for x in tx.inputs():
-                if x['type'] == 'coinbase': continue
-                addr = x.get('address')
-                if addr == None: continue
-                if addr == "(pubkey)":
-                    prevout_hash = x.get('prevout_hash')
-                    prevout_n = x.get('prevout_n')
-                    _addr = self.wallet.find_pay_to_pubkey_address(prevout_hash, prevout_n)
-                    if _addr:
-                        addr = _addr
-                input_addresses.append(addr)
-            for addr, v in tx.get_outputs():
-                output_addresses.append(addr)
-            out.append({
-                'txid': tx_hash,
-                'timestamp': timestamp,
-                'date': date,
-                'input_addresses': input_addresses,
-                'output_addresses': output_addresses,
-                'label': label,
-                'value': str(Decimal(value)/COIN) if value is not None else None,
-                'height': height,
-                'confirmations': conf
-            })
-        return out
+        kwargs = {'show_addresses': show_addresses}
+        if year:
+            import time
+            start_date = datetime.datetime(year, 1, 1)
+            end_date = datetime.datetime(year+1, 1, 1)
+            kwargs['from_timestamp'] = time.mktime(start_date.timetuple())
+            kwargs['to_timestamp'] = time.mktime(end_date.timetuple())
+        if show_fiat:
+            from .exchange_rate import FxThread
+            fx = FxThread(self.config, None)
+            kwargs['fx'] = fx
+        return json_encode(self.wallet.get_full_history(**kwargs))
 
     @command('w')
     def setlabel(self, key, label):
@@ -545,7 +518,7 @@ class Commands:
             if raw:
                 tx = Transaction(raw)
             else:
-                raise BaseException("Unknown transaction")
+                raise Exception("Unknown transaction")
         return tx.as_dict()
 
     @command('')
@@ -574,7 +547,7 @@ class Commands:
         """Return a payment request"""
         r = self.wallet.get_payment_request(key, self.config)
         if not r:
-            raise BaseException("Request not found")
+            raise Exception("Request not found")
         return self._format_request(r)
 
     #@command('w')
@@ -612,7 +585,7 @@ class Commands:
     @command('w')
     def addrequest(self, amount, memo='', expiration=None, force=False):
         """Create a payment request, using the first unused address of the wallet.
-        The address will be condidered as used after this operation.
+        The address will be considered as used after this operation.
         If no payment is received, the address will be considered as unused if the payment request is deleted from the wallet."""
         addr = self.wallet.get_unused_address()
         if addr is None:
@@ -627,12 +600,21 @@ class Commands:
         out = self.wallet.get_payment_request(addr, self.config)
         return self._format_request(out)
 
+    @command('w')
+    def addtransaction(self, tx):
+        """ Add a transaction to the wallet history """
+        tx = Transaction(tx)
+        if not self.wallet.add_transaction(tx.txid(), tx):
+            return False
+        self.wallet.save_transactions()
+        return tx.txid()
+
     @command('wp')
     def signrequest(self, address, password=None):
         "Sign payment request with an OpenAlias"
         alias = self.config.get('alias')
         if not alias:
-            raise BaseException('No alias in your configuration')
+            raise Exception('No alias in your configuration')
         alias_addr = self.wallet.contacts.resolve(alias)['address']
         self.wallet.sign_payment_request(address, alias, alias_addr, password)
 
@@ -649,18 +631,20 @@ class Commands:
 
     @command('n')
     def notify(self, address, URL):
-        """Watch an address. Everytime the address changes, a http POST is sent to the URL."""
+        """Watch an address. Every time the address changes, a http POST is sent to the URL."""
         def callback(x):
             import urllib.request
             headers = {'content-type':'application/json'}
             data = {'address':address, 'status':x.get('result')}
+            serialized_data = util.to_bytes(json.dumps(data))
             try:
-                req = urllib.request.Request(URL, json.dumps(data), headers)
+                req = urllib.request.Request(URL, serialized_data, headers)
                 response_stream = urllib.request.urlopen(req, timeout=5)
                 util.print_error('Got Response for %s' % address)
             except BaseException as e:
                 util.print_error(str(e))
-        self.network.send([('blockchain.address.subscribe', [address])], callback)
+        h = self.network.addr_to_scripthash(address)
+        self.network.send([('blockchain.scripthash.subscribe', [h])], callback)
         return True
 
     @command('wn')
@@ -668,6 +652,12 @@ class Commands:
         """ return wallet synchronization status """
         return self.wallet.is_up_to_date()
 
+    @command('n')
+    def getfeerate(self):
+        """Return current optimal fee rate per kilobyte, according
+        to config settings (static/dynamic)"""
+        return self.config.fee_per_kb()
+
     @command('')
     def help(self):
         # for the python console
@@ -708,7 +698,6 @@ command_options = {
     'from_addr':   ("-F", "Source address (must be a wallet address; use sweep to spend from non-wallet address)."),
     'change_addr': ("-c", "Change address. Default is a spare address, or the source address if it's not in the wallet"),
     'nbits':       (None, "Number of bits of entropy"),
-    'entropy':     (None, "Custom entropy"),
     'language':    ("-L", "Default language for wordlist"),
     'privkey':     (None, "Private key. Set to '?' to get a prompt."),
     'unsigned':    ("-u", "Do not sign transaction"),
@@ -721,6 +710,9 @@ command_options = {
     'pending':     (None, "Show only pending requests."),
     'expired':     (None, "Show only expired requests."),
     'paid':        (None, "Show only paid requests."),
+    'show_addresses': (None, "Show input and output addresses"),
+    'show_fiat':   (None, "Show fiat value of transactions"),
+    'year':        (None, "Show history for a given year"),
 }
 
 
@@ -731,7 +723,7 @@ arg_types = {
     'num': int,
     'nbits': int,
     'imax': int,
-    'entropy': int,
+    'year': int,
     'tx': tx_from_str,
     'pubkeys': json_loads,
     'jsontx': json_loads,
@@ -794,7 +786,7 @@ def subparser_call(self, parser, namespace, values, option_string=None):
         parser = self._name_parser_map[parser_name]
     except KeyError:
         tup = parser_name, ', '.join(self._name_parser_map)
-        msg = _('unknown parser %r (choices: %s)') % tup
+        msg = _('unknown parser {!r} (choices: {})').format(*tup)
         raise ArgumentError(self, msg)
     # parse all the remaining options into the namespace
     # store any unrecognized options on the object, so that the top
@@ -808,7 +800,7 @@ argparse._SubParsersAction.__call__ = subparser_call
 
 
 def add_network_options(parser):
-    parser.add_argument("-1", "--oneserver", action="store_true", dest="oneserver", default=False, help="connect to one server only")
+    parser.add_argument("-1", "--oneserver", action="store_true", dest="oneserver", default=None, help="connect to one server only")
     parser.add_argument("-s", "--server", dest="server", default=None, help="set server host:port:protocol, where protocol is either t (tcp) or s (ssl)")
     parser.add_argument("-p", "--proxy", dest="proxy", default=None, help="set proxy [type:]host[:port], where type is socks4,socks5 or http")
 
@@ -819,6 +811,7 @@ def add_global_options(parser):
     group.add_argument("-P", "--portable", action="store_true", dest="portable", default=False, help="Use local 'electrum_data' directory")
     group.add_argument("-w", "--wallet", dest="wallet_path", help="wallet path")
     group.add_argument("--testnet", action="store_true", dest="testnet", default=False, help="Use Testnet")
+    group.add_argument("--regtest", action="store_true", dest="regtest", default=False, help="Use Regtest")
 
 def get_parser():
     # create main parser

lib/constants.py

@@ -0,0 +1,105 @@
+# -*- coding: utf-8 -*-
+#
+# Electrum - lightweight Bitcoin client
+# Copyright (C) 2018 The Electrum developers
+#
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation files
+# (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge,
+# publish, distribute, sublicense, and/or sell copies of the Software,
+# and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import os
+import json
+
+
+def read_json(filename, default):
+    path = os.path.join(os.path.dirname(__file__), filename)
+    try:
+        with open(path, 'r') as f:
+            r = json.loads(f.read())
+    except:
+        r = default
+    return r
+
+
+class BitcoinMainnet:
+
+    TESTNET = False
+    WIF_PREFIX = 0x80
+    ADDRTYPE_P2PKH = bytes.fromhex('1CB8')
+    ADDRTYPE_P2SH = bytes.fromhex('1CBD')
+    GENESIS = "00040fe8ec8471911baa1db1266ea15dd06b4a8a5c453883c000b031973dce08"
+    DEFAULT_PORTS = {'t': '50021', 's': '50022'}
+    DEFAULT_SERVERS = read_json('servers.json', {})
+    CHECKPOINTS = []
+
+    XPRV_HEADERS = {
+        'standard':    0x0488ade4,  # xprv
+    }
+    XPUB_HEADERS = {
+        'standard':    0x0488b21e,  # xpub
+    }
+    DRKV_HEADER = 0x02fe52f8  # drkv
+    DRKP_HEADER = 0x02fe52cc  # drkp
+
+
+class BitcoinTestnet:
+
+    TESTNET = True
+    WIF_PREFIX = 0xEF
+    ADDRTYPE_P2PKH = bytes.fromhex('1D25')
+    ADDRTYPE_P2SH = bytes.fromhex('1CBA')
+    GENESIS = "05a60a92d99d85997cce3b87616c089f6124d7342af37106edc76126334a2c38"
+    DEFAULT_PORTS = {'t': '51021', 's': '51022'}
+    DEFAULT_SERVERS = read_json('servers_testnet.json', {})
+    CHECKPOINTS = []
+
+    XPRV_HEADERS = {
+        'standard':    0x04358394,  # tprv
+    }
+    XPUB_HEADERS = {
+        'standard':    0x043587cf,  # tpub
+    }
+    DRKV_HEADER = 0x3a8061a0  # DRKV
+    DRKP_HEADER = 0x3a805837  # DRKP
+
+
+class BitcoinRegtest(BitcoinTestnet):
+
+    GENESIS = "029f11d80ef9765602235e1bc9727e3eb6ba20839319f761fee920d63401e327"
+    DEFAULT_SERVERS = read_json('servers_regtest.json', {})
+    CHECKPOINTS = []
+
+
+# don't import net directly, import the module instead (so that net is singleton)
+net = BitcoinMainnet
+
+
+def set_mainnet():
+    global net
+    net = BitcoinMainnet
+
+
+def set_testnet():
+    global net
+    net = BitcoinTestnet
+
+
+def set_regtest():
+    global net
+    net = BitcoinRegtest

lib/contacts.py

@@ -22,10 +22,14 @@
 # SOFTWARE.
 import re
 import dns
+from dns.exception import DNSException
 import json
+import traceback
+import sys
 
 from . import bitcoin
 from . import dnssec
+from .util import export_meta, import_meta, print_error, to_string
 
 
 class Contacts(dict):
@@ -48,14 +52,15 @@ class Contacts(dict):
         self.storage.put('contacts', dict(self))
 
     def import_file(self, path):
-        try:
-            with open(path, 'r') as f:
-                d = self._validate(json.loads(f.read()))
-        except:
-            return
-        self.update(d)
+        import_meta(path, self._validate, self.load_meta)
+
+    def load_meta(self, data):
+        self.update(data)
         self.save()
 
+    def export_file(self, filename):
+        export_meta(self, filename)
+
     def __setitem__(self, key, value):
         dict.__setitem__(self, key, value)
         self.save()
@@ -92,10 +97,14 @@ class Contacts(dict):
     def resolve_openalias(self, url):
         # support email-style addresses, per the OA standard
         url = url.replace('@', '.')
-        records, validated = dnssec.query(url, dns.rdatatype.TXT)
+        try:
+            records, validated = dnssec.query(url, dns.rdatatype.TXT)
+        except DNSException as e:
+            print_error('Error resolving openalias: ', str(e))
+            return None
         prefix = 'zcash'
         for record in records:
-            string = record.strings[0]
+            string = to_string(record.strings[0], 'utf8')
             if string.startswith('oa1:' + prefix):
                 address = self.find_regex(string, r'recipient_address=([A-Za-z0-9]+)')
                 name = self.find_regex(string, r'recipient_name=([^;]+)')
@@ -113,13 +122,13 @@ class Contacts(dict):
             return None
             
     def _validate(self, data):
-        for k,v in list(data.items()):
+        for k, v in list(data.items()):
             if k == 'contacts':
                 return self._validate(v)
             if not bitcoin.is_address(k):
                 data.pop(k)
             else:
-                _type,_ = v
+                _type, _ = v
                 if _type != 'address':
                     data.pop(k)
         return data

lib/currencies.json

@@ -6,7 +6,6 @@
         "BTC"
     ],
     "CoinMarketCap": [
-        "BTC",
         "USD"
     ]
 }

lib/daemon.py

@@ -25,6 +25,8 @@
 import ast
 import os
 import time
+import traceback
+import sys
 
 # from jsonrpc import JSONRPCResponseManager
 import jsonrpclib
@@ -58,7 +60,7 @@ def get_fd_or_server(config):
     lockfile = get_lockfile(config)
     while True:
         try:
-            return os.open(lockfile, os.O_CREAT | os.O_EXCL | os.O_WRONLY), None
+            return os.open(lockfile, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644), None
         except OSError:
             pass
         server = get_server(config)
@@ -121,13 +123,12 @@ class Daemon(DaemonThread):
         self.config = config
         if config.get('offline'):
             self.network = None
-            self.fx = None
         else:
             self.network = Network(config)
             self.network.start()
-            self.fx = FxThread(config, self.network)
+        self.fx = FxThread(config, self.network)
+        if self.network:
             self.network.add_jobs([self.fx])
-
         self.gui = None
         self.wallets = {}
         # Setup JSONRPC server
@@ -172,8 +173,9 @@ class Daemon(DaemonThread):
         elif sub == 'load_wallet':
             path = config.get_wallet_path()
             wallet = self.load_wallet(path, config.get('password'))
-            self.cmd_runner.wallet = wallet
-            response = True
+            if wallet is not None:
+                self.cmd_runner.wallet = wallet
+            response = wallet is not None
         elif sub == 'close_wallet':
             path = config.get_wallet_path()
             if path in self.wallets:
@@ -184,6 +186,9 @@ class Daemon(DaemonThread):
         elif sub == 'status':
             if self.network:
                 p = self.network.get_parameters()
+                current_wallet = self.cmd_runner.wallet
+                current_wallet_path = current_wallet.storage.path \
+                                      if current_wallet else None
                 response = {
                     'path': self.network.config.path,
                     'server': p[0],
@@ -195,6 +200,7 @@ class Daemon(DaemonThread):
                     'version': ELECTRUM_VERSION,
                     'wallets': {k: w.is_up_to_date()
                                 for k, w in self.wallets.items()},
+                    'current_wallet': current_wallet_path,
                     'fee_per_kb': self.config.fee_per_kb(),
                 }
             else:
@@ -301,4 +307,8 @@ class Daemon(DaemonThread):
             gui_name = 'qt'
         gui = __import__('electrum_zcash_gui.' + gui_name, fromlist=['electrum_zcash_gui'])
         self.gui = gui.ElectrumGui(config, self, plugins)
-        self.gui.main()
+        try:
+            self.gui.main()
+        except BaseException as e:
+            traceback.print_exc(file=sys.stdout)
+            # app will exit now

lib/dnssec.py

@@ -199,7 +199,7 @@ def check_query(ns, sub, _type, keys):
     elif answer[1].rdtype == dns.rdatatype.RRSIG:
         rrset, rrsig = answer
     else:
-        raise BaseException('No signature set in record')
+        raise Exception('No signature set in record')
     if keys is None:
         keys = {dns.name.from_text(sub):rrset}
     dns.dnssec.validate(rrset, rrsig, keys)
@@ -248,7 +248,7 @@ def get_and_validate(ns, url, _type):
                 continue
             break
         else:
-            raise BaseException("DS does not match DNSKEY")
+            raise Exception("DS does not match DNSKEY")
         # set key for next iteration
         keys = {name: rrset}
     # get TXT record (signed by zone)

lib/exchange_rate.py

@@ -2,6 +2,8 @@ from datetime import datetime
 import inspect
 import requests
 import sys
+import os
+import json
 from threading import Thread
 import time
 import csv
@@ -39,7 +41,7 @@ class ExchangeBase(PrintError):
     def get_json(self, site, get_string):
         # APIs must have https
         url = ''.join(['https://', site, get_string])
-        response = requests.request('GET', url, headers={'User-Agent' : 'Electrum-Zcash'})
+        response = requests.request('GET', url, headers={'User-Agent' : 'Electrum-Zcash'}, timeout=10)
         return response.json()
 
     def get_csv(self, site, get_string):
@@ -65,28 +67,54 @@ class ExchangeBase(PrintError):
         t.setDaemon(True)
         t.start()
 
-    def get_historical_rates_safe(self, ccy):
+    def read_historical_rates(self, ccy, cache_dir):
+        filename = os.path.join(cache_dir, self.name() + '_'+ ccy)
+        if os.path.exists(filename):
+            timestamp = os.stat(filename).st_mtime
+            try:
+                with open(filename, 'r', encoding='utf-8') as f:
+                    h = json.loads(f.read())
+                h['timestamp'] = timestamp
+            except:
+                h = None
+        else:
+            h = None
+        if h:
+            self.history[ccy] = h
+            self.on_history()
+        return h
+
+    def get_historical_rates_safe(self, ccy, cache_dir):
         try:
             self.print_error("requesting fx history for", ccy)
-            self.history[ccy] = self.historical_rates(ccy)
+            h = self.request_history(ccy)
             self.print_error("received fx history for", ccy)
-            self.on_history()
         except BaseException as e:
             self.print_error("failed fx history:", e)
+            return
+        filename = os.path.join(cache_dir, self.name() + '_' + ccy)
+        with open(filename, 'w', encoding='utf-8') as f:
+            f.write(json.dumps(h))
+        h['timestamp'] = time.time()
+        self.history[ccy] = h
+        self.on_history()
 
-    def get_historical_rates(self, ccy):
-        result = self.history.get(ccy)
-        if not result and ccy in self.history_ccys():
-            t = Thread(target=self.get_historical_rates_safe, args=(ccy,))
+    def get_historical_rates(self, ccy, cache_dir):
+        if ccy not in self.history_ccys():
+            return
+        h = self.history.get(ccy)
+        if h is None:
+            h = self.read_historical_rates(ccy, cache_dir)
+        if h is None or h['timestamp'] < time.time() - 24*3600:
+            t = Thread(target=self.get_historical_rates_safe, args=(ccy, cache_dir))
             t.setDaemon(True)
             t.start()
-        return result
 
     def history_ccys(self):
         return []
 
     def historical_rate(self, ccy, d_t):
-        return self.history.get(ccy, {}).get(d_t.strftime('%Y-%m-%d'))
+        return self.history.get(ccy, {}).get(d_t.strftime('%Y-%m-%d'), 'NaN')
 
     def get_currencies(self):
         rates = self.get_rates('')
@@ -96,7 +124,7 @@ class ExchangeBase(PrintError):
 class Bittrex(ExchangeBase):
     def get_rates(self, ccy):
         json = self.get_json('bittrex.com',
-                             '/api/v1.1/public/getticker?market=BTC-DASH')
+                             '/api/v1.1/public/getticker?market=BTC-ZEC')
         quote_currencies = {}
         if not json.get('success', False):
             return quote_currencies
@@ -109,21 +137,20 @@ class Poloniex(ExchangeBase):
     def get_rates(self, ccy):
         json = self.get_json('poloniex.com', '/public?command=returnTicker')
         quote_currencies = {}
-        dash_ticker = json.get('BTC_DASH')
-        quote_currencies['BTC'] = Decimal(dash_ticker['last'])
+        zcash_ticker = json.get('BTC_ZEC')
+        quote_currencies['BTC'] = Decimal(zcash_ticker['last'])
         return quote_currencies
 
 
 class CoinMarketCap(ExchangeBase):
     def get_rates(self, ccy):
-        json = self.get_json('api.coinmarketcap.com', '/v1/ticker/dash/')
+        json = self.get_json('api.coinmarketcap.com', '/v1/ticker/1437/')
         quote_currencies = {}
         if not isinstance(json, list):
             return quote_currencies
         json = json[0]
         for ccy, key in [
             ('USD', 'price_usd'),
-            ('BTC', 'price_btc'),
         ]:
             quote_currencies[ccy] = Decimal(json[key])
         return quote_currencies
@@ -141,7 +168,7 @@ def get_exchanges_and_currencies():
     import os, json
     path = os.path.join(os.path.dirname(__file__), 'currencies.json')
     try:
-        with open(path, 'r') as f:
+        with open(path, 'r', encoding='utf-8') as f:
             return json.loads(f.read())
     except:
         pass
@@ -154,9 +181,11 @@ def get_exchanges_and_currencies():
         exchange = klass(None, None)
         try:
             d[name] = exchange.get_currencies()
+            print(name, "ok")
         except:
+            print(name, "error")
             continue
-    with open(path, 'w') as f:
+    with open(path, 'w', encoding='utf-8') as f:
         f.write(json.dumps(d, indent=4, sort_keys=True))
     return d
 
@@ -185,7 +214,10 @@ class FxThread(ThreadJob):
         self.history_used_spot = False
         self.ccy_combo = None
         self.hist_checkbox = None
+        self.cache_dir = os.path.join(config.path, 'cache')
         self.set_exchange(self.config_exchange())
+        if not os.path.exists(self.cache_dir):
+            os.mkdir(self.cache_dir)
 
     def get_currencies(self, h):
         d = get_exchanges_by_ccy(h)
@@ -208,7 +240,7 @@ class FxThread(ThreadJob):
         # This runs from the plugins thread which catches exceptions
         if self.is_enabled():
             if self.timeout ==0 and self.show_history():
-                self.exchange.get_historical_rates(self.ccy)
+                self.exchange.get_historical_rates(self.ccy, self.cache_dir)
             if self.timeout <= time.time():
                 self.timeout = time.time() + 150
                 self.exchange.update(self.ccy)
@@ -225,6 +257,12 @@ class FxThread(ThreadJob):
     def set_history_config(self, b):
         self.config.set_key('history_rates', bool(b))
 
+    def get_history_capital_gains_config(self):
+        return bool(self.config.get('history_rates_capital_gains', False))
+
+    def set_history_capital_gains_config(self, b):
+        self.config.set_key('history_rates_capital_gains', bool(b))
+
     def get_fiat_address_config(self):
         return bool(self.config.get('fiat_address'))
 
@@ -256,45 +294,65 @@ class FxThread(ThreadJob):
         # A new exchange means new fx quotes, initially empty.  Force
         # a quote refresh
         self.timeout = 0
+        self.exchange.read_historical_rates(self.ccy, self.cache_dir)
 
     def on_quotes(self):
-        self.network.trigger_callback('on_quotes')
+        if self.network:
+            self.network.trigger_callback('on_quotes')
 
     def on_history(self):
-        self.network.trigger_callback('on_history')
+        if self.network:
+            self.network.trigger_callback('on_history')
 
     def exchange_rate(self):
         '''Returns None, or the exchange rate as a Decimal'''
         rate = self.exchange.quotes.get(self.ccy)
-        if rate:
-            return Decimal(rate)
+        if rate is None:
+            return Decimal('NaN')
+        return Decimal(rate)
+
+    def format_amount(self, btc_balance):
+        rate = self.exchange_rate()
+        return '' if rate.is_nan() else "%s" % self.value_str(btc_balance, rate)
 
     def format_amount_and_units(self, btc_balance):
         rate = self.exchange_rate()
-        return '' if rate is None else "%s %s" % (self.value_str(btc_balance, rate), self.ccy)
+        return '' if rate.is_nan() else "%s %s" % (self.value_str(btc_balance, rate), self.ccy)
 
     def get_fiat_status_text(self, btc_balance, base_unit, decimal_point):
         rate = self.exchange_rate()
-        return _("  (No FX rate available)") if rate is None else " 1 %s~%s %s" % (base_unit,
+        return _("  (No FX rate available)") if rate.is_nan() else " 1 %s~%s %s" % (base_unit,
             self.value_str(COIN / (10**(8 - decimal_point)), rate), self.ccy)
 
+    def fiat_value(self, satoshis, rate):
+        return Decimal('NaN') if satoshis is None else Decimal(satoshis) / COIN * Decimal(rate)
+
     def value_str(self, satoshis, rate):
-        if satoshis is None:  # Can happen with incomplete history
-            return _("Unknown")
-        if rate:
-            value = Decimal(satoshis) / COIN * Decimal(rate)
-            return "%s" % (self.ccy_amount_str(value, True))
-        return _("No data")
+        return self.format_fiat(self.fiat_value(satoshis, rate))
+
+    def format_fiat(self, value):
+        if value.is_nan():
+            return _("No data")
+        return "%s" % (self.ccy_amount_str(value, True))
 
     def history_rate(self, d_t):
+        if d_t is None:
+            return Decimal('NaN')
         rate = self.exchange.historical_rate(self.ccy, d_t)
         # Frequently there is no rate for today, until tomorrow :)
         # Use spot quotes in that case
-        if rate is None and (datetime.today().date() - d_t.date()).days <= 2:
-            rate = self.exchange.quotes.get(self.ccy)
+        if rate == 'NaN' and (datetime.today().date() - d_t.date()).days <= 2:
+            rate = self.exchange.quotes.get(self.ccy, 'NaN')
             self.history_used_spot = True
-        return rate
+        return Decimal(rate)
 
     def historical_value_str(self, satoshis, d_t):
-        rate = self.history_rate(d_t)
-        return self.value_str(satoshis, rate)
+        return self.format_fiat(self.historical_value(satoshis, d_t))
+
+    def historical_value(self, satoshis, d_t):
+        return self.fiat_value(satoshis, self.history_rate(d_t))
+
+    def timestamp_rate(self, timestamp):
+        from electrum_zcash.util import timestamp_to_datetime
+        date = timestamp_to_datetime(timestamp)
+        return self.history_rate(date)

lib/interface.py

@@ -43,7 +43,7 @@ from . import pem
 
 
 def Connection(server, queue, config_path):
-    """Makes asynchronous connections to a remote electrum server.
+    """Makes asynchronous connections to a remote Electrum server.
     Returns the running thread that is making the connection.
 
     Once the thread has connected, it finishes, placing a tuple on the
@@ -144,7 +144,7 @@ class TcpConnection(threading.Thread, util.PrintError):
                     context = self.get_ssl_context(cert_reqs=ssl.CERT_REQUIRED, ca_certs=ca_path)
                     s = context.wrap_socket(s, do_handshake_on_connect=True)
                 except ssl.SSLError as e:
-                    print_error(e)
+                    self.print_error(e)
                     s = None
                 except:
                     return
@@ -172,8 +172,10 @@ class TcpConnection(threading.Thread, util.PrintError):
                 # workaround android bug
                 cert = re.sub("([^\n])-----END CERTIFICATE-----","\\1\n-----END CERTIFICATE-----",cert)
                 temporary_path = cert_path + '.temp'
-                with open(temporary_path,"w") as f:
+                with open(temporary_path, "w", encoding='utf-8') as f:
                     f.write(cert)
+                    f.flush()
+                    os.fsync(f.fileno())
             else:
                 is_new = False
 
@@ -199,7 +201,7 @@ class TcpConnection(threading.Thread, util.PrintError):
                         os.unlink(rej)
                     os.rename(temporary_path, rej)
                 else:
-                    with open(cert_path) as f:
+                    with open(cert_path, encoding='utf-8') as f:
                         cert = f.read()
                     try:
                         b = pem.dePem(cert, 'CERTIFICATE')
@@ -238,7 +240,7 @@ class TcpConnection(threading.Thread, util.PrintError):
 
 class Interface(util.PrintError):
     """The Interface class handles a socket connected to a single remote
-    electrum server.  It's exposed API is:
+    Electrum server.  Its exposed API is:
 
     - Member functions close(), fileno(), get_responses(), has_timed_out(),
       ping_required(), queue_request(), send_requests()
@@ -295,8 +297,8 @@ class Interface(util.PrintError):
         wire_requests = self.unsent_requests[0:n]
         try:
             self.pipe.send_all([make_dict(*r) for r in wire_requests])
-        except socket.error as e:
-            self.print_error("socket error:", e)
+        except BaseException as e:
+            self.print_error("pipe send error:", e)
             return False
         self.unsent_requests = self.unsent_requests[n:]
         for request in wire_requests:
@@ -396,7 +398,7 @@ def test_certificates():
     certs = os.listdir(mydir)
     for c in certs:
         p = os.path.join(mydir,c)
-        with open(p) as f:
+        with open(p, encoding='utf-8') as f:
             cert = f.read()
         check_cert(c, cert)
 

lib/keystore.py

@@ -28,8 +28,9 @@ from unicodedata import normalize
 
 from . import bitcoin
 from .bitcoin import *
-
-from .util import PrintError, InvalidPassword, hfu
+from . import constants
+from .util import (PrintError, InvalidPassword, hfu, WalletFileException,
+                   BitcoinException)
 from .mnemonic import Mnemonic, load_wordlist
 from .plugins import run_hook
 
@@ -45,6 +46,10 @@ class KeyStore(PrintError):
     def can_import(self):
         return False
 
+    def may_have_password(self):
+        """Returns whether the keystore can be encrypted with a password."""
+        raise NotImplementedError()
+
     def get_tx_derivations(self, tx):
         keypairs = {}
         for txin in tx.inputs():
@@ -71,6 +76,8 @@ class KeyStore(PrintError):
             return False
         return bool(self.get_tx_derivations(tx))
 
+    def ready_to_sign(self):
+        return not self.is_watching_only()
 
 
 class Software_KeyStore(KeyStore):
@@ -116,9 +123,6 @@ class Imported_KeyStore(Software_KeyStore):
     def is_deterministic(self):
         return False
 
-    def can_change_password(self):
-        return True
-
     def get_master_public_key(self):
         return None
 
@@ -138,7 +142,14 @@ class Imported_KeyStore(Software_KeyStore):
     def import_privkey(self, sec, password):
         txin_type, privkey, compressed = deserialize_privkey(sec)
         pubkey = public_key_from_private_key(privkey, compressed)
-        self.keypairs[pubkey] = pw_encode(sec, password)
+        # re-serialize the key so the internal storage format is consistent
+        serialized_privkey = serialize_privkey(
+            privkey, compressed, txin_type, internal_use=True)
+        # NOTE: if the same pubkey is reused for multiple addresses (script types),
+        # there will only be one pubkey-privkey pair for it in self.keypairs,
+        # and the privkey will encode a txin_type but that txin_type cannot be trusted.
+        # Removing keys complicates this further.
+        self.keypairs[pubkey] = pw_encode(serialized_privkey, password)
         return txin_type, pubkey
 
     def delete_imported_key(self, key):
@@ -196,9 +207,6 @@ class Deterministic_KeyStore(Software_KeyStore):
     def is_watching_only(self):
         return not self.has_seed()
 
-    def can_change_password(self):
-        return not self.is_watching_only()
-
     def add_seed(self, seed):
         if self.seed:
             raise Exception("a seed exists")
@@ -504,7 +512,7 @@ class Hardware_KeyStore(KeyStore, Xpub):
         }
 
     def unpaired(self):
-        '''A device paired with the wallet was diconnected.  This can be
+        '''A device paired with the wallet was disconnected.  This can be
         called in any thread context.'''
         self.print_error("unpaired")
 
@@ -522,9 +530,24 @@ class Hardware_KeyStore(KeyStore, Xpub):
         assert not self.has_seed()
         return False
 
-    def can_change_password(self):
-        return False
+    def get_password_for_storage_encryption(self):
+        from .storage import get_derivation_used_for_hw_device_encryption
+        client = self.plugin.get_client(self)
+        derivation = get_derivation_used_for_hw_device_encryption()
+        xpub = client.get_xpub(derivation, "standard")
+        password = self.get_pubkey_from_xpub(xpub, ())
+        return password
 
+    def has_usable_connection_with_device(self):
+        if not hasattr(self, 'plugin'):
+            return False
+        client = self.plugin.get_client(self, force_pair=False)
+        if client is None:
+            return False
+        return client.has_usable_connection_with_device()
+
+    def ready_to_sign(self):
+        return super().ready_to_sign() and self.has_usable_connection_with_device()
 
 
 def bip39_normalize_passphrase(passphrase):
@@ -571,10 +594,19 @@ def bip39_is_checksum_valid(mnemonic):
 def from_bip39_seed(seed, passphrase, derivation):
     k = BIP32_KeyStore({})
     bip32_seed = bip39_to_seed(seed, passphrase)
-    t = 'standard'  # bip43
-    k.add_xprv_from_seed(bip32_seed, t, derivation)
+    xtype = xtype_from_derivation(derivation)
+    k.add_xprv_from_seed(bip32_seed, xtype, derivation)
     return k
 
+
+def xtype_from_derivation(derivation):
+    """Returns the script type to be used for this derivation."""
+    if derivation.startswith("m/84'") or derivation.startswith("m/49'"):
+        raise Exception('Unknown bip43 derivation purpose %s' % derivation[:5])
+    else:
+        return 'standard'
+
+
 # extended pubkeys
 
 def is_xpubkey(x_pubkey):
@@ -599,7 +631,8 @@ def xpubkey_to_address(x_pubkey):
         mpk, s = Old_KeyStore.parse_xpubkey(x_pubkey)
         pubkey = Old_KeyStore.get_pubkey_from_mpk(mpk, s[0], s[1])
     else:
-        raise BaseException("Cannot parse pubkey")
+        raise BitcoinException("Cannot parse pubkey. prefix: {}"
+                               .format(x_pubkey[0:2]))
     if pubkey:
         address = public_key_to_p2pkh(bfh(pubkey))
     return pubkey, address
@@ -618,14 +651,15 @@ def hardware_keystore(d):
     if hw_type in hw_keystores:
         constructor = hw_keystores[hw_type]
         return constructor(d)
-    raise BaseException('unknown hardware type', hw_type)
+    raise WalletFileException('unknown hardware type: {}'.format(hw_type))
 
 def load_keystore(storage, name):
-    w = storage.get('wallet_type', 'standard')
     d = storage.get(name, {})
     t = d.get('type')
     if not t:
-        raise BaseException('wallet format requires update')
+        raise WalletFileException(
+            'Wallet format requires update.\n'
+            'Cannot find keystore for name {}'.format(name))
     if t == 'old':
         k = Old_KeyStore(d)
     elif t == 'imported':
@@ -635,7 +669,8 @@ def load_keystore(storage, name):
     elif t == 'hardware':
         k = hardware_keystore(d)
     else:
-        raise BaseException('unknown wallet type', t)
+        raise WalletFileException(
+            'Unknown type {} for keystore named {}'.format(t, name))
     return k
 
 
@@ -671,10 +706,9 @@ is_private_key = lambda x: is_xprv(x) or is_private_key_list(x)
 is_bip32_key = lambda x: is_xprv(x) or is_xpub(x)
 
 
-def bip44_derivation(account_id):
-    bip  = 44
-    coin = 1 if bitcoin.NetworkConstants.TESTNET else 133
-    return "m/%d'/%d'/%d'" % (bip, coin, int(account_id))
+def bip44_derivation(account_id, bip43_purpose=44):
+    coin = 1 if constants.net.TESTNET else 133
+    return "m/%d'/%d'/%d'" % (bip43_purpose, coin, int(account_id))
 
 def from_seed(seed, passphrase, is_p2sh):
     t = seed_type(seed)
@@ -690,7 +724,7 @@ def from_seed(seed, passphrase, is_p2sh):
         xtype = 'standard'
         keystore.add_xprv_from_seed(bip32_seed, xtype, der)
     else:
-        raise BaseException(t)
+        raise BitcoinException('Unexpected seed type {}'.format(t))
     return keystore
 
 def from_private_key_list(text):
@@ -724,5 +758,5 @@ def from_master_key(text):
     elif is_xpub(text):
         k = from_xpub(text)
     else:
-        raise BaseException('Invalid key')
+        raise BitcoinException('Invalid master key')
     return k

lib/mnemonic.py

@@ -91,7 +91,7 @@ def normalize_text(seed):
 
 def load_wordlist(filename):
     path = os.path.join(os.path.dirname(__file__), 'wordlist', filename)
-    with open(path, 'r') as f:
+    with open(path, 'r', encoding='utf-8') as f:
         s = f.read().strip()
     s = unicodedata.normalize('NFKD', s)
     lines = s.split('\n')
@@ -157,30 +157,24 @@ class Mnemonic(object):
             i = i*n + k
         return i
 
-    def check_seed(self, seed, custom_entropy):
-        assert is_new_seed(seed)
-        i = self.mnemonic_decode(seed)
-        return i % custom_entropy == 0
-
-    def make_seed(self, seed_type='standard', num_bits=132, custom_entropy=1):
+    def make_seed(self, seed_type='standard', num_bits=132):
         prefix = version.seed_prefix(seed_type)
-        # increase num_bits in order to obtain a uniform distibution for the last word
+        # increase num_bits in order to obtain a uniform distribution for the last word
         bpw = math.log(len(self.wordlist), 2)
-        num_bits = int(math.ceil(num_bits/bpw) * bpw)
-        # handle custom entropy; make sure we add at least 16 bits
-        n_custom = int(math.ceil(math.log(custom_entropy, 2)))
-        n = max(16, num_bits - n_custom)
-        print_error("make_seed", prefix, "adding %d bits"%n)
-        my_entropy = 1
-        while my_entropy < pow(2, n - bpw):
+        # rounding
+        n = int(math.ceil(num_bits/bpw) * bpw)
+        print_error("make_seed. prefix: '%s'"%prefix, "entropy: %d bits"%n)
+        entropy = 1
+        while entropy < pow(2, n - bpw):
             # try again if seed would not contain enough words
-            my_entropy = ecdsa.util.randrange(pow(2, n))
+            entropy = ecdsa.util.randrange(pow(2, n))
         nonce = 0
         while True:
             nonce += 1
-            i = custom_entropy * (my_entropy + nonce)
+            i = entropy + nonce
             seed = self.mnemonic_encode(i)
-            assert i == self.mnemonic_decode(seed)
+            if i != self.mnemonic_decode(seed):
+                raise Exception('Cannot extract same entropy from mnemonic!')
             if is_old_seed(seed):
                 continue
             if is_new_seed(seed, prefix):

lib/network.py

@@ -1,4 +1,3 @@
-
 # Electrum - Lightweight Bitcoin Client
 # Copyright (c) 2011-2016 Thomas Voegtlin
 #
@@ -38,10 +37,11 @@ import socks
 from . import util
 from . import bitcoin
 from .bitcoin import *
-from .blockchain import HDR_LEN, CHUNK_LEN
+from . import constants
 from .interface import Connection, Interface
 from . import blockchain
 from .version import ELECTRUM_VERSION, PROTOCOL_VERSION
+from .i18n import _
 
 
 NODES_RETRY_INTERVAL = 60
@@ -61,7 +61,7 @@ def parse_servers(result):
             for v in item[2]:
                 if re.match("[st]\d*", v):
                     protocol, port = v[0], v[1:]
-                    if port == '': port = bitcoin.NetworkConstants.DEFAULT_PORTS[protocol]
+                    if port == '': port = constants.net.DEFAULT_PORTS[protocol]
                     out[protocol] = port
                 elif re.match("v(.?)+", v):
                     version = v[1:]
@@ -95,7 +95,7 @@ def filter_protocol(hostmap, protocol = 's'):
 
 def pick_random_server(hostmap = None, protocol = 's', exclude_set = set()):
     if hostmap is None:
-        hostmap = bitcoin.NetworkConstants.DEFAULT_SERVERS
+        hostmap = constants.net.DEFAULT_SERVERS
     eligible = list(set(filter_protocol(hostmap, protocol)) - exclude_set)
     return random.choice(eligible) if eligible else None
 
@@ -139,8 +139,9 @@ def deserialize_proxy(s):
 
 
 def deserialize_server(server_str):
-    host, port, protocol = str(server_str).split(':')
-    assert protocol in 'st'
+    host, port, protocol = str(server_str).rsplit(':', 2)
+    if protocol not in 'st':
+        raise ValueError('invalid network protocol: {}'.format(protocol))
     int(port)    # Throw if cannot be converted to int
     return host, port, protocol
 
@@ -174,15 +175,16 @@ class Network(util.DaemonThread):
         if self.blockchain_index not in self.blockchains.keys():
             self.blockchain_index = 0
         # Server for addresses and transactions
-        self.default_server = self.config.get('server')
+        self.default_server = self.config.get('server', None)
         # Sanitize default server
-        try:
-            deserialize_server(self.default_server)
-        except:
-            self.default_server = None
+        if self.default_server:
+            try:
+                deserialize_server(self.default_server)
+            except:
+                self.print_error('Warning: failed to parse server-string; falling back to random.')
+                self.default_server = None
         if not self.default_server:
             self.default_server = pick_random_server()
-
         self.lock = threading.Lock()
         self.pending_sends = []
         self.message_id = 0
@@ -219,6 +221,7 @@ class Network(util.DaemonThread):
         self.interfaces = {}
         self.auto_connect = self.config.get('auto_connect', True)
         self.connecting = set()
+        self.requested_chunks = set()
         self.socket_queue = queue.Queue()
         self.start_network(deserialize_server(self.default_server)[2],
                            deserialize_proxy(self.config.get('proxy')))
@@ -244,7 +247,7 @@ class Network(util.DaemonThread):
             return []
         path = os.path.join(self.config.path, "recent_servers")
         try:
-            with open(path, "r") as f:
+            with open(path, "r", encoding='utf-8') as f:
                 data = f.read()
                 return json.loads(data)
         except:
@@ -256,7 +259,7 @@ class Network(util.DaemonThread):
         path = os.path.join(self.config.path, "recent_servers")
         s = json.dumps(self.recent_servers, indent=4, sort_keys=True)
         try:
-            with open(path, "w") as f:
+            with open(path, "w", encoding='utf-8') as f:
                 f.write(s)
         except:
             pass
@@ -306,6 +309,9 @@ class Network(util.DaemonThread):
         # Resend unanswered requests
         requests = self.unanswered_requests.values()
         self.unanswered_requests = {}
+        if self.interface.ping_required():
+            params = [ELECTRUM_VERSION, PROTOCOL_VERSION]
+            self.queue_request('server.version', params, self.interface)
         for request in requests:
             message_id = self.queue_request(request[0], request[1])
             self.unanswered_requests[message_id] = request
@@ -314,15 +320,14 @@ class Network(util.DaemonThread):
         self.queue_request('server.peers.subscribe', [])
         self.request_fee_estimates()
         self.queue_request('blockchain.relayfee', [])
-        if self.interface.ping_required():
-            params = [ELECTRUM_VERSION, PROTOCOL_VERSION]
-            self.queue_request('server.version', params, self.interface)
-        for h in self.subscribed_addresses:
+        for h in list(self.subscribed_addresses):
             self.queue_request('blockchain.scripthash.subscribe', [h])
 
     def request_fee_estimates(self):
+        from .simple_config import FEE_ETA_TARGETS
         self.config.requested_fee_estimates()
-        for i in bitcoin.FEE_TARGETS:
+        self.queue_request('mempool.get_fee_histogram', [])
+        for i in FEE_ETA_TARGETS:
             self.queue_request('blockchain.estimatefee', [i])
 
     def get_status_value(self, key):
@@ -332,6 +337,8 @@ class Network(util.DaemonThread):
             value = self.banner
         elif key == 'fee':
             value = self.config.fee_estimates
+        elif key == 'fee_histogram':
+            value = self.config.mempool_fees
         elif key == 'updated':
             value = (self.get_local_height(), self.get_server_height())
         elif key == 'servers':
@@ -359,7 +366,7 @@ class Network(util.DaemonThread):
         return list(self.interfaces.keys())
 
     def get_servers(self):
-        out = bitcoin.NetworkConstants.DEFAULT_SERVERS
+        out = constants.net.DEFAULT_SERVERS
         if self.irc_servers:
             out.update(filter_version(self.irc_servers.copy()))
         else:
@@ -543,6 +550,11 @@ class Network(util.DaemonThread):
         elif method == 'server.donation_address':
             if error is None:
                 self.donation_address = result
+        elif method == 'mempool.get_fee_histogram':
+            if error is None:
+                self.print_error('fee_histogram', result)
+                self.config.mempool_fees = result
+                self.notify('fee_histogram')
         elif method == 'blockchain.estimatefee':
             if error is None and result > 0:
                 i = params[0]
@@ -552,14 +564,12 @@ class Network(util.DaemonThread):
                 self.notify('fee')
         elif method == 'blockchain.relayfee':
             if error is None:
-                self.relay_fee = int(result * COIN)
+                self.relay_fee = int(result * COIN) if result is not None else None
                 self.print_error("relayfee", self.relay_fee)
-        elif method == 'blockchain.block.headers':
-            height, count = params
-            if count == 1:
-                self.on_get_header(interface, response, height)
-            elif count == CHUNK_LEN:
-                self.on_get_chunk(interface, response, height)
+        elif method == 'blockchain.block.get_chunk':
+            self.on_get_chunk(interface, response)
+        elif method == 'blockchain.block.get_header':
+            self.on_get_header(interface, response)
 
         for callback in callbacks:
             callback(response)
@@ -668,7 +678,7 @@ class Network(util.DaemonThread):
                     # check cached response for subscriptions
                     r = self.sub_cache.get(k)
                 if r is not None:
-                    util.print_error("cache hit", k)
+                    self.print_error("cache hit", k)
                     callback(r)
                 else:
                     message_id = self.queue_request(method, params)
@@ -707,7 +717,7 @@ class Network(util.DaemonThread):
         interface.mode = 'default'
         interface.request = None
         self.interfaces[server] = interface
-        self.queue_request('blockchain.headers.subscribe', [True], interface)
+        self.queue_request('blockchain.headers.subscribe', [], interface)
         if server == self.default_server:
             self.switch_to_interface(server)
         #self.notify('interfaces')
@@ -758,77 +768,77 @@ class Network(util.DaemonThread):
             if self.config.is_fee_estimates_update_required():
                 self.request_fee_estimates()
 
-    def request_chunk(self, interface, idx):
-        interface.print_error("requesting chunk %d" % idx)
-        self.queue_request('blockchain.block.headers',
-                           [CHUNK_LEN*idx, CHUNK_LEN], interface)
-        interface.request = idx
-        interface.req_time = time.time()
+    def request_chunk(self, interface, index):
+        if index in self.requested_chunks:
+            return
+        interface.print_error("requesting chunk %d" % index)
+        self.requested_chunks.add(index)
+        self.queue_request('blockchain.block.get_chunk', [index], interface)
 
-    def on_get_chunk(self, interface, response, height):
+    def on_get_chunk(self, interface, response):
         '''Handle receiving a chunk of block headers'''
         error = response.get('error')
         result = response.get('result')
-        if result is None or error is not None:
+        params = response.get('params')
+        blockchain = interface.blockchain
+        if result is None or params is None or error is not None:
             interface.print_error(error or 'bad response')
             return
-
-        hex_chunk = result.get('hex', None)
+        index = params[0]
         # Ignore unsolicited chunks
-        index = height // CHUNK_LEN
-        if interface.request != index or height / CHUNK_LEN != index:
+        if index not in self.requested_chunks:
+            interface.print_error("received chunk %d (unsolicited)" % index)
             return
-
-        connect = interface.blockchain.connect_chunk(index, hex_chunk)
-        # If not finished, get the next chunk
+        else:
+            interface.print_error("received chunk %d" % index)
+        self.requested_chunks.remove(index)
+        connect = blockchain.connect_chunk(index, result)
         if not connect:
             self.connection_down(interface.server)
             return
-        if interface.blockchain.height() < interface.tip:
+        # If not finished, get the next chunk
+        if index >= len(blockchain.checkpoints) and blockchain.height() < interface.tip:
             self.request_chunk(interface, index+1)
         else:
-            interface.request = None
             interface.mode = 'default'
-            interface.print_error('catch up done', interface.blockchain.height())
-            interface.blockchain.catch_up = None
+            interface.print_error('catch up done', blockchain.height())
+            blockchain.catch_up = None
         self.notify('updated')
 
     def request_header(self, interface, height):
         #interface.print_error("requesting header %d" % height)
-        self.queue_request('blockchain.block.headers', [height, 1], interface)
+        self.queue_request('blockchain.block.get_header', [height], interface)
         interface.request = height
         interface.req_time = time.time()
 
-    def on_get_header(self, interface, response, height):
+    def on_get_header(self, interface, response):
         '''Handle receiving a single block header'''
-        result = response.get('result', {})
-        hex_header = result.get('hex', None)
-
+        header = response.get('result')
+        if not header:
+            interface.print_error(response)
+            self.connection_down(interface.server)
+            return
+        height = header.get('block_height')
         if interface.request != height:
             interface.print_error("unsolicited header",interface.request, height)
             self.connection_down(interface.server)
             return
-
-        if not hex_header:
-            interface.print_error(response)
-            self.connection_down(interface.server)
-            return
-
-        if len(hex_header) != HDR_LEN*2:
-            interface.print_error('wrong header length', interface.request)
-            self.connection_down(interface.server)
-            return
-
-        header = blockchain.deserialize_header(bfh(hex_header), height)
-
         chain = blockchain.check_header(header)
         if interface.mode == 'backward':
-            if chain:
+            can_connect = blockchain.can_connect(header)
+            if can_connect and can_connect.catch_up is None:
+                interface.mode = 'catch_up'
+                interface.blockchain = can_connect
+                interface.blockchain.save_header(header)
+                next_height = height + 1
+                interface.blockchain.catch_up = interface.server
+            elif chain:
                 interface.print_error("binary search")
                 interface.mode = 'binary'
                 interface.blockchain = chain
                 interface.good = height
                 next_height = (interface.bad + interface.good) // 2
+                assert next_height >= self.max_checkpoint(), (interface.bad, interface.good)
             else:
                 if height == 0:
                     self.connection_down(interface.server)
@@ -837,7 +847,7 @@ class Network(util.DaemonThread):
                     interface.bad = height
                     interface.bad_header = header
                     delta = interface.tip - height
-                    next_height = max(0, interface.tip - 2 * delta)
+                    next_height = max(self.max_checkpoint(), interface.tip - 2 * delta)
 
         elif interface.mode == 'binary':
             if chain:
@@ -848,6 +858,7 @@ class Network(util.DaemonThread):
                 interface.bad_header = header
             if interface.bad != interface.good + 1:
                 next_height = (interface.bad + interface.good) // 2
+                assert next_height >= self.max_checkpoint()
             elif not interface.blockchain.can_connect(interface.bad_header, check_height=False):
                 self.connection_down(interface.server)
                 next_height = None
@@ -912,11 +923,11 @@ class Network(util.DaemonThread):
                 self.notify('updated')
 
         else:
-            raise BaseException(interface.mode)
+            raise Exception(interface.mode)
         # If not finished, get the next header
         if next_height:
             if interface.mode == 'catch_up' and interface.tip > next_height + 50:
-                self.request_chunk(interface, next_height // CHUNK_LEN)
+                self.request_chunk(interface, next_height // 2016)
             else:
                 self.request_header(interface, next_height)
         else:
@@ -957,33 +968,18 @@ class Network(util.DaemonThread):
 
     def init_headers_file(self):
         b = self.blockchains[0]
-        if b.get_hash(0) == bitcoin.NetworkConstants.GENESIS:
-            self.downloading_headers = False
-            return
         filename = b.path()
-        def download_thread():
-            try:
-                import urllib.request, socket
-                socket.setdefaulttimeout(30)
-                self.print_error("downloading ", bitcoin.NetworkConstants.HEADERS_URL)
-                urllib.request.urlretrieve(bitcoin.NetworkConstants.HEADERS_URL, filename + '.tmp')
-                os.rename(filename + '.tmp', filename)
-                self.print_error("done.")
-            except Exception:
-                self.print_error("download failed. creating file", filename)
-                open(filename, 'wb+').close()
-            b = self.blockchains[0]
-            with b.lock: b.update_size()
-            self.downloading_headers = False
-        self.downloading_headers = True
-        t = threading.Thread(target = download_thread)
-        t.daemon = True
-        t.start()
+        length = 80 * len(constants.net.CHECKPOINTS) * 2016
+        if not os.path.exists(filename) or os.path.getsize(filename) < length:
+            with open(filename, 'wb') as f:
+                if length>0:
+                    f.seek(length-1)
+                    f.write(b'\x00')
+        with b.lock:
+            b.update_size()
 
     def run(self):
         self.init_headers_file()
-        while self.is_running() and self.downloading_headers:
-            time.sleep(1)
         while self.is_running():
             self.maintain_sockets()
             self.wait_on_sockets()
@@ -994,17 +990,12 @@ class Network(util.DaemonThread):
         self.on_stop()
 
     def on_notify_header(self, interface, header):
-        height = header.get('height')
-        hex_header = header.get('hex')
-        if not height or not hex_header:
+        height = header.get('block_height')
+        if not height:
             return
-
-        if len(hex_header) != HDR_LEN*2:
-            interface.print_error('wrong header length', interface.request)
+        if height < self.max_checkpoint():
             self.connection_down(interface.server)
             return
-
-        header = blockchain.deserialize_header(bfh(hex_header), height)
         interface.tip_header = header
         interface.tip = height
         if interface.mode != 'default':
@@ -1029,14 +1020,17 @@ class Network(util.DaemonThread):
             interface.mode = 'backward'
             interface.bad = height
             interface.bad_header = header
-            self.request_header(interface, min(tip, height - 1))
+            self.request_header(interface, min(tip +1, height - 1))
         else:
             chain = self.blockchains[0]
             if chain.catch_up is None:
                 chain.catch_up = interface
                 interface.mode = 'catch_up'
                 interface.blockchain = chain
+                self.print_error("switching to catchup mode", tip,  self.blockchains)
                 self.request_header(interface, 0)
+            else:
+                self.print_error("chain already catching up with", chain.catch_up.server)
 
     def blockchain(self):
         if self.interface and self.interface.blockchain is not None:
@@ -1061,7 +1055,7 @@ class Network(util.DaemonThread):
                     self.switch_to_interface(i.server)
                     break
         else:
-            raise BaseException('blockchain not found', index)
+            raise Exception('blockchain not found', index)
 
         if self.interface:
             server = self.interface.server
@@ -1078,9 +1072,9 @@ class Network(util.DaemonThread):
         try:
             r = q.get(True, timeout)
         except queue.Empty:
-            raise BaseException('Server did not answer')
+            raise util.TimeoutException(_('Server did not answer'))
         if r.get('error'):
-            raise BaseException(r.get('error'))
+            raise Exception(r.get('error'))
         return r.get('result')
 
     def broadcast(self, tx, timeout=30):
@@ -1092,3 +1086,12 @@ class Network(util.DaemonThread):
         if out != tx_hash:
             return False, "error: " + out
         return True, out
+
+    def export_checkpoints(self, path):
+        # run manually from the console to generate checkpoints
+        cp = self.blockchain().get_checkpoints()
+        with open(path, 'w', encoding='utf-8') as f:
+            f.write(json.dumps(cp, indent=4))
+
+    def max_checkpoint(self):
+        return max(0, len(constants.net.CHECKPOINTS) * 2016 - 1)

lib/paymentrequest.py

@@ -40,6 +40,7 @@ except ImportError:
 from . import bitcoin
 from . import util
 from .util import print_error, bh2u, bfh
+from .util import export_meta, import_meta
 from . import transaction
 from . import x509
 from . import rsakey
@@ -88,13 +89,13 @@ def get_payment_request(url):
             error = "payment URL not pointing to a valid server"
     elif u.scheme == 'file':
         try:
-            with open(u.path, 'r') as f:
+            with open(u.path, 'r', encoding='utf-8') as f:
                 data = f.read()
         except IOError:
             data = None
             error = "payment URL not pointing to a valid file"
     else:
-        raise BaseException("unknown scheme", url)
+        raise Exception("unknown scheme", url)
     pr = PaymentRequest(data, error)
     return pr
 
@@ -147,7 +148,7 @@ class PaymentRequest:
             self.error = "Error: Cannot parse payment request"
             return False
         if not pr.signature:
-            # the address will be dispayed as requestor
+            # the address will be displayed as requestor
             self.requestor = None
             return True
         if pr.pki_type in ["x509+sha256", "x509+sha1"]:
@@ -339,9 +340,9 @@ def verify_cert_chain(chain):
             x.check_date()
         else:
             if not x.check_ca():
-                raise BaseException("ERROR: Supplied CA Certificate Error")
+                raise Exception("ERROR: Supplied CA Certificate Error")
     if not cert_num > 1:
-        raise BaseException("ERROR: CA Certificate Chain Not Provided by Payment Processor")
+        raise Exception("ERROR: CA Certificate Chain Not Provided by Payment Processor")
     # if the root CA is not supplied, add it to the chain
     ca = x509_chain[cert_num-1]
     if ca.getFingerprint() not in ca_list:
@@ -351,7 +352,7 @@ def verify_cert_chain(chain):
             root = ca_list[f]
             x509_chain.append(root)
         else:
-            raise BaseException("Supplied CA Not Found in Trusted CA Store.")
+            raise Exception("Supplied CA Not Found in Trusted CA Store.")
     # verify the chain of signatures
     cert_num = len(x509_chain)
     for i in range(1, cert_num):
@@ -372,10 +373,10 @@ def verify_cert_chain(chain):
             hashBytes = bytearray(hashlib.sha512(data).digest())
             verify = pubkey.verify(sig, x509.PREFIX_RSA_SHA512 + hashBytes)
         else:
-            raise BaseException("Algorithm not supported")
+            raise Exception("Algorithm not supported")
             util.print_error(self.error, algo.getComponentByName('algorithm'))
         if not verify:
-            raise BaseException("Certificate not Signed by Provided CA Certificate Chain")
+            raise Exception("Certificate not Signed by Provided CA Certificate Chain")
 
     return x509_chain[0], ca
 
@@ -384,9 +385,9 @@ def check_ssl_config(config):
     from . import pem
     key_path = config.get('ssl_privkey')
     cert_path = config.get('ssl_chain')
-    with open(key_path, 'r') as f:
+    with open(key_path, 'r', encoding='utf-8') as f:
         params = pem.parse_private_key(f.read())
-    with open(cert_path, 'r') as f:
+    with open(cert_path, 'r', encoding='utf-8') as f:
         s = f.read()
     bList = pem.dePemList(s, "CERTIFICATE")
     # verify chain
@@ -404,10 +405,10 @@ def check_ssl_config(config):
 
 def sign_request_with_x509(pr, key_path, cert_path):
     from . import pem
-    with open(key_path, 'r') as f:
+    with open(key_path, 'r', encoding='utf-8') as f:
         params = pem.parse_private_key(f.read())
         privkey = rsakey.RSAKey(*params)
-    with open(cert_path, 'r') as f:
+    with open(cert_path, 'r', encoding='utf-8') as f:
         s = f.read()
         bList = pem.dePemList(s, "CERTIFICATE")
     certificates = pb2.X509Certificates()
@@ -452,7 +453,11 @@ class InvoiceStore(object):
 
     def set_paid(self, pr, txid):
         pr.tx = txid
-        self.paid[txid] = pr.get_id()
+        pr_id = pr.get_id()
+        self.paid[txid] = pr_id
+        if pr_id not in self.invoices:
+            # in case the user had deleted it previously
+            self.add(pr)
 
     def load(self, d):
         for k, v in d.items():
@@ -467,24 +472,29 @@ class InvoiceStore(object):
                 continue
 
     def import_file(self, path):
-        try:
-            with open(path, 'r') as f:
-                d = json.loads(f.read())
-                self.load(d)
-        except:
-            traceback.print_exc(file=sys.stderr)
-            return
+        def validate(data):
+            return data  # TODO
+        import_meta(path, validate, self.on_import)
+
+    def on_import(self, data):
+        self.load(data)
         self.save()
 
-    def save(self):
-        l = {}
+    def export_file(self, filename):
+        export_meta(self.dump(), filename)
+
+    def dump(self):
+        d = {}
         for k, pr in self.invoices.items():
-            l[k] = {
+            d[k] = {
                 'hex': bh2u(pr.raw),
                 'requestor': pr.requestor,
                 'txid': pr.tx
             }
-        self.storage.put('invoices', l)
+        return d
+
+    def save(self):
+        self.storage.put('invoices', self.dump())
 
     def get_status(self, key):
         pr = self.get(key)

lib/plot.py

@@ -1,30 +1,32 @@
-from PyQt5.QtGui import *
-from electrum.i18n import _
-
-
 import datetime
 from collections import defaultdict
-from electrum.bitcoin import COIN
 
 import matplotlib
 matplotlib.use('Qt5Agg')
 import matplotlib.pyplot as plt
 import matplotlib.dates as md
-from matplotlib.patches import Ellipse
-from matplotlib.offsetbox import AnchoredOffsetbox, TextArea, DrawingArea, HPacker
+
+from .i18n import _
+from .bitcoin import COIN
 
 
-def plot_history(wallet, history):
+class NothingToPlotException(Exception):
+    def __str__(self):
+        return _("Nothing to plot.")
+
+
+def plot_history(history):
+    if len(history) == 0:
+        raise NothingToPlotException()
     hist_in = defaultdict(int)
     hist_out = defaultdict(int)
     for item in history:
-        tx_hash, height, confirmations, timestamp, value, balance = item
-        if not confirmations:
+        if not item['confirmations']:
             continue
-        if timestamp is None:
+        if item['timestamp'] is None:
             continue
-        value = value*1./COIN
-        date = datetime.datetime.fromtimestamp(timestamp)
+        value = item['value'].value/COIN
+        date = item['date']
         datenum = int(md.date2num(datetime.date(date.year, date.month, 1)))
         if value > 0:
             hist_in[datenum] += value
@@ -43,10 +45,19 @@ def plot_history(wallet, history):
     xfmt = md.DateFormatter('%Y-%m')
     ax.xaxis.set_major_formatter(xfmt)
     width = 20
-    dates, values = zip(*sorted(hist_in.items()))
-    r1 = axarr[0].bar(dates, values, width, label='incoming')
-    axarr[0].legend(loc='upper left')
-    dates, values = zip(*sorted(hist_out.items()))
-    r2 = axarr[1].bar(dates, values, width, color='r', label='outgoing')
-    axarr[1].legend(loc='upper left')
+
+    r1 = None
+    r2 = None
+    dates_values = list(zip(*sorted(hist_in.items())))
+    if dates_values and len(dates_values) == 2:
+        dates, values = dates_values
+        r1 = axarr[0].bar(dates, values, width, label='incoming')
+        axarr[0].legend(loc='upper left')
+    dates_values = list(zip(*sorted(hist_out.items())))
+    if dates_values and len(dates_values) == 2:
+        dates, values = dates_values
+        r2 = axarr[1].bar(dates, values, width, color='r', label='outgoing')
+        axarr[1].legend(loc='upper left')
+    if r1 is None and r2 is None:
+        raise NothingToPlotException()
     return plt

lib/plugins.py

@@ -312,6 +312,8 @@ class DeviceMgr(ThreadJob, PrintError):
         # What we recognise.  Each entry is a (vendor_id, product_id)
         # pair.
         self.recognised_hardware = set()
+        # Custom enumerate functions for devices we don't know about.
+        self.enumerate_func = set()
         # For synchronization
         self.lock = threading.RLock()
         self.hid_lock = threading.RLock()
@@ -334,6 +336,9 @@ class DeviceMgr(ThreadJob, PrintError):
         for pair in device_pairs:
             self.recognised_hardware.add(pair)
 
+    def register_enumerate_func(self, func):
+        self.enumerate_func.add(func)
+
     def create_client(self, device, handler, plugin):
         # Get from cache first
         client = self.client_lookup(device.id_)
@@ -362,15 +367,20 @@ class DeviceMgr(ThreadJob, PrintError):
             if not xpub in self.xpub_ids:
                 return
             _id = self.xpub_ids.pop(xpub)
-        client = self.client_lookup(_id)
-        self.clients.pop(client, None)
-        if client:
-            client.close()
+            self._close_client(_id)
 
     def unpair_id(self, id_):
         xpub = self.xpub_by_id(id_)
         if xpub:
             self.unpair_xpub(xpub)
+        else:
+            self._close_client(id_)
+
+    def _close_client(self, id_):
+        client = self.client_lookup(id_)
+        self.clients.pop(client, None)
+        if client:
+            client.close()
 
     def pair_xpub(self, xpub, id_):
         with self.lock:
@@ -393,7 +403,7 @@ class DeviceMgr(ThreadJob, PrintError):
     def client_for_keystore(self, plugin, handler, keystore, force_pair):
         self.print_error("getting client for keystore")
         if handler is None:
-            raise BaseException(_("Handler not found for") + ' ' + plugin.name + '\n' + _("A library is probably missing."))
+            raise Exception(_("Handler not found for") + ' ' + plugin.name + '\n' + _("A library is probably missing."))
         handler.update_status(False)
         devices = self.scan_devices()
         xpub = keystore.xpub
@@ -442,21 +452,23 @@ class DeviceMgr(ThreadJob, PrintError):
         # The user input has wrong PIN or passphrase, or cancelled input,
         # or it is not pairable
         raise DeviceUnpairableError(
-            _('Electrum-Zcash cannot pair with your %s.\n\n'
+            _('Electrum-Zcash cannot pair with your {}.\n\n'
               'Before you request Zcash coins to be sent to addresses in this '
               'wallet, ensure you can pair with your device, or that you have '
               'its seed (and passphrase, if any).  Otherwise all coins you '
-              'receive will be unspendable.') % plugin.device)
+              'receive will be unspendable.').format(plugin.device))
 
     def unpaired_device_infos(self, handler, plugin, devices=None):
         '''Returns a list of DeviceInfo objects: one for each connected,
         unpaired device accepted by the plugin.'''
+        if not plugin.libraries_available:
+            raise Exception('Missing libraries for {}'.format(plugin.name))
         if devices is None:
             devices = self.scan_devices()
         devices = [dev for dev in devices if not self.xpub_by_id(dev.id_)]
         infos = []
         for device in devices:
-            if not device.product_key in plugin.DEVICE_IDS:
+            if device.product_key not in plugin.DEVICE_IDS:
                 continue
             client = self.create_client(device, handler, plugin)
             if not client:
@@ -472,9 +484,14 @@ class DeviceMgr(ThreadJob, PrintError):
             infos = self.unpaired_device_infos(handler, plugin, devices)
             if infos:
                 break
-            msg = _('Please insert your %s.  Verify the cable is '
-                    'connected and that no other application is using it.\n\n'
-                    'Try to connect again?') % plugin.device
+            msg = _('Please insert your {}').format(plugin.device)
+            if keystore.label:
+                msg += ' ({})'.format(keystore.label)
+            msg += '. {}\n\n{}'.format(
+                _('Verify the cable is connected and that '
+                  'no other application is using it.'),
+                _('Try to connect again?')
+            )
             if not handler.yes_no_question(msg):
                 raise UserCancelled()
             devices = None
@@ -484,27 +501,27 @@ class DeviceMgr(ThreadJob, PrintError):
         for info in infos:
             if info.label == keystore.label:
                 return info
-        msg = _("Please select which %s device to use:") % plugin.device
-        descriptions = [info.label + ' (%s)'%(_("initialized") if info.initialized else _("wiped")) for info in infos]
+        msg = _("Please select which {} device to use:").format(plugin.device)
+        descriptions = [str(info.label) + ' (%s)'%(_("initialized") if info.initialized else _("wiped")) for info in infos]
         c = handler.query_choice(msg, descriptions)
         if c is None:
             raise UserCancelled()
         info = infos[c]
         # save new label
         keystore.set_label(info.label)
-        handler.win.wallet.save_keystore()
+        if handler.win.wallet is not None:
+            handler.win.wallet.save_keystore()
         return info
 
-    def scan_devices(self):
-        # All currently supported hardware libraries use hid, so we
-        # assume it here.  This can be easily abstracted if necessary.
-        # Note this import must be local so those without hardware
-        # wallet libraries are not affected.
-        import hid
-        self.print_error("scanning devices...")
+    def _scan_devices_with_hid(self):
+        try:
+            import hid
+        except ImportError:
+            return []
+
         with self.hid_lock:
             hid_list = hid.enumerate(0, 0)
-        # First see what's connected that we know about
+
         devices = []
         for d in hid_list:
             product_key = (d['vendor_id'], d['product_id'])
@@ -518,14 +535,31 @@ class DeviceMgr(ThreadJob, PrintError):
                 id_ += str(interface_number) + str(usage_page)
                 devices.append(Device(d['path'], interface_number,
                                       id_, product_key, usage_page))
+        return devices
 
-        # Now find out what was disconnected
+    def scan_devices(self):
+        self.print_error("scanning devices...")
+
+        # First see what's connected that we know about
+        devices = self._scan_devices_with_hid()
+
+        # Let plugin handlers enumerate devices we don't know about
+        for f in self.enumerate_func:
+            try:
+                new_devices = f()
+            except BaseException as e:
+                self.print_error('custom device enum failed. func {}, error {}'
+                                 .format(str(f), str(e)))
+            else:
+                devices.extend(new_devices)
+
+        # find out what was disconnected
         pairs = [(dev.path, dev.id_) for dev in devices]
         disconnected_ids = []
         with self.lock:
             connected = {}
             for client, pair in self.clients.items():
-                if pair in pairs:
+                if pair in pairs and client.has_usable_connection_with_device():
                     connected[client] = pair
                 else:
                     disconnected_ids.append(pair[1])

lib/qrscanner.py

@@ -29,18 +29,18 @@ import ctypes
 
 if sys.platform == 'darwin':
     name = 'libzbar.dylib'
-elif sys.platform == 'windows':
-    name = 'libzbar.dll'
+elif sys.platform in ('windows', 'win32'):
+    name = 'libzbar-0.dll'
 else:
     name = 'libzbar.so.0'
 
 try:
     libzbar = ctypes.cdll.LoadLibrary(name)
-except OSError:
+except BaseException:
     libzbar = None
 
 
-def scan_barcode(device='', timeout=-1, display=True, threaded=False):
+def scan_barcode(device='', timeout=-1, display=True, threaded=False, try_again=True):
     if libzbar is None:
         raise RuntimeError("Cannot start QR scanner; zbar not available.")
     libzbar.zbar_symbol_get_data.restype = ctypes.c_char_p
@@ -50,6 +50,10 @@ def scan_barcode(device='', timeout=-1, display=True, threaded=False):
     proc = libzbar.zbar_processor_create(threaded)
     libzbar.zbar_processor_request_size(proc, 640, 480)
     if libzbar.zbar_processor_init(proc, device.encode('utf-8'), display) != 0:
+        if try_again:
+            # workaround for a bug in "ZBar for Windows"
+            # libzbar.zbar_processor_init always seem to fail the first time around
+            return scan_barcode(device, timeout, display, threaded, try_again=False)
         raise RuntimeError("Can not start QR scanner; initialization failed.")
     libzbar.zbar_processor_set_visible(proc)
     if libzbar.zbar_process_one(proc, timeout):

lib/servers_regtest.json

@@ -0,0 +1,8 @@
+{
+    "127.0.0.1": {
+        "pruning": "-",
+        "s": "51022",
+        "t": "51021",
+        "version": "1.2"
+    }
+}

lib/servers_testnet.json

@@ -1,5 +1,5 @@
 {
-    "localhost": {
+    "127.0.0.1": {
         "pruning": "-",
         "s": "51022",
         "t": "51021",

lib/simple_config.py

@@ -5,11 +5,21 @@ import os
 import stat
 
 from copy import deepcopy
-from .util import user_dir, print_error, print_stderr, PrintError
 
-from .bitcoin import MAX_FEE_RATE, FEE_TARGETS
+from .util import (user_dir, print_error, PrintError,
+                   NoDynamicFeeEstimates, format_satoshis)
+from .i18n import _
+
+FEE_ETA_TARGETS = [25, 10, 5, 2]
+FEE_DEPTH_TARGETS = [10000000, 5000000, 2000000, 1000000, 500000, 200000, 100000]
+
+# satoshi per kbyte
+FEERATE_MAX_DYNAMIC = 150000
+FEERATE_WARNING_HIGH_FEE = 100000
+FEERATE_FALLBACK_STATIC_FEE = 1000
+FEERATE_DEFAULT_RELAY = 1000
+FEERATE_STATIC_VALUES = [150, 300, 500, 1000, 1500, 2500, 3500, 5000, 7500, 10000]
 
-SYSTEM_CONFIG_PATH = "/etc/electrum-zcash.conf"
 
 config = None
 
@@ -24,35 +34,37 @@ def set_config(c):
     config = c
 
 
+FINAL_CONFIG_VERSION = 2
+
+
 class SimpleConfig(PrintError):
     """
     The SimpleConfig class is responsible for handling operations involving
     configuration files.
 
-    There are 3 different sources of possible configuration values:
+    There are two different sources of possible configuration values:
         1. Command line options.
         2. User configuration (in the user's config directory)
-        3. System configuration (in /etc/)
-    They are taken in order (1. overrides config options set in 2., that
-    override config set in 3.)
+    They are taken in order (1. overrides config options set in 2.)
     """
-    fee_rates = [5000, 10000, 20000, 30000, 50000, 70000, 100000, 150000, 200000, 300000]
 
-    def __init__(self, options={}, read_system_config_function=None,
-                 read_user_config_function=None, read_user_dir_function=None):
+    def __init__(self, options=None, read_user_config_function=None,
+                 read_user_dir_function=None):
+
+        if options is None:
+            options = {}
 
         # This lock needs to be acquired for updating and reading the config in
         # a thread-safe way.
         self.lock = threading.RLock()
 
+        self.mempool_fees = {}
         self.fee_estimates = {}
         self.fee_estimates_last_updated = {}
         self.last_time_fee_estimates_requested = 0  # zero ensures immediate fees
 
         # The following two functions are there for dependency injection when
         # testing.
-        if read_system_config_function is None:
-            read_system_config_function = read_system_config
         if read_user_config_function is None:
             read_user_config_function = read_user_config
         if read_user_dir_function is None:
@@ -62,90 +74,154 @@ class SimpleConfig(PrintError):
 
         # The command line options
         self.cmdline_options = deepcopy(options)
-
-        # Portable wallets don't use a system config
-        if self.cmdline_options.get('portable', False):
-            self.system_config = {}
-        else:
-            self.system_config = read_system_config_function()
+        # don't allow to be set on CLI:
+        self.cmdline_options.pop('config_version', None)
 
         # Set self.path and read the user config
         self.user_config = {}  # for self.get in electrum_path()
         self.path = self.electrum_path()
         self.user_config = read_user_config_function(self.path)
-        # Upgrade obsolete keys
-        self.fixup_keys({'auto_cycle': 'auto_connect'})
+        if not self.user_config:
+            # avoid new config getting upgraded
+            self.user_config = {'config_version': FINAL_CONFIG_VERSION}
+
+        # config "upgrade" - CLI options
+        self.rename_config_keys(
+            self.cmdline_options, {'auto_cycle': 'auto_connect'}, True)
+
+        # config upgrade - user config
+        if self.requires_upgrade():
+            self.upgrade()
+
         # Make a singleton instance of 'self'
         set_config(self)
 
     def electrum_path(self):
-        # Read electrum_path from command line / system configuration
+        # Read electrum_path from command line
         # Otherwise use the user's default data directory.
         path = self.get('electrum_path')
         if path is None:
             path = self.user_dir()
 
+        def make_dir(path):
+            # Make directory if it does not yet exist.
+            if not os.path.exists(path):
+                if os.path.islink(path):
+                    raise Exception('Dangling link: ' + path)
+                os.mkdir(path)
+                os.chmod(path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
+
+        make_dir(path)
         if self.get('testnet'):
             path = os.path.join(path, 'testnet')
-
-        # Make directory if it does not yet exist.
-        if not os.path.exists(path):
-            if os.path.islink(path):
-                raise BaseException('Dangling link: ' + path)
-            os.makedirs(path)
-            os.chmod(path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
+            make_dir(path)
+        elif self.get('regtest'):
+            path = os.path.join(path, 'regtest')
+            make_dir(path)
 
         self.print_error("electrum-zcash directory", path)
         return path
 
-    def fixup_config_keys(self, config, keypairs):
+    def rename_config_keys(self, config, keypairs, deprecation_warning=False):
+        """Migrate old key names to new ones"""
         updated = False
         for old_key, new_key in keypairs.items():
             if old_key in config:
-                if not new_key in config:
+                if new_key not in config:
                     config[new_key] = config[old_key]
+                    if deprecation_warning:
+                        self.print_stderr('Note that the {} variable has been deprecated. '
+                                     'You should use {} instead.'.format(old_key, new_key))
                 del config[old_key]
                 updated = True
         return updated
 
-    def fixup_keys(self, keypairs):
-        '''Migrate old key names to new ones'''
-        self.fixup_config_keys(self.cmdline_options, keypairs)
-        self.fixup_config_keys(self.system_config, keypairs)
-        if self.fixup_config_keys(self.user_config, keypairs):
-            self.save_user_config()
-
-    def set_key(self, key, value, save = True):
+    def set_key(self, key, value, save=True):
         if not self.is_modifiable(key):
-            print_stderr("Warning: not changing config key '%s' set on the command line" % key)
+            self.print_stderr("Warning: not changing config key '%s' set on the command line" % key)
             return
+        self._set_key_in_user_config(key, value, save)
 
+    def _set_key_in_user_config(self, key, value, save=True):
         with self.lock:
-            self.user_config[key] = value
+            if value is not None:
+                self.user_config[key] = value
+            else:
+                self.user_config.pop(key, None)
             if save:
                 self.save_user_config()
-        return
 
     def get(self, key, default=None):
         with self.lock:
             out = self.cmdline_options.get(key)
             if out is None:
-                out = self.user_config.get(key)
-                if out is None:
-                    out = self.system_config.get(key, default)
+                out = self.user_config.get(key, default)
         return out
 
+    def requires_upgrade(self):
+        return self.get_config_version() < FINAL_CONFIG_VERSION
+
+    def upgrade(self):
+        with self.lock:
+            self.print_error('upgrading config')
+
+            self.convert_version_2()
+
+            self.set_key('config_version', FINAL_CONFIG_VERSION, save=True)
+
+    def convert_version_2(self):
+        if not self._is_upgrade_method_needed(1, 1):
+            return
+
+        self.rename_config_keys(self.user_config, {'auto_cycle': 'auto_connect'})
+
+        try:
+            # change server string FROM host:port:proto TO host:port:s
+            server_str = self.user_config.get('server')
+            host, port, protocol = str(server_str).rsplit(':', 2)
+            assert protocol in ('s', 't')
+            int(port)  # Throw if cannot be converted to int
+            server_str = '{}:{}:s'.format(host, port)
+            self._set_key_in_user_config('server', server_str)
+        except BaseException:
+            self._set_key_in_user_config('server', None)
+
+        self.set_key('config_version', 2)
+
+    def _is_upgrade_method_needed(self, min_version, max_version):
+        cur_version = self.get_config_version()
+        if cur_version > max_version:
+            return False
+        elif cur_version < min_version:
+            raise Exception(
+                ('config upgrade: unexpected version %d (should be %d-%d)'
+                 % (cur_version, min_version, max_version)))
+        else:
+            return True
+
+    def get_config_version(self):
+        config_version = self.get('config_version', 1)
+        if config_version > FINAL_CONFIG_VERSION:
+            self.print_stderr('WARNING: config version ({}) is higher than ours ({})'
+                             .format(config_version, FINAL_CONFIG_VERSION))
+        return config_version
+
     def is_modifiable(self, key):
-        return not key in self.cmdline_options
+        return key not in self.cmdline_options
 
     def save_user_config(self):
         if not self.path:
             return
         path = os.path.join(self.path, "config")
         s = json.dumps(self.user_config, indent=4, sort_keys=True)
-        with open(path, "w") as f:
-            f.write(s)
-        os.chmod(path, stat.S_IREAD | stat.S_IWRITE)
+        try:
+            with open(path, "w", encoding='utf-8') as f:
+                f.write(s)
+            os.chmod(path, stat.S_IREAD | stat.S_IWRITE)
+        except FileNotFoundError:
+            # datadir probably deleted while running...
+            if os.path.exists(self.path):  # or maybe not?
+                raise
 
     def get_wallet_path(self):
         """Set the path of the wallet."""
@@ -160,10 +236,14 @@ class SimpleConfig(PrintError):
             return path
 
         # default path
+        if not os.path.exists(self.path):
+            raise FileNotFoundError(
+                _('Electrum-Zcash datadir does not exist. Was it deleted while running?') + '\n' +
+                _('Should be at {}').format(self.path))
         dirpath = os.path.join(self.path, "wallets")
         if not os.path.exists(dirpath):
             if os.path.islink(dirpath):
-                raise BaseException('Dangling link: ' + dirpath)
+                raise Exception('Dangling link: ' + dirpath)
             os.mkdir(dirpath)
             os.chmod(dirpath, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
 
@@ -200,57 +280,203 @@ class SimpleConfig(PrintError):
             path = wallet.storage.path
             self.set_key('gui_last_wallet', path)
 
-    def max_fee_rate(self):
-        f = self.get('max_fee_rate', MAX_FEE_RATE)
-        if f==0:
-            f = MAX_FEE_RATE
-        return f
+    def impose_hard_limits_on_fee(func):
+        def get_fee_within_limits(self, *args, **kwargs):
+            fee = func(self, *args, **kwargs)
+            if fee is None:
+                return fee
+            fee = min(FEERATE_MAX_DYNAMIC, fee)
+            fee = max(FEERATE_DEFAULT_RELAY, fee)
+            return fee
+        return get_fee_within_limits
 
-    def dynfee(self, i):
+    @impose_hard_limits_on_fee
+    def eta_to_fee(self, i):
+        """Returns fee in sat/kbyte."""
         if i < 4:
-            j = FEE_TARGETS[i]
+            j = FEE_ETA_TARGETS[i]
             fee = self.fee_estimates.get(j)
         else:
             assert i == 4
             fee = self.fee_estimates.get(2)
             if fee is not None:
                 fee += fee/2
-        if fee is not None:
-            fee = min(5*MAX_FEE_RATE, fee)
         return fee
 
-    def reverse_dynfee(self, fee_per_kb):
+    def fee_to_depth(self, target_fee):
+        depth = 0
+        for fee, s in self.mempool_fees:
+            depth += s
+            if fee <= target_fee:
+                break
+        else:
+            return 0
+        return depth
+
+    @impose_hard_limits_on_fee
+    def depth_to_fee(self, i):
+        """Returns fee in sat/kbyte."""
+        target = self.depth_target(i)
+        depth = 0
+        for fee, s in self.mempool_fees:
+            depth += s
+            if depth > target:
+                break
+        else:
+            return 0
+        return fee * 1000
+
+    def depth_target(self, i):
+        return FEE_DEPTH_TARGETS[i]
+
+    def eta_target(self, i):
+        if i == len(FEE_ETA_TARGETS):
+            return 1
+        return FEE_ETA_TARGETS[i]
+
+    def fee_to_eta(self, fee_per_kb):
         import operator
-        l = list(self.fee_estimates.items()) + [(1, self.dynfee(4))]
+        l = list(self.fee_estimates.items()) + [(1, self.eta_to_fee(4))]
         dist = map(lambda x: (x[0], abs(x[1] - fee_per_kb)), l)
         min_target, min_value = min(dist, key=operator.itemgetter(1))
         if fee_per_kb < self.fee_estimates.get(25)/2:
             min_target = -1
         return min_target
 
+    def depth_tooltip(self, depth):
+        return "%.1f MB from tip"%(depth/1000000)
+
+    def eta_tooltip(self, x):
+        if x < 0:
+            return _('Low fee')
+        elif x == 1:
+            return _('In the next block')
+        else:
+            return _('Within {} blocks').format(x)
+
+    def get_fee_status(self):
+        dyn = self.is_dynfee()
+        mempool = self.use_mempool_fees()
+        pos = self.get_depth_level() if mempool else self.get_fee_level()
+        fee_rate = self.fee_per_kb()
+        target, tooltip = self.get_fee_text(pos, dyn, mempool, fee_rate)
+        return tooltip + '  [%s]'%target if dyn else target + '  [Static]'
+
+    def get_fee_text(self, pos, dyn, mempool, fee_rate):
+        """Returns (text, tooltip) where
+        text is what we target: static fee / num blocks to confirm in / mempool depth
+        tooltip is the corresponding estimate (e.g. num blocks for a static fee)
+        """
+        rate_str = (format_satoshis(fee_rate/1000, False, 0, 0, False)  + ' sat/byte') if fee_rate is not None else 'unknown'
+        if dyn:
+            if mempool:
+                depth = self.depth_target(pos)
+                text = self.depth_tooltip(depth)
+            else:
+                eta = self.eta_target(pos)
+                text = self.eta_tooltip(eta)
+            tooltip = rate_str
+        else:
+            text = rate_str
+            if mempool and self.has_fee_mempool():
+                depth = self.fee_to_depth(fee_rate)
+                tooltip = self.depth_tooltip(depth)
+            elif not mempool and self.has_fee_etas():
+                eta = self.fee_to_eta(fee_rate)
+                tooltip = self.eta_tooltip(eta)
+            else:
+                tooltip = ''
+        return text, tooltip
+
+    def get_depth_level(self):
+        maxp = len(FEE_DEPTH_TARGETS) - 1
+        return min(maxp, self.get('depth_level', 2))
+
+    def get_fee_level(self):
+        maxp = len(FEE_ETA_TARGETS)  # not (-1) to have "next block"
+        return min(maxp, self.get('fee_level', 2))
+
+    def get_fee_slider(self, dyn, mempool):
+        if dyn:
+            if mempool:
+                pos = self.get_depth_level()
+                maxp = len(FEE_DEPTH_TARGETS) - 1
+                fee_rate = self.depth_to_fee(pos)
+            else:
+                pos = self.get_fee_level()
+                maxp = len(FEE_ETA_TARGETS)  # not (-1) to have "next block"
+                fee_rate = self.eta_to_fee(pos)
+        else:
+            fee_rate = self.fee_per_kb(dyn=False)
+            pos = self.static_fee_index(fee_rate)
+            maxp = 9
+        return maxp, pos, fee_rate
+
     def static_fee(self, i):
-        return self.fee_rates[i]
+        return FEERATE_STATIC_VALUES[i]
 
     def static_fee_index(self, value):
-        dist = list(map(lambda x: abs(x - value), self.fee_rates))
+        if value is None:
+            raise TypeError('static fee cannot be None')
+        dist = list(map(lambda x: abs(x - value), FEERATE_STATIC_VALUES))
         return min(range(len(dist)), key=dist.__getitem__)
 
-    def has_fee_estimates(self):
-        return len(self.fee_estimates)==4
+    def has_fee_etas(self):
+        return len(self.fee_estimates) == 4
+
+    def has_fee_mempool(self):
+        return bool(self.mempool_fees)
+
+    def has_dynamic_fees_ready(self):
+        if self.use_mempool_fees():
+            return self.has_fee_mempool()
+        else:
+            return self.has_fee_etas()
 
     def is_dynfee(self):
-        return self.get('dynamic_fees', True)
+        return bool(self.get('dynamic_fees', True))
 
-    def fee_per_kb(self):
-        dyn = self.is_dynfee()
+    def use_mempool_fees(self):
+        return bool(self.get('mempool_fees', False))
+
+    def fee_per_kb(self, dyn=None, mempool=None):
+        """Returns sat/kvB fee to pay for a txn.
+        Note: might return None.
+        """
+        if dyn is None:
+            dyn = self.is_dynfee()
+        if mempool is None:
+            mempool = self.use_mempool_fees()
         if dyn:
-            fee_rate = self.dynfee(self.get('fee_level', 2))
+            if mempool:
+                fee_rate = self.depth_to_fee(self.get_depth_level())
+            else:
+                fee_rate = self.eta_to_fee(self.get_fee_level())
         else:
-            fee_rate = self.get('fee_per_kb', self.max_fee_rate()/2)
+            fee_rate = self.get('fee_per_kb', FEERATE_FALLBACK_STATIC_FEE)
         return fee_rate
 
+    def fee_per_byte(self):
+        """Returns sat/vB fee to pay for a txn.
+        Note: might return None.
+        """
+        fee_per_kb = self.fee_per_kb()
+        return fee_per_kb / 1000 if fee_per_kb is not None else None
+
     def estimate_fee(self, size):
-        return int(self.fee_per_kb() * size / 1000.)
+        fee_per_kb = self.fee_per_kb()
+        if fee_per_kb is None:
+            raise NoDynamicFeeEstimates()
+        return self.estimate_fee_for_feerate(fee_per_kb, size)
+
+    @classmethod
+    def estimate_fee_for_feerate(cls, fee_per_kb, size):
+        # note: We only allow integer sat/byte values atm.
+        # The GUI for simplicity reasons only displays integer sat/byte,
+        # and for the sake of consistency, we thus only use integer sat/byte in
+        # the backend too.
+        fee_per_byte = int(fee_per_kb / 1000)
+        return int(fee_per_byte * size)
 
     def update_fee_estimates(self, key, value):
         self.fee_estimates[key] = value
@@ -261,11 +487,7 @@ class SimpleConfig(PrintError):
         Returns True if an update should be requested.
         """
         now = time.time()
-        prev_updates = self.fee_estimates_last_updated.values()
-        oldest_fee_time = min(prev_updates) if prev_updates else 0
-        stale_fees = now - oldest_fee_time > 7200
-        old_request = now - self.last_time_fee_estimates_requested > 60
-        return stale_fees and old_request
+        return now - self.last_time_fee_estimates_requested > 60
 
     def requested_fee_estimates(self):
         self.last_time_fee_estimates_requested = time.time()
@@ -277,21 +499,6 @@ class SimpleConfig(PrintError):
         return device
 
 
-def read_system_config(path=SYSTEM_CONFIG_PATH):
-    """Parse and return the system config settings in /etc/electrum-zcash.conf."""
-    result = {}
-    if os.path.exists(path):
-        import configparser
-        p = configparser.ConfigParser()
-        try:
-            p.read(path)
-            for k, v in p.items('client'):
-                result[k] = v
-        except (configparser.NoSectionError, configparser.MissingSectionHeaderError):
-            pass
-
-    return result
-
 def read_user_config(path):
     """Parse and store the user config settings in electrum-zcash.conf into user_config[]."""
     if not path:
@@ -300,7 +507,7 @@ def read_user_config(path):
     if not os.path.exists(config_path):
         return {}
     try:
-        with open(config_path, "r") as f:
+        with open(config_path, "r", encoding='utf-8') as f:
             data = f.read()
         result = json.loads(data)
     except:

lib/storage.py

@@ -33,7 +33,7 @@ import pbkdf2, hmac, hashlib
 import base64
 import zlib
 
-from .util import PrintError, profiler
+from .util import PrintError, profiler, InvalidPassword, WalletFileException
 from .plugins import run_hook, plugin_loaders
 from .keystore import bip44_derivation
 from . import bitcoin
@@ -51,11 +51,20 @@ FINAL_SEED_VERSION = 16     # electrum >= 2.7 will set this to prevent
 def multisig_type(wallet_type):
     '''If wallet_type is mofn multi-sig, return [m, n],
     otherwise return None.'''
+    if not wallet_type:
+        return None
     match = re.match('(\d+)of(\d+)', wallet_type)
     if match:
         match = [int(x) for x in match.group(1, 2)]
     return match
 
+def get_derivation_used_for_hw_device_encryption():
+    return ("m"
+            "/4541509'"      # ascii 'ELE'  as decimal ("BIP43 purpose")
+            "/1112098098'")  # ascii 'BIE2' as decimal
+
+# storage encryption version
+STO_EV_PLAINTEXT, STO_EV_USER_PW, STO_EV_XPUB_PW = range(0, 3)
 
 class WalletStorage(PrintError):
 
@@ -68,11 +77,13 @@ class WalletStorage(PrintError):
         self.modified = False
         self.pubkey = None
         if self.file_exists():
-            with open(self.path, "r") as f:
+            with open(self.path, "r", encoding='utf-8') as f:
                 self.raw = f.read()
+            self._encryption_version = self._init_encryption_version()
             if not self.is_encrypted():
                 self.load_data(self.raw)
         else:
+            self._encryption_version = STO_EV_PLAINTEXT
             # avoid new wallets getting 'upgraded'
             self.put('seed_version', FINAL_SEED_VERSION)
 
@@ -102,15 +113,51 @@ class WalletStorage(PrintError):
 
         if not self.manual_upgrades:
             if self.requires_split():
-                raise BaseException("This wallet has multiple accounts and must be split")
+                raise WalletFileException("This wallet has multiple accounts and must be split")
             if self.requires_upgrade():
                 self.upgrade()
 
+    def is_past_initial_decryption(self):
+        """Return if storage is in a usable state for normal operations.
+
+        The value is True exactly
+            if encryption is disabled completely (self.is_encrypted() == False),
+            or if encryption is enabled but the contents have already been decrypted.
+        """
+        return bool(self.data)
+
     def is_encrypted(self):
+        """Return if storage encryption is currently enabled."""
+        return self.get_encryption_version() != STO_EV_PLAINTEXT
+
+    def is_encrypted_with_user_pw(self):
+        return self.get_encryption_version() == STO_EV_USER_PW
+
+    def is_encrypted_with_hw_device(self):
+        return self.get_encryption_version() == STO_EV_XPUB_PW
+
+    def get_encryption_version(self):
+        """Return the version of encryption used for this storage.
+
+        0: plaintext / no encryption
+
+        ECIES, private key derived from a password,
+        1: password is provided by user
+        2: password is derived from an xpub; used with hw wallets
+        """
+        return self._encryption_version
+
+    def _init_encryption_version(self):
         try:
-            return base64.b64decode(self.raw)[0:4] == b'BIE1'
+            magic = base64.b64decode(self.raw)[0:4]
+            if magic == b'BIE1':
+                return STO_EV_USER_PW
+            elif magic == b'BIE2':
+                return STO_EV_XPUB_PW
+            else:
+                return STO_EV_PLAINTEXT
         except:
-            return False
+            return STO_EV_PLAINTEXT
 
     def file_exists(self):
         return self.path and os.path.exists(self.path)
@@ -120,20 +167,50 @@ class WalletStorage(PrintError):
         ec_key = bitcoin.EC_KEY(secret)
         return ec_key
 
+    def _get_encryption_magic(self):
+        v = self._encryption_version
+        if v == STO_EV_USER_PW:
+            return b'BIE1'
+        elif v == STO_EV_XPUB_PW:
+            return b'BIE2'
+        else:
+            raise WalletFileException('no encryption magic for version: %s' % v)
+
     def decrypt(self, password):
         ec_key = self.get_key(password)
-        s = zlib.decompress(ec_key.decrypt_message(self.raw)) if self.raw else None
+        if self.raw:
+            enc_magic = self._get_encryption_magic()
+            s = zlib.decompress(ec_key.decrypt_message(self.raw, enc_magic))
+        else:
+            s = None
         self.pubkey = ec_key.get_public_key()
         s = s.decode('utf8')
         self.load_data(s)
 
-    def set_password(self, password, encrypt):
-        self.put('use_encryption', bool(password))
-        if encrypt and password:
+    def check_password(self, password):
+        """Raises an InvalidPassword exception on invalid password"""
+        if not self.is_encrypted():
+            return
+        if self.pubkey and self.pubkey != self.get_key(password).get_public_key():
+            raise InvalidPassword()
+
+    def set_keystore_encryption(self, enable):
+        self.put('use_encryption', enable)
+
+    def set_password(self, password, enc_version=None):
+        """Set a password to be used for encrypting this storage."""
+        if enc_version is None:
+            enc_version = self._encryption_version
+        if password and enc_version != STO_EV_PLAINTEXT:
             ec_key = self.get_key(password)
             self.pubkey = ec_key.get_public_key()
+            self._encryption_version = enc_version
         else:
             self.pubkey = None
+            self._encryption_version = STO_EV_PLAINTEXT
+        # make sure next storage.write() saves changes
+        with self.lock:
+            self.modified = True
 
     def get(self, key, default=None):
         with self.lock:
@@ -175,11 +252,12 @@ class WalletStorage(PrintError):
         if self.pubkey:
             s = bytes(s, 'utf8')
             c = zlib.compress(s)
-            s = bitcoin.encrypt_message(c, self.pubkey)
+            enc_magic = self._get_encryption_magic()
+            s = bitcoin.encrypt_message(c, self.pubkey, enc_magic)
             s = s.decode('utf8')
 
         temp_path = "%s.tmp.%s" % (self.path, os.getpid())
-        with open(temp_path, "w") as f:
+        with open(temp_path, "w", encoding='utf-8') as f:
             f.write(s)
             f.flush()
             os.fsync(f.fileno())
@@ -242,7 +320,7 @@ class WalletStorage(PrintError):
                 storage2.write()
                 result.append(new_path)
         else:
-            raise BaseException("This wallet has multiple accounts and must be split")
+            raise WalletFileException("This wallet has multiple accounts and must be split")
         return result
 
     def requires_upgrade(self):
@@ -263,6 +341,9 @@ class WalletStorage(PrintError):
         self.write()
 
     def convert_wallet_type(self):
+        if not self._is_upgrade_method_needed(0, 13):
+            return
+
         wallet_type = self.get('wallet_type')
         if wallet_type == 'btchip': wallet_type = 'ledger'
         if self.get('keystore') or self.get('x1/') or wallet_type=='imported':
@@ -338,7 +419,7 @@ class WalletStorage(PrintError):
                     d['seed'] = seed
                 self.put(key, d)
         else:
-            raise
+            raise WalletFileException('Unable to tell wallet type. Is this even a wallet file?')
         # remove junk
         self.put('master_public_key', None)
         self.put('master_public_keys', None)
@@ -445,6 +526,9 @@ class WalletStorage(PrintError):
         self.put('seed_version', 16)
 
     def convert_imported(self):
+        if not self._is_upgrade_method_needed(0, 13):
+            return
+
         # '/x' is the internal ID for imported accounts
         d = self.get('accounts', {}).get('/x', {}).get('imported',{})
         if not d:
@@ -458,7 +542,7 @@ class WalletStorage(PrintError):
             else:
                 addresses.append(addr)
         if addresses and keypairs:
-            raise BaseException('mixed addresses and privkeys')
+            raise WalletFileException('mixed addresses and privkeys')
         elif addresses:
             self.put('addresses', addresses)
             self.put('accounts', None)
@@ -468,9 +552,12 @@ class WalletStorage(PrintError):
             self.put('keypairs', keypairs)
             self.put('accounts', None)
         else:
-            raise BaseException('no addresses or privkeys')
+            raise WalletFileException('no addresses or privkeys')
 
     def convert_account(self):
+        if not self._is_upgrade_method_needed(0, 13):
+            return
+
         self.put('accounts', None)
 
     def _is_upgrade_method_needed(self, min_version, max_version):
@@ -478,9 +565,9 @@ class WalletStorage(PrintError):
         if cur_version > max_version:
             return False
         elif cur_version < min_version:
-            raise BaseException(
-                ('storage upgrade: unexpected version %d (should be %d-%d)'
-                 % (cur_version, min_version, max_version)))
+            raise WalletFileException(
+                'storage upgrade: unexpected version {} (should be {}-{})'
+                .format(cur_version, min_version, max_version))
         else:
             return True
 
@@ -496,7 +583,9 @@ class WalletStorage(PrintError):
         if not seed_version:
             seed_version = OLD_SEED_VERSION if len(self.get('master_public_key','')) == 128 else NEW_SEED_VERSION
         if seed_version > FINAL_SEED_VERSION:
-            raise BaseException('This version of Electrum is too old to open this wallet')
+            raise WalletFileException('This version of Electrum is too old to open this wallet.\n'
+                                      '(highest supported storage version: {}, version of this file: {})'
+                                      .format(FINAL_SEED_VERSION, seed_version))
         if seed_version >=12:
             return seed_version
         if seed_version not in [OLD_SEED_VERSION, NEW_SEED_VERSION]:
@@ -517,4 +606,4 @@ class WalletStorage(PrintError):
             else:
                 # creation was complete if electrum was run from source
                 msg += "\nPlease open this file with Electrum 1.9.8, and move your coins to a new wallet."
-        raise BaseException(msg)
+        raise WalletFileException(msg)

lib/synchronizer.py

@@ -50,6 +50,8 @@ class Synchronizer(ThreadJob):
         self.requested_histories = {}
         self.requested_addrs = set()
         self.lock = Lock()
+
+        self.initialized = False
         self.initialize()
 
     def parse_response(self, response):
@@ -84,11 +86,13 @@ class Synchronizer(ThreadJob):
         return bh2u(hashlib.sha256(status.encode('ascii')).digest())
 
     def on_address_status(self, response):
+        if self.wallet.synchronizer is None and self.initialized:
+            return  # we have been killed, this was just an orphan callback
         params, result = self.parse_response(response)
         if not params:
             return
         addr = params[0]
-        history = self.wallet.get_address_history(addr)
+        history = self.wallet.history.get(addr, [])
         if self.get_status(history) != result:
             if self.requested_histories.get(addr) is None:
                 self.requested_histories[addr] = result
@@ -98,12 +102,17 @@ class Synchronizer(ThreadJob):
             self.requested_addrs.remove(addr)
 
     def on_address_history(self, response):
+        if self.wallet.synchronizer is None and self.initialized:
+            return  # we have been killed, this was just an orphan callback
         params, result = self.parse_response(response)
         if not params:
             return
         addr = params[0]
+        server_status = self.requested_histories.get(addr)
+        if server_status is None:
+            self.print_error("receiving history (unsolicited)", addr, len(result))
+            return
         self.print_error("receiving history", addr, len(result))
-        server_status = self.requested_histories[addr]
         hashes = set(map(lambda item: item['tx_hash'], result))
         hist = list(map(lambda item: (item['tx_hash'], item['height']), result))
         # tx_fees
@@ -127,6 +136,8 @@ class Synchronizer(ThreadJob):
         self.requested_histories.pop(addr)
 
     def tx_response(self, response):
+        if self.wallet.synchronizer is None and self.initialized:
+            return  # we have been killed, this was just an orphan callback
         params, result = self.parse_response(response)
         if not params:
             return
@@ -177,6 +188,7 @@ class Synchronizer(ThreadJob):
         if self.requested_tx:
             self.print_error("missing tx", self.requested_tx)
         self.subscribe_to_addresses(set(self.wallet.get_addresses()))
+        self.initialized = True
 
     def run(self):
         '''Called from the network proxy thread main loop.'''

lib/tests/__init__.py

@@ -0,0 +1,16 @@
+import unittest
+
+from lib import constants
+
+
+class TestCaseForTestnet(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        super().setUpClass()
+        constants.set_testnet()
+
+    @classmethod
+    def tearDownClass(cls):
+        super().tearDownClass()
+        constants.set_mainnet()

lib/tests/test_bitcoin.py

@@ -11,8 +11,12 @@ from lib.bitcoin import (
     var_int, op_push, address_to_script, regenerate_key,
     verify_message, deserialize_privkey, serialize_privkey,
     is_b58_address, address_to_scripthash, is_minikey, is_compressed, is_xpub,
-    xpub_type, is_xprv, is_bip32_derivation, seed_type, NetworkConstants)
+    xpub_type, is_xprv, is_bip32_derivation, seed_type, EncodeBase58Check)
 from lib.util import bfh
+from lib import constants
+
+from . import TestCaseForTestnet
+
 
 try:
     import ecdsa
@@ -140,11 +144,11 @@ class Test_bitcoin(unittest.TestCase):
         self.assertEqual(op_push(0x4b), '4b')
         self.assertEqual(op_push(0x4c), '4c4c')
         self.assertEqual(op_push(0xfe), '4cfe')
-        self.assertEqual(op_push(0xff), '4dff00')
+        self.assertEqual(op_push(0xff), '4cff')
         self.assertEqual(op_push(0x100), '4d0001')
         self.assertEqual(op_push(0x1234), '4d3412')
         self.assertEqual(op_push(0xfffe), '4dfeff')
-        self.assertEqual(op_push(0xffff), '4effff0000')
+        self.assertEqual(op_push(0xffff), '4dffff')
         self.assertEqual(op_push(0x10000), '4e00000100')
         self.assertEqual(op_push(0x12345678), '4e78563412')
 
@@ -158,17 +162,7 @@ class Test_bitcoin(unittest.TestCase):
         self.assertEqual(address_to_script('t3grLzdTrjSSiCFXzxV5YCvkYZt2tJjDLau'), 'a914f47c8954e421031ad04ecd8e7752c9479206b9d387')
 
 
-class Test_bitcoin_testnet(unittest.TestCase):
-
-    @classmethod
-    def setUpClass(cls):
-        super().setUpClass()
-        NetworkConstants.set_testnet()
-
-    @classmethod
-    def tearDownClass(cls):
-        super().tearDownClass()
-        NetworkConstants.set_mainnet()
+class Test_bitcoin_testnet(TestCaseForTestnet):
 
     def test_address_to_script(self):
         # base58 P2PKH
@@ -250,11 +244,69 @@ class Test_xprv_xpub(unittest.TestCase):
         self.assertFalse(is_bip32_derivation(""))
         self.assertFalse(is_bip32_derivation("m/q8462"))
 
+    def test_version_bytes(self):
+        xprv_headers_b58 = {
+            'standard':    'xprv',
+        }
+        xpub_headers_b58 = {
+            'standard':    'xpub',
+        }
+        for xtype, xkey_header_bytes in constants.net.XPRV_HEADERS.items():
+            xkey_header_bytes = bfh("%08x" % xkey_header_bytes)
+            xkey_bytes = xkey_header_bytes + bytes([0] * 74)
+            xkey_b58 = EncodeBase58Check(xkey_bytes)
+            self.assertTrue(xkey_b58.startswith(xprv_headers_b58[xtype]))
+
+            xkey_bytes = xkey_header_bytes + bytes([255] * 74)
+            xkey_b58 = EncodeBase58Check(xkey_bytes)
+            self.assertTrue(xkey_b58.startswith(xprv_headers_b58[xtype]))
+
+        for xtype, xkey_header_bytes in constants.net.XPUB_HEADERS.items():
+            xkey_header_bytes = bfh("%08x" % xkey_header_bytes)
+            xkey_bytes = xkey_header_bytes + bytes([0] * 74)
+            xkey_b58 = EncodeBase58Check(xkey_bytes)
+            self.assertTrue(xkey_b58.startswith(xpub_headers_b58[xtype]))
+
+            xkey_bytes = xkey_header_bytes + bytes([255] * 74)
+            xkey_b58 = EncodeBase58Check(xkey_bytes)
+            self.assertTrue(xkey_b58.startswith(xpub_headers_b58[xtype]))
+
+
+class Test_xprv_xpub_testnet(TestCaseForTestnet):
+
+    def test_version_bytes(self):
+        xprv_headers_b58 = {
+            'standard':    'tprv',
+        }
+        xpub_headers_b58 = {
+            'standard':    'tpub',
+        }
+        for xtype, xkey_header_bytes in constants.net.XPRV_HEADERS.items():
+            xkey_header_bytes = bfh("%08x" % xkey_header_bytes)
+            xkey_bytes = xkey_header_bytes + bytes([0] * 74)
+            xkey_b58 = EncodeBase58Check(xkey_bytes)
+            self.assertTrue(xkey_b58.startswith(xprv_headers_b58[xtype]))
+
+            xkey_bytes = xkey_header_bytes + bytes([255] * 74)
+            xkey_b58 = EncodeBase58Check(xkey_bytes)
+            self.assertTrue(xkey_b58.startswith(xprv_headers_b58[xtype]))
+
+        for xtype, xkey_header_bytes in constants.net.XPUB_HEADERS.items():
+            xkey_header_bytes = bfh("%08x" % xkey_header_bytes)
+            xkey_bytes = xkey_header_bytes + bytes([0] * 74)
+            xkey_b58 = EncodeBase58Check(xkey_bytes)
+            self.assertTrue(xkey_b58.startswith(xpub_headers_b58[xtype]))
+
+            xkey_bytes = xkey_header_bytes + bytes([255] * 74)
+            xkey_b58 = EncodeBase58Check(xkey_bytes)
+            self.assertTrue(xkey_b58.startswith(xpub_headers_b58[xtype]))
+
 
 class Test_keyImport(unittest.TestCase):
 
     priv_pub_addr = (
            {'priv': 'KzMFjMC2MPadjvX5Cd7b8AKKjjpBSoRKUTpoAtN6B3J9ezWYyXS6',
+            'exported_privkey': 'p2pkh:KzMFjMC2MPadjvX5Cd7b8AKKjjpBSoRKUTpoAtN6B3J9ezWYyXS6',
             'pub': '02c6467b7e621144105ed3e4835b0b4ab7e35266a2ae1c4f8baa19e9ca93452997',
             'address': 't1QTbqnYayRQQ2QZDUgYAcZ6EAcuddyRMbu',
             'minikey' : False,
@@ -262,7 +314,17 @@ class Test_keyImport(unittest.TestCase):
             'compressed': True,
             'addr_encoding': 'base58',
             'scripthash': 'c9aecd1fef8d661a42c560bf75c8163e337099800b8face5ca3d1393a30508a7'},
+           {'priv': 'p2pkh:Kzj8VjwpZ99bQqVeUiRXrKuX9mLr1o6sWxFMCBJn1umC38BMiQTD',
+            'exported_privkey': 'p2pkh:Kzj8VjwpZ99bQqVeUiRXrKuX9mLr1o6sWxFMCBJn1umC38BMiQTD',
+            'pub': '0352d78b4b37e0f6d4e164423436f2925fa57817467178eca550a88f2821973c41',
+            'address': 't1ZQHZQpr51Z83vYLkuCgUnA7xgygSNEL8E',
+            'minikey': False,
+            'txin_type': 'p2pkh',
+            'compressed': True,
+            'addr_encoding': 'base58',
+            'scripthash': 'a9b2a76fc196c553b352186dfcca81fcf323a721cd8431328f8e9d54216818c1'},
            {'priv': '5Hxn5C4SQuiV6e62A1MtZmbSeQyrLFhu5uYks62pU5VBUygK2KD',
+            'exported_privkey': 'p2pkh:5Hxn5C4SQuiV6e62A1MtZmbSeQyrLFhu5uYks62pU5VBUygK2KD',
             'pub': '04e5fe91a20fac945845a5518450d23405ff3e3e1ce39827b47ee6d5db020a9075422d56a59195ada0035e4a52a238849f68e7a325ba5b2247013e0481c5c7cb3f',
             'address': 't1ZFtVnxGSXwNZjnsKVhiAGeEC8nJPuJDDp',
             'minikey': False,
@@ -270,8 +332,18 @@ class Test_keyImport(unittest.TestCase):
             'compressed': False,
             'addr_encoding': 'base58',
             'scripthash': 'f5914651408417e1166f725a5829ff9576d0dbf05237055bf13abd2af7f79473'},
+           {'priv': 'p2pkh:5KhYQCe1xd5g2tqpmmGpUWDpDuTbA8vnpbiCNDwMPAx29WNQYfN',
+            'exported_privkey': 'p2pkh:5KhYQCe1xd5g2tqpmmGpUWDpDuTbA8vnpbiCNDwMPAx29WNQYfN',
+            'pub': '048f0431b0776e8210376c81280011c2b68be43194cb00bd47b7e9aa66284b713ce09556cde3fee606051a07613f3c159ef3953b8927c96ae3dae94a6ba4182e0e',
+            'address': 't1LzMikhRjUTSEzTLdGUBsrBTFZhR9hwffh',
+            'minikey': False,
+            'txin_type': 'p2pkh',
+            'compressed': False,
+            'addr_encoding': 'base58',
+            'scripthash': '6dd2e07ad2de9ba8eec4bbe8467eb53f8845acff0d9e6f5627391acc22ff62df'},
            # from http://bitscan.com/articles/security/spotlight-on-mini-private-keys
            {'priv': 'SzavMBLoXU6kDrqtUVmffv',
+            'exported_privkey': 'p2pkh:L53fCHmQhbNp1B4JipfBtfeHZH7cAibzG9oK19XfiFzxHgAkz6JK',
             'pub': '02588d202afcc1ee4ab5254c7847ec25b9a135bbda0f2bc69ee1a714749fd77dc9',
             'address': 't1S9WvZLVKoLjXUAxssvGojtqx1pABJMJUU',
             'minikey': True,
@@ -309,6 +381,7 @@ class Test_keyImport(unittest.TestCase):
     def test_is_private_key(self):
         for priv_details in self.priv_pub_addr:
             self.assertTrue(is_private_key(priv_details['priv']))
+            self.assertTrue(is_private_key(priv_details['exported_privkey']))
             self.assertFalse(is_private_key(priv_details['pub']))
             self.assertFalse(is_private_key(priv_details['address']))
         self.assertFalse(is_private_key("not a privkey"))
@@ -317,8 +390,7 @@ class Test_keyImport(unittest.TestCase):
         for priv_details in self.priv_pub_addr:
             txin_type, privkey, compressed = deserialize_privkey(priv_details['priv'])
             priv2 = serialize_privkey(privkey, compressed, txin_type)
-            if not priv_details['minikey']:
-                self.assertEqual(priv_details['priv'], priv2)
+            self.assertEqual(priv_details['exported_privkey'], priv2)
 
     def test_address_to_scripthash(self):
         for priv_details in self.priv_pub_addr:

lib/tests/test_commands.py

@@ -0,0 +1,33 @@
+import unittest
+from decimal import Decimal
+
+from lib.commands import Commands
+
+
+class TestCommands(unittest.TestCase):
+
+    def test_setconfig_non_auth_number(self):
+        self.assertEqual(7777, Commands._setconfig_normalize_value('rpcport', "7777"))
+        self.assertEqual(7777, Commands._setconfig_normalize_value('rpcport', '7777'))
+        self.assertAlmostEqual(Decimal(2.3), Commands._setconfig_normalize_value('somekey', '2.3'))
+
+    def test_setconfig_non_auth_number_as_string(self):
+        self.assertEqual("7777", Commands._setconfig_normalize_value('somekey', "'7777'"))
+
+    def test_setconfig_non_auth_boolean(self):
+        self.assertEqual(True, Commands._setconfig_normalize_value('show_console_tab', "true"))
+        self.assertEqual(True, Commands._setconfig_normalize_value('show_console_tab', "True"))
+
+    def test_setconfig_non_auth_list(self):
+        self.assertEqual(['file:///var/www/', 'https://electrum.org'],
+            Commands._setconfig_normalize_value('url_rewrite', "['file:///var/www/','https://electrum.org']"))
+        self.assertEqual(['file:///var/www/', 'https://electrum.org'],
+            Commands._setconfig_normalize_value('url_rewrite', '["file:///var/www/","https://electrum.org"]'))
+
+    def test_setconfig_auth(self):
+        self.assertEqual("7777", Commands._setconfig_normalize_value('rpcuser', "7777"))
+        self.assertEqual("7777", Commands._setconfig_normalize_value('rpcuser', '7777'))
+        self.assertEqual("7777", Commands._setconfig_normalize_value('rpcpassword', '7777'))
+        self.assertEqual("2asd", Commands._setconfig_normalize_value('rpcpassword', '2asd'))
+        self.assertEqual("['file:///var/www/','https://electrum.org']",
+            Commands._setconfig_normalize_value('rpcpassword', "['file:///var/www/','https://electrum.org']"))

lib/tests/test_simple_config.py

@@ -6,8 +6,7 @@ import tempfile
 import shutil
 
 from io import StringIO
-from lib.simple_config import (SimpleConfig, read_system_config,
-                               read_user_config)
+from lib.simple_config import (SimpleConfig, read_user_config)
 
 
 class Test_SimpleConfig(unittest.TestCase):
@@ -37,18 +36,15 @@ class Test_SimpleConfig(unittest.TestCase):
 
     def test_simple_config_key_rename(self):
         """auto_cycle was renamed auto_connect"""
-        fake_read_system = lambda : {}
         fake_read_user = lambda _: {"auto_cycle": True}
         read_user_dir = lambda : self.user_dir
         config = SimpleConfig(options=self.options,
-                              read_system_config_function=fake_read_system,
                               read_user_config_function=fake_read_user,
                               read_user_dir_function=read_user_dir)
         self.assertEqual(config.get("auto_connect"), True)
         self.assertEqual(config.get("auto_cycle"), None)
         fake_read_user = lambda _: {"auto_connect": False, "auto_cycle": True}
         config = SimpleConfig(options=self.options,
-                              read_system_config_function=fake_read_system,
                               read_user_config_function=fake_read_user,
                               read_user_dir_function=read_user_dir)
         self.assertEqual(config.get("auto_connect"), False)
@@ -57,110 +53,51 @@ class Test_SimpleConfig(unittest.TestCase):
     def test_simple_config_command_line_overrides_everything(self):
         """Options passed by command line override all other configuration
         sources"""
-        fake_read_system = lambda : {"electrum_path": "a"}
         fake_read_user = lambda _: {"electrum_path": "b"}
         read_user_dir = lambda : self.user_dir
         config = SimpleConfig(options=self.options,
-                              read_system_config_function=fake_read_system,
                               read_user_config_function=fake_read_user,
                               read_user_dir_function=read_user_dir)
         self.assertEqual(self.options.get("electrum_path"),
                          config.get("electrum_path"))
 
-    def test_simple_config_user_config_overrides_system_config(self):
-        """Options passed in user config override system config."""
-        fake_read_system = lambda : {"electrum_path": self.electrum_dir}
-        fake_read_user = lambda _: {"electrum_path": "b"}
-        read_user_dir = lambda : self.user_dir
-        config = SimpleConfig(options={},
-                              read_system_config_function=fake_read_system,
-                              read_user_config_function=fake_read_user,
-                              read_user_dir_function=read_user_dir)
-        self.assertEqual("b", config.get("electrum_path"))
-
-    def test_simple_config_system_config_ignored_if_portable(self):
-        """If electrum is started with the "portable" flag, system
-        configuration is completely ignored."""
-        fake_read_system = lambda : {"some_key": "some_value"}
-        fake_read_user = lambda _: {}
-        read_user_dir = lambda : self.user_dir
-        config = SimpleConfig(options={"portable": True},
-                              read_system_config_function=fake_read_system,
-                              read_user_config_function=fake_read_user,
-                              read_user_dir_function=read_user_dir)
-        self.assertEqual(config.get("some_key"), None)
-
     def test_simple_config_user_config_is_used_if_others_arent_specified(self):
         """If no system-wide configuration and no command-line options are
         specified, the user configuration is used instead."""
-        fake_read_system = lambda : {}
         fake_read_user = lambda _: {"electrum_path": self.electrum_dir}
         read_user_dir = lambda : self.user_dir
         config = SimpleConfig(options={},
-                              read_system_config_function=fake_read_system,
                               read_user_config_function=fake_read_user,
                               read_user_dir_function=read_user_dir)
         self.assertEqual(self.options.get("electrum_path"),
                          config.get("electrum_path"))
 
     def test_cannot_set_options_passed_by_command_line(self):
-        fake_read_system = lambda : {}
         fake_read_user = lambda _: {"electrum_path": "b"}
         read_user_dir = lambda : self.user_dir
         config = SimpleConfig(options=self.options,
-                              read_system_config_function=fake_read_system,
                               read_user_config_function=fake_read_user,
                               read_user_dir_function=read_user_dir)
         config.set_key("electrum_path", "c")
         self.assertEqual(self.options.get("electrum_path"),
                          config.get("electrum_path"))
 
-    def test_can_set_options_from_system_config(self):
-        fake_read_system = lambda : {"electrum_path": self.electrum_dir}
-        fake_read_user = lambda _: {}
-        read_user_dir = lambda : self.user_dir
-        config = SimpleConfig(options={},
-                              read_system_config_function=fake_read_system,
-                              read_user_config_function=fake_read_user,
-                              read_user_dir_function=read_user_dir)
-        config.set_key("electrum_path", "c")
-        self.assertEqual("c", config.get("electrum_path"))
-
     def test_can_set_options_set_in_user_config(self):
         another_path = tempfile.mkdtemp()
-        fake_read_system = lambda : {}
         fake_read_user = lambda _: {"electrum_path": self.electrum_dir}
         read_user_dir = lambda : self.user_dir
         config = SimpleConfig(options={},
-                              read_system_config_function=fake_read_system,
-                              read_user_config_function=fake_read_user,
-                              read_user_dir_function=read_user_dir)
-        config.set_key("electrum_path", another_path)
-        self.assertEqual(another_path, config.get("electrum_path"))
-
-    def test_can_set_options_from_system_config_if_portable(self):
-        """If the "portable" flag is set, the user can overwrite system
-        configuration options."""
-        another_path = tempfile.mkdtemp()
-        fake_read_system = lambda : {"electrum_path": self.electrum_dir}
-        fake_read_user = lambda _: {}
-        read_user_dir = lambda : self.user_dir
-        config = SimpleConfig(options={"portable": True},
-                              read_system_config_function=fake_read_system,
                               read_user_config_function=fake_read_user,
                               read_user_dir_function=read_user_dir)
         config.set_key("electrum_path", another_path)
         self.assertEqual(another_path, config.get("electrum_path"))
 
     def test_user_config_is_not_written_with_read_only_config(self):
-        """The user config does not contain command-line options or system
-        options when saved."""
-        fake_read_system = lambda : {"something": "b"}
+        """The user config does not contain command-line options when saved."""
         fake_read_user = lambda _: {"something": "a"}
         read_user_dir = lambda : self.user_dir
         self.options.update({"something": "c"})
         config = SimpleConfig(options=self.options,
-                              read_system_config_function=fake_read_system,
                               read_user_config_function=fake_read_user,
                               read_user_dir_function=read_user_dir)
         config.save_user_config()
@@ -168,48 +105,10 @@ class Test_SimpleConfig(unittest.TestCase):
         with open(os.path.join(self.electrum_dir, "config"), "r") as f:
             contents = f.read()
         result = ast.literal_eval(contents)
+        result.pop('config_version', None)
         self.assertEqual({"something": "a"}, result)
 
 
-class TestSystemConfig(unittest.TestCase):
-
-    sample_conf = """
-[client]
-gap_limit = 5
-
-[something_else]
-everything = 42
-"""
-
-    def setUp(self):
-        super(TestSystemConfig, self).setUp()
-        self.thefile = tempfile.mkstemp(suffix=".electrum.test.conf")[1]
-
-    def tearDown(self):
-        super(TestSystemConfig, self).tearDown()
-        os.remove(self.thefile)
-
-    def test_read_system_config_file_does_not_exist(self):
-        somefile = "/foo/I/do/not/exist/electrum.conf"
-        result = read_system_config(somefile)
-        self.assertEqual({}, result)
-
-    def test_read_system_config_file_returns_file_options(self):
-        with open(self.thefile, "w") as f:
-            f.write(self.sample_conf)
-
-        result = read_system_config(self.thefile)
-        self.assertEqual({"gap_limit": "5"}, result)
-
-    def test_read_system_config_file_no_sections(self):
-
-        with open(self.thefile, "w") as f:
-            f.write("gap_limit = 5")  # The file has no sections at all
-
-        result = read_system_config(self.thefile)
-        self.assertEqual({}, result)
-
-
 class TestUserConfig(unittest.TestCase):
 
     def setUp(self):

lib/tests/test_transaction.py

@@ -1,10 +1,9 @@
 import unittest
+
 from lib import transaction
 from lib.bitcoin import TYPE_ADDRESS
-
 from lib.keystore import xpubkey_to_address
-
-from lib.util import bh2u
+from lib.util import bh2u, bfh
 
 unsigned_blob = '01000000012a5c9a94fcde98f5581cd00162c60a13936ceb75389ea65bf38633b424eb4031000000005701ff4c53ff0488b21e03ef2afea18000000089689bff23e1e7fb2f161daa37270a97a3d8c2e537584b2d304ecb47b86d21fc021b010d3bd425f8cf2e04824bfdf1f1f5ff1d51fadd9a41f9e3fb8dd3403b1bfe00000000ffffffff0140420f00000000001976a914230ac37834073a42146f11ef8414ae929feaafc388ac00000000'
 signed_blob = '01000000012a5c9a94fcde98f5581cd00162c60a13936ceb75389ea65bf38633b424eb4031000000006c493046022100a82bbc57a0136751e5433f41cf000b3f1a99c6744775e76ec764fb78c54ee100022100f9e80b7de89de861dc6fb0c1429d5da72c2b6b2ee2406bc9bfb1beedd729d985012102e61d176da16edd1d258a200ad9759ef63adf8e14cd97f53227bae35cdb84d2f6ffffffff0140420f00000000001976a914230ac37834073a42146f11ef8414ae929feaafc388ac00000000'
@@ -133,6 +132,11 @@ class TestTransaction(unittest.TestCase):
         self.assertEqual(tx.estimated_weight(), 772)
         self.assertEqual(tx.estimated_size(), 193)
 
+    def test_estimated_output_size(self):
+        estimated_output_size = transaction.Transaction.estimated_output_size
+        self.assertEqual(estimated_output_size('t1MZDS9LxiXasLqR5fMDK4kDa8TJjSFsMsq'), 34)
+        self.assertEqual(estimated_output_size('t3NSSQe2KNgLcTWy2WsiRAkr7NTtZ15fhLn'), 32)
+
     def test_errors(self):
         with self.assertRaises(TypeError):
             transaction.Transaction.pay_script(output_type=None, addr='')
@@ -148,29 +152,67 @@ class TestTransaction(unittest.TestCase):
         tx = transaction.Transaction(v2_blob)
         self.assertEqual(tx.txid(), "b97f9180173ab141b61b9f944d841e60feec691d6daab4d4d932b24dd36606fe")
 
+    def test_get_address_from_output_script(self):
+        # the inverse of this test is in test_bitcoin: test_address_to_script
+        addr_from_script = lambda script: transaction.get_address_from_output_script(bfh(script))
+        ADDR = transaction.TYPE_ADDRESS
+
+        # base58 p2pkh
+        self.assertEqual((ADDR, 't1MZDS9LxiXasLqR5fMDK4kDa8TJjSFsMsq'), addr_from_script('76a91428662c67561b95c79d2257d2a93d9d151c977e9188ac'))
+        self.assertEqual((ADDR, 't1U7SgL7CWNnawSvZD8k8JgwWUygasy2cp1'), addr_from_script('76a914704f4b81cadb7bf7e68c08cd3657220f680f863c88ac'))
+
+        # base58 p2sh
+        self.assertEqual((ADDR, 't3NSSQe2KNgLcTWy2WsiRAkr7NTtZ15fhLn'), addr_from_script('a9142a84cf00d47f699ee7bbc1dea5ec1bdecb4ac15487'))
+        self.assertEqual((ADDR, 't3grLzdTrjSSiCFXzxV5YCvkYZt2tJjDLau'), addr_from_script('a914f47c8954e421031ad04ecd8e7752c9479206b9d387'))
+
+#####
+
+    def _run_naive_tests_on_tx(self, raw_tx, txid):
+        tx = transaction.Transaction(raw_tx)
+        self.assertEqual(txid, tx.txid())
+        self.assertEqual(raw_tx, tx.serialize())
+        self.assertTrue(tx.estimated_size() >= 0)
+
     def test_txid_coinbase_to_p2pk(self):
-        tx = transaction.Transaction('01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff4103400d0302ef02062f503253482f522cfabe6d6dd90d39663d10f8fd25ec88338295d4c6ce1c90d4aeb368d8bdbadcc1da3b635801000000000000000474073e03ffffffff013c25cf2d01000000434104b0bd634234abbb1ba1e986e884185c61cf43e001f9137f23c2c409273eb16e6537a576782eba668a7ef8bd3b3cfb1edb7117ab65129b8a2e681f3c1e0908ef7bac00000000')
-        self.assertEqual('dbaf14e1c476e76ea05a8b71921a46d6b06f0a950f17c5f9f1a03b8fae467f10', tx.txid())
+        raw_tx = '01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff4103400d0302ef02062f503253482f522cfabe6d6dd90d39663d10f8fd25ec88338295d4c6ce1c90d4aeb368d8bdbadcc1da3b635801000000000000000474073e03ffffffff013c25cf2d01000000434104b0bd634234abbb1ba1e986e884185c61cf43e001f9137f23c2c409273eb16e6537a576782eba668a7ef8bd3b3cfb1edb7117ab65129b8a2e681f3c1e0908ef7bac00000000'
+        txid = 'dbaf14e1c476e76ea05a8b71921a46d6b06f0a950f17c5f9f1a03b8fae467f10'
+        self._run_naive_tests_on_tx(raw_tx, txid)
 
     def test_txid_coinbase_to_p2pkh(self):
-        tx = transaction.Transaction('01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff25033ca0030400001256124d696e656420627920425443204775696c640800000d41000007daffffffff01c00d1298000000001976a91427a1f12771de5cc3b73941664b2537c15316be4388ac00000000')
-        self.assertEqual('4328f9311c6defd9ae1bd7f4516b62acf64b361eb39dfcf09d9925c5fd5c61e8', tx.txid())
+        raw_tx = '01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff25033ca0030400001256124d696e656420627920425443204775696c640800000d41000007daffffffff01c00d1298000000001976a91427a1f12771de5cc3b73941664b2537c15316be4388ac00000000'
+        txid = '4328f9311c6defd9ae1bd7f4516b62acf64b361eb39dfcf09d9925c5fd5c61e8'
+        self._run_naive_tests_on_tx(raw_tx, txid)
 
     def test_txid_p2pk_to_p2pkh(self):
-        tx = transaction.Transaction('010000000118231a31d2df84f884ced6af11dc24306319577d4d7c340124a7e2dd9c314077000000004847304402200b6c45891aed48937241907bc3e3868ee4c792819821fcde33311e5a3da4789a02205021b59692b652a01f5f009bd481acac2f647a7d9c076d71d85869763337882e01fdffffff016c95052a010000001976a9149c4891e7791da9e622532c97f43863768264faaf88ac00000000')
-        self.assertEqual('90ba90a5b115106d26663fce6c6215b8699c5d4b2672dd30756115f3337dddf9', tx.txid())
+        raw_tx = '010000000118231a31d2df84f884ced6af11dc24306319577d4d7c340124a7e2dd9c314077000000004847304402200b6c45891aed48937241907bc3e3868ee4c792819821fcde33311e5a3da4789a02205021b59692b652a01f5f009bd481acac2f647a7d9c076d71d85869763337882e01fdffffff016c95052a010000001976a9149c4891e7791da9e622532c97f43863768264faaf88ac00000000'
+        txid = '90ba90a5b115106d26663fce6c6215b8699c5d4b2672dd30756115f3337dddf9'
+        self._run_naive_tests_on_tx(raw_tx, txid)
 
     def test_txid_p2pk_to_p2sh(self):
-        tx = transaction.Transaction('0100000001e4643183d6497823576d17ac2439fb97eba24be8137f312e10fcc16483bb2d070000000048473044022032bbf0394dfe3b004075e3cbb3ea7071b9184547e27f8f73f967c4b3f6a21fa4022073edd5ae8b7b638f25872a7a308bb53a848baa9b9cc70af45fcf3c683d36a55301fdffffff011821814a0000000017a9143c640bc28a346749c09615b50211cb051faff00f8700000000')
-        self.assertEqual('172bdf5a690b874385b98d7ab6f6af807356f03a26033c6a65ab79b4ac2085b5', tx.txid())
+        raw_tx = '0100000001e4643183d6497823576d17ac2439fb97eba24be8137f312e10fcc16483bb2d070000000048473044022032bbf0394dfe3b004075e3cbb3ea7071b9184547e27f8f73f967c4b3f6a21fa4022073edd5ae8b7b638f25872a7a308bb53a848baa9b9cc70af45fcf3c683d36a55301fdffffff011821814a0000000017a9143c640bc28a346749c09615b50211cb051faff00f8700000000'
+        txid = '172bdf5a690b874385b98d7ab6f6af807356f03a26033c6a65ab79b4ac2085b5'
+        self._run_naive_tests_on_tx(raw_tx, txid)
 
     def test_txid_p2pkh_to_p2pkh(self):
-        tx = transaction.Transaction('0100000001f9dd7d33f315617530dd72264b5d9c69b815626cce3f66266d1015b1a590ba90000000006a4730440220699bfee3d280a499daf4af5593e8750b54fef0557f3c9f717bfa909493a84f60022057718eec7985b7796bb8630bf6ea2e9bf2892ac21bd6ab8f741a008537139ffe012103b4289890b40590447b57f773b5843bf0400e9cead08be225fac587b3c2a8e973fdffffff01ec24052a010000001976a914ce9ff3d15ed5f3a3d94b583b12796d063879b11588ac00000000')
-        self.assertEqual('24737c68f53d4b519939119ed83b2a8d44d716d7f3ca98bcecc0fbb92c2085ce', tx.txid())
+        raw_tx = '0100000001f9dd7d33f315617530dd72264b5d9c69b815626cce3f66266d1015b1a590ba90000000006a4730440220699bfee3d280a499daf4af5593e8750b54fef0557f3c9f717bfa909493a84f60022057718eec7985b7796bb8630bf6ea2e9bf2892ac21bd6ab8f741a008537139ffe012103b4289890b40590447b57f773b5843bf0400e9cead08be225fac587b3c2a8e973fdffffff01ec24052a010000001976a914ce9ff3d15ed5f3a3d94b583b12796d063879b11588ac00000000'
+        txid = '24737c68f53d4b519939119ed83b2a8d44d716d7f3ca98bcecc0fbb92c2085ce'
+        self._run_naive_tests_on_tx(raw_tx, txid)
 
     def test_txid_p2pkh_to_p2sh(self):
-        tx = transaction.Transaction('010000000195232c30f6611b9f2f82ec63f5b443b132219c425e1824584411f3d16a7a54bc000000006b4830450221009f39ac457dc8ff316e5cc03161c9eff6212d8694ccb88d801dbb32e85d8ed100022074230bb05e99b85a6a50d2b71e7bf04d80be3f1d014ea038f93943abd79421d101210317be0f7e5478e087453b9b5111bdad586038720f16ac9658fd16217ffd7e5785fdffffff0200e40b540200000017a914d81df3751b9e7dca920678cc19cac8d7ec9010b08718dfd63c2c0000001976a914303c42b63569ff5b390a2016ff44651cd84c7c8988acc7010000')
-        self.assertEqual('155e4740fa59f374abb4e133b87247dccc3afc233cb97c2bf2b46bba3094aedc', tx.txid())
+        raw_tx = '010000000195232c30f6611b9f2f82ec63f5b443b132219c425e1824584411f3d16a7a54bc000000006b4830450221009f39ac457dc8ff316e5cc03161c9eff6212d8694ccb88d801dbb32e85d8ed100022074230bb05e99b85a6a50d2b71e7bf04d80be3f1d014ea038f93943abd79421d101210317be0f7e5478e087453b9b5111bdad586038720f16ac9658fd16217ffd7e5785fdffffff0200e40b540200000017a914d81df3751b9e7dca920678cc19cac8d7ec9010b08718dfd63c2c0000001976a914303c42b63569ff5b390a2016ff44651cd84c7c8988acc7010000'
+        txid = '155e4740fa59f374abb4e133b87247dccc3afc233cb97c2bf2b46bba3094aedc'
+        self._run_naive_tests_on_tx(raw_tx, txid)
+
+    # input: p2sh, not multisig
+    def test_txid_regression_issue_3899(self):
+        raw_tx = '0100000004328685b0352c981d3d451b471ae3bfc78b82565dc2a54049a81af273f0a9fd9c010000000b0009630330472d5fae685bffffffff328685b0352c981d3d451b471ae3bfc78b82565dc2a54049a81af273f0a9fd9c020000000b0009630359646d5fae6858ffffffff328685b0352c981d3d451b471ae3bfc78b82565dc2a54049a81af273f0a9fd9c030000000b000963034bd4715fae6854ffffffff328685b0352c981d3d451b471ae3bfc78b82565dc2a54049a81af273f0a9fd9c040000000b000963036de8705fae6860ffffffff0130750000000000001976a914b5abca61d20f9062fb1fdbb880d9d93bac36675188ac00000000'
+        txid = 'f570d5d1e965ee61bcc7005f8fefb1d3abbed9d7ddbe035e2a68fa07e5fc4a0d'
+        self._run_naive_tests_on_tx(raw_tx, txid)
+
+    def test_txid_negative_version_num(self):
+        raw_tx = 'f0b47b9a01ecf5e5c3bbf2cf1f71ecdc7f708b0b222432e914b394e24aad1494a42990ddfc000000008b483045022100852744642305a99ad74354e9495bf43a1f96ded470c256cd32e129290f1fa191022030c11d294af6a61b3da6ed2c0c296251d21d113cfd71ec11126517034b0dcb70014104a0fe6e4a600f859a0932f701d3af8e0ecd4be886d91045f06a5a6b931b95873aea1df61da281ba29cadb560dad4fc047cf47b4f7f2570da4c0b810b3dfa7e500ffffffff0240420f00000000001976a9147eeacb8a9265cd68c92806611f704fc55a21e1f588ac05f00d00000000001976a914eb3bd8ccd3ba6f1570f844b59ba3e0a667024a6a88acff7f0000'
+        txid = 'c659729a7fea5071361c2c1a68551ca2bf77679b27086cc415adeeb03852e369'
+        self._run_naive_tests_on_tx(raw_tx, txid)
 
 
 class NetworkMock(object):

lib/tests/test_wallet_vertical.py

@@ -5,44 +5,59 @@ import lib.bitcoin as bitcoin
 import lib.keystore as keystore
 import lib.storage as storage
 import lib.wallet as wallet
+from lib import constants
+
+from . import TestCaseForTestnet
 
 
-# TODO: 2fa
-class TestWalletKeystoreAddressIntegrity(unittest.TestCase):
+class WalletIntegrityHelper:
 
     gap_limit = 1  # make tests run faster
 
-    def _check_seeded_keystore_sanity(self, ks):
-        self.assertTrue (ks.is_deterministic())
-        self.assertFalse(ks.is_watching_only())
-        self.assertFalse(ks.can_import())
-        self.assertTrue (ks.has_seed())
+    @classmethod
+    def check_seeded_keystore_sanity(cls, test_obj, ks):
+        test_obj.assertTrue(ks.is_deterministic())
+        test_obj.assertFalse(ks.is_watching_only())
+        test_obj.assertFalse(ks.can_import())
+        test_obj.assertTrue(ks.has_seed())
 
-    def _check_xpub_keystore_sanity(self, ks):
-        self.assertTrue (ks.is_deterministic())
-        self.assertTrue (ks.is_watching_only())
-        self.assertFalse(ks.can_import())
-        self.assertFalse(ks.has_seed())
+    @classmethod
+    def check_xpub_keystore_sanity(cls, test_obj, ks):
+        test_obj.assertTrue(ks.is_deterministic())
+        test_obj.assertTrue(ks.is_watching_only())
+        test_obj.assertFalse(ks.can_import())
+        test_obj.assertFalse(ks.has_seed())
 
-    def _create_standard_wallet(self, ks):
+    @classmethod
+    def create_standard_wallet(cls, ks):
         store = storage.WalletStorage('if_this_exists_mocking_failed_648151893')
         store.put('keystore', ks.dump())
-        store.put('gap_limit', self.gap_limit)
+        store.put('gap_limit', cls.gap_limit)
         w = wallet.Standard_Wallet(store)
         w.synchronize()
         return w
 
-    def _create_multisig_wallet(self, ks1, ks2):
+    @classmethod
+    def create_multisig_wallet(cls, ks1, ks2, ks3=None):
+        """Creates a 2-of-2 or 2-of-3 multisig wallet."""
         store = storage.WalletStorage('if_this_exists_mocking_failed_648151893')
-        multisig_type = "%dof%d" % (2, 2)
-        store.put('wallet_type', multisig_type)
         store.put('x%d/' % 1, ks1.dump())
         store.put('x%d/' % 2, ks2.dump())
-        store.put('gap_limit', self.gap_limit)
+        if ks3 is None:
+            multisig_type = "%dof%d" % (2, 2)
+        else:
+            multisig_type = "%dof%d" % (2, 3)
+            store.put('x%d/' % 3, ks3.dump())
+        store.put('wallet_type', multisig_type)
+        store.put('gap_limit', cls.gap_limit)
         w = wallet.Multisig_Wallet(store)
         w.synchronize()
         return w
 
+
+# TODO passphrase/seed_extension
+class TestWalletKeystoreAddressIntegrityForMainnet(unittest.TestCase):
+
     @mock.patch.object(storage.WalletStorage, '_write')
     def test_electrum_seed_standard(self, mock_write):
         seed_words = 'cycle rocket west magnet parrot shuffle foot correct salt library feed song'
@@ -50,12 +65,14 @@ class TestWalletKeystoreAddressIntegrity(unittest.TestCase):
 
         ks = keystore.from_seed(seed_words, '', False)
 
-        self._check_seeded_keystore_sanity(ks)
+        WalletIntegrityHelper.check_seeded_keystore_sanity(self, ks)
         self.assertTrue(isinstance(ks, keystore.BIP32_KeyStore))
 
+        self.assertEqual(ks.xprv, 'xprv9s21ZrQH143K32jECVM729vWgGq4mUDJCk1ozqAStTphzQtCTuoFmFafNoG1g55iCnBTXUzz3zWnDb5CVLGiFvmaZjuazHDL8a81cPQ8KL6')
         self.assertEqual(ks.xpub, 'xpub661MyMwAqRbcFWohJWt7PHsFEJfZAvw9ZxwQoDa4SoMgsDDM1T7WK3u9E4edkC4ugRnZ8E4xDZRpk8Rnts3Nbt97dPwT52CwBdDWroaZf8U')
 
-        w = self._create_standard_wallet(ks)
+        w = WalletIntegrityHelper.create_standard_wallet(ks)
+        self.assertEqual(w.txin_type, 'p2pkh')
 
         self.assertEqual(w.get_receiving_addresses()[0], 't1fFMuEC9XFGsEUEPzpEE8jhxpcEMs369xJ')
         self.assertEqual(w.get_change_addresses()[0], 't1cKFzsmq8d97RtePBbqS1WebLofuXaXkzF')
@@ -67,12 +84,13 @@ class TestWalletKeystoreAddressIntegrity(unittest.TestCase):
 
         ks = keystore.from_seed(seed_words, '', False)
 
-        self._check_seeded_keystore_sanity(ks)
+        WalletIntegrityHelper.check_seeded_keystore_sanity(self, ks)
         self.assertTrue(isinstance(ks, keystore.Old_KeyStore))
 
         self.assertEqual(ks.mpk, 'e9d4b7866dd1e91c862aebf62a49548c7dbf7bcc6e4b7b8c9da820c7737968df9c09d5a3e271dc814a29981f81b3faaf2737b551ef5dcc6189cf0f8252c442b3')
 
-        w = self._create_standard_wallet(ks)
+        w = WalletIntegrityHelper.create_standard_wallet(ks)
+        self.assertEqual(w.txin_type, 'p2pkh')
 
         self.assertEqual(w.get_receiving_addresses()[0], 't1YAqEWYrfi9CbW5LgmayAvjDE5T5MgaYiD')
         self.assertEqual(w.get_change_addresses()[0], 't1cJ799hEFa5AHmB3ReeDo3Rr2X4quderf4')
@@ -86,9 +104,11 @@ class TestWalletKeystoreAddressIntegrity(unittest.TestCase):
 
         self.assertTrue(isinstance(ks, keystore.BIP32_KeyStore))
 
+        self.assertEqual(ks.xprv, 'xprv9zGLcNEb3cHUKizLVBz6RYeE9bEZAVPjH2pD1DEzCnPcsemWc3d3xTao8sfhfUmDLMq6e3RcEMEvJG1Et8dvfL8DV4h7mwm9J6AJsW9WXQD')
         self.assertEqual(ks.xpub, 'xpub6DFh1smUsyqmYD4obDX6ngaxhd53Zx7aeFjoobebm7vbkT6f9awJWFuGzBT9FQJEWFBL7UyhMXtYzRcwDuVbcxtv9Ce2W9eMm4KXLdvdbjv')
 
-        w = self._create_standard_wallet(ks)
+        w = WalletIntegrityHelper.create_standard_wallet(ks)
+        self.assertEqual(w.txin_type, 'p2pkh')
 
         self.assertEqual(w.get_receiving_addresses()[0], 't1PbiEBABXU1E4GEnE31cUPULDoYYWVMpjs')
         self.assertEqual(w.get_change_addresses()[0], 't1Z8gbq4eeVbg89ACGddq1T6yfEP9bQx9Ki')
@@ -99,15 +119,18 @@ class TestWalletKeystoreAddressIntegrity(unittest.TestCase):
         self.assertEqual(bitcoin.seed_type(seed_words), 'standard')
 
         ks1 = keystore.from_seed(seed_words, '', True)
-        self._check_seeded_keystore_sanity(ks1)
+        WalletIntegrityHelper.check_seeded_keystore_sanity(self, ks1)
         self.assertTrue(isinstance(ks1, keystore.BIP32_KeyStore))
+        self.assertEqual(ks1.xprv, 'xprv9s21ZrQH143K3t9vo23J3hajRbzvkRLJ6Y1zFrUFAfU3t8oooMPfb7f87cn5KntgqZs5nipZkCiBFo5ZtaSD2eDo7j7CMuFV8Zu6GYLTpY6')
         self.assertEqual(ks1.xpub, 'xpub661MyMwAqRbcGNEPu3aJQqXTydqR9t49Tkwb4Esrj112kw8xLthv8uybxvaki4Ygt9xiwZUQGeFTG7T2TUzR3eA4Zp3aq5RXsABHFBUrq4c')
 
+        # electrum seed: ghost into match ivory badge robot record tackle radar elbow traffic loud
         ks2 = keystore.from_xpub('xpub661MyMwAqRbcGfCPEkkyo5WmcrhTq8mi3xuBS7VEZ3LYvsgY1cCFDbenT33bdD12axvrmXhuX3xkAbKci3yZY9ZEk8vhLic7KNhLjqdh5ec')
-        self._check_xpub_keystore_sanity(ks2)
+        WalletIntegrityHelper.check_xpub_keystore_sanity(self, ks2)
         self.assertTrue(isinstance(ks2, keystore.BIP32_KeyStore))
 
-        w = self._create_multisig_wallet(ks1, ks2)
+        w = WalletIntegrityHelper.create_multisig_wallet(ks1, ks2)
+        self.assertEqual(w.txin_type, 'p2sh')
 
         self.assertEqual(w.get_receiving_addresses()[0], 't3KcK3kAJerAahSJhN6Pt729u7ficQqK4XX')
         self.assertEqual(w.get_change_addresses()[0], 't3PQ7wZhzpoywPLnD1nfcd5sPYLtw2qh5ak')
@@ -119,13 +142,17 @@ class TestWalletKeystoreAddressIntegrity(unittest.TestCase):
 
         ks1 = keystore.from_bip39_seed(seed_words, '', "m/45'/0")
         self.assertTrue(isinstance(ks1, keystore.BIP32_KeyStore))
+        self.assertEqual(ks1.xprv, 'xprv9vyEFyXf7pYVv4eDU3hhuCEAHPHNGuxX73nwtYdpbLcqwJCPwFKknAK8pHWuHHBirCzAPDZ7UJHrYdhLfn1NkGp9rk3rVz2aEqrT93qKRD9')
         self.assertEqual(ks1.xpub, 'xpub69xafV4YxC6o8Yiga5EiGLAtqR7rgNgNUGiYgw3S9g9pp6XYUne1KxdcfYtxwmA3eBrzMFuYcNQKfqsXCygCo4GxQFHfywxpUbKNfYvGJka')
 
+        # bip39 seed: tray machine cook badge night page project uncover ritual toward person enact
+        # der: m/45'/0
         ks2 = keystore.from_xpub('xpub6Bco9vrgo8rNUSi8Bjomn8xLA41DwPXeuPcgJamNRhTTyGVHsp8fZXaGzp9ypHoei16J6X3pumMAP1u3Dy4jTSWjm4GZowL7Dcn9u4uZC9W')
-        self._check_xpub_keystore_sanity(ks2)
+        WalletIntegrityHelper.check_xpub_keystore_sanity(self, ks2)
         self.assertTrue(isinstance(ks2, keystore.BIP32_KeyStore))
 
-        w = self._create_multisig_wallet(ks1, ks2)
+        w = WalletIntegrityHelper.create_multisig_wallet(ks1, ks2)
+        self.assertEqual(w.txin_type, 'p2sh')
 
         self.assertEqual(w.get_receiving_addresses()[0], 't3ZvKyVcMRf5rofUFgN8gk23jtCkRh12Lja')
         self.assertEqual(w.get_change_addresses()[0], 't3JaafdGtfhXJzCrupYct6tJBdD2XZUtm8H')

lib/transaction.py

@@ -32,6 +32,8 @@ from .util import print_error, profiler
 from . import bitcoin
 from .bitcoin import *
 import struct
+import traceback
+import sys
 
 #
 # Workalike python implementation of Bitcoin's CDataStream class.
@@ -227,10 +229,10 @@ opcodes = Enumeration("Opcodes", [
     "OP_WITHIN", "OP_RIPEMD160", "OP_SHA1", "OP_SHA256", "OP_HASH160",
     "OP_HASH256", "OP_CODESEPARATOR", "OP_CHECKSIG", "OP_CHECKSIGVERIFY", "OP_CHECKMULTISIG",
     "OP_CHECKMULTISIGVERIFY",
-    ("OP_SINGLEBYTE_END", 0xF0),
-    ("OP_DOUBLEBYTE_BEGIN", 0xF000),
-    "OP_PUBKEY", "OP_PUBKEYHASH",
-    ("OP_INVALIDOPCODE", 0xFFFF),
+    ("OP_NOP1", 0xB0),
+    ("OP_CHECKLOCKTIMEVERIFY", 0xB1), ("OP_CHECKSEQUENCEVERIFY", 0xB2),
+    "OP_NOP4", "OP_NOP5", "OP_NOP6", "OP_NOP7", "OP_NOP8", "OP_NOP9", "OP_NOP10",
+    ("OP_INVALIDOPCODE", 0xFF),
 ])
 
 
@@ -240,10 +242,6 @@ def script_GetOp(_bytes):
         vch = None
         opcode = _bytes[i]
         i += 1
-        if opcode >= opcodes.OP_SINGLEBYTE_END:
-            opcode <<= 8
-            opcode |= _bytes[i]
-            i += 1
 
         if opcode <= opcodes.OP_PUSHDATA4:
             nSize = opcode
@@ -303,7 +301,8 @@ def parse_scriptSig(d, _bytes):
         decoded = [ x for x in script_GetOp(_bytes) ]
     except Exception as e:
         # coinbase transactions raise an exception
-        print_error("cannot find address in input script", bh2u(_bytes))
+        print_error("parse_scriptSig: cannot find address in input script (coinbase?)",
+                    bh2u(_bytes))
         return
 
     match = [ opcodes.OP_PUSHDATA4 ]
@@ -320,9 +319,9 @@ def parse_scriptSig(d, _bytes):
             d['pubkeys'] = ["(pubkey)"]
         return
 
-    # non-generated TxIn transactions push a signature
-    # (seventy-something bytes) and then their public key
-    # (65 bytes) onto the stack:
+    # p2pkh TxIn transactions push a signature
+    # (71-73 bytes) and then their public key
+    # (33 or 65 bytes) onto the stack:
     match = [ opcodes.OP_PUSHDATA4, opcodes.OP_PUSHDATA4 ]
     if match_decoded(decoded, match):
         sig = bh2u(decoded[0][1])
@@ -331,7 +330,8 @@ def parse_scriptSig(d, _bytes):
             signatures = parse_sig([sig])
             pubkey, address = xpubkey_to_address(x_pubkey)
         except:
-            print_error("cannot find address in input script", bh2u(_bytes))
+            print_error("parse_scriptSig: cannot find address in input script (p2pkh?)",
+                        bh2u(_bytes))
             return
         d['type'] = 'p2pkh'
         d['signatures'] = signatures
@@ -343,37 +343,49 @@ def parse_scriptSig(d, _bytes):
 
     # p2sh transaction, m of n
     match = [ opcodes.OP_0 ] + [ opcodes.OP_PUSHDATA4 ] * (len(decoded) - 1)
-    if not match_decoded(decoded, match):
-        print_error("cannot find address in input script", bh2u(_bytes))
+    if match_decoded(decoded, match):
+        x_sig = [bh2u(x[1]) for x in decoded[1:-1]]
+        try:
+            m, n, x_pubkeys, pubkeys, redeemScript = parse_redeemScript(decoded[-1][1])
+        except NotRecognizedRedeemScript:
+            print_error("parse_scriptSig: cannot find address in input script (p2sh?)",
+                        bh2u(_bytes))
+            # we could still guess:
+            # d['address'] = hash160_to_p2sh(hash_160(decoded[-1][1]))
+            return
+        # write result in d
+        d['type'] = 'p2sh'
+        d['num_sig'] = m
+        d['signatures'] = parse_sig(x_sig)
+        d['x_pubkeys'] = x_pubkeys
+        d['pubkeys'] = pubkeys
+        d['redeemScript'] = redeemScript
+        d['address'] = hash160_to_p2sh(hash_160(bfh(redeemScript)))
         return
-    x_sig = [bh2u(x[1]) for x in decoded[1:-1]]
-    m, n, x_pubkeys, pubkeys, redeemScript = parse_redeemScript(decoded[-1][1])
-    # write result in d
-    d['type'] = 'p2sh'
-    d['num_sig'] = m
-    d['signatures'] = parse_sig(x_sig)
-    d['x_pubkeys'] = x_pubkeys
-    d['pubkeys'] = pubkeys
-    d['redeemScript'] = redeemScript
-    d['address'] = hash160_to_p2sh(hash_160(bfh(redeemScript)))
+
+    print_error("parse_scriptSig: cannot find address in input script (unknown)",
+                bh2u(_bytes))
 
 
 def parse_redeemScript(s):
     dec2 = [ x for x in script_GetOp(s) ]
-    m = dec2[0][0] - opcodes.OP_1 + 1
-    n = dec2[-2][0] - opcodes.OP_1 + 1
+    try:
+        m = dec2[0][0] - opcodes.OP_1 + 1
+        n = dec2[-2][0] - opcodes.OP_1 + 1
+    except IndexError:
+        raise NotRecognizedRedeemScript()
     op_m = opcodes.OP_1 + m - 1
     op_n = opcodes.OP_1 + n - 1
     match_multisig = [ op_m ] + [opcodes.OP_PUSHDATA4]*n + [ op_n, opcodes.OP_CHECKMULTISIG ]
     if not match_decoded(dec2, match_multisig):
-        print_error("cannot find address in input script", bh2u(s))
         raise NotRecognizedRedeemScript()
     x_pubkeys = [bh2u(x[1]) for x in dec2[1:-2]]
     pubkeys = [safe_parse_pubkey(x) for x in x_pubkeys]
     redeemScript = multisig_script(pubkeys, m)
     return m, n, x_pubkeys, pubkeys, redeemScript
 
-def get_address_from_output_script(_bytes):
+
+def get_address_from_output_script(_bytes, *, net=None):
     decoded = [x for x in script_GetOp(_bytes)]
 
     # The Genesis Block, self-payments, and pay-by-IP-address payments look like:
@@ -386,12 +398,12 @@ def get_address_from_output_script(_bytes):
     # DUP HASH160 20 BYTES:... EQUALVERIFY CHECKSIG
     match = [ opcodes.OP_DUP, opcodes.OP_HASH160, opcodes.OP_PUSHDATA4, opcodes.OP_EQUALVERIFY, opcodes.OP_CHECKSIG ]
     if match_decoded(decoded, match):
-        return TYPE_ADDRESS, hash160_to_p2pkh(decoded[2][1])
+        return TYPE_ADDRESS, hash160_to_p2pkh(decoded[2][1], net=net)
 
     # p2sh
     match = [ opcodes.OP_HASH160, opcodes.OP_PUSHDATA4, opcodes.OP_EQUAL ]
     if match_decoded(decoded, match):
-        return TYPE_ADDRESS, hash160_to_p2sh(decoded[1][1])
+        return TYPE_ADDRESS, hash160_to_p2sh(decoded[1][1], net=net)
 
     return TYPE_SCRIPT, bh2u(_bytes)
 
@@ -405,19 +417,23 @@ def parse_input(vds):
     d['prevout_hash'] = prevout_hash
     d['prevout_n'] = prevout_n
     d['sequence'] = sequence
+    d['x_pubkeys'] = []
+    d['pubkeys'] = []
+    d['signatures'] = {}
+    d['address'] = None
+    d['num_sig'] = 0
     if prevout_hash == '00'*32:
         d['type'] = 'coinbase'
         d['scriptSig'] = bh2u(scriptSig)
     else:
-        d['x_pubkeys'] = []
-        d['pubkeys'] = []
-        d['signatures'] = {}
-        d['address'] = None
         d['type'] = 'unknown'
-        d['num_sig'] = 0
         if scriptSig:
             d['scriptSig'] = bh2u(scriptSig)
-            parse_scriptSig(d, scriptSig)
+            try:
+                parse_scriptSig(d, scriptSig)
+            except BaseException:
+                traceback.print_exc(file=sys.stderr)
+                print_error('failed to parse scriptSig', bh2u(scriptSig))
         else:
             d['scriptSig'] = ''
 
@@ -479,7 +495,7 @@ class Transaction:
         elif isinstance(raw, dict):
             self.raw = raw['hex']
         else:
-            raise BaseException("cannot initialize transaction", raw)
+            raise Exception("cannot initialize transaction", raw)
         self._inputs = None
         self._outputs = None
         self.locktime = 0
@@ -503,6 +519,8 @@ class Transaction:
     @classmethod
     def get_sorted_pubkeys(self, txin):
         # sort pubkeys and x_pubkeys, using the order of pubkeys
+        if txin['type'] == 'coinbase':
+            return [], []
         x_pubkeys = txin['x_pubkeys']
         pubkeys = txin.get('pubkeys')
         if pubkeys is None:
@@ -603,6 +621,8 @@ class Transaction:
     def get_siglist(self, txin, estimate_size=False):
         # if we have enough signatures, we use the actual pubkeys
         # otherwise, use extended pubkeys (with bip32 derivation)
+        if txin['type'] == 'coinbase':
+            return [], []
         num_sig = txin.get('num_sig', 1)
         if estimate_size:
             pubkey_size = self.estimate_pubkey_size_for_txin(txin)
@@ -645,7 +665,9 @@ class Transaction:
         return script
 
     @classmethod
-    def is_txin_complete(self, txin):
+    def is_txin_complete(cls, txin):
+        if txin['type'] == 'coinbase':
+            return True
         num_sig = txin.get('num_sig', 1)
         x_signatures = txin['signatures']
         signatures = list(filter(None, x_signatures))
@@ -653,13 +675,13 @@ class Transaction:
 
     @classmethod
     def get_preimage_script(self, txin):
+        pubkeys, x_pubkeys = self.get_sorted_pubkeys(txin)
         if txin['type'] == 'p2pkh':
             return bitcoin.address_to_script(txin['address'])
         elif txin['type'] in ['p2sh']:
-            pubkeys, x_pubkeys = self.get_sorted_pubkeys(txin)
             return multisig_script(pubkeys, txin['num_sig'])
         elif txin['type'] == 'p2pk':
-            pubkey = txin['pubkeys'][0]
+            pubkey = pubkeys[0]
             return bitcoin.public_key_to_p2pk_script(pubkey)
         else:
             raise TypeError('Unknown txin type', txin['type'])
@@ -668,6 +690,14 @@ class Transaction:
     def serialize_outpoint(self, txin):
         return bh2u(bfh(txin['prevout_hash'])[::-1]) + int_to_hex(txin['prevout_n'], 4)
 
+    @classmethod
+    def get_outpoint_from_txin(cls, txin):
+        if txin['type'] == 'coinbase':
+            return None
+        prevout_hash = txin['prevout_hash']
+        prevout_n = txin['prevout_n']
+        return prevout_hash + ':%d' % prevout_n
+
     @classmethod
     def serialize_input(self, txin, script):
         # Prev hash and index
@@ -761,6 +791,13 @@ class Transaction:
         input_size = len(cls.serialize_input(txin, script)) // 2
         return 4 * input_size
 
+    @classmethod
+    def estimated_output_size(cls, address):
+        """Return an estimate of serialized output size in bytes."""
+        script = bitcoin.address_to_script(address)
+        # 8 byte value + 1 byte script len + script
+        return 9 + len(script) // 2
+
     @classmethod
     def virtual_size_from_weight(cls, weight):
         return weight // 4 + (weight % 4 > 0)
@@ -814,7 +851,8 @@ class Transaction:
                     private_key = bitcoin.MySigningKey.from_secret_exponent(secexp, curve = SECP256k1)
                     public_key = private_key.get_verifying_key()
                     sig = private_key.sign_digest_deterministic(pre_hash, hashfunc=hashlib.sha256, sigencode = ecdsa.util.sigencode_der_canonize)
-                    assert public_key.verify_digest(sig, pre_hash, sigdecode = ecdsa.util.sigdecode_der)
+                    if not public_key.verify_digest(sig, pre_hash, sigdecode = ecdsa.util.sigdecode_der_canonize):
+                        raise Exception('Sanity check verifying our own signature failed.')
                     txin['signatures'][j] = bh2u(sig) + '01'
                     #txin['x_pubkeys'][j] = pubkey
                     txin['pubkeys'][j] = pubkey # needed for fd keys

lib/util.py

@@ -41,28 +41,102 @@ def inv_dict(d):
 
 
 base_units = {'ZEC':8, 'mZEC':5, 'uZEC':2}
-fee_levels = [_('Within 25 blocks'), _('Within 10 blocks'), _('Within 5 blocks'), _('Within 2 blocks'), _('In the next block')]
 
 def normalize_version(v):
     return [int(x) for x in re.sub(r'(\.0+)*$','', v).split(".")]
 
 class NotEnoughFunds(Exception): pass
 
+
+class NoDynamicFeeEstimates(Exception):
+    def __str__(self):
+        return _('Dynamic fee estimates not available')
+
+
 class InvalidPassword(Exception):
     def __str__(self):
         return _("Incorrect password")
 
+
+class FileImportFailed(Exception):
+    def __init__(self, message=''):
+        self.message = str(message)
+
+    def __str__(self):
+        return _("Failed to import from file.") + "\n" + self.message
+
+
+class FileExportFailed(Exception):
+    def __init__(self, message=''):
+        self.message = str(message)
+
+    def __str__(self):
+        return _("Failed to export to file.") + "\n" + self.message
+
+
+class TimeoutException(Exception):
+    def __init__(self, message=''):
+        self.message = str(message)
+
+    def __str__(self):
+        if not self.message:
+            return _("Operation timed out.")
+        return self.message
+
+
+class WalletFileException(Exception): pass
+
+
+class BitcoinException(Exception): pass
+
+
 # Throw this exception to unwind the stack like when an error occurs.
 # However unlike other exceptions the user won't be informed.
 class UserCancelled(Exception):
     '''An exception that is suppressed from the user'''
     pass
 
+class Satoshis(object):
+    def __new__(cls, value):
+        self = super(Satoshis, cls).__new__(cls)
+        self.value = value
+        return self
+
+    def __repr__(self):
+        return 'Satoshis(%d)'%self.value
+
+    def __str__(self):
+        return format_satoshis(self.value) + " BTC"
+
+class Fiat(object):
+    def __new__(cls, value, ccy):
+        self = super(Fiat, cls).__new__(cls)
+        self.ccy = ccy
+        self.value = value
+        return self
+
+    def __repr__(self):
+        return 'Fiat(%s)'% self.__str__()
+
+    def __str__(self):
+        if self.value.is_nan():
+            return _('No Data')
+        else:
+            return "{:.2f}".format(self.value) + ' ' + self.ccy
+
 class MyEncoder(json.JSONEncoder):
     def default(self, obj):
         from .transaction import Transaction
         if isinstance(obj, Transaction):
             return obj.as_dict()
+        if isinstance(obj, Satoshis):
+            return str(obj)
+        if isinstance(obj, Fiat):
+            return str(obj)
+        if isinstance(obj, Decimal):
+            return str(obj)
+        if isinstance(obj, datetime):
+            return obj.isoformat(' ')[:-3]
         return super(MyEncoder, self).default(obj)
 
 class PrintError(object):
@@ -71,8 +145,12 @@ class PrintError(object):
         return self.__class__.__name__
 
     def print_error(self, *msg):
+        # only prints with --verbose flag
         print_error("[%s]" % self.diagnostic_name(), *msg)
 
+    def print_stderr(self, *msg):
+        print_stderr("[%s]" % self.diagnostic_name(), *msg)
+
     def print_msg(self, *msg):
         print_msg("[%s]" % self.diagnostic_name(), *msg)
 
@@ -247,8 +325,8 @@ def android_check_data_dir():
     old_electrum_dir = ext_dir + '/electrum-zcash'
     if not os.path.exists(data_dir) and os.path.exists(old_electrum_dir):
         import shutil
-        new_headers_path = android_headers_dir() + android_headers_file_name()
-        old_headers_path = old_electrum_dir + android_headers_file_name()
+        new_headers_path = android_headers_dir() + '/blockchain_headers'
+        old_headers_path = old_electrum_dir + '/blockchain_headers'
         if not os.path.exists(new_headers_path) and os.path.exists(old_headers_path):
             print_error("Moving headers file to", new_headers_path)
             shutil.move(old_headers_path, new_headers_path)
@@ -348,7 +426,7 @@ def format_satoshis(x, is_diff=False, num_zeros = 0, decimal_point = 8, whitespa
         return 'unknown'
     x = int(x)  # Some callers pass Decimal
     scale_factor = pow (10, decimal_point)
-    integer_part = "{:n}".format(int(abs(x) / scale_factor))
+    integer_part = "{:d}".format(int(abs(x) / scale_factor))
     if x < 0:
         integer_part = '-' + integer_part
     elif is_diff:
@@ -365,10 +443,9 @@ def format_satoshis(x, is_diff=False, num_zeros = 0, decimal_point = 8, whitespa
     return result
 
 def timestamp_to_datetime(timestamp):
-    try:
-        return datetime.fromtimestamp(timestamp)
-    except:
+    if timestamp is None:
         return None
+    return datetime.fromtimestamp(timestamp)
 
 def format_time(timestamp):
     date = timestamp_to_datetime(timestamp)
@@ -429,22 +506,22 @@ def time_difference(distance_in_time, include_seconds):
         return "over %d years" % (round(distance_in_minutes / 525600))
 
 mainnet_block_explorers = {
-    'blockexplorer.com': ('https://zcash.blockexplorer.com/blocks',
-                        {'tx': 'transactions', 'addr': 'addresses'}),
-    'system default': ('blockchain:',
-                        {'tx': 'tx', 'addr': 'address'}),
+    'blockexplorer.com': ('https://zcash.blockexplorer.com/blocks/',
+                        {'tx': 'transactions/', 'addr': 'addresses/'}),
+    'system default': ('blockchain:/',
+                        {'tx': 'tx/', 'addr': 'address/'}),
 }
 
 testnet_block_explorers = {
     'testnet.z.cash': ('https://explorer.testnet.z.cash/',
-                       {'tx': 'tx', 'addr': 'address'}),
-    'system default': ('blockchain:',
-                       {'tx': 'tx', 'addr': 'address'}),
+                       {'tx': 'tx/', 'addr': 'address/'}),
+    'system default': ('blockchain:/',
+                       {'tx': 'tx/', 'addr': 'address/'}),
 }
 
 def block_explorer_info():
-    from . import bitcoin
-    return testnet_block_explorers if bitcoin.NetworkConstants.TESTNET else mainnet_block_explorers
+    from . import constants
+    return testnet_block_explorers if constants.net.TESTNET else mainnet_block_explorers
 
 def block_explorer(config):
     return config.get('block_explorer', 'blockexplorer.com')
@@ -460,7 +537,7 @@ def block_explorer_URL(config, kind, item):
     if not kind_str:
         return
     url_parts = [be_tuple[0], kind_str, item]
-    return "/".join(url_parts)
+    return ''.join(url_parts)
 
 # URL decode
 #_ud = re.compile('%([0-9a-hA-H]{2})', re.MULTILINE)
@@ -472,12 +549,12 @@ def parse_URI(uri, on_pr=None):
 
     if ':' not in uri:
         if not bitcoin.is_address(uri):
-            raise BaseException("Not a Zcash address")
+            raise Exception("Not a Zcash address")
         return {'address': uri}
 
     u = urllib.parse.urlparse(uri)
     if u.scheme != 'zcash':
-        raise BaseException("Not a Zcash URI")
+        raise Exception("Not a Zcash URI")
     address = u.path
 
     # python for android fails to parse query
@@ -494,7 +571,7 @@ def parse_URI(uri, on_pr=None):
     out = {k: v[0] for k, v in pq.items()}
     if address:
         if not bitcoin.is_address(address):
-            raise BaseException("Invalid Zcash address:" + address)
+            raise Exception("Invalid Zcash address:" + address)
         out['address'] = address
     if 'amount' in out:
         am = out['amount']
@@ -643,10 +720,6 @@ class SocketPipe:
                 print_error("SSLError:", e)
                 time.sleep(0.1)
                 continue
-            except OSError as e:
-                print_error("OSError", e)
-                time.sleep(0.1)
-                continue
 
 
 class QueuePipe:
@@ -683,25 +756,56 @@ class QueuePipe:
             self.send(request)
 
 
-def check_www_dir(rdir):
-    import urllib, shutil, os
-    if not os.path.exists(rdir):
-        os.mkdir(rdir)
-    index = os.path.join(rdir, 'index.html')
-    if not os.path.exists(index):
-        print_error("copying index.html")
-        src = os.path.join(os.path.dirname(__file__), 'www', 'index.html')
-        shutil.copy(src, index)
-    files = [
-        "https://code.jquery.com/jquery-1.9.1.min.js",
-        "https://raw.githubusercontent.com/davidshimjs/qrcodejs/master/qrcode.js",
-        "https://code.jquery.com/ui/1.10.3/jquery-ui.js",
-        "https://code.jquery.com/ui/1.10.3/themes/smoothness/jquery-ui.css"
-    ]
-    for URL in files:
-        path = urllib.parse.urlsplit(URL).path
-        filename = os.path.basename(path)
-        path = os.path.join(rdir, filename)
-        if not os.path.exists(path):
-            print_error("downloading ", URL)
-            urllib.request.urlretrieve(URL, path)
+
+
+def setup_thread_excepthook():
+    """
+    Workaround for `sys.excepthook` thread bug from:
+    http://bugs.python.org/issue1230540
+
+    Call once from the main thread before creating any threads.
+    """
+
+    init_original = threading.Thread.__init__
+
+    def init(self, *args, **kwargs):
+
+        init_original(self, *args, **kwargs)
+        run_original = self.run
+
+        def run_with_except_hook(*args2, **kwargs2):
+            try:
+                run_original(*args2, **kwargs2)
+            except Exception:
+                sys.excepthook(*sys.exc_info())
+
+        self.run = run_with_except_hook
+
+    threading.Thread.__init__ = init
+
+
+def versiontuple(v):
+    return tuple(map(int, (v.split("."))))
+
+
+def import_meta(path, validater, load_meta):
+    try:
+        with open(path, 'r', encoding='utf-8') as f:
+            d = validater(json.loads(f.read()))
+        load_meta(d)
+    #backwards compatibility for JSONDecodeError
+    except ValueError:
+        traceback.print_exc(file=sys.stderr)
+        raise FileImportFailed(_("Invalid JSON code."))
+    except BaseException as e:
+        traceback.print_exc(file=sys.stdout)
+        raise FileImportFailed(e)
+
+
+def export_meta(meta, fileName):
+    try:
+        with open(fileName, 'w+', encoding='utf-8') as f:
+            json.dump(meta, f, indent=4, sort_keys=True)
+    except (IOError, os.error) as e:
+        traceback.print_exc(file=sys.stderr)
+        raise FileExportFailed(e)

lib/verifier.py

@@ -36,22 +36,37 @@ class SPV(ThreadJob):
         self.merkle_roots = {}
 
     def run(self):
+        interface = self.network.interface
+        if not interface:
+            return
+        blockchain = interface.blockchain
+        if not blockchain:
+            return
         lh = self.network.get_local_height()
         unverified = self.wallet.get_unverified_txs()
         for tx_hash, tx_height in unverified.items():
             # do not request merkle branch before headers are available
-            if (tx_height > 0) and (tx_hash not in self.merkle_roots) and (tx_height <= lh):
-                request = ('blockchain.transaction.get_merkle',
-                           [tx_hash, tx_height])
-                self.network.send([request], self.verify_merkle)
-                self.print_error('requested merkle', tx_hash)
-                self.merkle_roots[tx_hash] = None
+            if (tx_height > 0) and (tx_height <= lh):
+                header = blockchain.read_header(tx_height)
+                if header is None:
+                    index = tx_height // 2016
+                    if index < len(blockchain.checkpoints):
+                        self.network.request_chunk(interface, index)
+                else:
+                    if tx_hash not in self.merkle_roots:
+                        request = ('blockchain.transaction.get_merkle',
+                                   [tx_hash, tx_height])
+                        self.network.send([request], self.verify_merkle)
+                        self.print_error('requested merkle', tx_hash)
+                        self.merkle_roots[tx_hash] = None
 
         if self.network.blockchain() != self.blockchain:
             self.blockchain = self.network.blockchain()
             self.undo_verifications()
 
     def verify_merkle(self, r):
+        if self.wallet.verifier is None:
+            return  # we have been killed, this was just an orphan callback
         if r.get('error'):
             self.print_error('received an error:', r)
             return
@@ -64,17 +79,26 @@ class SPV(ThreadJob):
         pos = merkle.get('pos')
         merkle_root = self.hash_merkle_root(merkle['merkle'], tx_hash, pos)
         header = self.network.blockchain().read_header(tx_height)
-        if not header or header.get('merkle_root') != merkle_root:
-            # FIXME: we should make a fresh connection to a server to
-            # recover from this, as this TX will now never verify
-            self.print_error("merkle verification failed for", tx_hash)
+        # FIXME: if verification fails below,
+        # we should make a fresh connection to a server to
+        # recover from this, as this TX will now never verify
+        if not header:
+            self.print_error(
+                "merkle verification failed for {} (missing header {})"
+                .format(tx_hash, tx_height))
+            return
+        if header.get('merkle_root') != merkle_root:
+            self.print_error(
+                "merkle verification failed for {} (merkle root mismatch {} != {})"
+                .format(tx_hash, header.get('merkle_root'), merkle_root))
             return
         # we passed all the tests
         self.merkle_roots[tx_hash] = merkle_root
         self.print_error("verified %s" % tx_hash)
         self.wallet.add_verified_tx(tx_hash, (tx_height, header.get('timestamp'), pos))
 
-    def hash_merkle_root(self, merkle_s, target_hash, pos):
+    @classmethod
+    def hash_merkle_root(cls, merkle_s, target_hash, pos):
         h = hash_decode(target_hash)
         for i in range(len(merkle_s)):
             item = merkle_s[i]

lib/version.py

@@ -1,4 +1,4 @@
-ELECTRUM_VERSION = '3.0.6'   # version of the client package
+ELECTRUM_VERSION = '3.1.3'   # version of the client package
 PROTOCOL_VERSION = '1.2'     # protocol version requested
 
 # The hash of the mnemonic seed must begin with this

lib/websockets.py

@@ -64,7 +64,7 @@ class WsClientThread(util.DaemonThread):
         # read json file
         rdir = self.config.get('requests_dir')
         n = os.path.join(rdir, 'req', request_id[0], request_id[1], request_id, request_id + '.json')
-        with open(n) as f:
+        with open(n, encoding='utf-8') as f:
             s = f.read()
         d = json.loads(s)
         addr = d.get('address')

lib/www/example.php

@@ -1,17 +0,0 @@
-<?php
-
-require_once 'jsonRPCClient.php';
-$electrum = new jsonRPCClient('http://localhost:7777');
-
-echo '<b>Wallet balance</b><br />'."\n";
-try {
-
-    $balance = $electrum->getbalance();
-    echo 'confirmed: <i>'.$balance['confirmed'].'</i><br />'."\n";
-    echo 'unconfirmed: <i>'.$balance['unconfirmed'].'</i><br />'."\n";
-
-} catch (Exception $e) {
-    echo nl2br($e->getMessage()).'<br />'."\n";
-}
-
-?>

lib/x509.py

@@ -284,7 +284,7 @@ class X509(object):
         return self.AKI if self.AKI else repr(self.issuer)
 
     def get_common_name(self):
-        return self.subject.get('2.5.4.3', 'unknown').decode()
+        return self.subject.get('2.5.4.3', b'unknown').decode()
 
     def get_signature(self):
         return self.cert_sig_algo, self.signature, self.data
@@ -313,7 +313,7 @@ def load_certificates(ca_path):
     ca_list = {}
     ca_keyID = {}
     # ca_path = '/tmp/tmp.txt'
-    with open(ca_path, 'r') as f:
+    with open(ca_path, 'r', encoding='utf-8') as f:
         s = f.read()
     bList = pem.dePemList(s, "CERTIFICATE")
     for b in bList: