From fb4e0d6efecefba44bf9cebd01ad26233b400b4f Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Thu, 14 Jun 2018 23:27:03 +1200 Subject: [PATCH 1/8] Implement helper class for generating random values --- tv_rand.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 tv_rand.py diff --git a/tv_rand.py b/tv_rand.py new file mode 100644 index 0000000..3035286 --- /dev/null +++ b/tv_rand.py @@ -0,0 +1,28 @@ +import os +import struct + + +class Rand(object): + def __init__(self, random=os.urandom): + self._random = random + + def b(self, l): + return self._random(l) + + def v(self, l, f): + return struct.unpack(f, self.b(l))[0] + + def u8(self): + return self.v(1, 'b') + + def u32(self): + return self.v(4, ' 0 + + def a(self, vals): + return vals[self.u8() % len(vals)] From 704a2ac14d987f10e5e79c03903bbb472c35aa3d Mon Sep 17 00:00:00 2001 From: Ariel Gabizon Date: Wed, 13 Jun 2018 15:45:05 +0200 Subject: [PATCH 2/8] Implement ZIP 143 test vector generation No support for JoinSplits yet. Co-authored-by: Jack Grigg --- transaction.py | 117 ++++++++++++++++++++++++++++++++++++ zip_0143.py | 157 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 274 insertions(+) create mode 100644 transaction.py create mode 100644 zip_0143.py diff --git a/transaction.py b/transaction.py new file mode 100644 index 0000000..8cadb4a --- /dev/null +++ b/transaction.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +import struct + +MAX_MONEY = 21000000 * 100000000 +TX_EXPIRY_HEIGHT_THRESHOLD = 500000000 + +OVERWINTER_VERSION_GROUP_ID = 0x03C48270 +OVERWINTER_TX_VERSION = 3 + + +RAND_OPCODES = [ + 0x00, # OP_FALSE, + 0x51, # OP_1, + 0x52, # OP_2, + 0x53, # OP_3, + 0xac, # OP_CHECKSIG, + 0x63, # OP_IF, + 0x65, # OP_VERIF, + 0x6a, # OP_RETURN, +] + +class Script(object): + def __init__(self, rand): + self._script = bytes([ + rand.a(RAND_OPCODES) for i in range(rand.u8() % 10) + ]) + + def raw(self): + return self._script + + def __bytes__(self): + return struct.pack('b', len(self._script)) + self._script + + +class OutPoint(object): + def __init__(self, rand): + self.txid = rand.b(32) + self.n = rand.u32() + + def __bytes__(self): + return self.txid + struct.pack('= 2: + ret += struct.pack('b', 0) + + return ret diff --git a/zip_0143.py b/zip_0143.py new file mode 100644 index 0000000..b2b279c --- /dev/null +++ b/zip_0143.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +from pyblake2 import blake2b +import struct + +from transaction import ( + MAX_MONEY, + OVERWINTER_TX_VERSION, + Script, + Transaction, +) +from tv_output import render_args, render_tv +from tv_rand import Rand + + +SIGHASH_ALL = 1 +SIGHASH_NONE = 2 +SIGHASH_SINGLE = 3 +SIGHASH_ANYONECANPAY = 0x80 + +NOT_AN_INPUT = -1 # For portability of the test vectors + +def getHashPrevouts(tx): + digest = blake2b(digest_size=32, person=b'ZcashPrevoutHash') + for x in tx.vin: + digest.update(bytes(x.prevout)) + return digest.digest() + +def getHashSequence(tx): + digest = blake2b(digest_size=32, person=b'ZcashSequencHash') + for x in tx.vin: + digest.update(struct.pack(''), + ('scriptCode', 'Vec'), + ('nIn', 'u32'), + ('nHashType', 'u32'), + ('amount', 'u64'), + ('consensusBranchId', 'u32'), + ('sighash', '[u8; 32]'), + ), + test_vectors, + ) + + +if __name__ == '__main__': + main() From 9448eeeee59bcd0a3d3a8de4301d7fa48be77e8f Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 15 Jun 2018 00:42:06 +1200 Subject: [PATCH 3/8] Implement JoinSplits in ZIP 143 test vectors --- transaction.py | 75 +++++++++++++++++++++++++++++++++++++++++++++++++- zip_0143.py | 12 ++++++-- 2 files changed, 84 insertions(+), 3 deletions(-) diff --git a/transaction.py b/transaction.py index 8cadb4a..6dbf557 100644 --- a/transaction.py +++ b/transaction.py @@ -8,6 +8,66 @@ OVERWINTER_VERSION_GROUP_ID = 0x03C48270 OVERWINTER_TX_VERSION = 3 +# BN254 encoding of G1 elements. p[1] is big-endian. +def pack_g1(p): + return struct.pack('B', 0x02 | (1 if p[0] else 0)) + p[1] + +# BN254 encoding of G2 elements. p[1] is big-endian. +def pack_g2(p): + return struct.pack('B', 0x0a | (1 if p[0] else 0)) + p[1] + +class PHGRProof(object): + def __init__(self, rand): + self.g_A = (rand.bool(), rand.b(32)) + self.g_A_prime = (rand.bool(), rand.b(32)) + self.g_B = (rand.bool(), rand.b(64)) + self.g_B_prime = (rand.bool(), rand.b(32)) + self.g_C = (rand.bool(), rand.b(32)) + self.g_C_prime = (rand.bool(), rand.b(32)) + self.g_K = (rand.bool(), rand.b(32)) + self.g_H = (rand.bool(), rand.b(32)) + + def __bytes__(self): + return ( + pack_g1(self.g_A) + + pack_g1(self.g_A_prime) + + pack_g2(self.g_B) + + pack_g1(self.g_B_prime) + + pack_g1(self.g_C) + + pack_g1(self.g_C_prime) + + pack_g1(self.g_K) + + pack_g1(self.g_H) + ) + + +class JoinSplit(object): + def __init__(self, rand): + self.vpub_old = 0 + self.vpub_new = 0 + self.anchor = rand.b(32) + self.nullifiers = (rand.b(32), rand.b(32)) + self.commitments = (rand.b(32), rand.b(32)) + self.ephemeralKey = rand.b(32) + self.randomSeed = rand.b(32) + self.macs = (rand.b(32), rand.b(32)) + self.proof = PHGRProof(rand) + self.ciphertexts = (rand.b(601), rand.b(601)) + + def __bytes__(self): + return ( + struct.pack('= 2: + for i in range(rand.u8() % 3): + self.vJoinSplit.append(JoinSplit(rand)) + if len(self.vJoinSplit) > 0: + self.joinSplitPubKey = rand.b(32) # Potentially invalid + self.joinSplitSig = rand.b(64) # Invalid + def header(self): return self.nVersion | (1 << 31 if self.fOverwintered else 0) @@ -112,6 +180,11 @@ class Transaction(object): ret += struct.pack('= 2: - ret += struct.pack('b', 0) + ret += struct.pack('b', len(self.vJoinSplit)) + for jsdesc in self.vJoinSplit: + ret += bytes(jsdesc) + if len(self.vJoinSplit) > 0: + ret += self.joinSplitPubKey + ret += self.joinSplitSig return ret diff --git a/zip_0143.py b/zip_0143.py index b2b279c..f48fb70 100644 --- a/zip_0143.py +++ b/zip_0143.py @@ -37,9 +37,14 @@ def getHashOutputs(tx): digest.update(bytes(x)) return digest.digest() +def getHashJoinSplits(tx): + digest = blake2b(digest_size=32, person=b'ZcashJSplitsHash') + for jsdesc in tx.vJoinSplit: + digest.update(bytes(jsdesc)) + digest.update(tx.joinSplitPubKey) + return digest.digest() + -# Currently assumes the nHashType is SIGHASHALL -# and that there are no joinSplits def signature_hash(scriptCode, tx, nIn, nHashType, amount, consensusBranchId): hashPrevouts = b'\x00'*32 hashSequence = b'\x00'*32 @@ -63,6 +68,9 @@ def signature_hash(scriptCode, tx, nIn, nHashType, amount, consensusBranchId): digest.update(bytes(tx.vout[nIn])) hashOutputs = digest.digest() + if len(tx.vJoinSplit) > 0: + hashJoinSplits = getHashJoinSplits(tx) + digest = blake2b( digest_size=32, person=b'ZcashSigHash' + struct.pack(' Date: Sat, 16 Jun 2018 11:00:11 +1200 Subject: [PATCH 4/8] Don't reverse 32-byte transactions in Zcash output --- tv_output.py | 7 +++++-- zip_0143.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tv_output.py b/tv_output.py index 38c182f..8c8a417 100644 --- a/tv_output.py +++ b/tv_output.py @@ -43,7 +43,7 @@ def tv_json(filename, parts, vectors, bitcoin_flavoured): ', '.join([p[0] for p in parts]) )) print(' ' + ',\n '.join([ - json.dumps([tv_value_json(v[p[0]], bitcoin_flavoured) for p in parts]) for v in vectors + json.dumps([tv_value_json(v[p[0]], p[1].get('bitcoin_flavoured', bitcoin_flavoured)) for p in parts]) for v in vectors ])) print(']') @@ -93,7 +93,7 @@ def tv_part_rust(name, value, indent=3): def tv_rust(filename, parts, vectors): print(' struct TestVector {') - for p in parts: print(' %s: %s,' % p) + for p in parts: print(' %s: %s,' % (p[0], p[1]['rust'])) print(''' }; // From https://github.com/zcash-hackworks/zcash-test-vectors/blob/master/%s.py''' % ( @@ -124,6 +124,9 @@ def render_args(): return parser.parse_args() def render_tv(args, filename, parts, vectors): + # Convert older format + parts = [(p[0], p[1] if type(p[1]) == type({}) else {'rust': p[1]}) for p in parts] + if args.target == 'rust': tv_rust(filename, parts, vectors) elif args.target == 'zcash': diff --git a/zip_0143.py b/zip_0143.py index f48fb70..642af0b 100644 --- a/zip_0143.py +++ b/zip_0143.py @@ -149,7 +149,7 @@ def main(): args, 'zip_0143', ( - ('tx', 'Vec'), + ('tx', {'rust': 'Vec', 'bitcoin_flavoured': False}), ('scriptCode', 'Vec'), ('nIn', 'u32'), ('nHashType', 'u32'), From acdfe06c87cef45b4eaa715bd71d05c47f5305c0 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 6 Jul 2018 13:46:59 +0100 Subject: [PATCH 5/8] Implement CompactSize helper --- transaction.py | 10 ++++++---- zc_utils.py | 27 +++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 4 deletions(-) create mode 100644 zc_utils.py diff --git a/transaction.py b/transaction.py index 6dbf557..25aa5b2 100644 --- a/transaction.py +++ b/transaction.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 import struct +from zc_utils import write_compact_size + MAX_MONEY = 21000000 * 100000000 TX_EXPIRY_HEIGHT_THRESHOLD = 500000000 @@ -89,7 +91,7 @@ class Script(object): return self._script def __bytes__(self): - return struct.pack('b', len(self._script)) + self._script + return write_compact_size(len(self._script)) + self._script class OutPoint(object): @@ -167,11 +169,11 @@ class Transaction(object): self.nVersionGroupId == OVERWINTER_VERSION_GROUP_ID and \ self.nVersion == OVERWINTER_TX_VERSION - ret += struct.pack('b', len(self.vin)) + ret += write_compact_size(len(self.vin)) for x in self.vin: ret += bytes(x) - ret += struct.pack('b', len(self.vout)) + ret += write_compact_size(len(self.vout)) for x in self.vout: ret += bytes(x) @@ -180,7 +182,7 @@ class Transaction(object): ret += struct.pack('= 2: - ret += struct.pack('b', len(self.vJoinSplit)) + ret += write_compact_size(len(self.vJoinSplit)) for jsdesc in self.vJoinSplit: ret += bytes(jsdesc) if len(self.vJoinSplit) > 0: diff --git a/zc_utils.py b/zc_utils.py new file mode 100644 index 0000000..06809ac --- /dev/null +++ b/zc_utils.py @@ -0,0 +1,27 @@ +import struct + +def write_compact_size(n): + if n < 253: + return struct.pack('B', n) + elif n <= 0xFFFF: + return struct.pack('B', 253) + struct.pack(' Date: Tue, 21 Aug 2018 19:35:11 +0100 Subject: [PATCH 6/8] Use snake_case for ZIP 143 test vector part names --- zip_0143.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/zip_0143.py b/zip_0143.py index 642af0b..2ccd578 100644 --- a/zip_0143.py +++ b/zip_0143.py @@ -137,11 +137,11 @@ def main(): test_vectors.append({ 'tx': bytes(tx), - 'scriptCode': scriptCode.raw(), - 'nIn': nIn, - 'nHashType': nHashType, + 'script_code': scriptCode.raw(), + 'transparent_input': nIn, + 'hash_type': nHashType, 'amount': amount, - 'consensusBranchId': consensusBranchId, + 'consensus_branch_id': consensusBranchId, 'sighash': sighash, }) @@ -150,11 +150,11 @@ def main(): 'zip_0143', ( ('tx', {'rust': 'Vec', 'bitcoin_flavoured': False}), - ('scriptCode', 'Vec'), - ('nIn', 'u32'), - ('nHashType', 'u32'), + ('script_code', 'Vec'), + ('transparent_input', 'u32'), + ('hash_type', 'u32'), ('amount', 'u64'), - ('consensusBranchId', 'u32'), + ('consensus_branch_id', 'u32'), ('sighash', '[u8; 32]'), ), test_vectors, From a9676cc9b4e9ad86306f8c252e0307de9659215e Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Tue, 21 Aug 2018 19:44:02 +0100 Subject: [PATCH 7/8] Render Vec data correctly --- tv_output.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/tv_output.py b/tv_output.py index 8c8a417..9bc6766 100644 --- a/tv_output.py +++ b/tv_output.py @@ -63,6 +63,17 @@ def tv_bytes_rust(name, value, pad): pad, )) +def tv_vec_bytes_rust(name, value, pad): + print('''%s%s: vec![ + %s%s +%s],''' % ( + pad, + name, + pad, + chunk(hexlify(value)), + pad, + )) + def tv_option_bytes_rust(name, value, pad): if value: print('''%s%s: Some([ @@ -80,10 +91,13 @@ def tv_option_bytes_rust(name, value, pad): def tv_int_rust(name, value, pad): print('%s%s: %d,' % (pad, name, value)) -def tv_part_rust(name, value, indent=3): +def tv_part_rust(name, value, typ, indent=3): pad = ' ' * indent if type(value) == bytes: - tv_bytes_rust(name, value, pad) + if typ == 'Vec': + tv_vec_bytes_rust(name, value, pad) + else: + tv_bytes_rust(name, value, pad) elif isinstance(value, Some) or value is None: tv_option_bytes_rust(name, value, pad) elif type(value) == int: @@ -101,13 +115,13 @@ def tv_rust(filename, parts, vectors): )) if type(vectors) == type({}): print(' let test_vector = TestVector {') - for p in parts: tv_part_rust(p[0], vectors[p[0]]) + for p in parts: tv_part_rust(p[0], vectors[p[0]], p[1]['rust']) print(' };') elif type(vectors) == type([]): print(' let test_vectors = vec![') for vector in vectors: print(' TestVector {') - for p in parts: tv_part_rust(p[0], vector[p[0]], 4) + for p in parts: tv_part_rust(p[0], vector[p[0]], p[1]['rust'], 4) print(' },') print(' ];') else: From 17d146b5a95cef61e57a184972342a445af6da89 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Tue, 21 Aug 2018 20:13:11 +0100 Subject: [PATCH 8/8] Refactor Rust output rendering to support reformatting Used to convert a -1 for JSON to Option::None in Rust --- tv_output.py | 48 +++++++++++++++++++++++++++++++++++++----------- zip_0143.py | 11 +++++++---- 2 files changed, 44 insertions(+), 15 deletions(-) diff --git a/tv_output.py b/tv_output.py index 9bc6766..ce63ad8 100644 --- a/tv_output.py +++ b/tv_output.py @@ -88,18 +88,44 @@ def tv_option_bytes_rust(name, value, pad): else: print('%s%s: None,' % (pad, name)) +def tv_option_vec_bytes_rust(name, value, pad): + if value: + print('''%s%s: Some(vec![ + %s%s +%s]),''' % ( + pad, + name, + pad, + chunk(hexlify(value.thing)), + pad, + )) + else: + print('%s%s: None,' % (pad, name)) + def tv_int_rust(name, value, pad): print('%s%s: %d,' % (pad, name, value)) -def tv_part_rust(name, value, typ, indent=3): +def tv_option_int_rust(name, value, pad): + if value: + print('%s%s: Some(%d),' % (pad, name, value.thing)) + else: + print('%s%s: None,' % (pad, name)) + +def tv_part_rust(name, value, config, indent=3): + if 'rust_fmt' in config: + value = config['rust_fmt'](value) + pad = ' ' * indent - if type(value) == bytes: - if typ == 'Vec': - tv_vec_bytes_rust(name, value, pad) - else: - tv_bytes_rust(name, value, pad) - elif isinstance(value, Some) or value is None: + if config['rust_type'] == 'Option>': + tv_option_vec_bytes_rust(name, value, pad) + elif config['rust_type'] == 'Vec': + tv_vec_bytes_rust(name, value, pad) + elif config['rust_type'].startswith('Option<['): tv_option_bytes_rust(name, value, pad) + elif type(value) == bytes: + tv_bytes_rust(name, value, pad) + elif config['rust_type'].startswith('Option<'): + tv_option_int_rust(name, value, pad) elif type(value) == int: tv_int_rust(name, value, pad) else: @@ -107,7 +133,7 @@ def tv_part_rust(name, value, typ, indent=3): def tv_rust(filename, parts, vectors): print(' struct TestVector {') - for p in parts: print(' %s: %s,' % (p[0], p[1]['rust'])) + for p in parts: print(' %s: %s,' % (p[0], p[1]['rust_type'])) print(''' }; // From https://github.com/zcash-hackworks/zcash-test-vectors/blob/master/%s.py''' % ( @@ -115,13 +141,13 @@ def tv_rust(filename, parts, vectors): )) if type(vectors) == type({}): print(' let test_vector = TestVector {') - for p in parts: tv_part_rust(p[0], vectors[p[0]], p[1]['rust']) + for p in parts: tv_part_rust(p[0], vectors[p[0]], p[1]) print(' };') elif type(vectors) == type([]): print(' let test_vectors = vec![') for vector in vectors: print(' TestVector {') - for p in parts: tv_part_rust(p[0], vector[p[0]], p[1]['rust'], 4) + for p in parts: tv_part_rust(p[0], vector[p[0]], p[1], 4) print(' },') print(' ];') else: @@ -139,7 +165,7 @@ def render_args(): def render_tv(args, filename, parts, vectors): # Convert older format - parts = [(p[0], p[1] if type(p[1]) == type({}) else {'rust': p[1]}) for p in parts] + parts = [(p[0], p[1] if type(p[1]) == type({}) else {'rust_type': p[1]}) for p in parts] if args.target == 'rust': tv_rust(filename, parts, vectors) diff --git a/zip_0143.py b/zip_0143.py index 2ccd578..5d0ffb5 100644 --- a/zip_0143.py +++ b/zip_0143.py @@ -8,7 +8,7 @@ from transaction import ( Script, Transaction, ) -from tv_output import render_args, render_tv +from tv_output import render_args, render_tv, Some from tv_rand import Rand @@ -17,7 +17,7 @@ SIGHASH_NONE = 2 SIGHASH_SINGLE = 3 SIGHASH_ANYONECANPAY = 0x80 -NOT_AN_INPUT = -1 # For portability of the test vectors +NOT_AN_INPUT = -1 # For portability of the test vectors; replaced with None for Rust def getHashPrevouts(tx): digest = blake2b(digest_size=32, person=b'ZcashPrevoutHash') @@ -149,9 +149,12 @@ def main(): args, 'zip_0143', ( - ('tx', {'rust': 'Vec', 'bitcoin_flavoured': False}), + ('tx', {'rust_type': 'Vec', 'bitcoin_flavoured': False}), ('script_code', 'Vec'), - ('transparent_input', 'u32'), + ('transparent_input', { + 'rust_type': 'Option', + 'rust_fmt': lambda x: None if x == -1 else Some(x), + }), ('hash_type', 'u32'), ('amount', 'u64'), ('consensus_branch_id', 'u32'),