Merge pull request #49 from daira/large-uas

Updates for large UAs/UVKs

f4jumble.py

@@ -8,17 +8,22 @@ from pyblake2 import blake2b
 
 from tv_output import render_args, render_tv
 from tv_rand import Rand
+from utils import i2leosp
 
 
 # Maximum output length of BLAKE2b
 l_H = 64
 assert 8*l_H == 512
 
+MIN_l_M = 48
+MAX_l_M = 4194368
+assert MAX_l_M == 65537*l_H
+
 def instantiate(l_L, l_R):
     def H(i, u):
         digest = blake2b(
             digest_size=l_L,
-            person=b'UA_F4Jumble_H_' + bytes([i, 0]),
+            person=b'UA_F4Jumble_H' + bytes([i, 0, 0]),
         )
         digest.update(u)
         return digest.digest()
@@ -27,11 +32,11 @@ def instantiate(l_L, l_R):
         def inner(j):
             digest = blake2b(
                 digest_size=l_H,
-                person=b'UA_F4Jumble_G_' + bytes([i, j]),
+                person=b'UA_F4Jumble_G' + bytes([i]) + i2leosp(16, j),
             )
             digest.update(u)
             return digest.digest()
-        
+
         return b''.join([inner(j) for j in range(0, math.ceil(l_R/l_H))])[:l_R]
 
     return (H, G)
@@ -41,7 +46,7 @@ def xor(x, y):
 
 def f4jumble(M):
     l_M = len(M)
-    assert 48 <= l_M and l_M <= 16448
+    assert MIN_l_M <= l_M and l_M <= MAX_l_M
 
     l_L = min([l_H, l_M//2])
     l_R = l_M - l_L
@@ -58,7 +63,7 @@ def f4jumble(M):
 
 def f4jumble_inv(M):
     l_M = len(M)
-    assert 48 <= l_M and l_M <= 16448
+    assert MIN_l_M <= l_M and l_M <= MAX_l_M
 
     l_L = min([l_H, l_M//2])
     l_R = l_M - l_L
@@ -86,28 +91,28 @@ def main():
         return bytes(ret)
     rand = Rand(randbytes)
 
-    test_vectors = []
+    plain_test_vectors = []
+
     # Generate test vectors with various lengths:
     for l_M in [
-        48,
+        MIN_l_M,
         l_H,
         2*l_H,
         2*l_H + 1,
         3*l_H,
         3*l_H + 1,
-        (rand.u32() % 16400) + 48,
-        16448,
+        257*l_H,
+        257*l_H + 1,
     ]:
         M = rand.b(l_M)
         jumbled = f4jumble(M)
         assert len(jumbled) == len(M)
         assert f4jumble_inv(jumbled) == M
-        test_vectors.append(M)
 
-    test_vectors = [{
-        'normal': M,
-        'jumbled': f4jumble(M),
-    } for M in test_vectors]
+        plain_test_vectors.append({
+            'normal': M,
+            'jumbled': jumbled,
+        })
 
     render_tv(
         args,
@@ -116,7 +121,7 @@ def main():
             ('normal', 'Vec<u8>'),
             ('jumbled', 'Vec<u8>'),
         ),
-        test_vectors,
+        plain_test_vectors,
     )
 
 

f4jumble_long.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+import sys; assert sys.version_info[0] >= 3, "Python 3 required."
+
+from pyblake2 import blake2b
+
+from tv_output import render_args, render_tv
+from f4jumble import f4jumble, f4jumble_inv, MAX_l_M
+
+
+def main():
+    args = render_args()
+
+    hashed_test_vectors = []
+
+    for l_M in [
+        3246395,
+        MAX_l_M,
+    ]:
+        M = bytes([i & 0xFF for i in range(l_M)])
+        jumbled = f4jumble(M)
+        assert len(jumbled) == len(M)
+        assert f4jumble_inv(jumbled) == M
+
+        hashed_test_vectors.append({
+            'length': l_M,
+            'jumbled_hash': blake2b(jumbled).digest()
+        })
+
+    render_tv(
+        args,
+        'f4jumble_long',
+        (
+            ('length', 'usize'),
+            ('jumbled_hash', '[u8; 64]'),
+        ),
+        hashed_test_vectors,
+    )
+
+
+if __name__ == "__main__":
+    main()

orchard_empty_roots.py

@@ -1,3 +1,6 @@
+#!/usr/bin/env python3
+import sys; assert sys.version_info[0] >= 3, "Python 3 required."
+
 from orchard_merkle_tree import empty_roots
 from orchard_pallas import Fp
 from tv_output import render_args, render_tv

orchard_map_to_curve.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+import sys; assert sys.version_info[0] >= 3, "Python 3 required."
 
 from orchard_group_hash import map_to_curve_simple_swu
 from orchard_iso_pallas import Point as IsoPoint

orchard_note_encryption.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 import sys; assert sys.version_info[0] >= 3, "Python 3 required."
+
 import struct
 
 from chacha20poly1305 import ChaCha20Poly1305

orchard_poseidon.py

@@ -1,3 +1,6 @@
+#!/usr/bin/env python3
+import sys; assert sys.version_info[0] >= 3, "Python 3 required."
+
 from orchard_pallas import Fp
 import numpy as np
 from itertools import chain

orchard_poseidon_hash.py

@@ -1,3 +1,6 @@
+#!/usr/bin/env python3
+import sys; assert sys.version_info[0] >= 3, "Python 3 required."
+
 from orchard_pallas import Fp
 from orchard_poseidon import perm
 from utils import leos2ip

transaction.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 import struct
 
 from orchard_pallas import (

unified_address.py

@@ -4,17 +4,18 @@ import sys; assert sys.version_info[0] >= 3, "Python 3 required."
 import math
 import struct
 
-from pyblake2 import blake2b
 from bech32m import bech32_encode, bech32_decode, convertbits, Encoding
 
 from tv_output import render_args, render_tv, Some
 from tv_rand import Rand
+from zc_utils import write_compact_size, parse_compact_size
 from f4jumble import f4jumble, f4jumble_inv
 import sapling_key_components
 import orchard_key_components
 
+
 def tlv(typecode, value):
-    return b"".join([bytes([typecode, len(value)]), value])
+    return b"".join([write_compact_size(typecode), write_compact_size(len(value)), value])
 
 def padding(hrp):
     assert(len(hrp) <= 16)
@@ -51,46 +52,35 @@ def decode_unified(addr_str):
     suffix = decoded[-16:]
     # check trailing padding bytes
     assert suffix == padding(hrp)
-    decoded = decoded[:-16]
+    rest = decoded[:-16]
 
-    s = 0
-    acc = []
     result = {}
-    for b in decoded:
-        if s == 0:
-            receiver_type = b
-            s = 1
-        elif s == 1:
-            receiver_len = b
-            expected_len = {0: 20, 1: 20, 2: 43, 3: 43}.get(receiver_type)
-            if expected_len is not None:
-                assert receiver_len == expected_len, "incorrect receiver length"
-            s = 2
-        elif s == 2:
-            if len(acc) < receiver_len:
-                acc.append(b)
-            
-            if len(acc) == receiver_len:
-                if receiver_type == 0 or receiver_type == 1:
-                    assert not ('transparent' in result), "duplicate transparent receiver detected"
-                    assert len(acc) == 20
-                    result['transparent'] = bytes(acc)
-                    acc = []
-                    s = 0
+    while len(rest) > 0:
+        (receiver_type, rest) = parse_compact_size(rest)
+        (receiver_len, rest) = parse_compact_size(rest)
 
-                elif receiver_type == 2:
-                    assert not ('sapling' in result), "duplicate sapling receiver detected"
-                    assert len(acc) == 43
-                    result['sapling'] = bytes(acc)
-                    acc = []
-                    s = 0
+        expected_len = {0: 20, 1: 20, 2: 43, 3: 43}.get(receiver_type)
+        if expected_len is not None:
+            assert receiver_len == expected_len, "incorrect receiver length"
+
+        assert len(rest) >= receiver_len
+        (receiver, rest) = (rest[:receiver_len], rest[receiver_len:])
+
+        if receiver_type == 0 or receiver_type == 1:
+            assert not ('transparent' in result), "duplicate transparent receiver detected"
+            assert len(receiver) == 20
+            result['transparent'] = receiver
+
+        elif receiver_type == 2:
+            assert not ('sapling' in result), "duplicate sapling receiver detected"
+            assert len(receiver) == 43
+            result['sapling'] = receiver
+
+        elif receiver_type == 3:
+            assert not ('orchard' in result), "duplicate orchard receiver detected"
+            assert len(receiver) == 43
+            result['orchard'] = receiver
 
-                elif receiver_type == 3:
-                    assert not ('orchard' in result), "duplicate orchard receiver detected"
-                    assert len(acc) == 43
-                    result['orchard'] = bytes(acc)
-                    acc = []
-                    s = 0
     return result
 
 
@@ -130,13 +120,13 @@ def main():
             orchard_default_d = orchard_fvk.default_d()
             orchard_default_pk_d = orchard_fvk.default_pkd()
             orchard_raw_addr = b"".join([orchard_default_d[:11], bytes(orchard_default_pk_d)[:32]])
-        else: 
+        else:
             orchard_raw_addr = None
 
         is_p2pkh = rand.bool()
         receivers = [
-            orchard_raw_addr, 
-            sapling_raw_addr, 
+            orchard_raw_addr,
+            sapling_raw_addr,
             (is_p2pkh, t_addr)
         ]
         ua = encode_unified(receivers)

zc_utils.py

@@ -1,6 +1,12 @@
+#!/usr/bin/env python3
+import sys; assert sys.version_info[0] >= 3, "Python 3 required."
+
 import struct
 
-def write_compact_size(n):
+MAX_SIZE = 0x2000000
+
+def write_compact_size(n, allow_u64=False):
+    assert allow_u64 or n <= MAX_SIZE
     if n < 253:
         return struct.pack('B', n)
     elif n <= 0xFFFF:
@@ -10,18 +16,70 @@ def write_compact_size(n):
     else:
         return struct.pack('B', 255) + struct.pack('<Q', n)
 
-assert write_compact_size(0) == b'\x00'
-assert write_compact_size(1) == b'\x01'
-assert write_compact_size(252) == b'\xFC'
-assert write_compact_size(253) == b'\xFD\xFD\x00'
-assert write_compact_size(254) == b'\xFD\xFE\x00'
-assert write_compact_size(255) == b'\xFD\xFF\x00'
-assert write_compact_size(256) == b'\xFD\x00\x01'
-assert write_compact_size(0xFFFE) == b'\xFD\xFE\xFF'
-assert write_compact_size(0xFFFF) == b'\xFD\xFF\xFF'
-assert write_compact_size(0x010000) == b'\xFE\x00\x00\x01\x00'
-assert write_compact_size(0x010001) == b'\xFE\x01\x00\x01\x00'
-assert write_compact_size(0xFFFFFFFE) == b'\xFE\xFE\xFF\xFF\xFF'
-assert write_compact_size(0xFFFFFFFF) == b'\xFE\xFF\xFF\xFF\xFF'
-assert write_compact_size(0x0100000000) == b'\xFF\x00\x00\x00\x00\x01\x00\x00\x00'
-assert write_compact_size(0xFFFFFFFFFFFFFFFF) == b'\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF'
+def parse_compact_size(rest, allow_u64=False):
+    (n, rest) = parse_compact_u64(rest)
+    assert allow_u64 or n <= MAX_SIZE
+    return (n, rest)
+
+def parse_compact_u64(rest):
+    assert len(rest) >= 1
+    b = rest[0]
+    if b < 253:
+        return (b, rest[1:])
+    elif b == 253:
+        assert len(rest) >= 3
+        n = struct.unpack('<H', rest[1:3])[0]
+        assert n >= 253
+        return (n, rest[3:])
+    elif b == 254:
+        assert len(rest) >= 5
+        n = struct.unpack('<I', rest[1:5])[0]
+        assert n >= 0x10000
+        return (n, rest[5:])
+    else:
+        assert len(rest) >= 9
+        n = struct.unpack('<Q', rest[1:9])[0]
+        assert n >= 0x100000000
+        return (n, rest[9:])
+
+
+def assert_parse_fails(encoding, allow_u64):
+    try:
+        parse_compact_size(encoding, allow_u64)
+    except AssertionError:
+        pass
+    else:
+        raise AssertionError("parse_compact_size(%r) failed to raise AssertionError" % (encoding,))
+
+def test_round_trip(n, encoding, allow_u64):
+    assert write_compact_size(n, allow_u64) == encoding
+    assert parse_compact_size(encoding, allow_u64) == (n, b'')
+    assert parse_compact_size(encoding + b'*', allow_u64) == (n, b'*')
+    assert_parse_fails(encoding[:-1], allow_u64)
+
+for allow_u64 in (False, True):
+    test_round_trip(0, b'\x00', allow_u64)
+    test_round_trip(1, b'\x01', allow_u64)
+    test_round_trip(252, b'\xFC', allow_u64)
+    test_round_trip(253, b'\xFD\xFD\x00', allow_u64)
+    test_round_trip(254, b'\xFD\xFE\x00', allow_u64)
+    test_round_trip(255, b'\xFD\xFF\x00', allow_u64)
+    test_round_trip(256, b'\xFD\x00\x01', allow_u64)
+    test_round_trip(0xFFFE, b'\xFD\xFE\xFF', allow_u64)
+    test_round_trip(0xFFFF, b'\xFD\xFF\xFF', allow_u64)
+    test_round_trip(0x010000, b'\xFE\x00\x00\x01\x00', allow_u64)
+    test_round_trip(0x010001, b'\xFE\x01\x00\x01\x00', allow_u64)
+    test_round_trip(0x02000000, b'\xFE\x00\x00\x00\x02', allow_u64)
+
+    assert_parse_fails(b'\xFD\xFC\x00', allow_u64)
+    assert_parse_fails(b'\xFE\xFF\xFF\x00\x00', allow_u64)
+    assert_parse_fails(b'\xFF\xFF\xFF\xFF\xFF\x00\x00\x00\x00', allow_u64)
+
+assert_parse_fails(b'\xFE\x01\x00\x00\x02', False)
+assert_parse_fails(b'\xFF\x00\x00\x00\x00\x01\x00\x00\x00', False)
+assert_parse_fails(b'\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF', False)
+
+test_round_trip(0xFFFFFFFE, b'\xFE\xFE\xFF\xFF\xFF', True)
+test_round_trip(0xFFFFFFFF, b'\xFE\xFF\xFF\xFF\xFF', True)
+test_round_trip(0x0100000000, b'\xFF\x00\x00\x00\x00\x01\x00\x00\x00', True)
+test_round_trip(0xFFFFFFFFFFFFFFFF, b'\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF', True)

zip_0143.py

@@ -1,4 +1,6 @@
 #!/usr/bin/env python3
+import sys; assert sys.version_info[0] >= 3, "Python 3 required."
+
 from pyblake2 import blake2b
 import struct
 

zip_0243.py

@@ -1,4 +1,6 @@
 #!/usr/bin/env python3
+import sys; assert sys.version_info[0] >= 3, "Python 3 required."
+
 from pyblake2 import blake2b
 import struct
 

zip_0244.py

@@ -1,4 +1,6 @@
 #!/usr/bin/env python3
+import sys; assert sys.version_info[0] >= 3, "Python 3 required."
+
 from pyblake2 import blake2b
 import struct