From 12c344177803d0a5f21b20f748433df75a55bc97 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Tue, 11 Jul 2023 15:17:44 +0000 Subject: [PATCH 01/76] Migrate to Rust crate versions with fast spendability support --- backend-lib/Cargo.lock | 479 ++++++------------ backend-lib/Cargo.toml | 19 +- .../z/ecc/android/sdk/internal/Backend.kt | 15 +- .../android/sdk/internal/jni/RustBackend.kt | 45 +- .../sdk/internal/model/JniScanRange.kt | 41 ++ backend-lib/src/main/rust/lib.rs | 157 +++--- backend-lib/src/main/rust/utils.rs | 8 +- .../android/sdk/util/BalancePrinterUtil.kt | 12 - .../cash/z/ecc/fixture/FakeRustBackend.kt | 5 +- .../sdk/block/CompactBlockProcessor.kt | 35 +- .../android/sdk/internal/TypesafeBackend.kt | 17 +- .../sdk/internal/TypesafeBackendImpl.kt | 18 +- .../android/sdk/internal/model/ScanRange.kt | 19 + 13 files changed, 373 insertions(+), 497 deletions(-) create mode 100644 backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniScanRange.kt create mode 100644 sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanRange.kt diff --git a/backend-lib/Cargo.lock b/backend-lib/Cargo.lock index 98e9eebe..1a0a5687 100644 --- a/backend-lib/Cargo.lock +++ b/backend-lib/Cargo.lock @@ -40,15 +40,21 @@ dependencies = [ [[package]] name = "ahash" -version = "0.7.6" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" +checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" dependencies = [ - "getrandom", + "cfg-if", "once_cell", "version_check", ] +[[package]] +name = "allocator-api2" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56fc6cf8dc8c4158eed8649f9b8b0ea1518eb62b544fe9490d66fa0b349eafe9" + [[package]] name = "anyhow" version = "1.0.71" @@ -88,12 +94,6 @@ dependencies = [ "rustc-demangle", ] -[[package]] -name = "base-x" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" - [[package]] name = "base58" version = "0.1.0" @@ -148,7 +148,7 @@ dependencies = [ "hmac", "pbkdf2", "rand", - "sha2 0.10.6", + "sha2", "unicode-normalization", "zeroize", ] @@ -159,6 +159,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42" + [[package]] name = "bitvec" version = "1.0.1" @@ -205,15 +211,6 @@ dependencies = [ "generic-array 0.12.4", ] -[[package]] -name = "block-buffer" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" -dependencies = [ - "generic-array 0.14.7", -] - [[package]] name = "block-buffer" version = "0.10.4" @@ -247,11 +244,12 @@ dependencies = [ [[package]] name = "bs58" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "771fe0050b883fcc3ea2359b1a96bcfbc090b7116eae7c3c512c7a083fdf23d3" +checksum = "f5353f36341f7451062466f0b755b96ac3a9547e4d7f6b70d603fc721a7d7896" dependencies = [ - "sha2 0.9.9", + "sha2", + "tinyvec", ] [[package]] @@ -350,12 +348,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "const_fn" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbdcdcb6d86f71c5e97409ad45898af11cbc995b4ee8112d59095a28d376c935" - [[package]] name = "constant_time_eq" version = "0.2.5" @@ -448,15 +440,6 @@ dependencies = [ "generic-array 0.12.4", ] -[[package]] -name = "digest" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" -dependencies = [ - "generic-array 0.14.7", -] - [[package]] name = "digest" version = "0.10.6" @@ -468,33 +451,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "directories" -version = "5.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a49173b84e034382284f27f1af4dcbbd231ffa358c0fe316541a7337f376a35" -dependencies = [ - "dirs-sys", -] - -[[package]] -name = "dirs-sys" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" -dependencies = [ - "libc", - "option-ext", - "redox_users", - "windows-sys 0.48.0", -] - -[[package]] -name = "discard" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" - [[package]] name = "dlopen2" version = "0.4.1" @@ -527,8 +483,7 @@ checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" [[package]] name = "equihash" version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab579d7cf78477773b03e80bc2f89702ef02d7112c711d54ca93dcdce68533d5" +source = "git+https://github.com/zcash/librustzcash.git?rev=352e1c709ac4c95074812dcabeba3cd5565371d2#352e1c709ac4c95074812dcabeba3cd5565371d2" dependencies = [ "blake2b_simd", "byteorder", @@ -558,8 +513,7 @@ dependencies = [ [[package]] name = "f4jumble" version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a83e8d7fd0c526af4aad893b7c9fe41e2699ed8a776a6c74aecdeafe05afc75" +source = "git+https://github.com/zcash/librustzcash.git?rev=352e1c709ac4c95074812dcabeba3cd5565371d2#352e1c709ac4c95074812dcabeba3cd5565371d2" dependencies = [ "blake2b_simd", ] @@ -732,15 +686,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "hashbrown" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" -dependencies = [ - "ahash", -] - [[package]] name = "hashbrown" version = "0.12.3" @@ -748,12 +693,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] -name = "hashlink" -version = "0.7.0" +name = "hashbrown" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7249a3129cbc1ffccd74857f81464a323a152173cdb134e0fd81bc803b29facf" +checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" dependencies = [ - "hashbrown 0.11.2", + "ahash", + "allocator-api2", +] + +[[package]] +name = "hashlink" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "312f66718a2d7789ffef4f4b7b213138ed9f1eb3aa1d0d82fc99f88fb3ffd26f" +dependencies = [ + "hashbrown 0.14.0", ] [[package]] @@ -765,7 +720,20 @@ dependencies = [ "lazy_static", "rand_core", "ring", - "secp256k1", + "secp256k1 0.21.3", +] + +[[package]] +name = "hdwallet" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a03ba7d4c9ea41552cd4351965ff96883e629693ae85005c501bb4b9e1c48a7" +dependencies = [ + "lazy_static", + "rand_core", + "ring", + "secp256k1 0.26.0", + "thiserror", ] [[package]] @@ -775,7 +743,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "969c513e03167e65d4bb59f5c51ec3820210975044ad7f218ab801fc169760fa" dependencies = [ "base58", - "hdwallet", + "hdwallet 0.3.1", "hex", "ripemd160", ] @@ -817,12 +785,20 @@ dependencies = [ ] [[package]] -name = "incrementalmerkletree" -version = "0.3.1" +name = "home" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5ad43a3f5795945459d577f6589cf62a476e92c79b75e70cd954364e14ce17b" +checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" dependencies = [ - "serde", + "windows-sys 0.48.0", +] + +[[package]] +name = "incrementalmerkletree" +version = "0.4.0" +source = "git+https://github.com/zcash/incrementalmerkletree.git?rev=67111e29403c33f2e36d6924167f1d5f04ad0fc2#67111e29403c33f2e36d6924167f1d5f04ad0fc2" +dependencies = [ + "either", ] [[package]] @@ -922,6 +898,15 @@ dependencies = [ "subtle", ] +[[package]] +name = "known-folders" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b6f1427d9c43b1cce87434c4d9eca33f43bdbb6246a762aa823a582f74c1684" +dependencies = [ + "windows-sys 0.48.0", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -945,9 +930,9 @@ checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb" [[package]] name = "libsqlite3-sys" -version = "0.22.2" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290b64917f8b0cb885d9de0f9959fe1f775d7fa12f1da2db9001c1c8ab60f89d" +checksum = "afc22eff61b133b115c6e8c74e818c628d6d5e7a502afea6f64dee076dd94326" dependencies = [ "cc", "pkg-config", @@ -1135,17 +1120,10 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" -[[package]] -name = "option-ext" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" - [[package]] name = "orchard" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c6f418f2c25573923f81a091f38b4b19bc20f6c92b5070fb8f0711e64a2b998" +version = "0.5.0" +source = "git+https://github.com/zcash/orchard.git?rev=6ef89d5f154de2cf7b7dd87edb8d8c49158beebb#6ef89d5f154de2cf7b7dd87edb8d8c49158beebb" dependencies = [ "aes", "bitvec", @@ -1289,12 +1267,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "proc-macro-hack" -version = "0.5.20+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" - [[package]] name = "proc-macro2" version = "1.0.56" @@ -1456,33 +1428,13 @@ dependencies = [ "zeroize", ] -[[package]] -name = "redox_syscall" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" -dependencies = [ - "bitflags", -] - [[package]] name = "redox_syscall" version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" dependencies = [ - "bitflags", -] - -[[package]] -name = "redox_users" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" -dependencies = [ - "getrandom", - "redox_syscall 0.2.16", - "thiserror", + "bitflags 1.3.2", ] [[package]] @@ -1537,17 +1489,15 @@ dependencies = [ [[package]] name = "rusqlite" -version = "0.25.4" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c4b1eaf239b47034fb450ee9cdedd7d0226571689d8823030c4b6c2cb407152" +checksum = "549b9d036d571d42e6e85d1c1425e2ac83491075078ca9a15be021c56b1641f2" dependencies = [ - "bitflags", + "bitflags 2.3.3", "fallible-iterator", "fallible-streaming-iterator", "hashlink", - "lazy_static", "libsqlite3-sys", - "memchr", "smallvec", "time", ] @@ -1558,22 +1508,13 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" -[[package]] -name = "rustc_version" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" -dependencies = [ - "semver", -] - [[package]] name = "rustix" version = "0.37.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8bbfc1d1c7c40c01715f47d71444744a81669ca84e8b63e25a55e169b1f86433" dependencies = [ - "bitflags", + "bitflags 1.3.2", "errno", "io-lifetimes", "libc", @@ -1581,12 +1522,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "ryu" -version = "1.0.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" - [[package]] name = "same-file" version = "1.0.6" @@ -1610,9 +1545,9 @@ dependencies = [ [[package]] name = "schemer-rusqlite" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1f34ab2dd976536b8518fe5b868756cf6e6caec89f38d8b6f432ba27f1d5f27" +checksum = "0fb5ac1fa52c58e2c6a618e3149d464e7ad8d0effca74990ea29c1fe2338b3b1" dependencies = [ "rusqlite", "schemer", @@ -1631,7 +1566,16 @@ version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c42e6f1735c5f00f51e43e28d6634141f2bcad10931b2609ddd74a86d751260" dependencies = [ - "secp256k1-sys", + "secp256k1-sys 0.4.2", +] + +[[package]] +name = "secp256k1" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4124a35fe33ae14259c490fd70fa199a32b9ce9502f2ee6bc4f81ec06fa65894" +dependencies = [ + "secp256k1-sys 0.8.1", ] [[package]] @@ -1643,6 +1587,15 @@ dependencies = [ "cc", ] +[[package]] +name = "secp256k1-sys" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70a129b9e9efbfb223753b9163c4ab3b13cff7fd9c7f010fbac25ab4099fa07e" +dependencies = [ + "cc", +] + [[package]] name = "secrecy" version = "0.8.0" @@ -1652,21 +1605,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "semver" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" -dependencies = [ - "semver-parser", -] - -[[package]] -name = "semver-parser" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" - [[package]] name = "serde" version = "1.0.160" @@ -1687,45 +1625,6 @@ dependencies = [ "syn 2.0.15", ] -[[package]] -name = "serde_json" -version = "1.0.96" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" -dependencies = [ - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "sha1" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1da05c97445caa12d05e848c4a4fcbbea29e748ac28f7e80e9b010392063770" -dependencies = [ - "sha1_smol", -] - -[[package]] -name = "sha1_smol" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" - -[[package]] -name = "sha2" -version = "0.9.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" -dependencies = [ - "block-buffer 0.9.0", - "cfg-if", - "cpufeatures", - "digest 0.9.0", - "opaque-debug 0.3.0", -] - [[package]] name = "sha2" version = "0.10.6" @@ -1746,6 +1645,17 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shardtree" +version = "0.0.0" +source = "git+https://github.com/zcash/incrementalmerkletree.git?rev=67111e29403c33f2e36d6924167f1d5f04ad0fc2#67111e29403c33f2e36d6924167f1d5f04ad0fc2" +dependencies = [ + "bitflags 1.3.2", + "either", + "incrementalmerkletree", + "tracing", +] + [[package]] name = "smallvec" version = "1.10.0" @@ -1758,70 +1668,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" -[[package]] -name = "standback" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e113fb6f3de07a243d434a56ec6f186dfd51cb08448239fe7bcae73f87ff28ff" -dependencies = [ - "version_check", -] - [[package]] name = "static_assertions" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" -[[package]] -name = "stdweb" -version = "0.4.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5" -dependencies = [ - "discard", - "rustc_version", - "stdweb-derive", - "stdweb-internal-macros", - "stdweb-internal-runtime", - "wasm-bindgen", -] - -[[package]] -name = "stdweb-derive" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" -dependencies = [ - "proc-macro2", - "quote", - "serde", - "serde_derive", - "syn 1.0.109", -] - -[[package]] -name = "stdweb-internal-macros" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11" -dependencies = [ - "base-x", - "proc-macro2", - "quote", - "serde", - "serde_derive", - "serde_json", - "sha1", - "syn 1.0.109", -] - -[[package]] -name = "stdweb-internal-runtime" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0" - [[package]] name = "subtle" version = "2.4.1" @@ -1876,7 +1728,7 @@ checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998" dependencies = [ "cfg-if", "fastrand", - "redox_syscall 0.3.5", + "redox_syscall", "rustix", "windows-sys 0.45.0", ] @@ -1913,40 +1765,29 @@ dependencies = [ [[package]] name = "time" -version = "0.2.27" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4752a97f8eebd6854ff91f1c1824cd6160626ac4bd44287f7f4ea2035a02a242" +checksum = "59e399c068f43a5d116fedaf73b203fa4f9c519f17e2b34f63221d3792f81446" dependencies = [ - "const_fn", - "libc", - "standback", - "stdweb", + "itoa", + "serde", + "time-core", "time-macros", - "version_check", - "winapi", ] +[[package]] +name = "time-core" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" + [[package]] name = "time-macros" -version = "0.1.1" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "957e9c6e26f12cb6d0dd7fc776bb67a706312e7299aed74c8dd5b17ebb27e2f1" +checksum = "96ba15a897f3c86766b757e5ac7221554c6750054d74d5b28844fce5fb36a6c4" dependencies = [ - "proc-macro-hack", - "time-macros-impl", -] - -[[package]] -name = "time-macros-impl" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd3c141a1b43194f3f56a1411225df8646c55781d5f26db825b3d98507eb482f" -dependencies = [ - "proc-macro-hack", - "proc-macro2", - "quote", - "standback", - "syn 1.0.109", + "time-core", ] [[package]] @@ -2377,13 +2218,22 @@ dependencies = [ "tap", ] +[[package]] +name = "xdg" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "688597db5a750e9cad4511cb94729a078e274308099a0382b5b8203bbc767fee" +dependencies = [ + "home", +] + [[package]] name = "zcash-android-wallet-sdk" version = "0.0.4" dependencies = [ "dlopen2", "failure", - "hdwallet", + "hdwallet 0.3.1", "hdwallet-bitcoin", "hex", "jni", @@ -2392,8 +2242,9 @@ dependencies = [ "orchard", "paranoid-android", "rayon", + "rusqlite", "schemer", - "secp256k1", + "secp256k1 0.21.3", "secrecy", "tracing", "tracing-subscriber", @@ -2406,9 +2257,8 @@ dependencies = [ [[package]] name = "zcash_address" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52be35a205369d480378646bff9c9fedafd8efe8af1e0e54bb858f405883f2b2" +version = "0.3.0" +source = "git+https://github.com/zcash/librustzcash.git?rev=352e1c709ac4c95074812dcabeba3cd5565371d2#352e1c709ac4c95074812dcabeba3cd5565371d2" dependencies = [ "bech32", "bs58", @@ -2419,8 +2269,7 @@ dependencies = [ [[package]] name = "zcash_client_backend" version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a55db8d2cb4ca82a71fa66ccd9fa5b211f5ab90c866721311ddd85f8f90d0701" +source = "git+https://github.com/zcash/librustzcash.git?rev=352e1c709ac4c95074812dcabeba3cd5565371d2#352e1c709ac4c95074812dcabeba3cd5565371d2" dependencies = [ "base64", "bech32", @@ -2429,7 +2278,8 @@ dependencies = [ "byteorder", "crossbeam-channel", "group", - "hdwallet", + "hdwallet 0.4.1", + "incrementalmerkletree", "memuse", "nom", "orchard", @@ -2437,6 +2287,7 @@ dependencies = [ "prost", "rayon", "secrecy", + "shardtree", "subtle", "time", "tonic-build", @@ -2451,30 +2302,33 @@ dependencies = [ [[package]] name = "zcash_client_sqlite" version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6be255cd5f217fa8bbe599cf07d56ded68b1f43180dbeda1822bee851f5f1971" +source = "git+https://github.com/zcash/librustzcash.git?rev=352e1c709ac4c95074812dcabeba3cd5565371d2#352e1c709ac4c95074812dcabeba3cd5565371d2" dependencies = [ "bs58", + "byteorder", + "either", "group", - "hdwallet", + "hdwallet 0.4.1", + "incrementalmerkletree", "jubjub", "prost", "rusqlite", "schemer", "schemer-rusqlite", "secrecy", + "shardtree", "time", "tracing", "uuid", "zcash_client_backend", + "zcash_encoding", "zcash_primitives", ] [[package]] name = "zcash_encoding" version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f03391b81727875efa6ac0661a20883022b6fba92365dc121c48fa9b00c5aac0" +source = "git+https://github.com/zcash/librustzcash.git?rev=352e1c709ac4c95074812dcabeba3cd5565371d2#352e1c709ac4c95074812dcabeba3cd5565371d2" dependencies = [ "byteorder", "nonempty", @@ -2482,9 +2336,9 @@ dependencies = [ [[package]] name = "zcash_note_encryption" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eb2149e6cd5fbee36c5b87c601715a8c35554602f7fe84af38b636afa2db318" +checksum = "5b4580cd6cee12e44421dac43169be8d23791650816bdb34e6ddfa70ac89c1c5" dependencies = [ "chacha20", "chacha20poly1305", @@ -2495,9 +2349,8 @@ dependencies = [ [[package]] name = "zcash_primitives" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "914d2195a478d5b63191584dff126f552751115181857b290211ec88e68acc3e" +version = "0.12.0" +source = "git+https://github.com/zcash/librustzcash.git?rev=352e1c709ac4c95074812dcabeba3cd5565371d2#352e1c709ac4c95074812dcabeba3cd5565371d2" dependencies = [ "aes", "bip0039", @@ -2510,7 +2363,7 @@ dependencies = [ "ff", "fpe", "group", - "hdwallet", + "hdwallet 0.4.1", "hex", "incrementalmerkletree", "jubjub", @@ -2521,8 +2374,8 @@ dependencies = [ "rand", "rand_core", "ripemd", - "secp256k1", - "sha2 0.10.6", + "secp256k1 0.26.0", + "sha2", "subtle", "zcash_address", "zcash_encoding", @@ -2531,20 +2384,22 @@ dependencies = [ [[package]] name = "zcash_proofs" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5c8147884952748b00aa443d36511ae2d7b49acfec74cfd39c0959fbb61ef14" +version = "0.12.1" +source = "git+https://github.com/zcash/librustzcash.git?rev=352e1c709ac4c95074812dcabeba3cd5565371d2#352e1c709ac4c95074812dcabeba3cd5565371d2" dependencies = [ "bellman", "blake2b_simd", "bls12_381", - "directories", "group", + "home", + "incrementalmerkletree", "jubjub", + "known-folders", "lazy_static", "rand_core", "redjubjub", "tracing", + "xdg", "zcash_primitives", ] diff --git a/backend-lib/Cargo.toml b/backend-lib/Cargo.toml index 0d72c256..b24df510 100644 --- a/backend-lib/Cargo.toml +++ b/backend-lib/Cargo.toml @@ -16,15 +16,16 @@ hdwallet = "0.3.1" hdwallet-bitcoin = "0.3" hex = "0.4" jni = { version = "0.20", default-features = false } +rusqlite = "0.29" schemer = "0.2" secp256k1 = "0.21" secrecy = "0.8" -zcash_address = "0.2" +zcash_address = "0.3" zcash_client_backend = { version = "0.9", features = ["transparent-inputs", "unstable"] } zcash_client_sqlite = { version = "0.7.1", features = ["transparent-inputs", "unstable"] } -zcash_primitives = "0.11" -zcash_proofs = "0.11" -orchard = { version = "0.4", default-features = false } +zcash_primitives = "0.12" +zcash_proofs = "0.12" +orchard = { version = "0.5", default-features = false } # Initialization rayon = "1.7" @@ -39,6 +40,16 @@ tracing-subscriber = "0.3" dlopen2 = "0.4" libc = "0.2" +[patch.crates-io] +incrementalmerkletree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "67111e29403c33f2e36d6924167f1d5f04ad0fc2" } +shardtree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "67111e29403c33f2e36d6924167f1d5f04ad0fc2" } +orchard = { git = "https://github.com/zcash/orchard.git", rev = "6ef89d5f154de2cf7b7dd87edb8d8c49158beebb" } +zcash_address = { git = "https://github.com/zcash/librustzcash.git", rev = "352e1c709ac4c95074812dcabeba3cd5565371d2" } +zcash_client_backend = { git = "https://github.com/zcash/librustzcash.git", rev = "352e1c709ac4c95074812dcabeba3cd5565371d2" } +zcash_client_sqlite = { git = "https://github.com/zcash/librustzcash.git", rev = "352e1c709ac4c95074812dcabeba3cd5565371d2" } +zcash_primitives = { git = "https://github.com/zcash/librustzcash.git", rev = "352e1c709ac4c95074812dcabeba3cd5565371d2" } +zcash_proofs = { git = "https://github.com/zcash/librustzcash.git", rev = "352e1c709ac4c95074812dcabeba3cd5565371d2" } + ## Uncomment this to test librustzcash changes locally #[patch.crates-io] #zcash_address = { path = '../../clones/librustzcash/components/zcash_address' } diff --git a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/Backend.kt b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/Backend.kt index 295a76f6..35e75d5d 100644 --- a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/Backend.kt +++ b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/Backend.kt @@ -1,6 +1,7 @@ package cash.z.ecc.android.sdk.internal import cash.z.ecc.android.sdk.internal.model.JniBlockMeta +import cash.z.ecc.android.sdk.internal.model.JniScanRange import cash.z.ecc.android.sdk.internal.model.JniUnifiedSpendingKey import java.lang.RuntimeException import kotlin.jvm.Throws @@ -101,7 +102,13 @@ interface Backend { * @throws RuntimeException as a common indicator of the operation failure */ @Throws(RuntimeException::class) - suspend fun scanBlocks(limit: Long?) + suspend fun suggestScanRanges(): List + + /** + * @throws RuntimeException as a common indicator of the operation failure + */ + @Throws(RuntimeException::class) + suspend fun scanBlocks(fromHeight: Long, limit: Long) /** * @throws RuntimeException as a common indicator of the operation failure @@ -118,12 +125,6 @@ interface Backend { suspend fun rewindBlockMetadataToHeight(height: Long) - /** - * @param limit The limit provides an efficient way how to restrict the portion of blocks, which will be validated. - * @return Null if successful. If an error occurs, the height will be the blockheight where the error was detected. - */ - suspend fun validateCombinedChainOrErrorHeight(limit: Long?): Long? - suspend fun getVerifiedTransparentBalance(address: String): Long suspend fun getTotalTransparentBalance(address: String): Long diff --git a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/jni/RustBackend.kt b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/jni/RustBackend.kt index ed6dcbf5..3d257d79 100644 --- a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/jni/RustBackend.kt +++ b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/jni/RustBackend.kt @@ -207,22 +207,6 @@ class RustBackend private constructor( ) } - override suspend fun validateCombinedChainOrErrorHeight(limit: Long?) = - withContext(SdkDispatchers.DATABASE_IO) { - val validationResult = validateCombinedChain( - dbCachePath = fsBlockDbRoot.absolutePath, - dbDataPath = dataDbFile.absolutePath, - limit = limit ?: -1, - networkId = networkId - ) - - if (-1L == validationResult) { - null - } else { - validationResult - } - } - override suspend fun getVerifiedTransparentBalance(address: String): Long = withContext(SdkDispatchers.DATABASE_IO) { getVerifiedTransparentBalance( @@ -264,12 +248,22 @@ class RustBackend private constructor( ) } - override suspend fun scanBlocks(limit: Long?) { + override suspend fun suggestScanRanges(): List { + return withContext(SdkDispatchers.DATABASE_IO) { + suggestScanRanges( + dataDbFile.absolutePath, + networkId = networkId + ).asList() + } + } + + override suspend fun scanBlocks(fromHeight: Long, limit: Long) { return withContext(SdkDispatchers.DATABASE_IO) { scanBlocks( fsBlockDbRoot.absolutePath, dataDbFile.absolutePath, - limit ?: -1, + fromHeight, + limit, networkId = networkId ) } @@ -499,14 +493,6 @@ class RustBackend private constructor( height: Long ) - @JvmStatic - private external fun validateCombinedChain( - dbCachePath: String, - dbDataPath: String, - limit: Long, - networkId: Int - ): Long - @JvmStatic private external fun getNearestRewindHeight( dbDataPath: String, @@ -521,10 +507,17 @@ class RustBackend private constructor( networkId: Int ) + @JvmStatic + private external fun suggestScanRanges( + dbDataPath: String, + networkId: Int + ): Array + @JvmStatic private external fun scanBlocks( dbCachePath: String, dbDataPath: String, + fromHeight: Long, limit: Long, networkId: Int ) diff --git a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniScanRange.kt b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniScanRange.kt new file mode 100644 index 00000000..2be702f9 --- /dev/null +++ b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniScanRange.kt @@ -0,0 +1,41 @@ +package cash.z.ecc.android.sdk.internal.model + +import androidx.annotation.Keep +import co.electriccoin.lightwallet.client.model.CompactBlockUnsafe + +/** + * Serves as cross layer (Kotlin, Rust) communication class. + * + * @param startHeight the minimum height in the range (inclusive) - although it's type Long, it needs to be a UInt + * @param endHeight the maximum height in the range (exclusive) - although it's type Long, it needs to be a UInt + * @param priority the priority of the range for scanning + */ +@Keep +class JniScanRange( + val startHeight: Long, + val endHeight: Long, + val priority: Long +) { + init { + // We require some of the parameters below to be in the range of unsigned integer, because of the Rust layer + // implementation. + require(UINT_RANGE.contains(startHeight)) { + "Height $startHeight is outside of allowed range $UINT_RANGE" + } + require(UINT_RANGE.contains(endHeight)) { + "Height $endHeight is outside of allowed range $UINT_RANGE" + } + } + + companion object { + private val UINT_RANGE = 0.toLong()..UInt.MAX_VALUE.toLong() + + fun new(range: OpenEndRange, priority: Long): JniScanRange { + return JniScanRange( + startHeight = range.start, + endHeight = range.endExclusive, + priority = priority + ) + } + } +} diff --git a/backend-lib/src/main/rust/lib.rs b/backend-lib/src/main/rust/lib.rs index 283b3dc5..6f07b57e 100644 --- a/backend-lib/src/main/rust/lib.rs +++ b/backend-lib/src/main/rust/lib.rs @@ -1,5 +1,6 @@ use std::collections::HashMap; use std::convert::{TryFrom, TryInto}; +use std::num::NonZeroU32; use std::panic; use std::path::Path; use std::ptr; @@ -17,11 +18,12 @@ use tracing::{debug, error}; use tracing_subscriber::prelude::*; use tracing_subscriber::reload; use zcash_address::{ToAddress, ZcashAddress}; +use zcash_client_backend::data_api::scanning::{ScanPriority, ScanRange}; use zcash_client_backend::keys::{DecodingError, UnifiedSpendingKey}; use zcash_client_backend::{ address::{RecipientAddress, UnifiedAddress}, data_api::{ - chain::{self, scan_cached_blocks, validate_chain}, + chain::scan_cached_blocks, wallet::{ decrypt_and_store_transaction, input_selection::GreedyInputSelector, shield_transparent_funds, spend, @@ -68,7 +70,7 @@ mod zip317 { pub(super) use zcash_primitives::transaction::fees::zip317::*; } -const ANCHOR_OFFSET: u32 = 10; +const ANCHOR_OFFSET: NonZeroU32 = unsafe { NonZeroU32::new_unchecked(10) }; #[cfg(debug_assertions)] fn print_debug_state() { @@ -84,7 +86,7 @@ fn wallet_db( env: &JNIEnv<'_>, params: P, db_data: JString<'_>, -) -> Result, failure::Error> { +) -> Result, failure::Error> { WalletDb::for_path(utils::java_string_to_rust(&env, db_data), params) .map_err(|e| format_err!("Error opening wallet database connection: {}", e)) } @@ -256,11 +258,10 @@ pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_cr ) -> jobject { let res = panic::catch_unwind(|| { let network = parse_network(network_id as u32)?; - let db_data = wallet_db(&env, network, db_data)?; + let mut db_data = wallet_db(&env, network, db_data)?; let seed = SecretVec::new(env.convert_byte_array(seed).unwrap()); - let mut db_ops = db_data.get_update_ops()?; - let (account, usk) = db_ops + let (account, usk) = db_data .create_account(&seed) .map_err(|e| format_err!("Error while initializing accounts: {}", e))?; @@ -283,7 +284,7 @@ pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_in ) -> jboolean { let res = panic::catch_unwind(|| { let network = parse_network(network_id as u32)?; - let db_data = wallet_db(&env, network, db_data)?; + let mut db_data = wallet_db(&env, network, db_data)?; // TODO: avoid all this unwrapping and also surface errors, better let count = env.get_array_length(ufvks_arr).unwrap(); let ufvks = (0..count) @@ -307,7 +308,7 @@ pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_in .map(|(i, res)| res.map(|ufvk| (AccountId::from(i as u32), ufvk))) .collect::, _>>()?; - match init_accounts_table(&db_data, &ufvks) { + match init_accounts_table(&mut db_data, &ufvks) { Ok(()) => Ok(JNI_TRUE), Err(e) => Err(format_err!("Error while initializing accounts: {}", e)), } @@ -485,7 +486,7 @@ pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_in ) -> jboolean { let res = panic::catch_unwind(|| { let network = parse_network(network_id as u32)?; - let db_data = wallet_db(&env, network, db_data)?; + let mut db_data = wallet_db(&env, network, db_data)?; let hash = { let mut hash = hex::decode(utils::java_string_to_rust(&env, hash_string)).unwrap(); hash.reverse(); @@ -501,7 +502,7 @@ pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_in debug!("initializing blocks table with height {}", height); match init_blocks_table( - &db_data, + &mut db_data, (height as u32).try_into()?, hash, time, @@ -725,14 +726,14 @@ pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_ge // reported to users does not drop temporarily in a way that they don't expect. // `getVerifiedBalance` requires `ANCHOR_OFFSET` confirmations, which means it // always shows a spendable balance. - let min_confirmations = 0; + let min_confirmations = NonZeroU32::new(1).unwrap(); (&db_data) .get_target_and_anchor_heights(min_confirmations) .map_err(|e| format_err!("Error while fetching anchor height: {}", e)) .and_then(|opt_anchor| { opt_anchor - .map(|(_, a)| a) + .map(|(_, a)| a + 1) .ok_or(format_err!("Anchor height not available; scan required.")) }) .and_then(|anchor| { @@ -797,9 +798,7 @@ pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_ge let addr = utils::java_string_to_rust(&env, address); let taddr = TransparentAddress::decode(&network, &addr).unwrap(); - // We select all transparent funds including unmined coins, as that is the same - // set of UTXOs we shield from. - let min_confirmations = 0; + let min_confirmations = NonZeroU32::new(1).unwrap(); let amount = (&db_data) .get_target_and_anchor_heights(min_confirmations) @@ -873,8 +872,9 @@ pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_ge .get_memo(NoteId::ReceivedNoteId(id_note)) .map_err(|e| format_err!("An error occurred retrieving the memo, {}", e)) .and_then(|memo| match memo { - Memo::Empty => Ok("".to_string()), - Memo::Text(memo) => Ok(memo.into()), + Some(Memo::Empty) => Ok("".to_string()), + Some(Memo::Text(memo)) => Ok(memo.into()), + None => Err(format_err!("Memo not available")), _ => Err(format_err!("This memo does not contain UTF-8 text")), })?; @@ -900,8 +900,9 @@ pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_ge .get_memo(NoteId::SentNoteId(id_note)) .map_err(|e| format_err!("An error occurred retrieving the memo, {}", e)) .and_then(|memo| match memo { - Memo::Empty => Ok("".to_string()), - Memo::Text(memo) => Ok(memo.into()), + Some(Memo::Empty) => Ok("".to_string()), + Some(Memo::Text(memo)) => Ok(memo.into()), + None => Err(format_err!("Memo not available")), _ => Err(format_err!("This memo does not contain UTF-8 text")), })?; @@ -1051,44 +1052,6 @@ pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_re unwrap_exc_or(&env, res, ()) } -#[no_mangle] -pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_validateCombinedChain( - env: JNIEnv<'_>, - _: JClass<'_>, - db_cache: JString<'_>, - db_data: JString<'_>, - limit: jlong, - network_id: jint, -) -> jlong { - let res = panic::catch_unwind(|| { - let network = parse_network(network_id as u32)?; - let block_db = block_db(&env, db_cache)?; - let db_data = wallet_db(&env, network, db_data)?; - let validate_limit = u32::try_from(limit).ok(); - - let validate_from = (&db_data) - .get_max_height_hash() - .map_err(|e| format_err!("Error while validating chain: {}", e))?; - - let val_res = validate_chain(&block_db, validate_from, validate_limit); - - if let Err(e) = val_res { - match e { - chain::error::Error::Chain(e) => { - let upper_bound_u32 = u32::from(e.at_height()); - Ok(upper_bound_u32 as i64) - } - _ => Err(format_err!("Error while validating chain: {:?}", e)), - } - } else { - // All blocks are valid, so "highest invalid block height" is below genesis. - Ok(-1) - } - }); - - unwrap_exc_or(&env, res, 0) as jlong -} - #[no_mangle] pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_getNearestRewindHeight( env: JNIEnv<'_>, @@ -1135,8 +1098,7 @@ pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_re ) -> jboolean { let res = panic::catch_unwind(|| { let network = parse_network(network_id as u32)?; - let db_data = wallet_db(&env, network, db_data)?; - let mut db_data = db_data.get_update_ops()?; + let mut db_data = wallet_db(&env, network, db_data)?; let height = BlockHeight::try_from(height)?; db_data @@ -1148,23 +1110,78 @@ pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_re unwrap_exc_or(&env, res, JNI_FALSE) } +fn encode_scan_range<'a>( + env: &JNIEnv<'a>, + scan_range: ScanRange, +) -> jni::errors::Result> { + let priority = match scan_range.priority() { + ScanPriority::Scanned => 10, + ScanPriority::Historic => 20, + ScanPriority::OpenAdjacent => 30, + ScanPriority::FoundNote => 40, + ScanPriority::ChainTip => 50, + ScanPriority::Verify => 60, + }; + env.new_object( + "cash/z/ecc/android/sdk/internal/model/JniScanRange", + "(JJJ)V", + &[ + JValue::Long(i64::from(u32::from(scan_range.block_range().start))), + JValue::Long(i64::from(u32::from(scan_range.block_range().end))), + JValue::Long(priority), + ], + ) +} + +#[no_mangle] +pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_suggestScanRanges( + env: JNIEnv<'_>, + _: JClass<'_>, + db_data: JString<'_>, + network_id: jint, +) -> jobjectArray { + let res = panic::catch_unwind(|| { + let network = parse_network(network_id as u32)?; + let db_data = wallet_db(&env, network, db_data)?; + + let ranges = db_data + .suggest_scan_ranges() + .map_err(|e| format_err!("Error while fetching suggested scan ranges: {}", e))?; + + Ok(utils::rust_vec_to_java( + &env, + ranges, + "cash/z/ecc/android/sdk/internal/model/JniBlockMeta", + |env, scan_range| encode_scan_range(env, scan_range), + |env| { + encode_scan_range( + env, + ScanRange::from_parts((0.into())..(0.into()), ScanPriority::Scanned), + ) + }, + )) + }); + unwrap_exc_or(&env, res, ptr::null_mut()) +} + #[no_mangle] pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_scanBlocks( env: JNIEnv<'_>, _: JClass<'_>, db_cache: JString<'_>, db_data: JString<'_>, + from_height: jlong, limit: jlong, network_id: jint, ) -> jboolean { let res = panic::catch_unwind(|| { let network = parse_network(network_id as u32)?; let db_cache = block_db(&env, db_cache)?; - let db_data = wallet_db(&env, network, db_data)?; - let mut db_data = db_data.get_update_ops()?; - let limit = u32::try_from(limit).ok(); + let mut db_data = wallet_db(&env, network, db_data)?; + let from_height = BlockHeight::try_from(from_height)?; + let limit = usize::try_from(limit)?; - match scan_cached_blocks(&network, &db_cache, &mut db_data, limit) { + match scan_cached_blocks(&network, &db_cache, &mut db_data, from_height, limit) { Ok(()) => Ok(JNI_TRUE), Err(e) => Err(format_err!( "Rust error while scanning blocks (limit {:?}): {:?}", @@ -1199,8 +1216,7 @@ pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_pu txid.copy_from_slice(&txid_bytes); let script_pubkey = Script(env.convert_byte_array(script).unwrap()); - let db_data = wallet_db(&env, network, db_data)?; - let mut db_data = db_data.get_update_ops()?; + let mut db_data = wallet_db(&env, network, db_data)?; let addr = utils::java_string_to_rust(&env, address); let _address = TransparentAddress::decode(&network, &addr).unwrap(); @@ -1233,8 +1249,7 @@ pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_de ) -> jboolean { let res = panic::catch_unwind(|| { let network = parse_network(network_id as u32)?; - let db_data = wallet_db(&env, network, db_data)?; - let mut db_data = db_data.get_update_ops()?; + let mut db_data = wallet_db(&env, network, db_data)?; let tx_bytes = env.convert_byte_array(tx).unwrap(); // The consensus branch ID passed in here does not matter: // - v4 and below cache it internally, but all we do with this transaction while @@ -1292,8 +1307,7 @@ pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_cr ) -> jlong { let res = panic::catch_unwind(|| { let network = parse_network(network_id as u32)?; - let db_data = wallet_db(&env, network, db_data)?; - let mut db_data = db_data.get_update_ops()?; + let mut db_data = wallet_db(&env, network, db_data)?; let usk = decode_usk(&env, usk)?; let to = utils::java_string_to_rust(&env, to); let value = @@ -1382,14 +1396,13 @@ pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_sh ) -> jlong { let res = panic::catch_unwind(|| { let network = parse_network(network_id as u32)?; - let db_data = wallet_db(&env, network, db_data)?; - let mut db_data = db_data.get_update_ops()?; + let mut db_data = wallet_db(&env, network, db_data)?; let usk = decode_usk(&env, usk)?; let memo_bytes = env.convert_byte_array(memo).unwrap(); let spend_params = utils::java_string_to_rust(&env, spend_params); let output_params = utils::java_string_to_rust(&env, output_params); - let min_confirmations = 0; + let min_confirmations = NonZeroU32::new(1).unwrap(); let account = db_data .get_account_for_ufvk(&usk.to_unified_full_viewing_key())? diff --git a/backend-lib/src/main/rust/utils.rs b/backend-lib/src/main/rust/utils.rs index 70068f25..e3dc9b08 100644 --- a/backend-lib/src/main/rust/utils.rs +++ b/backend-lib/src/main/rust/utils.rs @@ -6,8 +6,6 @@ use jni::{ JNIEnv, }; -use std::ops::Deref; - pub(crate) mod exception; pub(crate) mod target_ndk; pub(crate) mod trace; @@ -27,17 +25,17 @@ pub(crate) fn rust_vec_to_java<'a, T, U, V, F, G>( ) -> jobjectArray where U: Desc<'a, JClass<'a>>, - V: Deref>, + V: Into>, F: Fn(&JNIEnv<'a>, T) -> JNIResult, G: Fn(&JNIEnv<'a>) -> JNIResult, { let jempty = empty_element(env).expect("Couldn't create Java string!"); let jret = env - .new_object_array(data.len() as jsize, element_class, *jempty) + .new_object_array(data.len() as jsize, element_class, jempty.into()) .expect("Couldn't create Java array!"); for (i, elem) in data.into_iter().enumerate() { let jelem = element_map(env, elem).expect("Couldn't map element to Java!"); - env.set_object_array_element(jret, i as jsize, *jelem) + env.set_object_array_element(jret, i as jsize, jelem.into()) .expect("Couldn't set Java array element!"); } jret diff --git a/sdk-lib/src/androidTest/java/cash/z/ecc/android/sdk/util/BalancePrinterUtil.kt b/sdk-lib/src/androidTest/java/cash/z/ecc/android/sdk/util/BalancePrinterUtil.kt index 53a7a611..8de49b6b 100644 --- a/sdk-lib/src/androidTest/java/cash/z/ecc/android/sdk/util/BalancePrinterUtil.kt +++ b/sdk-lib/src/androidTest/java/cash/z/ecc/android/sdk/util/BalancePrinterUtil.kt @@ -59,9 +59,6 @@ class BalancePrinterUtil { // val lastDownloaded = downloader.getLastDownloadedHeight() // val blockRange = (Math.max(birthday, lastDownloaded))..latestBlockHeight // downloadNewBlocks(blockRange) -// val error = validateNewBlocks(blockRange) -// twig("validation completed with result $error") -// assertEquals(-1, error) } private suspend fun deleteDb(dbName: String) { @@ -150,13 +147,4 @@ class BalancePrinterUtil { // } // } // } - -// private fun validateNewBlocks(range: IntRange?): Int { -// // val dummyWallet = initWallet("dummySeed") -// Twig.sprout("validating") -// twig("validating blocks in range $range") -// // val result = rustBackend.validateCombinedChain() -// Twig.clip("validating") -// return result -// } } diff --git a/sdk-lib/src/androidTest/java/cash/z/ecc/fixture/FakeRustBackend.kt b/sdk-lib/src/androidTest/java/cash/z/ecc/fixture/FakeRustBackend.kt index 685643dc..231561c6 100644 --- a/sdk-lib/src/androidTest/java/cash/z/ecc/fixture/FakeRustBackend.kt +++ b/sdk-lib/src/androidTest/java/cash/z/ecc/fixture/FakeRustBackend.kt @@ -18,9 +18,6 @@ internal class FakeRustBackend( } override suspend fun getLatestHeight(): Long = metadata.maxOf { it.height } - override suspend fun validateCombinedChainOrErrorHeight(limit: Long?): Long? { - TODO("Not yet implemented") - } override suspend fun getVerifiedTransparentBalance(address: String): Long { TODO("Not yet implemented") @@ -133,6 +130,6 @@ internal class FakeRustBackend( TODO("Not yet implemented") } - override suspend fun scanBlocks(limit: Long?) = + override suspend fun scanBlocks(fromHeight: Long, limit: Long) = error("Intentionally not implemented in mocked FakeRustBackend implementation.") } diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt index 39323eb1..36a6a8a1 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt @@ -742,27 +742,12 @@ class CompactBlockProcessor internal constructor( // Enrich batch model with fetched blocks. It's useful for later blocks deletion downloadStageResult.batch.blocks = downloadStageResult.stageResult.downloadedBlocks - // Run validation stage + // Run scanning stage (which also validates the fetched blocks) SyncStageResult( downloadStageResult.batch, - validateBatchOfBlocks( - backend = backend, - batch = downloadStageResult.batch - ) - ) - } - }.map { validateResult -> - Twig.debug { "Validation stage done with result: $validateResult" } - - if (validateResult.stageResult != BlockProcessingResult.Success) { - validateResult - } else { - // Run scanning stage - SyncStageResult( - validateResult.batch, scanBatchOfBlocks( backend = backend, - batch = validateResult.batch + batch = downloadStageResult.batch ) ) } @@ -904,24 +889,10 @@ class CompactBlockProcessor internal constructor( } } - @VisibleForTesting - internal suspend fun validateBatchOfBlocks(batch: BlockBatch, backend: TypesafeBackend): BlockProcessingResult { - Twig.verbose { "Starting to validate batch $batch" } - - val result = backend.validateCombinedChainOrErrorBlockHeight(batch.range.length()) - - return if (null == result) { - Twig.verbose { "Successfully validated batch $batch" } - BlockProcessingResult.Success - } else { - BlockProcessingResult.FailedValidateBlocks(result) - } - } - @VisibleForTesting internal suspend fun scanBatchOfBlocks(batch: BlockBatch, backend: TypesafeBackend): BlockProcessingResult { return runCatching { - backend.scanBlocks(batch.range.length()) + backend.scanBlocks(batch.range.start, batch.range.length()) }.onSuccess { Twig.verbose { "Successfully scanned batch $batch" } }.onFailure { diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackend.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackend.kt index 97274eda..492aa3ff 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackend.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackend.kt @@ -2,6 +2,7 @@ package cash.z.ecc.android.sdk.internal import cash.z.ecc.android.sdk.internal.model.Checkpoint import cash.z.ecc.android.sdk.internal.model.JniBlockMeta +import cash.z.ecc.android.sdk.internal.model.JniScanRange import cash.z.ecc.android.sdk.model.Account import cash.z.ecc.android.sdk.model.BlockHeight import cash.z.ecc.android.sdk.model.UnifiedFullViewingKey @@ -9,8 +10,6 @@ import cash.z.ecc.android.sdk.model.UnifiedSpendingKey import cash.z.ecc.android.sdk.model.WalletBalance import cash.z.ecc.android.sdk.model.Zatoshi import cash.z.ecc.android.sdk.model.ZcashNetwork -import java.lang.RuntimeException -import kotlin.jvm.Throws @Suppress("TooManyFunctions") internal interface TypesafeBackend { @@ -61,12 +60,6 @@ internal interface TypesafeBackend { suspend fun rewindBlockMetadataToHeight(height: BlockHeight) - /** - * @param limit The limit provides an efficient way how to restrict the portion of blocks, which will be validated. - * @return Null if successful. If an error occurs, the height will be the height where the error was detected. - */ - suspend fun validateCombinedChainOrErrorBlockHeight(limit: Long?): BlockHeight? - suspend fun getDownloadedUtxoBalance(address: String): WalletBalance @Suppress("LongParameterList") @@ -89,7 +82,13 @@ internal interface TypesafeBackend { * @throws RuntimeException as a common indicator of the operation failure */ @Throws(RuntimeException::class) - suspend fun scanBlocks(limit: Long?) + suspend fun scanBlocks(fromHeight: BlockHeight, limit: Long) + + /** + * @throws RuntimeException as a common indicator of the operation failure + */ + @Throws(RuntimeException::class) + suspend fun suggestScanRanges(): List suspend fun decryptAndStoreTransaction(tx: ByteArray) diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackendImpl.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackendImpl.kt index c4c572c1..e015df1d 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackendImpl.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackendImpl.kt @@ -2,6 +2,7 @@ package cash.z.ecc.android.sdk.internal import cash.z.ecc.android.sdk.internal.model.Checkpoint import cash.z.ecc.android.sdk.internal.model.JniBlockMeta +import cash.z.ecc.android.sdk.internal.model.JniScanRange import cash.z.ecc.android.sdk.model.Account import cash.z.ecc.android.sdk.model.BlockHeight import cash.z.ecc.android.sdk.model.UnifiedFullViewingKey @@ -115,19 +116,6 @@ internal class TypesafeBackendImpl(private val backend: Backend) : TypesafeBacke backend.rewindBlockMetadataToHeight(height.value) } - /** - * @param limit The limit provides an efficient way how to restrict the portion of blocks, which will be validated. - * @return Null if successful. If an error occurs, the height will be the height where the error was detected. - */ - override suspend fun validateCombinedChainOrErrorBlockHeight(limit: Long?): BlockHeight? { - return backend.validateCombinedChainOrErrorHeight(limit)?.let { - BlockHeight.new( - ZcashNetwork.from(backend.networkId), - it - ) - } - } - override suspend fun getDownloadedUtxoBalance(address: String): WalletBalance { // Note this implementation is not ideal because it requires two database queries without a transaction, which // makes the data potentially inconsistent. However the verified amount is queried first which makes this less @@ -168,7 +156,9 @@ internal class TypesafeBackendImpl(private val backend: Backend) : TypesafeBacke override suspend fun initDataDb(seed: ByteArray?): Int = backend.initDataDb(seed) - override suspend fun scanBlocks(limit: Long?) = backend.scanBlocks(limit) + override suspend fun scanBlocks(fromHeight: BlockHeight, limit: Long) = backend.scanBlocks(fromHeight.value, limit) + + override suspend fun suggestScanRanges(): List = backend.suggestScanRanges() override suspend fun decryptAndStoreTransaction(tx: ByteArray) = backend.decryptAndStoreTransaction(tx) diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanRange.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanRange.kt new file mode 100644 index 00000000..b7685078 --- /dev/null +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanRange.kt @@ -0,0 +1,19 @@ +package cash.z.ecc.android.sdk.internal.model + +import cash.z.ecc.android.sdk.model.BlockHeight + +internal data class ScanRange( + val range: OpenEndRange, + val priority: Long +) { + override fun toString() = "ScanRange(range=$range, priority=$priority)" + + companion object { + fun new(jni: JniScanRange): ScanRange { + return ScanRange( + range = jni.startHeight..jni.endHeight, + priority = priority + ) + } + } +} From 7cee1d7203694cb072deab32569adb2233777946 Mon Sep 17 00:00:00 2001 From: Honza Date: Thu, 13 Jul 2023 13:18:15 +0200 Subject: [PATCH 02/76] Make buildable + Solve Lint warnings --- .../cash/z/ecc/android/sdk/internal/jni/RustBackend.kt | 1 + .../cash/z/ecc/android/sdk/internal/model/JniScanRange.kt | 4 ++-- .../java/cash/z/ecc/fixture/FakeRustBackend.kt | 5 +++++ .../cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt | 2 +- .../cash/z/ecc/android/sdk/internal/model/ScanRange.kt | 8 +++++--- 5 files changed, 14 insertions(+), 6 deletions(-) diff --git a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/jni/RustBackend.kt b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/jni/RustBackend.kt index 3d257d79..96909383 100644 --- a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/jni/RustBackend.kt +++ b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/jni/RustBackend.kt @@ -5,6 +5,7 @@ import cash.z.ecc.android.sdk.internal.SdkDispatchers import cash.z.ecc.android.sdk.internal.ext.deleteRecursivelySuspend import cash.z.ecc.android.sdk.internal.ext.deleteSuspend import cash.z.ecc.android.sdk.internal.model.JniBlockMeta +import cash.z.ecc.android.sdk.internal.model.JniScanRange import cash.z.ecc.android.sdk.internal.model.JniUnifiedSpendingKey import kotlinx.coroutines.withContext import java.io.File diff --git a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniScanRange.kt b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniScanRange.kt index 2be702f9..dad660d4 100644 --- a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniScanRange.kt +++ b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniScanRange.kt @@ -1,7 +1,6 @@ package cash.z.ecc.android.sdk.internal.model import androidx.annotation.Keep -import co.electriccoin.lightwallet.client.model.CompactBlockUnsafe /** * Serves as cross layer (Kotlin, Rust) communication class. @@ -30,7 +29,8 @@ class JniScanRange( companion object { private val UINT_RANGE = 0.toLong()..UInt.MAX_VALUE.toLong() - fun new(range: OpenEndRange, priority: Long): JniScanRange { + @OptIn(ExperimentalStdlibApi::class) + fun new(range: OpenEndRange, priority: Long): JniScanRange { return JniScanRange( startHeight = range.start, endHeight = range.endExclusive, diff --git a/sdk-lib/src/androidTest/java/cash/z/ecc/fixture/FakeRustBackend.kt b/sdk-lib/src/androidTest/java/cash/z/ecc/fixture/FakeRustBackend.kt index 231561c6..8c9fd2ae 100644 --- a/sdk-lib/src/androidTest/java/cash/z/ecc/fixture/FakeRustBackend.kt +++ b/sdk-lib/src/androidTest/java/cash/z/ecc/fixture/FakeRustBackend.kt @@ -2,6 +2,7 @@ package cash.z.ecc.fixture import cash.z.ecc.android.sdk.internal.Backend import cash.z.ecc.android.sdk.internal.model.JniBlockMeta +import cash.z.ecc.android.sdk.internal.model.JniScanRange import cash.z.ecc.android.sdk.internal.model.JniUnifiedSpendingKey internal class FakeRustBackend( @@ -17,6 +18,10 @@ internal class FakeRustBackend( metadata.removeAll { it.height > height } } + override suspend fun suggestScanRanges(): List { + TODO("Not yet implemented") + } + override suspend fun getLatestHeight(): Long = metadata.maxOf { it.height } override suspend fun getVerifiedTransparentBalance(address: String): Long { diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt index 36a6a8a1..ccf94aea 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt @@ -892,7 +892,7 @@ class CompactBlockProcessor internal constructor( @VisibleForTesting internal suspend fun scanBatchOfBlocks(batch: BlockBatch, backend: TypesafeBackend): BlockProcessingResult { return runCatching { - backend.scanBlocks(batch.range.start, batch.range.length()) + backend.scanBlocks(batch.range.start.value, batch.range.length()) }.onSuccess { Twig.verbose { "Successfully scanned batch $batch" } }.onFailure { diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanRange.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanRange.kt index b7685078..7d6afc15 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanRange.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanRange.kt @@ -1,7 +1,9 @@ package cash.z.ecc.android.sdk.internal.model import cash.z.ecc.android.sdk.model.BlockHeight +import cash.z.ecc.android.sdk.model.ZcashNetwork +@OptIn(ExperimentalStdlibApi::class) internal data class ScanRange( val range: OpenEndRange, val priority: Long @@ -9,10 +11,10 @@ internal data class ScanRange( override fun toString() = "ScanRange(range=$range, priority=$priority)" companion object { - fun new(jni: JniScanRange): ScanRange { + fun new(jni: JniScanRange, zcashNetwork: ZcashNetwork): ScanRange { return ScanRange( - range = jni.startHeight..jni.endHeight, - priority = priority + range = BlockHeight.new(zcashNetwork, jni.startHeight).. Date: Thu, 13 Jul 2023 14:39:02 +0200 Subject: [PATCH 03/76] Clear validation stage --- .../android/sdk/internal/jni/RustBackend.kt | 3 +- .../sdk/block/CompactBlockProcessor.kt | 35 +++++++------------ 2 files changed, 14 insertions(+), 24 deletions(-) diff --git a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/jni/RustBackend.kt b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/jni/RustBackend.kt index 96909383..25de2dee 100644 --- a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/jni/RustBackend.kt +++ b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/jni/RustBackend.kt @@ -434,8 +434,7 @@ class RustBackend private constructor( @JvmStatic private external fun listTransparentReceivers(dbDataPath: String, account: Int, networkId: Int): Array - fun validateUnifiedSpendingKey(bytes: ByteArray) = - isValidSpendingKey(bytes) + fun validateUnifiedSpendingKey(bytes: ByteArray) = isValidSpendingKey(bytes) @JvmStatic private external fun isValidSpendingKey(bytes: ByteArray): Boolean diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt index ccf94aea..6a4bc80d 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt @@ -243,11 +243,6 @@ class CompactBlockProcessor internal constructor( checkErrorResult(result.failedAtHeight) } - is BlockProcessingResult.FailedValidateBlocks -> { - Twig.error { "Failed while validating blocks at height: ${result.failedAtHeight}" } - checkErrorResult(result.failedAtHeight) - } - is BlockProcessingResult.FailedScanBlocks -> { Twig.error { "Failed while scanning blocks at height: ${result.failedAtHeight}" } checkErrorResult(result.failedAtHeight) @@ -366,7 +361,7 @@ class CompactBlockProcessor internal constructor( } if (syncResult != BlockProcessingResult.Success) { - // Remove persisted but not validated and scanned blocks in case of any failure + // Remove persisted but not scanned blocks in case of any failure val lastScannedHeight = getLastScannedHeight(repository) downloader.rewindToHeight(lastScannedHeight) deleteAllBlockFiles( @@ -388,7 +383,6 @@ class CompactBlockProcessor internal constructor( object Reconnecting : BlockProcessingResult() data class FailedDownloadBlocks(val failedAtHeight: BlockHeight) : BlockProcessingResult() data class FailedScanBlocks(val failedAtHeight: BlockHeight) : BlockProcessingResult() - data class FailedValidateBlocks(val failedAtHeight: BlockHeight) : BlockProcessingResult() data class FailedDeleteBlocks(val failedAtHeight: BlockHeight) : BlockProcessingResult() data class FailedEnhance(val error: CompactBlockProcessorException.EnhanceTransactionError) : BlockProcessingResult() @@ -413,8 +407,8 @@ class CompactBlockProcessor internal constructor( runCatching { networkBlockHeightUnsafe?.toBlockHeight(network) }.getOrNull() } ?: return false - // If we find out that we previously downloaded, but not validated and scanned persisted blocks, we need - // to rewind the blocks above the last scanned height first. + // If we find out that we previously downloaded, but not scanned persisted blocks, we need to rewind the + // blocks above the last scanned height first. val lastScannedHeight = getLastScannedHeight(repository) val lastDownloadedHeight = getLastDownloadedHeight(downloader).let { BlockHeight.new( @@ -429,7 +423,7 @@ class CompactBlockProcessor internal constructor( Twig.verbose { "Clearing blocks of last persisted batch within the last scanned height " + "$lastScannedHeight and last download height $lastDownloadedHeight, as all these blocks " + - "possibly haven't been validated and scanned in the previous blocks sync attempt." + "possibly haven't been scanned in the previous blocks sync attempt." } downloader.rewindToHeight(lastScannedHeight) lastScannedHeight @@ -653,9 +647,9 @@ class CompactBlockProcessor internal constructor( /** * Default size of batches of blocks to request from the compact block service. Then it's also used as a default - * size of batches of blocks to validate and scan via librustzcash. For scanning action applies this - the - * smaller this number the more granular information can be provided about scan state. Unfortunately, it may - * also lead to a lot of overhead during scanning. + * size of batches of blocks to scan via librustzcash. For scanning action applies this - the smaller this + * number the more granular information can be provided about scan state. Unfortunately, it may also lead to + * a lot of overhead during scanning. */ internal const val SYNC_BATCH_SIZE = 10 @@ -1259,9 +1253,8 @@ class CompactBlockProcessor internal constructor( if (null != lastSyncedHeight) { val range = (targetHeight + 1)..lastSyncedHeight Twig.debug { - "We kept the cache blocks in place so we don't need to wait for the next " + - "scheduled download to rescan. Instead we will rescan and validate blocks " + - "${range.start}..${range.endInclusive}" + "We kept the cache blocks in place so we don't need to wait for the next scheduled download " + + "to rescan. Instead we will rescan blocks ${range.start}..${range.endInclusive}" } syncBlocksAndEnhanceTransactions( @@ -1413,11 +1406,9 @@ class CompactBlockProcessor internal constructor( * block height available from the server is greater than what we have locally. We move out * of this state once our local height matches the server. * - * **Validating** is when the blocks that have been downloaded are actively being validated to - * ensure that there are no gaps and that every block is chain-sequential to the previous - * block, which determines whether a reorg has happened on our watch. - * - * **Scanning** is when the blocks that have been downloaded are actively being decrypted. + * **Scanning** is when the blocks that have been downloaded are actively being decrypted and validated to + * ensure that there are no gaps and that every block is chain-sequential to the previous block, which + * determines whether a reorg has happened on our watch. * * **Deleting** is when the temporary block files being removed from the persistence. * @@ -1431,7 +1422,7 @@ class CompactBlockProcessor internal constructor( /** * [State] for when we are done with syncing the blocks, for now, i.e. all necessary stages done (download, - * validate, and scan). + * scan). */ class Synced(val syncedRange: ClosedRange?) : IConnected, ISyncing, State() From 57fe48e9fea8fc293f8838ac40e0563bf8edbc4b Mon Sep 17 00:00:00 2001 From: Honza Date: Mon, 17 Jul 2023 15:27:54 +0200 Subject: [PATCH 04/76] Switch to ScanRange model in sdk-lib module --- .../z/ecc/android/sdk/block/CompactBlockProcessor.kt | 2 +- .../cash/z/ecc/android/sdk/internal/TypesafeBackend.kt | 4 ++-- .../z/ecc/android/sdk/internal/TypesafeBackendImpl.kt | 9 +++++++-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt index 6a4bc80d..472b53ce 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt @@ -886,7 +886,7 @@ class CompactBlockProcessor internal constructor( @VisibleForTesting internal suspend fun scanBatchOfBlocks(batch: BlockBatch, backend: TypesafeBackend): BlockProcessingResult { return runCatching { - backend.scanBlocks(batch.range.start.value, batch.range.length()) + backend.scanBlocks(batch.range.start, batch.range.length()) }.onSuccess { Twig.verbose { "Successfully scanned batch $batch" } }.onFailure { diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackend.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackend.kt index 492aa3ff..96482493 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackend.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackend.kt @@ -2,7 +2,7 @@ package cash.z.ecc.android.sdk.internal import cash.z.ecc.android.sdk.internal.model.Checkpoint import cash.z.ecc.android.sdk.internal.model.JniBlockMeta -import cash.z.ecc.android.sdk.internal.model.JniScanRange +import cash.z.ecc.android.sdk.internal.model.ScanRange import cash.z.ecc.android.sdk.model.Account import cash.z.ecc.android.sdk.model.BlockHeight import cash.z.ecc.android.sdk.model.UnifiedFullViewingKey @@ -88,7 +88,7 @@ internal interface TypesafeBackend { * @throws RuntimeException as a common indicator of the operation failure */ @Throws(RuntimeException::class) - suspend fun suggestScanRanges(): List + suspend fun suggestScanRanges(): List suspend fun decryptAndStoreTransaction(tx: ByteArray) diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackendImpl.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackendImpl.kt index e015df1d..5794da45 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackendImpl.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackendImpl.kt @@ -2,7 +2,7 @@ package cash.z.ecc.android.sdk.internal import cash.z.ecc.android.sdk.internal.model.Checkpoint import cash.z.ecc.android.sdk.internal.model.JniBlockMeta -import cash.z.ecc.android.sdk.internal.model.JniScanRange +import cash.z.ecc.android.sdk.internal.model.ScanRange import cash.z.ecc.android.sdk.model.Account import cash.z.ecc.android.sdk.model.BlockHeight import cash.z.ecc.android.sdk.model.UnifiedFullViewingKey @@ -158,7 +158,12 @@ internal class TypesafeBackendImpl(private val backend: Backend) : TypesafeBacke override suspend fun scanBlocks(fromHeight: BlockHeight, limit: Long) = backend.scanBlocks(fromHeight.value, limit) - override suspend fun suggestScanRanges(): List = backend.suggestScanRanges() + override suspend fun suggestScanRanges(): List = backend.suggestScanRanges().map { jniScanRange -> + ScanRange.new( + jniScanRange, + network + ) + } override suspend fun decryptAndStoreTransaction(tx: ByteArray) = backend.decryptAndStoreTransaction(tx) From 3476ccace1d6e996eed87b7b04ba23af50efd1d9 Mon Sep 17 00:00:00 2001 From: Honza Date: Tue, 18 Jul 2023 14:25:40 +0200 Subject: [PATCH 05/76] Fix type returned from suggest_scan_ranges --- backend-lib/src/main/rust/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend-lib/src/main/rust/lib.rs b/backend-lib/src/main/rust/lib.rs index 6f07b57e..24620de5 100644 --- a/backend-lib/src/main/rust/lib.rs +++ b/backend-lib/src/main/rust/lib.rs @@ -1151,7 +1151,7 @@ pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_su Ok(utils::rust_vec_to_java( &env, ranges, - "cash/z/ecc/android/sdk/internal/model/JniBlockMeta", + "cash/z/ecc/android/sdk/internal/model/JniScanRange", |env, scan_range| encode_scan_range(env, scan_range), |env| { encode_scan_range( From 3c5d91b112e73acc0ca3e03208237c861197d2b2 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Tue, 18 Jul 2023 21:39:26 +0000 Subject: [PATCH 06/76] Expose `putSaplingSubtreeRoots` and `updateChainTip` Rust APIs --- .../z/ecc/android/sdk/internal/Backend.kt | 16 ++++ .../android/sdk/internal/jni/RustBackend.kt | 37 ++++++++ .../sdk/internal/model/JniSubtreeRoot.kt | 35 ++++++++ backend-lib/src/main/rust/lib.rs | 89 ++++++++++++++++++- .../cash/z/ecc/fixture/FakeRustBackend.kt | 12 +++ 5 files changed, 187 insertions(+), 2 deletions(-) create mode 100644 backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniSubtreeRoot.kt diff --git a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/Backend.kt b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/Backend.kt index 35e75d5d..b6f6ae45 100644 --- a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/Backend.kt +++ b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/Backend.kt @@ -2,6 +2,7 @@ package cash.z.ecc.android.sdk.internal import cash.z.ecc.android.sdk.internal.model.JniBlockMeta import cash.z.ecc.android.sdk.internal.model.JniScanRange +import cash.z.ecc.android.sdk.internal.model.JniSubtreeRoot import cash.z.ecc.android.sdk.internal.model.JniUnifiedSpendingKey import java.lang.RuntimeException import kotlin.jvm.Throws @@ -98,6 +99,21 @@ interface Backend { @Throws(RuntimeException::class) suspend fun rewindToHeight(height: Long) + /** + * @throws RuntimeException as a common indicator of the operation failure + */ + @Throws(RuntimeException::class) + suspend fun putSaplingSubtreeRoots( + startIndex: Long, + roots: List, + ) + + /** + * @throws RuntimeException as a common indicator of the operation failure + */ + @Throws(RuntimeException::class) + suspend fun updateChainTip(height: Long) + /** * @throws RuntimeException as a common indicator of the operation failure */ diff --git a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/jni/RustBackend.kt b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/jni/RustBackend.kt index 25de2dee..48e27a15 100644 --- a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/jni/RustBackend.kt +++ b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/jni/RustBackend.kt @@ -6,6 +6,7 @@ import cash.z.ecc.android.sdk.internal.ext.deleteRecursivelySuspend import cash.z.ecc.android.sdk.internal.ext.deleteSuspend import cash.z.ecc.android.sdk.internal.model.JniBlockMeta import cash.z.ecc.android.sdk.internal.model.JniScanRange +import cash.z.ecc.android.sdk.internal.model.JniSubtreeRoot import cash.z.ecc.android.sdk.internal.model.JniUnifiedSpendingKey import kotlinx.coroutines.withContext import java.io.File @@ -249,6 +250,27 @@ class RustBackend private constructor( ) } + override suspend fun putSaplingSubtreeRoots( + startIndex: Long, + roots: List, + ) = withContext(SdkDispatchers.DATABASE_IO) { + putSaplingSubtreeRoots( + dataDbFile.absolutePath, + startIndex, + roots.toTypedArray(), + networkId = networkId + ) + } + + override suspend fun updateChainTip(height: Long) = + withContext(SdkDispatchers.DATABASE_IO) { + updateChainTip( + dataDbFile.absolutePath, + height, + networkId = networkId + ) + } + override suspend fun suggestScanRanges(): List { return withContext(SdkDispatchers.DATABASE_IO) { suggestScanRanges( @@ -507,6 +529,21 @@ class RustBackend private constructor( networkId: Int ) + @JvmStatic + private external fun putSaplingSubtreeRoots( + dbDataPath: String, + startIndex: Long, + roots: Array, + networkId: Int + ) + + @JvmStatic + private external fun updateChainTip( + dbDataPath: String, + height: Long, + networkId: Int + ) + @JvmStatic private external fun suggestScanRanges( dbDataPath: String, diff --git a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniSubtreeRoot.kt b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniSubtreeRoot.kt new file mode 100644 index 00000000..31172d3e --- /dev/null +++ b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniSubtreeRoot.kt @@ -0,0 +1,35 @@ +package cash.z.ecc.android.sdk.internal.model + +import androidx.annotation.Keep +import co.electriccoin.lightwallet.client.model.CompactBlockUnsafe + +/** + * Serves as cross layer (Kotlin, Rust) communication class. + * + * @param rootHash the subtree's root hash + * @param completingBlockHeight the block height in which the subtree was completed - although it's type Long, it needs to be in UInt range + */ +@Keep +class JniSubtreeRoot( + val rootHash: ByteArray, + val completingBlockHeight: Long +) { + init { + // We require some of the parameters below to be in the range of unsigned integer, + // because of the Rust layer implementation. + require(UINT_RANGE.contains(completingBlockHeight)) { + "Height $completingBlockHeight is outside of allowed range $UINT_RANGE" + } + } + + companion object { + private val UINT_RANGE = 0.toLong()..UInt.MAX_VALUE.toLong() + + fun new(subtreeRoot: SubtreeRootUnsafe): JniSubtreeRoot { + return JniSubtreeRoot( + rootHash = subtreeRoot.rootHash, + completingBlockHeight = subtreeRoot.completingBlockHeight.toLong() + ) + } + } +} diff --git a/backend-lib/src/main/rust/lib.rs b/backend-lib/src/main/rust/lib.rs index 24620de5..4f81afc2 100644 --- a/backend-lib/src/main/rust/lib.rs +++ b/backend-lib/src/main/rust/lib.rs @@ -23,12 +23,12 @@ use zcash_client_backend::keys::{DecodingError, UnifiedSpendingKey}; use zcash_client_backend::{ address::{RecipientAddress, UnifiedAddress}, data_api::{ - chain::scan_cached_blocks, + chain::{scan_cached_blocks, CommitmentTreeRoot}, wallet::{ decrypt_and_store_transaction, input_selection::GreedyInputSelector, shield_transparent_funds, spend, }, - WalletRead, WalletWrite, + WalletCommitmentTrees, WalletRead, WalletWrite, }, encoding::AddressCodec, fees::DustOutputPolicy, @@ -48,6 +48,8 @@ use zcash_primitives::{ consensus::{BlockHeight, BranchId, Network, Parameters}, legacy::{Script, TransparentAddress}, memo::{Memo, MemoBytes}, + merkle_tree::HashSer, + sapling, transaction::{ components::{amount::NonNegativeAmount, Amount, OutPoint, TxOut}, Transaction, @@ -1110,6 +1112,89 @@ pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_re unwrap_exc_or(&env, res, JNI_FALSE) } +fn decode_sapling_subtree_root( + env: &JNIEnv<'_>, + obj: JObject<'_>, +) -> Result, failure::Error> { + let long_as_u32 = |name| -> Result { + Ok(u32::try_from(env.get_field(obj, name, "J")?.j()?)?) + }; + + fn byte_array( + env: &JNIEnv<'_>, + obj: JObject<'_>, + name: &str, + ) -> Result, failure::Error> { + let field = env.get_field(obj, name, "[B")?.l()?.into_raw(); + Ok(env.convert_byte_array(field)?[..].try_into()?) + } + + Ok(CommitmentTreeRoot::from_parts( + BlockHeight::from_u32(long_as_u32("completingBlockHeight")?), + sapling::Node::read(&byte_array(env, obj, "rootHash")?[..])?, + )) +} + +#[no_mangle] +pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_putSaplingSubtreeRoots( + env: JNIEnv<'_>, + _: JClass<'_>, + db_data: JString<'_>, + start_index: jlong, + roots: jobjectArray, + network_id: jint, +) -> jboolean { + let res = panic::catch_unwind(|| { + let network = parse_network(network_id as u32)?; + let mut db_data = wallet_db(&env, network, db_data)?; + + let start_index = if start_index >= 0 { + start_index as u64 + } else { + return Err(format_err!("Start index must be nonnegative.")); + }; + let roots = { + let count = env.get_array_length(roots).unwrap(); + (0..count) + .map(|i| { + env.get_object_array_element(roots, i) + .map_err(|e| e.into()) + .and_then(|jobj| decode_sapling_subtree_root(&env, jobj)) + }) + .collect::, _>>()? + }; + + db_data + .put_sapling_subtree_roots(start_index, &roots) + .map(|()| JNI_TRUE) + .map_err(|e| format_err!("Error while storing Sapling subtree roots: {}", e)) + }); + + unwrap_exc_or(&env, res, JNI_FALSE) +} + +#[no_mangle] +pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_updateChainTip( + env: JNIEnv<'_>, + _: JClass<'_>, + db_data: JString<'_>, + height: jlong, + network_id: jint, +) -> jboolean { + let res = panic::catch_unwind(|| { + let network = parse_network(network_id as u32)?; + let mut db_data = wallet_db(&env, network, db_data)?; + let height = BlockHeight::try_from(height)?; + + db_data + .update_chain_tip(height) + .map(|()| JNI_TRUE) + .map_err(|e| format_err!("Error while updating chain tip to height {}: {}", height, e)) + }); + + unwrap_exc_or(&env, res, JNI_FALSE) +} + fn encode_scan_range<'a>( env: &JNIEnv<'a>, scan_range: ScanRange, diff --git a/sdk-lib/src/androidTest/java/cash/z/ecc/fixture/FakeRustBackend.kt b/sdk-lib/src/androidTest/java/cash/z/ecc/fixture/FakeRustBackend.kt index 8c9fd2ae..abeec8bb 100644 --- a/sdk-lib/src/androidTest/java/cash/z/ecc/fixture/FakeRustBackend.kt +++ b/sdk-lib/src/androidTest/java/cash/z/ecc/fixture/FakeRustBackend.kt @@ -3,6 +3,7 @@ package cash.z.ecc.fixture import cash.z.ecc.android.sdk.internal.Backend import cash.z.ecc.android.sdk.internal.model.JniBlockMeta import cash.z.ecc.android.sdk.internal.model.JniScanRange +import cash.z.ecc.android.sdk.internal.model.JniSubtreeRoot import cash.z.ecc.android.sdk.internal.model.JniUnifiedSpendingKey internal class FakeRustBackend( @@ -18,6 +19,17 @@ internal class FakeRustBackend( metadata.removeAll { it.height > height } } + override suspend fun putSaplingSubtreeRoots( + startIndex: Long, + roots: List, + ) { + TODO("Not yet implemented") + } + + override suspend fun updateChainTip(height: Long) { + TODO("Not yet implemented") + } + override suspend fun suggestScanRanges(): List { TODO("Not yet implemented") } From f7bb2ae460c350ed1f2d76bb7b7460dca67842c5 Mon Sep 17 00:00:00 2001 From: Honza Date: Wed, 19 Jul 2023 09:06:12 +0200 Subject: [PATCH 07/76] Fix JniSubtreeRoot + extension and tests --- backend-lib/build.gradle.kts | 3 +++ .../ecc/android/sdk/internal/ext/NumberExt.kt | 7 ++++++ .../android/sdk/internal/jni/RustBackend.kt | 14 +++++------ .../sdk/internal/model/JniBlockMeta.kt | 15 ++++++------ .../sdk/internal/model/JniScanRange.kt | 11 ++++----- .../sdk/internal/model/JniSubtreeRoot.kt | 11 ++++----- .../android/sdk/internal/ext/NumberExtTest.kt | 24 +++++++++++++++++++ 7 files changed, 58 insertions(+), 27 deletions(-) create mode 100644 backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/NumberExt.kt create mode 100644 backend-lib/src/test/java/cash/z/ecc/android/sdk/internal/ext/NumberExtTest.kt diff --git a/backend-lib/build.gradle.kts b/backend-lib/build.gradle.kts index 21e0e8a6..aa0027a3 100644 --- a/backend-lib/build.gradle.kts +++ b/backend-lib/build.gradle.kts @@ -104,6 +104,9 @@ dependencies { implementation(libs.kotlinx.coroutines.core) implementation(libs.kotlinx.coroutines.android) + // Tests + testImplementation(libs.kotlin.test) + androidTestImplementation(libs.androidx.multidex) androidTestImplementation(libs.androidx.test.runner) androidTestImplementation(libs.androidx.test.junit) diff --git a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/NumberExt.kt b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/NumberExt.kt new file mode 100644 index 00000000..b8bcf686 --- /dev/null +++ b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/NumberExt.kt @@ -0,0 +1,7 @@ +@file:Suppress("ktlint:filename") + +package cash.z.ecc.android.sdk.internal.ext + +fun Long.isInUIntRange(): Boolean { + return this >= 0L && this <= UInt.MAX_VALUE.toLong() +} diff --git a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/jni/RustBackend.kt b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/jni/RustBackend.kt index 48e27a15..ec120317 100644 --- a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/jni/RustBackend.kt +++ b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/jni/RustBackend.kt @@ -254,13 +254,13 @@ class RustBackend private constructor( startIndex: Long, roots: List, ) = withContext(SdkDispatchers.DATABASE_IO) { - putSaplingSubtreeRoots( - dataDbFile.absolutePath, - startIndex, - roots.toTypedArray(), - networkId = networkId - ) - } + putSaplingSubtreeRoots( + dataDbFile.absolutePath, + startIndex, + roots.toTypedArray(), + networkId = networkId + ) + } override suspend fun updateChainTip(height: Long) = withContext(SdkDispatchers.DATABASE_IO) { diff --git a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniBlockMeta.kt b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniBlockMeta.kt index faf7f324..6dfbe4dd 100644 --- a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniBlockMeta.kt +++ b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniBlockMeta.kt @@ -1,6 +1,7 @@ package cash.z.ecc.android.sdk.internal.model import androidx.annotation.Keep +import cash.z.ecc.android.sdk.internal.ext.isInUIntRange import co.electriccoin.lightwallet.client.model.CompactBlockUnsafe /** @@ -23,20 +24,18 @@ class JniBlockMeta( init { // We require some of the parameters below to be in the range of unsigned integer, because of the Rust layer // implementation. - require(UINT_RANGE.contains(height)) { - "Height $height is outside of allowed range $UINT_RANGE" + require(height.isInUIntRange()) { + "Height $height is outside of allowed UInt range" } - require(UINT_RANGE.contains(saplingOutputsCount)) { - "SaplingOutputsCount $saplingOutputsCount is outside of allowed range $UINT_RANGE" + require(saplingOutputsCount.isInUIntRange()) { + "SaplingOutputsCount $saplingOutputsCount is outside of allowed UInt range" } - require(UINT_RANGE.contains(orchardOutputsCount)) { - "SaplingOutputsCount $orchardOutputsCount is outside of allowed range $UINT_RANGE" + require(orchardOutputsCount.isInUIntRange()) { + "SaplingOutputsCount $orchardOutputsCount is outside of allowed UInt range" } } companion object { - private val UINT_RANGE = 0.toLong()..UInt.MAX_VALUE.toLong() - fun new(block: CompactBlockUnsafe): JniBlockMeta { return JniBlockMeta( height = block.height, diff --git a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniScanRange.kt b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniScanRange.kt index dad660d4..a4a7228b 100644 --- a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniScanRange.kt +++ b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniScanRange.kt @@ -1,6 +1,7 @@ package cash.z.ecc.android.sdk.internal.model import androidx.annotation.Keep +import cash.z.ecc.android.sdk.internal.ext.isInUIntRange /** * Serves as cross layer (Kotlin, Rust) communication class. @@ -18,17 +19,15 @@ class JniScanRange( init { // We require some of the parameters below to be in the range of unsigned integer, because of the Rust layer // implementation. - require(UINT_RANGE.contains(startHeight)) { - "Height $startHeight is outside of allowed range $UINT_RANGE" + require(startHeight.isInUIntRange()) { + "Height $startHeight is outside of allowed UInt range" } - require(UINT_RANGE.contains(endHeight)) { - "Height $endHeight is outside of allowed range $UINT_RANGE" + require(endHeight.isInUIntRange()) { + "Height $endHeight is outside of allowed UInt range" } } companion object { - private val UINT_RANGE = 0.toLong()..UInt.MAX_VALUE.toLong() - @OptIn(ExperimentalStdlibApi::class) fun new(range: OpenEndRange, priority: Long): JniScanRange { return JniScanRange( diff --git a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniSubtreeRoot.kt b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniSubtreeRoot.kt index 31172d3e..9c473938 100644 --- a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniSubtreeRoot.kt +++ b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniSubtreeRoot.kt @@ -1,7 +1,8 @@ package cash.z.ecc.android.sdk.internal.model import androidx.annotation.Keep -import co.electriccoin.lightwallet.client.model.CompactBlockUnsafe +import cash.z.ecc.android.sdk.internal.ext.isInUIntRange +import co.electriccoin.lightwallet.client.model.SubtreeRootUnsafe /** * Serves as cross layer (Kotlin, Rust) communication class. @@ -17,18 +18,16 @@ class JniSubtreeRoot( init { // We require some of the parameters below to be in the range of unsigned integer, // because of the Rust layer implementation. - require(UINT_RANGE.contains(completingBlockHeight)) { - "Height $completingBlockHeight is outside of allowed range $UINT_RANGE" + require(completingBlockHeight.isInUIntRange()) { + "Height $completingBlockHeight is outside of allowed UInt range" } } companion object { - private val UINT_RANGE = 0.toLong()..UInt.MAX_VALUE.toLong() - fun new(subtreeRoot: SubtreeRootUnsafe): JniSubtreeRoot { return JniSubtreeRoot( rootHash = subtreeRoot.rootHash, - completingBlockHeight = subtreeRoot.completingBlockHeight.toLong() + completingBlockHeight = subtreeRoot.completingBlockHeight.value ) } } diff --git a/backend-lib/src/test/java/cash/z/ecc/android/sdk/internal/ext/NumberExtTest.kt b/backend-lib/src/test/java/cash/z/ecc/android/sdk/internal/ext/NumberExtTest.kt new file mode 100644 index 00000000..44d2ae68 --- /dev/null +++ b/backend-lib/src/test/java/cash/z/ecc/android/sdk/internal/ext/NumberExtTest.kt @@ -0,0 +1,24 @@ +package cash.z.ecc.android.sdk.internal.ext + +import kotlin.test.Test +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +class NumberExtTest { + @Test + fun is_in_range_test() { + assertTrue(1L.isInUIntRange()) + assertTrue(0L.isInUIntRange()) + assertTrue(UInt.MAX_VALUE.toLong().isInUIntRange()) + assertTrue(UInt.MIN_VALUE.toLong().isInUIntRange()) + } + + @Test + fun is_not_in_range_test() { + assertFalse(0L.minus(1L).isInUIntRange()) + assertFalse(Long.MAX_VALUE.isInUIntRange()) + assertFalse(Long.MIN_VALUE.isInUIntRange()) + assertFalse(UInt.MAX_VALUE.toLong().plus(1L).isInUIntRange()) + assertFalse(UInt.MIN_VALUE.toLong().minus(1L).isInUIntRange()) + } +} From e43e86889cf4add4cb64d973613aed5ec3ada5f6 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Wed, 19 Jul 2023 16:43:18 +0000 Subject: [PATCH 08/76] Migrate to Rust crate versions with fast spendability bugfixes --- backend-lib/Cargo.lock | 20 ++++++++++---------- backend-lib/Cargo.toml | 14 +++++++------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/backend-lib/Cargo.lock b/backend-lib/Cargo.lock index 1a0a5687..3b098597 100644 --- a/backend-lib/Cargo.lock +++ b/backend-lib/Cargo.lock @@ -483,7 +483,7 @@ checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" [[package]] name = "equihash" version = "0.2.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=352e1c709ac4c95074812dcabeba3cd5565371d2#352e1c709ac4c95074812dcabeba3cd5565371d2" +source = "git+https://github.com/zcash/librustzcash.git?rev=f4221889feee34bbfb87238141ba867b973f317f#f4221889feee34bbfb87238141ba867b973f317f" dependencies = [ "blake2b_simd", "byteorder", @@ -513,7 +513,7 @@ dependencies = [ [[package]] name = "f4jumble" version = "0.1.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=352e1c709ac4c95074812dcabeba3cd5565371d2#352e1c709ac4c95074812dcabeba3cd5565371d2" +source = "git+https://github.com/zcash/librustzcash.git?rev=f4221889feee34bbfb87238141ba867b973f317f#f4221889feee34bbfb87238141ba867b973f317f" dependencies = [ "blake2b_simd", ] @@ -796,7 +796,7 @@ dependencies = [ [[package]] name = "incrementalmerkletree" version = "0.4.0" -source = "git+https://github.com/zcash/incrementalmerkletree.git?rev=67111e29403c33f2e36d6924167f1d5f04ad0fc2#67111e29403c33f2e36d6924167f1d5f04ad0fc2" +source = "git+https://github.com/zcash/incrementalmerkletree.git?rev=bae25ad89c0c192bee625252d2d419bd56638e48#bae25ad89c0c192bee625252d2d419bd56638e48" dependencies = [ "either", ] @@ -1648,7 +1648,7 @@ dependencies = [ [[package]] name = "shardtree" version = "0.0.0" -source = "git+https://github.com/zcash/incrementalmerkletree.git?rev=67111e29403c33f2e36d6924167f1d5f04ad0fc2#67111e29403c33f2e36d6924167f1d5f04ad0fc2" +source = "git+https://github.com/zcash/incrementalmerkletree.git?rev=bae25ad89c0c192bee625252d2d419bd56638e48#bae25ad89c0c192bee625252d2d419bd56638e48" dependencies = [ "bitflags 1.3.2", "either", @@ -2258,7 +2258,7 @@ dependencies = [ [[package]] name = "zcash_address" version = "0.3.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=352e1c709ac4c95074812dcabeba3cd5565371d2#352e1c709ac4c95074812dcabeba3cd5565371d2" +source = "git+https://github.com/zcash/librustzcash.git?rev=f4221889feee34bbfb87238141ba867b973f317f#f4221889feee34bbfb87238141ba867b973f317f" dependencies = [ "bech32", "bs58", @@ -2269,7 +2269,7 @@ dependencies = [ [[package]] name = "zcash_client_backend" version = "0.9.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=352e1c709ac4c95074812dcabeba3cd5565371d2#352e1c709ac4c95074812dcabeba3cd5565371d2" +source = "git+https://github.com/zcash/librustzcash.git?rev=f4221889feee34bbfb87238141ba867b973f317f#f4221889feee34bbfb87238141ba867b973f317f" dependencies = [ "base64", "bech32", @@ -2302,7 +2302,7 @@ dependencies = [ [[package]] name = "zcash_client_sqlite" version = "0.7.1" -source = "git+https://github.com/zcash/librustzcash.git?rev=352e1c709ac4c95074812dcabeba3cd5565371d2#352e1c709ac4c95074812dcabeba3cd5565371d2" +source = "git+https://github.com/zcash/librustzcash.git?rev=f4221889feee34bbfb87238141ba867b973f317f#f4221889feee34bbfb87238141ba867b973f317f" dependencies = [ "bs58", "byteorder", @@ -2328,7 +2328,7 @@ dependencies = [ [[package]] name = "zcash_encoding" version = "0.2.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=352e1c709ac4c95074812dcabeba3cd5565371d2#352e1c709ac4c95074812dcabeba3cd5565371d2" +source = "git+https://github.com/zcash/librustzcash.git?rev=f4221889feee34bbfb87238141ba867b973f317f#f4221889feee34bbfb87238141ba867b973f317f" dependencies = [ "byteorder", "nonempty", @@ -2350,7 +2350,7 @@ dependencies = [ [[package]] name = "zcash_primitives" version = "0.12.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=352e1c709ac4c95074812dcabeba3cd5565371d2#352e1c709ac4c95074812dcabeba3cd5565371d2" +source = "git+https://github.com/zcash/librustzcash.git?rev=f4221889feee34bbfb87238141ba867b973f317f#f4221889feee34bbfb87238141ba867b973f317f" dependencies = [ "aes", "bip0039", @@ -2385,7 +2385,7 @@ dependencies = [ [[package]] name = "zcash_proofs" version = "0.12.1" -source = "git+https://github.com/zcash/librustzcash.git?rev=352e1c709ac4c95074812dcabeba3cd5565371d2#352e1c709ac4c95074812dcabeba3cd5565371d2" +source = "git+https://github.com/zcash/librustzcash.git?rev=f4221889feee34bbfb87238141ba867b973f317f#f4221889feee34bbfb87238141ba867b973f317f" dependencies = [ "bellman", "blake2b_simd", diff --git a/backend-lib/Cargo.toml b/backend-lib/Cargo.toml index b24df510..9c806a6c 100644 --- a/backend-lib/Cargo.toml +++ b/backend-lib/Cargo.toml @@ -41,14 +41,14 @@ dlopen2 = "0.4" libc = "0.2" [patch.crates-io] -incrementalmerkletree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "67111e29403c33f2e36d6924167f1d5f04ad0fc2" } -shardtree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "67111e29403c33f2e36d6924167f1d5f04ad0fc2" } +incrementalmerkletree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "bae25ad89c0c192bee625252d2d419bd56638e48" } +shardtree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "bae25ad89c0c192bee625252d2d419bd56638e48" } orchard = { git = "https://github.com/zcash/orchard.git", rev = "6ef89d5f154de2cf7b7dd87edb8d8c49158beebb" } -zcash_address = { git = "https://github.com/zcash/librustzcash.git", rev = "352e1c709ac4c95074812dcabeba3cd5565371d2" } -zcash_client_backend = { git = "https://github.com/zcash/librustzcash.git", rev = "352e1c709ac4c95074812dcabeba3cd5565371d2" } -zcash_client_sqlite = { git = "https://github.com/zcash/librustzcash.git", rev = "352e1c709ac4c95074812dcabeba3cd5565371d2" } -zcash_primitives = { git = "https://github.com/zcash/librustzcash.git", rev = "352e1c709ac4c95074812dcabeba3cd5565371d2" } -zcash_proofs = { git = "https://github.com/zcash/librustzcash.git", rev = "352e1c709ac4c95074812dcabeba3cd5565371d2" } +zcash_address = { git = "https://github.com/zcash/librustzcash.git", rev = "f4221889feee34bbfb87238141ba867b973f317f" } +zcash_client_backend = { git = "https://github.com/zcash/librustzcash.git", rev = "f4221889feee34bbfb87238141ba867b973f317f" } +zcash_client_sqlite = { git = "https://github.com/zcash/librustzcash.git", rev = "f4221889feee34bbfb87238141ba867b973f317f" } +zcash_primitives = { git = "https://github.com/zcash/librustzcash.git", rev = "f4221889feee34bbfb87238141ba867b973f317f" } +zcash_proofs = { git = "https://github.com/zcash/librustzcash.git", rev = "f4221889feee34bbfb87238141ba867b973f317f" } ## Uncomment this to test librustzcash changes locally #[patch.crates-io] From 6a68889df607ac5fb7d0d3c99537328ba5ded1b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Honza=20Rychnovsk=C3=BD?= Date: Thu, 20 Jul 2023 08:42:19 +0200 Subject: [PATCH 09/76] [#1123] Linear vs Non-linear decision mechanism --- .../sdk/internal/model/JniSubtreeRoot.kt | 10 +-- .../sdk/block/CompactBlockProcessor.kt | 86 +++++++++++++++++-- .../z/ecc/android/sdk/exception/Exceptions.kt | 5 ++ .../android/sdk/internal/TypesafeBackend.kt | 16 ++++ .../sdk/internal/TypesafeBackendImpl.kt | 15 ++++ .../internal/block/CompactBlockDownloader.kt | 45 +++++++++- .../android/sdk/internal/model/SubtreeRoot.kt | 43 ++++++++++ 7 files changed, 204 insertions(+), 16 deletions(-) create mode 100644 sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/SubtreeRoot.kt diff --git a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniSubtreeRoot.kt b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniSubtreeRoot.kt index 9c473938..9fd50ba9 100644 --- a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniSubtreeRoot.kt +++ b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniSubtreeRoot.kt @@ -2,13 +2,13 @@ package cash.z.ecc.android.sdk.internal.model import androidx.annotation.Keep import cash.z.ecc.android.sdk.internal.ext.isInUIntRange -import co.electriccoin.lightwallet.client.model.SubtreeRootUnsafe /** * Serves as cross layer (Kotlin, Rust) communication class. * * @param rootHash the subtree's root hash - * @param completingBlockHeight the block height in which the subtree was completed - although it's type Long, it needs to be in UInt range + * @param completingBlockHeight the block height in which the subtree was completed - although it's type Long, it needs + * to be in UInt range */ @Keep class JniSubtreeRoot( @@ -24,10 +24,10 @@ class JniSubtreeRoot( } companion object { - fun new(subtreeRoot: SubtreeRootUnsafe): JniSubtreeRoot { + fun new(rootHash: ByteArray, completingBlockHeight: Long): JniSubtreeRoot { return JniSubtreeRoot( - rootHash = subtreeRoot.rootHash, - completingBlockHeight = subtreeRoot.completingBlockHeight.value + rootHash = rootHash, + completingBlockHeight = completingBlockHeight ) } } diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt index 472b53ce..97645eff 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt @@ -24,6 +24,7 @@ import cash.z.ecc.android.sdk.internal.ext.toHexReversed import cash.z.ecc.android.sdk.internal.model.BlockBatch import cash.z.ecc.android.sdk.internal.model.DbTransactionOverview import cash.z.ecc.android.sdk.internal.model.JniBlockMeta +import cash.z.ecc.android.sdk.internal.model.SubtreeRoot import cash.z.ecc.android.sdk.internal.model.ext.from import cash.z.ecc.android.sdk.internal.model.ext.toBlockHeight import cash.z.ecc.android.sdk.internal.repository.DerivedDataRepository @@ -38,6 +39,8 @@ import co.electriccoin.lightwallet.client.model.BlockHeightUnsafe import co.electriccoin.lightwallet.client.model.GetAddressUtxosReplyUnsafe import co.electriccoin.lightwallet.client.model.LightWalletEndpointInfoUnsafe import co.electriccoin.lightwallet.client.model.Response +import co.electriccoin.lightwallet.client.model.ShieldedProtocolEnum +import co.electriccoin.lightwallet.client.model.SubtreeRootUnsafe import kotlinx.coroutines.delay import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.MutableStateFlow @@ -51,6 +54,7 @@ import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.onCompletion import kotlinx.coroutines.flow.onEach import kotlinx.coroutines.flow.takeWhile +import kotlinx.coroutines.flow.toList import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import java.util.Locale @@ -186,14 +190,26 @@ class CompactBlockProcessor internal constructor( lastKnownHeight = getLastScannedHeight(repository) ) - Twig.debug { "setup verified. processor starting" } + // Download note commitment tree data from lightwalletd to decide if we communicate with linear + // or non-linear node + var subTreeRootList: List? = null + retryUpTo(GET_SUBTREE_ROOTS_RETRIES) { + subTreeRootList = getSubtreeRoots().map { SubtreeRoot.new(it, network) } + Twig.info { "Fetched SubTreeRoot list: $subTreeRootList" } + } - // using do/while makes it easier to execute exactly one loop which helps with testing this processor quickly + Twig.debug { "Setup verified. Processor starting..." } + + // Using do/while makes it easier to execute exactly one loop which helps with testing this processor quickly // (because you can start and then immediately set isStopped=true to always get precisely one loop) do { retryWithBackoff(::onProcessorError, maxDelayMillis = MAX_BACKOFF_INTERVAL) { val result = processingMutex.withLockLogged("processNewBlocks") { - processNewBlocks() + if (subTreeRootList.isNullOrEmpty()) { + processNewBlocksInLinearOrder() + } else { + processNewBlocksInNonLinearOrder(subTreeRootList!!) + } } // immediately process again after failures in order to download new blocks right away when (result) { @@ -296,13 +312,13 @@ class CompactBlockProcessor internal constructor( throw error } - private suspend fun processNewBlocks(): BlockProcessingResult { - Twig.debug { "Beginning to process new blocks (with lower bound: $lowerBoundHeight)..." } + private suspend fun processNewBlocksInLinearOrder(): BlockProcessingResult { + Twig.debug { "Beginning to process new blocks with Linear approach (with lower bound: $lowerBoundHeight)..." } return if (!updateRanges()) { Twig.debug { "Disconnection detected! Attempting to reconnect!" } setState(State.Disconnected) - downloader.lightWalletClient.reconnect() + downloader.reconnect() BlockProcessingResult.Reconnecting } else if (_processorInfo.value.lastSyncRange.isNullOrEmpty()) { setState(State.Synced(_processorInfo.value.lastSyncRange)) @@ -330,6 +346,57 @@ class CompactBlockProcessor internal constructor( } } + @Throws(LightWalletException.GetSubtreeRootsException::class) + internal suspend fun getSubtreeRoots(): List { + Twig.debug { "Getting SubtreeRoots..." } + + return downloader.getSubtreeRoots( + startIndex = 1, + maxEntries = 0, + shieldedProtocol = ShieldedProtocolEnum.SAPLING + ).onEach { response -> + when (response) { + is Response.Success -> { + Twig.debug { "SubtreeRoots got successfully" } + } + is Response.Failure -> { + Twig.error { "Getting SubtreeRoots failed with: ${response.toThrowable()}" } + throw LightWalletException.GetSubtreeRootsException( + response.code, + response.description, + response.toThrowable() + ) + } + } + } + .filterIsInstance>() + .map { response -> + response.result + }.toList() + } + + private suspend fun processNewBlocksInNonLinearOrder(subTreeRootList: List): BlockProcessingResult { + Twig.debug { + "Beginning to process new blocks with DAG approach (with roots: $subTreeRootList, and lower " + + "bound: $lowerBoundHeight)..." + } + + // 2) Pass the commitment tree data to the database. + // wallet_db.put_sapling_subtree_roots(0, &roots).unwrap(); + + // 3) Download chain tip metadata from lightwalletd + // let tip_height: BlockHeight = unimplemented!(); + // Possibly call the modified updateRanges() to update just the latestBlockHeight field + + // 4) Notify the wallet of the updated chain tip. + // wallet_db.update_chain_tip(tip_height).map_err(Error::Wallet)?; + + // 5) Get the suggested scan ranges from the wallet database + // let mut scan_ranges = wallet_db.suggest_scan_ranges().map_err(Error::Wallet)?; + + return BlockProcessingResult.NoBlocksToProcess + } + @Suppress("ReturnCount") private suspend fun syncBlocksAndEnhanceTransactions( syncRange: ClosedRange, @@ -529,7 +596,7 @@ class CompactBlockProcessor internal constructor( retryUpTo(UTXO_FETCH_RETRIES) { val tAddresses = backend.listTransparentReceivers(account) - downloader.lightWalletClient.fetchUtxos( + downloader.fetchUtxos( tAddresses, BlockHeightUnsafe.from(startHeight) ).onEach { response -> @@ -640,6 +707,11 @@ class CompactBlockProcessor internal constructor( */ internal const val UTXO_FETCH_RETRIES = 3 + /** + * Get subtree roots default attempts at retrying. + */ + internal const val GET_SUBTREE_ROOTS_RETRIES = 3 + /** * The theoretical maximum number of blocks in a reorg, due to other bottlenecks in the protocol design. */ diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/exception/Exceptions.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/exception/Exceptions.kt index b248f41d..6737afcc 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/exception/Exceptions.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/exception/Exceptions.kt @@ -240,6 +240,11 @@ sealed class LightWalletException(message: String, cause: Throwable? = null) : S cause ) + class GetSubtreeRootsException(code: Int, description: String?, cause: Throwable) : SdkException( + "Failed to get subtree roots with code: $code due to: ${description ?: "-"}", + cause + ) + class FetchUtxosException(code: Int, description: String?, cause: Throwable) : SdkException( "Failed to fetch UTXOs with code: $code due to: ${description ?: "-"}", cause diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackend.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackend.kt index 96482493..a1ebbb4b 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackend.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackend.kt @@ -3,6 +3,7 @@ package cash.z.ecc.android.sdk.internal import cash.z.ecc.android.sdk.internal.model.Checkpoint import cash.z.ecc.android.sdk.internal.model.JniBlockMeta import cash.z.ecc.android.sdk.internal.model.ScanRange +import cash.z.ecc.android.sdk.internal.model.SubtreeRoot import cash.z.ecc.android.sdk.model.Account import cash.z.ecc.android.sdk.model.BlockHeight import cash.z.ecc.android.sdk.model.UnifiedFullViewingKey @@ -78,6 +79,21 @@ internal interface TypesafeBackend { suspend fun initDataDb(seed: ByteArray?): Int + /** + * @throws RuntimeException as a common indicator of the operation failure + */ + @Throws(RuntimeException::class) + suspend fun putSaplingSubtreeRoots( + startIndex: Long, + roots: List, + ) + + /** + * @throws RuntimeException as a common indicator of the operation failure + */ + @Throws(RuntimeException::class) + suspend fun updateChainTip(height: BlockHeight) + /** * @throws RuntimeException as a common indicator of the operation failure */ diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackendImpl.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackendImpl.kt index 5794da45..e8ddadd5 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackendImpl.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackendImpl.kt @@ -2,7 +2,9 @@ package cash.z.ecc.android.sdk.internal import cash.z.ecc.android.sdk.internal.model.Checkpoint import cash.z.ecc.android.sdk.internal.model.JniBlockMeta +import cash.z.ecc.android.sdk.internal.model.JniSubtreeRoot import cash.z.ecc.android.sdk.internal.model.ScanRange +import cash.z.ecc.android.sdk.internal.model.SubtreeRoot import cash.z.ecc.android.sdk.model.Account import cash.z.ecc.android.sdk.model.BlockHeight import cash.z.ecc.android.sdk.model.UnifiedFullViewingKey @@ -156,6 +158,19 @@ internal class TypesafeBackendImpl(private val backend: Backend) : TypesafeBacke override suspend fun initDataDb(seed: ByteArray?): Int = backend.initDataDb(seed) + override suspend fun putSaplingSubtreeRoots(startIndex: Long, roots: List) = + backend.putSaplingSubtreeRoots( + startIndex = startIndex, + roots = roots.map { + JniSubtreeRoot.new( + rootHash = it.rootHash, + completingBlockHeight = it.completingBlockHeight.value + ) + } + ) + + override suspend fun updateChainTip(height: BlockHeight) = backend.updateChainTip(height.value) + override suspend fun scanBlocks(fromHeight: BlockHeight, limit: Long) = backend.scanBlocks(fromHeight.value, limit) override suspend fun suggestScanRanges(): List = backend.suggestScanRanges().map { jniScanRange -> diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/block/CompactBlockDownloader.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/block/CompactBlockDownloader.kt index 6b1d7c77..f961de8e 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/block/CompactBlockDownloader.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/block/CompactBlockDownloader.kt @@ -12,7 +12,7 @@ import co.electriccoin.lightwallet.client.model.BlockHeightUnsafe import co.electriccoin.lightwallet.client.model.CompactBlockUnsafe import co.electriccoin.lightwallet.client.model.LightWalletEndpointInfoUnsafe import co.electriccoin.lightwallet.client.model.Response -import kotlinx.coroutines.Dispatchers +import co.electriccoin.lightwallet.client.model.ShieldedProtocolEnum import kotlinx.coroutines.Dispatchers.IO import kotlinx.coroutines.flow.filterIsInstance import kotlinx.coroutines.flow.map @@ -31,8 +31,7 @@ import kotlinx.coroutines.withContext */ open class CompactBlockDownloader private constructor(val compactBlockRepository: CompactBlockRepository) { - lateinit var lightWalletClient: LightWalletClient - private set + private lateinit var lightWalletClient: LightWalletClient constructor( lightWalletClient: LightWalletClient, @@ -129,11 +128,21 @@ open class CompactBlockDownloader private constructor(val compactBlockRepository * Stop this downloader and cleanup any resources being used. */ suspend fun stop() { - withContext(Dispatchers.IO) { + withContext(IO) { lightWalletClient.shutdown() } } + /** + * Reconnect to the same or a different server. This is useful when the connection is + * unrecoverable. That might be time to switch to a mirror or just reconnect. + */ + suspend fun reconnect() { + withContext(IO) { + lightWalletClient.reconnect() + } + } + /** * Fetch the details of a known transaction. * @@ -141,6 +150,34 @@ open class CompactBlockDownloader private constructor(val compactBlockRepository */ suspend fun fetchTransaction(txId: ByteArray) = lightWalletClient.fetchTransaction(txId) + /** + * Fetch all UTXOs for the given addresses and from the given height. + * + * @return Flow of UTXOs for the given [tAddresses] from the [startHeight] + */ + suspend fun fetchUtxos( + tAddresses: List, + startHeight: BlockHeightUnsafe + ) = lightWalletClient.fetchUtxos( + tAddresses = tAddresses, + startHeight = startHeight + ) + + /** + * Returns a stream of information about roots of subtrees of the Sapling and Orchard note commitment trees. + * + * @return a flow of information about roots of subtrees of the Sapling and Orchard note commitment trees. + */ + suspend fun getSubtreeRoots( + startIndex: Int, + shieldedProtocol: ShieldedProtocolEnum, + maxEntries: Int + ) = lightWalletClient.getSubtreeRoots( + startIndex = startIndex, + shieldedProtocol = shieldedProtocol, + maxEntries = maxEntries + ) + companion object { private const val GET_SERVER_INFO_RETRIES = 6 } diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/SubtreeRoot.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/SubtreeRoot.kt new file mode 100644 index 00000000..a58306ed --- /dev/null +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/SubtreeRoot.kt @@ -0,0 +1,43 @@ +package cash.z.ecc.android.sdk.internal.model + +import cash.z.ecc.android.sdk.model.BlockHeight +import cash.z.ecc.android.sdk.model.ZcashNetwork +import co.electriccoin.lightwallet.client.model.SubtreeRootUnsafe + +internal data class SubtreeRoot( + val rootHash: ByteArray, + val completingBlockHash: ByteArray, + val completingBlockHeight: BlockHeight +) { + override fun toString() = "SubtreeRoot(completingBlockHeight=${completingBlockHeight.value})" + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as SubtreeRoot + + if (!rootHash.contentEquals(other.rootHash)) return false + if (!completingBlockHash.contentEquals(other.completingBlockHash)) return false + if (completingBlockHeight != other.completingBlockHeight) return false + + return true + } + + override fun hashCode(): Int { + var result = rootHash.contentHashCode() + result = 31 * result + completingBlockHash.contentHashCode() + result = 31 * result + completingBlockHeight.hashCode() + return result + } + + companion object { + fun new(unsafe: SubtreeRootUnsafe, zcashNetwork: ZcashNetwork): SubtreeRoot { + return SubtreeRoot( + rootHash = unsafe.rootHash, + completingBlockHash = unsafe.completingBlockHash, + completingBlockHeight = BlockHeight.new(zcashNetwork, unsafe.completingBlockHeight.value) + ) + } + } +} From b811305e5330f18ee57e4c5dca211e80fe0a1d0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Honza=20Rychnovsk=C3=BD?= Date: Thu, 20 Jul 2023 15:42:46 +0200 Subject: [PATCH 10/76] [#1125] Download chain tip metadata --- .../sdk/block/CompactBlockProcessor.kt | 167 ++++++++++++------ .../z/ecc/android/sdk/exception/Exceptions.kt | 5 + .../android/sdk/internal/ext/WalletService.kt | 29 +++ 3 files changed, 150 insertions(+), 51 deletions(-) diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt index 97645eff..6713ba55 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt @@ -19,6 +19,7 @@ import cash.z.ecc.android.sdk.internal.block.CompactBlockDownloader import cash.z.ecc.android.sdk.internal.ext.isNullOrEmpty import cash.z.ecc.android.sdk.internal.ext.length import cash.z.ecc.android.sdk.internal.ext.retryUpTo +import cash.z.ecc.android.sdk.internal.ext.retryUpToAndContinue import cash.z.ecc.android.sdk.internal.ext.retryWithBackoff import cash.z.ecc.android.sdk.internal.ext.toHexReversed import cash.z.ecc.android.sdk.internal.model.BlockBatch @@ -192,11 +193,8 @@ class CompactBlockProcessor internal constructor( // Download note commitment tree data from lightwalletd to decide if we communicate with linear // or non-linear node - var subTreeRootList: List? = null - retryUpTo(GET_SUBTREE_ROOTS_RETRIES) { - subTreeRootList = getSubtreeRoots().map { SubtreeRoot.new(it, network) } - Twig.info { "Fetched SubTreeRoot list: $subTreeRootList" } - } + val subTreeRootList = getSubtreeRoots(downloader, network) + Twig.info { "Fetched SubTreeRoot list: $subTreeRootList" } Twig.debug { "Setup verified. Processor starting..." } @@ -208,7 +206,7 @@ class CompactBlockProcessor internal constructor( if (subTreeRootList.isNullOrEmpty()) { processNewBlocksInLinearOrder() } else { - processNewBlocksInNonLinearOrder(subTreeRootList!!) + processNewBlocksInNonLinearOrder(subTreeRootList) } } // immediately process again after failures in order to download new blocks right away @@ -316,7 +314,7 @@ class CompactBlockProcessor internal constructor( Twig.debug { "Beginning to process new blocks with Linear approach (with lower bound: $lowerBoundHeight)..." } return if (!updateRanges()) { - Twig.debug { "Disconnection detected! Attempting to reconnect!" } + Twig.debug { "Disconnection detected. Attempting to reconnect." } setState(State.Disconnected) downloader.reconnect() BlockProcessingResult.Reconnecting @@ -346,47 +344,31 @@ class CompactBlockProcessor internal constructor( } } - @Throws(LightWalletException.GetSubtreeRootsException::class) - internal suspend fun getSubtreeRoots(): List { - Twig.debug { "Getting SubtreeRoots..." } - - return downloader.getSubtreeRoots( - startIndex = 1, - maxEntries = 0, - shieldedProtocol = ShieldedProtocolEnum.SAPLING - ).onEach { response -> - when (response) { - is Response.Success -> { - Twig.debug { "SubtreeRoots got successfully" } - } - is Response.Failure -> { - Twig.error { "Getting SubtreeRoots failed with: ${response.toThrowable()}" } - throw LightWalletException.GetSubtreeRootsException( - response.code, - response.description, - response.toThrowable() - ) - } - } - } - .filterIsInstance>() - .map { response -> - response.result - }.toList() - } - private suspend fun processNewBlocksInNonLinearOrder(subTreeRootList: List): BlockProcessingResult { Twig.debug { "Beginning to process new blocks with DAG approach (with roots: $subTreeRootList, and lower " + "bound: $lowerBoundHeight)..." } - // 2) Pass the commitment tree data to the database. - // wallet_db.put_sapling_subtree_roots(0, &roots).unwrap(); + // Pass the commitment tree data to the database. + backend.putSaplingSubtreeRoots( + startIndex = 0, + roots = subTreeRootList + ) - // 3) Download chain tip metadata from lightwalletd - // let tip_height: BlockHeight = unimplemented!(); - // Possibly call the modified updateRanges() to update just the latestBlockHeight field + // Download chain tip metadata from lightwalletd + val chainTip = fetchLatestBlockHeight( + downloader = downloader, + network = network + ) ?: let { + Twig.debug { "Disconnection detected. Attempting to reconnect." } + setState(State.Disconnected) + downloader.reconnect() + return BlockProcessingResult.Reconnecting + } + + // Note: print to suppress unused warning + Twig.debug { "${chainTip.value}" } // 4) Notify the wallet of the updated chain tip. // wallet_db.update_chain_tip(tip_height).map_err(Error::Wallet)?; @@ -464,15 +446,7 @@ class CompactBlockProcessor internal constructor( private suspend fun updateRanges(): Boolean { // This fetches the latest height each time this method is called, which can be very inefficient // when downloading all of the blocks from the server - val networkBlockHeight = run { - val networkBlockHeightUnsafe = - when (val response = downloader.getLatestBlockHeight()) { - is Response.Success -> response.result - else -> null - } - - runCatching { networkBlockHeightUnsafe?.toBlockHeight(network) }.getOrNull() - } ?: return false + val networkBlockHeight = fetchLatestBlockHeight(downloader, network) ?: return false // If we find out that we previously downloaded, but not scanned persisted blocks, we need to rewind the // blocks above the last scanned height first. @@ -514,7 +488,9 @@ class CompactBlockProcessor internal constructor( /** * Confirm that the wallet data is properly setup for use. */ - // Need to refactor this to be less ugly and more testable + // TODO [#1127]: Refactor CompactBlockProcessor.verifySetup + // TODO [#1127]: Need to refactor this to be less ugly and more testable + // TODO [#1127]: https://github.com/zcash/zcash-android-wallet-sdk/issues/1127 @Suppress("NestedBlockDepth") private suspend fun verifySetup() { // verify that the data is initialized @@ -707,6 +683,11 @@ class CompactBlockProcessor internal constructor( */ internal const val UTXO_FETCH_RETRIES = 3 + /** + * Latest block height fetching default attempts at retrying. + */ + internal const val FETCH_LATEST_BLOCK_HEIGHT_RETRIES = 3 + /** * Get subtree roots default attempts at retrying. */ @@ -736,6 +717,90 @@ class CompactBlockProcessor internal constructor( */ internal const val REWIND_DISTANCE = 10 + /** + * This operation fetches and returns the latest block height (chain tip) + * + * @return Latest block height wrapped in BlockHeight object, or null in case of failure + */ + @VisibleForTesting + internal suspend fun fetchLatestBlockHeight( + downloader: CompactBlockDownloader, + network: ZcashNetwork + ): BlockHeight? { + Twig.debug { "Fetching latest block height..." } + + var latestBlockHeight: BlockHeight? = null + + retryUpToAndContinue(FETCH_LATEST_BLOCK_HEIGHT_RETRIES) { + when (val response = downloader.getLatestBlockHeight()) { + is Response.Success -> { + Twig.debug { "Latest block height fetched successfully with value: ${response.result.value}" } + latestBlockHeight = runCatching { + response.result.toBlockHeight(network) + }.getOrNull() + } + is Response.Failure -> { + Twig.error { "Fetching latest block height failed with: ${response.toThrowable()}" } + throw LightWalletException.GetLatestBlockHeightException( + response.code, + response.description, + response.toThrowable() + ) + } + } + } + + return latestBlockHeight + } + + /** + * This operation downloads note commitment tree data from the lightwalletd server to decide if we communicate + * with linear or non-linear node + * + * @return List of SubtreeRoot objects in case of the operation success, null otherwise + */ + @VisibleForTesting + internal suspend fun getSubtreeRoots( + downloader: CompactBlockDownloader, + network: ZcashNetwork + ): List? { + Twig.debug { "Fetching SubtreeRoots..." } + + var subTreeRootList: List? = null + + retryUpToAndContinue(GET_SUBTREE_ROOTS_RETRIES) { + subTreeRootList = downloader.getSubtreeRoots( + startIndex = 1, + maxEntries = 0, + shieldedProtocol = ShieldedProtocolEnum.SAPLING + ).onEach { response -> + when (response) { + is Response.Success -> { + Twig.debug { "SubtreeRoots got successfully" } + } + is Response.Failure -> { + Twig.error { "Fetching SubtreeRoots failed with: ${response.toThrowable()}" } + throw LightWalletException.GetSubtreeRootsException( + response.code, + response.description, + response.toThrowable() + ) + } + } + } + .filterIsInstance>() + .map { response -> + response.result + } + .toList() + .map { + SubtreeRoot.new(it, network) + } + } + + return subTreeRootList + } + /** * Requests, processes and persists all blocks from the given range. * diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/exception/Exceptions.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/exception/Exceptions.kt index 6737afcc..5b2809e4 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/exception/Exceptions.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/exception/Exceptions.kt @@ -249,6 +249,11 @@ sealed class LightWalletException(message: String, cause: Throwable? = null) : S "Failed to fetch UTXOs with code: $code due to: ${description ?: "-"}", cause ) + + class GetLatestBlockHeightException(code: Int, description: String?, cause: Throwable) : SdkException( + "Failed to fetch latest block height with code: $code due to: ${description ?: "-"}", + cause + ) } /** diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/WalletService.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/WalletService.kt index 2c96644d..047aca77 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/WalletService.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/WalletService.kt @@ -42,6 +42,35 @@ suspend inline fun retryUpTo( } } +/** + * Execute the given block and if it fails, retry up to [retries] more times. If none of the + * retries succeed, then leave the block execution unfinished and continue. + * + * @param retries the number of times to retry the block after the first attempt fails. + * @param initialDelayMillis the initial amount of time to wait before the first retry. + * @param block the code to execute, which will be wrapped in a try/catch and retried whenever an + * exception is thrown up to [retries] attempts. + */ +suspend inline fun retryUpToAndContinue( + retries: Int, + initialDelayMillis: Long = 500L, + block: (Int) -> Unit +) { + var failedAttempts = 0 + while (failedAttempts < retries) { + @Suppress("TooGenericExceptionCaught") + try { + block(failedAttempts) + return + } catch (t: Throwable) { + failedAttempts++ + val duration = (initialDelayMillis.toDouble() * Math.pow(2.0, failedAttempts.toDouble() - 1)).toLong() + Twig.warn(t) { "Retrying ($failedAttempts/$retries) in ${duration}s..." } + delay(duration) + } + } +} + /** * Execute the given block and if it fails, retry with an exponential backoff. * From c8b481a12f4137022371dc358daba803127a5463 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Honza=20Rychnovsk=C3=BD?= Date: Mon, 24 Jul 2023 09:42:37 +0200 Subject: [PATCH 11/76] [#1130] Initial Suggested Ranges Parsing * [#1130] Initial suggested ranges parsing --- .../sdk/block/CompactBlockProcessor.kt | 144 +++++++++++++++++- 1 file changed, 136 insertions(+), 8 deletions(-) diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt index 6713ba55..16e0ef29 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt @@ -25,6 +25,7 @@ import cash.z.ecc.android.sdk.internal.ext.toHexReversed import cash.z.ecc.android.sdk.internal.model.BlockBatch import cash.z.ecc.android.sdk.internal.model.DbTransactionOverview import cash.z.ecc.android.sdk.internal.model.JniBlockMeta +import cash.z.ecc.android.sdk.internal.model.ScanRange import cash.z.ecc.android.sdk.internal.model.SubtreeRoot import cash.z.ecc.android.sdk.internal.model.ext.from import cash.z.ecc.android.sdk.internal.model.ext.toBlockHeight @@ -179,7 +180,7 @@ class CompactBlockProcessor internal constructor( /** * Download compact blocks, verify and scan them until [stop] is called. */ - @Suppress("LongMethod") + @Suppress("LongMethod", "CyclomaticComplexMethod") suspend fun start() { verifySetup() @@ -194,7 +195,7 @@ class CompactBlockProcessor internal constructor( // Download note commitment tree data from lightwalletd to decide if we communicate with linear // or non-linear node val subTreeRootList = getSubtreeRoots(downloader, network) - Twig.info { "Fetched SubTreeRoot list: $subTreeRootList" } + Twig.info { "Fetched SubTreeRoot list size: ${subTreeRootList?.size ?: 0}" } Twig.debug { "Setup verified. Processor starting..." } @@ -221,6 +222,8 @@ class CompactBlockProcessor internal constructor( } BlockProcessingResult.NoBlocksToProcess -> { + // TODO [#1129]: Refactor work with lastSyncRange and lastSyncedHeight + // TODO [#1129]: https://github.com/zcash/zcash-android-wallet-sdk/issues/1129 val noWorkDone = _processorInfo.value.lastSyncRange?.isEmpty() ?: true val summary = if (noWorkDone) { "Nothing to process: no new blocks to sync" @@ -344,6 +347,31 @@ class CompactBlockProcessor internal constructor( } } + internal sealed class SuggestScanRangesResult { + data class Success(val ranges: List) : SuggestScanRangesResult() + + // Fix it: consider failed at height parameter and more concrete exception type + data class Failure(val exception: Throwable) : SuggestScanRangesResult() + } + + @Suppress("MagicNumber") + internal enum class SuggestScanRangePriority(val priority: Long) { + Scanned(10), + Historic(20), + OpenAdjacent(30), + FoundNote(40), + ChainTip(50), + Verify(60); + + companion object { + fun fromPriority(priority: Long): SuggestScanRangePriority { + Twig.verbose { "Current suggested scan range priority: $priority" } + return values().firstOrNull { it.priority == priority } ?: Scanned + } + } + } + + @Suppress("ReturnCount") private suspend fun processNewBlocksInNonLinearOrder(subTreeRootList: List): BlockProcessingResult { Twig.debug { "Beginning to process new blocks with DAG approach (with roots: $subTreeRootList, and lower " + @@ -367,14 +395,98 @@ class CompactBlockProcessor internal constructor( return BlockProcessingResult.Reconnecting } - // Note: print to suppress unused warning - Twig.debug { "${chainTip.value}" } + // Notify the wallet of the updated chain tip. + backend.updateChainTip(chainTip) - // 4) Notify the wallet of the updated chain tip. - // wallet_db.update_chain_tip(tip_height).map_err(Error::Wallet)?; + // Get the suggested scan ranges from the wallet database + when (val suggestedRangesResult = suggestScanRanges(backend)) { + is SuggestScanRangesResult.Failure -> { + Twig.error { "Process suggested scan ranges failure: ${suggestedRangesResult.exception}" } + // Fix it: process failure + } + is SuggestScanRangesResult.Success -> { + Twig.debug { "Process suggested scan ranges result: ${suggestedRangesResult.ranges}" } - // 5) Get the suggested scan ranges from the wallet database - // let mut scan_ranges = wallet_db.suggest_scan_ranges().map_err(Error::Wallet)?; + if (suggestedRangesResult.ranges.isEmpty()) { + // Nothing to sync - break out of the processing + return BlockProcessingResult.NoBlocksToProcess + } else { + val firstRangePriority = SuggestScanRangePriority.fromPriority( + suggestedRangesResult.ranges[0].priority + ) + when (firstRangePriority) { + SuggestScanRangePriority.Verify -> {} + SuggestScanRangePriority.ChainTip -> {} + SuggestScanRangePriority.FoundNote -> {} + SuggestScanRangePriority.OpenAdjacent -> {} + SuggestScanRangePriority.Historic -> {} + SuggestScanRangePriority.Scanned -> {} + } + } + } + } + + // Run the following loop until the wallet's view of the chain tip as of the previous wallet session is valid + // loop { + // // If there is a range of blocks that needs to be verified, it will always be returned as + // // the first element of the vector of suggested ranges. + // match scan_ranges.first() { + // Some(scan_range) if scan_range.priority() == ScanPriority::Verify => { + // // Download the blocks in `scan_range` into the block source, overwriting any + // // existing blocks in this range. + // unimplemented!(); + // + // // Scan the downloaded blocks + // let scan_result = scan_cached_blocks( + // &network, + // &block_source, + // &mut wallet_db, + // scan_range.block_range().start, + // scan_range.len() + // ); + // + // // Check for scanning errors that indicate that the wallet's chain tip is out of + // // sync with blockchain history. + // match scan_result { + // Ok(_) => { + // // At this point, the cache and scanned data are locally consistent (though + // // not necessarily consistent with the latest chain tip - this would be + // // discovered the next time this codepath is executed after new blocks are + // // received) so we can break out of the loop. + // break; + // } + // Err(Error::Scan(err)) if err.is_continuity_error() => { + // // Pick a height to rewind to, which must be at least one block before + // // the height at which the error occurred, but may be an earlier height + // // determined based on heuristics such as the platform, available bandwidth, + // // size of recent CompactBlocks, etc. + // let rewind_height = err.at_height().saturating_sub(10); + // + // // Rewind to the chosen height. + // wallet_db.truncate_to_height(rewind_height).map_err(Error::Wallet)?; + // + // // Delete cached blocks from rewind_height onwards. + // // + // // This does imply that assumed-valid blocks will be re-downloaded, but it + // // is also possible that in the intervening time, a chain reorg has + // // occurred that orphaned some of those blocks. + // unimplemented!(); + // } + // Err(other) => { + // // Handle or return other errors + // } + // } + // + // // Truncation will have updated the suggested scan ranges, so we now + // // re_request + // scan_ranges = wallet_db.suggest_scan_ranges().map_err(Error::Wallet)?; + // } + // _ => { + // // Nothing to verify; break out of the loop + // break; + // } + // } + // } return BlockProcessingResult.NoBlocksToProcess } @@ -801,6 +913,20 @@ class CompactBlockProcessor internal constructor( return subTreeRootList } + @VisibleForTesting + internal suspend fun suggestScanRanges(backend: TypesafeBackend): SuggestScanRangesResult { + return runCatching { + backend.suggestScanRanges() + }.onSuccess { ranges -> + Twig.info { "Successfully got newly suggested ranges: $ranges" } + }.onFailure { exception -> + Twig.error { "Failed to get newly suggested ranges with: $exception" } + }.fold( + onSuccess = { SuggestScanRangesResult.Success(it) }, + onFailure = { SuggestScanRangesResult.Failure(it) } + ) + } + /** * Requests, processes and persists all blocks from the given range. * @@ -1610,6 +1736,8 @@ class CompactBlockProcessor internal constructor( * @param firstUnenhancedHeight the height in which the enhancing should start, or null in case of no previous * transaction enhancing done yet */ + // TODO [#1129]: Refactor work with lastSyncRange and lastSyncedHeight + // TODO [#1129]: https://github.com/zcash/zcash-android-wallet-sdk/issues/1129 data class ProcessorInfo( val networkBlockHeight: BlockHeight?, val lastSyncedHeight: BlockHeight?, From 811081b28970bc22c2fdf8500ed98c512fe3a460 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Mon, 24 Jul 2023 13:08:13 +0000 Subject: [PATCH 12/76] Migrate to Rust revision with spend detection and migration perf fixes --- backend-lib/Cargo.lock | 16 ++++++++-------- backend-lib/Cargo.toml | 10 +++++----- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/backend-lib/Cargo.lock b/backend-lib/Cargo.lock index 3b098597..6439eea0 100644 --- a/backend-lib/Cargo.lock +++ b/backend-lib/Cargo.lock @@ -483,7 +483,7 @@ checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" [[package]] name = "equihash" version = "0.2.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=f4221889feee34bbfb87238141ba867b973f317f#f4221889feee34bbfb87238141ba867b973f317f" +source = "git+https://github.com/zcash/librustzcash.git?rev=46ca3202e47eabc48298add881d898878f510c87#46ca3202e47eabc48298add881d898878f510c87" dependencies = [ "blake2b_simd", "byteorder", @@ -513,7 +513,7 @@ dependencies = [ [[package]] name = "f4jumble" version = "0.1.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=f4221889feee34bbfb87238141ba867b973f317f#f4221889feee34bbfb87238141ba867b973f317f" +source = "git+https://github.com/zcash/librustzcash.git?rev=46ca3202e47eabc48298add881d898878f510c87#46ca3202e47eabc48298add881d898878f510c87" dependencies = [ "blake2b_simd", ] @@ -2258,7 +2258,7 @@ dependencies = [ [[package]] name = "zcash_address" version = "0.3.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=f4221889feee34bbfb87238141ba867b973f317f#f4221889feee34bbfb87238141ba867b973f317f" +source = "git+https://github.com/zcash/librustzcash.git?rev=46ca3202e47eabc48298add881d898878f510c87#46ca3202e47eabc48298add881d898878f510c87" dependencies = [ "bech32", "bs58", @@ -2269,7 +2269,7 @@ dependencies = [ [[package]] name = "zcash_client_backend" version = "0.9.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=f4221889feee34bbfb87238141ba867b973f317f#f4221889feee34bbfb87238141ba867b973f317f" +source = "git+https://github.com/zcash/librustzcash.git?rev=46ca3202e47eabc48298add881d898878f510c87#46ca3202e47eabc48298add881d898878f510c87" dependencies = [ "base64", "bech32", @@ -2302,7 +2302,7 @@ dependencies = [ [[package]] name = "zcash_client_sqlite" version = "0.7.1" -source = "git+https://github.com/zcash/librustzcash.git?rev=f4221889feee34bbfb87238141ba867b973f317f#f4221889feee34bbfb87238141ba867b973f317f" +source = "git+https://github.com/zcash/librustzcash.git?rev=46ca3202e47eabc48298add881d898878f510c87#46ca3202e47eabc48298add881d898878f510c87" dependencies = [ "bs58", "byteorder", @@ -2328,7 +2328,7 @@ dependencies = [ [[package]] name = "zcash_encoding" version = "0.2.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=f4221889feee34bbfb87238141ba867b973f317f#f4221889feee34bbfb87238141ba867b973f317f" +source = "git+https://github.com/zcash/librustzcash.git?rev=46ca3202e47eabc48298add881d898878f510c87#46ca3202e47eabc48298add881d898878f510c87" dependencies = [ "byteorder", "nonempty", @@ -2350,7 +2350,7 @@ dependencies = [ [[package]] name = "zcash_primitives" version = "0.12.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=f4221889feee34bbfb87238141ba867b973f317f#f4221889feee34bbfb87238141ba867b973f317f" +source = "git+https://github.com/zcash/librustzcash.git?rev=46ca3202e47eabc48298add881d898878f510c87#46ca3202e47eabc48298add881d898878f510c87" dependencies = [ "aes", "bip0039", @@ -2385,7 +2385,7 @@ dependencies = [ [[package]] name = "zcash_proofs" version = "0.12.1" -source = "git+https://github.com/zcash/librustzcash.git?rev=f4221889feee34bbfb87238141ba867b973f317f#f4221889feee34bbfb87238141ba867b973f317f" +source = "git+https://github.com/zcash/librustzcash.git?rev=46ca3202e47eabc48298add881d898878f510c87#46ca3202e47eabc48298add881d898878f510c87" dependencies = [ "bellman", "blake2b_simd", diff --git a/backend-lib/Cargo.toml b/backend-lib/Cargo.toml index 9c806a6c..4bcdeb7b 100644 --- a/backend-lib/Cargo.toml +++ b/backend-lib/Cargo.toml @@ -44,11 +44,11 @@ libc = "0.2" incrementalmerkletree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "bae25ad89c0c192bee625252d2d419bd56638e48" } shardtree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "bae25ad89c0c192bee625252d2d419bd56638e48" } orchard = { git = "https://github.com/zcash/orchard.git", rev = "6ef89d5f154de2cf7b7dd87edb8d8c49158beebb" } -zcash_address = { git = "https://github.com/zcash/librustzcash.git", rev = "f4221889feee34bbfb87238141ba867b973f317f" } -zcash_client_backend = { git = "https://github.com/zcash/librustzcash.git", rev = "f4221889feee34bbfb87238141ba867b973f317f" } -zcash_client_sqlite = { git = "https://github.com/zcash/librustzcash.git", rev = "f4221889feee34bbfb87238141ba867b973f317f" } -zcash_primitives = { git = "https://github.com/zcash/librustzcash.git", rev = "f4221889feee34bbfb87238141ba867b973f317f" } -zcash_proofs = { git = "https://github.com/zcash/librustzcash.git", rev = "f4221889feee34bbfb87238141ba867b973f317f" } +zcash_address = { git = "https://github.com/zcash/librustzcash.git", rev = "46ca3202e47eabc48298add881d898878f510c87" } +zcash_client_backend = { git = "https://github.com/zcash/librustzcash.git", rev = "46ca3202e47eabc48298add881d898878f510c87" } +zcash_client_sqlite = { git = "https://github.com/zcash/librustzcash.git", rev = "46ca3202e47eabc48298add881d898878f510c87" } +zcash_primitives = { git = "https://github.com/zcash/librustzcash.git", rev = "46ca3202e47eabc48298add881d898878f510c87" } +zcash_proofs = { git = "https://github.com/zcash/librustzcash.git", rev = "46ca3202e47eabc48298add881d898878f510c87" } ## Uncomment this to test librustzcash changes locally #[patch.crates-io] From 0b7c61927ca523803331fc7a96222f9e58e471c7 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Wed, 26 Jul 2023 02:37:26 +0000 Subject: [PATCH 13/76] Migrate to Rust revision with additional performance improvements --- backend-lib/Cargo.lock | 21 +++++++++++---------- backend-lib/Cargo.toml | 14 +++++++------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/backend-lib/Cargo.lock b/backend-lib/Cargo.lock index 6439eea0..3bbad617 100644 --- a/backend-lib/Cargo.lock +++ b/backend-lib/Cargo.lock @@ -483,7 +483,7 @@ checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" [[package]] name = "equihash" version = "0.2.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=46ca3202e47eabc48298add881d898878f510c87#46ca3202e47eabc48298add881d898878f510c87" +source = "git+https://github.com/zcash/librustzcash.git?rev=facd4ccac56bb646917513c2d87f5e8d8b59750c#facd4ccac56bb646917513c2d87f5e8d8b59750c" dependencies = [ "blake2b_simd", "byteorder", @@ -513,7 +513,7 @@ dependencies = [ [[package]] name = "f4jumble" version = "0.1.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=46ca3202e47eabc48298add881d898878f510c87#46ca3202e47eabc48298add881d898878f510c87" +source = "git+https://github.com/zcash/librustzcash.git?rev=facd4ccac56bb646917513c2d87f5e8d8b59750c#facd4ccac56bb646917513c2d87f5e8d8b59750c" dependencies = [ "blake2b_simd", ] @@ -796,7 +796,7 @@ dependencies = [ [[package]] name = "incrementalmerkletree" version = "0.4.0" -source = "git+https://github.com/zcash/incrementalmerkletree.git?rev=bae25ad89c0c192bee625252d2d419bd56638e48#bae25ad89c0c192bee625252d2d419bd56638e48" +source = "git+https://github.com/zcash/incrementalmerkletree.git?rev=da97e6c399c5acddedad2c1730dbb7ee55499a2f#da97e6c399c5acddedad2c1730dbb7ee55499a2f" dependencies = [ "either", ] @@ -1648,7 +1648,7 @@ dependencies = [ [[package]] name = "shardtree" version = "0.0.0" -source = "git+https://github.com/zcash/incrementalmerkletree.git?rev=bae25ad89c0c192bee625252d2d419bd56638e48#bae25ad89c0c192bee625252d2d419bd56638e48" +source = "git+https://github.com/zcash/incrementalmerkletree.git?rev=da97e6c399c5acddedad2c1730dbb7ee55499a2f#da97e6c399c5acddedad2c1730dbb7ee55499a2f" dependencies = [ "bitflags 1.3.2", "either", @@ -2258,7 +2258,7 @@ dependencies = [ [[package]] name = "zcash_address" version = "0.3.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=46ca3202e47eabc48298add881d898878f510c87#46ca3202e47eabc48298add881d898878f510c87" +source = "git+https://github.com/zcash/librustzcash.git?rev=facd4ccac56bb646917513c2d87f5e8d8b59750c#facd4ccac56bb646917513c2d87f5e8d8b59750c" dependencies = [ "bech32", "bs58", @@ -2269,7 +2269,7 @@ dependencies = [ [[package]] name = "zcash_client_backend" version = "0.9.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=46ca3202e47eabc48298add881d898878f510c87#46ca3202e47eabc48298add881d898878f510c87" +source = "git+https://github.com/zcash/librustzcash.git?rev=facd4ccac56bb646917513c2d87f5e8d8b59750c#facd4ccac56bb646917513c2d87f5e8d8b59750c" dependencies = [ "base64", "bech32", @@ -2302,7 +2302,7 @@ dependencies = [ [[package]] name = "zcash_client_sqlite" version = "0.7.1" -source = "git+https://github.com/zcash/librustzcash.git?rev=46ca3202e47eabc48298add881d898878f510c87#46ca3202e47eabc48298add881d898878f510c87" +source = "git+https://github.com/zcash/librustzcash.git?rev=facd4ccac56bb646917513c2d87f5e8d8b59750c#facd4ccac56bb646917513c2d87f5e8d8b59750c" dependencies = [ "bs58", "byteorder", @@ -2311,6 +2311,7 @@ dependencies = [ "hdwallet 0.4.1", "incrementalmerkletree", "jubjub", + "maybe-rayon", "prost", "rusqlite", "schemer", @@ -2328,7 +2329,7 @@ dependencies = [ [[package]] name = "zcash_encoding" version = "0.2.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=46ca3202e47eabc48298add881d898878f510c87#46ca3202e47eabc48298add881d898878f510c87" +source = "git+https://github.com/zcash/librustzcash.git?rev=facd4ccac56bb646917513c2d87f5e8d8b59750c#facd4ccac56bb646917513c2d87f5e8d8b59750c" dependencies = [ "byteorder", "nonempty", @@ -2350,7 +2351,7 @@ dependencies = [ [[package]] name = "zcash_primitives" version = "0.12.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=46ca3202e47eabc48298add881d898878f510c87#46ca3202e47eabc48298add881d898878f510c87" +source = "git+https://github.com/zcash/librustzcash.git?rev=facd4ccac56bb646917513c2d87f5e8d8b59750c#facd4ccac56bb646917513c2d87f5e8d8b59750c" dependencies = [ "aes", "bip0039", @@ -2385,7 +2386,7 @@ dependencies = [ [[package]] name = "zcash_proofs" version = "0.12.1" -source = "git+https://github.com/zcash/librustzcash.git?rev=46ca3202e47eabc48298add881d898878f510c87#46ca3202e47eabc48298add881d898878f510c87" +source = "git+https://github.com/zcash/librustzcash.git?rev=facd4ccac56bb646917513c2d87f5e8d8b59750c#facd4ccac56bb646917513c2d87f5e8d8b59750c" dependencies = [ "bellman", "blake2b_simd", diff --git a/backend-lib/Cargo.toml b/backend-lib/Cargo.toml index 4bcdeb7b..fbd251cf 100644 --- a/backend-lib/Cargo.toml +++ b/backend-lib/Cargo.toml @@ -41,14 +41,14 @@ dlopen2 = "0.4" libc = "0.2" [patch.crates-io] -incrementalmerkletree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "bae25ad89c0c192bee625252d2d419bd56638e48" } -shardtree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "bae25ad89c0c192bee625252d2d419bd56638e48" } +incrementalmerkletree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "da97e6c399c5acddedad2c1730dbb7ee55499a2f" } +shardtree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "da97e6c399c5acddedad2c1730dbb7ee55499a2f" } orchard = { git = "https://github.com/zcash/orchard.git", rev = "6ef89d5f154de2cf7b7dd87edb8d8c49158beebb" } -zcash_address = { git = "https://github.com/zcash/librustzcash.git", rev = "46ca3202e47eabc48298add881d898878f510c87" } -zcash_client_backend = { git = "https://github.com/zcash/librustzcash.git", rev = "46ca3202e47eabc48298add881d898878f510c87" } -zcash_client_sqlite = { git = "https://github.com/zcash/librustzcash.git", rev = "46ca3202e47eabc48298add881d898878f510c87" } -zcash_primitives = { git = "https://github.com/zcash/librustzcash.git", rev = "46ca3202e47eabc48298add881d898878f510c87" } -zcash_proofs = { git = "https://github.com/zcash/librustzcash.git", rev = "46ca3202e47eabc48298add881d898878f510c87" } +zcash_address = { git = "https://github.com/zcash/librustzcash.git", rev = "facd4ccac56bb646917513c2d87f5e8d8b59750c" } +zcash_client_backend = { git = "https://github.com/zcash/librustzcash.git", rev = "facd4ccac56bb646917513c2d87f5e8d8b59750c" } +zcash_client_sqlite = { git = "https://github.com/zcash/librustzcash.git", rev = "facd4ccac56bb646917513c2d87f5e8d8b59750c" } +zcash_primitives = { git = "https://github.com/zcash/librustzcash.git", rev = "facd4ccac56bb646917513c2d87f5e8d8b59750c" } +zcash_proofs = { git = "https://github.com/zcash/librustzcash.git", rev = "facd4ccac56bb646917513c2d87f5e8d8b59750c" } ## Uncomment this to test librustzcash changes locally #[patch.crates-io] From 92ac1e926e30f060800490cc5fda79df226cbf53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Honza=20Rychnovsk=C3=BD?= Date: Mon, 31 Jul 2023 09:56:31 +0200 Subject: [PATCH 14/76] [#1139][#1138] Process Priority.Verify blocks * [#1139][#1138] Process Priority.Verify blocks This makes the new subtask functions more testable. * Increase CI test timeout * [#1047][#1140] Improve CBD internal state handling * [#1047][1140] Improve CBD internal state handling This improves internal state handling in CompactBlockProcessor. Error handling included. And it prepares us for ContinuityError handling too. * [#1143] Handle ContinuityError --- .github/workflows/pull-request.yml | 4 +- .../sdk/block/CompactBlockProcessor.kt | 601 ++++++++++++------ .../z/ecc/android/sdk/exception/Exceptions.kt | 23 +- .../internal/ext/{Ext.kt => ExceptionExt.kt} | 25 + .../sdk/internal/ext/OpenEndRangeExt.kt | 6 + .../android/sdk/internal/ext/WalletService.kt | 17 +- .../z/ecc/android/sdk/model/BlockHeight.kt | 18 + .../z/ecc/android/sdk/ext/ExceptionExtTest.kt | 24 + .../ecc/android/sdk/model/BlockHeightTest.kt | 22 + 9 files changed, 526 insertions(+), 214 deletions(-) rename sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/{Ext.kt => ExceptionExt.kt} (59%) create mode 100644 sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/OpenEndRangeExt.kt create mode 100644 sdk-lib/src/test/java/cash/z/ecc/android/sdk/ext/ExceptionExtTest.kt diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index ef506506..9a63abd2 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -199,7 +199,7 @@ jobs: timeout-minutes: 30 uses: ./.github/actions/setup - name: Build and test - timeout-minutes: 25 + timeout-minutes: 30 run: | ./gradlew test - name: Collect Artifacts @@ -292,7 +292,7 @@ jobs: timeout-minutes: 30 uses: ./.github/actions/setup - name: Build and test - timeout-minutes: 25 + timeout-minutes: 30 env: ORG_GRADLE_PROJECT_ZCASH_EMULATOR_WTF_API_KEY: ${{ secrets.EMULATOR_WTF_API_KEY }} run: | diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt index 16e0ef29..13d1d26f 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt @@ -17,10 +17,12 @@ import cash.z.ecc.android.sdk.internal.Twig import cash.z.ecc.android.sdk.internal.TypesafeBackend import cash.z.ecc.android.sdk.internal.block.CompactBlockDownloader import cash.z.ecc.android.sdk.internal.ext.isNullOrEmpty +import cash.z.ecc.android.sdk.internal.ext.isScanContinuityError import cash.z.ecc.android.sdk.internal.ext.length import cash.z.ecc.android.sdk.internal.ext.retryUpTo import cash.z.ecc.android.sdk.internal.ext.retryUpToAndContinue import cash.z.ecc.android.sdk.internal.ext.retryWithBackoff +import cash.z.ecc.android.sdk.internal.ext.toClosedRange import cash.z.ecc.android.sdk.internal.ext.toHexReversed import cash.z.ecc.android.sdk.internal.model.BlockBatch import cash.z.ecc.android.sdk.internal.model.DbTransactionOverview @@ -207,12 +209,23 @@ class CompactBlockProcessor internal constructor( if (subTreeRootList.isNullOrEmpty()) { processNewBlocksInLinearOrder() } else { - processNewBlocksInNonLinearOrder(subTreeRootList) + processNewBlocksInNonLinearOrder( + backend = backend, + downloader = downloader, + repository = repository, + network = network, + subTreeRootList = subTreeRootList, + lastValidHeight = lowerBoundHeight, + firstUnenhancedHeight = _processorInfo.value.firstUnenhancedHeight + ) } } // immediately process again after failures in order to download new blocks right away when (result) { BlockProcessingResult.Reconnecting -> { + setState(State.Disconnected) + downloader.reconnect() + val napTime = calculatePollInterval(true) Twig.debug { "Unable to process new blocks because we are disconnected! Attempting to " + @@ -220,7 +233,6 @@ class CompactBlockProcessor internal constructor( } delay(napTime) } - BlockProcessingResult.NoBlocksToProcess -> { // TODO [#1129]: Refactor work with lastSyncRange and lastSyncedHeight // TODO [#1129]: https://github.com/zcash/zcash-android-wallet-sdk/issues/1129 @@ -238,44 +250,16 @@ class CompactBlockProcessor internal constructor( } delay(napTime) } - - is BlockProcessingResult.FailedEnhance -> { + is BlockProcessingResult.SyncFailure -> { Twig.error { - "Failed while enhancing transaction details at height: ${result.error.height} +" + - "with: ${result.error}" - } - checkErrorResult(result.error.height) - } - - is BlockProcessingResult.FailedDeleteBlocks -> { - Twig.error { - "Failed to delete temporary blocks files from the device disk. It will be retried on the" + - " next time, while downloading new blocks." + "Failed while processing blocks at height: ${result.failedAtHeight} with: " + + "${result.error}" } checkErrorResult(result.failedAtHeight) } - - is BlockProcessingResult.FailedDownloadBlocks -> { - Twig.error { "Failed while downloading blocks at height: ${result.failedAtHeight}" } - checkErrorResult(result.failedAtHeight) - } - - is BlockProcessingResult.FailedScanBlocks -> { - Twig.error { "Failed while scanning blocks at height: ${result.failedAtHeight}" } - checkErrorResult(result.failedAtHeight) - } - is BlockProcessingResult.Success -> { // Do nothing. } - - is BlockProcessingResult.DownloadSuccess -> { - // Do nothing. Syncing of blocks is in progress. - } - - BlockProcessingResult.UpdateBirthday -> { - // Do nothing. The birthday was just updated. - } } } } while (_state.value !is State.Stopped) @@ -317,9 +301,7 @@ class CompactBlockProcessor internal constructor( Twig.debug { "Beginning to process new blocks with Linear approach (with lower bound: $lowerBoundHeight)..." } return if (!updateRanges()) { - Twig.debug { "Disconnection detected. Attempting to reconnect." } - setState(State.Disconnected) - downloader.reconnect() + Twig.warn { "Disconnection detected. Attempting to reconnect." } BlockProcessingResult.Reconnecting } else if (_processorInfo.value.lastSyncRange.isNullOrEmpty()) { setState(State.Synced(_processorInfo.value.lastSyncRange)) @@ -349,9 +331,22 @@ class CompactBlockProcessor internal constructor( internal sealed class SuggestScanRangesResult { data class Success(val ranges: List) : SuggestScanRangesResult() + data class Failure(val failedAtHeight: BlockHeight, val exception: Throwable) : SuggestScanRangesResult() + } - // Fix it: consider failed at height parameter and more concrete exception type - data class Failure(val exception: Throwable) : SuggestScanRangesResult() + internal sealed class PutSaplingSubtreeRootsResult { + object Success : PutSaplingSubtreeRootsResult() + data class Failure(val failedAtHeight: BlockHeight, val exception: Throwable) : PutSaplingSubtreeRootsResult() + } + + internal sealed class UpdateChainTipResult { + data class Success(val height: BlockHeight) : UpdateChainTipResult() + data class Failure(val failedAtHeight: BlockHeight, val exception: Throwable) : UpdateChainTipResult() + } + + internal sealed class VerifySuggestedScanRange { + data class ShouldVerify(val scanRange: ScanRange) : VerifySuggestedScanRange() + object NoRangeToVerify : VerifySuggestedScanRange() } @Suppress("MagicNumber") @@ -371,123 +366,152 @@ class CompactBlockProcessor internal constructor( } } - @Suppress("ReturnCount") - private suspend fun processNewBlocksInNonLinearOrder(subTreeRootList: List): BlockProcessingResult { + // TODO [#1137]: Refactor processNewBlocksInNonLinearOrder + // TODO [#1137]: https://github.com/zcash/zcash-android-wallet-sdk/issues/1137 + @OptIn(ExperimentalStdlibApi::class) + @Suppress("ReturnCount", "LongMethod", "CyclomaticComplexMethod", "LongParameterList") + private suspend fun processNewBlocksInNonLinearOrder( + backend: TypesafeBackend, + downloader: CompactBlockDownloader, + repository: DerivedDataRepository, + network: ZcashNetwork, + subTreeRootList: List, + lastValidHeight: BlockHeight, + firstUnenhancedHeight: BlockHeight? + ): BlockProcessingResult { Twig.debug { "Beginning to process new blocks with DAG approach (with roots: $subTreeRootList, and lower " + - "bound: $lowerBoundHeight)..." + "bound: $lastValidHeight)..." } // Pass the commitment tree data to the database. - backend.putSaplingSubtreeRoots( - startIndex = 0, - roots = subTreeRootList - ) + when ( + val result = + putSaplingSubtreeRoots( + backend = backend, + startIndex = 0, + subTreeRootList = subTreeRootList, + lastValidHeight = lastValidHeight + ) + ) { + PutSaplingSubtreeRootsResult.Success -> { /* Lets continue to the next step */ } + is PutSaplingSubtreeRootsResult.Failure -> { + return BlockProcessingResult.SyncFailure(result.failedAtHeight, result.exception) + } + } // Download chain tip metadata from lightwalletd val chainTip = fetchLatestBlockHeight( downloader = downloader, network = network ) ?: let { - Twig.debug { "Disconnection detected. Attempting to reconnect." } - setState(State.Disconnected) - downloader.reconnect() + Twig.warn { "Disconnection detected. Attempting to reconnect." } return BlockProcessingResult.Reconnecting } - // Notify the wallet of the updated chain tip. - backend.updateChainTip(chainTip) + // Notify the wallet of the updated chain tip + when ( + val result = + updateChainTip( + backend = backend, + chainTip = chainTip, + lastValidHeight = lastValidHeight + ) + ) { + is UpdateChainTipResult.Success -> { /* Lets continue to the next step */ } + is UpdateChainTipResult.Failure -> { + return BlockProcessingResult.SyncFailure(result.failedAtHeight, result.exception) + } + } // Get the suggested scan ranges from the wallet database - when (val suggestedRangesResult = suggestScanRanges(backend)) { + var suggestedRangesResult = suggestScanRanges( + backend, + lastValidHeight + ) + when (suggestedRangesResult) { + is SuggestScanRangesResult.Success -> { /* Lets continue to the next step */ } is SuggestScanRangesResult.Failure -> { - Twig.error { "Process suggested scan ranges failure: ${suggestedRangesResult.exception}" } - // Fix it: process failure + Twig.error { + "Process suggested scan ranges failure: " + + "${(suggestedRangesResult as SuggestScanRangesResult.Failure).exception}" + } + return BlockProcessingResult.SyncFailure( + suggestedRangesResult.failedAtHeight, + suggestedRangesResult.exception + ) } - is SuggestScanRangesResult.Success -> { - Twig.debug { "Process suggested scan ranges result: ${suggestedRangesResult.ranges}" } + } - if (suggestedRangesResult.ranges.isEmpty()) { - // Nothing to sync - break out of the processing - return BlockProcessingResult.NoBlocksToProcess - } else { - val firstRangePriority = SuggestScanRangePriority.fromPriority( - suggestedRangesResult.ranges[0].priority - ) - when (firstRangePriority) { - SuggestScanRangePriority.Verify -> {} - SuggestScanRangePriority.ChainTip -> {} - SuggestScanRangePriority.FoundNote -> {} - SuggestScanRangePriority.OpenAdjacent -> {} - SuggestScanRangePriority.Historic -> {} - SuggestScanRangePriority.Scanned -> {} + // Parse and process ranges. If it recognizes a range with Priority.Verify, it runs the verification part. + var verifyRangeResult = shouldVerifySuggestedScanRanges(suggestedRangesResult) + + Twig.info { "Check for verification of ranges resulted with: $verifyRangeResult" } + + while (verifyRangeResult is VerifySuggestedScanRange.ShouldVerify) { + Twig.info { "Starting verification of range: $verifyRangeResult" } + + // Remove existing blocks as they'll be re-downloaded + downloader.rewindToHeight(verifyRangeResult.scanRange.range.start) + deleteAllBlockFiles( + downloader = downloader, + lastKnownHeight = verifyRangeResult.scanRange.range.start + ) + + var syncingResult: SyncingResult = SyncingResult.AllSuccess + runSyncingAndEnhancing( + backend = backend, + downloader = downloader, + repository = repository, + network = network, + syncRange = verifyRangeResult.scanRange.range.toClosedRange(), + withDownload = true, + enhanceStartHeight = firstUnenhancedHeight + ).collect { syncProgress -> + _progress.value = syncProgress.percentage + updateProgress(lastSyncedHeight = syncProgress.lastSyncedHeight) + + when (syncProgress.resultState) { + SyncingResult.UpdateBirthday -> { + updateBirthdayHeight() } + is SyncingResult.Failure -> { + syncingResult = syncProgress.resultState + return@collect + } else -> { + // Continue with processing + } + } + } + + if (syncingResult != SyncingResult.AllSuccess) { + // Remove persisted but not scanned blocks in case of any failure + val lastScannedHeight = getLastScannedHeight(repository) + downloader.rewindToHeight(lastScannedHeight) + deleteAllBlockFiles( + downloader = downloader, + lastKnownHeight = lastScannedHeight + ) + return (syncingResult as SyncingResult.Failure).toBlockProcessingResult() + } + + // Re-request suggested scan ranges + suggestedRangesResult = suggestScanRanges(backend, lowerBoundHeight) + when (suggestedRangesResult) { + is SuggestScanRangesResult.Success -> { + verifyRangeResult = shouldVerifySuggestedScanRanges(suggestedRangesResult) + } + is SuggestScanRangesResult.Failure -> { + Twig.error { "Process suggested scan ranges failure: ${suggestedRangesResult.exception}" } + return BlockProcessingResult.SyncFailure( + suggestedRangesResult.failedAtHeight, + suggestedRangesResult.exception + ) } } } - // Run the following loop until the wallet's view of the chain tip as of the previous wallet session is valid - // loop { - // // If there is a range of blocks that needs to be verified, it will always be returned as - // // the first element of the vector of suggested ranges. - // match scan_ranges.first() { - // Some(scan_range) if scan_range.priority() == ScanPriority::Verify => { - // // Download the blocks in `scan_range` into the block source, overwriting any - // // existing blocks in this range. - // unimplemented!(); - // - // // Scan the downloaded blocks - // let scan_result = scan_cached_blocks( - // &network, - // &block_source, - // &mut wallet_db, - // scan_range.block_range().start, - // scan_range.len() - // ); - // - // // Check for scanning errors that indicate that the wallet's chain tip is out of - // // sync with blockchain history. - // match scan_result { - // Ok(_) => { - // // At this point, the cache and scanned data are locally consistent (though - // // not necessarily consistent with the latest chain tip - this would be - // // discovered the next time this codepath is executed after new blocks are - // // received) so we can break out of the loop. - // break; - // } - // Err(Error::Scan(err)) if err.is_continuity_error() => { - // // Pick a height to rewind to, which must be at least one block before - // // the height at which the error occurred, but may be an earlier height - // // determined based on heuristics such as the platform, available bandwidth, - // // size of recent CompactBlocks, etc. - // let rewind_height = err.at_height().saturating_sub(10); - // - // // Rewind to the chosen height. - // wallet_db.truncate_to_height(rewind_height).map_err(Error::Wallet)?; - // - // // Delete cached blocks from rewind_height onwards. - // // - // // This does imply that assumed-valid blocks will be re-downloaded, but it - // // is also possible that in the intervening time, a chain reorg has - // // occurred that orphaned some of those blocks. - // unimplemented!(); - // } - // Err(other) => { - // // Handle or return other errors - // } - // } - // - // // Truncation will have updated the suggested scan ranges, so we now - // // re_request - // scan_ranges = wallet_db.suggest_scan_ranges().map_err(Error::Wallet)?; - // } - // _ => { - // // Nothing to verify; break out of the loop - // break; - // } - // } - // } - + // Note: this will be replaced with a suitable result at the end return BlockProcessingResult.NoBlocksToProcess } @@ -500,7 +524,7 @@ class CompactBlockProcessor internal constructor( _state.value = State.Syncing // Syncing last blocks and enhancing transactions - var syncResult: BlockProcessingResult = BlockProcessingResult.Success + var syncingResult: SyncingResult = SyncingResult.AllSuccess runSyncingAndEnhancing( backend = backend, downloader = downloader, @@ -513,15 +537,20 @@ class CompactBlockProcessor internal constructor( _progress.value = syncProgress.percentage updateProgress(lastSyncedHeight = syncProgress.lastSyncedHeight) - if (syncProgress.result == BlockProcessingResult.UpdateBirthday) { - updateBirthdayHeight() - } else if (syncProgress.result != BlockProcessingResult.Success) { - syncResult = syncProgress.result - return@collect + when (syncProgress.resultState) { + SyncingResult.UpdateBirthday -> { + updateBirthdayHeight() + } + is SyncingResult.Failure -> { + syncingResult = syncProgress.resultState + return@collect + } else -> { + // Continue with processing + } } } - if (syncResult != BlockProcessingResult.Success) { + if (syncingResult != SyncingResult.AllSuccess) { // Remove persisted but not scanned blocks in case of any failure val lastScannedHeight = getLastScannedHeight(repository) downloader.rewindToHeight(lastScannedHeight) @@ -529,8 +558,7 @@ class CompactBlockProcessor internal constructor( downloader = downloader, lastKnownHeight = lastScannedHeight ) - - return syncResult + return (syncingResult as SyncingResult.Failure).toBlockProcessingResult() } return BlockProcessingResult.Success @@ -539,14 +567,8 @@ class CompactBlockProcessor internal constructor( sealed class BlockProcessingResult { object NoBlocksToProcess : BlockProcessingResult() object Success : BlockProcessingResult() - data class DownloadSuccess(val downloadedBlocks: List?) : BlockProcessingResult() - object UpdateBirthday : BlockProcessingResult() object Reconnecting : BlockProcessingResult() - data class FailedDownloadBlocks(val failedAtHeight: BlockHeight) : BlockProcessingResult() - data class FailedScanBlocks(val failedAtHeight: BlockHeight) : BlockProcessingResult() - data class FailedDeleteBlocks(val failedAtHeight: BlockHeight) : BlockProcessingResult() - data class FailedEnhance(val error: CompactBlockProcessorException.EnhanceTransactionError) : - BlockProcessingResult() + data class SyncFailure(val failedAtHeight: BlockHeight, val error: Throwable) : BlockProcessingResult() } /** @@ -882,8 +904,10 @@ class CompactBlockProcessor internal constructor( retryUpToAndContinue(GET_SUBTREE_ROOTS_RETRIES) { subTreeRootList = downloader.getSubtreeRoots( - startIndex = 1, - maxEntries = 0, + // TODO [#1133]: DAG: Set the correct getSubtreeRoots inputs + // TODO [#1133]: https://github.com/zcash/zcash-android-wallet-sdk/issues/1133 + startIndex = 0, + maxEntries = 65536, shieldedProtocol = ShieldedProtocolEnum.SAPLING ).onEach { response -> when (response) { @@ -913,8 +937,81 @@ class CompactBlockProcessor internal constructor( return subTreeRootList } + /** + * Pass the commitment tree data to the database. + * + * @param backend Typesafe Rust backend + * @param startIndex Index to which put the data + * @param lastValidHeight The height to which rewind in case of any trouble + * @return PutSaplingSubtreeRootsResult + */ @VisibleForTesting - internal suspend fun suggestScanRanges(backend: TypesafeBackend): SuggestScanRangesResult { + internal suspend fun putSaplingSubtreeRoots( + backend: TypesafeBackend, + startIndex: Long = 0, + subTreeRootList: List, + lastValidHeight: BlockHeight + ): PutSaplingSubtreeRootsResult { + return runCatching { + backend.putSaplingSubtreeRoots( + startIndex = startIndex, + roots = subTreeRootList + ) + } + .onSuccess { + Twig.info { + "Sapling subtree roots put successfully with startIndex: $startIndex and roots: " + + "${subTreeRootList.size}" + } + } + .onFailure { + Twig.error { "Sapling subtree roots put failed with: $it" } + }.fold( + onSuccess = { PutSaplingSubtreeRootsResult.Success }, + onFailure = { PutSaplingSubtreeRootsResult.Failure(lastValidHeight, it) } + ) + } + + /** + * Notify the wallet of the updated chain tip. + * + * @param backend Typesafe Rust backend + * @param chainTip Height of latest block + * @param lastValidHeight The height to which rewind in case of any trouble + * @return UpdateChainTipResult + */ + @VisibleForTesting + internal suspend fun updateChainTip( + backend: TypesafeBackend, + chainTip: BlockHeight, + lastValidHeight: BlockHeight + ): UpdateChainTipResult { + return runCatching { + backend.updateChainTip(chainTip) + } + .onSuccess { + Twig.info { "Chain tip updated successfully with height: $chainTip" } + } + .onFailure { + Twig.info { "Chain tip update failed with: $it" } + }.fold( + onSuccess = { UpdateChainTipResult.Success(chainTip) }, + onFailure = { UpdateChainTipResult.Failure(lastValidHeight, it) } + ) + } + + /** + * Get the suggested scan ranges from the wallet database. + * + * @param backend Typesafe Rust backend + * @param lastValidHeight The height to which rewind in case of any trouble + * @return SuggestScanRangesResult + */ + @VisibleForTesting + internal suspend fun suggestScanRanges( + backend: TypesafeBackend, + lastValidHeight: BlockHeight + ): SuggestScanRangesResult { return runCatching { backend.suggestScanRanges() }.onSuccess { ranges -> @@ -923,10 +1020,76 @@ class CompactBlockProcessor internal constructor( Twig.error { "Failed to get newly suggested ranges with: $exception" } }.fold( onSuccess = { SuggestScanRangesResult.Success(it) }, - onFailure = { SuggestScanRangesResult.Failure(it) } + onFailure = { SuggestScanRangesResult.Failure(lastValidHeight, it) } ) } + /** + * Parse and process ranges. If it recognizes a range with Priority.Verify at the first position, it runs the + * verification part. + * + * @param suggestedRangesResult Wrapper for list of ranges to process + * @return VerifySuggestedScanRange + */ + @VisibleForTesting + internal fun shouldVerifySuggestedScanRanges( + suggestedRangesResult: SuggestScanRangesResult.Success + ): VerifySuggestedScanRange { + Twig.debug { "Check for Priority.Verify scan range result: ${suggestedRangesResult.ranges}" } + + return if (suggestedRangesResult.ranges.isEmpty()) { + VerifySuggestedScanRange.NoRangeToVerify + } else { + val firstRangePriority = SuggestScanRangePriority.fromPriority( + suggestedRangesResult.ranges[0].priority + ) + if (firstRangePriority == SuggestScanRangePriority.Verify) { + VerifySuggestedScanRange.ShouldVerify(suggestedRangesResult.ranges[0]) + } else { + VerifySuggestedScanRange.NoRangeToVerify + } + } + } + + @VisibleForTesting + internal sealed class SyncingResult { + object AllSuccess : SyncingResult() + data class DownloadSuccess(val downloadedBlocks: List?) : SyncingResult() + interface Failure { + val failedAtHeight: BlockHeight + val exception: CompactBlockProcessorException + fun toBlockProcessingResult(): BlockProcessingResult = + BlockProcessingResult.SyncFailure( + this.failedAtHeight, + this.exception + ) + } + data class DownloadFailed( + override val failedAtHeight: BlockHeight, + override val exception: CompactBlockProcessorException + ) : Failure, SyncingResult() + object ScanSuccess : SyncingResult() + data class ScanFailed( + override val failedAtHeight: BlockHeight, + override val exception: CompactBlockProcessorException + ) : Failure, SyncingResult() + object DeleteSuccess : SyncingResult() + data class DeleteFailed( + override val failedAtHeight: BlockHeight, + override val exception: CompactBlockProcessorException + ) : Failure, SyncingResult() + object EnhanceSuccess : SyncingResult() + data class EnhanceFailed( + override val failedAtHeight: BlockHeight, + override val exception: CompactBlockProcessorException + ) : Failure, SyncingResult() + object UpdateBirthday : SyncingResult() + data class ContinuityError( + override val failedAtHeight: BlockHeight, + override val exception: CompactBlockProcessorException + ) : Failure, SyncingResult() + } + /** * Requests, processes and persists all blocks from the given range. * @@ -959,7 +1122,7 @@ class CompactBlockProcessor internal constructor( BatchSyncProgress( percentage = PercentDecimal.ONE_HUNDRED_PERCENT, lastSyncedHeight = getLastScannedHeight(repository), - result = BlockProcessingResult.Success + resultState = SyncingResult.AllSuccess ) ) } else { @@ -986,13 +1149,13 @@ class CompactBlockProcessor internal constructor( batch = it ) } else { - BlockProcessingResult.DownloadSuccess(null) + SyncingResult.DownloadSuccess(null) } ) }.buffer(1).map { downloadStageResult -> Twig.debug { "Download stage done with result: $downloadStageResult" } - if (downloadStageResult.stageResult !is BlockProcessingResult.DownloadSuccess) { + if (downloadStageResult.stageResult !is SyncingResult.DownloadSuccess) { // In case of any failure, we just propagate the result downloadStageResult } else { @@ -1011,7 +1174,7 @@ class CompactBlockProcessor internal constructor( }.map { scanResult -> Twig.debug { "Scan stage done with result: $scanResult" } - if (scanResult.stageResult != BlockProcessingResult.Success) { + if (scanResult.stageResult != SyncingResult.ScanSuccess) { scanResult } else { // Run deletion stage @@ -1026,11 +1189,17 @@ class CompactBlockProcessor internal constructor( }.onEach { continuousResult -> Twig.debug { "Deletion stage done with result: $continuousResult" } + var resultState = if (continuousResult.stageResult == SyncingResult.DeleteSuccess) { + SyncingResult.AllSuccess + } else { + continuousResult.stageResult + } + emit( BatchSyncProgress( percentage = PercentDecimal(continuousResult.batch.order / batches.size.toFloat()), lastSyncedHeight = getLastScannedHeight(repository), - result = continuousResult.stageResult + resultState = resultState ) ) @@ -1040,7 +1209,7 @@ class CompactBlockProcessor internal constructor( // Enhance is run in case of the range is on or over its limit, or in case of any failure // state comes from the previous stages, or if the end of the sync range is reached if (enhancingRange.length() >= ENHANCE_BATCH_SIZE || - continuousResult.stageResult != BlockProcessingResult.Success || + resultState != SyncingResult.AllSuccess || continuousResult.batch.order == batches.size.toLong() ) { // Copy the range for use and reset for the next iteration @@ -1053,33 +1222,36 @@ class CompactBlockProcessor internal constructor( downloader = downloader ).collect { enhancingResult -> Twig.debug { "Enhancing result: $enhancingResult" } - // TODO [#1047]: CompactBlockProcessor: Consider a separate sub-stage result handling - // TODO [#1047]: https://github.com/zcash/zcash-android-wallet-sdk/issues/1047 - when (enhancingResult) { - is BlockProcessingResult.UpdateBirthday -> { + resultState = when (enhancingResult) { + is SyncingResult.UpdateBirthday -> { Twig.debug { "Birthday height update reporting" } + enhancingResult } - is BlockProcessingResult.FailedEnhance -> { + is SyncingResult.EnhanceFailed -> { Twig.error { "Enhancing failed for: $enhancingRange with $enhancingResult" } + enhancingResult } else -> { - // Transactions enhanced correctly + // Transactions enhanced correctly. Now we return common sync success state. + SyncingResult.AllSuccess } } emit( BatchSyncProgress( percentage = PercentDecimal(continuousResult.batch.order / batches.size.toFloat()), lastSyncedHeight = getLastScannedHeight(repository), - result = enhancingResult + resultState = resultState ) ) } } - - Twig.debug { "All sync stages done for the batch: ${continuousResult.batch}" } + Twig.debug { + "All sync stages done for the batch: ${continuousResult.batch} with result state: " + + "$resultState" + } }.takeWhile { batchProcessResult -> - batchProcessResult.stageResult == BlockProcessingResult.Success || - batchProcessResult.stageResult == BlockProcessingResult.UpdateBirthday + batchProcessResult.stageResult == SyncingResult.DeleteSuccess || + batchProcessResult.stageResult == SyncingResult.UpdateBirthday }.collect() } } @@ -1121,33 +1293,42 @@ class CompactBlockProcessor internal constructor( * @param batch the batch of blocks to download. */ @VisibleForTesting - @Throws(CompactBlockProcessorException.FailedDownload::class) - @Suppress("MagicNumber") internal suspend fun downloadBatchOfBlocks( downloader: CompactBlockDownloader, batch: BlockBatch - ): BlockProcessingResult { + ): SyncingResult { var downloadedBlocks = listOf() - retryUpTo(RETRIES, { CompactBlockProcessorException.FailedDownload(it) }) { failedAttempts -> + var downloadException: CompactBlockProcessorException.FailedDownloadException? = null + + retryUpToAndContinue( + retries = RETRIES, + exceptionWrapper = { + downloadException = CompactBlockProcessorException.FailedDownloadException(it) + downloadException!! + } + ) { failedAttempts -> + @Suppress("MagicNumber") if (failedAttempts == 0) { Twig.verbose { "Starting to download batch $batch" } } else { Twig.warn { "Retrying to download batch $batch after $failedAttempts failure(s)..." } } - downloadedBlocks = downloader.downloadBlockRange(batch.range) } Twig.verbose { "Successfully downloaded batch: $batch of $downloadedBlocks blocks" } return if (downloadedBlocks.isNotEmpty()) { - BlockProcessingResult.DownloadSuccess(downloadedBlocks) + SyncingResult.DownloadSuccess(downloadedBlocks) } else { - BlockProcessingResult.FailedDownloadBlocks(batch.range.start) + SyncingResult.DownloadFailed( + batch.range.start, + downloadException ?: CompactBlockProcessorException.FailedDownloadException() + ) } } @VisibleForTesting - internal suspend fun scanBatchOfBlocks(batch: BlockBatch, backend: TypesafeBackend): BlockProcessingResult { + internal suspend fun scanBatchOfBlocks(batch: BlockBatch, backend: TypesafeBackend): SyncingResult { return runCatching { backend.scanBlocks(batch.range.start, batch.range.length()) }.onSuccess { @@ -1155,8 +1336,21 @@ class CompactBlockProcessor internal constructor( }.onFailure { Twig.error { "Failed while scanning batch $batch with $it" } }.fold( - onSuccess = { BlockProcessingResult.Success }, - onFailure = { BlockProcessingResult.FailedScanBlocks(batch.range.start) } + onSuccess = { SyncingResult.ScanSuccess }, + onFailure = { + // Check if the error is continuity type + if (it.isScanContinuityError()) { + SyncingResult.ContinuityError( + failedAtHeight = batch.range.start - 1, // To ensure we later rewind below the failed height + exception = CompactBlockProcessorException.FailedScanException(it) + ) + } else { + SyncingResult.ScanFailed( + failedAtHeight = batch.range.start, + exception = CompactBlockProcessorException.FailedScanException(it) + ) + } + } ) } @@ -1164,13 +1358,16 @@ class CompactBlockProcessor internal constructor( internal suspend fun deleteAllBlockFiles( downloader: CompactBlockDownloader, lastKnownHeight: BlockHeight - ): BlockProcessingResult { + ): SyncingResult { Twig.verbose { "Starting to delete all temporary block files" } return if (downloader.compactBlockRepository.deleteAllCompactBlockFiles()) { Twig.verbose { "Successfully deleted all temporary block files" } - BlockProcessingResult.Success + SyncingResult.DeleteSuccess } else { - BlockProcessingResult.FailedDeleteBlocks(lastKnownHeight) + SyncingResult.DeleteFailed( + lastKnownHeight, + CompactBlockProcessorException.FailedDeleteException() + ) } } @@ -1178,18 +1375,21 @@ class CompactBlockProcessor internal constructor( internal suspend fun deleteFilesOfBatchOfBlocks( batch: BlockBatch, downloader: CompactBlockDownloader - ): BlockProcessingResult { + ): SyncingResult { Twig.verbose { "Starting to delete temporary block files from batch: $batch" } return batch.blocks?.let { blocks -> val deleted = downloader.compactBlockRepository.deleteCompactBlockFiles(blocks) if (deleted) { Twig.verbose { "Successfully deleted all temporary batched block files" } - BlockProcessingResult.Success + SyncingResult.DeleteSuccess } else { - BlockProcessingResult.FailedDeleteBlocks(batch.range.start) + SyncingResult.DeleteFailed( + batch.range.start, + CompactBlockProcessorException.FailedDeleteException() + ) } - } ?: BlockProcessingResult.Success + } ?: SyncingResult.DeleteSuccess } @VisibleForTesting @@ -1198,7 +1398,7 @@ class CompactBlockProcessor internal constructor( repository: DerivedDataRepository, backend: TypesafeBackend, downloader: CompactBlockDownloader - ): Flow = flow { + ): Flow = flow { Twig.debug { "Enhancing transaction details for blocks $range" } val newTxs = repository.findNewTransactions(range) @@ -1210,13 +1410,13 @@ class CompactBlockProcessor internal constructor( // If the first transaction has been added if (newTxs.size.toLong() == repository.getTransactionCount()) { Twig.debug { "Encountered the first transaction. This changes the birthday height!" } - emit(BlockProcessingResult.UpdateBirthday) + emit(SyncingResult.UpdateBirthday) } newTxs.filter { it.minedHeight != null }.onEach { newTransaction -> val trEnhanceResult = enhanceTransaction(newTransaction, backend, downloader) - if (trEnhanceResult is BlockProcessingResult.FailedEnhance) { - Twig.error { "Encountered transaction enhancing error: ${trEnhanceResult.error}" } + if (trEnhanceResult is SyncingResult.EnhanceFailed) { + Twig.error { "Encountered transaction enhancing error: ${trEnhanceResult.exception}" } emit(trEnhanceResult) // We intentionally do not terminate the batch enhancing here, just reporting it } @@ -1224,17 +1424,17 @@ class CompactBlockProcessor internal constructor( } Twig.debug { "Done enhancing transaction details" } - emit(BlockProcessingResult.Success) + emit(SyncingResult.EnhanceSuccess) } private suspend fun enhanceTransaction( transaction: DbTransactionOverview, backend: TypesafeBackend, downloader: CompactBlockDownloader - ): BlockProcessingResult { + ): SyncingResult { Twig.debug { "Starting enhancing transaction (id:${transaction.id} block:${transaction.minedHeight})" } if (transaction.minedHeight == null) { - return BlockProcessingResult.Success + return SyncingResult.EnhanceSuccess } return try { @@ -1259,9 +1459,12 @@ class CompactBlockProcessor internal constructor( ) Twig.debug { "Done enhancing transaction (id:${transaction.id} block:${transaction.minedHeight})" } - BlockProcessingResult.Success - } catch (e: CompactBlockProcessorException.EnhanceTransactionError) { - BlockProcessingResult.FailedEnhance(e) + SyncingResult.EnhanceSuccess + } catch (exception: CompactBlockProcessorException.EnhanceTransactionError) { + SyncingResult.EnhanceFailed( + transaction.minedHeight, + exception + ) } } @@ -1396,11 +1599,11 @@ class CompactBlockProcessor internal constructor( } private suspend fun handleChainError(errorHeight: BlockHeight) { - // TODO [#683]: consider an error object containing hash information + // TODO [#683]: Consider an error object containing hash information // TODO [#683]: https://github.com/zcash/zcash-android-wallet-sdk/issues/683 printValidationErrorInfo(errorHeight) determineLowerBound(errorHeight).let { lowerBound -> - Twig.debug { "handling chain error at $errorHeight by rewinding to block $lowerBound" } + Twig.debug { "Handling chain error at $errorHeight by rewinding to block $lowerBound" } onChainErrorListener?.invoke(errorHeight, lowerBound) rewindToNearestHeight(lowerBound, true) } @@ -1712,7 +1915,7 @@ class CompactBlockProcessor internal constructor( internal data class BatchSyncProgress( val percentage: PercentDecimal, val lastSyncedHeight: BlockHeight?, - val result: BlockProcessingResult + val resultState: SyncingResult ) /** @@ -1720,7 +1923,7 @@ class CompactBlockProcessor internal constructor( */ private data class SyncStageResult( val batch: BlockBatch, - val stageResult: BlockProcessingResult + val stageResult: SyncingResult ) /** diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/exception/Exceptions.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/exception/Exceptions.kt index 5b2809e4..ee3e8d27 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/exception/Exceptions.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/exception/Exceptions.kt @@ -79,13 +79,6 @@ sealed class CompactBlockProcessorException(message: String, cause: Throwable? = null ) class FailedReorgRepair(message: String) : CompactBlockProcessorException(message) - class FailedDownload(cause: Throwable? = null) : CompactBlockProcessorException( - "Error while downloading blocks. This most " + - "likely means the server is down or slow to respond. See logs for details.", - cause - ) - class Disconnected(cause: Throwable? = null) : - CompactBlockProcessorException("Disconnected Error. Unable to download blocks due to ${cause?.message}", cause) object Uninitialized : CompactBlockProcessorException( "Cannot process blocks because the wallet has not been" + " initialized. Verify that the seed phrase was properly created or imported. If so, then this problem" + @@ -95,6 +88,22 @@ sealed class CompactBlockProcessorException(message: String, cause: Throwable? = "Attempting to scan without an account. This is probably a setup error or a race condition." ) + class FailedDownloadException(cause: Throwable? = null) : CompactBlockProcessorException( + "Error while downloading blocks. This most likely means the server is down or slow to respond. " + + "See logs for details.", + cause + ) + class FailedScanException(cause: Throwable? = null) : CompactBlockProcessorException( + "Error while scanning blocks. This most likely means a problem with locally persisted data. " + + "See logs for details.", + cause + ) + class FailedDeleteException(cause: Throwable? = null) : CompactBlockProcessorException( + "Error while deleting block files. This most likely means the data are not persisted correctly." + + " See logs for details.", + cause + ) + open class EnhanceTransactionError( message: String, val height: BlockHeight, diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/Ext.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/ExceptionExt.kt similarity index 59% rename from sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/Ext.kt rename to sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/ExceptionExt.kt index 27a1f7cd..92b40975 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/Ext.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/ExceptionExt.kt @@ -44,3 +44,28 @@ internal inline fun tryWarn( } } } + +internal const val PREV_HASH_MISMATCH = "The parent hash of proposed block does not correspond to the block hash at " + + "height" // $NON-NLS +internal const val BLOCK_HEIGHT_DISCONTINUITY = "Block height discontinuity at height" // $NON-NLS +internal const val TREE_SIZE_MISMATCH = "note commitment tree size provided by a compact block did not match the " + + "expected size at height" // $NON-NLS + +/** + * Check whether this error is the result of a failed continuity while scanning new blocks in the Rust layer. + * + * @return true in case of the check match, false otherwise + */ +internal fun Throwable.isScanContinuityError(): Boolean { + val errorMessages = listOf( + PREV_HASH_MISMATCH, + BLOCK_HEIGHT_DISCONTINUITY, + TREE_SIZE_MISMATCH + ) + errorMessages.forEach { errMessage -> + if (this.message?.lowercase()?.contains(errMessage.lowercase()) == true) { + return true + } + } + return false +} diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/OpenEndRangeExt.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/OpenEndRangeExt.kt new file mode 100644 index 00000000..26ed7c6e --- /dev/null +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/OpenEndRangeExt.kt @@ -0,0 +1,6 @@ +package cash.z.ecc.android.sdk.internal.ext + +import cash.z.ecc.android.sdk.model.BlockHeight + +@OptIn(ExperimentalStdlibApi::class) +internal fun OpenEndRange.toClosedRange(): ClosedRange = start..endExclusive diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/WalletService.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/WalletService.kt index 047aca77..786e31d8 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/WalletService.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/WalletService.kt @@ -5,6 +5,7 @@ import cash.z.ecc.android.sdk.ext.ZcashSdk.MAX_BACKOFF_INTERVAL import cash.z.ecc.android.sdk.internal.Twig import kotlinx.coroutines.delay import java.io.File +import kotlin.math.pow import kotlin.random.Random /** @@ -35,7 +36,7 @@ suspend inline fun retryUpTo( if (failedAttempts > retries) { throw exceptionWrapper(t) } - val duration = (initialDelayMillis.toDouble() * Math.pow(2.0, failedAttempts.toDouble() - 1)).toLong() + val duration = (initialDelayMillis.toDouble() * 2.0.pow(failedAttempts.toDouble() - 1)).toLong() Twig.warn(t) { "Retrying ($failedAttempts/$retries) in ${duration}s..." } delay(duration) } @@ -47,12 +48,15 @@ suspend inline fun retryUpTo( * retries succeed, then leave the block execution unfinished and continue. * * @param retries the number of times to retry the block after the first attempt fails. + * @param exceptionWrapper a function that can wrap the final failure to add more useful information + * * or context. Default behavior is to just return the final exception. * @param initialDelayMillis the initial amount of time to wait before the first retry. * @param block the code to execute, which will be wrapped in a try/catch and retried whenever an * exception is thrown up to [retries] attempts. */ suspend inline fun retryUpToAndContinue( retries: Int, + exceptionWrapper: (Throwable) -> Throwable = { it }, initialDelayMillis: Long = 500L, block: (Int) -> Unit ) { @@ -64,7 +68,10 @@ suspend inline fun retryUpToAndContinue( return } catch (t: Throwable) { failedAttempts++ - val duration = (initialDelayMillis.toDouble() * Math.pow(2.0, failedAttempts.toDouble() - 1)).toLong() + if (failedAttempts > retries) { + exceptionWrapper(t) + } + val duration = (initialDelayMillis.toDouble() * 2.0.pow(failedAttempts.toDouble() - 1)).toLong() Twig.warn(t) { "Retrying ($failedAttempts/$retries) in ${duration}s..." } delay(duration) } @@ -102,10 +109,8 @@ suspend inline fun retryWithBackoff( sequence++ // initialDelay^(sequence/4) + jitter - var duration = Math.pow( - initialDelayMillis.toDouble(), - (sequence.toDouble() / 4.0) - ).toLong() + Random.nextLong(1000L) + var duration = initialDelayMillis.toDouble().pow((sequence.toDouble() / 4.0)).toLong() + + Random.nextLong(1000L) if (duration > maxDelayMillis) { duration = maxDelayMillis - Random.nextLong(1000L) // include jitter but don't exceed max delay sequence /= 2 diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/model/BlockHeight.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/model/BlockHeight.kt index 9e66c184..b15a3c1b 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/model/BlockHeight.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/model/BlockHeight.kt @@ -39,6 +39,24 @@ data class BlockHeight internal constructor(val value: Long) : Comparable= 0) { + "Cannot subtract negative value $other to BlockHeight" + } + + return BlockHeight(value - other.toLong()) + } + + operator fun minus(other: Long): BlockHeight { + require(other >= 0) { + "Cannot subtract negative value $other to BlockHeight" + } + + return BlockHeight(value + other) + } + companion object { private val UINT_RANGE = 0.toLong()..UInt.MAX_VALUE.toLong() diff --git a/sdk-lib/src/test/java/cash/z/ecc/android/sdk/ext/ExceptionExtTest.kt b/sdk-lib/src/test/java/cash/z/ecc/android/sdk/ext/ExceptionExtTest.kt new file mode 100644 index 00000000..fdb69bd3 --- /dev/null +++ b/sdk-lib/src/test/java/cash/z/ecc/android/sdk/ext/ExceptionExtTest.kt @@ -0,0 +1,24 @@ +package cash.z.ecc.android.sdk.ext + +import cash.z.ecc.android.sdk.internal.ext.PREV_HASH_MISMATCH +import cash.z.ecc.android.sdk.internal.ext.isScanContinuityError +import kotlin.test.Test +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +class ExceptionExtTest { + + @Test + fun is_scan_continuity_error() { + assertTrue { + RuntimeException(PREV_HASH_MISMATCH).isScanContinuityError() + } + } + + @Test + fun is_not_scan_continuity_error() { + assertFalse { + RuntimeException("Text").isScanContinuityError() + } + } +} diff --git a/sdk-lib/src/test/java/cash/z/ecc/android/sdk/model/BlockHeightTest.kt b/sdk-lib/src/test/java/cash/z/ecc/android/sdk/model/BlockHeightTest.kt index 4290bceb..42e1c325 100644 --- a/sdk-lib/src/test/java/cash/z/ecc/android/sdk/model/BlockHeightTest.kt +++ b/sdk-lib/src/test/java/cash/z/ecc/android/sdk/model/BlockHeightTest.kt @@ -68,4 +68,26 @@ class BlockHeightTest { ZcashNetwork.Mainnet.saplingActivationHeight + -1L } } + + @Test + fun subtraction_of_block_height_succeeds() { + val one = BlockHeight.new( + ZcashNetwork.Mainnet, + ZcashNetwork.Mainnet.saplingActivationHeight.value + + ZcashNetwork.Mainnet.saplingActivationHeight.value + ) + val two = BlockHeight.new(ZcashNetwork.Mainnet, ZcashNetwork.Mainnet.saplingActivationHeight.value) + + assertEquals(ZcashNetwork.Mainnet.saplingActivationHeight.value, (one - two).value) + } + + @Test + fun subtraction_of_long_succeeds() { + assertEquals(419_323L, (ZcashNetwork.Mainnet.saplingActivationHeight - 123L).value) + } + + @Test + fun subtraction_of_int_succeeds() { + assertEquals(419_323, (ZcashNetwork.Mainnet.saplingActivationHeight + 123).value) + } } From 7d098f67b938943ac50e802fe1fa1bbd354cd139 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Honza=20Rychnovsk=C3=BD?= Date: Mon, 31 Jul 2023 15:19:31 +0200 Subject: [PATCH 15/76] [#1146] Process the rest of the ranges * [#1146] Process the rest of the ranges --- .../sdk/block/CompactBlockProcessor.kt | 68 ++++++++++++++++++- 1 file changed, 65 insertions(+), 3 deletions(-) diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt index 13d1d26f..190a1ddc 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt @@ -502,7 +502,10 @@ class CompactBlockProcessor internal constructor( verifyRangeResult = shouldVerifySuggestedScanRanges(suggestedRangesResult) } is SuggestScanRangesResult.Failure -> { - Twig.error { "Process suggested scan ranges failure: ${suggestedRangesResult.exception}" } + Twig.error { + "Process suggested scan ranges failure: " + + "${(suggestedRangesResult as SuggestScanRangesResult.Failure).exception}" + } return BlockProcessingResult.SyncFailure( suggestedRangesResult.failedAtHeight, suggestedRangesResult.exception @@ -511,8 +514,67 @@ class CompactBlockProcessor internal constructor( } } - // Note: this will be replaced with a suitable result at the end - return BlockProcessingResult.NoBlocksToProcess + // Process the rest of ranges + + // Get the suggested scan ranges from the wallet database + suggestedRangesResult = suggestScanRanges( + backend, + lastValidHeight + ) + val scanRanges = when (suggestedRangesResult) { + is SuggestScanRangesResult.Success -> { suggestedRangesResult.ranges } + is SuggestScanRangesResult.Failure -> { + Twig.error { "Process suggested scan ranges failure: ${suggestedRangesResult.exception}" } + return BlockProcessingResult.SyncFailure( + suggestedRangesResult.failedAtHeight, + suggestedRangesResult.exception + ) + } + } + scanRanges.forEach { scanRange -> + Twig.debug { "Start processing the range: $scanRange" } + + // TODO [#1145]: Sync Historic range in reverse order + // TODO [#1145]: https://github.com/zcash/zcash-android-wallet-sdk/issues/1145 + var syncingResult: SyncingResult = SyncingResult.AllSuccess + runSyncingAndEnhancing( + backend = backend, + downloader = downloader, + repository = repository, + network = network, + syncRange = scanRange.range.toClosedRange(), + withDownload = true, + enhanceStartHeight = firstUnenhancedHeight + ).collect { syncProgress -> + _progress.value = syncProgress.percentage + updateProgress(lastSyncedHeight = syncProgress.lastSyncedHeight) + + when (syncProgress.resultState) { + SyncingResult.UpdateBirthday -> { + updateBirthdayHeight() + } + is SyncingResult.Failure -> { + syncingResult = syncProgress.resultState + return@collect + } else -> { + // Continue with processing + } + } + } + + if (syncingResult != SyncingResult.AllSuccess) { + // Remove persisted but not scanned blocks in case of any failure + val lastScannedHeight = getLastScannedHeight(repository) + downloader.rewindToHeight(lastScannedHeight) + deleteAllBlockFiles( + downloader = downloader, + lastKnownHeight = lastScannedHeight + ) + return (syncingResult as SyncingResult.Failure).toBlockProcessingResult() + } + } + + return BlockProcessingResult.Success } @Suppress("ReturnCount") From 3c31144963206390f49bd34d9b7b1165b4db8ecf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Honza=20Rychnovsk=C3=BD?= Date: Wed, 2 Aug 2023 10:11:04 +0200 Subject: [PATCH 16/76] [#1133] Set the correct getSubtreeRoots inputs * [#1133] Set the correct getSubtreeRoots inputs - It also better handles setState calls - It fixes OpenEndRange -> ClosedRange - And it improves logging in syncing mechanism * Increase build CI timeout --- .github/workflows/pull-request.yml | 2 +- .../sdk/block/CompactBlockProcessor.kt | 51 +++++++++---------- .../sdk/internal/ext/OpenEndRangeExt.kt | 2 +- .../android/sdk/internal/model/ScanRange.kt | 19 ++++++- .../z/ecc/android/sdk/model/ZcashNetwork.kt | 3 ++ .../ecc/android/sdk/model/ZcashNetworkTest.kt | 28 ++++++++++ 6 files changed, 74 insertions(+), 31 deletions(-) create mode 100644 sdk-lib/src/test/java/cash/z/ecc/android/sdk/model/ZcashNetworkTest.kt diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 9a63abd2..1656d313 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -336,7 +336,7 @@ jobs: run: | keytool -genkey -v -keystore $SIGNING_KEY_PATH -keypass android -storepass android -alias androiddebugkey -keyalg RSA -keysize 2048 -validity 100000 -dname "CN=, OU=, O=Test, L=, S=, C=" -noprompt - name: Build - timeout-minutes: 30 + timeout-minutes: 35 env: ORG_GRADLE_PROJECT_ZCASH_RELEASE_KEYSTORE_PATH: ${{ format('{0}/release.jks', env.home) }} ORG_GRADLE_PROJECT_ZCASH_RELEASE_KEYSTORE_PASSWORD: android diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt index 190a1ddc..60255756 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt @@ -29,6 +29,7 @@ import cash.z.ecc.android.sdk.internal.model.DbTransactionOverview import cash.z.ecc.android.sdk.internal.model.JniBlockMeta import cash.z.ecc.android.sdk.internal.model.ScanRange import cash.z.ecc.android.sdk.internal.model.SubtreeRoot +import cash.z.ecc.android.sdk.internal.model.SuggestScanRangePriority import cash.z.ecc.android.sdk.internal.model.ext.from import cash.z.ecc.android.sdk.internal.model.ext.toBlockHeight import cash.z.ecc.android.sdk.internal.repository.DerivedDataRepository @@ -234,6 +235,7 @@ class CompactBlockProcessor internal constructor( delay(napTime) } BlockProcessingResult.NoBlocksToProcess -> { + setState(State.Synced(_processorInfo.value.lastSyncRange)) // TODO [#1129]: Refactor work with lastSyncRange and lastSyncedHeight // TODO [#1129]: https://github.com/zcash/zcash-android-wallet-sdk/issues/1129 val noWorkDone = _processorInfo.value.lastSyncRange?.isEmpty() ?: true @@ -304,9 +306,10 @@ class CompactBlockProcessor internal constructor( Twig.warn { "Disconnection detected. Attempting to reconnect." } BlockProcessingResult.Reconnecting } else if (_processorInfo.value.lastSyncRange.isNullOrEmpty()) { - setState(State.Synced(_processorInfo.value.lastSyncRange)) + Twig.info { "No more blocks to process." } BlockProcessingResult.NoBlocksToProcess } else { + setState(State.Syncing) val syncRange = if (BenchmarkingExt.isBenchmarking()) { // We inject a benchmark test blocks range at this point to process only a restricted range of // blocks for a more reliable benchmark results. @@ -349,23 +352,6 @@ class CompactBlockProcessor internal constructor( object NoRangeToVerify : VerifySuggestedScanRange() } - @Suppress("MagicNumber") - internal enum class SuggestScanRangePriority(val priority: Long) { - Scanned(10), - Historic(20), - OpenAdjacent(30), - FoundNote(40), - ChainTip(50), - Verify(60); - - companion object { - fun fromPriority(priority: Long): SuggestScanRangePriority { - Twig.verbose { "Current suggested scan range priority: $priority" } - return values().firstOrNull { it.priority == priority } ?: Scanned - } - } - } - // TODO [#1137]: Refactor processNewBlocksInNonLinearOrder // TODO [#1137]: https://github.com/zcash/zcash-android-wallet-sdk/issues/1137 @OptIn(ExperimentalStdlibApi::class) @@ -424,6 +410,8 @@ class CompactBlockProcessor internal constructor( } } + setState(State.Syncing) + // Get the suggested scan ranges from the wallet database var suggestedRangesResult = suggestScanRanges( backend, @@ -583,8 +571,6 @@ class CompactBlockProcessor internal constructor( withDownload: Boolean, enhanceStartHeight: BlockHeight? ): BlockProcessingResult { - _state.value = State.Syncing - // Syncing last blocks and enhancing transactions var syncingResult: SyncingResult = SyncingResult.AllSuccess runSyncingAndEnhancing( @@ -969,15 +955,22 @@ class CompactBlockProcessor internal constructor( // TODO [#1133]: DAG: Set the correct getSubtreeRoots inputs // TODO [#1133]: https://github.com/zcash/zcash-android-wallet-sdk/issues/1133 startIndex = 0, - maxEntries = 65536, + maxEntries = if (network.isTestnet()) { + 65536 + } else { + 0 + }, shieldedProtocol = ShieldedProtocolEnum.SAPLING ).onEach { response -> when (response) { is Response.Success -> { - Twig.debug { "SubtreeRoots got successfully" } + Twig.verbose { + "SubtreeRoot got successfully: it's completingHeight: ${response.result + .completingBlockHeight}" + } } is Response.Failure -> { - Twig.error { "Fetching SubtreeRoots failed with: ${response.toThrowable()}" } + Twig.error { "Fetching SubtreeRoot failed with: ${response.toThrowable()}" } throw LightWalletException.GetSubtreeRootsException( response.code, response.description, @@ -1102,9 +1095,7 @@ class CompactBlockProcessor internal constructor( return if (suggestedRangesResult.ranges.isEmpty()) { VerifySuggestedScanRange.NoRangeToVerify } else { - val firstRangePriority = SuggestScanRangePriority.fromPriority( - suggestedRangesResult.ranges[0].priority - ) + val firstRangePriority = suggestedRangesResult.ranges[0].getSuggestScanRangePriority() if (firstRangePriority == SuggestScanRangePriority.Verify) { VerifySuggestedScanRange.ShouldVerify(suggestedRangesResult.ranges[0]) } else { @@ -1116,7 +1107,11 @@ class CompactBlockProcessor internal constructor( @VisibleForTesting internal sealed class SyncingResult { object AllSuccess : SyncingResult() - data class DownloadSuccess(val downloadedBlocks: List?) : SyncingResult() + data class DownloadSuccess(val downloadedBlocks: List?) : SyncingResult() { + override fun toString(): String { + return "DownloadSuccess with ${downloadedBlocks?.size ?: "none"} blocks" + } + } interface Failure { val failedAtHeight: BlockHeight val exception: CompactBlockProcessorException @@ -1188,7 +1183,7 @@ class CompactBlockProcessor internal constructor( ) ) } else { - Twig.debug { "Syncing blocks in range $syncRange" } + Twig.info { "Syncing blocks in range $syncRange" } val batches = getBatchedBlockList(syncRange, network) diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/OpenEndRangeExt.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/OpenEndRangeExt.kt index 26ed7c6e..dcd2ad4b 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/OpenEndRangeExt.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/OpenEndRangeExt.kt @@ -3,4 +3,4 @@ package cash.z.ecc.android.sdk.internal.ext import cash.z.ecc.android.sdk.model.BlockHeight @OptIn(ExperimentalStdlibApi::class) -internal fun OpenEndRange.toClosedRange(): ClosedRange = start..endExclusive +internal fun OpenEndRange.toClosedRange(): ClosedRange = start..endExclusive - 1 diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanRange.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanRange.kt index 7d6afc15..e6aee321 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanRange.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanRange.kt @@ -1,5 +1,6 @@ package cash.z.ecc.android.sdk.internal.model +import cash.z.ecc.android.sdk.internal.Twig import cash.z.ecc.android.sdk.model.BlockHeight import cash.z.ecc.android.sdk.model.ZcashNetwork @@ -8,7 +9,13 @@ internal data class ScanRange( val range: OpenEndRange, val priority: Long ) { - override fun toString() = "ScanRange(range=$range, priority=$priority)" + override fun toString() = "ScanRange(range=$range, priority=${getSuggestScanRangePriority()})" + + internal fun getSuggestScanRangePriority(): SuggestScanRangePriority { + Twig.verbose { "Current suggested scan range priority: $priority" } + return SuggestScanRangePriority.values() + .firstOrNull { it.priority == priority } ?: SuggestScanRangePriority.Scanned + } companion object { fun new(jni: JniScanRange, zcashNetwork: ZcashNetwork): ScanRange { @@ -19,3 +26,13 @@ internal data class ScanRange( } } } + +@Suppress("MagicNumber") +internal enum class SuggestScanRangePriority(val priority: Long) { + Scanned(10), + Historic(20), + OpenAdjacent(30), + FoundNote(40), + ChainTip(50), + Verify(60) +} diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/model/ZcashNetwork.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/model/ZcashNetwork.kt index 2844a6a8..ead0b1c3 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/model/ZcashNetwork.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/model/ZcashNetwork.kt @@ -11,6 +11,9 @@ data class ZcashNetwork( val saplingActivationHeight: BlockHeight, val orchardActivationHeight: BlockHeight ) { + fun isMainnet() = id == ID_MAINNET + + fun isTestnet() = id == ID_TESTNET @Suppress("MagicNumber") companion object { diff --git a/sdk-lib/src/test/java/cash/z/ecc/android/sdk/model/ZcashNetworkTest.kt b/sdk-lib/src/test/java/cash/z/ecc/android/sdk/model/ZcashNetworkTest.kt new file mode 100644 index 00000000..b20638be --- /dev/null +++ b/sdk-lib/src/test/java/cash/z/ecc/android/sdk/model/ZcashNetworkTest.kt @@ -0,0 +1,28 @@ +package cash.z.ecc.android.sdk.model + +import kotlin.test.Test +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +class ZcashNetworkTest { + + @Test + fun is_mainnet_succeed_test() { + assertTrue { ZcashNetwork.Mainnet.isMainnet() } + } + + @Test + fun is_mainnet_fail_test() { + assertFalse { ZcashNetwork.Testnet.isMainnet() } + } + + @Test + fun is_testnet_succeed_test() { + assertTrue { ZcashNetwork.Testnet.isTestnet() } + } + + @Test + fun is_testnet_fail_test() { + assertFalse { ZcashNetwork.Mainnet.isTestnet() } + } +} From 54d016e401702a38aed83ec83340970f9ccbd893 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Mon, 7 Aug 2023 19:17:28 +0000 Subject: [PATCH 17/76] Migrate to Rust revision with checkpoint and memo bugfixes --- backend-lib/Cargo.lock | 17 +++-- backend-lib/Cargo.toml | 10 +-- .../z/ecc/android/sdk/internal/Backend.kt | 15 +---- .../android/sdk/internal/jni/RustBackend.kt | 36 ++++------ backend-lib/src/main/rust/lib.rs | 66 +++++++------------ backend-lib/src/main/rust/utils.rs | 15 ++++- .../cash/z/ecc/fixture/FakeRustBackend.kt | 9 +-- .../cash/z/ecc/android/sdk/SdkSynchronizer.kt | 12 +--- .../z/ecc/android/sdk/exception/Exceptions.kt | 3 +- .../android/sdk/internal/TypesafeBackend.kt | 9 ++- .../sdk/internal/TypesafeBackendImpl.kt | 30 +++++---- .../db/derived/DbDerivedDataRepository.kt | 8 ++- .../internal/db/derived/TransactionTable.kt | 10 ++- .../sdk/internal/db/derived/TxOutputsView.kt | 6 +- .../repository/DerivedDataRepository.kt | 5 +- .../transaction/TransactionEncoderImpl.kt | 9 +-- 16 files changed, 114 insertions(+), 146 deletions(-) diff --git a/backend-lib/Cargo.lock b/backend-lib/Cargo.lock index 3bbad617..c2e1c9c9 100644 --- a/backend-lib/Cargo.lock +++ b/backend-lib/Cargo.lock @@ -483,7 +483,7 @@ checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" [[package]] name = "equihash" version = "0.2.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=facd4ccac56bb646917513c2d87f5e8d8b59750c#facd4ccac56bb646917513c2d87f5e8d8b59750c" +source = "git+https://github.com/zcash/librustzcash.git?rev=81d1928497683cdd3bb97e1411f66d3b2715dd85#81d1928497683cdd3bb97e1411f66d3b2715dd85" dependencies = [ "blake2b_simd", "byteorder", @@ -513,7 +513,7 @@ dependencies = [ [[package]] name = "f4jumble" version = "0.1.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=facd4ccac56bb646917513c2d87f5e8d8b59750c#facd4ccac56bb646917513c2d87f5e8d8b59750c" +source = "git+https://github.com/zcash/librustzcash.git?rev=81d1928497683cdd3bb97e1411f66d3b2715dd85#81d1928497683cdd3bb97e1411f66d3b2715dd85" dependencies = [ "blake2b_simd", ] @@ -2258,7 +2258,7 @@ dependencies = [ [[package]] name = "zcash_address" version = "0.3.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=facd4ccac56bb646917513c2d87f5e8d8b59750c#facd4ccac56bb646917513c2d87f5e8d8b59750c" +source = "git+https://github.com/zcash/librustzcash.git?rev=81d1928497683cdd3bb97e1411f66d3b2715dd85#81d1928497683cdd3bb97e1411f66d3b2715dd85" dependencies = [ "bech32", "bs58", @@ -2269,7 +2269,7 @@ dependencies = [ [[package]] name = "zcash_client_backend" version = "0.9.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=facd4ccac56bb646917513c2d87f5e8d8b59750c#facd4ccac56bb646917513c2d87f5e8d8b59750c" +source = "git+https://github.com/zcash/librustzcash.git?rev=81d1928497683cdd3bb97e1411f66d3b2715dd85#81d1928497683cdd3bb97e1411f66d3b2715dd85" dependencies = [ "base64", "bech32", @@ -2302,11 +2302,10 @@ dependencies = [ [[package]] name = "zcash_client_sqlite" version = "0.7.1" -source = "git+https://github.com/zcash/librustzcash.git?rev=facd4ccac56bb646917513c2d87f5e8d8b59750c#facd4ccac56bb646917513c2d87f5e8d8b59750c" +source = "git+https://github.com/zcash/librustzcash.git?rev=81d1928497683cdd3bb97e1411f66d3b2715dd85#81d1928497683cdd3bb97e1411f66d3b2715dd85" dependencies = [ "bs58", "byteorder", - "either", "group", "hdwallet 0.4.1", "incrementalmerkletree", @@ -2329,7 +2328,7 @@ dependencies = [ [[package]] name = "zcash_encoding" version = "0.2.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=facd4ccac56bb646917513c2d87f5e8d8b59750c#facd4ccac56bb646917513c2d87f5e8d8b59750c" +source = "git+https://github.com/zcash/librustzcash.git?rev=81d1928497683cdd3bb97e1411f66d3b2715dd85#81d1928497683cdd3bb97e1411f66d3b2715dd85" dependencies = [ "byteorder", "nonempty", @@ -2351,7 +2350,7 @@ dependencies = [ [[package]] name = "zcash_primitives" version = "0.12.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=facd4ccac56bb646917513c2d87f5e8d8b59750c#facd4ccac56bb646917513c2d87f5e8d8b59750c" +source = "git+https://github.com/zcash/librustzcash.git?rev=81d1928497683cdd3bb97e1411f66d3b2715dd85#81d1928497683cdd3bb97e1411f66d3b2715dd85" dependencies = [ "aes", "bip0039", @@ -2386,7 +2385,7 @@ dependencies = [ [[package]] name = "zcash_proofs" version = "0.12.1" -source = "git+https://github.com/zcash/librustzcash.git?rev=facd4ccac56bb646917513c2d87f5e8d8b59750c#facd4ccac56bb646917513c2d87f5e8d8b59750c" +source = "git+https://github.com/zcash/librustzcash.git?rev=81d1928497683cdd3bb97e1411f66d3b2715dd85#81d1928497683cdd3bb97e1411f66d3b2715dd85" dependencies = [ "bellman", "blake2b_simd", diff --git a/backend-lib/Cargo.toml b/backend-lib/Cargo.toml index fbd251cf..4ed3e727 100644 --- a/backend-lib/Cargo.toml +++ b/backend-lib/Cargo.toml @@ -44,11 +44,11 @@ libc = "0.2" incrementalmerkletree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "da97e6c399c5acddedad2c1730dbb7ee55499a2f" } shardtree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "da97e6c399c5acddedad2c1730dbb7ee55499a2f" } orchard = { git = "https://github.com/zcash/orchard.git", rev = "6ef89d5f154de2cf7b7dd87edb8d8c49158beebb" } -zcash_address = { git = "https://github.com/zcash/librustzcash.git", rev = "facd4ccac56bb646917513c2d87f5e8d8b59750c" } -zcash_client_backend = { git = "https://github.com/zcash/librustzcash.git", rev = "facd4ccac56bb646917513c2d87f5e8d8b59750c" } -zcash_client_sqlite = { git = "https://github.com/zcash/librustzcash.git", rev = "facd4ccac56bb646917513c2d87f5e8d8b59750c" } -zcash_primitives = { git = "https://github.com/zcash/librustzcash.git", rev = "facd4ccac56bb646917513c2d87f5e8d8b59750c" } -zcash_proofs = { git = "https://github.com/zcash/librustzcash.git", rev = "facd4ccac56bb646917513c2d87f5e8d8b59750c" } +zcash_address = { git = "https://github.com/zcash/librustzcash.git", rev = "81d1928497683cdd3bb97e1411f66d3b2715dd85" } +zcash_client_backend = { git = "https://github.com/zcash/librustzcash.git", rev = "81d1928497683cdd3bb97e1411f66d3b2715dd85" } +zcash_client_sqlite = { git = "https://github.com/zcash/librustzcash.git", rev = "81d1928497683cdd3bb97e1411f66d3b2715dd85" } +zcash_primitives = { git = "https://github.com/zcash/librustzcash.git", rev = "81d1928497683cdd3bb97e1411f66d3b2715dd85" } +zcash_proofs = { git = "https://github.com/zcash/librustzcash.git", rev = "81d1928497683cdd3bb97e1411f66d3b2715dd85" } ## Uncomment this to test librustzcash changes locally #[patch.crates-io] diff --git a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/Backend.kt b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/Backend.kt index b6f6ae45..35de48ad 100644 --- a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/Backend.kt +++ b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/Backend.kt @@ -4,8 +4,6 @@ import cash.z.ecc.android.sdk.internal.model.JniBlockMeta import cash.z.ecc.android.sdk.internal.model.JniScanRange import cash.z.ecc.android.sdk.internal.model.JniSubtreeRoot import cash.z.ecc.android.sdk.internal.model.JniUnifiedSpendingKey -import java.lang.RuntimeException -import kotlin.jvm.Throws /** * Contract defining the exposed capabilities of the Rust backend. @@ -27,13 +25,13 @@ interface Backend { to: String, value: Long, memo: ByteArray? = byteArrayOf() - ): Long + ): ByteArray suspend fun shieldToAddress( account: Int, unifiedSpendingKey: ByteArray, memo: ByteArray? = byteArrayOf() - ): Long + ): ByteArray suspend fun decryptAndStoreTransaction(tx: ByteArray) @@ -80,14 +78,7 @@ interface Backend { /** * @throws RuntimeException as a common indicator of the operation failure */ - @Throws(RuntimeException::class) - suspend fun getReceivedMemoAsUtf8(idNote: Long): String? - - /** - * @throws RuntimeException as a common indicator of the operation failure - */ - @Throws(RuntimeException::class) - suspend fun getSentMemoAsUtf8(idNote: Long): String? + suspend fun getMemoAsUtf8(txId: ByteArray, outputIndex: Int): String? suspend fun getVerifiedBalance(account: Int): Long diff --git a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/jni/RustBackend.kt b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/jni/RustBackend.kt index ec120317..a84be3f9 100644 --- a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/jni/RustBackend.kt +++ b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/jni/RustBackend.kt @@ -156,20 +156,12 @@ class RustBackend private constructor( return longValue } - override suspend fun getReceivedMemoAsUtf8(idNote: Long) = + override suspend fun getMemoAsUtf8(txId: ByteArray, outputIndex: Int) = withContext(SdkDispatchers.DATABASE_IO) { - getReceivedMemoAsUtf8( + getMemoAsUtf8( dataDbFile.absolutePath, - idNote, - networkId = networkId - ) - } - - override suspend fun getSentMemoAsUtf8(idNote: Long) = - withContext(SdkDispatchers.DATABASE_IO) { - getSentMemoAsUtf8( - dataDbFile.absolutePath, - idNote, + txId, + outputIndex, networkId = networkId ) } @@ -307,7 +299,7 @@ class RustBackend private constructor( to: String, value: Long, memo: ByteArray? - ): Long = withContext(SdkDispatchers.DATABASE_IO) { + ): ByteArray = withContext(SdkDispatchers.DATABASE_IO) { createToAddress( dataDbFile.absolutePath, unifiedSpendingKey, @@ -325,7 +317,7 @@ class RustBackend private constructor( account: Int, unifiedSpendingKey: ByteArray, memo: ByteArray? - ): Long { + ): ByteArray { return withContext(SdkDispatchers.DATABASE_IO) { shieldToAddress( dataDbFile.absolutePath, @@ -481,16 +473,10 @@ class RustBackend private constructor( ): Long @JvmStatic - private external fun getReceivedMemoAsUtf8( + private external fun getMemoAsUtf8( dbDataPath: String, - idNote: Long, - networkId: Int - ): String? - - @JvmStatic - private external fun getSentMemoAsUtf8( - dbDataPath: String, - dNote: Long, + txId: ByteArray, + outputIndex: Int, networkId: Int ): String? @@ -578,7 +564,7 @@ class RustBackend private constructor( outputParamsPath: String, networkId: Int, useZip317Fees: Boolean - ): Long + ): ByteArray @JvmStatic @Suppress("LongParameterList") @@ -590,7 +576,7 @@ class RustBackend private constructor( outputParamsPath: String, networkId: Int, useZip317Fees: Boolean - ): Long + ): ByteArray @JvmStatic private external fun branchIdForHeight(height: Long, networkId: Int): Long diff --git a/backend-lib/src/main/rust/lib.rs b/backend-lib/src/main/rust/lib.rs index 4f81afc2..b79b0a63 100644 --- a/backend-lib/src/main/rust/lib.rs +++ b/backend-lib/src/main/rust/lib.rs @@ -18,7 +18,10 @@ use tracing::{debug, error}; use tracing_subscriber::prelude::*; use tracing_subscriber::reload; use zcash_address::{ToAddress, ZcashAddress}; -use zcash_client_backend::data_api::scanning::{ScanPriority, ScanRange}; +use zcash_client_backend::data_api::{ + scanning::{ScanPriority, ScanRange}, + NoteId, ShieldedProtocol, +}; use zcash_client_backend::keys::{DecodingError, UnifiedSpendingKey}; use zcash_client_backend::{ address::{RecipientAddress, UnifiedAddress}, @@ -40,7 +43,7 @@ use zcash_client_sqlite::chain::init::init_blockmeta_db; use zcash_client_sqlite::{ chain::BlockMeta, wallet::init::{init_accounts_table, init_blocks_table, init_wallet_db, WalletMigrationError}, - FsBlockDb, NoteId, WalletDb, + FsBlockDb, WalletDb, }; use zcash_primitives::consensus::Network::{MainNetwork, TestNetwork}; use zcash_primitives::{ @@ -52,7 +55,7 @@ use zcash_primitives::{ sapling, transaction::{ components::{amount::NonNegativeAmount, Amount, OutPoint, TxOut}, - Transaction, + Transaction, TxId, }, zip32::{AccountId, DiversifierIndex}, }; @@ -859,47 +862,24 @@ pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_ge } #[no_mangle] -pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_getReceivedMemoAsUtf8( +pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_getMemoAsUtf8( env: JNIEnv<'_>, _: JClass<'_>, db_data: JString<'_>, - id_note: jlong, + txid_bytes: jbyteArray, + output_index: jint, network_id: jint, ) -> jstring { let res = panic::catch_unwind(|| { let network = parse_network(network_id as u32)?; let db_data = wallet_db(&env, network, db_data)?; - let memo = (&db_data) - .get_memo(NoteId::ReceivedNoteId(id_note)) - .map_err(|e| format_err!("An error occurred retrieving the memo, {}", e)) - .and_then(|memo| match memo { - Some(Memo::Empty) => Ok("".to_string()), - Some(Memo::Text(memo)) => Ok(memo.into()), - None => Err(format_err!("Memo not available")), - _ => Err(format_err!("This memo does not contain UTF-8 text")), - })?; - - let output = env.new_string(memo).expect("Couldn't create Java string!"); - Ok(output.into_raw()) - }); - unwrap_exc_or(&env, res, ptr::null_mut()) -} - -#[no_mangle] -pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_getSentMemoAsUtf8( - env: JNIEnv<'_>, - _: JClass<'_>, - db_data: JString<'_>, - id_note: jlong, - network_id: jint, -) -> jstring { - let res = panic::catch_unwind(|| { - let network = parse_network(network_id as u32)?; - let db_data = wallet_db(&env, network, db_data)?; + let txid_bytes = env.convert_byte_array(txid_bytes)?; + let txid = TxId::read(&txid_bytes[..])?; + let output_index = u16::try_from(output_index)?; let memo = (&db_data) - .get_memo(NoteId::SentNoteId(id_note)) + .get_memo(NoteId::new(txid, ShieldedProtocol::Sapling, output_index)) .map_err(|e| format_err!("An error occurred retrieving the memo, {}", e)) .and_then(|memo| match memo { Some(Memo::Empty) => Ok("".to_string()), @@ -1389,7 +1369,7 @@ pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_cr output_params: JString<'_>, network_id: jint, use_zip317_fees: jboolean, -) -> jlong { +) -> jbyteArray { let res = panic::catch_unwind(|| { let network = parse_network(network_id as u32)?; let mut db_data = wallet_db(&env, network, db_data)?; @@ -1433,7 +1413,7 @@ pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_cr }]) .map_err(|e| format_err!("Error creating transaction request: {:?}", e))?; - zip317_helper( + let txid = zip317_helper( (&mut db_data, prover, request), use_zip317_fees, |(wallet_db, prover, request), input_selector| { @@ -1462,9 +1442,11 @@ pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_cr ) .map_err(|e| format_err!("Error while creating transaction: {}", e)) }, - ) + )?; + + utils::rust_bytes_to_java(&env, txid.as_ref()) }); - unwrap_exc_or(&env, res, -1) + unwrap_exc_or(&env, res, ptr::null_mut()) } #[no_mangle] @@ -1478,7 +1460,7 @@ pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_sh output_params: JString<'_>, network_id: jint, use_zip317_fees: jboolean, -) -> jlong { +) -> jbyteArray { let res = panic::catch_unwind(|| { let network = parse_network(network_id as u32)?; let mut db_data = wallet_db(&env, network, db_data)?; @@ -1521,7 +1503,7 @@ pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_sh let shielding_threshold = NonNegativeAmount::from_u64(100000).unwrap(); - zip317_helper( + let txid = zip317_helper( (&mut db_data, prover), use_zip317_fees, |(wallet_db, prover), input_selector| { @@ -1552,9 +1534,11 @@ pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_sh ) .map_err(|e| format_err!("Error while shielding transaction: {}", e)) }, - ) + )?; + + utils::rust_bytes_to_java(&env, txid.as_ref()) }); - unwrap_exc_or(&env, res, -1) + unwrap_exc_or(&env, res, ptr::null_mut()) } #[no_mangle] diff --git a/backend-lib/src/main/rust/utils.rs b/backend-lib/src/main/rust/utils.rs index e3dc9b08..584f7d3e 100644 --- a/backend-lib/src/main/rust/utils.rs +++ b/backend-lib/src/main/rust/utils.rs @@ -1,8 +1,10 @@ +use core::slice; + use jni::{ descriptors::Desc, errors::Result as JNIResult, objects::{JClass, JObject, JString}, - sys::{jobjectArray, jsize}, + sys::{jbyteArray, jobjectArray, jsize}, JNIEnv, }; @@ -16,6 +18,17 @@ pub(crate) fn java_string_to_rust(env: &JNIEnv<'_>, jstring: JString<'_>) -> Str .into() } +pub(crate) fn rust_bytes_to_java( + env: &JNIEnv<'_>, + data: &[u8], +) -> Result { + // SAFETY: jbyte (i8) has the same size and alignment as u8. + let buf = unsafe { slice::from_raw_parts(data.as_ptr().cast(), data.len()) }; + let jret = env.new_byte_array(data.len() as jsize)?; + env.set_byte_array_region(jret, 0, buf)?; + Ok(jret) +} + pub(crate) fn rust_vec_to_java<'a, T, U, V, F, G>( env: &JNIEnv<'a>, data: Vec, diff --git a/sdk-lib/src/androidTest/java/cash/z/ecc/fixture/FakeRustBackend.kt b/sdk-lib/src/androidTest/java/cash/z/ecc/fixture/FakeRustBackend.kt index abeec8bb..02be697a 100644 --- a/sdk-lib/src/androidTest/java/cash/z/ecc/fixture/FakeRustBackend.kt +++ b/sdk-lib/src/androidTest/java/cash/z/ecc/fixture/FakeRustBackend.kt @@ -72,11 +72,11 @@ internal class FakeRustBackend( to: String, value: Long, memo: ByteArray? - ): Long { + ): ByteArray { TODO("Not yet implemented") } - override suspend fun shieldToAddress(account: Int, unifiedSpendingKey: ByteArray, memo: ByteArray?): Long { + override suspend fun shieldToAddress(account: Int, unifiedSpendingKey: ByteArray, memo: ByteArray?): ByteArray { TODO("Not yet implemented") } @@ -133,10 +133,7 @@ internal class FakeRustBackend( TODO("Not yet implemented") } - override suspend fun getReceivedMemoAsUtf8(idNote: Long): String? = - error("Intentionally not implemented in mocked FakeRustBackend implementation.") - - override suspend fun getSentMemoAsUtf8(idNote: Long): String? = + override suspend fun getMemoAsUtf8(txId: ByteArray, outputIndex: Int): String? = error("Intentionally not implemented in mocked FakeRustBackend implementation.") override suspend fun getVerifiedBalance(account: Int): Long { diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt index 6ecccb71..84473d1f 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt @@ -314,18 +314,10 @@ class SdkSynchronizer private constructor( } override fun getMemos(transactionOverview: TransactionOverview): Flow { - return storage.getNoteIds(transactionOverview.id).map { + return storage.getSaplingOutputIndices(transactionOverview.id).map { runCatching { - when (transactionOverview.isSentTransaction) { - true -> { - backend.getSentMemoAsUtf8(it) - } - false -> { - backend.getReceivedMemoAsUtf8(it) - } - } + backend.getMemoAsUtf8(transactionOverview.rawId.byteArray, it) }.onFailure { - // https://github.com/zcash/librustzcash/issues/834 Twig.error { "Failed to get memo with: $it" } }.onSuccess { Twig.debug { "Transaction memo queried: $it" } diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/exception/Exceptions.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/exception/Exceptions.kt index ee3e8d27..4e806eea 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/exception/Exceptions.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/exception/Exceptions.kt @@ -3,6 +3,7 @@ package cash.z.ecc.android.sdk.exception import cash.z.ecc.android.sdk.internal.SaplingParameters import cash.z.ecc.android.sdk.internal.model.Checkpoint import cash.z.ecc.android.sdk.model.BlockHeight +import cash.z.ecc.android.sdk.model.FirstClassByteArray import cash.z.ecc.android.sdk.model.ZcashNetwork import co.electriccoin.lightwallet.client.model.BlockHeightUnsafe @@ -283,7 +284,7 @@ sealed class TransactionEncoderException( object MissingParamsException : TransactionEncoderException( "Cannot send funds due to missing spend or output params and attempting to download them failed." ) - class TransactionNotFoundException(transactionId: Long) : TransactionEncoderException( + class TransactionNotFoundException(transactionId: FirstClassByteArray) : TransactionEncoderException( "Unable to find transactionId $transactionId in the repository. This means the wallet created a transaction " + "and then returned a row ID that does not actually exist. This is a scenario where the wallet should " + "have thrown an exception but failed to do so." diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackend.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackend.kt index a1ebbb4b..12250ecb 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackend.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackend.kt @@ -6,6 +6,7 @@ import cash.z.ecc.android.sdk.internal.model.ScanRange import cash.z.ecc.android.sdk.internal.model.SubtreeRoot import cash.z.ecc.android.sdk.model.Account import cash.z.ecc.android.sdk.model.BlockHeight +import cash.z.ecc.android.sdk.model.FirstClassByteArray import cash.z.ecc.android.sdk.model.UnifiedFullViewingKey import cash.z.ecc.android.sdk.model.UnifiedSpendingKey import cash.z.ecc.android.sdk.model.WalletBalance @@ -34,12 +35,12 @@ internal interface TypesafeBackend { to: String, value: Long, memo: ByteArray? = byteArrayOf() - ): Long + ): FirstClassByteArray suspend fun shieldToAddress( usk: UnifiedSpendingKey, memo: ByteArray? = byteArrayOf() - ): Long + ): FirstClassByteArray suspend fun getCurrentAddress(account: Account): String @@ -73,9 +74,7 @@ internal interface TypesafeBackend { height: BlockHeight ) - suspend fun getSentMemoAsUtf8(idNote: Long): String? - - suspend fun getReceivedMemoAsUtf8(idNote: Long): String? + suspend fun getMemoAsUtf8(txId: ByteArray, outputIndex: Int): String? suspend fun initDataDb(seed: ByteArray?): Int diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackendImpl.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackendImpl.kt index e8ddadd5..19c30242 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackendImpl.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackendImpl.kt @@ -7,6 +7,7 @@ import cash.z.ecc.android.sdk.internal.model.ScanRange import cash.z.ecc.android.sdk.internal.model.SubtreeRoot import cash.z.ecc.android.sdk.model.Account import cash.z.ecc.android.sdk.model.BlockHeight +import cash.z.ecc.android.sdk.model.FirstClassByteArray import cash.z.ecc.android.sdk.model.UnifiedFullViewingKey import cash.z.ecc.android.sdk.model.UnifiedSpendingKey import cash.z.ecc.android.sdk.model.WalletBalance @@ -53,21 +54,25 @@ internal class TypesafeBackendImpl(private val backend: Backend) : TypesafeBacke to: String, value: Long, memo: ByteArray? - ): Long = backend.createToAddress( - usk.account.value, - usk.copyBytes(), - to, - value, - memo + ): FirstClassByteArray = FirstClassByteArray( + backend.createToAddress( + usk.account.value, + usk.copyBytes(), + to, + value, + memo + ) ) override suspend fun shieldToAddress( usk: UnifiedSpendingKey, memo: ByteArray? - ): Long = backend.shieldToAddress( - usk.account.value, - usk.copyBytes(), - memo + ): FirstClassByteArray = FirstClassByteArray( + backend.shieldToAddress( + usk.account.value, + usk.copyBytes(), + memo + ) ) override suspend fun getCurrentAddress(account: Account): String { @@ -152,9 +157,8 @@ internal class TypesafeBackendImpl(private val backend: Backend) : TypesafeBacke ) } - override suspend fun getSentMemoAsUtf8(idNote: Long) = backend.getSentMemoAsUtf8(idNote) - - override suspend fun getReceivedMemoAsUtf8(idNote: Long): String? = backend.getReceivedMemoAsUtf8(idNote) + override suspend fun getMemoAsUtf8(txId: ByteArray, outputIndex: Int): String? = + backend.getMemoAsUtf8(txId, outputIndex) override suspend fun initDataDb(seed: ByteArray?): Int = backend.initDataDb(seed) diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/DbDerivedDataRepository.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/DbDerivedDataRepository.kt index 21160c9b..04ce09e8 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/DbDerivedDataRepository.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/DbDerivedDataRepository.kt @@ -4,6 +4,7 @@ import cash.z.ecc.android.sdk.internal.model.DbTransactionOverview import cash.z.ecc.android.sdk.internal.model.EncodedTransaction import cash.z.ecc.android.sdk.internal.repository.DerivedDataRepository import cash.z.ecc.android.sdk.model.BlockHeight +import cash.z.ecc.android.sdk.model.FirstClassByteArray import cash.z.ecc.android.sdk.model.TransactionRecipient import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.MutableStateFlow @@ -33,8 +34,8 @@ internal class DbDerivedDataRepository( return derivedDataDb.blockTable.count() > 0 } - override suspend fun findEncodedTransactionById(txId: Long): EncodedTransaction? { - return derivedDataDb.transactionTable.findEncodedTransactionById(txId) + override suspend fun findEncodedTransactionByTxId(txId: FirstClassByteArray): EncodedTransaction? { + return derivedDataDb.transactionTable.findEncodedTransactionByTxId(txId) } override suspend fun findNewTransactions(blockHeightRange: ClosedRange): List = @@ -63,7 +64,8 @@ internal class DbDerivedDataRepository( override val allTransactions: Flow> get() = invalidatingFlow.map { derivedDataDb.allTransactionView.getAllTransactions().toList() } - override fun getNoteIds(transactionId: Long) = derivedDataDb.txOutputsView.getNoteIds(transactionId) + override fun getSaplingOutputIndices(transactionId: Long) = + derivedDataDb.txOutputsView.getSaplingOutputIndices(transactionId) override fun getRecipients(transactionId: Long): Flow { return derivedDataDb.txOutputsView.getRecipients(transactionId) diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/TransactionTable.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/TransactionTable.kt index 48cee81a..6cac3446 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/TransactionTable.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/TransactionTable.kt @@ -45,7 +45,7 @@ internal class TransactionTable( private val SELECTION_TRANSACTION_ID_AND_RAW_NOT_NULL = String.format( Locale.ROOT, "%s = ? AND %s IS NOT NULL", // $NON-NLS - TransactionTableDefinition.COLUMN_INTEGER_ID, + TransactionTableDefinition.COLUMN_BLOB_TRANSACTION_ID, TransactionTableDefinition.COLUMN_BLOB_RAW ) } @@ -66,18 +66,16 @@ internal class TransactionTable( cursorParser = { it.getLong(0) } ).first() - suspend fun findEncodedTransactionById(id: Long): EncodedTransaction? { + suspend fun findEncodedTransactionByTxId(txId: FirstClassByteArray): EncodedTransaction? { return sqliteDatabase.queryAndMap( table = TransactionTableDefinition.TABLE_NAME, columns = PROJECTION_ENCODED_TRANSACTION, selection = SELECTION_TRANSACTION_ID_AND_RAW_NOT_NULL, - selectionArgs = arrayOf(id) + selectionArgs = arrayOf(txId) ) { - val txIdIndex = it.getColumnIndexOrThrow(TransactionTableDefinition.COLUMN_BLOB_TRANSACTION_ID) val rawIndex = it.getColumnIndexOrThrow(TransactionTableDefinition.COLUMN_BLOB_RAW) val heightIndex = it.getColumnIndexOrThrow(TransactionTableDefinition.COLUMN_INTEGER_EXPIRY_HEIGHT) - val txid = it.getBlob(txIdIndex) val raw = it.getBlob(rawIndex) val expiryHeight = if (it.isNull(heightIndex)) { null @@ -86,7 +84,7 @@ internal class TransactionTable( } EncodedTransaction( - FirstClassByteArray(txid), + txId, FirstClassByteArray(raw), expiryHeight ) diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/TxOutputsView.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/TxOutputsView.kt index becc0554..80432228 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/TxOutputsView.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/TxOutputsView.kt @@ -35,7 +35,7 @@ internal class TxOutputsView( ) } - fun getNoteIds(transactionId: Long) = + fun getSaplingOutputIndices(transactionId: Long) = sqliteDatabase.queryAndMap( table = TxOutputsViewDefinition.VIEW_NAME, columns = PROJECTION_ID, @@ -43,9 +43,9 @@ internal class TxOutputsView( selectionArgs = arrayOf(transactionId), orderBy = ORDER_BY, cursorParser = { - val idColumnIndex = it.getColumnIndex(TxOutputsViewDefinition.COLUMN_INTEGER_TRANSACTION_ID) + val idColumnIndex = it.getColumnIndex(TxOutputsViewDefinition.COLUMN_INTEGER_OUTPUT_INDEX) - it.getLong(idColumnIndex) + it.getInt(idColumnIndex) } ) diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/repository/DerivedDataRepository.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/repository/DerivedDataRepository.kt index b20da0e8..ba802ad0 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/repository/DerivedDataRepository.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/repository/DerivedDataRepository.kt @@ -3,6 +3,7 @@ package cash.z.ecc.android.sdk.internal.repository import cash.z.ecc.android.sdk.internal.model.DbTransactionOverview import cash.z.ecc.android.sdk.internal.model.EncodedTransaction import cash.z.ecc.android.sdk.model.BlockHeight +import cash.z.ecc.android.sdk.model.FirstClassByteArray import cash.z.ecc.android.sdk.model.TransactionRecipient import kotlinx.coroutines.flow.Flow @@ -46,7 +47,7 @@ internal interface DerivedDataRepository { * * @return the transaction or null when it cannot be found. */ - suspend fun findEncodedTransactionById(txId: Long): EncodedTransaction? + suspend fun findEncodedTransactionByTxId(txId: FirstClassByteArray): EncodedTransaction? /** * Find all the newly scanned transactions in the given range, including transactions (like @@ -105,7 +106,7 @@ internal interface DerivedDataRepository { val allTransactions: Flow> - fun getNoteIds(transactionId: Long): Flow + fun getSaplingOutputIndices(transactionId: Long): Flow fun getRecipients(transactionId: Long): Flow diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/transaction/TransactionEncoderImpl.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/transaction/TransactionEncoderImpl.kt index c43016a4..dbe53e75 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/transaction/TransactionEncoderImpl.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/transaction/TransactionEncoderImpl.kt @@ -7,6 +7,7 @@ import cash.z.ecc.android.sdk.internal.Twig import cash.z.ecc.android.sdk.internal.TypesafeBackend import cash.z.ecc.android.sdk.internal.model.EncodedTransaction import cash.z.ecc.android.sdk.internal.repository.DerivedDataRepository +import cash.z.ecc.android.sdk.model.FirstClassByteArray import cash.z.ecc.android.sdk.model.TransactionRecipient import cash.z.ecc.android.sdk.model.UnifiedSpendingKey import cash.z.ecc.android.sdk.model.Zatoshi @@ -47,7 +48,7 @@ internal class TransactionEncoderImpl( require(recipient is TransactionRecipient.Address) val transactionId = createSpend(usk, amount, recipient.addressValue, memo) - return repository.findEncodedTransactionById(transactionId) + return repository.findEncodedTransactionByTxId(transactionId) ?: throw TransactionEncoderException.TransactionNotFoundException(transactionId) } @@ -59,7 +60,7 @@ internal class TransactionEncoderImpl( require(recipient is TransactionRecipient.Account) val transactionId = createShieldingSpend(usk, memo) - return repository.findEncodedTransactionById(transactionId) + return repository.findEncodedTransactionByTxId(transactionId) ?: throw TransactionEncoderException.TransactionNotFoundException(transactionId) } @@ -121,7 +122,7 @@ internal class TransactionEncoderImpl( amount: Zatoshi, toAddress: String, memo: ByteArray? = byteArrayOf() - ): Long { + ): FirstClassByteArray { Twig.debug { "creating transaction to spend $amount zatoshi to" + " ${toAddress.masked()} with memo $memo" @@ -148,7 +149,7 @@ internal class TransactionEncoderImpl( private suspend fun createShieldingSpend( usk: UnifiedSpendingKey, memo: ByteArray? = byteArrayOf() - ): Long { + ): FirstClassByteArray { @Suppress("TooGenericExceptionCaught") return try { saplingParamTool.ensureParams(saplingParamTool.properties.paramsDirectory) From 6b487e979e92678ed4decb355328de5392d9abde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Honza=20Rychnovsk=C3=BD?= Date: Wed, 9 Aug 2023 10:59:15 +0200 Subject: [PATCH 18/76] [#1160] New SyncAlgorithm type API --- CHANGELOG.md | 14 ++++++- README.md | 9 +++++ .../z/ecc/android/sdk/WalletCoordinator.kt | 7 +++- .../cash/z/ecc/android/sdk/SdkSynchronizer.kt | 16 +++++--- .../cash/z/ecc/android/sdk/Synchronizer.kt | 38 ++++++++++++------- .../sdk/block/CompactBlockProcessor.kt | 27 ++++++++++--- 6 files changed, 83 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e770d4f..d6936d8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,18 @@ # Change Log +## Unreleased +- `CompactBlockProcessor` now processes compact blocks from the lightwalletd server in a non-linear order. This + feature shortens the time after which a wallet's spendable balance can be used. + +### Added +- New `syncAlgorithm` parameter of `Synchronizer.new()` and `WalletCoordinator()` to select preferred + `CompactBlockProcessor` block synchronizing algorithm. It can be of `CompactBlockProcessor.SyncAlgorithm.LINEAR` + or `NON_LINEAR`. The LINEAR type is automatically used if the client app does not specify otherwise. Please note + that the NON_LINEAR type is currently unstable and still under development. + ## 1.21.0-beta01 -Note: This is the last _1.x_ version release. The upcoming version _2.0_ brings the **Spend-before-Sync** feature, -which speeds up discovering the wallet's spendable balance. +Note: This is the last _1.x_ version release. The upcoming version _2.0_ brings the **Spend-before-Sync** feature, +which speeds up discovering the wallet's spendable balance. ### Changed - Updated dependencies: diff --git a/README.md b/README.md index 355902db..8e732ae7 100644 --- a/README.md +++ b/README.md @@ -53,3 +53,12 @@ Note that we aim for the main branch of this repository to be stable and releasa 1. Intel-based machines may have trouble building in Android Studio. The workaround is to add the following line to `~/.gradle/gradle.properties`: `ZCASH_IS_DEPENDENCY_LOCKING_ENABLED=false` 1. During builds, a warning will be printed that says "Unable to detect AGP versions for included builds. All projects in the build should use the same AGP version." This can be safely ignored. The version under build-conventions is the same as the version used elsewhere in the application. 1. Android Studio will warn about the Gradle checksum. This is a [known issue](https://github.com/gradle/gradle/issues/9361) and can be safely ignored. + +## Unstable Features +### Non-linear compact blocks synchronization +- CompactBlockProcessor now processes compact blocks from the lightwalletd server in a **non-linear** order. This +feature shortens the time after which a wallet's spendable balance can be used. +- Use the new `syncAlgorithm` parameter of `Synchronizer.new()` or `WalletCoordinator()` to select preferred + `CompactBlockProcessor` block synchronizing algorithm. It can be of `CompactBlockProcessor.SyncAlgorithm.LINEAR` + or `NON_LINEAR`. The LINEAR type is automatically used if the client app does not specify otherwise. Please note + that the NON_LINEAR type is currently unstable and still under development. diff --git a/sdk-incubator-lib/src/main/java/cash/z/ecc/android/sdk/WalletCoordinator.kt b/sdk-incubator-lib/src/main/java/cash/z/ecc/android/sdk/WalletCoordinator.kt index 5f478b84..e0a1e98d 100644 --- a/sdk-incubator-lib/src/main/java/cash/z/ecc/android/sdk/WalletCoordinator.kt +++ b/sdk-incubator-lib/src/main/java/cash/z/ecc/android/sdk/WalletCoordinator.kt @@ -1,6 +1,7 @@ package cash.z.ecc.android.sdk import android.content.Context +import cash.z.ecc.android.sdk.block.CompactBlockProcessor import cash.z.ecc.android.sdk.ext.onFirst import cash.z.ecc.android.sdk.internal.Twig import cash.z.ecc.android.sdk.model.PersistableWallet @@ -33,6 +34,8 @@ import java.util.UUID /** * @param persistableWallet flow of the user's stored wallet. Null indicates that no wallet has been stored. + * + * @param syncAlgorithm The CompactBlockProcess's type of block syncing algorithm */ /* * One area where this class needs to change before it can be moved out of the incubator is that we need to be able to @@ -43,7 +46,8 @@ import java.util.UUID */ class WalletCoordinator( context: Context, - val persistableWallet: Flow + val persistableWallet: Flow, + val syncAlgorithm: CompactBlockProcessor.SyncAlgorithm = CompactBlockProcessor.SyncAlgorithm.NON_LINEAR ) { private val applicationContext = context.applicationContext @@ -80,6 +84,7 @@ class WalletCoordinator( lightWalletEndpoint = LightWalletEndpoint.defaultForNetwork(persistableWallet.network), birthday = persistableWallet.birthday, seed = persistableWallet.seedPhrase.toByteArray(), + syncAlgorithm = syncAlgorithm ) trySend(InternalSynchronizerStatus.Available(closeableSynchronizer)) diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt index 84473d1f..3db96abe 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt @@ -109,6 +109,10 @@ class SdkSynchronizer private constructor( private val mutex = Mutex() /** + * Convenience method to create new SdkSynchronizer instance. + * + * @return Synchronizer instance as CloseableSynchronizer + * * @throws IllegalStateException If multiple instances of synchronizer with the same network+alias are * active at the same time. Call `close` to finish one synchronizer before starting another one with the same * network+alias. @@ -708,12 +712,14 @@ internal object DefaultSynchronizerFactory { backend: TypesafeBackend, downloader: CompactBlockDownloader, repository: DerivedDataRepository, - birthdayHeight: BlockHeight + birthdayHeight: BlockHeight, + syncAlgorithm: CompactBlockProcessor.SyncAlgorithm ): CompactBlockProcessor = CompactBlockProcessor( - downloader, - repository, - backend, - birthdayHeight + downloader = downloader, + repository = repository, + backend = backend, + minimumHeight = birthdayHeight, + syncAlgorithm = syncAlgorithm ) } diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/Synchronizer.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/Synchronizer.kt index 33507f97..d3bb3f72 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/Synchronizer.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/Synchronizer.kt @@ -398,20 +398,28 @@ interface Synchronizer { * Primary method that SDK clients will use to construct a synchronizer. * * @param zcashNetwork the network to use. + * * @param alias A string used to segregate multiple wallets in the filesystem. This implies the string * should not contain characters unsuitable for the platform's filesystem. The default value is * generally used unless an SDK client needs to support multiple wallets. + * * @param lightWalletEndpoint Server endpoint. See [cash.z.ecc.android.sdk.model.defaultForNetwork]. If a * client wishes to change the server endpoint, the active synchronizer will need to be stopped and a new * instance created with a new value. + * * @param seed the wallet's seed phrase. This is required the first time a new wallet is set up. For * subsequent calls, seed is only needed if [InitializerException.SeedRequired] is thrown. + * * @param birthday Block height representing the "birthday" of the wallet. When creating a new wallet, see * [BlockHeight.ofLatestCheckpoint]. When restoring an existing wallet, use block height that was first used * to create the wallet. If that value is unknown, null is acceptable but will result in longer * sync times. After sync completes, the birthday can be determined from [Synchronizer.latestBirthdayHeight]. + * + * @param syncAlgorithm The CompactBlockProcess's type of block syncing algorithm + * * @throws InitializerException.SeedRequired Indicates clients need to call this method again, providing the * seed bytes. + * * @throws IllegalStateException If multiple instances of synchronizer with the same network+alias are * active at the same time. Call `close` to finish one synchronizer before starting another one with the same * network+alias. @@ -427,7 +435,8 @@ interface Synchronizer { alias: String = ZcashSdk.DEFAULT_ALIAS, lightWalletEndpoint: LightWalletEndpoint, seed: ByteArray?, - birthday: BlockHeight? + birthday: BlockHeight?, + syncAlgorithm: CompactBlockProcessor.SyncAlgorithm = CompactBlockProcessor.SyncAlgorithm.LINEAR ): CloseableSynchronizer { val applicationContext = context.applicationContext @@ -482,20 +491,20 @@ interface Synchronizer { service ) val processor = DefaultSynchronizerFactory.defaultProcessor( - backend, - downloader, - repository, - birthday - ?: zcashNetwork.saplingActivationHeight + backend = backend, + downloader = downloader, + repository = repository, + birthdayHeight = birthday ?: zcashNetwork.saplingActivationHeight, + syncAlgorithm = syncAlgorithm ) return SdkSynchronizer.new( - zcashNetwork, - alias, - repository, - txManager, - processor, - backend + zcashNetwork = zcashNetwork, + alias = alias, + repository = repository, + txManager = txManager, + processor = processor, + backend = backend ) } @@ -513,9 +522,10 @@ interface Synchronizer { alias: String = ZcashSdk.DEFAULT_ALIAS, lightWalletEndpoint: LightWalletEndpoint, seed: ByteArray?, - birthday: BlockHeight? + birthday: BlockHeight?, + syncAlgorithm: CompactBlockProcessor.SyncAlgorithm = CompactBlockProcessor.SyncAlgorithm.LINEAR ): CloseableSynchronizer = runBlocking { - new(context, zcashNetwork, alias, lightWalletEndpoint, seed, birthday) + new(context, zcashNetwork, alias, lightWalletEndpoint, seed, birthday, syncAlgorithm) } /** diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt index 60255756..7f5710d0 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt @@ -85,6 +85,8 @@ import kotlin.time.toDuration * reorgs as a backstop to make sure we do not rewind beyond sapling activation. It also is factored * in when considering initial range to download. In most cases, this should be the birthday height * of the current wallet--the height before which we do not need to scan for transactions. + * + * @property syncAlgorithm The type of block syncing algorithm which should be preferably used */ @OpenForTesting @Suppress("TooManyFunctions", "LargeClass") @@ -92,7 +94,8 @@ class CompactBlockProcessor internal constructor( val downloader: CompactBlockDownloader, private val repository: DerivedDataRepository, private val backend: TypesafeBackend, - minimumHeight: BlockHeight + minimumHeight: BlockHeight, + private val syncAlgorithm: SyncAlgorithm ) { /** * Callback for any non-trivial errors that occur while processing compact blocks. @@ -196,8 +199,12 @@ class CompactBlockProcessor internal constructor( ) // Download note commitment tree data from lightwalletd to decide if we communicate with linear - // or non-linear node - val subTreeRootList = getSubtreeRoots(downloader, network) + // or non-linear node. It depends on the syncAlgorithm property on the first place. + val subTreeRootList = if (syncAlgorithm == SyncAlgorithm.LINEAR) { + emptyList() + } else { + getSubtreeRoots(downloader, network) + } Twig.info { "Fetched SubTreeRoot list size: ${subTreeRootList?.size ?: 0}" } Twig.debug { "Setup verified. Processor starting..." } @@ -205,7 +212,10 @@ class CompactBlockProcessor internal constructor( // Using do/while makes it easier to execute exactly one loop which helps with testing this processor quickly // (because you can start and then immediately set isStopped=true to always get precisely one loop) do { - retryWithBackoff(::onProcessorError, maxDelayMillis = MAX_BACKOFF_INTERVAL) { + retryWithBackoff( + onErrorListener = ::onProcessorError, + maxDelayMillis = MAX_BACKOFF_INTERVAL + ) { val result = processingMutex.withLockLogged("processNewBlocks") { if (subTreeRootList.isNullOrEmpty()) { processNewBlocksInLinearOrder() @@ -366,7 +376,7 @@ class CompactBlockProcessor internal constructor( firstUnenhancedHeight: BlockHeight? ): BlockProcessingResult { Twig.debug { - "Beginning to process new blocks with DAG approach (with roots: $subTreeRootList, and lower " + + "Beginning to process new blocks with Non-linear approach (with roots: $subTreeRootList, and lower " + "bound: $lastValidHeight)..." } @@ -952,7 +962,7 @@ class CompactBlockProcessor internal constructor( retryUpToAndContinue(GET_SUBTREE_ROOTS_RETRIES) { subTreeRootList = downloader.getSubtreeRoots( - // TODO [#1133]: DAG: Set the correct getSubtreeRoots inputs + // TODO [#1133]: SbS: Set the correct getSubtreeRoots inputs // TODO [#1133]: https://github.com/zcash/zcash-android-wallet-sdk/issues/1133 startIndex = 0, maxEntries = if (network.isTestnet()) { @@ -2043,6 +2053,11 @@ class CompactBlockProcessor internal constructor( val hash: String? ) + enum class SyncAlgorithm { + LINEAR, + NON_LINEAR + } + // // Helper Extensions // From 70cd1e030c54f8a6441ab1fb2592beb0fbd580c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Honza=20Rychnovsk=C3=BD?= Date: Thu, 10 Aug 2023 12:15:40 +0200 Subject: [PATCH 19/76] [#1157] Get roots: recover failed communication --- .../sdk/block/CompactBlockProcessor.kt | 86 +++++++++++++------ 1 file changed, 60 insertions(+), 26 deletions(-) diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt index 7f5710d0..47999868 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt @@ -200,12 +200,12 @@ class CompactBlockProcessor internal constructor( // Download note commitment tree data from lightwalletd to decide if we communicate with linear // or non-linear node. It depends on the syncAlgorithm property on the first place. - val subTreeRootList = if (syncAlgorithm == SyncAlgorithm.LINEAR) { - emptyList() + var subTreeRootResult = if (syncAlgorithm == SyncAlgorithm.LINEAR) { + GetSubtreeRootsResult.UseLinear } else { getSubtreeRoots(downloader, network) } - Twig.info { "Fetched SubTreeRoot list size: ${subTreeRootList?.size ?: 0}" } + Twig.info { "Fetched SubTreeRoot result: $subTreeRootResult, with preferred sync algorithm: $syncAlgorithm" } Twig.debug { "Setup verified. Processor starting..." } @@ -217,18 +217,32 @@ class CompactBlockProcessor internal constructor( maxDelayMillis = MAX_BACKOFF_INTERVAL ) { val result = processingMutex.withLockLogged("processNewBlocks") { - if (subTreeRootList.isNullOrEmpty()) { - processNewBlocksInLinearOrder() - } else { - processNewBlocksInNonLinearOrder( - backend = backend, - downloader = downloader, - repository = repository, - network = network, - subTreeRootList = subTreeRootList, - lastValidHeight = lowerBoundHeight, - firstUnenhancedHeight = _processorInfo.value.firstUnenhancedHeight - ) + when (subTreeRootResult) { + is GetSubtreeRootsResult.UseNonLinear -> { + processNewBlocksInNonLinearOrder( + backend = backend, + downloader = downloader, + repository = repository, + network = network, + subTreeRootList = (subTreeRootResult as GetSubtreeRootsResult.UseNonLinear) + .subTreeRootList, + lastValidHeight = lowerBoundHeight, + firstUnenhancedHeight = _processorInfo.value.firstUnenhancedHeight + ) + } + GetSubtreeRootsResult.UseLinear -> { + // Forced by syncAlgorithm parameter or caused by an empty response result + processNewBlocksInLinearOrder() + } + is GetSubtreeRootsResult.OtherFailure -> { + // Server possible replied with some kind of unsupported error + processNewBlocksInLinearOrder() + } + GetSubtreeRootsResult.FailureConnection -> { + // SubtreeRoot fetching retry + subTreeRootResult = getSubtreeRoots(downloader, network) + BlockProcessingResult.Reconnecting + } } } // immediately process again after failures in order to download new blocks right away @@ -347,6 +361,13 @@ class CompactBlockProcessor internal constructor( data class Failure(val failedAtHeight: BlockHeight, val exception: Throwable) : SuggestScanRangesResult() } + internal sealed class GetSubtreeRootsResult { + data class UseNonLinear(val subTreeRootList: List) : GetSubtreeRootsResult() + object UseLinear : GetSubtreeRootsResult() + object FailureConnection : GetSubtreeRootsResult() + data class OtherFailure(val exception: Throwable) : GetSubtreeRootsResult() + } + internal sealed class PutSaplingSubtreeRootsResult { object Success : PutSaplingSubtreeRootsResult() data class Failure(val failedAtHeight: BlockHeight, val exception: Throwable) : PutSaplingSubtreeRootsResult() @@ -947,23 +968,21 @@ class CompactBlockProcessor internal constructor( /** * This operation downloads note commitment tree data from the lightwalletd server to decide if we communicate - * with linear or non-linear node + * with node which provides linear or non-linear synchronization mode. * - * @return List of SubtreeRoot objects in case of the operation success, null otherwise + * @return GetSubtreeRootsResult as a wrapper for the lightwalletd response result */ @VisibleForTesting internal suspend fun getSubtreeRoots( downloader: CompactBlockDownloader, network: ZcashNetwork - ): List? { + ): GetSubtreeRootsResult { Twig.debug { "Fetching SubtreeRoots..." } - var subTreeRootList: List? = null + var result: GetSubtreeRootsResult = GetSubtreeRootsResult.UseLinear retryUpToAndContinue(GET_SUBTREE_ROOTS_RETRIES) { - subTreeRootList = downloader.getSubtreeRoots( - // TODO [#1133]: SbS: Set the correct getSubtreeRoots inputs - // TODO [#1133]: https://github.com/zcash/zcash-android-wallet-sdk/issues/1133 + downloader.getSubtreeRoots( startIndex = 0, maxEntries = if (network.isTestnet()) { 65536 @@ -980,12 +999,22 @@ class CompactBlockProcessor internal constructor( } } is Response.Failure -> { - Twig.error { "Fetching SubtreeRoot failed with: ${response.toThrowable()}" } - throw LightWalletException.GetSubtreeRootsException( + val error = LightWalletException.GetSubtreeRootsException( response.code, response.description, response.toThrowable() ) + if (response is Response.Failure.Server.Unavailable) { + Twig.error { + "Fetching SubtreeRoot failed due to server communication problem with " + + "failure: ${response.toThrowable()}" + } + result = GetSubtreeRootsResult.FailureConnection + } else { + Twig.error { "Fetching SubtreeRoot failed with failure: ${response.toThrowable()}" } + result = GetSubtreeRootsResult.OtherFailure(error) + } + throw error } } } @@ -996,10 +1025,15 @@ class CompactBlockProcessor internal constructor( .toList() .map { SubtreeRoot.new(it, network) + }.let { + result = if (it.isEmpty()) { + GetSubtreeRootsResult.UseLinear + } else { + GetSubtreeRootsResult.UseNonLinear(it) + } } } - - return subTreeRootList + return result } /** From 91f5cbe24de0c765b0f95b0b05cdaec166aa1a2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Honza=20Rychnovsk=C3=BD?= Date: Mon, 14 Aug 2023 09:48:55 +0200 Subject: [PATCH 20/76] [#1129] Refactor ProcessorInfo * Add rewind button to Demo app To be able to test refactored CompactBlockProcessor rewind feature. * Rename non-linear to spend-before-sync * [#1129] Refactor lastSyncRange lastSyncedHeight * Fix updateRange function As the overallSyncRange needs to be reset in case of all ranges are processed. * [#1166] Remove alsoClearBlockCache parameter --- CHANGELOG.md | 18 +- README.md | 8 +- .../android/sdk/darkside/test/TestWallet.kt | 2 +- .../z/ecc/android/sdk/demoapp/Navigation.kt | 3 +- .../demos/getbalance/GetBalanceFragment.kt | 11 - .../ListTransactionsFragment.kt | 11 - .../demos/listutxos/ListUtxosFragment.kt | 11 - .../sdk/demoapp/demos/send/SendFragment.kt | 11 - .../demoapp/fixture/WalletSnapshotFixture.kt | 1 - .../demoapp/ui/screen/home/view/HomeView.kt | 28 +- .../screen/home/viewmodel/WalletViewModel.kt | 12 + .../z/ecc/android/sdk/WalletCoordinator.kt | 4 +- .../cash/z/ecc/android/sdk/util/TestWallet.kt | 2 +- .../cash/z/ecc/android/sdk/SdkSynchronizer.kt | 4 +- .../cash/z/ecc/android/sdk/Synchronizer.kt | 17 +- .../sdk/block/CompactBlockProcessor.kt | 282 +++++++----------- .../transaction/TransactionEncoderImpl.kt | 2 +- 17 files changed, 186 insertions(+), 241 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d6936d8f..cdffafb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,14 +1,24 @@ # Change Log ## Unreleased -- `CompactBlockProcessor` now processes compact blocks from the lightwalletd server in a non-linear order. This - feature shortens the time after which a wallet's spendable balance can be used. +- `CompactBlockProcessor` now processes compact blocks from the lightwalletd server with spend-before-sync algorithm + (i.e. non-linear order). This feature shortens the time after which a wallet's spendable balance can be used. ### Added - New `syncAlgorithm` parameter of `Synchronizer.new()` and `WalletCoordinator()` to select preferred `CompactBlockProcessor` block synchronizing algorithm. It can be of `CompactBlockProcessor.SyncAlgorithm.LINEAR` - or `NON_LINEAR`. The LINEAR type is automatically used if the client app does not specify otherwise. Please note - that the NON_LINEAR type is currently unstable and still under development. + or `SPEND_BEFORE_SYNC`. The LINEAR type is automatically used if the client app does not specify otherwise. Please + note that the SPEND_BEFORE_SYNC type is currently unstable and still under development. + +### Removed +- `CompactBlockProcessor.ProcessorInfo.lastSyncHeight` which the SDK is no longer able to provide because of the new + **SpendBeforeSync** synchronization algorithm adoption. Use `CompactBlockProcessor.ProcessorInfo.overallSyncRange` + which contains all blocks in case of `SpendBeforeSync` synchronization algorithm. No internal change was made in + case of older Linear synchronization algorithm. +- `CompactBlockProcessor.ProcessorInfo.isSyncing`. Use `Synchronizer.status` instead. +- `CompactBlockProcessor.ProcessorInfo.syncProgress`. Use `Synchronizer.progress` instead. +- `alsoClearBlockCache` parameter from rewind functions of `Synchronizer` and `CompactBlockProcessor` as it take no + affect on the current rewind functionality result. ## 1.21.0-beta01 Note: This is the last _1.x_ version release. The upcoming version _2.0_ brings the **Spend-before-Sync** feature, diff --git a/README.md b/README.md index 8e732ae7..a2e1cb64 100644 --- a/README.md +++ b/README.md @@ -55,10 +55,10 @@ Note that we aim for the main branch of this repository to be stable and releasa 1. Android Studio will warn about the Gradle checksum. This is a [known issue](https://github.com/gradle/gradle/issues/9361) and can be safely ignored. ## Unstable Features -### Non-linear compact blocks synchronization -- CompactBlockProcessor now processes compact blocks from the lightwalletd server in a **non-linear** order. This +### Spend-before-Sync compact blocks synchronization algorithm +- CompactBlockProcessor now processes compact blocks from the lightwalletd server in a **spend-before-sync** order. This feature shortens the time after which a wallet's spendable balance can be used. - Use the new `syncAlgorithm` parameter of `Synchronizer.new()` or `WalletCoordinator()` to select preferred `CompactBlockProcessor` block synchronizing algorithm. It can be of `CompactBlockProcessor.SyncAlgorithm.LINEAR` - or `NON_LINEAR`. The LINEAR type is automatically used if the client app does not specify otherwise. Please note - that the NON_LINEAR type is currently unstable and still under development. + or `SPEND_BEFORE_SYNC`. The LINEAR type is automatically used if the client app does not specify otherwise. Please + note that the SPEND_BEFORE_SYNC type is currently unstable and still under development. diff --git a/darkside-test-lib/src/androidTest/java/cash/z/ecc/android/sdk/darkside/test/TestWallet.kt b/darkside-test-lib/src/androidTest/java/cash/z/ecc/android/sdk/darkside/test/TestWallet.kt index 8e4cc667..61d9b2dc 100644 --- a/darkside-test-lib/src/androidTest/java/cash/z/ecc/android/sdk/darkside/test/TestWallet.kt +++ b/darkside-test-lib/src/androidTest/java/cash/z/ecc/android/sdk/darkside/test/TestWallet.kt @@ -105,7 +105,7 @@ class TestWallet( } suspend fun rewindToHeight(height: BlockHeight): TestWallet { - synchronizer.rewindToNearestHeight(height, false) + synchronizer.rewindToNearestHeight(height) return this } diff --git a/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/Navigation.kt b/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/Navigation.kt index bb9a2c4c..7f431d51 100644 --- a/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/Navigation.kt +++ b/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/Navigation.kt @@ -63,7 +63,8 @@ internal fun ComposeActivity.Navigation() { } }, goTransactions = { navController.navigateJustOnce(TRANSACTIONS) }, - resetSdk = { walletViewModel.resetSdk() } + resetSdk = { walletViewModel.resetSdk() }, + rewind = { walletViewModel.rewind() } ) } } diff --git a/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/demos/getbalance/GetBalanceFragment.kt b/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/demos/getbalance/GetBalanceFragment.kt index cc6e6129..a0f892df 100644 --- a/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/demos/getbalance/GetBalanceFragment.kt +++ b/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/demos/getbalance/GetBalanceFragment.kt @@ -9,7 +9,6 @@ import androidx.lifecycle.repeatOnLifecycle import cash.z.ecc.android.bip39.Mnemonics import cash.z.ecc.android.bip39.toSeed import cash.z.ecc.android.sdk.Synchronizer -import cash.z.ecc.android.sdk.block.CompactBlockProcessor import cash.z.ecc.android.sdk.demoapp.BaseDemoFragment import cash.z.ecc.android.sdk.demoapp.databinding.FragmentGetBalanceBinding import cash.z.ecc.android.sdk.demoapp.ext.requireApplicationContext @@ -89,12 +88,6 @@ class GetBalanceFragment : BaseDemoFragment() { .flatMapLatest { it.progress } .collect { onProgress(it) } } - launch { - sharedViewModel.synchronizerFlow - .filterNotNull() - .flatMapLatest { it.processorInfo } - .collect { onProcessorInfoUpdated(it) } - } launch { sharedViewModel.synchronizerFlow .filterNotNull() @@ -179,10 +172,6 @@ class GetBalanceFragment : BaseDemoFragment() { binding.textStatus.text = "Syncing blocks...${percent.toPercentage()}%" } } - - private fun onProcessorInfoUpdated(info: CompactBlockProcessor.ProcessorInfo) { - if (info.isSyncing) binding.textStatus.text = "Syncing blocks...${info.syncProgress}%" - } } @Suppress("MagicNumber") diff --git a/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/demos/listtransactions/ListTransactionsFragment.kt b/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/demos/listtransactions/ListTransactionsFragment.kt index 9f607fce..2288ac47 100644 --- a/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/demos/listtransactions/ListTransactionsFragment.kt +++ b/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/demos/listtransactions/ListTransactionsFragment.kt @@ -8,7 +8,6 @@ import androidx.lifecycle.lifecycleScope import androidx.lifecycle.repeatOnLifecycle import androidx.recyclerview.widget.LinearLayoutManager import cash.z.ecc.android.sdk.Synchronizer -import cash.z.ecc.android.sdk.block.CompactBlockProcessor import cash.z.ecc.android.sdk.demoapp.BaseDemoFragment import cash.z.ecc.android.sdk.demoapp.databinding.FragmentListTransactionsBinding import cash.z.ecc.android.sdk.internal.Twig @@ -55,12 +54,6 @@ class ListTransactionsFragment : BaseDemoFragment() { .flatMapLatest { it.progress } .collect { onProgress(it) } } - launch { - sharedViewModel.synchronizerFlow - .filterNotNull() - .flatMapLatest { it.processorInfo } - .collect { onProcessorInfoUpdated(it) } - } launch { sharedViewModel.synchronizerFlow .filterNotNull() @@ -198,10 +191,6 @@ class ListUtxosFragment : BaseDemoFragment() { } } - private fun onProcessorInfoUpdated(info: CompactBlockProcessor.ProcessorInfo) { - if (info.isSyncing) binding.textStatus.text = "Syncing blocks...${info.syncProgress}%" - } - @Suppress("MagicNumber") private fun onProgress(percent: PercentDecimal) { if (percent.isLessThanHundredPercent()) binding.textStatus.text = "Syncing blocks...${percent.toPercentage()}%" diff --git a/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/demos/send/SendFragment.kt b/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/demos/send/SendFragment.kt index 8b89145b..439f565a 100644 --- a/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/demos/send/SendFragment.kt +++ b/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/demos/send/SendFragment.kt @@ -9,7 +9,6 @@ import androidx.lifecycle.Lifecycle import androidx.lifecycle.lifecycleScope import androidx.lifecycle.repeatOnLifecycle import cash.z.ecc.android.sdk.Synchronizer -import cash.z.ecc.android.sdk.block.CompactBlockProcessor import cash.z.ecc.android.sdk.demoapp.BaseDemoFragment import cash.z.ecc.android.sdk.demoapp.DemoConstants import cash.z.ecc.android.sdk.demoapp.databinding.FragmentSendBinding @@ -93,12 +92,6 @@ class SendFragment : BaseDemoFragment() { .flatMapLatest { it.progress } .collect { onProgress(it) } } - launch { - sharedViewModel.synchronizerFlow - .filterNotNull() - .flatMapLatest { it.processorInfo } - .collect { onProcessorInfoUpdated(it) } - } launch { sharedViewModel.synchronizerFlow .filterNotNull() @@ -133,10 +126,6 @@ class SendFragment : BaseDemoFragment() { } } - private fun onProcessorInfoUpdated(info: CompactBlockProcessor.ProcessorInfo) { - if (info.isSyncing) binding.textStatus.text = "Syncing blocks...${info.syncProgress}%" - } - @Suppress("MagicNumber") private fun onBalance(balance: WalletBalance?) { this.balance = balance diff --git a/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/fixture/WalletSnapshotFixture.kt b/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/fixture/WalletSnapshotFixture.kt index 6a2bf39e..cad402d5 100644 --- a/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/fixture/WalletSnapshotFixture.kt +++ b/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/fixture/WalletSnapshotFixture.kt @@ -22,7 +22,6 @@ object WalletSnapshotFixture { fun new( status: Synchronizer.Status = STATUS, processorInfo: CompactBlockProcessor.ProcessorInfo = CompactBlockProcessor.ProcessorInfo( - null, null, null, null diff --git a/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/ui/screen/home/view/HomeView.kt b/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/ui/screen/home/view/HomeView.kt index c6f0305a..5afad436 100644 --- a/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/ui/screen/home/view/HomeView.kt +++ b/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/ui/screen/home/view/HomeView.kt @@ -43,7 +43,8 @@ private fun ComposablePreviewHome() { goAddressDetails = {}, goTransactions = {}, goTestnetFaucet = {}, - resetSdk = {} + resetSdk = {}, + rewind = {}, ) } } @@ -59,9 +60,15 @@ fun Home( goTransactions: () -> Unit, goTestnetFaucet: () -> Unit, resetSdk: () -> Unit, + rewind: () -> Unit, ) { Scaffold(topBar = { - HomeTopAppBar(isTestnet, goTestnetFaucet, resetSdk) + HomeTopAppBar( + isTestnet, + goTestnetFaucet, + resetSdk, + rewind + ) }) { paddingValues -> HomeMainContent( paddingValues = paddingValues, @@ -79,7 +86,8 @@ fun Home( private fun HomeTopAppBar( isTestnet: Boolean, goTestnetFaucet: () -> Unit, - resetSdk: () -> Unit + resetSdk: () -> Unit, + rewind: () -> Unit ) { TopAppBar( title = { Text(text = stringResource(id = R.string.app_name)) }, @@ -87,7 +95,8 @@ private fun HomeTopAppBar( DebugMenu( isTestnet, goTestnetFaucet = goTestnetFaucet, - resetSdk = resetSdk + resetSdk = resetSdk, + rewind = rewind, ) } ) @@ -97,7 +106,8 @@ private fun HomeTopAppBar( private fun DebugMenu( isTestnet: Boolean, goTestnetFaucet: () -> Unit, - resetSdk: () -> Unit + resetSdk: () -> Unit, + rewind: () -> Unit ) { var expanded by rememberSaveable { mutableStateOf(false) } IconButton(onClick = { expanded = true }) { @@ -117,6 +127,14 @@ private fun DebugMenu( } ) } + + DropdownMenuItem( + text = { Text("Quick Rewind") }, + onClick = { + rewind() + expanded = false + } + ) DropdownMenuItem( text = { Text("Reset SDK") }, onClick = { diff --git a/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/ui/screen/home/viewmodel/WalletViewModel.kt b/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/ui/screen/home/viewmodel/WalletViewModel.kt index da0f9967..e2a55937 100644 --- a/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/ui/screen/home/viewmodel/WalletViewModel.kt +++ b/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/ui/screen/home/viewmodel/WalletViewModel.kt @@ -234,6 +234,18 @@ class WalletViewModel(application: Application) : AndroidViewModel(application) fun resetSdk() { walletCoordinator.resetSdk() } + + /** + * This rewinds to the nearest height, i.e. 14 days back from the current chain tip. + */ + fun rewind() { + val synchronizer = synchronizer.value + if (null != synchronizer) { + viewModelScope.launch { + synchronizer.quickRewind() + } + } + } } /** diff --git a/sdk-incubator-lib/src/main/java/cash/z/ecc/android/sdk/WalletCoordinator.kt b/sdk-incubator-lib/src/main/java/cash/z/ecc/android/sdk/WalletCoordinator.kt index e0a1e98d..da3224f3 100644 --- a/sdk-incubator-lib/src/main/java/cash/z/ecc/android/sdk/WalletCoordinator.kt +++ b/sdk-incubator-lib/src/main/java/cash/z/ecc/android/sdk/WalletCoordinator.kt @@ -47,7 +47,7 @@ import java.util.UUID class WalletCoordinator( context: Context, val persistableWallet: Flow, - val syncAlgorithm: CompactBlockProcessor.SyncAlgorithm = CompactBlockProcessor.SyncAlgorithm.NON_LINEAR + val syncAlgorithm: CompactBlockProcessor.SyncAlgorithm = CompactBlockProcessor.SyncAlgorithm.SPEND_BEFORE_SYNC ) { private val applicationContext = context.applicationContext @@ -132,7 +132,7 @@ class WalletCoordinator( synchronizerMutex.withLock { synchronizer.value?.let { it.latestBirthdayHeight?.let { height -> - it.rewindToNearestHeight(height, true) + it.rewindToNearestHeight(height) return true } } diff --git a/sdk-lib/src/androidTest/java/cash/z/ecc/android/sdk/util/TestWallet.kt b/sdk-lib/src/androidTest/java/cash/z/ecc/android/sdk/util/TestWallet.kt index 992dbae8..522d0af0 100644 --- a/sdk-lib/src/androidTest/java/cash/z/ecc/android/sdk/util/TestWallet.kt +++ b/sdk-lib/src/androidTest/java/cash/z/ecc/android/sdk/util/TestWallet.kt @@ -106,7 +106,7 @@ class TestWallet( } suspend fun rewindToHeight(height: BlockHeight): TestWallet { - synchronizer.rewindToNearestHeight(height, false) + synchronizer.rewindToNearestHeight(height) return this } diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt index 3db96abe..05eae408 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt @@ -309,8 +309,8 @@ class SdkSynchronizer private constructor( override suspend fun getNearestRewindHeight(height: BlockHeight): BlockHeight = processor.getNearestRewindHeight(height) - override suspend fun rewindToNearestHeight(height: BlockHeight, alsoClearBlockCache: Boolean) { - processor.rewindToNearestHeight(height, alsoClearBlockCache) + override suspend fun rewindToNearestHeight(height: BlockHeight) { + processor.rewindToNearestHeight(height) } override suspend fun quickRewind() { diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/Synchronizer.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/Synchronizer.kt index d3bb3f72..0745b8bb 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/Synchronizer.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/Synchronizer.kt @@ -268,16 +268,27 @@ interface Synchronizer { */ suspend fun getTransparentBalance(tAddr: String): WalletBalance - suspend fun getNearestRewindHeight(height: BlockHeight): BlockHeight - /** * Returns the safest height to which we can rewind, given a desire to rewind to the height * provided. Due to how witness incrementing works, a wallet cannot simply rewind to any * arbitrary height. This handles all that complexity yet remains flexible in the future as * improvements are made. */ - suspend fun rewindToNearestHeight(height: BlockHeight, alsoClearBlockCache: Boolean = false) + suspend fun getNearestRewindHeight(height: BlockHeight): BlockHeight + /** + * Rewinds to the safest height to which we can rewind, given a desire to rewind to the height + * provided. Due to how witness incrementing works, a wallet cannot simply rewind to any + * arbitrary height. This handles all that complexity yet remains flexible in the future as + * improvements are made. + */ + suspend fun rewindToNearestHeight(height: BlockHeight) + + /** + * Rewinds to the safest height approximately 14 days backward from the current chain tip. Due to how witness + * incrementing works, a wallet cannot simply rewind to any arbitrary height. This handles all that complexity + * yet remains flexible in the future as improvements are made. + */ suspend fun quickRewind() /** diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt index 47999868..a9068cd9 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt @@ -66,7 +66,6 @@ import java.util.Locale import java.util.concurrent.atomic.AtomicInteger import kotlin.math.max import kotlin.math.min -import kotlin.math.roundToInt import kotlin.time.Duration import kotlin.time.Duration.Companion.days import kotlin.time.DurationUnit @@ -139,7 +138,7 @@ class CompactBlockProcessor internal constructor( private val _state: MutableStateFlow = MutableStateFlow(State.Initialized) private val _progress = MutableStateFlow(PercentDecimal.ZERO_PERCENT) - private val _processorInfo = MutableStateFlow(ProcessorInfo(null, null, null, null)) + private val _processorInfo = MutableStateFlow(ProcessorInfo(null, null, null)) private val _networkHeight = MutableStateFlow(null) private val processingMutex = Mutex() @@ -199,7 +198,7 @@ class CompactBlockProcessor internal constructor( ) // Download note commitment tree data from lightwalletd to decide if we communicate with linear - // or non-linear node. It depends on the syncAlgorithm property on the first place. + // or spend-before-sync node. It depends on the syncAlgorithm property on the first place. var subTreeRootResult = if (syncAlgorithm == SyncAlgorithm.LINEAR) { GetSubtreeRootsResult.UseLinear } else { @@ -219,7 +218,7 @@ class CompactBlockProcessor internal constructor( val result = processingMutex.withLockLogged("processNewBlocks") { when (subTreeRootResult) { is GetSubtreeRootsResult.UseNonLinear -> { - processNewBlocksInNonLinearOrder( + processNewBlocksInSbSOrder( backend = backend, downloader = downloader, repository = repository, @@ -259,10 +258,10 @@ class CompactBlockProcessor internal constructor( delay(napTime) } BlockProcessingResult.NoBlocksToProcess -> { - setState(State.Synced(_processorInfo.value.lastSyncRange)) + setState(State.Synced(_processorInfo.value.overallSyncRange)) // TODO [#1129]: Refactor work with lastSyncRange and lastSyncedHeight // TODO [#1129]: https://github.com/zcash/zcash-android-wallet-sdk/issues/1129 - val noWorkDone = _processorInfo.value.lastSyncRange?.isEmpty() ?: true + val noWorkDone = _processorInfo.value.overallSyncRange?.isEmpty() ?: true val summary = if (noWorkDone) { "Nothing to process: no new blocks to sync" } else { @@ -326,10 +325,10 @@ class CompactBlockProcessor internal constructor( private suspend fun processNewBlocksInLinearOrder(): BlockProcessingResult { Twig.debug { "Beginning to process new blocks with Linear approach (with lower bound: $lowerBoundHeight)..." } - return if (!updateRanges()) { + return if (!updateRange(null)) { Twig.warn { "Disconnection detected. Attempting to reconnect." } BlockProcessingResult.Reconnecting - } else if (_processorInfo.value.lastSyncRange.isNullOrEmpty()) { + } else if (_processorInfo.value.overallSyncRange.isNullOrEmpty()) { Twig.info { "No more blocks to process." } BlockProcessingResult.NoBlocksToProcess } else { @@ -345,7 +344,7 @@ class CompactBlockProcessor internal constructor( } benchmarkBlockRange } else { - _processorInfo.value.lastSyncRange!! + _processorInfo.value.overallSyncRange!! } syncBlocksAndEnhanceTransactions( @@ -383,11 +382,14 @@ class CompactBlockProcessor internal constructor( object NoRangeToVerify : VerifySuggestedScanRange() } - // TODO [#1137]: Refactor processNewBlocksInNonLinearOrder + // TODO [#1137]: Refactor processNewBlocksInSbSOrder // TODO [#1137]: https://github.com/zcash/zcash-android-wallet-sdk/issues/1137 + /** + * This function process the missing blocks in non-linear order with Spend-before-Sync algorithm. + */ @OptIn(ExperimentalStdlibApi::class) @Suppress("ReturnCount", "LongMethod", "CyclomaticComplexMethod", "LongParameterList") - private suspend fun processNewBlocksInNonLinearOrder( + private suspend fun processNewBlocksInSbSOrder( backend: TypesafeBackend, downloader: CompactBlockDownloader, repository: DerivedDataRepository, @@ -397,7 +399,7 @@ class CompactBlockProcessor internal constructor( firstUnenhancedHeight: BlockHeight? ): BlockProcessingResult { Twig.debug { - "Beginning to process new blocks with Non-linear approach (with roots: $subTreeRootList, and lower " + + "Beginning to process new blocks with Spend-before-Sync approach (with roots: $subTreeRootList, and lower" + "bound: $lastValidHeight)..." } @@ -441,15 +443,15 @@ class CompactBlockProcessor internal constructor( } } - setState(State.Syncing) - // Get the suggested scan ranges from the wallet database var suggestedRangesResult = suggestScanRanges( backend, lastValidHeight ) - when (suggestedRangesResult) { - is SuggestScanRangesResult.Success -> { /* Lets continue to the next step */ } + val updateRangeResult = when (suggestedRangesResult) { + is SuggestScanRangesResult.Success -> { + updateRange(suggestedRangesResult.ranges) + } is SuggestScanRangesResult.Failure -> { Twig.error { "Process suggested scan ranges failure: " + @@ -462,6 +464,16 @@ class CompactBlockProcessor internal constructor( } } + if (!updateRangeResult) { + Twig.warn { "Disconnection detected. Attempting to reconnect." } + return BlockProcessingResult.Reconnecting + } else if (_processorInfo.value.overallSyncRange.isNullOrEmpty()) { + Twig.info { "No more blocks to process." } + return BlockProcessingResult.NoBlocksToProcess + } + + setState(State.Syncing) + // Parse and process ranges. If it recognizes a range with Priority.Verify, it runs the verification part. var verifyRangeResult = shouldVerifySuggestedScanRanges(suggestedRangesResult) @@ -487,8 +499,7 @@ class CompactBlockProcessor internal constructor( withDownload = true, enhanceStartHeight = firstUnenhancedHeight ).collect { syncProgress -> - _progress.value = syncProgress.percentage - updateProgress(lastSyncedHeight = syncProgress.lastSyncedHeight) + setProgress(syncProgress.percentage) when (syncProgress.resultState) { SyncingResult.UpdateBirthday -> { @@ -565,8 +576,7 @@ class CompactBlockProcessor internal constructor( withDownload = true, enhanceStartHeight = firstUnenhancedHeight ).collect { syncProgress -> - _progress.value = syncProgress.percentage - updateProgress(lastSyncedHeight = syncProgress.lastSyncedHeight) + setProgress(syncProgress.percentage) when (syncProgress.resultState) { SyncingResult.UpdateBirthday -> { @@ -613,8 +623,7 @@ class CompactBlockProcessor internal constructor( withDownload = withDownload, enhanceStartHeight = enhanceStartHeight ).collect { syncProgress -> - _progress.value = syncProgress.percentage - updateProgress(lastSyncedHeight = syncProgress.lastSyncedHeight) + setProgress(syncProgress.percentage) when (syncProgress.resultState) { SyncingResult.UpdateBirthday -> { @@ -652,11 +661,17 @@ class CompactBlockProcessor internal constructor( /** * Gets the latest range info and then uses that initialInfo to update (and transmit) - * the scan/download ranges that require processing. + * the info that require processing. This function is universal and works for both [SyncAlgorithm.LINEAR] and + * [SyncAlgorithm.SPEND_BEFORE_SYNC] algorithms. + * + * @param ranges The ranges which we obtained from the rust layer to proceed in case of + * [SyncAlgorithm.SPEND_BEFORE_SYNC] algorithm, or null in case of [SyncAlgorithm.LINEAR] algorithm, as it will + * be computed. * * @return true when the update succeeds. */ - private suspend fun updateRanges(): Boolean { + @OptIn(ExperimentalStdlibApi::class) + private suspend fun updateRange(ranges: List?): Boolean { // This fetches the latest height each time this method is called, which can be very inefficient // when downloading all of the blocks from the server val networkBlockHeight = fetchLatestBlockHeight(downloader, network) ?: return false @@ -688,10 +703,30 @@ class CompactBlockProcessor internal constructor( // Get the first un-enhanced transaction from the repository val firstUnenhancedHeight = getFirstUnenhancedHeight(repository) - updateProgress( + // The sync range computation depends on the used sync algorithm. The LINEAR one will compute it by itself, + // when the SPEND_BEFORE_SYNC will use the one obtained from the rust layer + val syncRange = if (ranges == null) { + lastSyncedHeight + 1..networkBlockHeight + } else if (ranges.isNotEmpty()) { + var resultRange = ranges[0].range.start..ranges[0].range.endExclusive + ranges.forEach { nextRange -> + if (nextRange.range.start < resultRange.start) { + resultRange = nextRange.range.start..resultRange.endInclusive + } + if (nextRange.range.endExclusive > resultRange.endInclusive) { + resultRange = resultRange.start..nextRange.range.endExclusive + } + } + resultRange + } else { + // Empty ranges most likely means that the SbS is done and the Rust layer replied with an empty suggested + // ranges + null + } + + setProcessorInfo( networkBlockHeight = networkBlockHeight, - lastSyncedHeight = lastSyncedHeight, - lastSyncRange = lastSyncedHeight + 1..networkBlockHeight, + overallSyncRange = syncRange, firstUnenhancedHeight = firstUnenhancedHeight ) @@ -968,7 +1003,7 @@ class CompactBlockProcessor internal constructor( /** * This operation downloads note commitment tree data from the lightwalletd server to decide if we communicate - * with node which provides linear or non-linear synchronization mode. + * with linear or spend-before-sync node * * @return GetSubtreeRootsResult as a wrapper for the lightwalletd response result */ @@ -1675,30 +1710,42 @@ class CompactBlockProcessor internal constructor( * * @param networkBlockHeight the latest block available to lightwalletd that may or may not be * downloaded by this wallet yet. - * @param lastSyncedHeight the height up to which the wallet last synced. This determines - * where the next sync will begin. - * @param lastSyncRange the inclusive range to sync. This represents what we most recently + * @param overallSyncRange the inclusive range to sync. This represents what we most recently * wanted to sync. In most cases, it will be an invalid range because we'd like to sync blocks * that we don't yet have. * @param firstUnenhancedHeight the height at which the enhancing should start. Use null if you have no * preferences. The height will be calculated automatically for you to continue where it previously ended, or * it'll be set to the sync start height in case of the first sync attempt. */ - private fun updateProgress( + private fun setProcessorInfo( networkBlockHeight: BlockHeight? = _processorInfo.value.networkBlockHeight, - lastSyncedHeight: BlockHeight? = _processorInfo.value.lastSyncedHeight, - lastSyncRange: ClosedRange? = _processorInfo.value.lastSyncRange, + overallSyncRange: ClosedRange? = _processorInfo.value.overallSyncRange, firstUnenhancedHeight: BlockHeight? = _processorInfo.value.firstUnenhancedHeight, ) { _networkHeight.value = networkBlockHeight _processorInfo.value = ProcessorInfo( networkBlockHeight = networkBlockHeight, - lastSyncedHeight = lastSyncedHeight, - lastSyncRange = lastSyncRange, + overallSyncRange = overallSyncRange, firstUnenhancedHeight = firstUnenhancedHeight ) } + /** + * Emit an instance of progress. + * + * @param progress the block syncing progress of type [PercentDecimal] in the range of [0, 1] + */ + private fun setProgress(progress: PercentDecimal = _progress.value) { + _progress.value = progress + } + + /** + * Transmits the given state for this processor. + */ + private suspend fun setState(newState: State) { + _state.value = newState + } + private suspend fun handleChainError(errorHeight: BlockHeight) { // TODO [#683]: Consider an error object containing hash information // TODO [#683]: https://github.com/zcash/zcash-android-wallet-sdk/issues/683 @@ -1706,7 +1753,7 @@ class CompactBlockProcessor internal constructor( determineLowerBound(errorHeight).let { lowerBound -> Twig.debug { "Handling chain error at $errorHeight by rewinding to block $lowerBound" } onChainErrorListener?.invoke(errorHeight, lowerBound) - rewindToNearestHeight(lowerBound, true) + rewindToNearestHeight(lowerBound) } } @@ -1729,106 +1776,43 @@ class CompactBlockProcessor internal constructor( * Rewind back at least two weeks worth of blocks. */ suspend fun quickRewind() { - val height = max(_processorInfo.value.lastSyncedHeight, repository.lastScannedHeight()) + val height = repository.lastScannedHeight() val blocksPer14Days = 14.days.inWholeMilliseconds / ZcashSdk.BLOCK_INTERVAL_MILLIS.toInt() val twoWeeksBack = BlockHeight.new( network, (height.value - blocksPer14Days).coerceAtLeast(lowerBoundHeight.value) ) - rewindToNearestHeight(twoWeeksBack, false) + rewindToNearestHeight(twoWeeksBack) } - /** - * @param alsoClearBlockCache when true, also clear the block cache which forces a redownload of - * blocks. Otherwise, the cached blocks will be used in the rescan, which in most cases, is fine. - */ @Suppress("LongMethod") - suspend fun rewindToNearestHeight( - height: BlockHeight, - alsoClearBlockCache: Boolean = false - ) { + suspend fun rewindToNearestHeight(height: BlockHeight) { processingMutex.withLockLogged("rewindToHeight") { - val lastSyncedHeight = _processorInfo.value.lastSyncedHeight val lastLocalBlock = repository.lastScannedHeight() val targetHeight = getNearestRewindHeight(height) Twig.debug { - "Rewinding from $lastSyncedHeight to requested height: $height using target height: " + - "$targetHeight with last local block: $lastLocalBlock" + "Rewinding to requested height: $height using target height: $targetHeight with last local block:" + + " $lastLocalBlock" } - if (null == lastSyncedHeight && targetHeight < lastLocalBlock) { + if (targetHeight < lastLocalBlock) { Twig.debug { "Rewinding because targetHeight is less than lastLocalBlock." } runCatching { backend.rewindToHeight(targetHeight) + downloader.rewindToHeight(targetHeight) }.onFailure { Twig.error { "Rewinding to the targetHeight $targetHeight failed with $it" } - } - } else if (null != lastSyncedHeight && targetHeight < lastSyncedHeight) { - Twig.debug { "Rewinding because targetHeight is less than lastSyncedHeight." } - runCatching { - backend.rewindToHeight(targetHeight) - }.onFailure { - Twig.error { "Rewinding to the targetHeight $targetHeight failed with $it" } + }.onSuccess { + Twig.info { "Rewind to $targetHeight was successful." } + setState(newState = State.Syncing) + setProgress(progress = PercentDecimal.ZERO_PERCENT) + setProcessorInfo(overallSyncRange = null) } } else { - Twig.debug { - "Not rewinding dataDb because the last synced height is $lastSyncedHeight and the" + - " last local block is $lastLocalBlock both of which are less than the target height of " + - "$targetHeight" - } - } - - val currentNetworkBlockHeight = _processorInfo.value.networkBlockHeight - - if (alsoClearBlockCache) { - Twig.debug { - "Also clearing block cache back to $targetHeight. These rewound blocks will download " + - "in the next scheduled scan" - } - downloader.rewindToHeight(targetHeight) - // communicate that the wallet is no longer synced because it might remain this way for 20+ second - // because we only download on 20s time boundaries so we can't trigger any immediate action - setState(State.Syncing) - if (null == currentNetworkBlockHeight) { - updateProgress( - lastSyncedHeight = targetHeight, - lastSyncRange = null - ) - } else { - updateProgress( - lastSyncedHeight = targetHeight, - lastSyncRange = (targetHeight + 1)..currentNetworkBlockHeight - ) - } - _progress.value = PercentDecimal.ZERO_PERCENT - } else { - if (null == currentNetworkBlockHeight) { - updateProgress( - lastSyncedHeight = targetHeight, - lastSyncRange = null - ) - } else { - updateProgress( - lastSyncedHeight = targetHeight, - lastSyncRange = (targetHeight + 1)..currentNetworkBlockHeight - ) - } - - _progress.value = PercentDecimal.ZERO_PERCENT - - if (null != lastSyncedHeight) { - val range = (targetHeight + 1)..lastSyncedHeight - Twig.debug { - "We kept the cache blocks in place so we don't need to wait for the next scheduled download " + - "to rescan. Instead we will rescan blocks ${range.start}..${range.endInclusive}" - } - - syncBlocksAndEnhanceTransactions( - syncRange = range, - withDownload = false, - enhanceStartHeight = null - ) + Twig.info { + "Not rewinding dataDb because last local block is $lastLocalBlock which is less than the target " + + "height of $targetHeight" } } } @@ -1944,13 +1928,6 @@ class CompactBlockProcessor internal constructor( suspend fun getUtxoCacheBalance(address: String): WalletBalance = backend.getDownloadedUtxoBalance(address) - /** - * Transmits the given state for this processor. - */ - private suspend fun setState(newState: State) { - _state.value = newState - } - /** * Sealed class representing the various states of this processor. */ @@ -2032,64 +2009,33 @@ class CompactBlockProcessor internal constructor( * * @param networkBlockHeight the latest block available to lightwalletd that may or may not be * downloaded by this wallet yet. - * @param lastSyncedHeight the height up to which the wallet last synced. This determines - * where the next sync will begin. - * @param lastSyncRange inclusive range to sync. Meaning, if the range is 10..10, + * @param overallSyncRange inclusive range to sync. Meaning, if the range is 10..10, * then we will download exactly block 10. If the range is 11..10, then we want to download * block 11 but can't. * @param firstUnenhancedHeight the height in which the enhancing should start, or null in case of no previous * transaction enhancing done yet */ - // TODO [#1129]: Refactor work with lastSyncRange and lastSyncedHeight - // TODO [#1129]: https://github.com/zcash/zcash-android-wallet-sdk/issues/1129 data class ProcessorInfo( val networkBlockHeight: BlockHeight?, - val lastSyncedHeight: BlockHeight?, - val lastSyncRange: ClosedRange?, + val overallSyncRange: ClosedRange?, val firstUnenhancedHeight: BlockHeight? - ) { - /** - * Determines whether this instance is actively syncing compact blocks. - * - * @return true when there are more than zero blocks remaining to sync. - */ - val isSyncing: Boolean - get() = - lastSyncedHeight != null && - lastSyncRange != null && - !lastSyncRange.isEmpty() && - lastSyncedHeight < lastSyncRange.endInclusive - - /** - * The amount of sync progress from 0 to 100. - */ - @Suppress("MagicNumber") - val syncProgress - get() = when { - lastSyncedHeight == null -> 0 - lastSyncRange == null -> 100 - lastSyncedHeight >= lastSyncRange.endInclusive -> 100 - else -> { - // when lastSyncedHeight == lastSyncedRange.first, we have synced one block, thus the offsets - val blocksSynced = - (lastSyncedHeight.value - lastSyncRange.start.value + 1).coerceAtLeast(0) - // we sync the range inclusively so 100..100 is one block to sync, thus the offset - val numberOfBlocks = - lastSyncRange.endInclusive.value - lastSyncRange.start.value + 1 - // take the percentage then convert and round - ((blocksSynced.toFloat() / numberOfBlocks) * 100.0f).coerceAtMost(100.0f).roundToInt() - } - } - } + ) data class ValidationErrorInfo( val errorHeight: BlockHeight, val hash: String? ) + /** + * Algorithm used to sync the SDK with the blockchain + */ enum class SyncAlgorithm { + // Linear sync processes the un-synced blocks in a linear way up to the chain tip LINEAR, - NON_LINEAR + + // Spend before Sync processes the un-synced blocks non-linearly, in prioritised ranges relevant to the stored + // wallet. Note: This feature is in development (alpha version) so use carefully. + SPEND_BEFORE_SYNC } // @@ -2120,11 +2066,3 @@ private fun LightWalletEndpointInfoUnsafe.matchingNetwork(network: String): Bool } return chainName.toId() == network.toId() } - -private fun max(a: BlockHeight?, b: BlockHeight) = if (null == a) { - b -} else if (a.value > b.value) { - a -} else { - b -} diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/transaction/TransactionEncoderImpl.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/transaction/TransactionEncoderImpl.kt index dbe53e75..54dc4b80 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/transaction/TransactionEncoderImpl.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/transaction/TransactionEncoderImpl.kt @@ -125,7 +125,7 @@ internal class TransactionEncoderImpl( ): FirstClassByteArray { Twig.debug { "creating transaction to spend $amount zatoshi to" + - " ${toAddress.masked()} with memo $memo" + " ${toAddress.masked()} with memo: ${memo?.contentToString()}" } @Suppress("TooGenericExceptionCaught") From fd17e7ef0e585be143ca636abb45c11eeda0bf8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Honza=20Rychnovsk=C3=BD?= Date: Tue, 15 Aug 2023 13:07:38 +0200 Subject: [PATCH 21/76] [#1159] Update sync progress reporting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [#1168] Checkpoints update * [#1159] Updated sync progress reporting * Increase test_robo_demo_app timeout * Migrate to ClosedEndRange It’s better to transform suggested ranges from OpenEndRange to ClosedRange as soon as possible to avoid its handling in the rest of the logic. * Improve all batch count calculating * Subsequent SbS sync algorithm renaming --- .github/workflows/pull-request.yml | 2 +- .../sdk/block/CompactBlockProcessor.kt | 172 +++++++++++------- .../android/sdk/internal/model/BlockBatch.kt | 6 +- .../android/sdk/internal/model/ScanRange.kt | 12 +- 4 files changed, 124 insertions(+), 68 deletions(-) diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 1656d313..1ecbe9ad 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -395,7 +395,7 @@ jobs: with: name: Demo app release binaries - name: Robo test - timeout-minutes: 15 + timeout-minutes: 20 env: # Path depends on `release_build` job, plus path of `Download a single artifact` step BINARIES_ZIP_PATH: binaries.zip diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt index a9068cd9..af376755 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt @@ -22,7 +22,6 @@ import cash.z.ecc.android.sdk.internal.ext.length import cash.z.ecc.android.sdk.internal.ext.retryUpTo import cash.z.ecc.android.sdk.internal.ext.retryUpToAndContinue import cash.z.ecc.android.sdk.internal.ext.retryWithBackoff -import cash.z.ecc.android.sdk.internal.ext.toClosedRange import cash.z.ecc.android.sdk.internal.ext.toHexReversed import cash.z.ecc.android.sdk.internal.model.BlockBatch import cash.z.ecc.android.sdk.internal.model.DbTransactionOverview @@ -217,13 +216,13 @@ class CompactBlockProcessor internal constructor( ) { val result = processingMutex.withLockLogged("processNewBlocks") { when (subTreeRootResult) { - is GetSubtreeRootsResult.UseNonLinear -> { + is GetSubtreeRootsResult.UseSbS -> { processNewBlocksInSbSOrder( backend = backend, downloader = downloader, repository = repository, network = network, - subTreeRootList = (subTreeRootResult as GetSubtreeRootsResult.UseNonLinear) + subTreeRootList = (subTreeRootResult as GetSubtreeRootsResult.UseSbS) .subTreeRootList, lastValidHeight = lowerBoundHeight, firstUnenhancedHeight = _processorInfo.value.firstUnenhancedHeight @@ -323,7 +322,7 @@ class CompactBlockProcessor internal constructor( } private suspend fun processNewBlocksInLinearOrder(): BlockProcessingResult { - Twig.debug { "Beginning to process new blocks with Linear approach (with lower bound: $lowerBoundHeight)..." } + Twig.info { "Beginning to process new blocks with Linear approach (with lower bound: $lowerBoundHeight)..." } return if (!updateRange(null)) { Twig.warn { "Disconnection detected. Attempting to reconnect." } @@ -347,7 +346,7 @@ class CompactBlockProcessor internal constructor( _processorInfo.value.overallSyncRange!! } - syncBlocksAndEnhanceTransactions( + syncBlocksAndEnhanceTransactionsLinearly( syncRange = syncRange, withDownload = true, enhanceStartHeight = _processorInfo.value.firstUnenhancedHeight @@ -361,7 +360,8 @@ class CompactBlockProcessor internal constructor( } internal sealed class GetSubtreeRootsResult { - data class UseNonLinear(val subTreeRootList: List) : GetSubtreeRootsResult() + // SbS: Spend-before-Sync + data class UseSbS(val subTreeRootList: List) : GetSubtreeRootsResult() object UseLinear : GetSubtreeRootsResult() object FailureConnection : GetSubtreeRootsResult() data class OtherFailure(val exception: Throwable) : GetSubtreeRootsResult() @@ -398,7 +398,7 @@ class CompactBlockProcessor internal constructor( lastValidHeight: BlockHeight, firstUnenhancedHeight: BlockHeight? ): BlockProcessingResult { - Twig.debug { + Twig.info { "Beginning to process new blocks with Spend-before-Sync approach (with roots: $subTreeRootList, and lower" + "bound: $lastValidHeight)..." } @@ -473,6 +473,8 @@ class CompactBlockProcessor internal constructor( } setState(State.Syncing) + val allBatchCount = getBatchCount(suggestedRangesResult.ranges.map { it.range }).toFloat() + var lastBatchOrder: Long = 0 // Parse and process ranges. If it recognizes a range with Priority.Verify, it runs the verification part. var verifyRangeResult = shouldVerifySuggestedScanRanges(suggestedRangesResult) @@ -490,23 +492,26 @@ class CompactBlockProcessor internal constructor( ) var syncingResult: SyncingResult = SyncingResult.AllSuccess - runSyncingAndEnhancing( + runSyncingAndEnhancingOnRange( backend = backend, downloader = downloader, repository = repository, network = network, - syncRange = verifyRangeResult.scanRange.range.toClosedRange(), + syncRange = verifyRangeResult.scanRange.range, withDownload = true, - enhanceStartHeight = firstUnenhancedHeight - ).collect { syncProgress -> - setProgress(syncProgress.percentage) + enhanceStartHeight = firstUnenhancedHeight, + lastBatchOrder = lastBatchOrder + ).collect { rangeSyncProgress -> + setProgress(PercentDecimal(rangeSyncProgress.overallOrder / allBatchCount)) + // We need to update lastBatchOrder for the next ranges processing + lastBatchOrder = rangeSyncProgress.overallOrder - when (syncProgress.resultState) { + when (rangeSyncProgress.resultState) { SyncingResult.UpdateBirthday -> { updateBirthdayHeight() } is SyncingResult.Failure -> { - syncingResult = syncProgress.resultState + syncingResult = rangeSyncProgress.resultState return@collect } else -> { // Continue with processing @@ -567,23 +572,25 @@ class CompactBlockProcessor internal constructor( // TODO [#1145]: Sync Historic range in reverse order // TODO [#1145]: https://github.com/zcash/zcash-android-wallet-sdk/issues/1145 var syncingResult: SyncingResult = SyncingResult.AllSuccess - runSyncingAndEnhancing( + runSyncingAndEnhancingOnRange( backend = backend, downloader = downloader, repository = repository, network = network, - syncRange = scanRange.range.toClosedRange(), + syncRange = scanRange.range, withDownload = true, - enhanceStartHeight = firstUnenhancedHeight - ).collect { syncProgress -> - setProgress(syncProgress.percentage) + enhanceStartHeight = firstUnenhancedHeight, + lastBatchOrder = lastBatchOrder + ).collect { rangeSyncProgress -> + setProgress(PercentDecimal(rangeSyncProgress.overallOrder / allBatchCount)) + lastBatchOrder = rangeSyncProgress.overallOrder - when (syncProgress.resultState) { + when (rangeSyncProgress.resultState) { SyncingResult.UpdateBirthday -> { updateBirthdayHeight() } is SyncingResult.Failure -> { - syncingResult = syncProgress.resultState + syncingResult = rangeSyncProgress.resultState return@collect } else -> { // Continue with processing @@ -607,30 +614,33 @@ class CompactBlockProcessor internal constructor( } @Suppress("ReturnCount") - private suspend fun syncBlocksAndEnhanceTransactions( + private suspend fun syncBlocksAndEnhanceTransactionsLinearly( syncRange: ClosedRange, withDownload: Boolean, enhanceStartHeight: BlockHeight? ): BlockProcessingResult { - // Syncing last blocks and enhancing transactions var syncingResult: SyncingResult = SyncingResult.AllSuccess - runSyncingAndEnhancing( + val allBatchCount = getBatchCount(listOf(syncRange)).toFloat() + + // Syncing last blocks and enhancing transactions + runSyncingAndEnhancingOnRange( backend = backend, downloader = downloader, repository = repository, network = network, syncRange = syncRange, withDownload = withDownload, - enhanceStartHeight = enhanceStartHeight - ).collect { syncProgress -> - setProgress(syncProgress.percentage) + enhanceStartHeight = enhanceStartHeight, + lastBatchOrder = 0 + ).collect { rangeSyncProgress -> + setProgress(PercentDecimal(rangeSyncProgress.overallOrder / allBatchCount)) - when (syncProgress.resultState) { + when (rangeSyncProgress.resultState) { SyncingResult.UpdateBirthday -> { updateBirthdayHeight() } is SyncingResult.Failure -> { - syncingResult = syncProgress.resultState + syncingResult = rangeSyncProgress.resultState return@collect } else -> { // Continue with processing @@ -708,13 +718,13 @@ class CompactBlockProcessor internal constructor( val syncRange = if (ranges == null) { lastSyncedHeight + 1..networkBlockHeight } else if (ranges.isNotEmpty()) { - var resultRange = ranges[0].range.start..ranges[0].range.endExclusive + var resultRange = ranges[0].range.start..ranges[0].range.endInclusive ranges.forEach { nextRange -> if (nextRange.range.start < resultRange.start) { resultRange = nextRange.range.start..resultRange.endInclusive } - if (nextRange.range.endExclusive > resultRange.endInclusive) { - resultRange = resultRange.start..nextRange.range.endExclusive + if (nextRange.range.endInclusive > resultRange.endInclusive) { + resultRange = resultRange.start..nextRange.range.endInclusive } } resultRange @@ -1064,7 +1074,7 @@ class CompactBlockProcessor internal constructor( result = if (it.isEmpty()) { GetSubtreeRootsResult.UseLinear } else { - GetSubtreeRootsResult.UseNonLinear(it) + GetSubtreeRootsResult.UseSbS(it) } } } @@ -1135,7 +1145,7 @@ class CompactBlockProcessor internal constructor( } /** - * Get the suggested scan ranges from the wallet database. + * Get the suggested scan ranges from the wallet database via the rust layer. * * @param backend Typesafe Rust backend * @param lastValidHeight The height to which rewind in case of any trouble @@ -1238,12 +1248,14 @@ class CompactBlockProcessor internal constructor( * processed existing blocks * @param enhanceStartHeight the height in which the enhancing should start, or null in case of no previous * transaction enhancing done yet + * @param lastBatchOrder is the order of the last processed batch. It comes from a previous range processing + * and is necessary for calculating cross ranges batch order of currently processing batches. - * @return Flow of BatchSyncProgress sync and enhancement results + * @return Flow of [BatchSyncProgress] sync and enhancement results */ @VisibleForTesting @Suppress("LongParameterList", "LongMethod") - internal suspend fun runSyncingAndEnhancing( + internal suspend fun runSyncingAndEnhancingOnRange( backend: TypesafeBackend, downloader: CompactBlockDownloader, repository: DerivedDataRepository, @@ -1251,20 +1263,19 @@ class CompactBlockProcessor internal constructor( syncRange: ClosedRange, withDownload: Boolean, enhanceStartHeight: BlockHeight?, + lastBatchOrder: Long ): Flow = flow { if (syncRange.isEmpty()) { Twig.debug { "No blocks to sync" } emit( BatchSyncProgress( - percentage = PercentDecimal.ONE_HUNDRED_PERCENT, - lastSyncedHeight = getLastScannedHeight(repository), resultState = SyncingResult.AllSuccess ) ) } else { Twig.info { "Syncing blocks in range $syncRange" } - val batches = getBatchedBlockList(syncRange, network) + val batches = getBatchedBlockList(lastBatchOrder, syncRange, network) // Check for the last enhanced height and eventually set is as the beginning of the next enhancing range var enhancingRange = if (enhanceStartHeight != null) { @@ -1333,8 +1344,8 @@ class CompactBlockProcessor internal constructor( emit( BatchSyncProgress( - percentage = PercentDecimal(continuousResult.batch.order / batches.size.toFloat()), - lastSyncedHeight = getLastScannedHeight(repository), + inRangeOrder = continuousResult.batch.inRangeOrder, + overallOrder = continuousResult.batch.crossRangesOrder, resultState = resultState ) ) @@ -1342,11 +1353,11 @@ class CompactBlockProcessor internal constructor( // Increment and compare the range for triggering the enhancing enhancingRange = enhancingRange.start..continuousResult.batch.range.endInclusive - // Enhance is run in case of the range is on or over its limit, or in case of any failure + // Enhancing is run in case of the range is on or over its limit, or in case of any failure // state comes from the previous stages, or if the end of the sync range is reached if (enhancingRange.length() >= ENHANCE_BATCH_SIZE || resultState != SyncingResult.AllSuccess || - continuousResult.batch.order == batches.size.toLong() + continuousResult.batch.inRangeOrder == batches.size.toLong() ) { // Copy the range for use and reset for the next iteration val currentEnhancingRange = enhancingRange @@ -1374,16 +1385,16 @@ class CompactBlockProcessor internal constructor( } emit( BatchSyncProgress( - percentage = PercentDecimal(continuousResult.batch.order / batches.size.toFloat()), - lastSyncedHeight = getLastScannedHeight(repository), + inRangeOrder = continuousResult.batch.inRangeOrder, + overallOrder = continuousResult.batch.crossRangesOrder, resultState = resultState ) ) } } - Twig.debug { - "All sync stages done for the batch: ${continuousResult.batch} with result state: " + - "$resultState" + Twig.info { + "All sync stages done for the batch ${continuousResult.batch.inRangeOrder}/${batches.size}:" + + " ${continuousResult.batch} with result state: $resultState" } }.takeWhile { batchProcessResult -> batchProcessResult.stageResult == SyncingResult.DeleteSuccess || @@ -1392,20 +1403,51 @@ class CompactBlockProcessor internal constructor( } } + /** + * Returns count of batches of blocks across all ranges. It works the same when triggered from the Linear + * synchronization or from the SbS synchronization. + * + * @param syncRanges List of ranges of all blocks to process + * + * @return Count of all batches for processing + */ + private fun getBatchCount(syncRanges: List>): Long { + var allRangesBatchCount = 0L + var allMissingBlocksCount = 0L + + syncRanges.forEach { range -> + val missingBlockCount = range.endInclusive.value - range.start.value + 1 + val batchCount = ( + missingBlockCount / SYNC_BATCH_SIZE + + (if (missingBlockCount.rem(SYNC_BATCH_SIZE) == 0L) 0 else 1) + ) + allMissingBlocksCount += missingBlockCount + allRangesBatchCount += batchCount + } + + Twig.debug { + "Found $allMissingBlocksCount missing blocks, syncing in $allRangesBatchCount batches of " + + "$SYNC_BATCH_SIZE..." + } + return allRangesBatchCount + } + + /** + * Prepare list of all [BlockBatch] internal objects to be processed during a range of + * blocks processing + * + * @param lastBatchOrder The index of the last previously processed batch + * @param syncRange Current range to be processed + * @param network The network we are operating on + * + * @return List of [BlockBatch] to for synchronization + */ private fun getBatchedBlockList( + lastBatchOrder: Long, syncRange: ClosedRange, network: ZcashNetwork ): List { - val missingBlockCount = syncRange.endInclusive.value - syncRange.start.value + 1 - val batchCount = ( - missingBlockCount / SYNC_BATCH_SIZE + - (if (missingBlockCount.rem(SYNC_BATCH_SIZE) == 0L) 0 else 1) - ) - - Twig.debug { - "Found $missingBlockCount missing blocks, syncing in $batchCount batches of $SYNC_BATCH_SIZE..." - } - + val batchCount = getBatchCount(listOf(syncRange)) var start = syncRange.start return buildList { for (index in 1..batchCount) { @@ -1417,7 +1459,13 @@ class CompactBlockProcessor internal constructor( ) ) // subtract 1 on the first value because the range is inclusive - add(BlockBatch(index, start..end)) + add( + BlockBatch( + inRangeOrder = index, + crossRangesOrder = lastBatchOrder + index, + range = start..end + ) + ) start = end + 1 } } @@ -1991,9 +2039,9 @@ class CompactBlockProcessor internal constructor( * Progress model class for sharing the whole batch sync progress out of the sync process. */ internal data class BatchSyncProgress( - val percentage: PercentDecimal, - val lastSyncedHeight: BlockHeight?, - val resultState: SyncingResult + val inRangeOrder: Long = 0, + val overallOrder: Long = 0, + val resultState: SyncingResult = SyncingResult.AllSuccess ) /** diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/BlockBatch.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/BlockBatch.kt index 5891307b..9b8cdf4b 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/BlockBatch.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/BlockBatch.kt @@ -3,9 +3,11 @@ package cash.z.ecc.android.sdk.internal.model import cash.z.ecc.android.sdk.model.BlockHeight internal data class BlockBatch( - val order: Long, + val inRangeOrder: Long, + val crossRangesOrder: Long, val range: ClosedRange, var blocks: List? = null ) { - override fun toString() = "BlockBatch(order=$order, range=$range, blocks=${blocks?.size ?: "null"})" + override fun toString() = "BlockBatch(crossRangesOrder=$crossRangesOrder, inRangeOrder=$inRangeOrder, " + + "range=$range, blocks=${blocks?.size ?: "null"})" } diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanRange.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanRange.kt index e6aee321..16b1ca01 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanRange.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanRange.kt @@ -1,12 +1,12 @@ package cash.z.ecc.android.sdk.internal.model import cash.z.ecc.android.sdk.internal.Twig +import cash.z.ecc.android.sdk.internal.ext.toClosedRange import cash.z.ecc.android.sdk.model.BlockHeight import cash.z.ecc.android.sdk.model.ZcashNetwork -@OptIn(ExperimentalStdlibApi::class) internal data class ScanRange( - val range: OpenEndRange, + val range: ClosedRange, val priority: Long ) { override fun toString() = "ScanRange(range=$range, priority=${getSuggestScanRangePriority()})" @@ -18,9 +18,15 @@ internal data class ScanRange( } companion object { + /** + * Note that this function also transforms the suggested ranges from [OpenEndRange] to [ClosedRange] so the + * rest of the logic can safely work with a unified range type. + */ + @OptIn(ExperimentalStdlibApi::class) fun new(jni: JniScanRange, zcashNetwork: ZcashNetwork): ScanRange { return ScanRange( - range = BlockHeight.new(zcashNetwork, jni.startHeight).. Date: Wed, 16 Aug 2023 12:59:52 +0200 Subject: [PATCH 22/76] Minor formatting changes --- .../java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt | 1 - .../android/sdk/internal/transaction/TransactionEncoderImpl.kt | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt index af376755..fa877c4a 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt @@ -387,7 +387,6 @@ class CompactBlockProcessor internal constructor( /** * This function process the missing blocks in non-linear order with Spend-before-Sync algorithm. */ - @OptIn(ExperimentalStdlibApi::class) @Suppress("ReturnCount", "LongMethod", "CyclomaticComplexMethod", "LongParameterList") private suspend fun processNewBlocksInSbSOrder( backend: TypesafeBackend, diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/transaction/TransactionEncoderImpl.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/transaction/TransactionEncoderImpl.kt index 54dc4b80..7630551c 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/transaction/TransactionEncoderImpl.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/transaction/TransactionEncoderImpl.kt @@ -125,7 +125,7 @@ internal class TransactionEncoderImpl( ): FirstClassByteArray { Twig.debug { "creating transaction to spend $amount zatoshi to" + - " ${toAddress.masked()} with memo: ${memo?.contentToString()}" + " ${toAddress.masked()} with memo: ${memo?.decodeToString()}" } @Suppress("TooGenericExceptionCaught") From cc3129d0a523d25b1c26e6ee83d00e9b9e4c6fef Mon Sep 17 00:00:00 2001 From: Honza Date: Wed, 16 Aug 2023 14:06:16 +0200 Subject: [PATCH 23/76] Fix selectionArgs input type --- .../z/ecc/android/sdk/internal/db/derived/TransactionTable.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/TransactionTable.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/TransactionTable.kt index 6cac3446..ed99bb02 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/TransactionTable.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/TransactionTable.kt @@ -71,7 +71,7 @@ internal class TransactionTable( table = TransactionTableDefinition.TABLE_NAME, columns = PROJECTION_ENCODED_TRANSACTION, selection = SELECTION_TRANSACTION_ID_AND_RAW_NOT_NULL, - selectionArgs = arrayOf(txId) + selectionArgs = arrayOf(txId.byteArray) ) { val rawIndex = it.getColumnIndexOrThrow(TransactionTableDefinition.COLUMN_BLOB_RAW) val heightIndex = it.getColumnIndexOrThrow(TransactionTableDefinition.COLUMN_INTEGER_EXPIRY_HEIGHT) From 6e7d2404dcde6b2cab78fb64d2e7b4b5af63314d Mon Sep 17 00:00:00 2001 From: Honza Date: Wed, 16 Aug 2023 14:25:17 +0200 Subject: [PATCH 24/76] Fix query db projection in getSaplingOutputIndices --- .../ecc/android/sdk/internal/db/derived/TxOutputsView.kt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/TxOutputsView.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/TxOutputsView.kt index 80432228..4a150ca3 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/TxOutputsView.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/TxOutputsView.kt @@ -20,7 +20,7 @@ internal class TxOutputsView( TxOutputsViewDefinition.COLUMN_INTEGER_TRANSACTION_ID ) - private val PROJECTION_ID = arrayOf(TxOutputsViewDefinition.COLUMN_INTEGER_TRANSACTION_ID) + private val PROJECTION_OUTPUT_INDEX = arrayOf(TxOutputsViewDefinition.COLUMN_INTEGER_OUTPUT_INDEX) private val PROJECTION_RECIPIENT = arrayOf( TxOutputsViewDefinition.COLUMN_STRING_TO_ADDRESS, @@ -38,14 +38,14 @@ internal class TxOutputsView( fun getSaplingOutputIndices(transactionId: Long) = sqliteDatabase.queryAndMap( table = TxOutputsViewDefinition.VIEW_NAME, - columns = PROJECTION_ID, + columns = PROJECTION_OUTPUT_INDEX, selection = SELECT_BY_TRANSACTION_ID_AND_NOT_CHANGE, selectionArgs = arrayOf(transactionId), orderBy = ORDER_BY, cursorParser = { - val idColumnIndex = it.getColumnIndex(TxOutputsViewDefinition.COLUMN_INTEGER_OUTPUT_INDEX) + val idColumnOutputIndex = it.getColumnIndex(TxOutputsViewDefinition.COLUMN_INTEGER_OUTPUT_INDEX) - it.getInt(idColumnIndex) + it.getInt(idColumnOutputIndex) } ) From 700bd1fb5954d413bb032a52de67874f33d9df16 Mon Sep 17 00:00:00 2001 From: Honza Date: Thu, 17 Aug 2023 08:34:05 +0200 Subject: [PATCH 25/76] Changelog update --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cdffafb2..0b43dcdb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,9 @@ which speeds up discovering the wallet's spendable balance. - etc. - Checkpoints +### Fixed +- `Synchronizer.getMemos()` now correctly returns a flow of strings for sent and received transactions. + ## 1.20.0-beta01 - The SDK internally migrated from `BackendExt` rust backend extension functions to more type-safe `TypesafeBackend`. - `Synchronizer.getMemos()` now internally handles expected `RuntimeException` from the rust layer and transforms it From 61f618caf98ce0877e809a371a2b4ef18e6b8c51 Mon Sep 17 00:00:00 2001 From: Honza Date: Thu, 17 Aug 2023 09:03:26 +0200 Subject: [PATCH 26/76] Remove OpenEndRange type entirely --- .../sdk/internal/jni/RustDerivationToolTest.kt | 2 -- .../z/ecc/android/sdk/internal/model/JniScanRange.kt | 11 ----------- .../z/ecc/android/sdk/internal/ext/OpenEndRangeExt.kt | 6 ------ .../z/ecc/android/sdk/internal/model/ScanRange.kt | 10 ++++------ 4 files changed, 4 insertions(+), 25 deletions(-) delete mode 100644 sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/OpenEndRangeExt.kt diff --git a/backend-lib/src/androidTest/java/cash/z/ecc/android/sdk/internal/jni/RustDerivationToolTest.kt b/backend-lib/src/androidTest/java/cash/z/ecc/android/sdk/internal/jni/RustDerivationToolTest.kt index f5efd906..f7a7d4af 100644 --- a/backend-lib/src/androidTest/java/cash/z/ecc/android/sdk/internal/jni/RustDerivationToolTest.kt +++ b/backend-lib/src/androidTest/java/cash/z/ecc/android/sdk/internal/jni/RustDerivationToolTest.kt @@ -1,7 +1,6 @@ package cash.z.ecc.android.sdk.internal.jni import cash.z.ecc.android.bip39.Mnemonics -import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.test.runTest import org.junit.Test import kotlin.test.assertContentEquals @@ -15,7 +14,6 @@ class RustDerivationToolTest { } @Test - @OptIn(ExperimentalCoroutinesApi::class) fun create_spending_key_does_not_mutate_passed_bytes() = runTest { val bytesOne = Mnemonics.MnemonicCode(SEED_PHRASE).toEntropy() val bytesTwo = Mnemonics.MnemonicCode(SEED_PHRASE).toEntropy() diff --git a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniScanRange.kt b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniScanRange.kt index a4a7228b..c1c676b2 100644 --- a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniScanRange.kt +++ b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniScanRange.kt @@ -26,15 +26,4 @@ class JniScanRange( "Height $endHeight is outside of allowed UInt range" } } - - companion object { - @OptIn(ExperimentalStdlibApi::class) - fun new(range: OpenEndRange, priority: Long): JniScanRange { - return JniScanRange( - startHeight = range.start, - endHeight = range.endExclusive, - priority = priority - ) - } - } } diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/OpenEndRangeExt.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/OpenEndRangeExt.kt deleted file mode 100644 index dcd2ad4b..00000000 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/OpenEndRangeExt.kt +++ /dev/null @@ -1,6 +0,0 @@ -package cash.z.ecc.android.sdk.internal.ext - -import cash.z.ecc.android.sdk.model.BlockHeight - -@OptIn(ExperimentalStdlibApi::class) -internal fun OpenEndRange.toClosedRange(): ClosedRange = start..endExclusive - 1 diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanRange.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanRange.kt index 16b1ca01..fcbc3b8d 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanRange.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanRange.kt @@ -1,7 +1,6 @@ package cash.z.ecc.android.sdk.internal.model import cash.z.ecc.android.sdk.internal.Twig -import cash.z.ecc.android.sdk.internal.ext.toClosedRange import cash.z.ecc.android.sdk.model.BlockHeight import cash.z.ecc.android.sdk.model.ZcashNetwork @@ -19,14 +18,13 @@ internal data class ScanRange( companion object { /** - * Note that this function also transforms the suggested ranges from [OpenEndRange] to [ClosedRange] so the - * rest of the logic can safely work with a unified range type. + * Note that this function subtracts 1 from [JniScanRange.endHeight] as the rest of the logic works with + * [ClosedRange] and the endHeight is exclusive. */ - @OptIn(ExperimentalStdlibApi::class) fun new(jni: JniScanRange, zcashNetwork: ZcashNetwork): ScanRange { return ScanRange( - range = (BlockHeight.new(zcashNetwork, jni.startHeight).. Date: Thu, 17 Aug 2023 18:22:56 -0600 Subject: [PATCH 27/76] Migrate to Rust revision with scanning and spending fixes --- backend-lib/Cargo.lock | 16 ++++++++-------- backend-lib/Cargo.toml | 10 +++++----- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/backend-lib/Cargo.lock b/backend-lib/Cargo.lock index c2e1c9c9..1dffe3e4 100644 --- a/backend-lib/Cargo.lock +++ b/backend-lib/Cargo.lock @@ -483,7 +483,7 @@ checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" [[package]] name = "equihash" version = "0.2.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=81d1928497683cdd3bb97e1411f66d3b2715dd85#81d1928497683cdd3bb97e1411f66d3b2715dd85" +source = "git+https://github.com/zcash/librustzcash.git?rev=f61d60bb967308b4c84defcfd6c8c7a70e1c276d#f61d60bb967308b4c84defcfd6c8c7a70e1c276d" dependencies = [ "blake2b_simd", "byteorder", @@ -513,7 +513,7 @@ dependencies = [ [[package]] name = "f4jumble" version = "0.1.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=81d1928497683cdd3bb97e1411f66d3b2715dd85#81d1928497683cdd3bb97e1411f66d3b2715dd85" +source = "git+https://github.com/zcash/librustzcash.git?rev=f61d60bb967308b4c84defcfd6c8c7a70e1c276d#f61d60bb967308b4c84defcfd6c8c7a70e1c276d" dependencies = [ "blake2b_simd", ] @@ -2258,7 +2258,7 @@ dependencies = [ [[package]] name = "zcash_address" version = "0.3.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=81d1928497683cdd3bb97e1411f66d3b2715dd85#81d1928497683cdd3bb97e1411f66d3b2715dd85" +source = "git+https://github.com/zcash/librustzcash.git?rev=f61d60bb967308b4c84defcfd6c8c7a70e1c276d#f61d60bb967308b4c84defcfd6c8c7a70e1c276d" dependencies = [ "bech32", "bs58", @@ -2269,7 +2269,7 @@ dependencies = [ [[package]] name = "zcash_client_backend" version = "0.9.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=81d1928497683cdd3bb97e1411f66d3b2715dd85#81d1928497683cdd3bb97e1411f66d3b2715dd85" +source = "git+https://github.com/zcash/librustzcash.git?rev=f61d60bb967308b4c84defcfd6c8c7a70e1c276d#f61d60bb967308b4c84defcfd6c8c7a70e1c276d" dependencies = [ "base64", "bech32", @@ -2302,7 +2302,7 @@ dependencies = [ [[package]] name = "zcash_client_sqlite" version = "0.7.1" -source = "git+https://github.com/zcash/librustzcash.git?rev=81d1928497683cdd3bb97e1411f66d3b2715dd85#81d1928497683cdd3bb97e1411f66d3b2715dd85" +source = "git+https://github.com/zcash/librustzcash.git?rev=f61d60bb967308b4c84defcfd6c8c7a70e1c276d#f61d60bb967308b4c84defcfd6c8c7a70e1c276d" dependencies = [ "bs58", "byteorder", @@ -2328,7 +2328,7 @@ dependencies = [ [[package]] name = "zcash_encoding" version = "0.2.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=81d1928497683cdd3bb97e1411f66d3b2715dd85#81d1928497683cdd3bb97e1411f66d3b2715dd85" +source = "git+https://github.com/zcash/librustzcash.git?rev=f61d60bb967308b4c84defcfd6c8c7a70e1c276d#f61d60bb967308b4c84defcfd6c8c7a70e1c276d" dependencies = [ "byteorder", "nonempty", @@ -2350,7 +2350,7 @@ dependencies = [ [[package]] name = "zcash_primitives" version = "0.12.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=81d1928497683cdd3bb97e1411f66d3b2715dd85#81d1928497683cdd3bb97e1411f66d3b2715dd85" +source = "git+https://github.com/zcash/librustzcash.git?rev=f61d60bb967308b4c84defcfd6c8c7a70e1c276d#f61d60bb967308b4c84defcfd6c8c7a70e1c276d" dependencies = [ "aes", "bip0039", @@ -2385,7 +2385,7 @@ dependencies = [ [[package]] name = "zcash_proofs" version = "0.12.1" -source = "git+https://github.com/zcash/librustzcash.git?rev=81d1928497683cdd3bb97e1411f66d3b2715dd85#81d1928497683cdd3bb97e1411f66d3b2715dd85" +source = "git+https://github.com/zcash/librustzcash.git?rev=f61d60bb967308b4c84defcfd6c8c7a70e1c276d#f61d60bb967308b4c84defcfd6c8c7a70e1c276d" dependencies = [ "bellman", "blake2b_simd", diff --git a/backend-lib/Cargo.toml b/backend-lib/Cargo.toml index 4ed3e727..63a2cc9d 100644 --- a/backend-lib/Cargo.toml +++ b/backend-lib/Cargo.toml @@ -44,11 +44,11 @@ libc = "0.2" incrementalmerkletree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "da97e6c399c5acddedad2c1730dbb7ee55499a2f" } shardtree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "da97e6c399c5acddedad2c1730dbb7ee55499a2f" } orchard = { git = "https://github.com/zcash/orchard.git", rev = "6ef89d5f154de2cf7b7dd87edb8d8c49158beebb" } -zcash_address = { git = "https://github.com/zcash/librustzcash.git", rev = "81d1928497683cdd3bb97e1411f66d3b2715dd85" } -zcash_client_backend = { git = "https://github.com/zcash/librustzcash.git", rev = "81d1928497683cdd3bb97e1411f66d3b2715dd85" } -zcash_client_sqlite = { git = "https://github.com/zcash/librustzcash.git", rev = "81d1928497683cdd3bb97e1411f66d3b2715dd85" } -zcash_primitives = { git = "https://github.com/zcash/librustzcash.git", rev = "81d1928497683cdd3bb97e1411f66d3b2715dd85" } -zcash_proofs = { git = "https://github.com/zcash/librustzcash.git", rev = "81d1928497683cdd3bb97e1411f66d3b2715dd85" } +zcash_address = { git = "https://github.com/zcash/librustzcash.git", rev = "f61d60bb967308b4c84defcfd6c8c7a70e1c276d" } +zcash_client_backend = { git = "https://github.com/zcash/librustzcash.git", rev = "f61d60bb967308b4c84defcfd6c8c7a70e1c276d" } +zcash_client_sqlite = { git = "https://github.com/zcash/librustzcash.git", rev = "f61d60bb967308b4c84defcfd6c8c7a70e1c276d" } +zcash_primitives = { git = "https://github.com/zcash/librustzcash.git", rev = "f61d60bb967308b4c84defcfd6c8c7a70e1c276d" } +zcash_proofs = { git = "https://github.com/zcash/librustzcash.git", rev = "f61d60bb967308b4c84defcfd6c8c7a70e1c276d" } ## Uncomment this to test librustzcash changes locally #[patch.crates-io] From 7277a7ecca285a00b82c1c85dc1c3e657a3a0a29 Mon Sep 17 00:00:00 2001 From: Honza Date: Fri, 18 Aug 2023 08:55:40 +0200 Subject: [PATCH 28/76] Fix ktlint warning --- .../java/cash/z/ecc/android/sdk/internal/model/ScanRange.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanRange.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanRange.kt index fcbc3b8d..9ce0973f 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanRange.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanRange.kt @@ -23,8 +23,8 @@ internal data class ScanRange( */ fun new(jni: JniScanRange, zcashNetwork: ZcashNetwork): ScanRange { return ScanRange( - range = BlockHeight.new(zcashNetwork, jni.startHeight).. - (BlockHeight.new(zcashNetwork, jni.endHeight) - 1), + range = + BlockHeight.new(zcashNetwork, jni.startHeight)..(BlockHeight.new(zcashNetwork, jni.endHeight) - 1), priority = jni.priority ) } From 9795610bb9b3537b4e43e28ef7f8f05b90aea766 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Honza=20Rychnovsk=C3=BD?= Date: Fri, 18 Aug 2023 13:25:54 +0200 Subject: [PATCH 29/76] [#1136] Tests for the new helper extensions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Jni objects attribute constraint tests
 * Extend continuity error test * ScanRange model tests * [#1174] Move model classes out of the CompactBlockProcessor * [#1174] SbS: Move model classes out of the CompactBlockProcessor * Move SyncingResult out of the processor * Add issue link --- .../sdk/internal/model/JniBlockMetaTest.kt | 32 ++++++ .../sdk/internal/model/JniScanRangeTest.kt | 28 +++++ .../sdk/internal/model/JniSubtreeRootTest.kt | 26 +++++ .../demoapp/fixture/WalletSnapshotFixture.kt | 2 +- .../screen/home/viewmodel/WalletSnapshot.kt | 2 +- .../screen/home/viewmodel/WalletViewModel.kt | 2 +- .../z/ecc/android/sdk/WalletCoordinator.kt | 2 +- .../processor/CompactBlockProcessorTest.kt | 8 ++ .../cash/z/ecc/android/sdk/SdkSynchronizer.kt | 12 +-- .../cash/z/ecc/android/sdk/Synchronizer.kt | 2 +- .../{ => processor}/CompactBlockProcessor.kt | 100 ++---------------- .../processor/model/BatchSyncProgress.kt | 11 ++ .../processor/model/GetSubtreeRootsResult.kt | 14 +++ .../model/PutSaplingSubtreeRootsResult.kt | 11 ++ .../model/SuggestScanRangesResult.kt | 12 +++ .../block/processor/model/SyncStageResult.kt | 11 ++ .../block/processor/model/SyncingResult.kt | 51 +++++++++ .../processor/model/UpdateChainTipResult.kt | 11 ++ .../model/VerifySuggestedScanRange.kt | 11 ++ .../android/sdk/internal/ext/WalletService.kt | 11 -- .../android/sdk/internal/model/ScanRange.kt | 2 - .../z/ecc/android/sdk/ext/ExceptionExtTest.kt | 18 ++-- .../android/sdk/fixture/ScanRangeFixture.kt | 17 +++ .../sdk/internal/model/ScanRangeTest.kt | 29 +++++ 24 files changed, 304 insertions(+), 121 deletions(-) create mode 100644 backend-lib/src/test/java/cash/z/ecc/android/sdk/internal/model/JniBlockMetaTest.kt create mode 100644 backend-lib/src/test/java/cash/z/ecc/android/sdk/internal/model/JniScanRangeTest.kt create mode 100644 backend-lib/src/test/java/cash/z/ecc/android/sdk/internal/model/JniSubtreeRootTest.kt create mode 100644 sdk-lib/src/androidTest/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessorTest.kt rename sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/{ => processor}/CompactBlockProcessor.kt (95%) create mode 100644 sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/BatchSyncProgress.kt create mode 100644 sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/GetSubtreeRootsResult.kt create mode 100644 sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/PutSaplingSubtreeRootsResult.kt create mode 100644 sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/SuggestScanRangesResult.kt create mode 100644 sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/SyncStageResult.kt create mode 100644 sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/SyncingResult.kt create mode 100644 sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/UpdateChainTipResult.kt create mode 100644 sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/VerifySuggestedScanRange.kt create mode 100644 sdk-lib/src/test/java/cash/z/ecc/android/sdk/fixture/ScanRangeFixture.kt create mode 100644 sdk-lib/src/test/java/cash/z/ecc/android/sdk/internal/model/ScanRangeTest.kt diff --git a/backend-lib/src/test/java/cash/z/ecc/android/sdk/internal/model/JniBlockMetaTest.kt b/backend-lib/src/test/java/cash/z/ecc/android/sdk/internal/model/JniBlockMetaTest.kt new file mode 100644 index 00000000..8194c06a --- /dev/null +++ b/backend-lib/src/test/java/cash/z/ecc/android/sdk/internal/model/JniBlockMetaTest.kt @@ -0,0 +1,32 @@ +package cash.z.ecc.android.sdk.internal.model + +import kotlin.test.Test +import kotlin.test.assertFailsWith +import kotlin.test.assertIs + +class JniBlockMetaTest { + @Test + fun attributes_within_constraints() { + val instance = JniBlockMeta( + height = UInt.MAX_VALUE.toLong(), + hash = byteArrayOf(), + time = 0L, + saplingOutputsCount = UInt.MIN_VALUE.toLong(), + orchardOutputsCount = UInt.MIN_VALUE.toLong() + ) + assertIs(instance) + } + + @Test + fun attributes_not_in_constraints() { + assertFailsWith(IllegalArgumentException::class) { + JniBlockMeta( + height = Long.MAX_VALUE, + hash = byteArrayOf(), + time = 0L, + saplingOutputsCount = Long.MIN_VALUE, + orchardOutputsCount = Long.MIN_VALUE + ) + } + } +} diff --git a/backend-lib/src/test/java/cash/z/ecc/android/sdk/internal/model/JniScanRangeTest.kt b/backend-lib/src/test/java/cash/z/ecc/android/sdk/internal/model/JniScanRangeTest.kt new file mode 100644 index 00000000..ada935b5 --- /dev/null +++ b/backend-lib/src/test/java/cash/z/ecc/android/sdk/internal/model/JniScanRangeTest.kt @@ -0,0 +1,28 @@ +package cash.z.ecc.android.sdk.internal.model + +import kotlin.test.Test +import kotlin.test.assertFailsWith +import kotlin.test.assertIs + +class JniScanRangeTest { + @Test + fun attributes_within_constraints() { + val instance = JniScanRange( + startHeight = UInt.MIN_VALUE.toLong(), + endHeight = UInt.MAX_VALUE.toLong(), + priority = 10 + ) + assertIs(instance) + } + + @Test + fun attributes_not_in_constraints() { + assertFailsWith(IllegalArgumentException::class) { + JniScanRange( + startHeight = Long.MIN_VALUE, + endHeight = Long.MAX_VALUE, + priority = 10 + ) + } + } +} diff --git a/backend-lib/src/test/java/cash/z/ecc/android/sdk/internal/model/JniSubtreeRootTest.kt b/backend-lib/src/test/java/cash/z/ecc/android/sdk/internal/model/JniSubtreeRootTest.kt new file mode 100644 index 00000000..a9690f6d --- /dev/null +++ b/backend-lib/src/test/java/cash/z/ecc/android/sdk/internal/model/JniSubtreeRootTest.kt @@ -0,0 +1,26 @@ +package cash.z.ecc.android.sdk.internal.model + +import kotlin.test.Test +import kotlin.test.assertFailsWith +import kotlin.test.assertIs + +class JniSubtreeRootTest { + @Test + fun attributes_within_constraints() { + val instance = JniSubtreeRoot( + rootHash = byteArrayOf(), + completingBlockHeight = UInt.MAX_VALUE.toLong() + ) + assertIs(instance) + } + + @Test + fun attributes_not_in_constraints() { + assertFailsWith(IllegalArgumentException::class) { + JniSubtreeRoot( + rootHash = byteArrayOf(), + completingBlockHeight = Long.MAX_VALUE + ) + } + } +} diff --git a/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/fixture/WalletSnapshotFixture.kt b/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/fixture/WalletSnapshotFixture.kt index cad402d5..01544e21 100644 --- a/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/fixture/WalletSnapshotFixture.kt +++ b/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/fixture/WalletSnapshotFixture.kt @@ -1,7 +1,7 @@ package cash.z.ecc.android.sdk.demoapp.fixture import cash.z.ecc.android.sdk.Synchronizer -import cash.z.ecc.android.sdk.block.CompactBlockProcessor +import cash.z.ecc.android.sdk.block.processor.CompactBlockProcessor import cash.z.ecc.android.sdk.demoapp.ui.screen.home.viewmodel.SynchronizerError import cash.z.ecc.android.sdk.demoapp.ui.screen.home.viewmodel.WalletSnapshot import cash.z.ecc.android.sdk.model.PercentDecimal diff --git a/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/ui/screen/home/viewmodel/WalletSnapshot.kt b/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/ui/screen/home/viewmodel/WalletSnapshot.kt index 78cb3a01..31d9b1c5 100644 --- a/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/ui/screen/home/viewmodel/WalletSnapshot.kt +++ b/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/ui/screen/home/viewmodel/WalletSnapshot.kt @@ -1,7 +1,7 @@ package cash.z.ecc.android.sdk.demoapp.ui.screen.home.viewmodel import cash.z.ecc.android.sdk.Synchronizer -import cash.z.ecc.android.sdk.block.CompactBlockProcessor +import cash.z.ecc.android.sdk.block.processor.CompactBlockProcessor import cash.z.ecc.android.sdk.ext.ZcashSdk import cash.z.ecc.android.sdk.model.PercentDecimal import cash.z.ecc.android.sdk.model.WalletBalance diff --git a/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/ui/screen/home/viewmodel/WalletViewModel.kt b/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/ui/screen/home/viewmodel/WalletViewModel.kt index e2a55937..a2966530 100644 --- a/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/ui/screen/home/viewmodel/WalletViewModel.kt +++ b/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/ui/screen/home/viewmodel/WalletViewModel.kt @@ -7,7 +7,7 @@ import cash.z.ecc.android.bip39.Mnemonics import cash.z.ecc.android.bip39.toSeed import cash.z.ecc.android.sdk.Synchronizer import cash.z.ecc.android.sdk.WalletCoordinator -import cash.z.ecc.android.sdk.block.CompactBlockProcessor +import cash.z.ecc.android.sdk.block.processor.CompactBlockProcessor import cash.z.ecc.android.sdk.demoapp.getInstance import cash.z.ecc.android.sdk.demoapp.preference.EncryptedPreferenceKeys import cash.z.ecc.android.sdk.demoapp.preference.EncryptedPreferenceSingleton diff --git a/sdk-incubator-lib/src/main/java/cash/z/ecc/android/sdk/WalletCoordinator.kt b/sdk-incubator-lib/src/main/java/cash/z/ecc/android/sdk/WalletCoordinator.kt index da3224f3..9275f16f 100644 --- a/sdk-incubator-lib/src/main/java/cash/z/ecc/android/sdk/WalletCoordinator.kt +++ b/sdk-incubator-lib/src/main/java/cash/z/ecc/android/sdk/WalletCoordinator.kt @@ -1,7 +1,7 @@ package cash.z.ecc.android.sdk import android.content.Context -import cash.z.ecc.android.sdk.block.CompactBlockProcessor +import cash.z.ecc.android.sdk.block.processor.CompactBlockProcessor import cash.z.ecc.android.sdk.ext.onFirst import cash.z.ecc.android.sdk.internal.Twig import cash.z.ecc.android.sdk.model.PersistableWallet diff --git a/sdk-lib/src/androidTest/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessorTest.kt b/sdk-lib/src/androidTest/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessorTest.kt new file mode 100644 index 00000000..54df3b6e --- /dev/null +++ b/sdk-lib/src/androidTest/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessorTest.kt @@ -0,0 +1,8 @@ +package cash.z.ecc.android.sdk.block.processor + +// TODO [#1094]: Consider fake SDK sync related components +// TODO [#1094]: Testing the CompactBlockProcessor is only available once we can mock the necessary core components like +// [CompactBlockDownloader], [DerivedDataRepository], or [TypesafeBackend] +// TODO [#1094]: https://github.com/zcash/zcash-android-wallet-sdk/issues/1094 +@Suppress("EmptyClassBlock") +class CompactBlockProcessorTest diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt index 05eae408..e2efed16 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt @@ -5,12 +5,12 @@ import cash.z.ecc.android.sdk.Synchronizer.Status.DISCONNECTED import cash.z.ecc.android.sdk.Synchronizer.Status.STOPPED import cash.z.ecc.android.sdk.Synchronizer.Status.SYNCED import cash.z.ecc.android.sdk.Synchronizer.Status.SYNCING -import cash.z.ecc.android.sdk.block.CompactBlockProcessor -import cash.z.ecc.android.sdk.block.CompactBlockProcessor.State.Disconnected -import cash.z.ecc.android.sdk.block.CompactBlockProcessor.State.Initialized -import cash.z.ecc.android.sdk.block.CompactBlockProcessor.State.Stopped -import cash.z.ecc.android.sdk.block.CompactBlockProcessor.State.Synced -import cash.z.ecc.android.sdk.block.CompactBlockProcessor.State.Syncing +import cash.z.ecc.android.sdk.block.processor.CompactBlockProcessor +import cash.z.ecc.android.sdk.block.processor.CompactBlockProcessor.State.Disconnected +import cash.z.ecc.android.sdk.block.processor.CompactBlockProcessor.State.Initialized +import cash.z.ecc.android.sdk.block.processor.CompactBlockProcessor.State.Stopped +import cash.z.ecc.android.sdk.block.processor.CompactBlockProcessor.State.Synced +import cash.z.ecc.android.sdk.block.processor.CompactBlockProcessor.State.Syncing import cash.z.ecc.android.sdk.exception.TransactionEncoderException import cash.z.ecc.android.sdk.exception.TransactionSubmitException import cash.z.ecc.android.sdk.ext.ConsensusBranchId diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/Synchronizer.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/Synchronizer.kt index 0745b8bb..ed531c32 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/Synchronizer.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/Synchronizer.kt @@ -1,7 +1,7 @@ package cash.z.ecc.android.sdk import android.content.Context -import cash.z.ecc.android.sdk.block.CompactBlockProcessor +import cash.z.ecc.android.sdk.block.processor.CompactBlockProcessor import cash.z.ecc.android.sdk.ext.ZcashSdk import cash.z.ecc.android.sdk.internal.Derivation import cash.z.ecc.android.sdk.internal.SaplingParamTool diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessor.kt similarity index 95% rename from sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt rename to sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessor.kt index fa877c4a..af26c40e 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessor.kt @@ -1,8 +1,16 @@ -package cash.z.ecc.android.sdk.block +package cash.z.ecc.android.sdk.block.processor import androidx.annotation.VisibleForTesting import cash.z.ecc.android.sdk.BuildConfig import cash.z.ecc.android.sdk.annotation.OpenForTesting +import cash.z.ecc.android.sdk.block.processor.model.BatchSyncProgress +import cash.z.ecc.android.sdk.block.processor.model.GetSubtreeRootsResult +import cash.z.ecc.android.sdk.block.processor.model.PutSaplingSubtreeRootsResult +import cash.z.ecc.android.sdk.block.processor.model.SuggestScanRangesResult +import cash.z.ecc.android.sdk.block.processor.model.SyncStageResult +import cash.z.ecc.android.sdk.block.processor.model.SyncingResult +import cash.z.ecc.android.sdk.block.processor.model.UpdateChainTipResult +import cash.z.ecc.android.sdk.block.processor.model.VerifySuggestedScanRange import cash.z.ecc.android.sdk.exception.CompactBlockProcessorException import cash.z.ecc.android.sdk.exception.CompactBlockProcessorException.EnhanceTransactionError.EnhanceTxDecryptError import cash.z.ecc.android.sdk.exception.CompactBlockProcessorException.EnhanceTransactionError.EnhanceTxDownloadError @@ -258,8 +266,6 @@ class CompactBlockProcessor internal constructor( } BlockProcessingResult.NoBlocksToProcess -> { setState(State.Synced(_processorInfo.value.overallSyncRange)) - // TODO [#1129]: Refactor work with lastSyncRange and lastSyncedHeight - // TODO [#1129]: https://github.com/zcash/zcash-android-wallet-sdk/issues/1129 val noWorkDone = _processorInfo.value.overallSyncRange?.isEmpty() ?: true val summary = if (noWorkDone) { "Nothing to process: no new blocks to sync" @@ -354,34 +360,6 @@ class CompactBlockProcessor internal constructor( } } - internal sealed class SuggestScanRangesResult { - data class Success(val ranges: List) : SuggestScanRangesResult() - data class Failure(val failedAtHeight: BlockHeight, val exception: Throwable) : SuggestScanRangesResult() - } - - internal sealed class GetSubtreeRootsResult { - // SbS: Spend-before-Sync - data class UseSbS(val subTreeRootList: List) : GetSubtreeRootsResult() - object UseLinear : GetSubtreeRootsResult() - object FailureConnection : GetSubtreeRootsResult() - data class OtherFailure(val exception: Throwable) : GetSubtreeRootsResult() - } - - internal sealed class PutSaplingSubtreeRootsResult { - object Success : PutSaplingSubtreeRootsResult() - data class Failure(val failedAtHeight: BlockHeight, val exception: Throwable) : PutSaplingSubtreeRootsResult() - } - - internal sealed class UpdateChainTipResult { - data class Success(val height: BlockHeight) : UpdateChainTipResult() - data class Failure(val failedAtHeight: BlockHeight, val exception: Throwable) : UpdateChainTipResult() - } - - internal sealed class VerifySuggestedScanRange { - data class ShouldVerify(val scanRange: ScanRange) : VerifySuggestedScanRange() - object NoRangeToVerify : VerifySuggestedScanRange() - } - // TODO [#1137]: Refactor processNewBlocksInSbSOrder // TODO [#1137]: https://github.com/zcash/zcash-android-wallet-sdk/issues/1137 /** @@ -1192,49 +1170,6 @@ class CompactBlockProcessor internal constructor( } } - @VisibleForTesting - internal sealed class SyncingResult { - object AllSuccess : SyncingResult() - data class DownloadSuccess(val downloadedBlocks: List?) : SyncingResult() { - override fun toString(): String { - return "DownloadSuccess with ${downloadedBlocks?.size ?: "none"} blocks" - } - } - interface Failure { - val failedAtHeight: BlockHeight - val exception: CompactBlockProcessorException - fun toBlockProcessingResult(): BlockProcessingResult = - BlockProcessingResult.SyncFailure( - this.failedAtHeight, - this.exception - ) - } - data class DownloadFailed( - override val failedAtHeight: BlockHeight, - override val exception: CompactBlockProcessorException - ) : Failure, SyncingResult() - object ScanSuccess : SyncingResult() - data class ScanFailed( - override val failedAtHeight: BlockHeight, - override val exception: CompactBlockProcessorException - ) : Failure, SyncingResult() - object DeleteSuccess : SyncingResult() - data class DeleteFailed( - override val failedAtHeight: BlockHeight, - override val exception: CompactBlockProcessorException - ) : Failure, SyncingResult() - object EnhanceSuccess : SyncingResult() - data class EnhanceFailed( - override val failedAtHeight: BlockHeight, - override val exception: CompactBlockProcessorException - ) : Failure, SyncingResult() - object UpdateBirthday : SyncingResult() - data class ContinuityError( - override val failedAtHeight: BlockHeight, - override val exception: CompactBlockProcessorException - ) : Failure, SyncingResult() - } - /** * Requests, processes and persists all blocks from the given range. * @@ -2034,23 +1969,6 @@ class CompactBlockProcessor internal constructor( object Initialized : State() } - /** - * Progress model class for sharing the whole batch sync progress out of the sync process. - */ - internal data class BatchSyncProgress( - val inRangeOrder: Long = 0, - val overallOrder: Long = 0, - val resultState: SyncingResult = SyncingResult.AllSuccess - ) - - /** - * Progress model class for sharing particular sync stage result internally in the sync process. - */ - private data class SyncStageResult( - val batch: BlockBatch, - val stageResult: SyncingResult - ) - /** * Data class for holding detailed information about the processor. * diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/BatchSyncProgress.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/BatchSyncProgress.kt new file mode 100644 index 00000000..4c425bf3 --- /dev/null +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/BatchSyncProgress.kt @@ -0,0 +1,11 @@ +package cash.z.ecc.android.sdk.block.processor.model + +/** + * Progress model class for sharing the whole batch synchronization progress out of the synchronization process. + */ +internal data class BatchSyncProgress( + val inRangeOrder: Long = 0, + val overallOrder: Long = 0, + val resultState: SyncingResult = + SyncingResult.AllSuccess +) diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/GetSubtreeRootsResult.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/GetSubtreeRootsResult.kt new file mode 100644 index 00000000..f6fb2f96 --- /dev/null +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/GetSubtreeRootsResult.kt @@ -0,0 +1,14 @@ +package cash.z.ecc.android.sdk.block.processor.model + +import cash.z.ecc.android.sdk.internal.model.SubtreeRoot + +/** + * Internal class for get subtree roots action result. + */ +internal sealed class GetSubtreeRootsResult { + // SbS: Spend-before-Sync + data class UseSbS(val subTreeRootList: List) : GetSubtreeRootsResult() + object UseLinear : GetSubtreeRootsResult() + object FailureConnection : GetSubtreeRootsResult() + data class OtherFailure(val exception: Throwable) : GetSubtreeRootsResult() +} diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/PutSaplingSubtreeRootsResult.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/PutSaplingSubtreeRootsResult.kt new file mode 100644 index 00000000..d72c4ce6 --- /dev/null +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/PutSaplingSubtreeRootsResult.kt @@ -0,0 +1,11 @@ +package cash.z.ecc.android.sdk.block.processor.model + +import cash.z.ecc.android.sdk.model.BlockHeight + +/** + * Internal class for sharing put sapling subtree roots action result. + */ +internal sealed class PutSaplingSubtreeRootsResult { + object Success : PutSaplingSubtreeRootsResult() + data class Failure(val failedAtHeight: BlockHeight, val exception: Throwable) : PutSaplingSubtreeRootsResult() +} diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/SuggestScanRangesResult.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/SuggestScanRangesResult.kt new file mode 100644 index 00000000..2b886365 --- /dev/null +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/SuggestScanRangesResult.kt @@ -0,0 +1,12 @@ +package cash.z.ecc.android.sdk.block.processor.model + +import cash.z.ecc.android.sdk.internal.model.ScanRange +import cash.z.ecc.android.sdk.model.BlockHeight + +/** + * Internal class for sharing suggested scan ranges action result. + */ +internal sealed class SuggestScanRangesResult { + data class Success(val ranges: List) : SuggestScanRangesResult() + data class Failure(val failedAtHeight: BlockHeight, val exception: Throwable) : SuggestScanRangesResult() +} diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/SyncStageResult.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/SyncStageResult.kt new file mode 100644 index 00000000..9195254c --- /dev/null +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/SyncStageResult.kt @@ -0,0 +1,11 @@ +package cash.z.ecc.android.sdk.block.processor.model + +import cash.z.ecc.android.sdk.internal.model.BlockBatch + +/** + * Common progress model class for sharing a batch synchronization stage result internally in the synchronization loop. + */ +internal data class SyncStageResult( + val batch: BlockBatch, + val stageResult: SyncingResult +) diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/SyncingResult.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/SyncingResult.kt new file mode 100644 index 00000000..6b32a7a5 --- /dev/null +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/SyncingResult.kt @@ -0,0 +1,51 @@ +package cash.z.ecc.android.sdk.block.processor.model + +import cash.z.ecc.android.sdk.block.processor.CompactBlockProcessor +import cash.z.ecc.android.sdk.exception.CompactBlockProcessorException +import cash.z.ecc.android.sdk.internal.model.JniBlockMeta +import cash.z.ecc.android.sdk.model.BlockHeight + +/** + * Internal class for the overall synchronization process result reporting. + */ +internal sealed class SyncingResult { + object AllSuccess : SyncingResult() + data class DownloadSuccess(val downloadedBlocks: List?) : SyncingResult() { + override fun toString(): String { + return "DownloadSuccess with ${downloadedBlocks?.size ?: "none"} blocks" + } + } + interface Failure { + val failedAtHeight: BlockHeight + val exception: CompactBlockProcessorException + fun toBlockProcessingResult(): CompactBlockProcessor.BlockProcessingResult = + CompactBlockProcessor.BlockProcessingResult.SyncFailure( + this.failedAtHeight, + this.exception + ) + } + data class DownloadFailed( + override val failedAtHeight: BlockHeight, + override val exception: CompactBlockProcessorException + ) : Failure, SyncingResult() + object ScanSuccess : SyncingResult() + data class ScanFailed( + override val failedAtHeight: BlockHeight, + override val exception: CompactBlockProcessorException + ) : Failure, SyncingResult() + object DeleteSuccess : SyncingResult() + data class DeleteFailed( + override val failedAtHeight: BlockHeight, + override val exception: CompactBlockProcessorException + ) : Failure, SyncingResult() + object EnhanceSuccess : SyncingResult() + data class EnhanceFailed( + override val failedAtHeight: BlockHeight, + override val exception: CompactBlockProcessorException + ) : Failure, SyncingResult() + object UpdateBirthday : SyncingResult() + data class ContinuityError( + override val failedAtHeight: BlockHeight, + override val exception: CompactBlockProcessorException + ) : Failure, SyncingResult() +} diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/UpdateChainTipResult.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/UpdateChainTipResult.kt new file mode 100644 index 00000000..53d218f5 --- /dev/null +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/UpdateChainTipResult.kt @@ -0,0 +1,11 @@ +package cash.z.ecc.android.sdk.block.processor.model + +import cash.z.ecc.android.sdk.model.BlockHeight + +/** + * Internal class for sharing update chain tip action result. + */ +internal sealed class UpdateChainTipResult { + data class Success(val height: BlockHeight) : UpdateChainTipResult() + data class Failure(val failedAtHeight: BlockHeight, val exception: Throwable) : UpdateChainTipResult() +} diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/VerifySuggestedScanRange.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/VerifySuggestedScanRange.kt new file mode 100644 index 00000000..90ed3824 --- /dev/null +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/VerifySuggestedScanRange.kt @@ -0,0 +1,11 @@ +package cash.z.ecc.android.sdk.block.processor.model + +import cash.z.ecc.android.sdk.internal.model.ScanRange + +/** + * Internal class for sharing verify suggested scan range action result. + */ +internal sealed class VerifySuggestedScanRange { + data class ShouldVerify(val scanRange: ScanRange) : VerifySuggestedScanRange() + object NoRangeToVerify : VerifySuggestedScanRange() +} diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/WalletService.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/WalletService.kt index 786e31d8..743b39df 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/WalletService.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/WalletService.kt @@ -1,10 +1,8 @@ package cash.z.ecc.android.sdk.internal.ext -import android.content.Context import cash.z.ecc.android.sdk.ext.ZcashSdk.MAX_BACKOFF_INTERVAL import cash.z.ecc.android.sdk.internal.Twig import kotlinx.coroutines.delay -import java.io.File import kotlin.math.pow import kotlin.random.Random @@ -120,12 +118,3 @@ suspend inline fun retryWithBackoff( } } } - -/** - * Return true if the given database already exists. - * - * @return true when the given database exists in the given context. - */ -internal fun dbExists(appContext: Context, dbFileName: String): Boolean { - return File(appContext.getDatabasePath(dbFileName).absolutePath).exists() -} diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanRange.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanRange.kt index 9ce0973f..39053bf3 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanRange.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanRange.kt @@ -1,6 +1,5 @@ package cash.z.ecc.android.sdk.internal.model -import cash.z.ecc.android.sdk.internal.Twig import cash.z.ecc.android.sdk.model.BlockHeight import cash.z.ecc.android.sdk.model.ZcashNetwork @@ -11,7 +10,6 @@ internal data class ScanRange( override fun toString() = "ScanRange(range=$range, priority=${getSuggestScanRangePriority()})" internal fun getSuggestScanRangePriority(): SuggestScanRangePriority { - Twig.verbose { "Current suggested scan range priority: $priority" } return SuggestScanRangePriority.values() .firstOrNull { it.priority == priority } ?: SuggestScanRangePriority.Scanned } diff --git a/sdk-lib/src/test/java/cash/z/ecc/android/sdk/ext/ExceptionExtTest.kt b/sdk-lib/src/test/java/cash/z/ecc/android/sdk/ext/ExceptionExtTest.kt index fdb69bd3..1288d9a2 100644 --- a/sdk-lib/src/test/java/cash/z/ecc/android/sdk/ext/ExceptionExtTest.kt +++ b/sdk-lib/src/test/java/cash/z/ecc/android/sdk/ext/ExceptionExtTest.kt @@ -1,6 +1,8 @@ package cash.z.ecc.android.sdk.ext +import cash.z.ecc.android.sdk.internal.ext.BLOCK_HEIGHT_DISCONTINUITY import cash.z.ecc.android.sdk.internal.ext.PREV_HASH_MISMATCH +import cash.z.ecc.android.sdk.internal.ext.TREE_SIZE_MISMATCH import cash.z.ecc.android.sdk.internal.ext.isScanContinuityError import kotlin.test.Test import kotlin.test.assertFalse @@ -10,15 +12,19 @@ class ExceptionExtTest { @Test fun is_scan_continuity_error() { - assertTrue { - RuntimeException(PREV_HASH_MISMATCH).isScanContinuityError() - } + assertTrue { RuntimeException(PREV_HASH_MISMATCH).isScanContinuityError() } + assertTrue { RuntimeException(TREE_SIZE_MISMATCH).isScanContinuityError() } + assertTrue { RuntimeException(BLOCK_HEIGHT_DISCONTINUITY).isScanContinuityError() } + + assertTrue { RuntimeException(PREV_HASH_MISMATCH.lowercase()).isScanContinuityError() } + + assertTrue { RuntimeException(PREV_HASH_MISMATCH.plus("Text")).isScanContinuityError() } } @Test fun is_not_scan_continuity_error() { - assertFalse { - RuntimeException("Text").isScanContinuityError() - } + assertFalse { RuntimeException("Text").isScanContinuityError() } + assertFalse { RuntimeException("").isScanContinuityError() } + assertFalse { RuntimeException(PREV_HASH_MISMATCH.drop(1)).isScanContinuityError() } } } diff --git a/sdk-lib/src/test/java/cash/z/ecc/android/sdk/fixture/ScanRangeFixture.kt b/sdk-lib/src/test/java/cash/z/ecc/android/sdk/fixture/ScanRangeFixture.kt new file mode 100644 index 00000000..249dc099 --- /dev/null +++ b/sdk-lib/src/test/java/cash/z/ecc/android/sdk/fixture/ScanRangeFixture.kt @@ -0,0 +1,17 @@ +package cash.z.ecc.android.sdk.fixture + +import cash.z.ecc.android.sdk.internal.model.ScanRange +import cash.z.ecc.android.sdk.internal.model.SuggestScanRangePriority +import cash.z.ecc.android.sdk.model.BlockHeight +import cash.z.ecc.android.sdk.model.ZcashNetwork + +object ScanRangeFixture { + internal val DEFAULT_CLOSED_RANGE = + ZcashNetwork.Testnet.saplingActivationHeight..ZcashNetwork.Testnet.saplingActivationHeight + 9 + internal val DEFAULT_PRIORITY = SuggestScanRangePriority.Verify.priority + + internal fun new( + range: ClosedRange = DEFAULT_CLOSED_RANGE, + priority: Long = DEFAULT_PRIORITY + ) = ScanRange(range, priority) +} diff --git a/sdk-lib/src/test/java/cash/z/ecc/android/sdk/internal/model/ScanRangeTest.kt b/sdk-lib/src/test/java/cash/z/ecc/android/sdk/internal/model/ScanRangeTest.kt new file mode 100644 index 00000000..20f6580a --- /dev/null +++ b/sdk-lib/src/test/java/cash/z/ecc/android/sdk/internal/model/ScanRangeTest.kt @@ -0,0 +1,29 @@ +package cash.z.ecc.android.sdk.internal.model + +import cash.z.ecc.android.sdk.fixture.ScanRangeFixture +import cash.z.ecc.android.sdk.internal.ext.isNotEmpty +import cash.z.ecc.android.sdk.internal.ext.length +import cash.z.ecc.android.sdk.model.ZcashNetwork +import kotlin.test.Test +import kotlin.test.assertTrue + +class ScanRangeTest { + @Test + fun get_suggest_scan_range_priority_test() { + val scanRange = ScanRangeFixture.new( + priority = SuggestScanRangePriority.Verify.priority + ) + assertTrue { + scanRange.getSuggestScanRangePriority() == SuggestScanRangePriority.Verify + } + } + + @Test + fun scan_range_boundaries_test() { + val scanRange = ScanRangeFixture.new( + range = ZcashNetwork.Testnet.saplingActivationHeight..ZcashNetwork.Testnet.saplingActivationHeight + 9 + ) + assertTrue { scanRange.range.isNotEmpty() } + assertTrue { scanRange.range.length() == 10L } + } +} From 0e40b3a80736140b49daacf5a69dc1535d741178 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Honza=20Rychnovsk=C3=BD?= Date: Fri, 25 Aug 2023 15:55:35 +0200 Subject: [PATCH 30/76] [#1181] Stop update balance before updateChainTip --- .../src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt index e2efed16..7c593504 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt @@ -395,11 +395,10 @@ class SdkSynchronizer private constructor( private fun CoroutineScope.onReady() { Twig.debug { "Starting synchronizer…" } - // Triggering UTXOs fetch and transparent balance update at the beginning of the block sync right after the app - // start, as it makes the transparent transactions appearance faster + // Triggering UTXOs and transactions fetching at the beginning of the block synchronization right after the + // app starts makes the transparent transactions appear faster. launch(CoroutineExceptionHandler(::onCriticalError)) { refreshUtxos(Account.DEFAULT) - refreshTransparentBalance() refreshTransactions() } From 32069bea0a18cce113556cd7c7d10c822085f217 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Tue, 29 Aug 2023 01:00:32 +0100 Subject: [PATCH 31/76] Migrate to Rust revision with scan queue initialization --- backend-lib/Cargo.lock | 16 ++++++++-------- backend-lib/Cargo.toml | 10 +++++----- backend-lib/src/main/rust/lib.rs | 1 + .../ecc/android/sdk/internal/model/ScanRange.kt | 1 + 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/backend-lib/Cargo.lock b/backend-lib/Cargo.lock index 1dffe3e4..ce036664 100644 --- a/backend-lib/Cargo.lock +++ b/backend-lib/Cargo.lock @@ -483,7 +483,7 @@ checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" [[package]] name = "equihash" version = "0.2.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=f61d60bb967308b4c84defcfd6c8c7a70e1c276d#f61d60bb967308b4c84defcfd6c8c7a70e1c276d" +source = "git+https://github.com/zcash/librustzcash.git?rev=b580c42bdc9cb0d0fb06a3975783be1bb34395e8#b580c42bdc9cb0d0fb06a3975783be1bb34395e8" dependencies = [ "blake2b_simd", "byteorder", @@ -513,7 +513,7 @@ dependencies = [ [[package]] name = "f4jumble" version = "0.1.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=f61d60bb967308b4c84defcfd6c8c7a70e1c276d#f61d60bb967308b4c84defcfd6c8c7a70e1c276d" +source = "git+https://github.com/zcash/librustzcash.git?rev=b580c42bdc9cb0d0fb06a3975783be1bb34395e8#b580c42bdc9cb0d0fb06a3975783be1bb34395e8" dependencies = [ "blake2b_simd", ] @@ -2258,7 +2258,7 @@ dependencies = [ [[package]] name = "zcash_address" version = "0.3.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=f61d60bb967308b4c84defcfd6c8c7a70e1c276d#f61d60bb967308b4c84defcfd6c8c7a70e1c276d" +source = "git+https://github.com/zcash/librustzcash.git?rev=b580c42bdc9cb0d0fb06a3975783be1bb34395e8#b580c42bdc9cb0d0fb06a3975783be1bb34395e8" dependencies = [ "bech32", "bs58", @@ -2269,7 +2269,7 @@ dependencies = [ [[package]] name = "zcash_client_backend" version = "0.9.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=f61d60bb967308b4c84defcfd6c8c7a70e1c276d#f61d60bb967308b4c84defcfd6c8c7a70e1c276d" +source = "git+https://github.com/zcash/librustzcash.git?rev=b580c42bdc9cb0d0fb06a3975783be1bb34395e8#b580c42bdc9cb0d0fb06a3975783be1bb34395e8" dependencies = [ "base64", "bech32", @@ -2302,7 +2302,7 @@ dependencies = [ [[package]] name = "zcash_client_sqlite" version = "0.7.1" -source = "git+https://github.com/zcash/librustzcash.git?rev=f61d60bb967308b4c84defcfd6c8c7a70e1c276d#f61d60bb967308b4c84defcfd6c8c7a70e1c276d" +source = "git+https://github.com/zcash/librustzcash.git?rev=b580c42bdc9cb0d0fb06a3975783be1bb34395e8#b580c42bdc9cb0d0fb06a3975783be1bb34395e8" dependencies = [ "bs58", "byteorder", @@ -2328,7 +2328,7 @@ dependencies = [ [[package]] name = "zcash_encoding" version = "0.2.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=f61d60bb967308b4c84defcfd6c8c7a70e1c276d#f61d60bb967308b4c84defcfd6c8c7a70e1c276d" +source = "git+https://github.com/zcash/librustzcash.git?rev=b580c42bdc9cb0d0fb06a3975783be1bb34395e8#b580c42bdc9cb0d0fb06a3975783be1bb34395e8" dependencies = [ "byteorder", "nonempty", @@ -2350,7 +2350,7 @@ dependencies = [ [[package]] name = "zcash_primitives" version = "0.12.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=f61d60bb967308b4c84defcfd6c8c7a70e1c276d#f61d60bb967308b4c84defcfd6c8c7a70e1c276d" +source = "git+https://github.com/zcash/librustzcash.git?rev=b580c42bdc9cb0d0fb06a3975783be1bb34395e8#b580c42bdc9cb0d0fb06a3975783be1bb34395e8" dependencies = [ "aes", "bip0039", @@ -2385,7 +2385,7 @@ dependencies = [ [[package]] name = "zcash_proofs" version = "0.12.1" -source = "git+https://github.com/zcash/librustzcash.git?rev=f61d60bb967308b4c84defcfd6c8c7a70e1c276d#f61d60bb967308b4c84defcfd6c8c7a70e1c276d" +source = "git+https://github.com/zcash/librustzcash.git?rev=b580c42bdc9cb0d0fb06a3975783be1bb34395e8#b580c42bdc9cb0d0fb06a3975783be1bb34395e8" dependencies = [ "bellman", "blake2b_simd", diff --git a/backend-lib/Cargo.toml b/backend-lib/Cargo.toml index 63a2cc9d..4c8952cc 100644 --- a/backend-lib/Cargo.toml +++ b/backend-lib/Cargo.toml @@ -44,11 +44,11 @@ libc = "0.2" incrementalmerkletree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "da97e6c399c5acddedad2c1730dbb7ee55499a2f" } shardtree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "da97e6c399c5acddedad2c1730dbb7ee55499a2f" } orchard = { git = "https://github.com/zcash/orchard.git", rev = "6ef89d5f154de2cf7b7dd87edb8d8c49158beebb" } -zcash_address = { git = "https://github.com/zcash/librustzcash.git", rev = "f61d60bb967308b4c84defcfd6c8c7a70e1c276d" } -zcash_client_backend = { git = "https://github.com/zcash/librustzcash.git", rev = "f61d60bb967308b4c84defcfd6c8c7a70e1c276d" } -zcash_client_sqlite = { git = "https://github.com/zcash/librustzcash.git", rev = "f61d60bb967308b4c84defcfd6c8c7a70e1c276d" } -zcash_primitives = { git = "https://github.com/zcash/librustzcash.git", rev = "f61d60bb967308b4c84defcfd6c8c7a70e1c276d" } -zcash_proofs = { git = "https://github.com/zcash/librustzcash.git", rev = "f61d60bb967308b4c84defcfd6c8c7a70e1c276d" } +zcash_address = { git = "https://github.com/zcash/librustzcash.git", rev = "b580c42bdc9cb0d0fb06a3975783be1bb34395e8" } +zcash_client_backend = { git = "https://github.com/zcash/librustzcash.git", rev = "b580c42bdc9cb0d0fb06a3975783be1bb34395e8" } +zcash_client_sqlite = { git = "https://github.com/zcash/librustzcash.git", rev = "b580c42bdc9cb0d0fb06a3975783be1bb34395e8" } +zcash_primitives = { git = "https://github.com/zcash/librustzcash.git", rev = "b580c42bdc9cb0d0fb06a3975783be1bb34395e8" } +zcash_proofs = { git = "https://github.com/zcash/librustzcash.git", rev = "b580c42bdc9cb0d0fb06a3975783be1bb34395e8" } ## Uncomment this to test librustzcash changes locally #[patch.crates-io] diff --git a/backend-lib/src/main/rust/lib.rs b/backend-lib/src/main/rust/lib.rs index b79b0a63..3a1d5b67 100644 --- a/backend-lib/src/main/rust/lib.rs +++ b/backend-lib/src/main/rust/lib.rs @@ -1180,6 +1180,7 @@ fn encode_scan_range<'a>( scan_range: ScanRange, ) -> jni::errors::Result> { let priority = match scan_range.priority() { + ScanPriority::Ignored => 0, ScanPriority::Scanned => 10, ScanPriority::Historic => 20, ScanPriority::OpenAdjacent => 30, diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanRange.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanRange.kt index 39053bf3..e309eb31 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanRange.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanRange.kt @@ -31,6 +31,7 @@ internal data class ScanRange( @Suppress("MagicNumber") internal enum class SuggestScanRangePriority(val priority: Long) { + Ignored(0), Scanned(10), Historic(20), OpenAdjacent(30), From f23aca38a6dee8012a2e195ded7f813c99fd17ad Mon Sep 17 00:00:00 2001 From: Honza Date: Tue, 29 Aug 2023 10:30:15 +0200 Subject: [PATCH 32/76] Update getSuggestScanRangePriority fallback value --- .../java/cash/z/ecc/android/sdk/internal/model/ScanRange.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanRange.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanRange.kt index e309eb31..10693c61 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanRange.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanRange.kt @@ -11,7 +11,7 @@ internal data class ScanRange( internal fun getSuggestScanRangePriority(): SuggestScanRangePriority { return SuggestScanRangePriority.values() - .firstOrNull { it.priority == priority } ?: SuggestScanRangePriority.Scanned + .firstOrNull { it.priority == priority } ?: SuggestScanRangePriority.Ignored } companion object { From 10a7aa7f3f99b9fcda31f7b7041d22023c7c4aae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Honza=20Rychnovsk=C3=BD?= Date: Wed, 30 Aug 2023 08:19:55 +0200 Subject: [PATCH 33/76] [#1180] Frequent SbS synchronization restarting * [#1180] Frequent SbS synchronization restarting - Processing blocks with SbS split into preparation and processing functions, which can be called repeatedly. - Refactored other parts of the synchronization mechanism - Closes #1180 - This also partly solves #1137 * Update .gitignore * Update LINCENSE documentation * [#1177] Checkpoints update * Fix Ktlint warning --- .gitignore | 1 + LICENSE | 2 +- .../block/processor/CompactBlockProcessor.kt | 351 ++++++++++++------ .../processor/model/SbSPreparationResult.kt | 28 ++ .../block/processor/model/SyncingResult.kt | 7 +- .../internal/block/CompactBlockDownloader.kt | 4 +- .../android/sdk/internal/ext/WalletService.kt | 2 +- .../processor/CompactBlockProcessorTest.kt | 30 ++ 8 files changed, 295 insertions(+), 130 deletions(-) create mode 100644 sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/SbSPreparationResult.kt create mode 100644 sdk-lib/src/test/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessorTest.kt diff --git a/.gitignore b/.gitignore index 90af866e..fc0bfdd3 100644 --- a/.gitignore +++ b/.gitignore @@ -51,6 +51,7 @@ captures/ .idea/workspace.xml .idea/protoeditor.xml .idea/appInsightsSettings.xml +.idea/migrations.xml *.iml # Keystore files diff --git a/LICENSE b/LICENSE index 51ce0d39..4e6dfe52 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2017-2021 Electric Coin Company +Copyright (c) 2017-2023 Electric Coin Company Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessor.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessor.kt index af26c40e..591e8315 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessor.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessor.kt @@ -6,6 +6,7 @@ import cash.z.ecc.android.sdk.annotation.OpenForTesting import cash.z.ecc.android.sdk.block.processor.model.BatchSyncProgress import cash.z.ecc.android.sdk.block.processor.model.GetSubtreeRootsResult import cash.z.ecc.android.sdk.block.processor.model.PutSaplingSubtreeRootsResult +import cash.z.ecc.android.sdk.block.processor.model.SbSPreparationResult import cash.z.ecc.android.sdk.block.processor.model.SuggestScanRangesResult import cash.z.ecc.android.sdk.block.processor.model.SyncStageResult import cash.z.ecc.android.sdk.block.processor.model.SyncingResult @@ -27,8 +28,8 @@ import cash.z.ecc.android.sdk.internal.block.CompactBlockDownloader import cash.z.ecc.android.sdk.internal.ext.isNullOrEmpty import cash.z.ecc.android.sdk.internal.ext.isScanContinuityError import cash.z.ecc.android.sdk.internal.ext.length -import cash.z.ecc.android.sdk.internal.ext.retryUpTo import cash.z.ecc.android.sdk.internal.ext.retryUpToAndContinue +import cash.z.ecc.android.sdk.internal.ext.retryUpToAndThrow import cash.z.ecc.android.sdk.internal.ext.retryWithBackoff import cash.z.ecc.android.sdk.internal.ext.toHexReversed import cash.z.ecc.android.sdk.internal.model.BlockBatch @@ -75,6 +76,7 @@ import kotlin.math.max import kotlin.math.min import kotlin.time.Duration import kotlin.time.Duration.Companion.days +import kotlin.time.Duration.Companion.minutes import kotlin.time.DurationUnit import kotlin.time.toDuration @@ -149,6 +151,20 @@ class CompactBlockProcessor internal constructor( private val _networkHeight = MutableStateFlow(null) private val processingMutex = Mutex() + /** + * The synchronization-related variable that holds the all batch count computed in the first initial synchronization + * loop. It is supposed to keep the same value across the synchronization refreshes with [runSbSSyncingPreparation] + * as happens in [SyncAlgorithm.SPEND_BEFORE_SYNC] synchronization function. + */ + private var allBatchCount: Long = 0 + + /** + * Another synchronization-related variable that holds the order of a currently processing batch of blocks. It + * is supposed to preserve its value across the synchronization refreshes with [runSbSSyncingPreparation] as + * happens in [SyncAlgorithm.SPEND_BEFORE_SYNC] synchronization function. + */ + private var lastBatchOrder: Long = 0 + /** * Flow of birthday heights. The birthday is essentially the first block that the wallet cares * about. Any prior block can be ignored. This is not a fixed value because the height is @@ -225,13 +241,28 @@ class CompactBlockProcessor internal constructor( val result = processingMutex.withLockLogged("processNewBlocks") { when (subTreeRootResult) { is GetSubtreeRootsResult.UseSbS -> { + // Pass the commitment tree data to the database + when ( + val result = putSaplingSubtreeRoots( + backend = backend, + startIndex = 0, + subTreeRootList = (subTreeRootResult as GetSubtreeRootsResult.UseSbS) + .subTreeRootList, + lastValidHeight = lowerBoundHeight + ) + ) { + PutSaplingSubtreeRootsResult.Success -> { + // Lets continue with the next step + } + is PutSaplingSubtreeRootsResult.Failure -> { + BlockProcessingResult.SyncFailure(result.failedAtHeight, result.exception) + } + } processNewBlocksInSbSOrder( backend = backend, downloader = downloader, repository = repository, network = network, - subTreeRootList = (subTreeRootResult as GetSubtreeRootsResult.UseSbS) - .subTreeRootList, lastValidHeight = lowerBoundHeight, firstUnenhancedHeight = _processorInfo.value.firstUnenhancedHeight ) @@ -251,7 +282,8 @@ class CompactBlockProcessor internal constructor( } } } - // immediately process again after failures in order to download new blocks right away + + // Immediately process again after failures in order to download new blocks right away when (result) { BlockProcessingResult.Reconnecting -> { setState(State.Disconnected) @@ -264,6 +296,10 @@ class CompactBlockProcessor internal constructor( } delay(napTime) } + BlockProcessingResult.RestartSynchronization -> { + Twig.info { "Planned restarting of block synchronization..." } + // No nap time set to immediately continue with refreshed block synchronization + } BlockProcessingResult.NoBlocksToProcess -> { setState(State.Synced(_processorInfo.value.overallSyncRange)) val noWorkDone = _processorInfo.value.overallSyncRange?.isEmpty() ?: true @@ -371,93 +407,43 @@ class CompactBlockProcessor internal constructor( downloader: CompactBlockDownloader, repository: DerivedDataRepository, network: ZcashNetwork, - subTreeRootList: List, lastValidHeight: BlockHeight, firstUnenhancedHeight: BlockHeight? ): BlockProcessingResult { Twig.info { - "Beginning to process new blocks with Spend-before-Sync approach (with roots: $subTreeRootList, and lower" + - "bound: $lastValidHeight)..." + "Beginning to process new blocks with Spend-before-Sync approach with lower bound: $lastValidHeight)..." } - // Pass the commitment tree data to the database. - when ( - val result = - putSaplingSubtreeRoots( - backend = backend, - startIndex = 0, - subTreeRootList = subTreeRootList, - lastValidHeight = lastValidHeight - ) - ) { - PutSaplingSubtreeRootsResult.Success -> { /* Lets continue to the next step */ } - is PutSaplingSubtreeRootsResult.Failure -> { - return BlockProcessingResult.SyncFailure(result.failedAtHeight, result.exception) - } - } - - // Download chain tip metadata from lightwalletd - val chainTip = fetchLatestBlockHeight( + // This step covers these operations fetchLatestBlockHeight, updateChainTip, suggestScanRanges, updateRange, + // and shouldVerifySuggestedScanRanges + val preparationResult = runSbSSyncingPreparation( + backend = backend, downloader = downloader, - network = network - ) ?: let { - Twig.warn { "Disconnection detected. Attempting to reconnect." } - return BlockProcessingResult.Reconnecting - } - - // Notify the wallet of the updated chain tip - when ( - val result = - updateChainTip( - backend = backend, - chainTip = chainTip, - lastValidHeight = lastValidHeight - ) - ) { - is UpdateChainTipResult.Success -> { /* Lets continue to the next step */ } - is UpdateChainTipResult.Failure -> { - return BlockProcessingResult.SyncFailure(result.failedAtHeight, result.exception) - } - } - - // Get the suggested scan ranges from the wallet database - var suggestedRangesResult = suggestScanRanges( - backend, - lastValidHeight + network = network, + lastValidHeight = lastValidHeight ) - val updateRangeResult = when (suggestedRangesResult) { - is SuggestScanRangesResult.Success -> { - updateRange(suggestedRangesResult.ranges) + when (preparationResult) { + is SbSPreparationResult.ProcessFailure -> { + return preparationResult.toBlockProcessingResult() } - is SuggestScanRangesResult.Failure -> { - Twig.error { - "Process suggested scan ranges failure: " + - "${(suggestedRangesResult as SuggestScanRangesResult.Failure).exception}" - } - return BlockProcessingResult.SyncFailure( - suggestedRangesResult.failedAtHeight, - suggestedRangesResult.exception - ) + SbSPreparationResult.ConnectionFailure -> { + return BlockProcessingResult.Reconnecting + } + SbSPreparationResult.NoMoreBlocksToProcess -> { + return BlockProcessingResult.NoBlocksToProcess + } + is SbSPreparationResult.Success -> { + Twig.info { "Preparation phase done with result: $preparationResult" } + // Continue processing ranges } } - if (!updateRangeResult) { - Twig.warn { "Disconnection detected. Attempting to reconnect." } - return BlockProcessingResult.Reconnecting - } else if (_processorInfo.value.overallSyncRange.isNullOrEmpty()) { - Twig.info { "No more blocks to process." } - return BlockProcessingResult.NoBlocksToProcess - } - - setState(State.Syncing) - val allBatchCount = getBatchCount(suggestedRangesResult.ranges.map { it.range }).toFloat() - var lastBatchOrder: Long = 0 - - // Parse and process ranges. If it recognizes a range with Priority.Verify, it runs the verification part. - var verifyRangeResult = shouldVerifySuggestedScanRanges(suggestedRangesResult) - - Twig.info { "Check for verification of ranges resulted with: $verifyRangeResult" } + var verifyRangeResult = preparationResult.verifyRangeResult + var suggestedRangesResult = preparationResult.suggestedRangesResult + val allBatchCountLocal = preparationResult.allBatchCount + val lastPreparationTime = System.currentTimeMillis() + // Running synchronization for the [ScanRange.SuggestScanRangePriority.Verify] range while (verifyRangeResult is VerifySuggestedScanRange.ShouldVerify) { Twig.info { "Starting verification of range: $verifyRangeResult" } @@ -479,9 +465,11 @@ class CompactBlockProcessor internal constructor( enhanceStartHeight = firstUnenhancedHeight, lastBatchOrder = lastBatchOrder ).collect { rangeSyncProgress -> - setProgress(PercentDecimal(rangeSyncProgress.overallOrder / allBatchCount)) - // We need to update lastBatchOrder for the next ranges processing - lastBatchOrder = rangeSyncProgress.overallOrder + // We need to update lastBatchOrder for the processing of the following range. It can occasionally + // be over the precomputed all-batch count in case of inter-syncing failure. + lastBatchOrder = min(rangeSyncProgress.overallOrder, allBatchCountLocal) + + setProgress(PercentDecimal(lastBatchOrder / allBatchCountLocal.toFloat())) when (rangeSyncProgress.resultState) { SyncingResult.UpdateBirthday -> { @@ -496,15 +484,19 @@ class CompactBlockProcessor internal constructor( } } - if (syncingResult != SyncingResult.AllSuccess) { - // Remove persisted but not scanned blocks in case of any failure - val lastScannedHeight = getLastScannedHeight(repository) - downloader.rewindToHeight(lastScannedHeight) - deleteAllBlockFiles( - downloader = downloader, - lastKnownHeight = lastScannedHeight - ) - return (syncingResult as SyncingResult.Failure).toBlockProcessingResult() + when (syncingResult) { + is SyncingResult.AllSuccess -> { + // Continue with processing the rest of the ranges + } else -> { + // An error came - remove persisted but not scanned blocks + val lastScannedHeight = getLastScannedHeight(repository) + downloader.rewindToHeight(lastScannedHeight) + deleteAllBlockFiles( + downloader = downloader, + lastKnownHeight = lastScannedHeight + ) + return (syncingResult as SyncingResult.Failure).toBlockProcessingResult() + } } // Re-request suggested scan ranges @@ -514,10 +506,7 @@ class CompactBlockProcessor internal constructor( verifyRangeResult = shouldVerifySuggestedScanRanges(suggestedRangesResult) } is SuggestScanRangesResult.Failure -> { - Twig.error { - "Process suggested scan ranges failure: " + - "${(suggestedRangesResult as SuggestScanRangesResult.Failure).exception}" - } + Twig.error { "Process suggested scan ranges failure: ${suggestedRangesResult.exception}" } return BlockProcessingResult.SyncFailure( suggestedRangesResult.failedAtHeight, suggestedRangesResult.exception @@ -527,12 +516,6 @@ class CompactBlockProcessor internal constructor( } // Process the rest of ranges - - // Get the suggested scan ranges from the wallet database - suggestedRangesResult = suggestScanRanges( - backend, - lastValidHeight - ) val scanRanges = when (suggestedRangesResult) { is SuggestScanRangesResult.Success -> { suggestedRangesResult.ranges } is SuggestScanRangesResult.Failure -> { @@ -558,36 +541,138 @@ class CompactBlockProcessor internal constructor( withDownload = true, enhanceStartHeight = firstUnenhancedHeight, lastBatchOrder = lastBatchOrder - ).collect { rangeSyncProgress -> - setProgress(PercentDecimal(rangeSyncProgress.overallOrder / allBatchCount)) - lastBatchOrder = rangeSyncProgress.overallOrder + ).map { rangeSyncProgress -> + // We need to update lastBatchOrder for the processing of the following range. It can occasionally + // be over the precomputed all-batch count in case of inter-syncing failure. + lastBatchOrder = min(rangeSyncProgress.overallOrder, allBatchCountLocal) + + setProgress(PercentDecimal(lastBatchOrder / allBatchCountLocal.toFloat())) when (rangeSyncProgress.resultState) { SyncingResult.UpdateBirthday -> { updateBirthdayHeight() + SyncingResult.AllSuccess } is SyncingResult.Failure -> { - syncingResult = rangeSyncProgress.resultState - return@collect + rangeSyncProgress.resultState } else -> { - // Continue with processing + // First, check the time and refresh the prepare phase inputs, if needed + val currentTimeMillis = System.currentTimeMillis() + if (shouldRefreshPreparation( + lastPreparationTime, + currentTimeMillis, + SBS_SYNCHRONIZATION_RESTART_TIMEOUT + ) + ) { + SyncingResult.RestartSynchronization + } else { + // Continue with processing + SyncingResult.AllSuccess + } } } - } + }.takeWhile { + syncingResult = it + it == SyncingResult.AllSuccess + }.collect() - if (syncingResult != SyncingResult.AllSuccess) { - // Remove persisted but not scanned blocks in case of any failure - val lastScannedHeight = getLastScannedHeight(repository) - downloader.rewindToHeight(lastScannedHeight) - deleteAllBlockFiles( - downloader = downloader, - lastKnownHeight = lastScannedHeight + when (syncingResult) { + is SyncingResult.AllSuccess -> { + // Continue with processing the rest of the ranges + } + is SyncingResult.RestartSynchronization -> { + // Restarting the synchronization process + return BlockProcessingResult.RestartSynchronization + } else -> { + // An error came - remove persisted but not scanned blocks + val lastScannedHeight = getLastScannedHeight(repository) + downloader.rewindToHeight(lastScannedHeight) + deleteAllBlockFiles( + downloader = downloader, + lastKnownHeight = lastScannedHeight + ) + return (syncingResult as SyncingResult.Failure).toBlockProcessingResult() + } + } + } + return BlockProcessingResult.Success + } + + @Suppress("ReturnCount") + internal suspend fun runSbSSyncingPreparation( + backend: TypesafeBackend, + downloader: CompactBlockDownloader, + network: ZcashNetwork, + lastValidHeight: BlockHeight + ): SbSPreparationResult { + // Download chain tip metadata from lightwalletd + val chainTip = fetchLatestBlockHeight( + downloader = downloader, + network = network + ) ?: let { + Twig.warn { "Disconnection detected. Attempting to reconnect." } + return SbSPreparationResult.ConnectionFailure + } + + // Notify the underlying rust layer about the updated chain tip + when ( + val result = + updateChainTip( + backend = backend, + chainTip = chainTip, + lastValidHeight = lastValidHeight + ) + ) { + is UpdateChainTipResult.Success -> { /* Lets continue to the next step */ } + is UpdateChainTipResult.Failure -> { + return SbSPreparationResult.ProcessFailure( + result.failedAtHeight, + result.exception ) - return (syncingResult as SyncingResult.Failure).toBlockProcessingResult() } } - return BlockProcessingResult.Success + // Get the suggested scan ranges from the wallet database + val suggestedRangesResult = suggestScanRanges( + backend, + lastValidHeight + ) + val updateRangeResult = when (suggestedRangesResult) { + is SuggestScanRangesResult.Success -> { + updateRange(suggestedRangesResult.ranges) + } + is SuggestScanRangesResult.Failure -> { + Twig.error { "Process suggested scan ranges failure: ${suggestedRangesResult.exception}" } + return SbSPreparationResult.ProcessFailure( + suggestedRangesResult.failedAtHeight, + suggestedRangesResult.exception + ) + } + } + + if (!updateRangeResult) { + Twig.warn { "Disconnection detected. Attempting to reconnect." } + return SbSPreparationResult.ConnectionFailure + } else if (_processorInfo.value.overallSyncRange.isNullOrEmpty()) { + Twig.info { "No more blocks to process." } + return SbSPreparationResult.NoMoreBlocksToProcess + } + + setState(State.Syncing) + allBatchCount = max(allBatchCount, getBatchCount(suggestedRangesResult.ranges.map { it.range })) + lastBatchOrder = max(lastBatchOrder, 0) + + // Parse and process ranges. If it recognizes a range with Priority.Verify, it runs the verification part. + val verifyRangeResult = shouldVerifySuggestedScanRanges(suggestedRangesResult) + + Twig.info { "Check for verification of ranges resulted with: $verifyRangeResult" } + + return SbSPreparationResult.Success( + suggestedRangesResult = suggestedRangesResult, + verifyRangeResult = verifyRangeResult, + allBatchCount = allBatchCount, + lastBatchOrder = lastBatchOrder + ) } @Suppress("ReturnCount") @@ -597,7 +682,7 @@ class CompactBlockProcessor internal constructor( enhanceStartHeight: BlockHeight? ): BlockProcessingResult { var syncingResult: SyncingResult = SyncingResult.AllSuccess - val allBatchCount = getBatchCount(listOf(syncRange)).toFloat() + allBatchCount = getBatchCount(listOf(syncRange)) // Syncing last blocks and enhancing transactions runSyncingAndEnhancingOnRange( @@ -610,7 +695,7 @@ class CompactBlockProcessor internal constructor( enhanceStartHeight = enhanceStartHeight, lastBatchOrder = 0 ).collect { rangeSyncProgress -> - setProgress(PercentDecimal(rangeSyncProgress.overallOrder / allBatchCount)) + setProgress(PercentDecimal(rangeSyncProgress.overallOrder / allBatchCount.toFloat())) when (rangeSyncProgress.resultState) { SyncingResult.UpdateBirthday -> { @@ -643,6 +728,7 @@ class CompactBlockProcessor internal constructor( object NoBlocksToProcess : BlockProcessingResult() object Success : BlockProcessingResult() object Reconnecting : BlockProcessingResult() + object RestartSynchronization : BlockProcessingResult() data class SyncFailure(val failedAtHeight: BlockHeight, val error: Throwable) : BlockProcessingResult() } @@ -657,7 +743,6 @@ class CompactBlockProcessor internal constructor( * * @return true when the update succeeds. */ - @OptIn(ExperimentalStdlibApi::class) private suspend fun updateRange(ranges: List?): Boolean { // This fetches the latest height each time this method is called, which can be very inefficient // when downloading all of the blocks from the server @@ -804,7 +889,7 @@ class CompactBlockProcessor internal constructor( if (failedUtxoFetches < 9) { // there are 3 attempts per block @Suppress("TooGenericExceptionCaught") try { - retryUpTo(UTXO_FETCH_RETRIES) { + retryUpToAndThrow(UTXO_FETCH_RETRIES) { val tAddresses = backend.listTransparentReceivers(account) downloader.fetchUtxos( @@ -952,6 +1037,24 @@ class CompactBlockProcessor internal constructor( */ internal const val REWIND_DISTANCE = 10 + /** + * Limit millis value for restarting currently running block synchronization that runs under the + * [SyncAlgorithm.SPEND_BEFORE_SYNC] synchronization. + */ + internal val SBS_SYNCHRONIZATION_RESTART_TIMEOUT = 10.minutes.inWholeMilliseconds + + /** + * Check for the next restart of the block synchronization preparation phase. This function is only SbS + * synchronization algorithm-related. + */ + internal fun shouldRefreshPreparation( + lastPreparationTime: Long, + currentTimeMillis: Long, + limitTime: Long + ): Boolean { + return (currentTimeMillis - lastPreparationTime) >= limitTime + } + /** * This operation fetches and returns the latest block height (chain tip) * @@ -1173,6 +1276,8 @@ class CompactBlockProcessor internal constructor( /** * Requests, processes and persists all blocks from the given range. * + * Works the same for both [SyncAlgorithm.LINEAR] and [SyncAlgorithm.SPEND_BEFORE_SYNC] algorithms. + * * @param backend the Rust backend component * @param downloader the compact block downloader component * @param repository the derived data repository component @@ -1302,10 +1407,10 @@ class CompactBlockProcessor internal constructor( backend = backend, downloader = downloader ).collect { enhancingResult -> - Twig.debug { "Enhancing result: $enhancingResult" } + Twig.info { "Enhancing result: $enhancingResult" } resultState = when (enhancingResult) { is SyncingResult.UpdateBirthday -> { - Twig.debug { "Birthday height update reporting" } + Twig.info { "Birthday height update reporting" } enhancingResult } is SyncingResult.EnhanceFailed -> { @@ -1594,7 +1699,7 @@ class CompactBlockProcessor internal constructor( downloader: CompactBlockDownloader ): ByteArray { var transactionDataResult: ByteArray? = null - retryUpTo(TRANSACTION_FETCH_RETRIES) { failedAttempts -> + retryUpToAndThrow(TRANSACTION_FETCH_RETRIES) { failedAttempts -> if (failedAttempts == 0) { Twig.debug { "Starting to fetch transaction (id:$id, block:$minedHeight)" } } else { diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/SbSPreparationResult.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/SbSPreparationResult.kt new file mode 100644 index 00000000..21401dfc --- /dev/null +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/SbSPreparationResult.kt @@ -0,0 +1,28 @@ +package cash.z.ecc.android.sdk.block.processor.model + +import cash.z.ecc.android.sdk.block.processor.CompactBlockProcessor +import cash.z.ecc.android.sdk.model.BlockHeight + +/** + * Internal class for sharing pre-synchronization steps result. + */ +internal sealed class SbSPreparationResult { + object ConnectionFailure : SbSPreparationResult() + data class ProcessFailure( + val failedAtHeight: BlockHeight, + val exception: Throwable + ) : SbSPreparationResult() { + fun toBlockProcessingResult(): CompactBlockProcessor.BlockProcessingResult = + CompactBlockProcessor.BlockProcessingResult.SyncFailure( + this.failedAtHeight, + this.exception + ) + } + data class Success( + val suggestedRangesResult: SuggestScanRangesResult, + val verifyRangeResult: VerifySuggestedScanRange, + val allBatchCount: Long, + val lastBatchOrder: Long + ) : SbSPreparationResult() + object NoMoreBlocksToProcess : SbSPreparationResult() +} diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/SyncingResult.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/SyncingResult.kt index 6b32a7a5..7fa47866 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/SyncingResult.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/SyncingResult.kt @@ -9,11 +9,12 @@ import cash.z.ecc.android.sdk.model.BlockHeight * Internal class for the overall synchronization process result reporting. */ internal sealed class SyncingResult { + override fun toString(): String = this::class.java.simpleName + object AllSuccess : SyncingResult() + object RestartSynchronization : SyncingResult() data class DownloadSuccess(val downloadedBlocks: List?) : SyncingResult() { - override fun toString(): String { - return "DownloadSuccess with ${downloadedBlocks?.size ?: "none"} blocks" - } + override fun toString() = "${this::class.java.simpleName} with ${downloadedBlocks?.size ?: "none"} blocks" } interface Failure { val failedAtHeight: BlockHeight diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/block/CompactBlockDownloader.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/block/CompactBlockDownloader.kt index f961de8e..2b633215 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/block/CompactBlockDownloader.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/block/CompactBlockDownloader.kt @@ -2,7 +2,7 @@ package cash.z.ecc.android.sdk.internal.block import cash.z.ecc.android.sdk.exception.LightWalletException import cash.z.ecc.android.sdk.internal.Twig -import cash.z.ecc.android.sdk.internal.ext.retryUpTo +import cash.z.ecc.android.sdk.internal.ext.retryUpToAndThrow import cash.z.ecc.android.sdk.internal.model.JniBlockMeta import cash.z.ecc.android.sdk.internal.model.ext.from import cash.z.ecc.android.sdk.internal.repository.CompactBlockRepository @@ -111,7 +111,7 @@ open class CompactBlockDownloader private constructor(val compactBlockRepository compactBlockRepository.getLatestHeight() suspend fun getServerInfo(): LightWalletEndpointInfoUnsafe? = withContext(IO) { - retryUpTo(GET_SERVER_INFO_RETRIES) { + retryUpToAndThrow(GET_SERVER_INFO_RETRIES) { when (val response = lightWalletClient.getServerInfo()) { is Response.Success -> return@withContext response.result else -> { diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/WalletService.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/WalletService.kt index 743b39df..3f470546 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/WalletService.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/WalletService.kt @@ -17,7 +17,7 @@ import kotlin.random.Random * @param block the code to execute, which will be wrapped in a try/catch and retried whenever an * exception is thrown up to [retries] attempts. */ -suspend inline fun retryUpTo( +suspend inline fun retryUpToAndThrow( retries: Int, exceptionWrapper: (Throwable) -> Throwable = { it }, initialDelayMillis: Long = 500L, diff --git a/sdk-lib/src/test/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessorTest.kt b/sdk-lib/src/test/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessorTest.kt new file mode 100644 index 00000000..13526cde --- /dev/null +++ b/sdk-lib/src/test/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessorTest.kt @@ -0,0 +1,30 @@ +package cash.z.ecc.android.sdk.block.processor + +import kotlin.test.Test +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +class CompactBlockProcessorTest { + + @Test + fun should_refresh_preparation_test() { + assertTrue { + CompactBlockProcessor.shouldRefreshPreparation( + lastPreparationTime = CompactBlockProcessor.SBS_SYNCHRONIZATION_RESTART_TIMEOUT, + currentTimeMillis = CompactBlockProcessor.SBS_SYNCHRONIZATION_RESTART_TIMEOUT * 2, + limitTime = CompactBlockProcessor.SBS_SYNCHRONIZATION_RESTART_TIMEOUT + ) + } + } + + @Test + fun should_not_refresh_preparation_test() { + assertFalse { + CompactBlockProcessor.shouldRefreshPreparation( + lastPreparationTime = CompactBlockProcessor.SBS_SYNCHRONIZATION_RESTART_TIMEOUT, + currentTimeMillis = CompactBlockProcessor.SBS_SYNCHRONIZATION_RESTART_TIMEOUT, + limitTime = CompactBlockProcessor.SBS_SYNCHRONIZATION_RESTART_TIMEOUT + ) + } + } +} From c1d917244adffbb323313f39006664a9b4cf227b Mon Sep 17 00:00:00 2001 From: Honza Date: Wed, 30 Aug 2023 11:47:54 +0200 Subject: [PATCH 34/76] Update usage of SyncAlgorithm in Demo-app --- .../z/ecc/android/sdk/demoapp/WalletCoordinatorFactory.kt | 7 ++++++- .../main/java/cash/z/ecc/android/sdk/WalletCoordinator.kt | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/WalletCoordinatorFactory.kt b/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/WalletCoordinatorFactory.kt index 6f1728d2..9a7a8b5f 100644 --- a/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/WalletCoordinatorFactory.kt +++ b/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/WalletCoordinatorFactory.kt @@ -2,6 +2,7 @@ package cash.z.ecc.android.sdk.demoapp import android.content.Context import cash.z.ecc.android.sdk.WalletCoordinator +import cash.z.ecc.android.sdk.block.processor.CompactBlockProcessor import cash.z.ecc.android.sdk.demoapp.preference.EncryptedPreferenceKeys import cash.z.ecc.android.sdk.demoapp.preference.EncryptedPreferenceSingleton import cash.z.ecc.android.sdk.demoapp.util.LazyWithArgument @@ -20,7 +21,11 @@ private val lazy = LazyWithArgument { emitAll(EncryptedPreferenceKeys.PERSISTABLE_WALLET.observe(encryptedPreferenceProvider)) } - WalletCoordinator(it, persistableWalletFlow) + WalletCoordinator( + context = it, + persistableWallet = persistableWalletFlow, + syncAlgorithm = CompactBlockProcessor.SyncAlgorithm.SPEND_BEFORE_SYNC + ) } fun WalletCoordinator.Companion.getInstance(context: Context) = lazy.getInstance(context) diff --git a/sdk-incubator-lib/src/main/java/cash/z/ecc/android/sdk/WalletCoordinator.kt b/sdk-incubator-lib/src/main/java/cash/z/ecc/android/sdk/WalletCoordinator.kt index 9275f16f..eac7e1e1 100644 --- a/sdk-incubator-lib/src/main/java/cash/z/ecc/android/sdk/WalletCoordinator.kt +++ b/sdk-incubator-lib/src/main/java/cash/z/ecc/android/sdk/WalletCoordinator.kt @@ -47,7 +47,7 @@ import java.util.UUID class WalletCoordinator( context: Context, val persistableWallet: Flow, - val syncAlgorithm: CompactBlockProcessor.SyncAlgorithm = CompactBlockProcessor.SyncAlgorithm.SPEND_BEFORE_SYNC + val syncAlgorithm: CompactBlockProcessor.SyncAlgorithm = CompactBlockProcessor.SyncAlgorithm.LINEAR ) { private val applicationContext = context.applicationContext From 83ea5b55c3dcacc7d8f63fa3a2169e989ce5f982 Mon Sep 17 00:00:00 2001 From: Honza Date: Mon, 4 Sep 2023 09:13:49 +0200 Subject: [PATCH 35/76] Fix ktlint warning --- .../main/java/cash/z/ecc/android/sdk/internal/ext/NumberExt.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/NumberExt.kt b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/NumberExt.kt index b8bcf686..0ad93baf 100644 --- a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/NumberExt.kt +++ b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/NumberExt.kt @@ -1,4 +1,4 @@ -@file:Suppress("ktlint:filename") +@file:Suppress("ktlint:standard:filename") package cash.z.ecc.android.sdk.internal.ext From 648b31a5558b544889fe79d3038fee31f1019365 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Honza=20Rychnovsk=C3=BD?= Date: Mon, 4 Sep 2023 20:27:46 +0200 Subject: [PATCH 36/76] [#1170] Regular transaction flow emitting * [#1170] Regular transaction flow emitting Although zcash/librustzcash#918 will provide information on when to trigger transaction polling, this partly solves the issue as well * Update changelog --- CHANGELOG.md | 5 +++- demo-app/build.gradle.kts | 1 + .../{Transactions.kt => TransactionsView.kt} | 9 ++++--- gradle.properties | 1 + .../block/processor/CompactBlockProcessor.kt | 27 +++++++++++++++++-- settings.gradle.kts | 2 ++ 6 files changed, 39 insertions(+), 6 deletions(-) rename demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/ui/screen/transactions/view/{Transactions.kt => TransactionsView.kt} (94%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b43dcdb..d9ca029b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,7 +37,10 @@ which speeds up discovering the wallet's spendable balance. - Checkpoints ### Fixed -- `Synchronizer.getMemos()` now correctly returns a flow of strings for sent and received transactions. +- `Synchronizer.getMemos()` now correctly returns a flow of strings for sent and received transactions. Issue **#1154**. +- `CompactBlockProcessor` now triggers transaction polling while block synchronization is in progress as expected. + Clients will be notified briefly after every new transaction is discovered via `Synchronizer.transactions` API. + Issue **#1170**. ## 1.20.0-beta01 - The SDK internally migrated from `BackendExt` rust backend extension functions to more type-safe `TypesafeBackend`. diff --git a/demo-app/build.gradle.kts b/demo-app/build.gradle.kts index 0fe25e31..55f174cd 100644 --- a/demo-app/build.gradle.kts +++ b/demo-app/build.gradle.kts @@ -137,6 +137,7 @@ dependencies { implementation(libs.bundles.grpc) implementation(libs.kotlinx.datetime) + implementation(libs.kotlinx.immutable) } fladle { diff --git a/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/ui/screen/transactions/view/Transactions.kt b/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/ui/screen/transactions/view/TransactionsView.kt similarity index 94% rename from demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/ui/screen/transactions/view/Transactions.kt rename to demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/ui/screen/transactions/view/TransactionsView.kt index 24ced1b1..a9f9522d 100644 --- a/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/ui/screen/transactions/view/Transactions.kt +++ b/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/ui/screen/transactions/view/TransactionsView.kt @@ -31,6 +31,8 @@ import cash.z.ecc.android.sdk.demoapp.R import cash.z.ecc.android.sdk.internal.Twig import cash.z.ecc.android.sdk.model.TransactionOverview import cash.z.ecc.android.sdk.model.WalletAddresses +import kotlinx.collections.immutable.ImmutableList +import kotlinx.collections.immutable.toPersistentList import kotlinx.coroutines.flow.flow import kotlinx.coroutines.flow.toList import kotlinx.coroutines.launch @@ -41,7 +43,7 @@ private fun ComposablePreview() { MaterialTheme { // TODO [#1090]: Demo: Add Addresses and Transactions Compose Previews // TODO [#1090]: https://github.com/zcash/zcash-android-wallet-sdk/issues/1090 - // Transactions() + // TransactionsView() } } @@ -72,6 +74,7 @@ fun Transactions( paddingValues = paddingValues, synchronizer, synchronizer.transactions.collectAsStateWithLifecycle(initialValue = emptyList()).value + .toPersistentList() ) } } @@ -110,7 +113,7 @@ private fun TransactionsTopAppBar( private fun TransactionsMainContent( paddingValues: PaddingValues, synchronizer: Synchronizer, - transactions: List + transactions: ImmutableList ) { val queryScope = rememberCoroutineScope() Column( @@ -127,7 +130,7 @@ private fun TransactionsMainContent( val memos = synchronizer.getMemos(it) queryScope.launch { memos.toList().run { - Twig.debug { + Twig.info { "Transaction memos: count: $size, contains: ${joinToString().ifEmpty { "-" }}" } } diff --git a/gradle.properties b/gradle.properties index bed50169..a5456236 100644 --- a/gradle.properties +++ b/gradle.properties @@ -136,6 +136,7 @@ JAVAX_ANNOTATION_VERSION=1.3.2 JUNIT_VERSION=5.9.3 KOTLINX_COROUTINES_VERSION=1.7.3 KOTLINX_DATETIME_VERSION=0.4.0 +KOTLINX_IMMUTABLE_COLLECTIONS_VERSION=0.3.5 KOTLIN_VERSION=1.9.10 MOCKITO_KOTLIN_VERSION=2.2.0 MOCKITO_VERSION=5.4.0 diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessor.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessor.kt index 591e8315..065647a0 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessor.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessor.kt @@ -475,6 +475,11 @@ class CompactBlockProcessor internal constructor( SyncingResult.UpdateBirthday -> { updateBirthdayHeight() } + SyncingResult.EnhanceSuccess -> { + Twig.info { "Triggering transaction refresh now" } + // Invalidate transaction data + refreshTransactions(transactionStorage = repository) + } is SyncingResult.Failure -> { syncingResult = rangeSyncProgress.resultState return@collect @@ -553,6 +558,12 @@ class CompactBlockProcessor internal constructor( updateBirthdayHeight() SyncingResult.AllSuccess } + SyncingResult.EnhanceSuccess -> { + Twig.info { "Triggering transaction refresh now" } + // Invalidate transaction data and return the common batch syncing success result to the caller + refreshTransactions(transactionStorage = repository) + SyncingResult.AllSuccess + } is SyncingResult.Failure -> { rangeSyncProgress.resultState } else -> { @@ -598,6 +609,13 @@ class CompactBlockProcessor internal constructor( return BlockProcessingResult.Success } + /** + * This invalidates transaction storage to trigger data refreshing for its subscribers. + */ + private fun refreshTransactions(transactionStorage: DerivedDataRepository) { + transactionStorage.invalidate() + } + @Suppress("ReturnCount") internal suspend fun runSbSSyncingPreparation( backend: TypesafeBackend, @@ -701,6 +719,11 @@ class CompactBlockProcessor internal constructor( SyncingResult.UpdateBirthday -> { updateBirthdayHeight() } + SyncingResult.EnhanceSuccess -> { + Twig.info { "Triggering transaction refresh now" } + // Invalidate transaction data + refreshTransactions(transactionStorage = repository) + } is SyncingResult.Failure -> { syncingResult = rangeSyncProgress.resultState return@collect @@ -1418,8 +1441,8 @@ class CompactBlockProcessor internal constructor( enhancingResult } else -> { - // Transactions enhanced correctly. Now we return common sync success state. - SyncingResult.AllSuccess + // Transactions enhanced correctly. Let's continue with block processing. + enhancingResult } } emit( diff --git a/settings.gradle.kts b/settings.gradle.kts index af130448..7792523f 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -110,6 +110,7 @@ dependencyResolutionManagement { val kotlinVersion = extra["KOTLIN_VERSION"].toString() val kotlinxCoroutinesVersion = extra["KOTLINX_COROUTINES_VERSION"].toString() val kotlinxDateTimeVersion = extra["KOTLINX_DATETIME_VERSION"].toString() + val kotlinxImmutableCollectionsVersion = extra["KOTLINX_IMMUTABLE_COLLECTIONS_VERSION"].toString() val mockitoKotlinVersion = extra["MOCKITO_KOTLIN_VERSION"].toString() val mockitoVersion = extra["MOCKITO_VERSION"].toString() val protocVersion = extra["PROTOC_VERSION"].toString() @@ -167,6 +168,7 @@ dependencyResolutionManagement { library("kotlinx-coroutines-android", "org.jetbrains.kotlinx:kotlinx-coroutines-android:$kotlinxCoroutinesVersion") library("kotlinx-coroutines-core", "org.jetbrains.kotlinx:kotlinx-coroutines-core:$kotlinxCoroutinesVersion") library("kotlinx-datetime", "org.jetbrains.kotlinx:kotlinx-datetime:$kotlinxDateTimeVersion") + library("kotlinx-immutable", "org.jetbrains.kotlinx:kotlinx-collections-immutable:$kotlinxImmutableCollectionsVersion") library("material", "com.google.android.material:material:$googleMaterialVersion") library("zcashwalletplgn", "com.github.zcash:zcash-android-wallet-plugins:$zcashWalletPluginVersion") From 11d9529f2b974eb543eebb75881d6d393d016289 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Wed, 6 Sep 2023 01:54:59 +0000 Subject: [PATCH 37/76] Migrate to latest Rust revision - Account birthdays - Scan progress - Bugfixes --- backend-lib/Cargo.lock | 18 +- backend-lib/Cargo.toml | 11 +- .../z/ecc/android/sdk/internal/Backend.kt | 27 +-- .../android/sdk/internal/jni/RustBackend.kt | 76 +++---- .../sdk/internal/model/JniScanProgress.kt | 16 ++ backend-lib/src/main/rust/lib.rs | 215 +++++++----------- .../client/model/TreeStateUnsafe.kt | 28 +++ .../cash/z/ecc/fixture/FakeRustBackend.kt | 23 +- .../cash/z/ecc/android/sdk/SdkSynchronizer.kt | 12 +- .../cash/z/ecc/android/sdk/Synchronizer.kt | 10 +- .../android/sdk/internal/TypesafeBackend.kt | 20 +- .../sdk/internal/TypesafeBackendImpl.kt | 37 ++- .../sdk/internal/db/derived/DerivedDataDb.kt | 22 +- .../android/sdk/internal/model/Checkpoint.kt | 8 + .../sdk/internal/model/ScanProgress.kt | 17 ++ 15 files changed, 256 insertions(+), 284 deletions(-) create mode 100644 backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniScanProgress.kt create mode 100644 lightwallet-client-lib/src/main/java/co/electriccoin/lightwallet/client/model/TreeStateUnsafe.kt create mode 100644 sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanProgress.kt diff --git a/backend-lib/Cargo.lock b/backend-lib/Cargo.lock index ce036664..b8e5a6ee 100644 --- a/backend-lib/Cargo.lock +++ b/backend-lib/Cargo.lock @@ -483,7 +483,7 @@ checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" [[package]] name = "equihash" version = "0.2.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=b580c42bdc9cb0d0fb06a3975783be1bb34395e8#b580c42bdc9cb0d0fb06a3975783be1bb34395e8" +source = "git+https://github.com/zcash/librustzcash.git?rev=b4772e948d1b5153a8bf13d29ae10259eb930f53#b4772e948d1b5153a8bf13d29ae10259eb930f53" dependencies = [ "blake2b_simd", "byteorder", @@ -513,7 +513,7 @@ dependencies = [ [[package]] name = "f4jumble" version = "0.1.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=b580c42bdc9cb0d0fb06a3975783be1bb34395e8#b580c42bdc9cb0d0fb06a3975783be1bb34395e8" +source = "git+https://github.com/zcash/librustzcash.git?rev=b4772e948d1b5153a8bf13d29ae10259eb930f53#b4772e948d1b5153a8bf13d29ae10259eb930f53" dependencies = [ "blake2b_simd", ] @@ -2241,6 +2241,7 @@ dependencies = [ "log-panics", "orchard", "paranoid-android", + "prost", "rayon", "rusqlite", "schemer", @@ -2258,7 +2259,7 @@ dependencies = [ [[package]] name = "zcash_address" version = "0.3.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=b580c42bdc9cb0d0fb06a3975783be1bb34395e8#b580c42bdc9cb0d0fb06a3975783be1bb34395e8" +source = "git+https://github.com/zcash/librustzcash.git?rev=b4772e948d1b5153a8bf13d29ae10259eb930f53#b4772e948d1b5153a8bf13d29ae10259eb930f53" dependencies = [ "bech32", "bs58", @@ -2269,7 +2270,7 @@ dependencies = [ [[package]] name = "zcash_client_backend" version = "0.9.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=b580c42bdc9cb0d0fb06a3975783be1bb34395e8#b580c42bdc9cb0d0fb06a3975783be1bb34395e8" +source = "git+https://github.com/zcash/librustzcash.git?rev=b4772e948d1b5153a8bf13d29ae10259eb930f53#b4772e948d1b5153a8bf13d29ae10259eb930f53" dependencies = [ "base64", "bech32", @@ -2279,6 +2280,7 @@ dependencies = [ "crossbeam-channel", "group", "hdwallet 0.4.1", + "hex", "incrementalmerkletree", "memuse", "nom", @@ -2302,7 +2304,7 @@ dependencies = [ [[package]] name = "zcash_client_sqlite" version = "0.7.1" -source = "git+https://github.com/zcash/librustzcash.git?rev=b580c42bdc9cb0d0fb06a3975783be1bb34395e8#b580c42bdc9cb0d0fb06a3975783be1bb34395e8" +source = "git+https://github.com/zcash/librustzcash.git?rev=b4772e948d1b5153a8bf13d29ae10259eb930f53#b4772e948d1b5153a8bf13d29ae10259eb930f53" dependencies = [ "bs58", "byteorder", @@ -2328,7 +2330,7 @@ dependencies = [ [[package]] name = "zcash_encoding" version = "0.2.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=b580c42bdc9cb0d0fb06a3975783be1bb34395e8#b580c42bdc9cb0d0fb06a3975783be1bb34395e8" +source = "git+https://github.com/zcash/librustzcash.git?rev=b4772e948d1b5153a8bf13d29ae10259eb930f53#b4772e948d1b5153a8bf13d29ae10259eb930f53" dependencies = [ "byteorder", "nonempty", @@ -2350,7 +2352,7 @@ dependencies = [ [[package]] name = "zcash_primitives" version = "0.12.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=b580c42bdc9cb0d0fb06a3975783be1bb34395e8#b580c42bdc9cb0d0fb06a3975783be1bb34395e8" +source = "git+https://github.com/zcash/librustzcash.git?rev=b4772e948d1b5153a8bf13d29ae10259eb930f53#b4772e948d1b5153a8bf13d29ae10259eb930f53" dependencies = [ "aes", "bip0039", @@ -2385,7 +2387,7 @@ dependencies = [ [[package]] name = "zcash_proofs" version = "0.12.1" -source = "git+https://github.com/zcash/librustzcash.git?rev=b580c42bdc9cb0d0fb06a3975783be1bb34395e8#b580c42bdc9cb0d0fb06a3975783be1bb34395e8" +source = "git+https://github.com/zcash/librustzcash.git?rev=b4772e948d1b5153a8bf13d29ae10259eb930f53#b4772e948d1b5153a8bf13d29ae10259eb930f53" dependencies = [ "bellman", "blake2b_simd", diff --git a/backend-lib/Cargo.toml b/backend-lib/Cargo.toml index 4c8952cc..bf44266f 100644 --- a/backend-lib/Cargo.toml +++ b/backend-lib/Cargo.toml @@ -16,6 +16,7 @@ hdwallet = "0.3.1" hdwallet-bitcoin = "0.3" hex = "0.4" jni = { version = "0.20", default-features = false } +prost = "0.11" rusqlite = "0.29" schemer = "0.2" secp256k1 = "0.21" @@ -44,11 +45,11 @@ libc = "0.2" incrementalmerkletree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "da97e6c399c5acddedad2c1730dbb7ee55499a2f" } shardtree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "da97e6c399c5acddedad2c1730dbb7ee55499a2f" } orchard = { git = "https://github.com/zcash/orchard.git", rev = "6ef89d5f154de2cf7b7dd87edb8d8c49158beebb" } -zcash_address = { git = "https://github.com/zcash/librustzcash.git", rev = "b580c42bdc9cb0d0fb06a3975783be1bb34395e8" } -zcash_client_backend = { git = "https://github.com/zcash/librustzcash.git", rev = "b580c42bdc9cb0d0fb06a3975783be1bb34395e8" } -zcash_client_sqlite = { git = "https://github.com/zcash/librustzcash.git", rev = "b580c42bdc9cb0d0fb06a3975783be1bb34395e8" } -zcash_primitives = { git = "https://github.com/zcash/librustzcash.git", rev = "b580c42bdc9cb0d0fb06a3975783be1bb34395e8" } -zcash_proofs = { git = "https://github.com/zcash/librustzcash.git", rev = "b580c42bdc9cb0d0fb06a3975783be1bb34395e8" } +zcash_address = { git = "https://github.com/zcash/librustzcash.git", rev = "b4772e948d1b5153a8bf13d29ae10259eb930f53" } +zcash_client_backend = { git = "https://github.com/zcash/librustzcash.git", rev = "b4772e948d1b5153a8bf13d29ae10259eb930f53" } +zcash_client_sqlite = { git = "https://github.com/zcash/librustzcash.git", rev = "b4772e948d1b5153a8bf13d29ae10259eb930f53" } +zcash_primitives = { git = "https://github.com/zcash/librustzcash.git", rev = "b4772e948d1b5153a8bf13d29ae10259eb930f53" } +zcash_proofs = { git = "https://github.com/zcash/librustzcash.git", rev = "b4772e948d1b5153a8bf13d29ae10259eb930f53" } ## Uncomment this to test librustzcash changes locally #[patch.crates-io] diff --git a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/Backend.kt b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/Backend.kt index 35de48ad..b9410d35 100644 --- a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/Backend.kt +++ b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/Backend.kt @@ -1,6 +1,7 @@ package cash.z.ecc.android.sdk.internal import cash.z.ecc.android.sdk.internal.model.JniBlockMeta +import cash.z.ecc.android.sdk.internal.model.JniScanProgress import cash.z.ecc.android.sdk.internal.model.JniScanRange import cash.z.ecc.android.sdk.internal.model.JniSubtreeRoot import cash.z.ecc.android.sdk.internal.model.JniUnifiedSpendingKey @@ -35,27 +36,9 @@ interface Backend { suspend fun decryptAndStoreTransaction(tx: ByteArray) - /** - * @param keys A list of UFVKs to initialize the accounts table with - * @throws RuntimeException as a common indicator of the operation failure - */ - @Throws(RuntimeException::class) - suspend fun initAccountsTable(vararg keys: String) - - /** - * @throws RuntimeException as a common indicator of the operation failure - */ - @Throws(RuntimeException::class) - suspend fun initBlocksTable( - checkpointHeight: Long, - checkpointHash: String, - checkpointTime: Long, - checkpointSaplingTree: String, - ) - suspend fun initDataDb(seed: ByteArray?): Int - suspend fun createAccount(seed: ByteArray): JniUnifiedSpendingKey + suspend fun createAccount(seed: ByteArray, treeState: ByteArray, recoverUntil: Long?): JniUnifiedSpendingKey fun isValidShieldedAddr(addr: String): Boolean @@ -105,6 +88,12 @@ interface Backend { @Throws(RuntimeException::class) suspend fun updateChainTip(height: Long) + /** + * @throws RuntimeException as a common indicator of the operation failure + */ + @Throws(RuntimeException::class) + suspend fun getScanProgress(): JniScanProgress? + /** * @throws RuntimeException as a common indicator of the operation failure */ diff --git a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/jni/RustBackend.kt b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/jni/RustBackend.kt index a84be3f9..858a09cc 100644 --- a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/jni/RustBackend.kt +++ b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/jni/RustBackend.kt @@ -5,6 +5,7 @@ import cash.z.ecc.android.sdk.internal.SdkDispatchers import cash.z.ecc.android.sdk.internal.ext.deleteRecursivelySuspend import cash.z.ecc.android.sdk.internal.ext.deleteSuspend import cash.z.ecc.android.sdk.internal.model.JniBlockMeta +import cash.z.ecc.android.sdk.internal.model.JniScanProgress import cash.z.ecc.android.sdk.internal.model.JniScanRange import cash.z.ecc.android.sdk.internal.model.JniSubtreeRoot import cash.z.ecc.android.sdk.internal.model.JniUnifiedSpendingKey @@ -68,42 +69,17 @@ class RustBackend private constructor( ) } - override suspend fun createAccount(seed: ByteArray): JniUnifiedSpendingKey { + override suspend fun createAccount( + seed: ByteArray, + treeState: ByteArray, + recoverUntil: Long? + ): JniUnifiedSpendingKey { return withContext(SdkDispatchers.DATABASE_IO) { createAccount( dataDbFile.absolutePath, seed, - networkId = networkId - ) - } - } - - /** - * @param keys A list of UFVKs to initialize the accounts table with - */ - override suspend fun initAccountsTable(vararg keys: String) { - return withContext(SdkDispatchers.DATABASE_IO) { - initAccountsTableWithKeys( - dataDbFile.absolutePath, - keys, - networkId = networkId - ) - } - } - - override suspend fun initBlocksTable( - checkpointHeight: Long, - checkpointHash: String, - checkpointTime: Long, - checkpointSaplingTree: String, - ) { - return withContext(SdkDispatchers.DATABASE_IO) { - initBlocksTable( - dataDbFile.absolutePath, - checkpointHeight, - checkpointHash, - checkpointTime, - checkpointSaplingTree, + treeState, + recoverUntil ?: -1, networkId = networkId ) } @@ -263,6 +239,14 @@ class RustBackend private constructor( ) } + override suspend fun getScanProgress(): JniScanProgress = + withContext(SdkDispatchers.DATABASE_IO) { + getScanProgress( + dataDbFile.absolutePath, + networkId = networkId + ) + } + override suspend fun suggestScanRanges(): List { return withContext(SdkDispatchers.DATABASE_IO) { suggestScanRanges( @@ -412,25 +396,13 @@ class RustBackend private constructor( private external fun initDataDb(dbDataPath: String, seed: ByteArray?, networkId: Int): Int @JvmStatic - private external fun initAccountsTableWithKeys( + private external fun createAccount( dbDataPath: String, - ufvks: Array, + seed: ByteArray, + treeState: ByteArray, + recoverUntil: Long, networkId: Int - ) - - @JvmStatic - @Suppress("LongParameterList") - private external fun initBlocksTable( - dbDataPath: String, - height: Long, - hash: String, - time: Long, - saplingTree: String, - networkId: Int - ) - - @JvmStatic - private external fun createAccount(dbDataPath: String, seed: ByteArray, networkId: Int): JniUnifiedSpendingKey + ): JniUnifiedSpendingKey @JvmStatic private external fun getCurrentAddress( @@ -530,6 +502,12 @@ class RustBackend private constructor( networkId: Int ) + @JvmStatic + private external fun getScanProgress( + dbDataPath: String, + networkId: Int + ): JniScanProgress + @JvmStatic private external fun suggestScanRanges( dbDataPath: String, diff --git a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniScanProgress.kt b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniScanProgress.kt new file mode 100644 index 00000000..514ea216 --- /dev/null +++ b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniScanProgress.kt @@ -0,0 +1,16 @@ +package cash.z.ecc.android.sdk.internal.model + +import androidx.annotation.Keep + +/** + * Serves as cross layer (Kotlin, Rust) communication class. + * + * @param numerator the numerator of the progress ratio + * @param endHeight the denominator of the progress ratio + */ +@Keep +class JniScanProgress( + val numerator: Long, + val denominator: Long +) { +} diff --git a/backend-lib/src/main/rust/lib.rs b/backend-lib/src/main/rust/lib.rs index 3a1d5b67..1e2027f6 100644 --- a/backend-lib/src/main/rust/lib.rs +++ b/backend-lib/src/main/rust/lib.rs @@ -1,4 +1,3 @@ -use std::collections::HashMap; use std::convert::{TryFrom, TryInto}; use std::num::NonZeroU32; use std::panic; @@ -12,6 +11,7 @@ use jni::{ sys::{jboolean, jbyteArray, jint, jlong, jobject, jobjectArray, jstring, JNI_FALSE, JNI_TRUE}, JNIEnv, }; +use prost::Message; use schemer::MigratorError; use secrecy::{ExposeSecret, SecretVec}; use tracing::{debug, error}; @@ -20,7 +20,7 @@ use tracing_subscriber::reload; use zcash_address::{ToAddress, ZcashAddress}; use zcash_client_backend::data_api::{ scanning::{ScanPriority, ScanRange}, - NoteId, ShieldedProtocol, + AccountBirthday, NoteId, Ratio, ShieldedProtocol, }; use zcash_client_backend::keys::{DecodingError, UnifiedSpendingKey}; use zcash_client_backend::{ @@ -36,13 +36,14 @@ use zcash_client_backend::{ encoding::AddressCodec, fees::DustOutputPolicy, keys::{Era, UnifiedFullViewingKey}, + proto::service::TreeState, wallet::{OvkPolicy, WalletTransparentOutput}, zip321::{Payment, TransactionRequest}, }; use zcash_client_sqlite::chain::init::init_blockmeta_db; use zcash_client_sqlite::{ chain::BlockMeta, - wallet::init::{init_accounts_table, init_blocks_table, init_wallet_db, WalletMigrationError}, + wallet::init::{init_wallet_db, WalletMigrationError}, FsBlockDb, WalletDb, }; use zcash_primitives::consensus::Network::{MainNetwork, TestNetwork}; @@ -75,7 +76,8 @@ mod zip317 { pub(super) use zcash_primitives::transaction::fees::zip317::*; } -const ANCHOR_OFFSET: NonZeroU32 = unsafe { NonZeroU32::new_unchecked(10) }; +const ANCHOR_OFFSET_U32: u32 = 10; +const ANCHOR_OFFSET: NonZeroU32 = unsafe { NonZeroU32::new_unchecked(ANCHOR_OFFSET_U32) }; #[cfg(debug_assertions)] fn print_debug_state() { @@ -259,15 +261,32 @@ pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_cr _: JClass<'_>, db_data: JString<'_>, seed: jbyteArray, + treestate: jbyteArray, + recover_until: jlong, network_id: jint, ) -> jobject { + use zcash_client_backend::data_api::BirthdayError; + let res = panic::catch_unwind(|| { let network = parse_network(network_id as u32)?; let mut db_data = wallet_db(&env, network, db_data)?; let seed = SecretVec::new(env.convert_byte_array(seed).unwrap()); + let treestate = TreeState::decode(&env.convert_byte_array(treestate).unwrap()[..]) + .map_err(|e| format_err!("Invalid TreeState: {}", e))?; + let recover_until = recover_until.try_into().ok(); + + let birthday = + AccountBirthday::from_treestate(treestate, recover_until).map_err(|e| match e { + BirthdayError::HeightInvalid(e) => { + format_err!("Invalid TreeState: Invalid height: {}", e) + } + BirthdayError::Decode(e) => { + format_err!("Invalid TreeState: Invalid frontier encoding: {}", e) + } + })?; let (account, usk) = db_data - .create_account(&seed) + .create_account(&seed, birthday) .map_err(|e| format_err!("Error while initializing accounts: {}", e))?; encode_usk(&env, account, usk) @@ -275,52 +294,6 @@ pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_cr unwrap_exc_or(&env, res, ptr::null_mut()) } -/// Initialises the data database with the given set of unified full viewing keys. -/// -/// This should only be used in special cases for implementing wallet recovery; prefer -/// `RustBackend.createAccount` for normal account creation purposes. -#[no_mangle] -pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_initAccountsTableWithKeys( - env: JNIEnv<'_>, - _: JClass<'_>, - db_data: JString<'_>, - ufvks_arr: jobjectArray, - network_id: jint, -) -> jboolean { - let res = panic::catch_unwind(|| { - let network = parse_network(network_id as u32)?; - let mut db_data = wallet_db(&env, network, db_data)?; - // TODO: avoid all this unwrapping and also surface errors, better - let count = env.get_array_length(ufvks_arr).unwrap(); - let ufvks = (0..count) - .map(|i| env.get_object_array_element(ufvks_arr, i)) - .map(|jstr| utils::java_string_to_rust(&env, jstr.unwrap().into())) - .map(|ufvkstr| { - UnifiedFullViewingKey::decode(&network, &ufvkstr).map_err(|e| { - if e.starts_with("UFVK is for network") { - let (network_name, other) = if network == TestNetwork { - ("testnet", "mainnet") - } else { - ("mainnet", "testnet") - }; - format_err!("Error: Wrong network! Unable to decode viewing key for {}. Check whether this is a key for {}.", network_name, other) - } else { - format_err!("Invalid Unified Full Viewing Key: {}", e) - } - }) - }) - .enumerate() // TODO: Pass account IDs across the FFI. - .map(|(i, res)| res.map(|ufvk| (AccountId::from(i as u32), ufvk))) - .collect::, _>>()?; - - match init_accounts_table(&mut db_data, &ufvks) { - Ok(()) => Ok(JNI_TRUE), - Err(e) => Err(format_err!("Error while initializing accounts: {}", e)), - } - }); - unwrap_exc_or(&env, res, JNI_FALSE) -} - /// Derives and returns a unified spending key from the given seed for the given account ID. /// /// Returns the newly created [ZIP 316] account identifier, along with the binary encoding @@ -478,48 +451,6 @@ pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustDerivation unwrap_exc_or(&env, res, ptr::null_mut()) } -#[no_mangle] -pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_initBlocksTable( - env: JNIEnv<'_>, - _: JClass<'_>, - db_data: JString<'_>, - height: jlong, - hash_string: JString<'_>, - time: jlong, - sapling_tree_string: JString<'_>, - network_id: jint, -) -> jboolean { - let res = panic::catch_unwind(|| { - let network = parse_network(network_id as u32)?; - let mut db_data = wallet_db(&env, network, db_data)?; - let hash = { - let mut hash = hex::decode(utils::java_string_to_rust(&env, hash_string)).unwrap(); - hash.reverse(); - BlockHash::from_slice(&hash) - }; - let time = if time >= 0 && time <= jlong::from(u32::max_value()) { - time as u32 - } else { - return Err(format_err!("time argument must fit in a u32")); - }; - let sapling_tree = - hex::decode(utils::java_string_to_rust(&env, sapling_tree_string)).unwrap(); - - debug!("initializing blocks table with height {}", height); - match init_blocks_table( - &mut db_data, - (height as u32).try_into()?, - hash, - time, - &sapling_tree, - ) { - Ok(()) => Ok(JNI_TRUE), - Err(e) => Err(format_err!("Error while initializing blocks table: {}", e)), - } - }); - unwrap_exc_or(&env, res, JNI_FALSE) -} - #[no_mangle] pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_getCurrentAddress( env: JNIEnv<'_>, @@ -726,27 +657,20 @@ pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_ge let db_data = wallet_db(&env, network, db_data)?; let account = AccountId::from(u32::try_from(accountj)?); - // We query the unverified balance including unmined transactions. Shielded notes - // in unmined transactions are never spendable, but this ensures that the balance - // reported to users does not drop temporarily in a way that they don't expect. - // `getVerifiedBalance` requires `ANCHOR_OFFSET` confirmations, which means it - // always shows a spendable balance. - let min_confirmations = NonZeroU32::new(1).unwrap(); - - (&db_data) - .get_target_and_anchor_heights(min_confirmations) - .map_err(|e| format_err!("Error while fetching anchor height: {}", e)) - .and_then(|opt_anchor| { - opt_anchor - .map(|(_, a)| a + 1) - .ok_or(format_err!("Anchor height not available; scan required.")) - }) - .and_then(|anchor| { - (&db_data) - .get_balance_at(account, anchor) - .map_err(|e| format_err!("Error while fetching verified balance: {}", e)) - }) - .map(|amount| amount.into()) + if let Some(wallet_summary) = db_data + .get_wallet_summary(0) + .map_err(|e| format_err!("Error while fetching balance: {}", e))? + { + wallet_summary + .account_balances() + .get(&account) + .ok_or_else(|| format_err!("Unknown account")) + .map(|balances| Amount::from(balances.sapling_balance.total()).into()) + } else { + // `None` means that the caller has not yet called `updateChainTip` on a + // brand-new wallet, so we can assume the balance is zero. + Ok(0) + } }); unwrap_exc_or(&env, res, -1) } @@ -842,20 +766,20 @@ pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_ge let db_data = wallet_db(&env, network, db_data)?; let account = AccountId::from(u32::try_from(account)?); - (&db_data) - .get_target_and_anchor_heights(ANCHOR_OFFSET) - .map_err(|e| format_err!("Error while fetching anchor height: {}", e)) - .and_then(|opt_anchor| { - opt_anchor - .map(|(_, a)| a) - .ok_or(format_err!("Anchor height not available; scan required.")) - }) - .and_then(|anchor| { - (&db_data) - .get_balance_at(account, anchor) - .map_err(|e| format_err!("Error while fetching verified balance: {}", e)) - }) - .map(|amount| amount.into()) + if let Some(wallet_summary) = db_data + .get_wallet_summary(ANCHOR_OFFSET_U32) + .map_err(|e| format_err!("Error while fetching verified balance: {}", e))? + { + wallet_summary + .account_balances() + .get(&account) + .ok_or_else(|| format_err!("Unknown account")) + .map(|balances| Amount::from(balances.sapling_balance.spendable_value).into()) + } else { + // `None` means that the caller has not yet called `updateChainTip` on a + // brand-new wallet, so we can assume the balance is zero. + Ok(0) + } }); unwrap_exc_or(&env, res, -1) @@ -1175,6 +1099,41 @@ pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_up unwrap_exc_or(&env, res, JNI_FALSE) } +fn encode_scan_progress(env: &JNIEnv<'_>, progress: Ratio) -> Result { + let output = env.new_object( + "cash/z/ecc/android/sdk/internal/model/JniScanProgress", + "(JJ)V", + &[ + JValue::Long(*progress.numerator() as i64), + JValue::Long(*progress.denominator() as i64), + ], + )?; + Ok(output.into_raw()) +} + +#[no_mangle] +pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_getScanProgress( + env: JNIEnv<'_>, + _: JClass<'_>, + db_data: JString<'_>, + network_id: jint, +) -> jobject { + let res = panic::catch_unwind(|| { + let network = parse_network(network_id as u32)?; + let db_data = wallet_db(&env, network, db_data)?; + + match db_data + .get_wallet_summary(0) + .map_err(|e| format_err!("Error while fetching scan progress: {}", e))? + .and_then(|wallet_summary| wallet_summary.scan_progress()) + { + Some(progress) => encode_scan_progress(&env, progress), + None => Ok(ptr::null_mut()), + } + }); + unwrap_exc_or(&env, res, ptr::null_mut()) +} + fn encode_scan_range<'a>( env: &JNIEnv<'a>, scan_range: ScanRange, diff --git a/lightwallet-client-lib/src/main/java/co/electriccoin/lightwallet/client/model/TreeStateUnsafe.kt b/lightwallet-client-lib/src/main/java/co/electriccoin/lightwallet/client/model/TreeStateUnsafe.kt new file mode 100644 index 00000000..b258238c --- /dev/null +++ b/lightwallet-client-lib/src/main/java/co/electriccoin/lightwallet/client/model/TreeStateUnsafe.kt @@ -0,0 +1,28 @@ +package co.electriccoin.lightwallet.client.model + +import cash.z.wallet.sdk.internal.rpc.Service.TreeState + +class TreeStateUnsafe( + val encoded: ByteArray +) { + companion object { + fun new(treeState: TreeState): TreeStateUnsafe { + return TreeStateUnsafe(treeState.toByteArray()) + } + + fun fromParts( + height: Long, + hash: String, + time: Int, + tree: String + ): TreeStateUnsafe { + val treeState = TreeState.newBuilder() + .setHeight(height) + .setHash(hash) + .setTime(time) + .setSaplingTree(tree) + .build() + return TreeStateUnsafe.new(treeState) + } + } +} \ No newline at end of file diff --git a/sdk-lib/src/androidTest/java/cash/z/ecc/fixture/FakeRustBackend.kt b/sdk-lib/src/androidTest/java/cash/z/ecc/fixture/FakeRustBackend.kt index 02be697a..3333134c 100644 --- a/sdk-lib/src/androidTest/java/cash/z/ecc/fixture/FakeRustBackend.kt +++ b/sdk-lib/src/androidTest/java/cash/z/ecc/fixture/FakeRustBackend.kt @@ -30,6 +30,10 @@ internal class FakeRustBackend( TODO("Not yet implemented") } + override suspend fun getScanProgress(): JniScanProgress { + TODO("Not yet implemented") + } + override suspend fun suggestScanRanges(): List { TODO("Not yet implemented") } @@ -83,23 +87,14 @@ internal class FakeRustBackend( override suspend fun decryptAndStoreTransaction(tx: ByteArray) = error("Intentionally not implemented in mocked FakeRustBackend implementation.") - override suspend fun initAccountsTable(vararg keys: String) { - TODO("Not yet implemented") - } - - override suspend fun initBlocksTable( - checkpointHeight: Long, - checkpointHash: String, - checkpointTime: Long, - checkpointSaplingTree: String - ) { - TODO("Not yet implemented") - } - override suspend fun initDataDb(seed: ByteArray?): Int = error("Intentionally not implemented in mocked FakeRustBackend implementation.") - override suspend fun createAccount(seed: ByteArray): JniUnifiedSpendingKey = + override suspend fun createAccount( + seed: ByteArray, + treeState: ByteArray, + recoverUntil: Long? + ): JniUnifiedSpendingKey = error("Intentionally not implemented in mocked FakeRustBackend implementation.") override fun isValidShieldedAddr(addr: String): Boolean = diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt index 7c593504..5ec48443 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt @@ -515,8 +515,12 @@ class SdkSynchronizer private constructor( // // Not ready to be a public API; internal for testing only - internal suspend fun createAccount(seed: ByteArray): UnifiedSpendingKey = - backend.createAccountAndGetSpendingKey(seed) + internal suspend fun createAccount( + seed: ByteArray, + checkpoint: Checkpoint, + recoverUntil: BlockHeight? + ): UnifiedSpendingKey = + backend.createAccountAndGetSpendingKey(seed, checkpoint, recoverUntil) /** * Returns the current Unified Address for this account. @@ -660,7 +664,7 @@ internal object DefaultSynchronizerFactory { zcashNetwork: ZcashNetwork, checkpoint: Checkpoint, seed: ByteArray?, - viewingKeys: List + numberOfAccounts: Int ): DerivedDataRepository = DbDerivedDataRepository( DerivedDataDb.new( @@ -670,7 +674,7 @@ internal object DefaultSynchronizerFactory { zcashNetwork, checkpoint, seed, - viewingKeys + numberOfAccounts ) ) diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/Synchronizer.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/Synchronizer.kt index ed531c32..4ce70472 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/Synchronizer.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/Synchronizer.kt @@ -476,14 +476,6 @@ interface Synchronizer { DefaultSynchronizerFactory .defaultCompactBlockRepository(coordinator.fsBlockDbRoot(zcashNetwork, alias), backend) - val viewingKeys = seed?.let { - DerivationTool.getInstance().deriveUnifiedFullViewingKeys( - seed, - zcashNetwork, - Derivation.DEFAULT_NUMBER_OF_ACCOUNTS - ).toList() - } ?: emptyList() - val repository = DefaultSynchronizerFactory.defaultDerivedDataRepository( applicationContext, backend, @@ -491,7 +483,7 @@ interface Synchronizer { zcashNetwork, loadedCheckpoint, seed, - viewingKeys + Derivation.DEFAULT_NUMBER_OF_ACCOUNTS ) val service = DefaultSynchronizerFactory.defaultService(applicationContext, lightWalletEndpoint) diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackend.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackend.kt index 12250ecb..7bc9eee7 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackend.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackend.kt @@ -2,6 +2,7 @@ package cash.z.ecc.android.sdk.internal import cash.z.ecc.android.sdk.internal.model.Checkpoint import cash.z.ecc.android.sdk.internal.model.JniBlockMeta +import cash.z.ecc.android.sdk.internal.model.ScanProgress import cash.z.ecc.android.sdk.internal.model.ScanRange import cash.z.ecc.android.sdk.internal.model.SubtreeRoot import cash.z.ecc.android.sdk.model.Account @@ -18,16 +19,11 @@ internal interface TypesafeBackend { val network: ZcashNetwork - suspend fun initAccountsTable(vararg keys: UnifiedFullViewingKey) - - suspend fun initAccountsTable( + suspend fun createAccountAndGetSpendingKey( seed: ByteArray, - numberOfAccounts: Int - ): List - - suspend fun initBlocksTable(checkpoint: Checkpoint) - - suspend fun createAccountAndGetSpendingKey(seed: ByteArray): UnifiedSpendingKey + checkpoint: Checkpoint, + recoverUntil: BlockHeight? + ): UnifiedSpendingKey @Suppress("LongParameterList") suspend fun createToAddress( @@ -99,6 +95,12 @@ internal interface TypesafeBackend { @Throws(RuntimeException::class) suspend fun scanBlocks(fromHeight: BlockHeight, limit: Long) + /** + * @throws RuntimeException as a common indicator of the operation failure + */ + @Throws(RuntimeException::class) + suspend fun getScanProgress(): ScanProgress? + /** * @throws RuntimeException as a common indicator of the operation failure */ diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackendImpl.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackendImpl.kt index 19c30242..fb1989a3 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackendImpl.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackendImpl.kt @@ -3,6 +3,7 @@ package cash.z.ecc.android.sdk.internal import cash.z.ecc.android.sdk.internal.model.Checkpoint import cash.z.ecc.android.sdk.internal.model.JniBlockMeta import cash.z.ecc.android.sdk.internal.model.JniSubtreeRoot +import cash.z.ecc.android.sdk.internal.model.ScanProgress import cash.z.ecc.android.sdk.internal.model.ScanRange import cash.z.ecc.android.sdk.internal.model.SubtreeRoot import cash.z.ecc.android.sdk.model.Account @@ -22,32 +23,20 @@ internal class TypesafeBackendImpl(private val backend: Backend) : TypesafeBacke override val network: ZcashNetwork get() = ZcashNetwork.from(backend.networkId) - override suspend fun initAccountsTable(vararg keys: UnifiedFullViewingKey) { - val ufvks = Array(keys.size) { keys[it].encoding } - @Suppress("SpreadOperator") - backend.initAccountsTable(*ufvks) - } - - override suspend fun initAccountsTable( + override suspend fun createAccountAndGetSpendingKey( seed: ByteArray, - numberOfAccounts: Int - ): List { - return DerivationTool.getInstance().deriveUnifiedFullViewingKeys(seed, network, numberOfAccounts) - } - - override suspend fun initBlocksTable(checkpoint: Checkpoint) { - backend.initBlocksTable( - checkpoint.height.value, - checkpoint.hash, - checkpoint.epochSeconds, - checkpoint.tree + checkpoint: Checkpoint, + recoverUntil: BlockHeight? + ): UnifiedSpendingKey { + return UnifiedSpendingKey( + backend.createAccount( + seed, + checkpoint.treeState().encoded, + recoverUntil?.value + ) ) } - override suspend fun createAccountAndGetSpendingKey(seed: ByteArray): UnifiedSpendingKey { - return UnifiedSpendingKey(backend.createAccount(seed)) - } - @Suppress("LongParameterList") override suspend fun createToAddress( usk: UnifiedSpendingKey, @@ -177,6 +166,10 @@ internal class TypesafeBackendImpl(private val backend: Backend) : TypesafeBacke override suspend fun scanBlocks(fromHeight: BlockHeight, limit: Long) = backend.scanBlocks(fromHeight.value, limit) + override suspend fun getScanProgress(): ScanProgress? = backend.getScanProgress()?.let { jniScanProgress -> + ScanProgress.new(jniScanProgress) + } + override suspend fun suggestScanRanges(): List = backend.suggestScanRanges().map { jniScanRange -> ScanRange.new( jniScanRange, diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/DerivedDataDb.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/DerivedDataDb.kt index 06656faa..907512c5 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/DerivedDataDb.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/DerivedDataDb.kt @@ -47,27 +47,15 @@ internal class DerivedDataDb private constructor( zcashNetwork: ZcashNetwork, checkpoint: Checkpoint, seed: ByteArray?, - viewingKeys: List + numberOfAccounts: Int ): DerivedDataDb { backend.initDataDb(seed) - runCatching { - // TODO [#681]: consider converting these to typed exceptions in the welding layer - // TODO [#681]: https://github.com/zcash/zcash-android-wallet-sdk/issues/681 - tryWarn( - message = "Did not initialize the blocks table. It probably was already initialized.", - ifContains = "table is not empty" - ) { - backend.initBlocksTable(checkpoint) + // If a seed is provided, fill in the accounts. + seed?.let { + for (i in 1..numberOfAccounts) { + backend.createAccountAndGetSpendingKey(it, checkpoint, null) } - tryWarn( - message = "Did not initialize the accounts table. It probably was already initialized.", - ifContains = "table is not empty" - ) { - backend.initAccountsTable(*viewingKeys.toTypedArray()) - } - }.onFailure { - Twig.error { "Failed to init derived data database with $it" } } val database = ReadOnlySupportSqliteOpenHelper.openExistingDatabaseAsReadOnly( diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/Checkpoint.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/Checkpoint.kt index d42acfb3..a4b3b177 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/Checkpoint.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/Checkpoint.kt @@ -1,6 +1,8 @@ package cash.z.ecc.android.sdk.internal.model import cash.z.ecc.android.sdk.model.BlockHeight +import cash.z.wallet.sdk.internal.rpc.Service.TreeState +import co.electriccoin.lightwallet.client.model.TreeStateUnsafe /** * Represents a checkpoint, which is used to speed sync times. @@ -18,5 +20,11 @@ internal data class Checkpoint( // Note: this field does NOT match the name of the JSON, so will break with field-based JSON parsing val tree: String ) { + fun treeState(): TreeStateUnsafe { + // TODO: epochSeconds should be a Uint32, and for some reason the generated + // Protobuf type Service.TreeState uses Int for this. + return TreeStateUnsafe.fromParts(height.value, hash, epochSeconds.toInt(), tree) + } + internal companion object } diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanProgress.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanProgress.kt new file mode 100644 index 00000000..b56004c2 --- /dev/null +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanProgress.kt @@ -0,0 +1,17 @@ +package cash.z.ecc.android.sdk.internal.model + +internal data class ScanProgress( + val numerator: Long, + val denominator: Long +) { + override fun toString() = "ScanProgress($numerator / $denominator)" + + companion object { + fun new(jni: JniScanProgress): ScanProgress { + return ScanProgress( + numerator = jni.numerator, + denominator = jni.denominator + ) + } + } +} From 9e334fd375530af1442838e372d315d1e012e568 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Honza=20Rychnovsk=C3=BD?= Date: Wed, 6 Sep 2023 09:10:32 +0200 Subject: [PATCH 38/76] [#1109][#1206] Regular balance flows emission * [#1206] Optimize SYNC_BATCH_SIZE * [#1109] Regular balance flows emission --- CHANGELOG.md | 2 + .../cash/z/ecc/android/sdk/SdkSynchronizer.kt | 42 ++++++------- .../block/processor/CompactBlockProcessor.kt | 60 +++++++++++++++---- 3 files changed, 73 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9ca029b..9f5cd109 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,8 @@ ## Unreleased - `CompactBlockProcessor` now processes compact blocks from the lightwalletd server with spend-before-sync algorithm (i.e. non-linear order). This feature shortens the time after which a wallet's spendable balance can be used. +- The block synchronization mechanism is about one-third faster thanks to the optimized +`CompactBlockProcessor.SYNC_BATCH_SIZE`. Issue **#1206**. ### Added - New `syncAlgorithm` parameter of `Synchronizer.new()` and `WalletCoordinator()` to select preferred diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt index 5ec48443..b40257ec 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt @@ -176,18 +176,14 @@ class SdkSynchronizer private constructor( } } - // pools - private val _orchardBalances = MutableStateFlow(null) - private val _saplingBalances = MutableStateFlow(null) - private val _transparentBalances = MutableStateFlow(null) - private val _status = MutableStateFlow(DISCONNECTED) var coroutineScope: CoroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Main) - override val orchardBalances = _orchardBalances.asStateFlow() - override val saplingBalances = _saplingBalances.asStateFlow() - override val transparentBalances = _transparentBalances.asStateFlow() + override val orchardBalances = processor.orchardBalances.asStateFlow() + override val saplingBalances = processor.saplingBalances.asStateFlow() + override val transparentBalances = processor.transparentBalances.asStateFlow() + override val transactions get() = combine(processor.networkHeight, storage.allTransactions) { networkHeight, allTransactions -> val latestBlockHeight = networkHeight ?: storage.lastScannedHeight() @@ -362,36 +358,42 @@ class SdkSynchronizer private constructor( storage.invalidate() } - // - // Private API - // - /** - * Calculate the latest balance, based on the blocks that have been scanned and transmit this - * information into the flow of [balances]. + * Calculate the latest balance based on the blocks that have been scanned and transmit this information into the + * [transparentBalances] and [saplingBalances] flow. The [orchardBalances] flow is still not filled with proper data + * because of the current limited Orchard support. */ suspend fun refreshAllBalances() { - refreshSaplingBalance() - refreshTransparentBalance() + processor.checkAllBalances() // TODO [#682]: refresh orchard balance // TODO [#682]: https://github.com/zcash/zcash-android-wallet-sdk/issues/682 Twig.warn { "Warning: Orchard balance does not yet refresh. Only some of the plumbing is in place." } } + /** + * Calculate the latest Sapling balance based on the blocks that have been scanned and transmit this information + * into the [saplingBalances] flow. + */ suspend fun refreshSaplingBalance() { - Twig.debug { "refreshing sapling balance" } - _saplingBalances.value = processor.getBalanceInfo(Account.DEFAULT) + processor.checkSaplingBalance() } + /** + * Calculate the latest Transparent balance based on the blocks that have been scanned and transmit this information + * into the [saplingBalances] flow. + */ suspend fun refreshTransparentBalance() { - Twig.debug { "refreshing transparent balance" } - _transparentBalances.value = processor.getUtxoCacheBalance(getTransparentAddress(Account.DEFAULT)) + processor.checkTransparentBalance() } suspend fun isValidAddress(address: String): Boolean { return !validateAddress(address).isNotValid } + // + // Private API + // + private fun CoroutineScope.onReady() { Twig.debug { "Starting synchronizer…" } diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessor.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessor.kt index 065647a0..02fef611 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessor.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessor.kt @@ -149,6 +149,12 @@ class CompactBlockProcessor internal constructor( private val _progress = MutableStateFlow(PercentDecimal.ZERO_PERCENT) private val _processorInfo = MutableStateFlow(ProcessorInfo(null, null, null)) private val _networkHeight = MutableStateFlow(null) + + // pools + internal val saplingBalances = MutableStateFlow(null) + internal val orchardBalances = MutableStateFlow(null) + internal val transparentBalances = MutableStateFlow(null) + private val processingMutex = Mutex() /** @@ -470,6 +476,7 @@ class CompactBlockProcessor internal constructor( lastBatchOrder = min(rangeSyncProgress.overallOrder, allBatchCountLocal) setProgress(PercentDecimal(lastBatchOrder / allBatchCountLocal.toFloat())) + checkAllBalances() when (rangeSyncProgress.resultState) { SyncingResult.UpdateBirthday -> { @@ -478,7 +485,7 @@ class CompactBlockProcessor internal constructor( SyncingResult.EnhanceSuccess -> { Twig.info { "Triggering transaction refresh now" } // Invalidate transaction data - refreshTransactions(transactionStorage = repository) + checkTransactions(transactionStorage = repository) } is SyncingResult.Failure -> { syncingResult = rangeSyncProgress.resultState @@ -552,6 +559,7 @@ class CompactBlockProcessor internal constructor( lastBatchOrder = min(rangeSyncProgress.overallOrder, allBatchCountLocal) setProgress(PercentDecimal(lastBatchOrder / allBatchCountLocal.toFloat())) + checkAllBalances() when (rangeSyncProgress.resultState) { SyncingResult.UpdateBirthday -> { @@ -561,7 +569,7 @@ class CompactBlockProcessor internal constructor( SyncingResult.EnhanceSuccess -> { Twig.info { "Triggering transaction refresh now" } // Invalidate transaction data and return the common batch syncing success result to the caller - refreshTransactions(transactionStorage = repository) + checkTransactions(transactionStorage = repository) SyncingResult.AllSuccess } is SyncingResult.Failure -> { @@ -609,13 +617,6 @@ class CompactBlockProcessor internal constructor( return BlockProcessingResult.Success } - /** - * This invalidates transaction storage to trigger data refreshing for its subscribers. - */ - private fun refreshTransactions(transactionStorage: DerivedDataRepository) { - transactionStorage.invalidate() - } - @Suppress("ReturnCount") internal suspend fun runSbSSyncingPreparation( backend: TypesafeBackend, @@ -714,6 +715,7 @@ class CompactBlockProcessor internal constructor( lastBatchOrder = 0 ).collect { rangeSyncProgress -> setProgress(PercentDecimal(rangeSyncProgress.overallOrder / allBatchCount.toFloat())) + checkAllBalances() when (rangeSyncProgress.resultState) { SyncingResult.UpdateBirthday -> { @@ -722,7 +724,7 @@ class CompactBlockProcessor internal constructor( SyncingResult.EnhanceSuccess -> { Twig.info { "Triggering transaction refresh now" } // Invalidate transaction data - refreshTransactions(transactionStorage = repository) + checkTransactions(transactionStorage = repository) } is SyncingResult.Failure -> { syncingResult = rangeSyncProgress.resultState @@ -747,6 +749,42 @@ class CompactBlockProcessor internal constructor( return BlockProcessingResult.Success } + /** + * This invalidates transaction storage to trigger data refreshing for its subscribers. + */ + private fun checkTransactions(transactionStorage: DerivedDataRepository) { + transactionStorage.invalidate() + } + + /** + * Calculate the latest balances, based on the blocks that have been scanned and transmit this + * information into the related internal flows. Note that the Orchard balance is not supported. + */ + internal suspend fun checkAllBalances() { + checkSaplingBalance() + checkTransparentBalance() + // TODO [#682]: refresh orchard balance + // TODO [#682]: https://github.com/zcash/zcash-android-wallet-sdk/issues/682 + } + + /** + * Calculate the latest Sapling balance, based on the blocks that have been scanned and transmit this + * information into the internal [saplingBalances] flow. + */ + internal suspend fun checkSaplingBalance() { + Twig.debug { "Checking Sapling balance" } + saplingBalances.value = getBalanceInfo(Account.DEFAULT) + } + + /** + * Calculate the latest Transparent balance, based on the blocks that have been scanned and transmit this + * information into the internal [transparentBalances] flow. + */ + internal suspend fun checkTransparentBalance() { + Twig.debug { "Checking Transparent balance" } + transparentBalances.value = getUtxoCacheBalance(getTransparentAddress(backend, Account.DEFAULT)) + } + sealed class BlockProcessingResult { object NoBlocksToProcess : BlockProcessingResult() object Success : BlockProcessingResult() @@ -1047,7 +1085,7 @@ class CompactBlockProcessor internal constructor( * number the more granular information can be provided about scan state. Unfortunately, it may also lead to * a lot of overhead during scanning. */ - internal const val SYNC_BATCH_SIZE = 10 + internal const val SYNC_BATCH_SIZE = 100 /** * Default size of batch of blocks for running the transaction enhancing. From 7b7275b5bde8a019d900d9e0d376ff0cae1b45e8 Mon Sep 17 00:00:00 2001 From: Honza Date: Wed, 6 Sep 2023 15:24:27 +0200 Subject: [PATCH 39/76] Migrate to latest Rust revision follow-up changes --- .../z/ecc/android/sdk/internal/Backend.kt | 13 +++++++ .../z/ecc/android/sdk/internal/Derivation.kt | 1 + .../android/sdk/internal/jni/RustBackend.kt | 4 +-- .../sdk/internal/model/JniScanProgress.kt | 13 ++++++- .../client/model/TreeStateUnsafe.kt | 4 +-- .../cash/z/ecc/fixture/FakeRustBackend.kt | 1 + .../cash/z/ecc/android/sdk/SdkSynchronizer.kt | 16 ++++++--- .../cash/z/ecc/android/sdk/Synchronizer.kt | 4 +-- .../block/processor/CompactBlockProcessor.kt | 16 +++++---- .../android/sdk/internal/TypesafeBackend.kt | 1 - .../sdk/internal/TypesafeBackendImpl.kt | 2 -- .../sdk/internal/db/derived/DerivedDataDb.kt | 35 ++++++++++++------- .../android/sdk/internal/model/Checkpoint.kt | 7 ++-- .../android/sdk/internal/model/ScanRange.kt | 2 +- 14 files changed, 81 insertions(+), 38 deletions(-) diff --git a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/Backend.kt b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/Backend.kt index b9410d35..308f04d9 100644 --- a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/Backend.kt +++ b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/Backend.kt @@ -38,6 +38,10 @@ interface Backend { suspend fun initDataDb(seed: ByteArray?): Int + /** + * @throws RuntimeException as a common indicator of the operation failure + */ + @Throws(RuntimeException::class) suspend fun createAccount(seed: ByteArray, treeState: ByteArray, recoverUntil: Long?): JniUnifiedSpendingKey fun isValidShieldedAddr(addr: String): Boolean @@ -54,6 +58,10 @@ interface Backend { suspend fun listTransparentReceivers(account: Int): List + /** + * @throws RuntimeException as a common indicator of the operation failure + */ + @Throws(RuntimeException::class) suspend fun getBalance(account: Int): Long fun getBranchIdForHeight(height: Long): Long @@ -61,8 +69,13 @@ interface Backend { /** * @throws RuntimeException as a common indicator of the operation failure */ + @Throws(RuntimeException::class) suspend fun getMemoAsUtf8(txId: ByteArray, outputIndex: Int): String? + /** + * @throws RuntimeException as a common indicator of the operation failure + */ + @Throws(RuntimeException::class) suspend fun getVerifiedBalance(account: Int): Long suspend fun getNearestRewindHeight(height: Long): Long diff --git a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/Derivation.kt b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/Derivation.kt index 3f61d159..bd9ce2da 100644 --- a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/Derivation.kt +++ b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/Derivation.kt @@ -40,5 +40,6 @@ interface Derivation { companion object { const val DEFAULT_NUMBER_OF_ACCOUNTS = 1 + val DEFAULT_RECOVERY_UNTIL_HEIGHT = null } } diff --git a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/jni/RustBackend.kt b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/jni/RustBackend.kt index 858a09cc..0395dfc0 100644 --- a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/jni/RustBackend.kt +++ b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/jni/RustBackend.kt @@ -239,7 +239,7 @@ class RustBackend private constructor( ) } - override suspend fun getScanProgress(): JniScanProgress = + override suspend fun getScanProgress(): JniScanProgress? = withContext(SdkDispatchers.DATABASE_IO) { getScanProgress( dataDbFile.absolutePath, @@ -506,7 +506,7 @@ class RustBackend private constructor( private external fun getScanProgress( dbDataPath: String, networkId: Int - ): JniScanProgress + ): JniScanProgress? @JvmStatic private external fun suggestScanRanges( diff --git a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniScanProgress.kt b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniScanProgress.kt index 514ea216..305ccaea 100644 --- a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniScanProgress.kt +++ b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniScanProgress.kt @@ -6,11 +6,22 @@ import androidx.annotation.Keep * Serves as cross layer (Kotlin, Rust) communication class. * * @param numerator the numerator of the progress ratio - * @param endHeight the denominator of the progress ratio + * @param denominator the denominator of the progress ratio */ @Keep class JniScanProgress( val numerator: Long, val denominator: Long ) { + init { + require(numerator >= 0L) { + "Numerator $numerator is outside of allowed range [0, Long.MAX_VALUE]" + } + require(denominator >= 1L) { + "Denominator $denominator is outside of allowed range [1, Long.MAX_VALUE]" + } + require(numerator.toFloat().div(denominator) >= 0f) { + "Result of ${numerator.toFloat()}/$denominator is outside of allowed range" + } + } } diff --git a/lightwallet-client-lib/src/main/java/co/electriccoin/lightwallet/client/model/TreeStateUnsafe.kt b/lightwallet-client-lib/src/main/java/co/electriccoin/lightwallet/client/model/TreeStateUnsafe.kt index b258238c..3f1740a1 100644 --- a/lightwallet-client-lib/src/main/java/co/electriccoin/lightwallet/client/model/TreeStateUnsafe.kt +++ b/lightwallet-client-lib/src/main/java/co/electriccoin/lightwallet/client/model/TreeStateUnsafe.kt @@ -22,7 +22,7 @@ class TreeStateUnsafe( .setTime(time) .setSaplingTree(tree) .build() - return TreeStateUnsafe.new(treeState) + return new(treeState) } } -} \ No newline at end of file +} diff --git a/sdk-lib/src/androidTest/java/cash/z/ecc/fixture/FakeRustBackend.kt b/sdk-lib/src/androidTest/java/cash/z/ecc/fixture/FakeRustBackend.kt index 3333134c..829116fb 100644 --- a/sdk-lib/src/androidTest/java/cash/z/ecc/fixture/FakeRustBackend.kt +++ b/sdk-lib/src/androidTest/java/cash/z/ecc/fixture/FakeRustBackend.kt @@ -2,6 +2,7 @@ package cash.z.ecc.fixture import cash.z.ecc.android.sdk.internal.Backend import cash.z.ecc.android.sdk.internal.model.JniBlockMeta +import cash.z.ecc.android.sdk.internal.model.JniScanProgress import cash.z.ecc.android.sdk.internal.model.JniScanRange import cash.z.ecc.android.sdk.internal.model.JniSubtreeRoot import cash.z.ecc.android.sdk.internal.model.JniUnifiedSpendingKey diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt index b40257ec..4d2e32ba 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt @@ -40,7 +40,6 @@ import cash.z.ecc.android.sdk.model.BlockHeight import cash.z.ecc.android.sdk.model.PercentDecimal import cash.z.ecc.android.sdk.model.TransactionOverview import cash.z.ecc.android.sdk.model.TransactionRecipient -import cash.z.ecc.android.sdk.model.UnifiedFullViewingKey import cash.z.ecc.android.sdk.model.UnifiedSpendingKey import cash.z.ecc.android.sdk.model.WalletBalance import cash.z.ecc.android.sdk.model.Zatoshi @@ -521,8 +520,13 @@ class SdkSynchronizer private constructor( seed: ByteArray, checkpoint: Checkpoint, recoverUntil: BlockHeight? - ): UnifiedSpendingKey = - backend.createAccountAndGetSpendingKey(seed, checkpoint, recoverUntil) + ): UnifiedSpendingKey? { + return runCatching { + backend.createAccountAndGetSpendingKey(seed, checkpoint, recoverUntil) + }.onFailure { + Twig.error(it) { "Create account failed." } + }.getOrNull() + } /** * Returns the current Unified Address for this account. @@ -666,7 +670,8 @@ internal object DefaultSynchronizerFactory { zcashNetwork: ZcashNetwork, checkpoint: Checkpoint, seed: ByteArray?, - numberOfAccounts: Int + numberOfAccounts: Int, + recoverUntil: BlockHeight? ): DerivedDataRepository = DbDerivedDataRepository( DerivedDataDb.new( @@ -676,7 +681,8 @@ internal object DefaultSynchronizerFactory { zcashNetwork, checkpoint, seed, - numberOfAccounts + numberOfAccounts, + recoverUntil ) ) diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/Synchronizer.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/Synchronizer.kt index 4ce70472..d37e9067 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/Synchronizer.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/Synchronizer.kt @@ -16,7 +16,6 @@ import cash.z.ecc.android.sdk.model.WalletBalance import cash.z.ecc.android.sdk.model.Zatoshi import cash.z.ecc.android.sdk.model.ZcashNetwork import cash.z.ecc.android.sdk.tool.CheckpointTool -import cash.z.ecc.android.sdk.tool.DerivationTool import cash.z.ecc.android.sdk.type.AddressType import cash.z.ecc.android.sdk.type.ConsensusMatchType import co.electriccoin.lightwallet.client.model.LightWalletEndpoint @@ -483,7 +482,8 @@ interface Synchronizer { zcashNetwork, loadedCheckpoint, seed, - Derivation.DEFAULT_NUMBER_OF_ACCOUNTS + Derivation.DEFAULT_NUMBER_OF_ACCOUNTS, + Derivation.DEFAULT_RECOVERY_UNTIL_HEIGHT, ) val service = DefaultSynchronizerFactory.defaultService(applicationContext, lightWalletEndpoint) diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessor.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessor.kt index 02fef611..185cc36c 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessor.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessor.kt @@ -2058,18 +2058,20 @@ class CompactBlockProcessor internal constructor( * @param account the account to check for balance info. * * @return an instance of WalletBalance containing information about available and total funds. + * + * @throws RustLayerException.BalanceException if any error occurs while getting the balances via the Rust layer */ suspend fun getBalanceInfo(account: Account): WalletBalance { - @Suppress("TooGenericExceptionCaught") - return try { + return runCatching { val balanceTotal = backend.getBalance(account) - Twig.debug { "found total balance: $balanceTotal" } + Twig.info { "Found total balance: $balanceTotal" } val balanceAvailable = backend.getVerifiedBalance(account) - Twig.debug { "found available balance: $balanceAvailable" } + Twig.info { "Found available balance: $balanceAvailable" } WalletBalance(balanceTotal, balanceAvailable) - } catch (t: Throwable) { - Twig.debug { "failed to get balance due to $t" } - throw RustLayerException.BalanceException(t) + }.onFailure { + Twig.error(it) { "Failed to get balance due to ${it.localizedMessage}" } + }.getOrElse { + throw RustLayerException.BalanceException(it) } } diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackend.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackend.kt index 7bc9eee7..289f55c2 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackend.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackend.kt @@ -8,7 +8,6 @@ import cash.z.ecc.android.sdk.internal.model.SubtreeRoot import cash.z.ecc.android.sdk.model.Account import cash.z.ecc.android.sdk.model.BlockHeight import cash.z.ecc.android.sdk.model.FirstClassByteArray -import cash.z.ecc.android.sdk.model.UnifiedFullViewingKey import cash.z.ecc.android.sdk.model.UnifiedSpendingKey import cash.z.ecc.android.sdk.model.WalletBalance import cash.z.ecc.android.sdk.model.Zatoshi diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackendImpl.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackendImpl.kt index fb1989a3..2d5d248a 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackendImpl.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackendImpl.kt @@ -9,12 +9,10 @@ import cash.z.ecc.android.sdk.internal.model.SubtreeRoot import cash.z.ecc.android.sdk.model.Account import cash.z.ecc.android.sdk.model.BlockHeight import cash.z.ecc.android.sdk.model.FirstClassByteArray -import cash.z.ecc.android.sdk.model.UnifiedFullViewingKey import cash.z.ecc.android.sdk.model.UnifiedSpendingKey import cash.z.ecc.android.sdk.model.WalletBalance import cash.z.ecc.android.sdk.model.Zatoshi import cash.z.ecc.android.sdk.model.ZcashNetwork -import cash.z.ecc.android.sdk.tool.DerivationTool import kotlinx.coroutines.withContext @Suppress("TooManyFunctions") diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/DerivedDataDb.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/DerivedDataDb.kt index 907512c5..8f16167c 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/DerivedDataDb.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/DerivedDataDb.kt @@ -6,9 +6,8 @@ import cash.z.ecc.android.sdk.internal.NoBackupContextWrapper import cash.z.ecc.android.sdk.internal.Twig import cash.z.ecc.android.sdk.internal.TypesafeBackend import cash.z.ecc.android.sdk.internal.db.ReadOnlySupportSqliteOpenHelper -import cash.z.ecc.android.sdk.internal.ext.tryWarn import cash.z.ecc.android.sdk.internal.model.Checkpoint -import cash.z.ecc.android.sdk.model.UnifiedFullViewingKey +import cash.z.ecc.android.sdk.model.BlockHeight import cash.z.ecc.android.sdk.model.ZcashNetwork import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext @@ -39,7 +38,7 @@ internal class DerivedDataDb private constructor( // SqliteOpenHelper is happy private const val DATABASE_VERSION = 8 - @Suppress("LongParameterList", "SpreadOperator") + @Suppress("LongParameterList") suspend fun new( context: Context, backend: TypesafeBackend, @@ -47,17 +46,11 @@ internal class DerivedDataDb private constructor( zcashNetwork: ZcashNetwork, checkpoint: Checkpoint, seed: ByteArray?, - numberOfAccounts: Int + numberOfAccounts: Int, + recoverUntil: BlockHeight? ): DerivedDataDb { backend.initDataDb(seed) - // If a seed is provided, fill in the accounts. - seed?.let { - for (i in 1..numberOfAccounts) { - backend.createAccountAndGetSpendingKey(it, checkpoint, null) - } - } - val database = ReadOnlySupportSqliteOpenHelper.openExistingDatabaseAsReadOnly( NoBackupContextWrapper( context, @@ -67,7 +60,25 @@ internal class DerivedDataDb private constructor( DATABASE_VERSION ) - return DerivedDataDb(zcashNetwork, database) + val dataDb = DerivedDataDb(zcashNetwork, database) + + // If a seed is provided, fill in the accounts. + seed?.let { checkedSeed -> + // toInt() should be safe because we expect very few accounts + val missingAccounts = numberOfAccounts - dataDb.accountTable.count().toInt() + require(missingAccounts >= 0) { + "Unexpected number of accounts: $missingAccounts" + } + repeat(missingAccounts) { + runCatching { + backend.createAccountAndGetSpendingKey(checkedSeed, checkpoint, recoverUntil) + }.onFailure { + Twig.error(it) { "Create account failed." } + } + } + } + + return dataDb } } } diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/Checkpoint.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/Checkpoint.kt index a4b3b177..6173c6c1 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/Checkpoint.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/Checkpoint.kt @@ -1,7 +1,7 @@ package cash.z.ecc.android.sdk.internal.model +import cash.z.ecc.android.sdk.internal.ext.isInUIntRange import cash.z.ecc.android.sdk.model.BlockHeight -import cash.z.wallet.sdk.internal.rpc.Service.TreeState import co.electriccoin.lightwallet.client.model.TreeStateUnsafe /** @@ -21,8 +21,9 @@ internal data class Checkpoint( val tree: String ) { fun treeState(): TreeStateUnsafe { - // TODO: epochSeconds should be a Uint32, and for some reason the generated - // Protobuf type Service.TreeState uses Int for this. + require(epochSeconds.isInUIntRange()) { + "epochSeconds $epochSeconds is outside of allowed UInt range" + } return TreeStateUnsafe.fromParts(height.value, hash, epochSeconds.toInt(), tree) } diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanRange.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanRange.kt index 10693c61..adfc5b16 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanRange.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanRange.kt @@ -10,7 +10,7 @@ internal data class ScanRange( override fun toString() = "ScanRange(range=$range, priority=${getSuggestScanRangePriority()})" internal fun getSuggestScanRangePriority(): SuggestScanRangePriority { - return SuggestScanRangePriority.values() + return SuggestScanRangePriority.entries .firstOrNull { it.priority == priority } ?: SuggestScanRangePriority.Ignored } From 42cf9d6466c1f2e214d1498c7ccaa7629d16c684 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Wed, 6 Sep 2023 20:28:45 +0100 Subject: [PATCH 40/76] Migrate to Rust revision that exposes scanned heights --- backend-lib/Cargo.lock | 16 +++--- backend-lib/Cargo.toml | 10 ++-- .../z/ecc/android/sdk/internal/Backend.kt | 25 +++++++++- .../android/sdk/internal/jni/RustBackend.kt | 46 +++++++++++++++-- backend-lib/src/main/rust/lib.rs | 50 ++++++++++++++++++- .../cash/z/ecc/fixture/FakeRustBackend.kt | 10 +++- .../android/sdk/internal/TypesafeBackend.kt | 25 +++++++++- .../sdk/internal/TypesafeBackendImpl.kt | 22 +++++++- .../block/FileCompactBlockRepository.kt | 2 +- 9 files changed, 183 insertions(+), 23 deletions(-) diff --git a/backend-lib/Cargo.lock b/backend-lib/Cargo.lock index b8e5a6ee..5b476257 100644 --- a/backend-lib/Cargo.lock +++ b/backend-lib/Cargo.lock @@ -483,7 +483,7 @@ checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" [[package]] name = "equihash" version = "0.2.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=b4772e948d1b5153a8bf13d29ae10259eb930f53#b4772e948d1b5153a8bf13d29ae10259eb930f53" +source = "git+https://github.com/zcash/librustzcash.git?rev=3d0ec004a275d062487f6fabc4061ee0b4687ff0#3d0ec004a275d062487f6fabc4061ee0b4687ff0" dependencies = [ "blake2b_simd", "byteorder", @@ -513,7 +513,7 @@ dependencies = [ [[package]] name = "f4jumble" version = "0.1.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=b4772e948d1b5153a8bf13d29ae10259eb930f53#b4772e948d1b5153a8bf13d29ae10259eb930f53" +source = "git+https://github.com/zcash/librustzcash.git?rev=3d0ec004a275d062487f6fabc4061ee0b4687ff0#3d0ec004a275d062487f6fabc4061ee0b4687ff0" dependencies = [ "blake2b_simd", ] @@ -2259,7 +2259,7 @@ dependencies = [ [[package]] name = "zcash_address" version = "0.3.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=b4772e948d1b5153a8bf13d29ae10259eb930f53#b4772e948d1b5153a8bf13d29ae10259eb930f53" +source = "git+https://github.com/zcash/librustzcash.git?rev=3d0ec004a275d062487f6fabc4061ee0b4687ff0#3d0ec004a275d062487f6fabc4061ee0b4687ff0" dependencies = [ "bech32", "bs58", @@ -2270,7 +2270,7 @@ dependencies = [ [[package]] name = "zcash_client_backend" version = "0.9.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=b4772e948d1b5153a8bf13d29ae10259eb930f53#b4772e948d1b5153a8bf13d29ae10259eb930f53" +source = "git+https://github.com/zcash/librustzcash.git?rev=3d0ec004a275d062487f6fabc4061ee0b4687ff0#3d0ec004a275d062487f6fabc4061ee0b4687ff0" dependencies = [ "base64", "bech32", @@ -2304,7 +2304,7 @@ dependencies = [ [[package]] name = "zcash_client_sqlite" version = "0.7.1" -source = "git+https://github.com/zcash/librustzcash.git?rev=b4772e948d1b5153a8bf13d29ae10259eb930f53#b4772e948d1b5153a8bf13d29ae10259eb930f53" +source = "git+https://github.com/zcash/librustzcash.git?rev=3d0ec004a275d062487f6fabc4061ee0b4687ff0#3d0ec004a275d062487f6fabc4061ee0b4687ff0" dependencies = [ "bs58", "byteorder", @@ -2330,7 +2330,7 @@ dependencies = [ [[package]] name = "zcash_encoding" version = "0.2.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=b4772e948d1b5153a8bf13d29ae10259eb930f53#b4772e948d1b5153a8bf13d29ae10259eb930f53" +source = "git+https://github.com/zcash/librustzcash.git?rev=3d0ec004a275d062487f6fabc4061ee0b4687ff0#3d0ec004a275d062487f6fabc4061ee0b4687ff0" dependencies = [ "byteorder", "nonempty", @@ -2352,7 +2352,7 @@ dependencies = [ [[package]] name = "zcash_primitives" version = "0.12.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=b4772e948d1b5153a8bf13d29ae10259eb930f53#b4772e948d1b5153a8bf13d29ae10259eb930f53" +source = "git+https://github.com/zcash/librustzcash.git?rev=3d0ec004a275d062487f6fabc4061ee0b4687ff0#3d0ec004a275d062487f6fabc4061ee0b4687ff0" dependencies = [ "aes", "bip0039", @@ -2387,7 +2387,7 @@ dependencies = [ [[package]] name = "zcash_proofs" version = "0.12.1" -source = "git+https://github.com/zcash/librustzcash.git?rev=b4772e948d1b5153a8bf13d29ae10259eb930f53#b4772e948d1b5153a8bf13d29ae10259eb930f53" +source = "git+https://github.com/zcash/librustzcash.git?rev=3d0ec004a275d062487f6fabc4061ee0b4687ff0#3d0ec004a275d062487f6fabc4061ee0b4687ff0" dependencies = [ "bellman", "blake2b_simd", diff --git a/backend-lib/Cargo.toml b/backend-lib/Cargo.toml index bf44266f..186b7013 100644 --- a/backend-lib/Cargo.toml +++ b/backend-lib/Cargo.toml @@ -45,11 +45,11 @@ libc = "0.2" incrementalmerkletree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "da97e6c399c5acddedad2c1730dbb7ee55499a2f" } shardtree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "da97e6c399c5acddedad2c1730dbb7ee55499a2f" } orchard = { git = "https://github.com/zcash/orchard.git", rev = "6ef89d5f154de2cf7b7dd87edb8d8c49158beebb" } -zcash_address = { git = "https://github.com/zcash/librustzcash.git", rev = "b4772e948d1b5153a8bf13d29ae10259eb930f53" } -zcash_client_backend = { git = "https://github.com/zcash/librustzcash.git", rev = "b4772e948d1b5153a8bf13d29ae10259eb930f53" } -zcash_client_sqlite = { git = "https://github.com/zcash/librustzcash.git", rev = "b4772e948d1b5153a8bf13d29ae10259eb930f53" } -zcash_primitives = { git = "https://github.com/zcash/librustzcash.git", rev = "b4772e948d1b5153a8bf13d29ae10259eb930f53" } -zcash_proofs = { git = "https://github.com/zcash/librustzcash.git", rev = "b4772e948d1b5153a8bf13d29ae10259eb930f53" } +zcash_address = { git = "https://github.com/zcash/librustzcash.git", rev = "3d0ec004a275d062487f6fabc4061ee0b4687ff0" } +zcash_client_backend = { git = "https://github.com/zcash/librustzcash.git", rev = "3d0ec004a275d062487f6fabc4061ee0b4687ff0" } +zcash_client_sqlite = { git = "https://github.com/zcash/librustzcash.git", rev = "3d0ec004a275d062487f6fabc4061ee0b4687ff0" } +zcash_primitives = { git = "https://github.com/zcash/librustzcash.git", rev = "3d0ec004a275d062487f6fabc4061ee0b4687ff0" } +zcash_proofs = { git = "https://github.com/zcash/librustzcash.git", rev = "3d0ec004a275d062487f6fabc4061ee0b4687ff0" } ## Uncomment this to test librustzcash changes locally #[patch.crates-io] diff --git a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/Backend.kt b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/Backend.kt index 308f04d9..ed61af6b 100644 --- a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/Backend.kt +++ b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/Backend.kt @@ -101,6 +101,29 @@ interface Backend { @Throws(RuntimeException::class) suspend fun updateChainTip(height: Long) + /** + * Returns the height to which the wallet has been fully scanned. + * + * This is the height for which the wallet has fully trial-decrypted this and all + * preceding blocks above the wallet's birthday height. + * + * @return The height to which the wallet has been fully scanned, or Null if no blocks have been scanned. + * @throws RuntimeException as a common indicator of the operation failure + */ + suspend fun getFullyScannedHeight(): Long? + + /** + * Returns the maximum height that the wallet has scanned. + * + * If the wallet is fully synced, this will be equivalent to `getFullyScannedHeight`; + * otherwise the maximal scanned height is likely to be greater than the fully scanned + * height due to the fact that out-of-order scanning can leave gaps. + * + * @return The maximum height that the wallet has scanned, or Null if no blocks have been scanned. + * @throws RuntimeException as a common indicator of the operation failure + */ + suspend fun getMaxScannedHeight(): Long? + /** * @throws RuntimeException as a common indicator of the operation failure */ @@ -128,7 +151,7 @@ interface Backend { /** * @return The latest height in the CompactBlock cache metadata DB, or Null if no blocks have been cached. */ - suspend fun getLatestHeight(): Long? + suspend fun getLatestCacheHeight(): Long? suspend fun findBlockMetadata(height: Long): JniBlockMeta? diff --git a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/jni/RustBackend.kt b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/jni/RustBackend.kt index 0395dfc0..b1307c3f 100644 --- a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/jni/RustBackend.kt +++ b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/jni/RustBackend.kt @@ -150,9 +150,9 @@ class RustBackend private constructor( ) } - override suspend fun getLatestHeight() = + override suspend fun getLatestCacheHeight() = withContext(SdkDispatchers.DATABASE_IO) { - val height = getLatestHeight(fsBlockDbRoot.absolutePath) + val height = getLatestCacheHeight(fsBlockDbRoot.absolutePath) if (-1L == height) { null @@ -239,6 +239,34 @@ class RustBackend private constructor( ) } + override suspend fun getFullyScannedHeight() = + withContext(SdkDispatchers.DATABASE_IO) { + val height = getFullyScannedHeight( + dataDbFile.absolutePath, + networkId = networkId + ) + + if (-1L == height) { + null + } else { + height + } + } + + override suspend fun getMaxScannedHeight() = + withContext(SdkDispatchers.DATABASE_IO) { + val height = getMaxScannedHeight( + dataDbFile.absolutePath, + networkId = networkId + ) + + if (-1L == height) { + null + } else { + height + } + } + override suspend fun getScanProgress(): JniScanProgress? = withContext(SdkDispatchers.DATABASE_IO) { getScanProgress( @@ -459,7 +487,7 @@ class RustBackend private constructor( ) @JvmStatic - private external fun getLatestHeight(dbCachePath: String): Long + private external fun getLatestCacheHeight(dbCachePath: String): Long @JvmStatic private external fun findBlockMetadata( @@ -502,6 +530,18 @@ class RustBackend private constructor( networkId: Int ) + @JvmStatic + private external fun getFullyScannedHeight( + dbDataPath: String, + networkId: Int + ): Long + + @JvmStatic + private external fun getMaxScannedHeight( + dbDataPath: String, + networkId: Int + ): Long + @JvmStatic private external fun getScanProgress( dbDataPath: String, diff --git a/backend-lib/src/main/rust/lib.rs b/backend-lib/src/main/rust/lib.rs index 1e2027f6..61ed06d7 100644 --- a/backend-lib/src/main/rust/lib.rs +++ b/backend-lib/src/main/rust/lib.rs @@ -891,7 +891,7 @@ pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_wr } #[no_mangle] -pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_getLatestHeight( +pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_getLatestCacheHeight( env: JNIEnv<'_>, _: JClass<'_>, fsblockdb_root: JString<'_>, @@ -1099,6 +1099,54 @@ pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_up unwrap_exc_or(&env, res, JNI_FALSE) } +#[no_mangle] +pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_getFullyScannedHeight( + env: JNIEnv<'_>, + _: JClass<'_>, + db_data: JString<'_>, + network_id: jint, +) -> jlong { + let res = panic::catch_unwind(|| { + let network = parse_network(network_id as u32)?; + let db_data = wallet_db(&env, network, db_data)?; + + match db_data.block_fully_scanned() { + Ok(Some(metadata)) => Ok(i64::from(u32::from(metadata.block_height()))), + // Use -1 to return null across the FFI. + Ok(None) => Ok(-1), + Err(e) => Err(format_err!( + "Failed to read block metadata from WalletDb: {:?}", + e + )), + } + }); + unwrap_exc_or(&env, res, -1) +} + +#[no_mangle] +pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_getMaxScannedHeight( + env: JNIEnv<'_>, + _: JClass<'_>, + db_data: JString<'_>, + network_id: jint, +) -> jlong { + let res = panic::catch_unwind(|| { + let network = parse_network(network_id as u32)?; + let db_data = wallet_db(&env, network, db_data)?; + + match db_data.block_max_scanned() { + Ok(Some(metadata)) => Ok(i64::from(u32::from(metadata.block_height()))), + // Use -1 to return null across the FFI. + Ok(None) => Ok(-1), + Err(e) => Err(format_err!( + "Failed to read block metadata from WalletDb: {:?}", + e + )), + } + }); + unwrap_exc_or(&env, res, -1) +} + fn encode_scan_progress(env: &JNIEnv<'_>, progress: Ratio) -> Result { let output = env.new_object( "cash/z/ecc/android/sdk/internal/model/JniScanProgress", diff --git a/sdk-lib/src/androidTest/java/cash/z/ecc/fixture/FakeRustBackend.kt b/sdk-lib/src/androidTest/java/cash/z/ecc/fixture/FakeRustBackend.kt index 829116fb..0a7762c8 100644 --- a/sdk-lib/src/androidTest/java/cash/z/ecc/fixture/FakeRustBackend.kt +++ b/sdk-lib/src/androidTest/java/cash/z/ecc/fixture/FakeRustBackend.kt @@ -31,6 +31,14 @@ internal class FakeRustBackend( TODO("Not yet implemented") } + override suspend fun getFullyScannedHeight(): Long? { + TODO("Not yet implemented") + } + + override suspend fun getMaxScannedHeight(): Long? { + TODO("Not yet implemented") + } + override suspend fun getScanProgress(): JniScanProgress { TODO("Not yet implemented") } @@ -39,7 +47,7 @@ internal class FakeRustBackend( TODO("Not yet implemented") } - override suspend fun getLatestHeight(): Long = metadata.maxOf { it.height } + override suspend fun getLatestCacheHeight(): Long = metadata.maxOf { it.height } override suspend fun getVerifiedTransparentBalance(address: String): Long { TODO("Not yet implemented") diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackend.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackend.kt index 289f55c2..9e56d5c7 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackend.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackend.kt @@ -51,7 +51,7 @@ internal interface TypesafeBackend { suspend fun rewindToHeight(height: BlockHeight) - suspend fun getLatestBlockHeight(): BlockHeight? + suspend fun getLatestCacheHeight(): BlockHeight? suspend fun findBlockMetadata(height: BlockHeight): JniBlockMeta? @@ -88,6 +88,29 @@ internal interface TypesafeBackend { @Throws(RuntimeException::class) suspend fun updateChainTip(height: BlockHeight) + /** + * Returns the height to which the wallet has been fully scanned. + * + * This is the height for which the wallet has fully trial-decrypted this and all + * preceding blocks above the wallet's birthday height. + * + * @return The height to which the wallet has been fully scanned, or Null if no blocks have been scanned. + * @throws RuntimeException as a common indicator of the operation failure + */ + suspend fun getFullyScannedHeight(): BlockHeight? + + /** + * Returns the maximum height that the wallet has scanned. + * + * If the wallet is fully synced, this will be equivalent to `getFullyScannedHeight`; + * otherwise the maximal scanned height is likely to be greater than the fully scanned + * height due to the fact that out-of-order scanning can leave gaps. + * + * @return The maximum height that the wallet has scanned, or Null if no blocks have been scanned. + * @throws RuntimeException as a common indicator of the operation failure + */ + suspend fun getMaxScannedHeight(): BlockHeight? + /** * @throws RuntimeException as a common indicator of the operation failure */ diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackendImpl.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackendImpl.kt index 2d5d248a..22a9d381 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackendImpl.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackendImpl.kt @@ -93,8 +93,8 @@ internal class TypesafeBackendImpl(private val backend: Backend) : TypesafeBacke backend.rewindToHeight(height.value) } - override suspend fun getLatestBlockHeight(): BlockHeight? { - return backend.getLatestHeight()?.let { + override suspend fun getLatestCacheHeight(): BlockHeight? { + return backend.getLatestCacheHeight()?.let { BlockHeight.new( ZcashNetwork.from(backend.networkId), it @@ -162,6 +162,24 @@ internal class TypesafeBackendImpl(private val backend: Backend) : TypesafeBacke override suspend fun updateChainTip(height: BlockHeight) = backend.updateChainTip(height.value) + override suspend fun getFullyScannedHeight(): BlockHeight? { + return backend.getFullyScannedHeight()?.let { + BlockHeight.new( + ZcashNetwork.from(backend.networkId), + it + ) + } + } + + override suspend fun getMaxScannedHeight(): BlockHeight? { + return backend.getMaxScannedHeight()?.let { + BlockHeight.new( + ZcashNetwork.from(backend.networkId), + it + ) + } + } + override suspend fun scanBlocks(fromHeight: BlockHeight, limit: Long) = backend.scanBlocks(fromHeight.value, limit) override suspend fun getScanProgress(): ScanProgress? = backend.getScanProgress()?.let { jniScanProgress -> diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/storage/block/FileCompactBlockRepository.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/storage/block/FileCompactBlockRepository.kt index d74441cc..4ce75751 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/storage/block/FileCompactBlockRepository.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/storage/block/FileCompactBlockRepository.kt @@ -25,7 +25,7 @@ internal class FileCompactBlockRepository( private val backend: TypesafeBackend ) : CompactBlockRepository { - override suspend fun getLatestHeight() = backend.getLatestBlockHeight() + override suspend fun getLatestHeight() = backend.getLatestCacheHeight() override suspend fun findCompactBlock(height: BlockHeight) = backend.findBlockMetadata(height) From 33a7a2000fb71724de7bfe3ca20d9ae3409cfc00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Honza=20Rychnovsk=C3=BD?= Date: Thu, 7 Sep 2023 11:52:03 +0200 Subject: [PATCH 41/76] [#1210] Eliminate LINEAR sync support --- CHANGELOG.md | 6 - README.md | 8 +- .../sdk/demoapp/WalletCoordinatorFactory.kt | 4 +- .../z/ecc/android/sdk/WalletCoordinator.kt | 9 +- .../cash/z/ecc/android/sdk/SdkSynchronizer.kt | 6 +- .../cash/z/ecc/android/sdk/Synchronizer.kt | 15 +- .../block/processor/CompactBlockProcessor.kt | 212 ++++-------------- .../processor/model/GetSubtreeRootsResult.kt | 7 +- .../processor/CompactBlockProcessorTest.kt | 12 +- 9 files changed, 66 insertions(+), 213 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f5cd109..d466553a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,12 +6,6 @@ - The block synchronization mechanism is about one-third faster thanks to the optimized `CompactBlockProcessor.SYNC_BATCH_SIZE`. Issue **#1206**. -### Added -- New `syncAlgorithm` parameter of `Synchronizer.new()` and `WalletCoordinator()` to select preferred - `CompactBlockProcessor` block synchronizing algorithm. It can be of `CompactBlockProcessor.SyncAlgorithm.LINEAR` - or `SPEND_BEFORE_SYNC`. The LINEAR type is automatically used if the client app does not specify otherwise. Please - note that the SPEND_BEFORE_SYNC type is currently unstable and still under development. - ### Removed - `CompactBlockProcessor.ProcessorInfo.lastSyncHeight` which the SDK is no longer able to provide because of the new **SpendBeforeSync** synchronization algorithm adoption. Use `CompactBlockProcessor.ProcessorInfo.overallSyncRange` diff --git a/README.md b/README.md index a2e1cb64..b41ecd0a 100644 --- a/README.md +++ b/README.md @@ -56,9 +56,5 @@ Note that we aim for the main branch of this repository to be stable and releasa ## Unstable Features ### Spend-before-Sync compact blocks synchronization algorithm -- CompactBlockProcessor now processes compact blocks from the lightwalletd server in a **spend-before-sync** order. This -feature shortens the time after which a wallet's spendable balance can be used. -- Use the new `syncAlgorithm` parameter of `Synchronizer.new()` or `WalletCoordinator()` to select preferred - `CompactBlockProcessor` block synchronizing algorithm. It can be of `CompactBlockProcessor.SyncAlgorithm.LINEAR` - or `SPEND_BEFORE_SYNC`. The LINEAR type is automatically used if the client app does not specify otherwise. Please - note that the SPEND_BEFORE_SYNC type is currently unstable and still under development. +- CompactBlockProcessor now processes compact blocks from the lightwalletd server in the **spend-before-sync** order. This +feature shortens the time after which a wallet's spendable balance can be used. Please note that this new block synchronization algorithm is still under development. diff --git a/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/WalletCoordinatorFactory.kt b/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/WalletCoordinatorFactory.kt index 9a7a8b5f..8cb1c6b8 100644 --- a/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/WalletCoordinatorFactory.kt +++ b/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/WalletCoordinatorFactory.kt @@ -2,7 +2,6 @@ package cash.z.ecc.android.sdk.demoapp import android.content.Context import cash.z.ecc.android.sdk.WalletCoordinator -import cash.z.ecc.android.sdk.block.processor.CompactBlockProcessor import cash.z.ecc.android.sdk.demoapp.preference.EncryptedPreferenceKeys import cash.z.ecc.android.sdk.demoapp.preference.EncryptedPreferenceSingleton import cash.z.ecc.android.sdk.demoapp.util.LazyWithArgument @@ -23,8 +22,7 @@ private val lazy = LazyWithArgument { WalletCoordinator( context = it, - persistableWallet = persistableWalletFlow, - syncAlgorithm = CompactBlockProcessor.SyncAlgorithm.SPEND_BEFORE_SYNC + persistableWallet = persistableWalletFlow ) } diff --git a/sdk-incubator-lib/src/main/java/cash/z/ecc/android/sdk/WalletCoordinator.kt b/sdk-incubator-lib/src/main/java/cash/z/ecc/android/sdk/WalletCoordinator.kt index eac7e1e1..8f6ea95a 100644 --- a/sdk-incubator-lib/src/main/java/cash/z/ecc/android/sdk/WalletCoordinator.kt +++ b/sdk-incubator-lib/src/main/java/cash/z/ecc/android/sdk/WalletCoordinator.kt @@ -1,7 +1,6 @@ package cash.z.ecc.android.sdk import android.content.Context -import cash.z.ecc.android.sdk.block.processor.CompactBlockProcessor import cash.z.ecc.android.sdk.ext.onFirst import cash.z.ecc.android.sdk.internal.Twig import cash.z.ecc.android.sdk.model.PersistableWallet @@ -34,8 +33,6 @@ import java.util.UUID /** * @param persistableWallet flow of the user's stored wallet. Null indicates that no wallet has been stored. - * - * @param syncAlgorithm The CompactBlockProcess's type of block syncing algorithm */ /* * One area where this class needs to change before it can be moved out of the incubator is that we need to be able to @@ -46,8 +43,7 @@ import java.util.UUID */ class WalletCoordinator( context: Context, - val persistableWallet: Flow, - val syncAlgorithm: CompactBlockProcessor.SyncAlgorithm = CompactBlockProcessor.SyncAlgorithm.LINEAR + val persistableWallet: Flow ) { private val applicationContext = context.applicationContext @@ -83,8 +79,7 @@ class WalletCoordinator( zcashNetwork = persistableWallet.network, lightWalletEndpoint = LightWalletEndpoint.defaultForNetwork(persistableWallet.network), birthday = persistableWallet.birthday, - seed = persistableWallet.seedPhrase.toByteArray(), - syncAlgorithm = syncAlgorithm + seed = persistableWallet.seedPhrase.toByteArray() ) trySend(InternalSynchronizerStatus.Available(closeableSynchronizer)) diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt index 4d2e32ba..53a8bd2c 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt @@ -723,14 +723,12 @@ internal object DefaultSynchronizerFactory { backend: TypesafeBackend, downloader: CompactBlockDownloader, repository: DerivedDataRepository, - birthdayHeight: BlockHeight, - syncAlgorithm: CompactBlockProcessor.SyncAlgorithm + birthdayHeight: BlockHeight ): CompactBlockProcessor = CompactBlockProcessor( downloader = downloader, repository = repository, backend = backend, - minimumHeight = birthdayHeight, - syncAlgorithm = syncAlgorithm + minimumHeight = birthdayHeight ) } diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/Synchronizer.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/Synchronizer.kt index d37e9067..1b0ec62e 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/Synchronizer.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/Synchronizer.kt @@ -162,7 +162,7 @@ interface Synchronizer { * Sends zatoshi. * * @param usk the unified spending key associated with the notes that will be spent. - * @param zatoshi the amount of zatoshi to send. + * @param amount the amount of zatoshi to send. * @param toAddress the recipient's address. * @param memo the optional memo to include as part of the transaction. * @@ -425,8 +425,6 @@ interface Synchronizer { * to create the wallet. If that value is unknown, null is acceptable but will result in longer * sync times. After sync completes, the birthday can be determined from [Synchronizer.latestBirthdayHeight]. * - * @param syncAlgorithm The CompactBlockProcess's type of block syncing algorithm - * * @throws InitializerException.SeedRequired Indicates clients need to call this method again, providing the * seed bytes. * @@ -445,8 +443,7 @@ interface Synchronizer { alias: String = ZcashSdk.DEFAULT_ALIAS, lightWalletEndpoint: LightWalletEndpoint, seed: ByteArray?, - birthday: BlockHeight?, - syncAlgorithm: CompactBlockProcessor.SyncAlgorithm = CompactBlockProcessor.SyncAlgorithm.LINEAR + birthday: BlockHeight? ): CloseableSynchronizer { val applicationContext = context.applicationContext @@ -497,8 +494,7 @@ interface Synchronizer { backend = backend, downloader = downloader, repository = repository, - birthdayHeight = birthday ?: zcashNetwork.saplingActivationHeight, - syncAlgorithm = syncAlgorithm + birthdayHeight = birthday ?: zcashNetwork.saplingActivationHeight ) return SdkSynchronizer.new( @@ -525,10 +521,9 @@ interface Synchronizer { alias: String = ZcashSdk.DEFAULT_ALIAS, lightWalletEndpoint: LightWalletEndpoint, seed: ByteArray?, - birthday: BlockHeight?, - syncAlgorithm: CompactBlockProcessor.SyncAlgorithm = CompactBlockProcessor.SyncAlgorithm.LINEAR + birthday: BlockHeight? ): CloseableSynchronizer = runBlocking { - new(context, zcashNetwork, alias, lightWalletEndpoint, seed, birthday, syncAlgorithm) + new(context, zcashNetwork, alias, lightWalletEndpoint, seed, birthday) } /** diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessor.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessor.kt index 185cc36c..5e894a18 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessor.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessor.kt @@ -46,8 +46,6 @@ import cash.z.ecc.android.sdk.model.BlockHeight import cash.z.ecc.android.sdk.model.PercentDecimal import cash.z.ecc.android.sdk.model.WalletBalance import cash.z.ecc.android.sdk.model.ZcashNetwork -import co.electriccoin.lightwallet.client.ext.BenchmarkingExt -import co.electriccoin.lightwallet.client.fixture.BenchmarkingBlockRangeFixture import co.electriccoin.lightwallet.client.model.BlockHeightUnsafe import co.electriccoin.lightwallet.client.model.GetAddressUtxosReplyUnsafe import co.electriccoin.lightwallet.client.model.LightWalletEndpointInfoUnsafe @@ -93,8 +91,6 @@ import kotlin.time.toDuration * reorgs as a backstop to make sure we do not rewind beyond sapling activation. It also is factored * in when considering initial range to download. In most cases, this should be the birthday height * of the current wallet--the height before which we do not need to scan for transactions. - * - * @property syncAlgorithm The type of block syncing algorithm which should be preferably used */ @OpenForTesting @Suppress("TooManyFunctions", "LargeClass") @@ -102,8 +98,7 @@ class CompactBlockProcessor internal constructor( val downloader: CompactBlockDownloader, private val repository: DerivedDataRepository, private val backend: TypesafeBackend, - minimumHeight: BlockHeight, - private val syncAlgorithm: SyncAlgorithm + minimumHeight: BlockHeight ) { /** * Callback for any non-trivial errors that occur while processing compact blocks. @@ -160,14 +155,14 @@ class CompactBlockProcessor internal constructor( /** * The synchronization-related variable that holds the all batch count computed in the first initial synchronization * loop. It is supposed to keep the same value across the synchronization refreshes with [runSbSSyncingPreparation] - * as happens in [SyncAlgorithm.SPEND_BEFORE_SYNC] synchronization function. + * as happens in spend-before-sync synchronization function. */ private var allBatchCount: Long = 0 /** * Another synchronization-related variable that holds the order of a currently processing batch of blocks. It * is supposed to preserve its value across the synchronization refreshes with [runSbSSyncingPreparation] as - * happens in [SyncAlgorithm.SPEND_BEFORE_SYNC] synchronization function. + * happens in spend-before-sync synchronization function. */ private var lastBatchOrder: Long = 0 @@ -227,13 +222,9 @@ class CompactBlockProcessor internal constructor( ) // Download note commitment tree data from lightwalletd to decide if we communicate with linear - // or spend-before-sync node. It depends on the syncAlgorithm property on the first place. - var subTreeRootResult = if (syncAlgorithm == SyncAlgorithm.LINEAR) { - GetSubtreeRootsResult.UseLinear - } else { - getSubtreeRoots(downloader, network) - } - Twig.info { "Fetched SubTreeRoot result: $subTreeRootResult, with preferred sync algorithm: $syncAlgorithm" } + // or spend-before-sync node. + var subTreeRootResult = getSubtreeRoots(downloader, network) + Twig.info { "Fetched SubTreeRoot result: $subTreeRootResult" } Twig.debug { "Setup verified. Processor starting..." } @@ -246,13 +237,13 @@ class CompactBlockProcessor internal constructor( ) { val result = processingMutex.withLockLogged("processNewBlocks") { when (subTreeRootResult) { - is GetSubtreeRootsResult.UseSbS -> { + is GetSubtreeRootsResult.SpendBeforeSync -> { // Pass the commitment tree data to the database when ( val result = putSaplingSubtreeRoots( backend = backend, startIndex = 0, - subTreeRootList = (subTreeRootResult as GetSubtreeRootsResult.UseSbS) + subTreeRootList = (subTreeRootResult as GetSubtreeRootsResult.SpendBeforeSync) .subTreeRootList, lastValidHeight = lowerBoundHeight ) @@ -273,13 +264,30 @@ class CompactBlockProcessor internal constructor( firstUnenhancedHeight = _processorInfo.value.firstUnenhancedHeight ) } - GetSubtreeRootsResult.UseLinear -> { - // Forced by syncAlgorithm parameter or caused by an empty response result - processNewBlocksInLinearOrder() + GetSubtreeRootsResult.Linear -> { + // This is caused by an empty response result. Although the spend-before-sync + // synchronization algorithm is not supported, we can get the entire block range as we + // previously did for the linear sync type. + processNewBlocksInSbSOrder( + backend = backend, + downloader = downloader, + repository = repository, + network = network, + lastValidHeight = lowerBoundHeight, + firstUnenhancedHeight = _processorInfo.value.firstUnenhancedHeight + ) } is GetSubtreeRootsResult.OtherFailure -> { - // Server possible replied with some kind of unsupported error - processNewBlocksInLinearOrder() + // The server possibly replied with some unsupported error. We still approach + // spend-before-sync synchronization. + processNewBlocksInSbSOrder( + backend = backend, + downloader = downloader, + repository = repository, + network = network, + lastValidHeight = lowerBoundHeight, + firstUnenhancedHeight = _processorInfo.value.firstUnenhancedHeight + ) } GetSubtreeRootsResult.FailureConnection -> { // SubtreeRoot fetching retry @@ -369,39 +377,6 @@ class CompactBlockProcessor internal constructor( throw error } - private suspend fun processNewBlocksInLinearOrder(): BlockProcessingResult { - Twig.info { "Beginning to process new blocks with Linear approach (with lower bound: $lowerBoundHeight)..." } - - return if (!updateRange(null)) { - Twig.warn { "Disconnection detected. Attempting to reconnect." } - BlockProcessingResult.Reconnecting - } else if (_processorInfo.value.overallSyncRange.isNullOrEmpty()) { - Twig.info { "No more blocks to process." } - BlockProcessingResult.NoBlocksToProcess - } else { - setState(State.Syncing) - val syncRange = if (BenchmarkingExt.isBenchmarking()) { - // We inject a benchmark test blocks range at this point to process only a restricted range of - // blocks for a more reliable benchmark results. - val benchmarkBlockRange = BenchmarkingBlockRangeFixture.new().let { - // Convert range of Longs to range of BlockHeights - BlockHeight.new(ZcashNetwork.Mainnet, it.start)..( - BlockHeight.new(ZcashNetwork.Mainnet, it.endInclusive) - ) - } - benchmarkBlockRange - } else { - _processorInfo.value.overallSyncRange!! - } - - syncBlocksAndEnhanceTransactionsLinearly( - syncRange = syncRange, - withDownload = true, - enhanceStartHeight = _processorInfo.value.firstUnenhancedHeight - ) - } - } - // TODO [#1137]: Refactor processNewBlocksInSbSOrder // TODO [#1137]: https://github.com/zcash/zcash-android-wallet-sdk/issues/1137 /** @@ -580,7 +555,7 @@ class CompactBlockProcessor internal constructor( if (shouldRefreshPreparation( lastPreparationTime, currentTimeMillis, - SBS_SYNCHRONIZATION_RESTART_TIMEOUT + SYNCHRONIZATION_RESTART_TIMEOUT ) ) { SyncingResult.RestartSynchronization @@ -651,6 +626,9 @@ class CompactBlockProcessor internal constructor( } } + // TODO [#1211]: Re-enable block synchronization benchmark test + // TODO [#1211]: https://github.com/zcash/zcash-android-wallet-sdk/issues/1211 + // Get the suggested scan ranges from the wallet database val suggestedRangesResult = suggestScanRanges( backend, @@ -694,61 +672,6 @@ class CompactBlockProcessor internal constructor( ) } - @Suppress("ReturnCount") - private suspend fun syncBlocksAndEnhanceTransactionsLinearly( - syncRange: ClosedRange, - withDownload: Boolean, - enhanceStartHeight: BlockHeight? - ): BlockProcessingResult { - var syncingResult: SyncingResult = SyncingResult.AllSuccess - allBatchCount = getBatchCount(listOf(syncRange)) - - // Syncing last blocks and enhancing transactions - runSyncingAndEnhancingOnRange( - backend = backend, - downloader = downloader, - repository = repository, - network = network, - syncRange = syncRange, - withDownload = withDownload, - enhanceStartHeight = enhanceStartHeight, - lastBatchOrder = 0 - ).collect { rangeSyncProgress -> - setProgress(PercentDecimal(rangeSyncProgress.overallOrder / allBatchCount.toFloat())) - checkAllBalances() - - when (rangeSyncProgress.resultState) { - SyncingResult.UpdateBirthday -> { - updateBirthdayHeight() - } - SyncingResult.EnhanceSuccess -> { - Twig.info { "Triggering transaction refresh now" } - // Invalidate transaction data - checkTransactions(transactionStorage = repository) - } - is SyncingResult.Failure -> { - syncingResult = rangeSyncProgress.resultState - return@collect - } else -> { - // Continue with processing - } - } - } - - if (syncingResult != SyncingResult.AllSuccess) { - // Remove persisted but not scanned blocks in case of any failure - val lastScannedHeight = getLastScannedHeight(repository) - downloader.rewindToHeight(lastScannedHeight) - deleteAllBlockFiles( - downloader = downloader, - lastKnownHeight = lastScannedHeight - ) - return (syncingResult as SyncingResult.Failure).toBlockProcessingResult() - } - - return BlockProcessingResult.Success - } - /** * This invalidates transaction storage to trigger data refreshing for its subscribers. */ @@ -795,52 +718,22 @@ class CompactBlockProcessor internal constructor( /** * Gets the latest range info and then uses that initialInfo to update (and transmit) - * the info that require processing. This function is universal and works for both [SyncAlgorithm.LINEAR] and - * [SyncAlgorithm.SPEND_BEFORE_SYNC] algorithms. + * the info that require processing. * - * @param ranges The ranges which we obtained from the rust layer to proceed in case of - * [SyncAlgorithm.SPEND_BEFORE_SYNC] algorithm, or null in case of [SyncAlgorithm.LINEAR] algorithm, as it will - * be computed. + * @param ranges The ranges which we obtained from the rust layer to proceed * * @return true when the update succeeds. */ - private suspend fun updateRange(ranges: List?): Boolean { + private suspend fun updateRange(ranges: List): Boolean { // This fetches the latest height each time this method is called, which can be very inefficient // when downloading all of the blocks from the server val networkBlockHeight = fetchLatestBlockHeight(downloader, network) ?: return false - // If we find out that we previously downloaded, but not scanned persisted blocks, we need to rewind the - // blocks above the last scanned height first. - val lastScannedHeight = getLastScannedHeight(repository) - val lastDownloadedHeight = getLastDownloadedHeight(downloader).let { - BlockHeight.new( - network, - max( - it?.value ?: 0, - lowerBoundHeight.value - ) - ) - } - val lastSyncedHeight = if (lastDownloadedHeight.value - lastScannedHeight.value > 0) { - Twig.verbose { - "Clearing blocks of last persisted batch within the last scanned height " + - "$lastScannedHeight and last download height $lastDownloadedHeight, as all these blocks " + - "possibly haven't been scanned in the previous blocks sync attempt." - } - downloader.rewindToHeight(lastScannedHeight) - lastScannedHeight - } else { - lastDownloadedHeight - } - // Get the first un-enhanced transaction from the repository val firstUnenhancedHeight = getFirstUnenhancedHeight(repository) - // The sync range computation depends on the used sync algorithm. The LINEAR one will compute it by itself, - // when the SPEND_BEFORE_SYNC will use the one obtained from the rust layer - val syncRange = if (ranges == null) { - lastSyncedHeight + 1..networkBlockHeight - } else if (ranges.isNotEmpty()) { + // The overall sync range computation + val syncRange = if (ranges.isNotEmpty()) { var resultRange = ranges[0].range.start..ranges[0].range.endInclusive ranges.forEach { nextRange -> if (nextRange.range.start < resultRange.start) { @@ -852,7 +745,7 @@ class CompactBlockProcessor internal constructor( } resultRange } else { - // Empty ranges most likely means that the SbS is done and the Rust layer replied with an empty suggested + // Empty ranges most likely means that the sync is done and the Rust layer replied with an empty suggested // ranges null } @@ -1099,10 +992,9 @@ class CompactBlockProcessor internal constructor( internal const val REWIND_DISTANCE = 10 /** - * Limit millis value for restarting currently running block synchronization that runs under the - * [SyncAlgorithm.SPEND_BEFORE_SYNC] synchronization. + * Limit millis value for restarting currently running block synchronization. */ - internal val SBS_SYNCHRONIZATION_RESTART_TIMEOUT = 10.minutes.inWholeMilliseconds + internal val SYNCHRONIZATION_RESTART_TIMEOUT = 10.minutes.inWholeMilliseconds /** * Check for the next restart of the block synchronization preparation phase. This function is only SbS @@ -1165,7 +1057,7 @@ class CompactBlockProcessor internal constructor( ): GetSubtreeRootsResult { Twig.debug { "Fetching SubtreeRoots..." } - var result: GetSubtreeRootsResult = GetSubtreeRootsResult.UseLinear + var result: GetSubtreeRootsResult = GetSubtreeRootsResult.Linear retryUpToAndContinue(GET_SUBTREE_ROOTS_RETRIES) { downloader.getSubtreeRoots( @@ -1213,9 +1105,9 @@ class CompactBlockProcessor internal constructor( SubtreeRoot.new(it, network) }.let { result = if (it.isEmpty()) { - GetSubtreeRootsResult.UseLinear + GetSubtreeRootsResult.Linear } else { - GetSubtreeRootsResult.UseSbS(it) + GetSubtreeRootsResult.SpendBeforeSync(it) } } } @@ -1337,8 +1229,6 @@ class CompactBlockProcessor internal constructor( /** * Requests, processes and persists all blocks from the given range. * - * Works the same for both [SyncAlgorithm.LINEAR] and [SyncAlgorithm.SPEND_BEFORE_SYNC] algorithms. - * * @param backend the Rust backend component * @param downloader the compact block downloader component * @param repository the derived data repository component @@ -2159,18 +2049,6 @@ class CompactBlockProcessor internal constructor( val hash: String? ) - /** - * Algorithm used to sync the SDK with the blockchain - */ - enum class SyncAlgorithm { - // Linear sync processes the un-synced blocks in a linear way up to the chain tip - LINEAR, - - // Spend before Sync processes the un-synced blocks non-linearly, in prioritised ranges relevant to the stored - // wallet. Note: This feature is in development (alpha version) so use carefully. - SPEND_BEFORE_SYNC - } - // // Helper Extensions // diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/GetSubtreeRootsResult.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/GetSubtreeRootsResult.kt index f6fb2f96..a87d2ceb 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/GetSubtreeRootsResult.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/GetSubtreeRootsResult.kt @@ -6,9 +6,8 @@ import cash.z.ecc.android.sdk.internal.model.SubtreeRoot * Internal class for get subtree roots action result. */ internal sealed class GetSubtreeRootsResult { - // SbS: Spend-before-Sync - data class UseSbS(val subTreeRootList: List) : GetSubtreeRootsResult() - object UseLinear : GetSubtreeRootsResult() - object FailureConnection : GetSubtreeRootsResult() + data class SpendBeforeSync(val subTreeRootList: List) : GetSubtreeRootsResult() + data object Linear : GetSubtreeRootsResult() + data object FailureConnection : GetSubtreeRootsResult() data class OtherFailure(val exception: Throwable) : GetSubtreeRootsResult() } diff --git a/sdk-lib/src/test/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessorTest.kt b/sdk-lib/src/test/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessorTest.kt index 13526cde..1df6c35c 100644 --- a/sdk-lib/src/test/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessorTest.kt +++ b/sdk-lib/src/test/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessorTest.kt @@ -10,9 +10,9 @@ class CompactBlockProcessorTest { fun should_refresh_preparation_test() { assertTrue { CompactBlockProcessor.shouldRefreshPreparation( - lastPreparationTime = CompactBlockProcessor.SBS_SYNCHRONIZATION_RESTART_TIMEOUT, - currentTimeMillis = CompactBlockProcessor.SBS_SYNCHRONIZATION_RESTART_TIMEOUT * 2, - limitTime = CompactBlockProcessor.SBS_SYNCHRONIZATION_RESTART_TIMEOUT + lastPreparationTime = CompactBlockProcessor.SYNCHRONIZATION_RESTART_TIMEOUT, + currentTimeMillis = CompactBlockProcessor.SYNCHRONIZATION_RESTART_TIMEOUT * 2, + limitTime = CompactBlockProcessor.SYNCHRONIZATION_RESTART_TIMEOUT ) } } @@ -21,9 +21,9 @@ class CompactBlockProcessorTest { fun should_not_refresh_preparation_test() { assertFalse { CompactBlockProcessor.shouldRefreshPreparation( - lastPreparationTime = CompactBlockProcessor.SBS_SYNCHRONIZATION_RESTART_TIMEOUT, - currentTimeMillis = CompactBlockProcessor.SBS_SYNCHRONIZATION_RESTART_TIMEOUT, - limitTime = CompactBlockProcessor.SBS_SYNCHRONIZATION_RESTART_TIMEOUT + lastPreparationTime = CompactBlockProcessor.SYNCHRONIZATION_RESTART_TIMEOUT, + currentTimeMillis = CompactBlockProcessor.SYNCHRONIZATION_RESTART_TIMEOUT, + limitTime = CompactBlockProcessor.SYNCHRONIZATION_RESTART_TIMEOUT ) } } From 14d854f96e8978286ed2bfeea6c1e489f9ca003c Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Thu, 7 Sep 2023 15:19:37 +0100 Subject: [PATCH 42/76] rust: Return 0 from `getBalance` and `getVerifiedBalance` for unknown accounts This works around zcash/librustzcash#948, where known accounts without history are not reported in the `WalletSummary`. We assume that the Kotlin side of the SDK does not call these FFI methods with unknown accounts. --- backend-lib/src/main/rust/lib.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/backend-lib/src/main/rust/lib.rs b/backend-lib/src/main/rust/lib.rs index 61ed06d7..64701f27 100644 --- a/backend-lib/src/main/rust/lib.rs +++ b/backend-lib/src/main/rust/lib.rs @@ -661,11 +661,11 @@ pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_ge .get_wallet_summary(0) .map_err(|e| format_err!("Error while fetching balance: {}", e))? { - wallet_summary + Ok(wallet_summary .account_balances() .get(&account) - .ok_or_else(|| format_err!("Unknown account")) .map(|balances| Amount::from(balances.sapling_balance.total()).into()) + .unwrap_or(0)) } else { // `None` means that the caller has not yet called `updateChainTip` on a // brand-new wallet, so we can assume the balance is zero. @@ -770,11 +770,11 @@ pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_ge .get_wallet_summary(ANCHOR_OFFSET_U32) .map_err(|e| format_err!("Error while fetching verified balance: {}", e))? { - wallet_summary + Ok(wallet_summary .account_balances() .get(&account) - .ok_or_else(|| format_err!("Unknown account")) .map(|balances| Amount::from(balances.sapling_balance.spendable_value).into()) + .unwrap_or(0)) } else { // `None` means that the caller has not yet called `updateChainTip` on a // brand-new wallet, so we can assume the balance is zero. From fc14082a1c84b456e356823da0cab7f11ef9f9ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Honza=20Rychnovsk=C3=BD?= Date: Thu, 7 Sep 2023 18:00:34 +0200 Subject: [PATCH 43/76] [#1213] Remove `BlockTable` and its APIs * [#1213] Remove `count()` from BlockTable API * [#1213] Remove `firstScannedHeight()` from BlockTable API * [#1213] Remove `findBlockHash()` from BlockTable * [#1213] Remove `lastScannedHeight()` from BlockTable * [#1213] Remove `BlockTable` entirely --- CHANGELOG.md | 22 ++-- .../z/ecc/android/sdk/internal/Backend.kt | 11 ++ .../cash/z/ecc/android/sdk/SdkSynchronizer.kt | 11 +- .../block/processor/CompactBlockProcessor.kt | 117 +++++++++++------- .../model/GetMaxScannedHeightResult.kt | 12 ++ .../block/processor/model/SyncingResult.kt | 4 +- .../z/ecc/android/sdk/exception/Exceptions.kt | 7 +- .../sdk/internal/db/derived/BlockTable.kt | 88 ------------- .../db/derived/DbDerivedDataRepository.kt | 14 --- .../sdk/internal/db/derived/DerivedDataDb.kt | 12 +- .../sdk/internal/model/ScanProgress.kt | 2 +- .../repository/DerivedDataRepository.kt | 24 ---- .../transaction/TransactionEncoderImpl.kt | 4 +- .../android/sdk/model/TransactionOverview.kt | 8 +- 14 files changed, 136 insertions(+), 200 deletions(-) create mode 100644 sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/GetMaxScannedHeightResult.kt delete mode 100644 sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/BlockTable.kt diff --git a/CHANGELOG.md b/CHANGELOG.md index d466553a..de2c712c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,21 @@ - `CompactBlockProcessor.ProcessorInfo.isSyncing`. Use `Synchronizer.status` instead. - `CompactBlockProcessor.ProcessorInfo.syncProgress`. Use `Synchronizer.progress` instead. - `alsoClearBlockCache` parameter from rewind functions of `Synchronizer` and `CompactBlockProcessor` as it take no - affect on the current rewind functionality result. + effect on the current rewind functionality result. +- Internally, we removed access to the shared block table from the Kotlin layer, which resulted in eliminating these + APIs: + - `SdkSynchornizer.findBlockHash()` + - `SdkSynchornizer.findBlockHashAsHex()` + +### Changed +- `CompactBlockProcessor.quickRewind()` and `CompactBlockProcessor.rewindToNearestHeight()` now might fail due to + internal changes in getting scanned height. Thus, these functions return `Boolean` results. + +### Fixed +- `Synchronizer.getMemos()` now correctly returns a flow of strings for sent and received transactions. Issue **#1154**. +- `CompactBlockProcessor` now triggers transaction polling while block synchronization is in progress as expected. + Clients will be notified briefly after every new transaction is discovered via `Synchronizer.transactions` API. + Issue **#1170**. ## 1.21.0-beta01 Note: This is the last _1.x_ version release. The upcoming version _2.0_ brings the **Spend-before-Sync** feature, @@ -32,12 +46,6 @@ which speeds up discovering the wallet's spendable balance. - etc. - Checkpoints -### Fixed -- `Synchronizer.getMemos()` now correctly returns a flow of strings for sent and received transactions. Issue **#1154**. -- `CompactBlockProcessor` now triggers transaction polling while block synchronization is in progress as expected. - Clients will be notified briefly after every new transaction is discovered via `Synchronizer.transactions` API. - Issue **#1170**. - ## 1.20.0-beta01 - The SDK internally migrated from `BackendExt` rust backend extension functions to more type-safe `TypesafeBackend`. - `Synchronizer.getMemos()` now internally handles expected `RuntimeException` from the rust layer and transforms it diff --git a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/Backend.kt b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/Backend.kt index ed61af6b..d0958510 100644 --- a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/Backend.kt +++ b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/Backend.kt @@ -36,6 +36,17 @@ interface Backend { suspend fun decryptAndStoreTransaction(tx: ByteArray) + /** + * Sets up the internal structure of the data database. + * + * If `seed` is `null`, database migrations will be attempted without it. + * + * @return 0 if successful, 1 if the seed must be provided in order to execute the requested migrations, or -1 + * otherwise. + * + * @throws RuntimeException as a common indicator of the operation failure + */ + @Throws(RuntimeException::class) suspend fun initDataDb(seed: ByteArray?): Int /** diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt index 53a8bd2c..ef0a1d37 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt @@ -24,7 +24,6 @@ import cash.z.ecc.android.sdk.internal.db.DatabaseCoordinator import cash.z.ecc.android.sdk.internal.db.derived.DbDerivedDataRepository import cash.z.ecc.android.sdk.internal.db.derived.DerivedDataDb import cash.z.ecc.android.sdk.internal.ext.isNullOrEmpty -import cash.z.ecc.android.sdk.internal.ext.toHexReversed import cash.z.ecc.android.sdk.internal.ext.tryNull import cash.z.ecc.android.sdk.internal.jni.RustBackend import cash.z.ecc.android.sdk.internal.model.Checkpoint @@ -185,7 +184,7 @@ class SdkSynchronizer private constructor( override val transactions get() = combine(processor.networkHeight, storage.allTransactions) { networkHeight, allTransactions -> - val latestBlockHeight = networkHeight ?: storage.lastScannedHeight() + val latestBlockHeight = networkHeight ?: backend.getMaxScannedHeight() allTransactions.map { TransactionOverview.new(it, latestBlockHeight) } } @@ -341,14 +340,6 @@ class SdkSynchronizer private constructor( // to do with the underlying data // TODO [#682]: https://github.com/zcash/zcash-android-wallet-sdk/issues/682 - suspend fun findBlockHash(height: BlockHeight): ByteArray? { - return storage.findBlockHash(height) - } - - suspend fun findBlockHashAsHex(height: BlockHeight): String? { - return findBlockHash(height)?.toHexReversed() - } - suspend fun getTransactionCount(): Int { return storage.getTransactionCount().toInt() } diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessor.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessor.kt index 5e894a18..e301663b 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessor.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessor.kt @@ -4,6 +4,7 @@ import androidx.annotation.VisibleForTesting import cash.z.ecc.android.sdk.BuildConfig import cash.z.ecc.android.sdk.annotation.OpenForTesting import cash.z.ecc.android.sdk.block.processor.model.BatchSyncProgress +import cash.z.ecc.android.sdk.block.processor.model.GetMaxScannedHeightResult import cash.z.ecc.android.sdk.block.processor.model.GetSubtreeRootsResult import cash.z.ecc.android.sdk.block.processor.model.PutSaplingSubtreeRootsResult import cash.z.ecc.android.sdk.block.processor.model.SbSPreparationResult @@ -218,7 +219,10 @@ class CompactBlockProcessor internal constructor( // Clear any undeleted left over block files from previous sync attempts deleteAllBlockFiles( downloader = downloader, - lastKnownHeight = getLastScannedHeight(repository) + lastKnownHeight = when (val result = getMaxScannedHeight(backend)) { + is GetMaxScannedHeightResult.Success -> result.height + else -> null + } ) // Download note commitment tree data from lightwalletd to decide if we communicate with linear @@ -347,7 +351,7 @@ class CompactBlockProcessor internal constructor( stop() } - suspend fun checkErrorResult(failedHeight: BlockHeight) { + suspend fun checkErrorResult(failedHeight: BlockHeight?) { if (consecutiveChainErrors.get() >= RETRIES) { val errorMessage = "ERROR: unable to resolve reorg at height $failedHeight after " + "${consecutiveChainErrors.get()} correction attempts!" @@ -476,8 +480,13 @@ class CompactBlockProcessor internal constructor( // Continue with processing the rest of the ranges } else -> { // An error came - remove persisted but not scanned blocks - val lastScannedHeight = getLastScannedHeight(repository) - downloader.rewindToHeight(lastScannedHeight) + val lastScannedHeight = when (val result = getMaxScannedHeight(backend)) { + is GetMaxScannedHeightResult.Success -> result.height + else -> null + } + lastScannedHeight?.let { + downloader.rewindToHeight(lastScannedHeight) + } deleteAllBlockFiles( downloader = downloader, lastKnownHeight = lastScannedHeight @@ -579,8 +588,13 @@ class CompactBlockProcessor internal constructor( return BlockProcessingResult.RestartSynchronization } else -> { // An error came - remove persisted but not scanned blocks - val lastScannedHeight = getLastScannedHeight(repository) - downloader.rewindToHeight(lastScannedHeight) + val lastScannedHeight = when (val result = getMaxScannedHeight(backend)) { + is GetMaxScannedHeightResult.Success -> result.height + else -> null + } + lastScannedHeight?.let { + downloader.rewindToHeight(lastScannedHeight) + } deleteAllBlockFiles( downloader = downloader, lastKnownHeight = lastScannedHeight @@ -713,7 +727,7 @@ class CompactBlockProcessor internal constructor( object Success : BlockProcessingResult() object Reconnecting : BlockProcessingResult() object RestartSynchronization : BlockProcessingResult() - data class SyncFailure(val failedAtHeight: BlockHeight, val error: Throwable) : BlockProcessingResult() + data class SyncFailure(val failedAtHeight: BlockHeight?, val error: Throwable) : BlockProcessingResult() } /** @@ -767,10 +781,7 @@ class CompactBlockProcessor internal constructor( // TODO [#1127]: https://github.com/zcash/zcash-android-wallet-sdk/issues/1127 @Suppress("NestedBlockDepth") private suspend fun verifySetup() { - // verify that the data is initialized - val error = if (!repository.isInitialized()) { - CompactBlockProcessorException.Uninitialized - } else if (repository.getAccountCount() == 0) { + val error = if (repository.getAccountCount() == 0) { CompactBlockProcessorException.NoAccount } else { // verify that the server is correct @@ -1531,7 +1542,7 @@ class CompactBlockProcessor internal constructor( @VisibleForTesting internal suspend fun deleteAllBlockFiles( downloader: CompactBlockDownloader, - lastKnownHeight: BlockHeight + lastKnownHeight: BlockHeight? ): SyncingResult { Twig.verbose { "Starting to delete all temporary block files" } return if (downloader.compactBlockRepository.deleteAllCompactBlockFiles()) { @@ -1691,8 +1702,26 @@ class CompactBlockProcessor internal constructor( * @return the last scanned height reported by the repository. */ @VisibleForTesting - internal suspend fun getLastScannedHeight(repository: DerivedDataRepository) = - repository.lastScannedHeight() + internal suspend fun getMaxScannedHeight(backend: TypesafeBackend): GetMaxScannedHeightResult { + return runCatching { + backend.getMaxScannedHeight() + }.onSuccess { + Twig.verbose { "Successfully called getMaxScannedHeight with result: $it" } + }.onFailure { + Twig.error { "Failed to call getMaxScannedHeight with result: $it" } + }.fold( + onSuccess = { + if (it == null) { + GetMaxScannedHeightResult.None + } else { + GetMaxScannedHeightResult.Success(it) + } + }, + onFailure = { + GetMaxScannedHeightResult.Failure(it) + } + ) + } /** * Get the height of the first un-enhanced transaction detail from the repository. @@ -1784,14 +1813,14 @@ class CompactBlockProcessor internal constructor( _state.value = newState } - private suspend fun handleChainError(errorHeight: BlockHeight) { - // TODO [#683]: Consider an error object containing hash information - // TODO [#683]: https://github.com/zcash/zcash-android-wallet-sdk/issues/683 + private suspend fun handleChainError(errorHeight: BlockHeight?) { printValidationErrorInfo(errorHeight) - determineLowerBound(errorHeight).let { lowerBound -> - Twig.debug { "Handling chain error at $errorHeight by rewinding to block $lowerBound" } - onChainErrorListener?.invoke(errorHeight, lowerBound) - rewindToNearestHeight(lowerBound) + errorHeight?.let { + determineLowerBound(errorHeight).let { lowerBound -> + Twig.debug { "Handling chain error at $errorHeight by rewinding to block $lowerBound" } + onChainErrorListener?.invoke(errorHeight, lowerBound) + rewindToNearestHeight(lowerBound) + } } } @@ -1813,20 +1842,26 @@ class CompactBlockProcessor internal constructor( /** * Rewind back at least two weeks worth of blocks. */ - suspend fun quickRewind() { - val height = repository.lastScannedHeight() + suspend fun quickRewind(): Boolean { + val height = when (val result = getMaxScannedHeight(backend)) { + is GetMaxScannedHeightResult.Success -> result.height + else -> return false + } val blocksPer14Days = 14.days.inWholeMilliseconds / ZcashSdk.BLOCK_INTERVAL_MILLIS.toInt() val twoWeeksBack = BlockHeight.new( network, (height.value - blocksPer14Days).coerceAtLeast(lowerBoundHeight.value) ) - rewindToNearestHeight(twoWeeksBack) + return rewindToNearestHeight(twoWeeksBack) } @Suppress("LongMethod") - suspend fun rewindToNearestHeight(height: BlockHeight) { + suspend fun rewindToNearestHeight(height: BlockHeight): Boolean { processingMutex.withLockLogged("rewindToHeight") { - val lastLocalBlock = repository.lastScannedHeight() + val lastLocalBlock = when (val result = getMaxScannedHeight(backend)) { + is GetMaxScannedHeightResult.Success -> result.height + else -> return false + } val targetHeight = getNearestRewindHeight(height) Twig.debug { @@ -1854,40 +1889,37 @@ class CompactBlockProcessor internal constructor( } } } + return true } /** insightful function for debugging these critical errors */ - private suspend fun printValidationErrorInfo(errorHeight: BlockHeight, count: Int = 11) { + private suspend fun printValidationErrorInfo(errorHeight: BlockHeight?, count: Int = 11) { // Note: blocks are public information so it's okay to print them but, still, let's not unless we're // debugging something if (!BuildConfig.DEBUG) { return } - var errorInfo = fetchValidationErrorInfo(errorHeight) - Twig.debug { "validation failed at block ${errorInfo.errorHeight} with hash: ${errorInfo.hash}" } + if (errorHeight == null) { + Twig.debug { "Validation failed at unspecified block height" } + return + } - errorInfo = fetchValidationErrorInfo(errorHeight + 1) - Twig.debug { "the next block is ${errorInfo.errorHeight} with hash: ${errorInfo.hash}" } + var errorInfo = ValidationErrorInfo(errorHeight) + Twig.debug { "Validation failed at block ${errorInfo.errorHeight}" } + + errorInfo = ValidationErrorInfo(errorHeight + 1) + Twig.debug { "The next block is ${errorInfo.errorHeight}" } Twig.debug { "=================== BLOCKS [$errorHeight..${errorHeight.value + count - 1}]: START ========" } repeat(count) { i -> val height = errorHeight + i val block = downloader.compactBlockRepository.findCompactBlock(height) - // sometimes the initial block was inserted via checkpoint and will not appear in the cache. We can get - // the hash another way. - val checkedHash = block?.hash ?: repository.findBlockHash(height) - Twig.debug { "block: $height\thash=${checkedHash?.toHexReversed()}" } + Twig.debug { "block: $height\thash=${block?.hash?.toHexReversed()}" } } Twig.debug { "=================== BLOCKS [$errorHeight..${errorHeight.value + count - 1}]: END ========" } } - private suspend fun fetchValidationErrorInfo(errorHeight: BlockHeight): ValidationErrorInfo { - val hash = repository.findBlockHash(errorHeight + 1)?.toHexReversed() - - return ValidationErrorInfo(errorHeight, hash) - } - /** * Called for every noteworthy error. * @@ -2045,8 +2077,7 @@ class CompactBlockProcessor internal constructor( ) data class ValidationErrorInfo( - val errorHeight: BlockHeight, - val hash: String? + val errorHeight: BlockHeight ) // diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/GetMaxScannedHeightResult.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/GetMaxScannedHeightResult.kt new file mode 100644 index 00000000..2eba89b3 --- /dev/null +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/GetMaxScannedHeightResult.kt @@ -0,0 +1,12 @@ +package cash.z.ecc.android.sdk.block.processor.model + +import cash.z.ecc.android.sdk.model.BlockHeight + +/** + * Internal class for sharing get max scanned height action result. + */ +internal sealed class GetMaxScannedHeightResult { + data class Success(val height: BlockHeight) : GetMaxScannedHeightResult() + data object None : GetMaxScannedHeightResult() + data class Failure(val exception: Throwable) : GetMaxScannedHeightResult() +} diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/SyncingResult.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/SyncingResult.kt index 7fa47866..be8046ae 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/SyncingResult.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/SyncingResult.kt @@ -17,7 +17,7 @@ internal sealed class SyncingResult { override fun toString() = "${this::class.java.simpleName} with ${downloadedBlocks?.size ?: "none"} blocks" } interface Failure { - val failedAtHeight: BlockHeight + val failedAtHeight: BlockHeight? val exception: CompactBlockProcessorException fun toBlockProcessingResult(): CompactBlockProcessor.BlockProcessingResult = CompactBlockProcessor.BlockProcessingResult.SyncFailure( @@ -36,7 +36,7 @@ internal sealed class SyncingResult { ) : Failure, SyncingResult() object DeleteSuccess : SyncingResult() data class DeleteFailed( - override val failedAtHeight: BlockHeight, + override val failedAtHeight: BlockHeight?, override val exception: CompactBlockProcessorException ) : Failure, SyncingResult() object EnhanceSuccess : SyncingResult() diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/exception/Exceptions.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/exception/Exceptions.kt index 4e806eea..2630a669 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/exception/Exceptions.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/exception/Exceptions.kt @@ -80,10 +80,11 @@ sealed class CompactBlockProcessorException(message: String, cause: Throwable? = null ) class FailedReorgRepair(message: String) : CompactBlockProcessorException(message) - object Uninitialized : CompactBlockProcessorException( + class Uninitialized(cause: Throwable? = null) : CompactBlockProcessorException( "Cannot process blocks because the wallet has not been" + " initialized. Verify that the seed phrase was properly created or imported. If so, then this problem" + - " can be fixed by re-importing the wallet." + " can be fixed by re-importing the wallet.", + cause ) object NoAccount : CompactBlockProcessorException( "Attempting to scan without an account. This is probably a setup error or a race condition." @@ -294,7 +295,7 @@ sealed class TransactionEncoderException( " with id $transactionId, does not have any raw data. This is a scenario where the wallet should have " + "thrown an exception but failed to do so." ) - class IncompleteScanException(lastScannedHeight: BlockHeight) : TransactionEncoderException( + class IncompleteScanException(lastScannedHeight: BlockHeight?) : TransactionEncoderException( "Cannot" + " create spending transaction because scanning is incomplete. We must scan up to the" + " latest height to know which consensus rules to apply. However, the last scanned" + diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/BlockTable.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/BlockTable.kt deleted file mode 100644 index 9f04bb33..00000000 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/BlockTable.kt +++ /dev/null @@ -1,88 +0,0 @@ -package cash.z.ecc.android.sdk.internal.db.derived - -import androidx.sqlite.db.SupportSQLiteDatabase -import cash.z.ecc.android.sdk.internal.db.queryAndMap -import cash.z.ecc.android.sdk.model.BlockHeight -import cash.z.ecc.android.sdk.model.ZcashNetwork -import kotlinx.coroutines.flow.first -import kotlinx.coroutines.flow.firstOrNull -import java.util.Locale - -internal class BlockTable(private val zcashNetwork: ZcashNetwork, private val sqliteDatabase: SupportSQLiteDatabase) { - companion object { - - private val SELECTION_MIN_HEIGHT = arrayOf( - String.format( - Locale.ROOT, - "MIN(%s)", // $NON-NLS - BlockTableDefinition.COLUMN_LONG_HEIGHT - ) - ) - - private val SELECTION_MAX_HEIGHT = arrayOf( - String.format( - Locale.ROOT, - "MAX(%s)", // $NON-NLS - BlockTableDefinition.COLUMN_LONG_HEIGHT - ) - ) - - private val SELECTION_BLOCK_HEIGHT = String.format( - Locale.ROOT, - "%s = ?", // $NON-NLS - BlockTableDefinition.COLUMN_LONG_HEIGHT - ) - - private val PROJECTION_COUNT = arrayOf("COUNT(*)") // $NON-NLS - - private val PROJECTION_HASH = arrayOf(BlockTableDefinition.COLUMN_BLOB_HASH) - } - - suspend fun count() = sqliteDatabase.queryAndMap( - BlockTableDefinition.TABLE_NAME, - columns = PROJECTION_COUNT, - cursorParser = { it.getLong(0) } - ).first() - - suspend fun firstScannedHeight(): BlockHeight { - // Note that we assume the Rust layer will add the birthday height as the first block - val heightLong = - sqliteDatabase.queryAndMap( - table = BlockTableDefinition.TABLE_NAME, - columns = SELECTION_MIN_HEIGHT, - cursorParser = { it.getLong(0) } - ).first() - - return BlockHeight.new(zcashNetwork, heightLong) - } - - suspend fun lastScannedHeight(): BlockHeight { - // Note that we assume the Rust layer will add the birthday height as the first block - val heightLong = - sqliteDatabase.queryAndMap( - table = BlockTableDefinition.TABLE_NAME, - columns = SELECTION_MAX_HEIGHT, - cursorParser = { it.getLong(0) } - ).first() - - return BlockHeight.new(zcashNetwork, heightLong) - } - - suspend fun findBlockHash(blockHeight: BlockHeight): ByteArray? { - return sqliteDatabase.queryAndMap( - table = BlockTableDefinition.TABLE_NAME, - columns = PROJECTION_HASH, - selection = SELECTION_BLOCK_HEIGHT, - selectionArgs = arrayOf(blockHeight.value), - cursorParser = { it.getBlob(0) } - ).firstOrNull() - } -} - -internal object BlockTableDefinition { - const val TABLE_NAME = "blocks" // $NON-NLS - - const val COLUMN_LONG_HEIGHT = "height" // $NON-NLS - - const val COLUMN_BLOB_HASH = "hash" // $NON-NLS -} diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/DbDerivedDataRepository.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/DbDerivedDataRepository.kt index 04ce09e8..9d0e7710 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/DbDerivedDataRepository.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/DbDerivedDataRepository.kt @@ -18,22 +18,10 @@ internal class DbDerivedDataRepository( ) : DerivedDataRepository { private val invalidatingFlow = MutableStateFlow(UUID.randomUUID()) - override suspend fun lastScannedHeight(): BlockHeight { - return derivedDataDb.blockTable.lastScannedHeight() - } - override suspend fun firstUnenhancedHeight(): BlockHeight? { return derivedDataDb.allTransactionView.firstUnenhancedHeight() } - override suspend fun firstScannedHeight(): BlockHeight { - return derivedDataDb.blockTable.firstScannedHeight() - } - - override suspend fun isInitialized(): Boolean { - return derivedDataDb.blockTable.count() > 0 - } - override suspend fun findEncodedTransactionByTxId(txId: FirstClassByteArray): EncodedTransaction? { return derivedDataDb.transactionTable.findEncodedTransactionByTxId(txId) } @@ -49,8 +37,6 @@ internal class DbDerivedDataRepository( override suspend fun findMatchingTransactionId(rawTransactionId: ByteArray) = derivedDataDb.transactionTable .findDatabaseId(rawTransactionId) - override suspend fun findBlockHash(height: BlockHeight) = derivedDataDb.blockTable.findBlockHash(height) - override suspend fun getTransactionCount() = derivedDataDb.transactionTable.count() override fun invalidate() { diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/DerivedDataDb.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/DerivedDataDb.kt index 8f16167c..0948a8c9 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/DerivedDataDb.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/DerivedDataDb.kt @@ -2,6 +2,7 @@ package cash.z.ecc.android.sdk.internal.db.derived import android.content.Context import androidx.sqlite.db.SupportSQLiteDatabase +import cash.z.ecc.android.sdk.exception.CompactBlockProcessorException import cash.z.ecc.android.sdk.internal.NoBackupContextWrapper import cash.z.ecc.android.sdk.internal.Twig import cash.z.ecc.android.sdk.internal.TypesafeBackend @@ -19,8 +20,6 @@ internal class DerivedDataDb private constructor( ) { val accountTable = AccountTable(sqliteDatabase) - val blockTable = BlockTable(zcashNetwork, sqliteDatabase) - val transactionTable = TransactionTable(zcashNetwork, sqliteDatabase) val allTransactionView = AllTransactionView(zcashNetwork, sqliteDatabase) @@ -49,7 +48,14 @@ internal class DerivedDataDb private constructor( numberOfAccounts: Int, recoverUntil: BlockHeight? ): DerivedDataDb { - backend.initDataDb(seed) + runCatching { + val result = backend.initDataDb(seed) + if (result < 0) { + throw CompactBlockProcessorException.Uninitialized() + } + }.onFailure { + throw CompactBlockProcessorException.Uninitialized(it) + } val database = ReadOnlySupportSqliteOpenHelper.openExistingDatabaseAsReadOnly( NoBackupContextWrapper( diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanProgress.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanProgress.kt index b56004c2..afb02478 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanProgress.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanProgress.kt @@ -4,7 +4,7 @@ internal data class ScanProgress( val numerator: Long, val denominator: Long ) { - override fun toString() = "ScanProgress($numerator / $denominator)" + override fun toString() = "ScanProgress($numerator/$denominator) -> ${numerator / (denominator.toFloat())}" companion object { fun new(jni: JniScanProgress): ScanProgress { diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/repository/DerivedDataRepository.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/repository/DerivedDataRepository.kt index ba802ad0..b0821771 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/repository/DerivedDataRepository.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/repository/DerivedDataRepository.kt @@ -13,13 +13,6 @@ import kotlinx.coroutines.flow.Flow @Suppress("TooManyFunctions") internal interface DerivedDataRepository { - /** - * The last height scanned by this repository. - * - * @return the last height scanned by this repository. - */ - suspend fun lastScannedHeight(): BlockHeight - /** * The height of the first transaction that hasn't been enhanced yet. * @@ -28,18 +21,6 @@ internal interface DerivedDataRepository { */ suspend fun firstUnenhancedHeight(): BlockHeight? - /** - * The height of the first block in this repository. This is typically the checkpoint that was - * used to initialize this wallet. If we overwrite this block, it breaks our ability to spend - * funds. - */ - suspend fun firstScannedHeight(): BlockHeight - - /** - * @return true when this repository has been initialized and seeded with the initial checkpoint. - */ - suspend fun isInitialized(): Boolean - /** * Find the encoded transaction associated with the given id. * @@ -76,11 +57,6 @@ internal interface DerivedDataRepository { suspend fun findMatchingTransactionId(rawTransactionId: ByteArray): Long? - // TODO [#681]: begin converting these into Data Access API. For now, just collect the desired - // operations and iterate/refactor, later - // TODO [#681]: https://github.com/zcash/zcash-android-wallet-sdk/issues/681 - suspend fun findBlockHash(height: BlockHeight): ByteArray? - suspend fun getTransactionCount(): Long /** diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/transaction/TransactionEncoderImpl.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/transaction/TransactionEncoderImpl.kt index 7630551c..8a4c626e 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/transaction/TransactionEncoderImpl.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/transaction/TransactionEncoderImpl.kt @@ -98,8 +98,8 @@ internal class TransactionEncoderImpl( backend.isValidUnifiedAddr(address) override suspend fun getConsensusBranchId(): Long { - val height = repository.lastScannedHeight() - if (height < backend.network.saplingActivationHeight) { + val height = backend.getMaxScannedHeight() + if (height == null || height < backend.network.saplingActivationHeight) { throw TransactionEncoderException.IncompleteScanException(height) } return backend.getBranchIdForHeight(height) diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/model/TransactionOverview.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/model/TransactionOverview.kt index 0a783572..6bac5fe6 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/model/TransactionOverview.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/model/TransactionOverview.kt @@ -33,7 +33,7 @@ data class TransactionOverview internal constructor( companion object { internal fun new( dbTransactionOverview: DbTransactionOverview, - latestBlockHeight: BlockHeight + latestBlockHeight: BlockHeight? ): TransactionOverview { return TransactionOverview( dbTransactionOverview.id, @@ -69,11 +69,13 @@ enum class TransactionState { private const val MIN_CONFIRMATIONS = 10 internal fun new( - latestBlockHeight: BlockHeight, + latestBlockHeight: BlockHeight?, minedHeight: BlockHeight?, expiryHeight: BlockHeight? ): TransactionState { - return if (minedHeight != null && (latestBlockHeight.value - minedHeight.value) >= MIN_CONFIRMATIONS) { + return if (latestBlockHeight == null) { + Pending + } else if (minedHeight != null && (latestBlockHeight.value - minedHeight.value) >= MIN_CONFIRMATIONS) { Confirmed } else if (minedHeight != null && (latestBlockHeight.value - minedHeight.value) < MIN_CONFIRMATIONS) { Pending From 59ab627c8267078f4c3995623de586bc920b0c1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Honza=20Rychnovsk=C3=BD?= Date: Thu, 7 Sep 2023 19:18:45 +0200 Subject: [PATCH 44/76] [#1218] Adopt new getScanProgress API --- .../block/processor/CompactBlockProcessor.kt | 58 ++++++++++++++++++- .../processor/model/GetScanProgressResult.kt | 17 ++++++ 2 files changed, 73 insertions(+), 2 deletions(-) create mode 100644 sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/GetScanProgressResult.kt diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessor.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessor.kt index e301663b..8f23e946 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessor.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessor.kt @@ -5,6 +5,7 @@ import cash.z.ecc.android.sdk.BuildConfig import cash.z.ecc.android.sdk.annotation.OpenForTesting import cash.z.ecc.android.sdk.block.processor.model.BatchSyncProgress import cash.z.ecc.android.sdk.block.processor.model.GetMaxScannedHeightResult +import cash.z.ecc.android.sdk.block.processor.model.GetScanProgressResult import cash.z.ecc.android.sdk.block.processor.model.GetSubtreeRootsResult import cash.z.ecc.android.sdk.block.processor.model.PutSaplingSubtreeRootsResult import cash.z.ecc.android.sdk.block.processor.model.SbSPreparationResult @@ -454,7 +455,20 @@ class CompactBlockProcessor internal constructor( // be over the precomputed all-batch count in case of inter-syncing failure. lastBatchOrder = min(rangeSyncProgress.overallOrder, allBatchCountLocal) - setProgress(PercentDecimal(lastBatchOrder / allBatchCountLocal.toFloat())) + // TODO [#1219]: Remove calculated sync progress leftovers + // TODO [#1219]: https://github.com/zcash/zcash-android-wallet-sdk/issues/1219 + Twig.info { + "Progress calculated: " + + "${PercentDecimal(lastBatchOrder / allBatchCountLocal.toFloat()).decimal}" + } + when (val result = getScanProgress(backend)) { + is GetScanProgressResult.Success -> { + val resultProgress = result.toPercentDecimal() + Twig.info { "Progress from rust: ${resultProgress.decimal}" } + setProgress(resultProgress) + } + else -> { /* Do not report the progress in case of any error */ } + } checkAllBalances() when (rangeSyncProgress.resultState) { @@ -542,7 +556,20 @@ class CompactBlockProcessor internal constructor( // be over the precomputed all-batch count in case of inter-syncing failure. lastBatchOrder = min(rangeSyncProgress.overallOrder, allBatchCountLocal) - setProgress(PercentDecimal(lastBatchOrder / allBatchCountLocal.toFloat())) + // TODO [#1219]: Remove calculated sync progress leftovers + // TODO [#1219]: https://github.com/zcash/zcash-android-wallet-sdk/issues/1219 + Twig.info { + "Progress calculated: " + + "${PercentDecimal(lastBatchOrder / allBatchCountLocal.toFloat()).decimal}" + } + when (val result = getScanProgress(backend)) { + is GetScanProgressResult.Success -> { + val resultProgress = result.toPercentDecimal() + Twig.info { "Progress from rust: ${resultProgress.decimal}" } + setProgress(resultProgress) + } + else -> { /* Do not report the progress in case of any error */ } + } checkAllBalances() when (rangeSyncProgress.resultState) { @@ -1237,6 +1264,33 @@ class CompactBlockProcessor internal constructor( } } + /** + * Get the current block scanning progress. + * + * @return the last scanning progress calculated by the Rust layer and wrapped in [GetScanProgressResult] + */ + @VisibleForTesting + internal suspend fun getScanProgress(backend: TypesafeBackend): GetScanProgressResult { + return runCatching { + backend.getScanProgress() + }.onSuccess { + Twig.verbose { "Successfully called getScanProgress with result: $it" } + }.onFailure { + Twig.error { "Failed to call getScanProgress with result: $it" } + }.fold( + onSuccess = { + if (it == null) { + GetScanProgressResult.None + } else { + GetScanProgressResult.Success(it) + } + }, + onFailure = { + GetScanProgressResult.Failure(it) + } + ) + } + /** * Requests, processes and persists all blocks from the given range. * diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/GetScanProgressResult.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/GetScanProgressResult.kt new file mode 100644 index 00000000..3a219087 --- /dev/null +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/GetScanProgressResult.kt @@ -0,0 +1,17 @@ +package cash.z.ecc.android.sdk.block.processor.model + +import cash.z.ecc.android.sdk.internal.model.ScanProgress +import cash.z.ecc.android.sdk.model.PercentDecimal + +/** + * Internal class for sharing get scan progress action result. + */ +internal sealed class GetScanProgressResult { + data class Success(val progress: ScanProgress) : GetScanProgressResult() { + fun toPercentDecimal() = + PercentDecimal(progress.numerator / progress.denominator.toFloat()) + } + + data object None : GetScanProgressResult() + data class Failure(val exception: Throwable) : GetScanProgressResult() +} From 9207dd0b58cdcfe55dba082e8082ce593732948b Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Thu, 7 Sep 2023 18:58:31 +0100 Subject: [PATCH 45/76] Migrate to Rust version with bugfixes This also reverts the prior change to `getBalance` / `getVerifiedBalance` now that the fix for zcash/librustzcash#948 is present. --- backend-lib/Cargo.lock | 16 ++++++++-------- backend-lib/Cargo.toml | 10 +++++----- backend-lib/src/main/rust/lib.rs | 8 ++++---- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/backend-lib/Cargo.lock b/backend-lib/Cargo.lock index 5b476257..39f7b5e2 100644 --- a/backend-lib/Cargo.lock +++ b/backend-lib/Cargo.lock @@ -483,7 +483,7 @@ checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" [[package]] name = "equihash" version = "0.2.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=3d0ec004a275d062487f6fabc4061ee0b4687ff0#3d0ec004a275d062487f6fabc4061ee0b4687ff0" +source = "git+https://github.com/zcash/librustzcash.git?rev=85156e2b7cb5c53c26c786ea7c5c22fe5cbbcb2d#85156e2b7cb5c53c26c786ea7c5c22fe5cbbcb2d" dependencies = [ "blake2b_simd", "byteorder", @@ -513,7 +513,7 @@ dependencies = [ [[package]] name = "f4jumble" version = "0.1.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=3d0ec004a275d062487f6fabc4061ee0b4687ff0#3d0ec004a275d062487f6fabc4061ee0b4687ff0" +source = "git+https://github.com/zcash/librustzcash.git?rev=85156e2b7cb5c53c26c786ea7c5c22fe5cbbcb2d#85156e2b7cb5c53c26c786ea7c5c22fe5cbbcb2d" dependencies = [ "blake2b_simd", ] @@ -2259,7 +2259,7 @@ dependencies = [ [[package]] name = "zcash_address" version = "0.3.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=3d0ec004a275d062487f6fabc4061ee0b4687ff0#3d0ec004a275d062487f6fabc4061ee0b4687ff0" +source = "git+https://github.com/zcash/librustzcash.git?rev=85156e2b7cb5c53c26c786ea7c5c22fe5cbbcb2d#85156e2b7cb5c53c26c786ea7c5c22fe5cbbcb2d" dependencies = [ "bech32", "bs58", @@ -2270,7 +2270,7 @@ dependencies = [ [[package]] name = "zcash_client_backend" version = "0.9.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=3d0ec004a275d062487f6fabc4061ee0b4687ff0#3d0ec004a275d062487f6fabc4061ee0b4687ff0" +source = "git+https://github.com/zcash/librustzcash.git?rev=85156e2b7cb5c53c26c786ea7c5c22fe5cbbcb2d#85156e2b7cb5c53c26c786ea7c5c22fe5cbbcb2d" dependencies = [ "base64", "bech32", @@ -2304,7 +2304,7 @@ dependencies = [ [[package]] name = "zcash_client_sqlite" version = "0.7.1" -source = "git+https://github.com/zcash/librustzcash.git?rev=3d0ec004a275d062487f6fabc4061ee0b4687ff0#3d0ec004a275d062487f6fabc4061ee0b4687ff0" +source = "git+https://github.com/zcash/librustzcash.git?rev=85156e2b7cb5c53c26c786ea7c5c22fe5cbbcb2d#85156e2b7cb5c53c26c786ea7c5c22fe5cbbcb2d" dependencies = [ "bs58", "byteorder", @@ -2330,7 +2330,7 @@ dependencies = [ [[package]] name = "zcash_encoding" version = "0.2.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=3d0ec004a275d062487f6fabc4061ee0b4687ff0#3d0ec004a275d062487f6fabc4061ee0b4687ff0" +source = "git+https://github.com/zcash/librustzcash.git?rev=85156e2b7cb5c53c26c786ea7c5c22fe5cbbcb2d#85156e2b7cb5c53c26c786ea7c5c22fe5cbbcb2d" dependencies = [ "byteorder", "nonempty", @@ -2352,7 +2352,7 @@ dependencies = [ [[package]] name = "zcash_primitives" version = "0.12.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=3d0ec004a275d062487f6fabc4061ee0b4687ff0#3d0ec004a275d062487f6fabc4061ee0b4687ff0" +source = "git+https://github.com/zcash/librustzcash.git?rev=85156e2b7cb5c53c26c786ea7c5c22fe5cbbcb2d#85156e2b7cb5c53c26c786ea7c5c22fe5cbbcb2d" dependencies = [ "aes", "bip0039", @@ -2387,7 +2387,7 @@ dependencies = [ [[package]] name = "zcash_proofs" version = "0.12.1" -source = "git+https://github.com/zcash/librustzcash.git?rev=3d0ec004a275d062487f6fabc4061ee0b4687ff0#3d0ec004a275d062487f6fabc4061ee0b4687ff0" +source = "git+https://github.com/zcash/librustzcash.git?rev=85156e2b7cb5c53c26c786ea7c5c22fe5cbbcb2d#85156e2b7cb5c53c26c786ea7c5c22fe5cbbcb2d" dependencies = [ "bellman", "blake2b_simd", diff --git a/backend-lib/Cargo.toml b/backend-lib/Cargo.toml index 186b7013..679be163 100644 --- a/backend-lib/Cargo.toml +++ b/backend-lib/Cargo.toml @@ -45,11 +45,11 @@ libc = "0.2" incrementalmerkletree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "da97e6c399c5acddedad2c1730dbb7ee55499a2f" } shardtree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "da97e6c399c5acddedad2c1730dbb7ee55499a2f" } orchard = { git = "https://github.com/zcash/orchard.git", rev = "6ef89d5f154de2cf7b7dd87edb8d8c49158beebb" } -zcash_address = { git = "https://github.com/zcash/librustzcash.git", rev = "3d0ec004a275d062487f6fabc4061ee0b4687ff0" } -zcash_client_backend = { git = "https://github.com/zcash/librustzcash.git", rev = "3d0ec004a275d062487f6fabc4061ee0b4687ff0" } -zcash_client_sqlite = { git = "https://github.com/zcash/librustzcash.git", rev = "3d0ec004a275d062487f6fabc4061ee0b4687ff0" } -zcash_primitives = { git = "https://github.com/zcash/librustzcash.git", rev = "3d0ec004a275d062487f6fabc4061ee0b4687ff0" } -zcash_proofs = { git = "https://github.com/zcash/librustzcash.git", rev = "3d0ec004a275d062487f6fabc4061ee0b4687ff0" } +zcash_address = { git = "https://github.com/zcash/librustzcash.git", rev = "85156e2b7cb5c53c26c786ea7c5c22fe5cbbcb2d" } +zcash_client_backend = { git = "https://github.com/zcash/librustzcash.git", rev = "85156e2b7cb5c53c26c786ea7c5c22fe5cbbcb2d" } +zcash_client_sqlite = { git = "https://github.com/zcash/librustzcash.git", rev = "85156e2b7cb5c53c26c786ea7c5c22fe5cbbcb2d" } +zcash_primitives = { git = "https://github.com/zcash/librustzcash.git", rev = "85156e2b7cb5c53c26c786ea7c5c22fe5cbbcb2d" } +zcash_proofs = { git = "https://github.com/zcash/librustzcash.git", rev = "85156e2b7cb5c53c26c786ea7c5c22fe5cbbcb2d" } ## Uncomment this to test librustzcash changes locally #[patch.crates-io] diff --git a/backend-lib/src/main/rust/lib.rs b/backend-lib/src/main/rust/lib.rs index 64701f27..61ed06d7 100644 --- a/backend-lib/src/main/rust/lib.rs +++ b/backend-lib/src/main/rust/lib.rs @@ -661,11 +661,11 @@ pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_ge .get_wallet_summary(0) .map_err(|e| format_err!("Error while fetching balance: {}", e))? { - Ok(wallet_summary + wallet_summary .account_balances() .get(&account) + .ok_or_else(|| format_err!("Unknown account")) .map(|balances| Amount::from(balances.sapling_balance.total()).into()) - .unwrap_or(0)) } else { // `None` means that the caller has not yet called `updateChainTip` on a // brand-new wallet, so we can assume the balance is zero. @@ -770,11 +770,11 @@ pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_ge .get_wallet_summary(ANCHOR_OFFSET_U32) .map_err(|e| format_err!("Error while fetching verified balance: {}", e))? { - Ok(wallet_summary + wallet_summary .account_balances() .get(&account) + .ok_or_else(|| format_err!("Unknown account")) .map(|balances| Amount::from(balances.sapling_balance.spendable_value).into()) - .unwrap_or(0)) } else { // `None` means that the caller has not yet called `updateChainTip` on a // brand-new wallet, so we can assume the balance is zero. From c6032b47bd2f9edcab8aeaaa1c434f24329ec217 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Honza=20Rychnovsk=C3=BD?= Date: Fri, 8 Sep 2023 16:56:02 +0200 Subject: [PATCH 46/76] [#1208][#1215] Pass proper `recoverUntil` - Closes #1208 - CLoses #1215 --- CHANGELOG.md | 2 + .../z/ecc/android/sdk/internal/Derivation.kt | 1 - .../sdk/internal/model/JniScanProgress.kt | 3 + .../android/sdk/darkside/test/TestWallet.kt | 5 +- .../sdk/sample/demoapp/SampleCodeTest.kt | 5 +- .../android/sdk/demoapp/ComposeActivity.kt | 2 +- .../android/sdk/demoapp/SharedViewModel.kt | 3 + .../screen/home/viewmodel/WalletViewModel.kt | 15 +++- .../z/ecc/android/sdk/WalletCoordinator.kt | 3 +- .../android/sdk/model/PersistableWallet.kt | 17 ++++- .../sdk/integration/TestnetIntegrationTest.kt | 5 +- .../sdk/internal/SdkSynchronizerTest.kt | 19 ++++- .../android/sdk/util/BalancePrinterUtil.kt | 5 +- .../ecc/android/sdk/util/DataDbScannerUtil.kt | 5 +- .../cash/z/ecc/android/sdk/util/TestWallet.kt | 5 +- .../cash/z/ecc/android/sdk/Synchronizer.kt | 73 +++++++++++++++---- .../block/processor/CompactBlockProcessor.kt | 5 ++ 17 files changed, 142 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index de2c712c..1258256c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,8 @@ ### Changed - `CompactBlockProcessor.quickRewind()` and `CompactBlockProcessor.rewindToNearestHeight()` now might fail due to internal changes in getting scanned height. Thus, these functions return `Boolean` results. +- `Synchronizer.new()` requires a new `walletInitMode` parameter of type `WalletInitMode`, which describes wallet + initialization mode. See related function and sealed class documentation. ### Fixed - `Synchronizer.getMemos()` now correctly returns a flow of strings for sent and received transactions. Issue **#1154**. diff --git a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/Derivation.kt b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/Derivation.kt index bd9ce2da..3f61d159 100644 --- a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/Derivation.kt +++ b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/Derivation.kt @@ -40,6 +40,5 @@ interface Derivation { companion object { const val DEFAULT_NUMBER_OF_ACCOUNTS = 1 - val DEFAULT_RECOVERY_UNTIL_HEIGHT = null } } diff --git a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniScanProgress.kt b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniScanProgress.kt index 305ccaea..ce4a9c57 100644 --- a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniScanProgress.kt +++ b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniScanProgress.kt @@ -23,5 +23,8 @@ class JniScanProgress( require(numerator.toFloat().div(denominator) >= 0f) { "Result of ${numerator.toFloat()}/$denominator is outside of allowed range" } + require(numerator.toFloat().div(denominator) <= 1f) { + "Result of ${numerator.toFloat()}/$denominator is outside of allowed range" + } } } diff --git a/darkside-test-lib/src/androidTest/java/cash/z/ecc/android/sdk/darkside/test/TestWallet.kt b/darkside-test-lib/src/androidTest/java/cash/z/ecc/android/sdk/darkside/test/TestWallet.kt index 61d9b2dc..6f33bd53 100644 --- a/darkside-test-lib/src/androidTest/java/cash/z/ecc/android/sdk/darkside/test/TestWallet.kt +++ b/darkside-test-lib/src/androidTest/java/cash/z/ecc/android/sdk/darkside/test/TestWallet.kt @@ -5,6 +5,7 @@ import cash.z.ecc.android.bip39.Mnemonics import cash.z.ecc.android.bip39.toSeed import cash.z.ecc.android.sdk.SdkSynchronizer import cash.z.ecc.android.sdk.Synchronizer +import cash.z.ecc.android.sdk.WalletInitMode import cash.z.ecc.android.sdk.internal.Twig import cash.z.ecc.android.sdk.model.Account import cash.z.ecc.android.sdk.model.BlockHeight @@ -65,7 +66,9 @@ class TestWallet( alias, endpoint, seed, - startHeight + startHeight, + // Using existing wallet init mode as simplification for the test + walletInitMode = WalletInitMode.ExistingWallet ) as SdkSynchronizer val available get() = synchronizer.saplingBalances.value?.available diff --git a/demo-app/src/androidTest/java/cash/z/wallet/sdk/sample/demoapp/SampleCodeTest.kt b/demo-app/src/androidTest/java/cash/z/wallet/sdk/sample/demoapp/SampleCodeTest.kt index 9d03e048..304023df 100644 --- a/demo-app/src/androidTest/java/cash/z/wallet/sdk/sample/demoapp/SampleCodeTest.kt +++ b/demo-app/src/androidTest/java/cash/z/wallet/sdk/sample/demoapp/SampleCodeTest.kt @@ -2,6 +2,7 @@ package cash.z.wallet.sdk.sample.demoapp import androidx.test.platform.app.InstrumentationRegistry import cash.z.ecc.android.sdk.Synchronizer +import cash.z.ecc.android.sdk.WalletInitMode import cash.z.ecc.android.sdk.demoapp.util.fromResources import cash.z.ecc.android.sdk.ext.convertZecToZatoshi import cash.z.ecc.android.sdk.ext.toHex @@ -202,7 +203,9 @@ class SampleCodeTest { network, lightWalletEndpoint = LightWalletEndpoint.defaultForNetwork(network), seed = seed, - birthday = null + birthday = null, + // Using existing wallet init mode as simplification for the test + walletInitMode = WalletInitMode.ExistingWallet ) } diff --git a/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/ComposeActivity.kt b/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/ComposeActivity.kt index 4170a2a2..8580655b 100644 --- a/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/ComposeActivity.kt +++ b/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/ComposeActivity.kt @@ -40,7 +40,7 @@ class ComposeActivity : ComponentActivity() { } SecretState.None -> { Seed( - ZcashNetwork.fromResources(applicationContext), + zcashNetwork = ZcashNetwork.fromResources(applicationContext), onExistingWallet = { walletViewModel.persistExistingWallet(it) }, onNewWallet = { walletViewModel.persistNewWallet() } ) diff --git a/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/SharedViewModel.kt b/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/SharedViewModel.kt index f989ffa2..0035c267 100644 --- a/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/SharedViewModel.kt +++ b/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/SharedViewModel.kt @@ -6,6 +6,7 @@ import androidx.lifecycle.viewModelScope import cash.z.ecc.android.bip39.Mnemonics import cash.z.ecc.android.bip39.toSeed import cash.z.ecc.android.sdk.Synchronizer +import cash.z.ecc.android.sdk.WalletInitMode import cash.z.ecc.android.sdk.demoapp.util.fromResources import cash.z.ecc.android.sdk.ext.onFirst import cash.z.ecc.android.sdk.internal.Twig @@ -82,6 +83,8 @@ class SharedViewModel(application: Application) : AndroidViewModel(application) } else { birthdayHeight.value }, + // We use restore mode as this is always initialization with an older seed + walletInitMode = WalletInitMode.RestoreWallet, alias = OLD_UI_SYNCHRONIZER_ALIAS ) diff --git a/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/ui/screen/home/viewmodel/WalletViewModel.kt b/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/ui/screen/home/viewmodel/WalletViewModel.kt index a2966530..dd2c3a3e 100644 --- a/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/ui/screen/home/viewmodel/WalletViewModel.kt +++ b/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/ui/screen/home/viewmodel/WalletViewModel.kt @@ -7,6 +7,7 @@ import cash.z.ecc.android.bip39.Mnemonics import cash.z.ecc.android.bip39.toSeed import cash.z.ecc.android.sdk.Synchronizer import cash.z.ecc.android.sdk.WalletCoordinator +import cash.z.ecc.android.sdk.WalletInitMode import cash.z.ecc.android.sdk.block.processor.CompactBlockProcessor import cash.z.ecc.android.sdk.demoapp.getInstance import cash.z.ecc.android.sdk.demoapp.preference.EncryptedPreferenceKeys @@ -48,7 +49,6 @@ import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withContext import kotlin.time.Duration.Companion.seconds -import kotlin.time.ExperimentalTime // To make this more multiplatform compatible, we need to remove the dependency on Context // for loading the preferences. @@ -101,7 +101,7 @@ class WalletViewModel(application: Application) : AndroidViewModel(application) null ) - @OptIn(ExperimentalCoroutinesApi::class, ExperimentalTime::class) + @OptIn(ExperimentalCoroutinesApi::class) val walletSnapshot: StateFlow = synchronizer .flatMapLatest { if (null == it) { @@ -141,10 +141,11 @@ class WalletViewModel(application: Application) : AndroidViewModel(application) */ fun persistNewWallet() { val application = getApplication() + PersistableWallet.walletInitMode = WalletInitMode.NewWallet viewModelScope.launch { val newWallet = PersistableWallet.new(application, ZcashNetwork.fromResources(application)) - persistExistingWallet(newWallet) + persistWallet(newWallet) } } @@ -153,6 +154,14 @@ class WalletViewModel(application: Application) : AndroidViewModel(application) * to see the side effects. This would be used for a user restoring a wallet from a backup. */ fun persistExistingWallet(persistableWallet: PersistableWallet) { + PersistableWallet.walletInitMode = WalletInitMode.RestoreWallet + persistWallet(persistableWallet) + } + + /** + * Persists a wallet asynchronously. Clients observe [secretState] to see the side effects. + */ + private fun persistWallet(persistableWallet: PersistableWallet) { val application = getApplication() viewModelScope.launch { diff --git a/sdk-incubator-lib/src/main/java/cash/z/ecc/android/sdk/WalletCoordinator.kt b/sdk-incubator-lib/src/main/java/cash/z/ecc/android/sdk/WalletCoordinator.kt index 8f6ea95a..5f1f9e52 100644 --- a/sdk-incubator-lib/src/main/java/cash/z/ecc/android/sdk/WalletCoordinator.kt +++ b/sdk-incubator-lib/src/main/java/cash/z/ecc/android/sdk/WalletCoordinator.kt @@ -79,7 +79,8 @@ class WalletCoordinator( zcashNetwork = persistableWallet.network, lightWalletEndpoint = LightWalletEndpoint.defaultForNetwork(persistableWallet.network), birthday = persistableWallet.birthday, - seed = persistableWallet.seedPhrase.toByteArray() + seed = persistableWallet.seedPhrase.toByteArray(), + walletInitMode = PersistableWallet.walletInitMode, ) trySend(InternalSynchronizerStatus.Available(closeableSynchronizer)) diff --git a/sdk-incubator-lib/src/main/java/cash/z/ecc/android/sdk/model/PersistableWallet.kt b/sdk-incubator-lib/src/main/java/cash/z/ecc/android/sdk/model/PersistableWallet.kt index 9840e9ed..370431c7 100644 --- a/sdk-incubator-lib/src/main/java/cash/z/ecc/android/sdk/model/PersistableWallet.kt +++ b/sdk-incubator-lib/src/main/java/cash/z/ecc/android/sdk/model/PersistableWallet.kt @@ -3,6 +3,7 @@ package cash.z.ecc.android.sdk.model import android.app.Application import cash.z.ecc.android.bip39.Mnemonics import cash.z.ecc.android.bip39.toEntropy +import cash.z.ecc.android.sdk.WalletInitMode import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext import org.json.JSONObject @@ -41,6 +42,10 @@ data class PersistableWallet( internal const val KEY_BIRTHDAY = "birthday" internal const val KEY_SEED_PHRASE = "seed_phrase" + // Note: This is not the ideal way to hold such a value. But we also want to avoid persisting the wallet + // initialization mode with the persistable wallet. + var walletInitMode: WalletInitMode = WalletInitMode.ExistingWallet + fun from(jsonObject: JSONObject): PersistableWallet { when (val version = jsonObject.getInt(KEY_VERSION)) { VERSION_1 -> { @@ -56,7 +61,11 @@ data class PersistableWallet( } val seedPhrase = jsonObject.getString(KEY_SEED_PHRASE) - return PersistableWallet(network, birthday, SeedPhrase.new(seedPhrase)) + return PersistableWallet( + network = network, + birthday = birthday, + seedPhrase = SeedPhrase.new(seedPhrase) + ) } else -> { throw IllegalArgumentException("Unsupported version $version") @@ -72,7 +81,11 @@ data class PersistableWallet( val seedPhrase = newSeedPhrase() - return PersistableWallet(zcashNetwork, birthday, seedPhrase) + return PersistableWallet( + zcashNetwork, + birthday, + seedPhrase + ) } } } diff --git a/sdk-lib/src/androidTest/java/cash/z/ecc/android/sdk/integration/TestnetIntegrationTest.kt b/sdk-lib/src/androidTest/java/cash/z/ecc/android/sdk/integration/TestnetIntegrationTest.kt index 02224d19..05aadfb1 100644 --- a/sdk-lib/src/androidTest/java/cash/z/ecc/android/sdk/integration/TestnetIntegrationTest.kt +++ b/sdk-lib/src/androidTest/java/cash/z/ecc/android/sdk/integration/TestnetIntegrationTest.kt @@ -4,6 +4,7 @@ import androidx.test.filters.LargeTest import androidx.test.platform.app.InstrumentationRegistry import cash.z.ecc.android.sdk.Synchronizer import cash.z.ecc.android.sdk.Synchronizer.Status.SYNCED +import cash.z.ecc.android.sdk.WalletInitMode import cash.z.ecc.android.sdk.ext.ZcashSdk import cash.z.ecc.android.sdk.ext.onFirst import cash.z.ecc.android.sdk.internal.Twig @@ -141,7 +142,9 @@ class TestnetIntegrationTest : ScopedTest() { lightWalletEndpoint = lightWalletEndpoint, seed = seed, - birthday = BlockHeight.new(ZcashNetwork.Testnet, birthdayHeight) + birthday = BlockHeight.new(ZcashNetwork.Testnet, birthdayHeight), + // Using existing wallet init mode as simplification for the test + walletInitMode = WalletInitMode.ExistingWallet ) } } diff --git a/sdk-lib/src/androidTest/java/cash/z/ecc/android/sdk/internal/SdkSynchronizerTest.kt b/sdk-lib/src/androidTest/java/cash/z/ecc/android/sdk/internal/SdkSynchronizerTest.kt index 7ace2e5e..d41fcb67 100644 --- a/sdk-lib/src/androidTest/java/cash/z/ecc/android/sdk/internal/SdkSynchronizerTest.kt +++ b/sdk-lib/src/androidTest/java/cash/z/ecc/android/sdk/internal/SdkSynchronizerTest.kt @@ -4,6 +4,7 @@ import androidx.test.filters.SmallTest import androidx.test.platform.app.InstrumentationRegistry import cash.z.ecc.android.bip39.Mnemonics import cash.z.ecc.android.sdk.Synchronizer +import cash.z.ecc.android.sdk.WalletInitMode import cash.z.ecc.android.sdk.fixture.WalletFixture import cash.z.ecc.android.sdk.model.ZcashNetwork import cash.z.ecc.android.sdk.model.defaultForNetwork @@ -29,7 +30,9 @@ class SdkSynchronizerTest { alias, LightWalletEndpoint.defaultForNetwork(ZcashNetwork.Mainnet), Mnemonics.MnemonicCode(WalletFixture.SEED_PHRASE).toEntropy(), - birthday = null + birthday = null, + // Using existing wallet init mode as simplification for the test + walletInitMode = WalletInitMode.ExistingWallet ).use { assertFailsWith { Synchronizer.new( @@ -38,7 +41,9 @@ class SdkSynchronizerTest { alias, LightWalletEndpoint.defaultForNetwork(ZcashNetwork.Mainnet), Mnemonics.MnemonicCode(WalletFixture.SEED_PHRASE).toEntropy(), - birthday = null + birthday = null, + // Using existing wallet init mode as simplification for the test + walletInitMode = WalletInitMode.ExistingWallet ) } } @@ -51,6 +56,8 @@ class SdkSynchronizerTest { // Random alias so that repeated invocations of this test will have a clean starting state val alias = UUID.randomUUID().toString() + // TODO [#1094]: Consider fake SDK sync related components + // TODO [#1094]: https://github.com/zcash/zcash-android-wallet-sdk/issues/1094 // In the future, inject fake networking component so that it doesn't require hitting the network Synchronizer.new( InstrumentationRegistry.getInstrumentation().context, @@ -58,7 +65,9 @@ class SdkSynchronizerTest { alias, LightWalletEndpoint.defaultForNetwork(ZcashNetwork.Mainnet), Mnemonics.MnemonicCode(WalletFixture.SEED_PHRASE).toEntropy(), - birthday = null + birthday = null, + // Using existing wallet init mode as simplification for the test + walletInitMode = WalletInitMode.ExistingWallet ).use {} // Second instance should succeed because first one was closed @@ -68,7 +77,9 @@ class SdkSynchronizerTest { alias, LightWalletEndpoint.defaultForNetwork(ZcashNetwork.Mainnet), Mnemonics.MnemonicCode(WalletFixture.SEED_PHRASE).toEntropy(), - birthday = null + birthday = null, + // Using existing wallet init mode as simplification for the test + walletInitMode = WalletInitMode.ExistingWallet ).use {} } } diff --git a/sdk-lib/src/androidTest/java/cash/z/ecc/android/sdk/util/BalancePrinterUtil.kt b/sdk-lib/src/androidTest/java/cash/z/ecc/android/sdk/util/BalancePrinterUtil.kt index 8de49b6b..59805a03 100644 --- a/sdk-lib/src/androidTest/java/cash/z/ecc/android/sdk/util/BalancePrinterUtil.kt +++ b/sdk-lib/src/androidTest/java/cash/z/ecc/android/sdk/util/BalancePrinterUtil.kt @@ -3,6 +3,7 @@ package cash.z.ecc.android.sdk.util import androidx.test.platform.app.InstrumentationRegistry import cash.z.ecc.android.sdk.CloseableSynchronizer import cash.z.ecc.android.sdk.Synchronizer +import cash.z.ecc.android.sdk.WalletInitMode import cash.z.ecc.android.sdk.internal.Twig import cash.z.ecc.android.sdk.internal.ext.deleteSuspend import cash.z.ecc.android.sdk.internal.model.Checkpoint @@ -96,7 +97,9 @@ class BalancePrinterUtil { lightWalletEndpoint = LightWalletEndpoint .defaultForNetwork(network), seed = seed, - birthday = birthdayHeight + birthday = birthdayHeight, + // Using existing wallet init mode as simplification for the test + walletInitMode = WalletInitMode.ExistingWallet ) // deleteDb(dataDbPath) diff --git a/sdk-lib/src/androidTest/java/cash/z/ecc/android/sdk/util/DataDbScannerUtil.kt b/sdk-lib/src/androidTest/java/cash/z/ecc/android/sdk/util/DataDbScannerUtil.kt index ec510f49..f570a852 100644 --- a/sdk-lib/src/androidTest/java/cash/z/ecc/android/sdk/util/DataDbScannerUtil.kt +++ b/sdk-lib/src/androidTest/java/cash/z/ecc/android/sdk/util/DataDbScannerUtil.kt @@ -4,6 +4,7 @@ import androidx.test.platform.app.InstrumentationRegistry import cash.z.ecc.android.sdk.CloseableSynchronizer import cash.z.ecc.android.sdk.SdkSynchronizer import cash.z.ecc.android.sdk.Synchronizer +import cash.z.ecc.android.sdk.WalletInitMode import cash.z.ecc.android.sdk.model.BlockHeight import cash.z.ecc.android.sdk.model.ZcashNetwork import cash.z.ecc.android.sdk.model.defaultForNetwork @@ -72,7 +73,9 @@ class DataDbScannerUtil { birthday = BlockHeight.new( ZcashNetwork.Mainnet, birthdayHeight - ) + ), + // Using existing wallet init mode as simplification for the test + walletInitMode = WalletInitMode.ExistingWallet ) println("sync!") diff --git a/sdk-lib/src/androidTest/java/cash/z/ecc/android/sdk/util/TestWallet.kt b/sdk-lib/src/androidTest/java/cash/z/ecc/android/sdk/util/TestWallet.kt index 522d0af0..919388da 100644 --- a/sdk-lib/src/androidTest/java/cash/z/ecc/android/sdk/util/TestWallet.kt +++ b/sdk-lib/src/androidTest/java/cash/z/ecc/android/sdk/util/TestWallet.kt @@ -5,6 +5,7 @@ import cash.z.ecc.android.bip39.Mnemonics import cash.z.ecc.android.bip39.toSeed import cash.z.ecc.android.sdk.SdkSynchronizer import cash.z.ecc.android.sdk.Synchronizer +import cash.z.ecc.android.sdk.WalletInitMode import cash.z.ecc.android.sdk.internal.Twig import cash.z.ecc.android.sdk.internal.deriveUnifiedSpendingKey import cash.z.ecc.android.sdk.internal.jni.RustDerivationTool @@ -66,7 +67,9 @@ class TestWallet( alias, lightWalletEndpoint = endpoint, seed = seed, - startHeight + startHeight, + // Using existing wallet init mode as simplification for the test + walletInitMode = WalletInitMode.ExistingWallet ) as SdkSynchronizer val available get() = synchronizer.saplingBalances.value?.available diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/Synchronizer.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/Synchronizer.kt index 1b0ec62e..ee37fdf6 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/Synchronizer.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/Synchronizer.kt @@ -5,7 +5,9 @@ import cash.z.ecc.android.sdk.block.processor.CompactBlockProcessor import cash.z.ecc.android.sdk.ext.ZcashSdk import cash.z.ecc.android.sdk.internal.Derivation import cash.z.ecc.android.sdk.internal.SaplingParamTool +import cash.z.ecc.android.sdk.internal.Twig import cash.z.ecc.android.sdk.internal.db.DatabaseCoordinator +import cash.z.ecc.android.sdk.internal.model.ext.toBlockHeight import cash.z.ecc.android.sdk.model.Account import cash.z.ecc.android.sdk.model.BlockHeight import cash.z.ecc.android.sdk.model.PercentDecimal @@ -19,6 +21,7 @@ import cash.z.ecc.android.sdk.tool.CheckpointTool import cash.z.ecc.android.sdk.type.AddressType import cash.z.ecc.android.sdk.type.ConsensusMatchType import co.electriccoin.lightwallet.client.model.LightWalletEndpoint +import co.electriccoin.lightwallet.client.model.Response import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.runBlocking @@ -425,6 +428,12 @@ interface Synchronizer { * to create the wallet. If that value is unknown, null is acceptable but will result in longer * sync times. After sync completes, the birthday can be determined from [Synchronizer.latestBirthdayHeight]. * + * @param walletInitMode a required parameter with one of [WalletInitMode] values. Use + * [WalletInitMode.NewWallet] when starting synchronizer for a newly created wallet. Or use + * [WalletInitMode.RestoreWallet] when restoring an existing wallet that was created at some point in the + * past. Or use the last [WalletInitMode.ExistingWallet] type for a wallet which is already initialized + * and needs follow-up block synchronization. + * * @throws InitializerException.SeedRequired Indicates clients need to call this method again, providing the * seed bytes. * @@ -443,7 +452,8 @@ interface Synchronizer { alias: String = ZcashSdk.DEFAULT_ALIAS, lightWalletEndpoint: LightWalletEndpoint, seed: ByteArray?, - birthday: BlockHeight? + birthday: BlockHeight?, + walletInitMode: WalletInitMode ): CloseableSynchronizer { val applicationContext = context.applicationContext @@ -472,20 +482,39 @@ interface Synchronizer { DefaultSynchronizerFactory .defaultCompactBlockRepository(coordinator.fsBlockDbRoot(zcashNetwork, alias), backend) + val service = DefaultSynchronizerFactory.defaultService(applicationContext, lightWalletEndpoint) + val downloader = DefaultSynchronizerFactory.defaultDownloader(service, blockStore) + + val chainTip = when (walletInitMode) { + is WalletInitMode.RestoreWallet -> { + when (val response = downloader.getLatestBlockHeight()) { + is Response.Success -> { + Twig.info { "Chain tip for recovery until param fetched: ${response.result.value}" } + runCatching { response.result.toBlockHeight(zcashNetwork) }.getOrNull() + } + is Response.Failure -> { + Twig.error { "Chain tip fetch for recovery until failed with: ${response.toThrowable()}" } + null + } + } + } + else -> { + null + } + } + val repository = DefaultSynchronizerFactory.defaultDerivedDataRepository( - applicationContext, - backend, - coordinator.dataDbFile(zcashNetwork, alias), - zcashNetwork, - loadedCheckpoint, - seed, - Derivation.DEFAULT_NUMBER_OF_ACCOUNTS, - Derivation.DEFAULT_RECOVERY_UNTIL_HEIGHT, + context = applicationContext, + rustBackend = backend, + databaseFile = coordinator.dataDbFile(zcashNetwork, alias), + zcashNetwork = zcashNetwork, + checkpoint = loadedCheckpoint, + seed = seed, + numberOfAccounts = Derivation.DEFAULT_NUMBER_OF_ACCOUNTS, + recoverUntil = chainTip, ) - val service = DefaultSynchronizerFactory.defaultService(applicationContext, lightWalletEndpoint) val encoder = DefaultSynchronizerFactory.defaultEncoder(backend, saplingParamTool, repository) - val downloader = DefaultSynchronizerFactory.defaultDownloader(service, blockStore) val txManager = DefaultSynchronizerFactory.defaultTxManager( encoder, service @@ -521,9 +550,10 @@ interface Synchronizer { alias: String = ZcashSdk.DEFAULT_ALIAS, lightWalletEndpoint: LightWalletEndpoint, seed: ByteArray?, - birthday: BlockHeight? + birthday: BlockHeight?, + walletInitMode: WalletInitMode ): CloseableSynchronizer = runBlocking { - new(context, zcashNetwork, alias, lightWalletEndpoint, seed, birthday) + new(context, zcashNetwork, alias, lightWalletEndpoint, seed, birthday, walletInitMode) } /** @@ -548,6 +578,23 @@ interface Synchronizer { } } +/** + * Sealed class describing wallet initialization mode. + * + * Use [NewWallet] type if the seed was just created as part of a + * new wallet initialization. + * + * Use [RestoreWallet] type if an existed wallet is initialized + * from a restored seed with older birthday height. + * + * Use [ExistingWallet] type if the wallet is already initialized. + */ +sealed class WalletInitMode { + data object NewWallet : WalletInitMode() + data object RestoreWallet : WalletInitMode() + data object ExistingWallet : WalletInitMode() +} + interface CloseableSynchronizer : Synchronizer, Closeable /** diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessor.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessor.kt index 8f23e946..403aa941 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessor.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessor.kt @@ -340,6 +340,8 @@ class CompactBlockProcessor internal constructor( "Failed while processing blocks at height: ${result.failedAtHeight} with: " + "${result.error}" } + // TODO [#1222]: Enrich BlockProcessingResult.SyncFailure with root cause + // TODO [#1222]: https://github.com/zcash/zcash-android-wallet-sdk/issues/1222 checkErrorResult(result.failedAtHeight) } is BlockProcessingResult.Success -> { @@ -754,6 +756,9 @@ class CompactBlockProcessor internal constructor( object Success : BlockProcessingResult() object Reconnecting : BlockProcessingResult() object RestartSynchronization : BlockProcessingResult() + + // TODO [#1222]: Enrich BlockProcessingResult.SyncFailure with root cause + // TODO [#1222]: https://github.com/zcash/zcash-android-wallet-sdk/issues/1222 data class SyncFailure(val failedAtHeight: BlockHeight?, val error: Throwable) : BlockProcessingResult() } From 3721b8b6c4ab4d076e5d84a671037d238df4004a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Honza=20Rychnovsk=C3=BD?= Date: Fri, 8 Sep 2023 17:35:56 +0200 Subject: [PATCH 47/76] [#1225] Disable deprecated darkside tests * [#1225] Disable deprecated darkside tests - Closes #1225 --- .../sdk/darkside/TransparentIntegrationTest.kt | 4 +++- .../sdk/darkside/reorgs/InboundTxTests.kt | 11 ++++++++--- .../sdk/darkside/reorgs/ReorgSetupTest.kt | 15 ++++++++++----- .../sdk/darkside/reorgs/ReorgSmallTest.kt | 16 ++++++++++------ .../darkside/test/DarksideTestCoordinator.kt | 17 ++++++----------- 5 files changed, 37 insertions(+), 26 deletions(-) diff --git a/darkside-test-lib/src/androidTest/java/cash/z/ecc/android/sdk/darkside/TransparentIntegrationTest.kt b/darkside-test-lib/src/androidTest/java/cash/z/ecc/android/sdk/darkside/TransparentIntegrationTest.kt index 0c58b536..b25021a8 100644 --- a/darkside-test-lib/src/androidTest/java/cash/z/ecc/android/sdk/darkside/TransparentIntegrationTest.kt +++ b/darkside-test-lib/src/androidTest/java/cash/z/ecc/android/sdk/darkside/TransparentIntegrationTest.kt @@ -10,11 +10,13 @@ import org.junit.runner.RunWith /** * Integration test to run in order to catch any regressions in transparent behavior. */ +// TODO [#1224]: Refactor and re-enable disabled darkside tests +// TODO [#1224]: https://github.com/zcash/zcash-android-wallet-sdk/issues/1224 @RunWith(AndroidJUnit4::class) class TransparentIntegrationTest : DarksideTest() { @Before fun setup() = runOnce { - sithLord.await() + // sithLord.await() } @Test diff --git a/darkside-test-lib/src/androidTest/java/cash/z/ecc/android/sdk/darkside/reorgs/InboundTxTests.kt b/darkside-test-lib/src/androidTest/java/cash/z/ecc/android/sdk/darkside/reorgs/InboundTxTests.kt index 8e0e136c..4f014e1a 100644 --- a/darkside-test-lib/src/androidTest/java/cash/z/ecc/android/sdk/darkside/reorgs/InboundTxTests.kt +++ b/darkside-test-lib/src/androidTest/java/cash/z/ecc/android/sdk/darkside/reorgs/InboundTxTests.kt @@ -6,15 +6,19 @@ import cash.z.ecc.android.sdk.darkside.test.ScopedTest import cash.z.ecc.android.sdk.model.BlockHeight import cash.z.ecc.android.sdk.model.ZcashNetwork import org.junit.BeforeClass +import org.junit.Ignore import org.junit.Test import org.junit.runner.RunWith +// TODO [#1224]: Refactor and re-enable disabled darkside tests +// TODO [#1224]: https://github.com/zcash/zcash-android-wallet-sdk/issues/1224 @RunWith(AndroidJUnit4::class) class InboundTxTests : ScopedTest() { @Test + @Ignore("Temporarily disabled") fun testTargetBlock_synced() { - validator.validateMinHeightSynced(firstBlock) + // validator.validateMinHeightSynced(firstBlock) } @Test @@ -28,10 +32,11 @@ class InboundTxTests : ScopedTest() { } @Test + @Ignore("Temporarily disabled") fun testTxCountAfter() { // add 2 transactions to block 663188 and 'mine' that block addTransactions(targetTxBlock, tx663174, tx663188) - sithLord.await(timeout = 30_000L, targetHeight = targetTxBlock) + // sithLord.await(timeout = 30_000L, targetHeight = targetTxBlock) validator.validateTxCount(2) } @@ -91,7 +96,7 @@ class InboundTxTests : ScopedTest() { .stageEmptyBlocks(firstBlock + 1, 100) .applyTipHeight(BlockHeight.new(ZcashNetwork.Mainnet, targetTxBlock.value - 1)) - sithLord.await() + // sithLord.await() } } } diff --git a/darkside-test-lib/src/androidTest/java/cash/z/ecc/android/sdk/darkside/reorgs/ReorgSetupTest.kt b/darkside-test-lib/src/androidTest/java/cash/z/ecc/android/sdk/darkside/reorgs/ReorgSetupTest.kt index 0d1334c1..f4e82721 100644 --- a/darkside-test-lib/src/androidTest/java/cash/z/ecc/android/sdk/darkside/reorgs/ReorgSetupTest.kt +++ b/darkside-test-lib/src/androidTest/java/cash/z/ecc/android/sdk/darkside/reorgs/ReorgSetupTest.kt @@ -3,34 +3,39 @@ package cash.z.ecc.android.sdk.darkside.reorgs import androidx.test.ext.junit.runners.AndroidJUnit4 import cash.z.ecc.android.sdk.darkside.test.DarksideTestCoordinator import cash.z.ecc.android.sdk.darkside.test.ScopedTest -import cash.z.ecc.android.sdk.model.BlockHeight -import cash.z.ecc.android.sdk.model.ZcashNetwork import org.junit.Before import org.junit.BeforeClass +import org.junit.Ignore import org.junit.Test import org.junit.runner.RunWith +// TODO [#1224]: Refactor and re-enable disabled darkside tests +// TODO [#1224]: https://github.com/zcash/zcash-android-wallet-sdk/issues/1224 @RunWith(AndroidJUnit4::class) class ReorgSetupTest : ScopedTest() { + /* private val birthdayHeight = BlockHeight.new(ZcashNetwork.Mainnet, 663150) private val targetHeight = BlockHeight.new(ZcashNetwork.Mainnet, 663250) + */ @Before fun setup() { - sithLord.await() + // sithLord.await() } @Test + @Ignore("Temporarily disabled") fun testBeforeReorg_minHeight() = timeout(30_000L) { // validate that we are synced, at least to the birthday height - validator.validateMinHeightSynced(birthdayHeight) + // validator.validateMinHeightSynced(birthdayHeight) } @Test + @Ignore("Temporarily disabled") fun testBeforeReorg_maxHeight() = timeout(30_000L) { // validate that we are not synced beyond the target height - validator.validateMaxHeightSynced(targetHeight) + // validator.validateMaxHeightSynced(targetHeight) } companion object { diff --git a/darkside-test-lib/src/androidTest/java/cash/z/ecc/android/sdk/darkside/reorgs/ReorgSmallTest.kt b/darkside-test-lib/src/androidTest/java/cash/z/ecc/android/sdk/darkside/reorgs/ReorgSmallTest.kt index 3de23724..bf58e840 100644 --- a/darkside-test-lib/src/androidTest/java/cash/z/ecc/android/sdk/darkside/reorgs/ReorgSmallTest.kt +++ b/darkside-test-lib/src/androidTest/java/cash/z/ecc/android/sdk/darkside/reorgs/ReorgSmallTest.kt @@ -3,45 +3,49 @@ package cash.z.ecc.android.sdk.darkside.reorgs import androidx.test.ext.junit.runners.AndroidJUnit4 import cash.z.ecc.android.sdk.darkside.test.DarksideTestCoordinator import cash.z.ecc.android.sdk.darkside.test.ScopedTest -import cash.z.ecc.android.sdk.model.BlockHeight -import cash.z.ecc.android.sdk.model.ZcashNetwork import org.junit.Assert.assertTrue import org.junit.Before import org.junit.BeforeClass +import org.junit.Ignore import org.junit.Test import org.junit.runner.RunWith @RunWith(AndroidJUnit4::class) class ReorgSmallTest : ScopedTest() { + /* private val targetHeight = BlockHeight.new( ZcashNetwork.Mainnet, 663250 ) private val hashBeforeReorg = "09ec0d5de30d290bc5a2318fbf6a2427a81c7db4790ce0e341a96aeac77108b9" private val hashAfterReorg = "tbd" + */ @Before fun setup() { - sithLord.await() + // sithLord.await() } @Test + @Ignore("Temporarily disabled") fun testBeforeReorg_latestBlockHash() = timeout(30_000L) { - validator.validateBlockHash(targetHeight, hashBeforeReorg) + // validator.validateBlockHash(targetHeight, hashBeforeReorg) } @Test + @Ignore("Temporarily disabled") fun testAfterReorg_callbackTriggered() = timeout(30_000L) { hadReorg = false // sithLord.triggerSmallReorg() - sithLord.await() +// sithLord.await() assertTrue(hadReorg) } @Test + @Ignore("Temporarily disabled") fun testAfterReorg_latestBlockHash() = timeout(30_000L) { - validator.validateBlockHash(targetHeight, hashAfterReorg) + // validator.validateBlockHash(targetHeight, hashAfterReorg) } companion object { diff --git a/darkside-test-lib/src/androidTest/java/cash/z/ecc/android/sdk/darkside/test/DarksideTestCoordinator.kt b/darkside-test-lib/src/androidTest/java/cash/z/ecc/android/sdk/darkside/test/DarksideTestCoordinator.kt index 76526d06..912258f6 100644 --- a/darkside-test-lib/src/androidTest/java/cash/z/ecc/android/sdk/darkside/test/DarksideTestCoordinator.kt +++ b/darkside-test-lib/src/androidTest/java/cash/z/ecc/android/sdk/darkside/test/DarksideTestCoordinator.kt @@ -2,7 +2,6 @@ package cash.z.ecc.android.sdk.darkside.test import androidx.test.core.app.ApplicationProvider import androidx.test.platform.app.InstrumentationRegistry -import cash.z.ecc.android.sdk.Synchronizer import cash.z.ecc.android.sdk.model.Account import cash.z.ecc.android.sdk.model.BlockHeight import cash.z.ecc.android.sdk.model.Darkside @@ -13,15 +12,14 @@ import co.electriccoin.lightwallet.client.model.BlockHeightUnsafe import co.electriccoin.lightwallet.client.model.LightWalletEndpoint import io.grpc.StatusRuntimeException import kotlinx.coroutines.delay -import kotlinx.coroutines.flow.filter import kotlinx.coroutines.flow.first -import kotlinx.coroutines.flow.map import kotlinx.coroutines.runBlocking import org.junit.Assert import org.junit.Assert.assertEquals -import org.junit.Assert.assertNotNull import org.junit.Assert.assertTrue +// TODO [#1224]: Refactor and re-enable disabled darkside tests +// TODO [#1224]: https://github.com/zcash/zcash-android-wallet-sdk/issues/1224 class DarksideTestCoordinator(val wallet: TestWallet) { constructor( alias: String = "DarksideTestCoordinator", @@ -95,6 +93,7 @@ class DarksideTestCoordinator(val wallet: TestWallet) { * Waits for, at most, the given amount of time for the synchronizer to download and scan blocks * and reach a 'SYNCED' status. */ + /* fun await(timeout: Long = 60_000L, targetHeight: BlockHeight? = null) = runBlocking { ScopedTest.timeoutWith(this, timeout) { synchronizer.status.map { status -> @@ -110,6 +109,7 @@ class DarksideTestCoordinator(val wallet: TestWallet) { }.filter { it == Synchronizer.Status.SYNCED }.first() } } + */ // /** // * Send a transaction and wait until it has been fully created and successfully submitted, which @@ -135,13 +135,6 @@ class DarksideTestCoordinator(val wallet: TestWallet) { inner class DarksideTestValidator { - fun validateHasBlock(height: BlockHeight) { - runBlocking { - assertTrue(synchronizer.findBlockHashAsHex(height) != null) - assertTrue(synchronizer.findBlockHash(height)?.size ?: 0 > 0) - } - } - fun validateLatestHeight(height: BlockHeight) = runBlocking { val info = synchronizer.processorInfo.first() val networkBlockHeight = info.networkBlockHeight @@ -152,6 +145,7 @@ class DarksideTestCoordinator(val wallet: TestWallet) { ) } + /* fun validateMinHeightSynced(minHeight: BlockHeight) = runBlocking { val info = synchronizer.processorInfo.first() val lastSyncedHeight = info.lastSyncedHeight @@ -177,6 +171,7 @@ class DarksideTestCoordinator(val wallet: TestWallet) { val hash = runBlocking { synchronizer.findBlockHashAsHex(height) } assertEquals(expectedHash, hash) } + */ fun onReorg(callback: (errorHeight: BlockHeight, rewindHeight: BlockHeight) -> Unit) { synchronizer.onChainErrorHandler = callback From 899b487ee4124740dd73ec56263ffdfa6b70fe34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Honza=20Rychnovsk=C3=BD?= Date: Fri, 8 Sep 2023 18:12:27 +0200 Subject: [PATCH 48/76] [#1219] Remove calculated sync progress Closes #1219 --- .../block/processor/CompactBlockProcessor.kt | 69 ++++--------------- .../processor/model/BatchSyncProgress.kt | 3 +- .../processor/model/SbSPreparationResult.kt | 4 +- .../android/sdk/internal/model/BlockBatch.kt | 6 +- 4 files changed, 16 insertions(+), 66 deletions(-) diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessor.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessor.kt index 403aa941..3287be24 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessor.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessor.kt @@ -154,20 +154,6 @@ class CompactBlockProcessor internal constructor( private val processingMutex = Mutex() - /** - * The synchronization-related variable that holds the all batch count computed in the first initial synchronization - * loop. It is supposed to keep the same value across the synchronization refreshes with [runSbSSyncingPreparation] - * as happens in spend-before-sync synchronization function. - */ - private var allBatchCount: Long = 0 - - /** - * Another synchronization-related variable that holds the order of a currently processing batch of blocks. It - * is supposed to preserve its value across the synchronization refreshes with [runSbSSyncingPreparation] as - * happens in spend-before-sync synchronization function. - */ - private var lastBatchOrder: Long = 0 - /** * Flow of birthday heights. The birthday is essentially the first block that the wallet cares * about. Any prior block can be ignored. This is not a fixed value because the height is @@ -428,7 +414,6 @@ class CompactBlockProcessor internal constructor( var verifyRangeResult = preparationResult.verifyRangeResult var suggestedRangesResult = preparationResult.suggestedRangesResult - val allBatchCountLocal = preparationResult.allBatchCount val lastPreparationTime = System.currentTimeMillis() // Running synchronization for the [ScanRange.SuggestScanRangePriority.Verify] range @@ -450,19 +435,9 @@ class CompactBlockProcessor internal constructor( network = network, syncRange = verifyRangeResult.scanRange.range, withDownload = true, - enhanceStartHeight = firstUnenhancedHeight, - lastBatchOrder = lastBatchOrder + enhanceStartHeight = firstUnenhancedHeight ).collect { rangeSyncProgress -> - // We need to update lastBatchOrder for the processing of the following range. It can occasionally - // be over the precomputed all-batch count in case of inter-syncing failure. - lastBatchOrder = min(rangeSyncProgress.overallOrder, allBatchCountLocal) - - // TODO [#1219]: Remove calculated sync progress leftovers - // TODO [#1219]: https://github.com/zcash/zcash-android-wallet-sdk/issues/1219 - Twig.info { - "Progress calculated: " + - "${PercentDecimal(lastBatchOrder / allBatchCountLocal.toFloat()).decimal}" - } + // Update sync progress when (val result = getScanProgress(backend)) { is GetScanProgressResult.Success -> { val resultProgress = result.toPercentDecimal() @@ -551,19 +526,9 @@ class CompactBlockProcessor internal constructor( network = network, syncRange = scanRange.range, withDownload = true, - enhanceStartHeight = firstUnenhancedHeight, - lastBatchOrder = lastBatchOrder + enhanceStartHeight = firstUnenhancedHeight ).map { rangeSyncProgress -> - // We need to update lastBatchOrder for the processing of the following range. It can occasionally - // be over the precomputed all-batch count in case of inter-syncing failure. - lastBatchOrder = min(rangeSyncProgress.overallOrder, allBatchCountLocal) - - // TODO [#1219]: Remove calculated sync progress leftovers - // TODO [#1219]: https://github.com/zcash/zcash-android-wallet-sdk/issues/1219 - Twig.info { - "Progress calculated: " + - "${PercentDecimal(lastBatchOrder / allBatchCountLocal.toFloat()).decimal}" - } + // Update sync progress when (val result = getScanProgress(backend)) { is GetScanProgressResult.Success -> { val resultProgress = result.toPercentDecimal() @@ -699,8 +664,6 @@ class CompactBlockProcessor internal constructor( } setState(State.Syncing) - allBatchCount = max(allBatchCount, getBatchCount(suggestedRangesResult.ranges.map { it.range })) - lastBatchOrder = max(lastBatchOrder, 0) // Parse and process ranges. If it recognizes a range with Priority.Verify, it runs the verification part. val verifyRangeResult = shouldVerifySuggestedScanRanges(suggestedRangesResult) @@ -709,9 +672,7 @@ class CompactBlockProcessor internal constructor( return SbSPreparationResult.Success( suggestedRangesResult = suggestedRangesResult, - verifyRangeResult = verifyRangeResult, - allBatchCount = allBatchCount, - lastBatchOrder = lastBatchOrder + verifyRangeResult = verifyRangeResult ) } @@ -1322,8 +1283,7 @@ class CompactBlockProcessor internal constructor( network: ZcashNetwork, syncRange: ClosedRange, withDownload: Boolean, - enhanceStartHeight: BlockHeight?, - lastBatchOrder: Long + enhanceStartHeight: BlockHeight? ): Flow = flow { if (syncRange.isEmpty()) { Twig.debug { "No blocks to sync" } @@ -1335,7 +1295,7 @@ class CompactBlockProcessor internal constructor( } else { Twig.info { "Syncing blocks in range $syncRange" } - val batches = getBatchedBlockList(lastBatchOrder, syncRange, network) + val batches = getBatchedBlockList(syncRange, network) // Check for the last enhanced height and eventually set is as the beginning of the next enhancing range var enhancingRange = if (enhanceStartHeight != null) { @@ -1404,8 +1364,7 @@ class CompactBlockProcessor internal constructor( emit( BatchSyncProgress( - inRangeOrder = continuousResult.batch.inRangeOrder, - overallOrder = continuousResult.batch.crossRangesOrder, + order = continuousResult.batch.order, resultState = resultState ) ) @@ -1417,7 +1376,7 @@ class CompactBlockProcessor internal constructor( // state comes from the previous stages, or if the end of the sync range is reached if (enhancingRange.length() >= ENHANCE_BATCH_SIZE || resultState != SyncingResult.AllSuccess || - continuousResult.batch.inRangeOrder == batches.size.toLong() + continuousResult.batch.order == batches.size.toLong() ) { // Copy the range for use and reset for the next iteration val currentEnhancingRange = enhancingRange @@ -1445,15 +1404,14 @@ class CompactBlockProcessor internal constructor( } emit( BatchSyncProgress( - inRangeOrder = continuousResult.batch.inRangeOrder, - overallOrder = continuousResult.batch.crossRangesOrder, + order = continuousResult.batch.order, resultState = resultState ) ) } } Twig.info { - "All sync stages done for the batch ${continuousResult.batch.inRangeOrder}/${batches.size}:" + + "All sync stages done for the batch ${continuousResult.batch.order}/${batches.size}:" + " ${continuousResult.batch} with result state: $resultState" } }.takeWhile { batchProcessResult -> @@ -1496,14 +1454,12 @@ class CompactBlockProcessor internal constructor( * Prepare list of all [BlockBatch] internal objects to be processed during a range of * blocks processing * - * @param lastBatchOrder The index of the last previously processed batch * @param syncRange Current range to be processed * @param network The network we are operating on * * @return List of [BlockBatch] to for synchronization */ private fun getBatchedBlockList( - lastBatchOrder: Long, syncRange: ClosedRange, network: ZcashNetwork ): List { @@ -1521,8 +1477,7 @@ class CompactBlockProcessor internal constructor( add( BlockBatch( - inRangeOrder = index, - crossRangesOrder = lastBatchOrder + index, + order = index, range = start..end ) ) diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/BatchSyncProgress.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/BatchSyncProgress.kt index 4c425bf3..293a0184 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/BatchSyncProgress.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/BatchSyncProgress.kt @@ -4,8 +4,7 @@ package cash.z.ecc.android.sdk.block.processor.model * Progress model class for sharing the whole batch synchronization progress out of the synchronization process. */ internal data class BatchSyncProgress( - val inRangeOrder: Long = 0, - val overallOrder: Long = 0, + val order: Long = 0, val resultState: SyncingResult = SyncingResult.AllSuccess ) diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/SbSPreparationResult.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/SbSPreparationResult.kt index 21401dfc..c4613394 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/SbSPreparationResult.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/SbSPreparationResult.kt @@ -20,9 +20,7 @@ internal sealed class SbSPreparationResult { } data class Success( val suggestedRangesResult: SuggestScanRangesResult, - val verifyRangeResult: VerifySuggestedScanRange, - val allBatchCount: Long, - val lastBatchOrder: Long + val verifyRangeResult: VerifySuggestedScanRange ) : SbSPreparationResult() object NoMoreBlocksToProcess : SbSPreparationResult() } diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/BlockBatch.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/BlockBatch.kt index 9b8cdf4b..5891307b 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/BlockBatch.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/BlockBatch.kt @@ -3,11 +3,9 @@ package cash.z.ecc.android.sdk.internal.model import cash.z.ecc.android.sdk.model.BlockHeight internal data class BlockBatch( - val inRangeOrder: Long, - val crossRangesOrder: Long, + val order: Long, val range: ClosedRange, var blocks: List? = null ) { - override fun toString() = "BlockBatch(crossRangesOrder=$crossRangesOrder, inRangeOrder=$inRangeOrder, " + - "range=$range, blocks=${blocks?.size ?: "null"})" + override fun toString() = "BlockBatch(order=$order, range=$range, blocks=${blocks?.size ?: "null"})" } From 0d001495d551a8873523b5d0ca31dcd94631b700 Mon Sep 17 00:00:00 2001 From: Honza Date: Fri, 8 Sep 2023 20:28:49 +0200 Subject: [PATCH 49/76] Documentation update --- CHANGELOG.md | 2 +- README.md | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1258256c..01350cc7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # Change Log ## Unreleased -- `CompactBlockProcessor` now processes compact blocks from the lightwalletd server with spend-before-sync algorithm +- `CompactBlockProcessor` now processes compact blocks from the lightwalletd server with **Spend-before-Sync** algorithm (i.e. non-linear order). This feature shortens the time after which a wallet's spendable balance can be used. - The block synchronization mechanism is about one-third faster thanks to the optimized `CompactBlockProcessor.SYNC_BATCH_SIZE`. Issue **#1206**. diff --git a/README.md b/README.md index b41ecd0a..f02c0803 100644 --- a/README.md +++ b/README.md @@ -56,5 +56,6 @@ Note that we aim for the main branch of this repository to be stable and releasa ## Unstable Features ### Spend-before-Sync compact blocks synchronization algorithm -- CompactBlockProcessor now processes compact blocks from the lightwalletd server in the **spend-before-sync** order. This -feature shortens the time after which a wallet's spendable balance can be used. Please note that this new block synchronization algorithm is still under development. +`CompactBlockProcessor` now processes compact blocks from the lightwalletd server in non-linear order with the +**Spend-before-Sync** algorithm. This feature speeds up discovering the wallet's spendable balance. Please note that +this new block synchronization algorithm is still under development. From 1ef0304f922e5bfcee856f3df4015732f1009838 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 8 Sep 2023 21:45:54 +0100 Subject: [PATCH 50/76] rust: cargo update --- backend-lib/Cargo.lock | 539 ++++++++++++++++++----------------------- 1 file changed, 234 insertions(+), 305 deletions(-) diff --git a/backend-lib/Cargo.lock b/backend-lib/Cargo.lock index 39f7b5e2..dd60ad62 100644 --- a/backend-lib/Cargo.lock +++ b/backend-lib/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "addr2line" -version = "0.19.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a76fd60b23679b7d19bd066031410fb7e458ccc5e958eb5c325888ce4baedc97" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" dependencies = [ "gimli", ] @@ -29,9 +29,9 @@ dependencies = [ [[package]] name = "aes" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "433cfd6710c9986c576a25ca913c39d66a6474107b406f34f91d4a8923395241" +checksum = "ac1f845298e95f983ff1944b728ae08b8cebab80d684f0a832ed0fc74dfa27e2" dependencies = [ "cfg-if", "cipher", @@ -50,16 +50,25 @@ dependencies = [ ] [[package]] -name = "allocator-api2" -version = "0.2.15" +name = "aho-corasick" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56fc6cf8dc8c4158eed8649f9b8b0ea1518eb62b544fe9490d66fa0b349eafe9" +checksum = "0c378d78423fdad8089616f827526ee33c19f2fddbd5de1629152c9593ba4783" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" [[package]] name = "anyhow" -version = "1.0.71" +version = "1.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" +checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" [[package]] name = "arrayref" @@ -69,9 +78,9 @@ checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" [[package]] name = "arrayvec" -version = "0.7.2" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" +checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "autocfg" @@ -81,9 +90,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "backtrace" -version = "0.3.67" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "233d376d6d185f2a3093e58f283f60f880315b6c60075b01f36b3b85154564ca" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" dependencies = [ "addr2line", "cc", @@ -102,9 +111,9 @@ checksum = "5024ee8015f02155eee35c711107ddd9a9bf3cb689cf2a9089c97e79b6e1ae83" [[package]] name = "base64" -version = "0.21.0" +version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" +checksum = "414dcefbc63d77c526a76b3afcf6fbb9b5e2791c19c3aa2297733208750c6e53" [[package]] name = "base64ct" @@ -161,9 +170,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.3.3" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42" +checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" [[package]] name = "bitvec" @@ -254,9 +263,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.12.1" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b1ce199063694f33ffb7dd4e0ee620741495c32833cde5aa08f02a0bf96f0c8" +checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" [[package]] name = "byte-tools" @@ -272,9 +281,9 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "bytes" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" [[package]] name = "cbc" @@ -287,9 +296,12 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.79" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +dependencies = [ + "libc", +] [[package]] name = "cesu8" @@ -350,15 +362,15 @@ dependencies = [ [[package]] name = "constant_time_eq" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13418e745008f7349ec7e449155f419a61b92b58a99cc3616942b926825ec76b" +checksum = "21a53c0a4d288377e7415b53dcfc3c04da5cdc2cc95c8d5ac178b58f0b861ad6" [[package]] name = "cpufeatures" -version = "0.2.7" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e4c1eaa2012c47becbbad2ab175484c2a84d1185b566fb2cc5b8707343dfe58" +checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" dependencies = [ "libc", ] @@ -386,9 +398,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.14" +version = "0.9.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695" +checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" dependencies = [ "autocfg", "cfg-if", @@ -399,9 +411,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.15" +version = "0.8.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" dependencies = [ "cfg-if", ] @@ -431,6 +443,12 @@ dependencies = [ "petgraph", ] +[[package]] +name = "deranged" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" + [[package]] name = "digest" version = "0.8.1" @@ -442,9 +460,9 @@ dependencies = [ [[package]] name = "digest" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", "crypto-common", @@ -476,9 +494,9 @@ dependencies = [ [[package]] name = "either" -version = "1.8.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" [[package]] name = "equihash" @@ -490,14 +508,20 @@ dependencies = [ ] [[package]] -name = "errno" -version = "0.3.1" +name = "equivalent" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "errno" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd" dependencies = [ "errno-dragonfly", "libc", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -554,12 +578,9 @@ checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" [[package]] name = "fastrand" -version = "1.9.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" -dependencies = [ - "instant", -] +checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" [[package]] name = "ff" @@ -619,9 +640,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" dependencies = [ "cfg-if", "libc", @@ -630,9 +651,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.27.2" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad0a93d233ebf96623465aad4046a8d3aa4da22d4f4beba5388838c8a434bbb4" +checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" [[package]] name = "group" @@ -686,12 +707,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - [[package]] name = "hashbrown" version = "0.14.0" @@ -704,11 +719,11 @@ dependencies = [ [[package]] name = "hashlink" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "312f66718a2d7789ffef4f4b7b213138ed9f1eb3aa1d0d82fc99f88fb3ffd26f" +checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" dependencies = [ - "hashbrown 0.14.0", + "hashbrown", ] [[package]] @@ -756,18 +771,9 @@ checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" -version = "0.2.6" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" -dependencies = [ - "libc", -] - -[[package]] -name = "hermit-abi" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" +checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" [[package]] name = "hex" @@ -781,7 +787,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest 0.10.6", + "digest 0.10.7", ] [[package]] @@ -790,7 +796,7 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" dependencies = [ - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -803,12 +809,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.9.3" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" dependencies = [ - "autocfg", - "hashbrown 0.12.3", + "equivalent", + "hashbrown", ] [[package]] @@ -820,26 +826,6 @@ dependencies = [ "generic-array 0.14.7", ] -[[package]] -name = "instant" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "io-lifetimes" -version = "1.0.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220" -dependencies = [ - "hermit-abi 0.3.1", - "libc", - "windows-sys 0.48.0", -] - [[package]] name = "itertools" version = "0.10.5" @@ -851,9 +837,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.6" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] name = "jni" @@ -877,9 +863,9 @@ checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" [[package]] name = "js-sys" -version = "0.3.61" +version = "0.3.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" +checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" dependencies = [ "wasm-bindgen", ] @@ -904,7 +890,7 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b6f1427d9c43b1cce87434c4d9eca33f43bdbb6246a762aa823a582f74c1684" dependencies = [ - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -918,15 +904,15 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.142" +version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a987beff54b60ffa6d51982e1aa1146bc42f19bd26be28b0586f252fccf5317" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" [[package]] name = "libm" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb" +checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4" [[package]] name = "libsqlite3-sys" @@ -941,18 +927,15 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.3.6" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b64f40e5e03e0d54f03845c8197d0291253cdbedfb1cb46b13c2c117554a9f4c" +checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503" [[package]] name = "log" -version = "0.4.17" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" -dependencies = [ - "cfg-if", -] +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] name = "log-panics" @@ -975,15 +958,15 @@ dependencies = [ [[package]] name = "memchr" -version = "2.5.0" +version = "2.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" [[package]] name = "memoffset" -version = "0.8.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" dependencies = [ "autocfg", ] @@ -1005,9 +988,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.6.2" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" dependencies = [ "adler", ] @@ -1055,9 +1038,9 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" +checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" dependencies = [ "autocfg", "num-integer", @@ -1076,37 +1059,37 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" dependencies = [ "autocfg", ] [[package]] name = "num_cpus" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.2.6", + "hermit-abi", "libc", ] [[package]] name = "object" -version = "0.30.3" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea86265d3d3dcb6a27fc51bd29a4bf387fae9d2986b823079d4986af253eb439" +checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" dependencies = [ "memchr", ] [[package]] name = "once_cell" -version = "1.17.1" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "opaque-debug" @@ -1208,21 +1191,21 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271779f35b581956db91a3e55737327a03aa051e90b1c47aeb189508533adfd7" dependencies = [ - "digest 0.10.6", + "digest 0.10.7", "password-hash", ] [[package]] name = "percent-encoding" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" +checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "petgraph" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4" +checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ "fixedbitset", "indexmap", @@ -1230,15 +1213,15 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.9" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" [[package]] name = "pkg-config" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" +checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" [[package]] name = "poly1305" @@ -1269,9 +1252,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.56" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" +checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" dependencies = [ "unicode-ident", ] @@ -1332,9 +1315,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.26" +version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" dependencies = [ "proc-macro2", ] @@ -1399,9 +1382,9 @@ dependencies = [ [[package]] name = "reddsa" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54b34d2c0df43159d2ff79d3cf929c9f11415529127344edb8160ad2be499fcd" +checksum = "78a5191930e84973293aa5f532b513404460cd2216c1cfb76d08748c15b40b02" dependencies = [ "blake2b_simd", "byteorder", @@ -1439,18 +1422,32 @@ dependencies = [ [[package]] name = "regex" -version = "1.8.1" +version = "1.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af83e617f331cc6ae2da5443c602dfa5af81e517212d9d611a5b3ba1777b5370" +checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47" dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795" +dependencies = [ + "aho-corasick", + "memchr", "regex-syntax", ] [[package]] name = "regex-syntax" -version = "0.7.1" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5996294f19bd3aae0453a862ad728f60e6600695733dd5df01da90c54363a3c" +checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" [[package]] name = "ring" @@ -1473,7 +1470,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd124222d17ad93a644ed9d011a40f4fb64aa54275c08cc216524a9ea82fb09f" dependencies = [ - "digest 0.10.6", + "digest 0.10.7", ] [[package]] @@ -1493,7 +1490,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "549b9d036d571d42e6e85d1c1425e2ac83491075078ca9a15be021c56b1641f2" dependencies = [ - "bitflags 2.3.3", + "bitflags 2.4.0", "fallible-iterator", "fallible-streaming-iterator", "hashlink", @@ -1510,16 +1507,15 @@ checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" [[package]] name = "rustix" -version = "0.37.18" +version = "0.38.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bbfc1d1c7c40c01715f47d71444744a81669ca84e8b63e25a55e169b1f86433" +checksum = "c0c3dde1fc030af041adc40e79c0e7fbcf431dd24870053d187d7c66e4b87453" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.4.0", "errno", - "io-lifetimes", "libc", "linux-raw-sys", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -1556,9 +1552,9 @@ dependencies = [ [[package]] name = "scopeguard" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "secp256k1" @@ -1607,33 +1603,33 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.160" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c" +checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.160" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291a097c63d8497e00160b166a967a4a79c64f3facdd01cbd7502231688d77df" +checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.31", ] [[package]] name = "sha2" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" +checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" dependencies = [ "cfg-if", "cpufeatures", - "digest 0.10.6", + "digest 0.10.7", ] [[package]] @@ -1658,9 +1654,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" +checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" [[package]] name = "spin" @@ -1693,9 +1689,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.15" +version = "2.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822" +checksum = "718fa2415bcb8d8bd775917a1bf12a7931b6dfa890753378538118181e0cb398" dependencies = [ "proc-macro2", "quote", @@ -1722,35 +1718,35 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tempfile" -version = "3.5.0" +version = "3.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998" +checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" dependencies = [ "cfg-if", "fastrand", "redox_syscall", "rustix", - "windows-sys 0.45.0", + "windows-sys", ] [[package]] name = "thiserror" -version = "1.0.40" +version = "1.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" +checksum = "9d6d7a740b8a666a7e828dd00da9c0dc290dff53154ea77ac109281de90589b7" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.40" +version = "1.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" +checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.31", ] [[package]] @@ -1765,10 +1761,11 @@ dependencies = [ [[package]] name = "time" -version = "0.3.23" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59e399c068f43a5d116fedaf73b203fa4f9c519f17e2b34f63221d3792f81446" +checksum = "17f6bb557fd245c28e6411aa56b6403c689ad95061f50e4be16c274e70a17e48" dependencies = [ + "deranged", "itoa", "serde", "time-core", @@ -1783,9 +1780,9 @@ checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" [[package]] name = "time-macros" -version = "0.2.10" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96ba15a897f3c86766b757e5ac7221554c6750054d74d5b28844fce5fb36a6c4" +checksum = "1a942f44339478ef67935ab2bbaec2fb0322496cf3cbe84b261e06ac3814c572" dependencies = [ "time-core", ] @@ -1832,20 +1829,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.24" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74" +checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.31", ] [[package]] name = "tracing-core" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" dependencies = [ "once_cell", "valuable", @@ -1896,9 +1893,9 @@ dependencies = [ [[package]] name = "unicode-ident" -version = "1.0.8" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" +checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" [[package]] name = "unicode-normalization" @@ -1917,9 +1914,9 @@ checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" [[package]] name = "universal-hash" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d3160b73c9a19f7e2939a2fdad446c57c1bbbbf4d919d3213ff1267a580d8b5" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" dependencies = [ "crypto-common", "subtle", @@ -1933,9 +1930,9 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" [[package]] name = "uuid" -version = "1.3.2" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dad5567ad0cf5b760e5665964bec1b47dfd077ba8a2544b513f3556d3d239a2" +checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d" [[package]] name = "valuable" @@ -1957,9 +1954,9 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "walkdir" -version = "2.3.3" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698" +checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" dependencies = [ "same-file", "winapi-util", @@ -1973,9 +1970,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.84" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" +checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -1983,24 +1980,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.84" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" +checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.31", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.84" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" +checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2008,28 +2005,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.84" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" +checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.31", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.84" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" +checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" [[package]] name = "web-sys" -version = "0.3.61" +version = "0.3.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" +checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" dependencies = [ "js-sys", "wasm-bindgen", @@ -2037,13 +2034,14 @@ dependencies = [ [[package]] name = "which" -version = "4.4.0" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" dependencies = [ "either", - "libc", + "home", "once_cell", + "rustix", ] [[package]] @@ -2077,137 +2075,71 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows-sys" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" -dependencies = [ - "windows-targets 0.42.2", -] - [[package]] name = "windows-sys" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets 0.48.0", + "windows-targets", ] [[package]] name = "windows-targets" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", -] - -[[package]] -name = "windows-targets" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" -dependencies = [ - "windows_aarch64_gnullvm 0.48.0", - "windows_aarch64_msvc 0.48.0", - "windows_i686_gnu 0.48.0", - "windows_i686_msvc 0.48.0", - "windows_x86_64_gnu 0.48.0", - "windows_x86_64_gnullvm 0.48.0", - "windows_x86_64_msvc 0.48.0", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] [[package]] name = "windows_aarch64_gnullvm" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_msvc" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_i686_gnu" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" - -[[package]] -name = "windows_i686_gnu" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_msvc" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" - -[[package]] -name = "windows_i686_msvc" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_x86_64_gnu" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_msvc" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "wyz" @@ -2220,12 +2152,9 @@ dependencies = [ [[package]] name = "xdg" -version = "2.5.0" +version = "2.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "688597db5a750e9cad4511cb94729a078e274308099a0382b5b8203bbc767fee" -dependencies = [ - "home", -] +checksum = "213b7324336b53d2414b2db8537e56544d981803139155afa84f76eeebb7a546" [[package]] name = "zcash-android-wallet-sdk" @@ -2422,5 +2351,5 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.31", ] From ab6a831f502c9bac86f6eaa9ceb6c03f46eb7e69 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 8 Sep 2023 21:46:57 +0100 Subject: [PATCH 51/76] Migrate to release candidates of Rust crates --- backend-lib/Cargo.lock | 84 +++++++++++++++++++++++------------------- backend-lib/Cargo.toml | 22 +++-------- 2 files changed, 53 insertions(+), 53 deletions(-) diff --git a/backend-lib/Cargo.lock b/backend-lib/Cargo.lock index dd60ad62..770ea51e 100644 --- a/backend-lib/Cargo.lock +++ b/backend-lib/Cargo.lock @@ -501,7 +501,8 @@ checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" [[package]] name = "equihash" version = "0.2.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=85156e2b7cb5c53c26c786ea7c5c22fe5cbbcb2d#85156e2b7cb5c53c26c786ea7c5c22fe5cbbcb2d" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab579d7cf78477773b03e80bc2f89702ef02d7112c711d54ca93dcdce68533d5" dependencies = [ "blake2b_simd", "byteorder", @@ -537,7 +538,8 @@ dependencies = [ [[package]] name = "f4jumble" version = "0.1.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=85156e2b7cb5c53c26c786ea7c5c22fe5cbbcb2d#85156e2b7cb5c53c26c786ea7c5c22fe5cbbcb2d" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a83e8d7fd0c526af4aad893b7c9fe41e2699ed8a776a6c74aecdeafe05afc75" dependencies = [ "blake2b_simd", ] @@ -801,8 +803,9 @@ dependencies = [ [[package]] name = "incrementalmerkletree" -version = "0.4.0" -source = "git+https://github.com/zcash/incrementalmerkletree.git?rev=da97e6c399c5acddedad2c1730dbb7ee55499a2f#da97e6c399c5acddedad2c1730dbb7ee55499a2f" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "361c467824d4d9d4f284be4b2608800839419dccc4d4608f28345237fe354623" dependencies = [ "either", ] @@ -1105,8 +1108,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "orchard" -version = "0.5.0" -source = "git+https://github.com/zcash/orchard.git?rev=6ef89d5f154de2cf7b7dd87edb8d8c49158beebb#6ef89d5f154de2cf7b7dd87edb8d8c49158beebb" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d31e68534df32024dcc89a8390ec6d7bef65edd87d91b45cfb481a2eb2d77c5" dependencies = [ "aes", "bitvec", @@ -1242,12 +1246,12 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "prettyplease" -version = "0.1.25" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" +checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" dependencies = [ "proc-macro2", - "syn 1.0.109", + "syn 2.0.31", ] [[package]] @@ -1261,9 +1265,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.11.9" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +checksum = "aa8473a65b88506c106c28ae905ca4a2b83a2993640467a41bb3080627ddfd2c" dependencies = [ "bytes", "prost-derive", @@ -1271,44 +1275,44 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.11.9" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" +checksum = "30d3e647e9eb04ddfef78dfee2d5b3fefdf94821c84b710a3d8ebc89ede8b164" dependencies = [ "bytes", "heck", "itertools", - "lazy_static", "log", "multimap", + "once_cell", "petgraph", "prettyplease", "prost", "prost-types", "regex", - "syn 1.0.109", + "syn 2.0.31", "tempfile", "which", ] [[package]] name = "prost-derive" -version = "0.11.9" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +checksum = "56075c27b20ae524d00f247b8a4dc333e5784f889fe63099f8e626bc8d73486c" dependencies = [ "anyhow", "itertools", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.31", ] [[package]] name = "prost-types" -version = "0.11.9" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" +checksum = "cebe0a918c97f86c217b0f76fd754e966f8b9f41595095cf7d74cb4e59d730f6" dependencies = [ "prost", ] @@ -1643,10 +1647,11 @@ dependencies = [ [[package]] name = "shardtree" -version = "0.0.0" -source = "git+https://github.com/zcash/incrementalmerkletree.git?rev=da97e6c399c5acddedad2c1730dbb7ee55499a2f#da97e6c399c5acddedad2c1730dbb7ee55499a2f" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c19f96dde3a8693874f7e7c53d95616569b4009379a903789efbd448f4ea9cc7" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.4.0", "either", "incrementalmerkletree", "tracing", @@ -1804,15 +1809,15 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tonic-build" -version = "0.9.2" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6fdaae4c2c638bb70fe42803a26fbd6fc6ac8c72f5c59f67ecc2a2dcabf4b07" +checksum = "8b477abbe1d18c0b08f56cd01d1bc288668c5b5cfd19b2ae1886bbf599c546f1" dependencies = [ "prettyplease", "proc-macro2", "prost-build", "quote", - "syn 1.0.109", + "syn 2.0.31", ] [[package]] @@ -2188,7 +2193,8 @@ dependencies = [ [[package]] name = "zcash_address" version = "0.3.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=85156e2b7cb5c53c26c786ea7c5c22fe5cbbcb2d#85156e2b7cb5c53c26c786ea7c5c22fe5cbbcb2d" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8944af5c206cf2e37020ad54618e1825501b98548d35a638b73e0ec5762df8d5" dependencies = [ "bech32", "bs58", @@ -2198,8 +2204,9 @@ dependencies = [ [[package]] name = "zcash_client_backend" -version = "0.9.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=85156e2b7cb5c53c26c786ea7c5c22fe5cbbcb2d#85156e2b7cb5c53c26c786ea7c5c22fe5cbbcb2d" +version = "0.10.0-rc.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dc7204683d2ef58f87a78168f7135edd2612a888d1663812e642d5835b32607" dependencies = [ "base64", "bech32", @@ -2232,8 +2239,9 @@ dependencies = [ [[package]] name = "zcash_client_sqlite" -version = "0.7.1" -source = "git+https://github.com/zcash/librustzcash.git?rev=85156e2b7cb5c53c26c786ea7c5c22fe5cbbcb2d#85156e2b7cb5c53c26c786ea7c5c22fe5cbbcb2d" +version = "0.8.0-rc.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8095730f4fcc20c5eeacee2a9edec10f1e600770e254fcdba6bc6abb2ebd320" dependencies = [ "bs58", "byteorder", @@ -2259,7 +2267,8 @@ dependencies = [ [[package]] name = "zcash_encoding" version = "0.2.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=85156e2b7cb5c53c26c786ea7c5c22fe5cbbcb2d#85156e2b7cb5c53c26c786ea7c5c22fe5cbbcb2d" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f03391b81727875efa6ac0661a20883022b6fba92365dc121c48fa9b00c5aac0" dependencies = [ "byteorder", "nonempty", @@ -2280,8 +2289,9 @@ dependencies = [ [[package]] name = "zcash_primitives" -version = "0.12.0" -source = "git+https://github.com/zcash/librustzcash.git?rev=85156e2b7cb5c53c26c786ea7c5c22fe5cbbcb2d#85156e2b7cb5c53c26c786ea7c5c22fe5cbbcb2d" +version = "0.13.0-rc.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc4391d9325e0a51a7cbff02b5c4b5472d66087bd9c903ddb12dea7ec22f3e0" dependencies = [ "aes", "bip0039", @@ -2315,15 +2325,15 @@ dependencies = [ [[package]] name = "zcash_proofs" -version = "0.12.1" -source = "git+https://github.com/zcash/librustzcash.git?rev=85156e2b7cb5c53c26c786ea7c5c22fe5cbbcb2d#85156e2b7cb5c53c26c786ea7c5c22fe5cbbcb2d" +version = "0.13.0-rc.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48f22eff3bdc382327ef28f809024ddc89ec6d903ba71be629b2cbea34afdda2" dependencies = [ "bellman", "blake2b_simd", "bls12_381", "group", "home", - "incrementalmerkletree", "jubjub", "known-folders", "lazy_static", diff --git a/backend-lib/Cargo.toml b/backend-lib/Cargo.toml index 679be163..cbfc95e1 100644 --- a/backend-lib/Cargo.toml +++ b/backend-lib/Cargo.toml @@ -16,17 +16,17 @@ hdwallet = "0.3.1" hdwallet-bitcoin = "0.3" hex = "0.4" jni = { version = "0.20", default-features = false } -prost = "0.11" +prost = "0.12" rusqlite = "0.29" schemer = "0.2" secp256k1 = "0.21" secrecy = "0.8" zcash_address = "0.3" -zcash_client_backend = { version = "0.9", features = ["transparent-inputs", "unstable"] } -zcash_client_sqlite = { version = "0.7.1", features = ["transparent-inputs", "unstable"] } -zcash_primitives = "0.12" -zcash_proofs = "0.12" -orchard = { version = "0.5", default-features = false } +zcash_client_backend = { version = "=0.10.0-rc.1", features = ["transparent-inputs", "unstable"] } +zcash_client_sqlite = { version = "=0.8.0-rc.1", features = ["transparent-inputs", "unstable"] } +zcash_primitives = "=0.13.0-rc.1" +zcash_proofs = "=0.13.0-rc.1" +orchard = { version = "0.6", default-features = false } # Initialization rayon = "1.7" @@ -41,16 +41,6 @@ tracing-subscriber = "0.3" dlopen2 = "0.4" libc = "0.2" -[patch.crates-io] -incrementalmerkletree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "da97e6c399c5acddedad2c1730dbb7ee55499a2f" } -shardtree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "da97e6c399c5acddedad2c1730dbb7ee55499a2f" } -orchard = { git = "https://github.com/zcash/orchard.git", rev = "6ef89d5f154de2cf7b7dd87edb8d8c49158beebb" } -zcash_address = { git = "https://github.com/zcash/librustzcash.git", rev = "85156e2b7cb5c53c26c786ea7c5c22fe5cbbcb2d" } -zcash_client_backend = { git = "https://github.com/zcash/librustzcash.git", rev = "85156e2b7cb5c53c26c786ea7c5c22fe5cbbcb2d" } -zcash_client_sqlite = { git = "https://github.com/zcash/librustzcash.git", rev = "85156e2b7cb5c53c26c786ea7c5c22fe5cbbcb2d" } -zcash_primitives = { git = "https://github.com/zcash/librustzcash.git", rev = "85156e2b7cb5c53c26c786ea7c5c22fe5cbbcb2d" } -zcash_proofs = { git = "https://github.com/zcash/librustzcash.git", rev = "85156e2b7cb5c53c26c786ea7c5c22fe5cbbcb2d" } - ## Uncomment this to test librustzcash changes locally #[patch.crates-io] #zcash_address = { path = '../../clones/librustzcash/components/zcash_address' } From 9284e60e2aef950c2c17f79d17affc7bbd631108 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 8 Sep 2023 21:48:48 +0100 Subject: [PATCH 52/76] rust: dlopen2 0.6 --- backend-lib/Cargo.lock | 10 +++++----- backend-lib/Cargo.toml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/backend-lib/Cargo.lock b/backend-lib/Cargo.lock index 770ea51e..acc36f5f 100644 --- a/backend-lib/Cargo.lock +++ b/backend-lib/Cargo.lock @@ -471,9 +471,9 @@ dependencies = [ [[package]] name = "dlopen2" -version = "0.4.1" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b121caccfc363e4d9a4589528f3bef7c71b83c6ed01c8dc68cbeeb7fd29ec698" +checksum = "6bc2c7ed06fd72a8513ded8d0d2f6fd2655a85d6885c48cae8625d80faf28c03" dependencies = [ "dlopen2_derive", "libc", @@ -483,13 +483,13 @@ dependencies = [ [[package]] name = "dlopen2_derive" -version = "0.2.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a09ac8bb8c16a282264c379dffba707b9c998afc7506009137f3c6136888078" +checksum = "f2b99bf03862d7f545ebc28ddd33a665b50865f4dfd84031a393823879bd4c54" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.31", ] [[package]] diff --git a/backend-lib/Cargo.toml b/backend-lib/Cargo.toml index cbfc95e1..c87cff94 100644 --- a/backend-lib/Cargo.toml +++ b/backend-lib/Cargo.toml @@ -38,7 +38,7 @@ tracing = "0.1" tracing-subscriber = "0.3" # Conditional access to newer NDK features -dlopen2 = "0.4" +dlopen2 = "0.6" libc = "0.2" ## Uncomment this to test librustzcash changes locally From d3d54d7f15723f9fd2ebc24174854bab02ca68d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Honza=20Rychnovsk=C3=BD?= Date: Sat, 9 Sep 2023 19:07:52 +0200 Subject: [PATCH 53/76] Filter out zero denominator in Rust layer Co-authored-by: Kris Nuttycombe --- backend-lib/src/main/rust/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend-lib/src/main/rust/lib.rs b/backend-lib/src/main/rust/lib.rs index 61ed06d7..4b165b34 100644 --- a/backend-lib/src/main/rust/lib.rs +++ b/backend-lib/src/main/rust/lib.rs @@ -1173,7 +1173,7 @@ pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_ge match db_data .get_wallet_summary(0) .map_err(|e| format_err!("Error while fetching scan progress: {}", e))? - .and_then(|wallet_summary| wallet_summary.scan_progress()) + .and_then(|summary| summary.scan_progress().filter(|r| r.denominator() > 0)) { Some(progress) => encode_scan_progress(&env, progress), None => Ok(ptr::null_mut()), From a43c5f130e952adc2f625e0bced659bb65ca2af1 Mon Sep 17 00:00:00 2001 From: Honza Date: Sat, 9 Sep 2023 17:57:57 +0200 Subject: [PATCH 54/76] Change createAccountAndGetSpendingKey API The checkpoint parameter changed for treeState: ByteArray --- .../main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt | 8 ++++++-- .../cash/z/ecc/android/sdk/internal/TypesafeBackend.kt | 3 +-- .../z/ecc/android/sdk/internal/TypesafeBackendImpl.kt | 9 ++++----- .../ecc/android/sdk/internal/db/derived/DerivedDataDb.kt | 6 +++++- 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt index ef0a1d37..0f0dd65d 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt @@ -509,11 +509,15 @@ class SdkSynchronizer private constructor( // Not ready to be a public API; internal for testing only internal suspend fun createAccount( seed: ByteArray, - checkpoint: Checkpoint, + treeState: ByteArray, recoverUntil: BlockHeight? ): UnifiedSpendingKey? { return runCatching { - backend.createAccountAndGetSpendingKey(seed, checkpoint, recoverUntil) + backend.createAccountAndGetSpendingKey( + seed = seed, + treeState = treeState, + recoverUntil = recoverUntil + ) }.onFailure { Twig.error(it) { "Create account failed." } }.getOrNull() diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackend.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackend.kt index 9e56d5c7..0b12c44c 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackend.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackend.kt @@ -1,6 +1,5 @@ package cash.z.ecc.android.sdk.internal -import cash.z.ecc.android.sdk.internal.model.Checkpoint import cash.z.ecc.android.sdk.internal.model.JniBlockMeta import cash.z.ecc.android.sdk.internal.model.ScanProgress import cash.z.ecc.android.sdk.internal.model.ScanRange @@ -20,7 +19,7 @@ internal interface TypesafeBackend { suspend fun createAccountAndGetSpendingKey( seed: ByteArray, - checkpoint: Checkpoint, + treeState: ByteArray, recoverUntil: BlockHeight? ): UnifiedSpendingKey diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackendImpl.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackendImpl.kt index 22a9d381..9063e5e6 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackendImpl.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackendImpl.kt @@ -1,6 +1,5 @@ package cash.z.ecc.android.sdk.internal -import cash.z.ecc.android.sdk.internal.model.Checkpoint import cash.z.ecc.android.sdk.internal.model.JniBlockMeta import cash.z.ecc.android.sdk.internal.model.JniSubtreeRoot import cash.z.ecc.android.sdk.internal.model.ScanProgress @@ -23,14 +22,14 @@ internal class TypesafeBackendImpl(private val backend: Backend) : TypesafeBacke override suspend fun createAccountAndGetSpendingKey( seed: ByteArray, - checkpoint: Checkpoint, + treeState: ByteArray, recoverUntil: BlockHeight? ): UnifiedSpendingKey { return UnifiedSpendingKey( backend.createAccount( - seed, - checkpoint.treeState().encoded, - recoverUntil?.value + seed = seed, + treeState = treeState, + recoverUntil = recoverUntil?.value ) ) } diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/DerivedDataDb.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/DerivedDataDb.kt index 0948a8c9..32c65f78 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/DerivedDataDb.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/DerivedDataDb.kt @@ -77,7 +77,11 @@ internal class DerivedDataDb private constructor( } repeat(missingAccounts) { runCatching { - backend.createAccountAndGetSpendingKey(checkedSeed, checkpoint, recoverUntil) + backend.createAccountAndGetSpendingKey( + seed = checkedSeed, + treeState = checkpoint.treeState().encoded, + recoverUntil = recoverUntil + ) }.onFailure { Twig.error(it) { "Create account failed." } } From 79ba05f23b257aa120c4a6ef0d434650db1488c0 Mon Sep 17 00:00:00 2001 From: Honza Date: Sat, 9 Sep 2023 18:58:38 +0200 Subject: [PATCH 55/76] Change denominator requirement - Added safe progress ratio calculating function - Added test --- .../sdk/internal/model/JniScanProgress.kt | 6 +++-- .../processor/model/GetScanProgressResult.kt | 5 ++-- .../sdk/internal/model/ScanProgress.kt | 17 ++++++++++--- .../sdk/fixture/ScanProgressFixture.kt | 13 ++++++++++ .../sdk/internal/model/ScanProgressTest.kt | 24 +++++++++++++++++++ 5 files changed, 57 insertions(+), 8 deletions(-) create mode 100644 sdk-lib/src/test/java/cash/z/ecc/android/sdk/fixture/ScanProgressFixture.kt create mode 100644 sdk-lib/src/test/java/cash/z/ecc/android/sdk/internal/model/ScanProgressTest.kt diff --git a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniScanProgress.kt b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniScanProgress.kt index ce4a9c57..7756c4e0 100644 --- a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniScanProgress.kt +++ b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniScanProgress.kt @@ -17,8 +17,10 @@ class JniScanProgress( require(numerator >= 0L) { "Numerator $numerator is outside of allowed range [0, Long.MAX_VALUE]" } - require(denominator >= 1L) { - "Denominator $denominator is outside of allowed range [1, Long.MAX_VALUE]" + // Note that we allow the denominator to be 0 because of the internal rust implementation, and we treat this + // special case as 0 progress ratio in the receiver [ScanProgress] class. + require(denominator >= 0L) { + "Denominator $denominator is outside of allowed range [0, Long.MAX_VALUE]" } require(numerator.toFloat().div(denominator) >= 0f) { "Result of ${numerator.toFloat()}/$denominator is outside of allowed range" diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/GetScanProgressResult.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/GetScanProgressResult.kt index 3a219087..a3cb9449 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/GetScanProgressResult.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/model/GetScanProgressResult.kt @@ -7,9 +7,8 @@ import cash.z.ecc.android.sdk.model.PercentDecimal * Internal class for sharing get scan progress action result. */ internal sealed class GetScanProgressResult { - data class Success(val progress: ScanProgress) : GetScanProgressResult() { - fun toPercentDecimal() = - PercentDecimal(progress.numerator / progress.denominator.toFloat()) + data class Success(val scanProgress: ScanProgress) : GetScanProgressResult() { + fun toPercentDecimal() = PercentDecimal(scanProgress.getSafeRation()) } data object None : GetScanProgressResult() diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanProgress.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanProgress.kt index afb02478..79f4cd79 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanProgress.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/ScanProgress.kt @@ -1,10 +1,21 @@ package cash.z.ecc.android.sdk.internal.model internal data class ScanProgress( - val numerator: Long, - val denominator: Long + private val numerator: Long, + private val denominator: Long ) { - override fun toString() = "ScanProgress($numerator/$denominator) -> ${numerator / (denominator.toFloat())}" + override fun toString() = "ScanProgress($numerator/$denominator) -> ${getSafeRation()}" + + /** + * Returns progress ratio in [0, 1] range. Any out-of-range value is treated as 0. + */ + fun getSafeRation() = numerator.toFloat().div(denominator).let { ration -> + if (ration < 0f || ration > 1f) { + 0f + } else { + ration + } + } companion object { fun new(jni: JniScanProgress): ScanProgress { diff --git a/sdk-lib/src/test/java/cash/z/ecc/android/sdk/fixture/ScanProgressFixture.kt b/sdk-lib/src/test/java/cash/z/ecc/android/sdk/fixture/ScanProgressFixture.kt new file mode 100644 index 00000000..f3fd93b2 --- /dev/null +++ b/sdk-lib/src/test/java/cash/z/ecc/android/sdk/fixture/ScanProgressFixture.kt @@ -0,0 +1,13 @@ +package cash.z.ecc.android.sdk.fixture + +import cash.z.ecc.android.sdk.internal.model.ScanProgress + +object ScanProgressFixture { + internal const val DEFAULT_NUMERATOR = 50L + internal const val DEFAULT_DENOMINATOR = 100L + + internal fun new( + numerator: Long = DEFAULT_NUMERATOR, + denominator: Long = DEFAULT_DENOMINATOR + ) = ScanProgress(numerator, denominator) +} diff --git a/sdk-lib/src/test/java/cash/z/ecc/android/sdk/internal/model/ScanProgressTest.kt b/sdk-lib/src/test/java/cash/z/ecc/android/sdk/internal/model/ScanProgressTest.kt new file mode 100644 index 00000000..826e5051 --- /dev/null +++ b/sdk-lib/src/test/java/cash/z/ecc/android/sdk/internal/model/ScanProgressTest.kt @@ -0,0 +1,24 @@ +package cash.z.ecc.android.sdk.internal.model + +import cash.z.ecc.android.sdk.fixture.ScanProgressFixture +import kotlin.test.Test +import kotlin.test.assertEquals + +class ScanProgressTest { + @Test + fun get_valid_ratio_test() { + val scanProgress = ScanProgressFixture.new() + assertEquals( + scanProgress.getSafeRation(), + ScanProgressFixture.DEFAULT_NUMERATOR.toFloat().div(ScanProgressFixture.DEFAULT_DENOMINATOR) + ) + } + + @Test + fun get_fallback_ratio_test() { + val scanProgress = ScanProgressFixture.new( + denominator = 0 + ) + assertEquals(0f, scanProgress.getSafeRation()) + } +} From ffc1c9973861e4f3760837d1f2af841eaf8f2f90 Mon Sep 17 00:00:00 2001 From: Honza Date: Sat, 9 Sep 2023 19:03:04 +0200 Subject: [PATCH 56/76] JniScanRange height parameters constraint --- .../java/cash/z/ecc/android/sdk/internal/model/JniScanRange.kt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniScanRange.kt b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniScanRange.kt index c1c676b2..640dd173 100644 --- a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniScanRange.kt +++ b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniScanRange.kt @@ -25,5 +25,8 @@ class JniScanRange( require(endHeight.isInUIntRange()) { "Height $endHeight is outside of allowed UInt range" } + require(endHeight >= startHeight) { + "End height $endHeight must be greater than start height $startHeight." + } } } From 1b6039c36800beea17dbe156e85e1e4f45c1212f Mon Sep 17 00:00:00 2001 From: Honza Date: Sat, 9 Sep 2023 19:11:40 +0200 Subject: [PATCH 57/76] Reverting denominator constraint back The Rust layer now filters out the zero denominator but keeping the ScanProgress new safe ratio function with its test. --- .../z/ecc/android/sdk/internal/model/JniScanProgress.kt | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniScanProgress.kt b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniScanProgress.kt index 7756c4e0..ce4a9c57 100644 --- a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniScanProgress.kt +++ b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniScanProgress.kt @@ -17,10 +17,8 @@ class JniScanProgress( require(numerator >= 0L) { "Numerator $numerator is outside of allowed range [0, Long.MAX_VALUE]" } - // Note that we allow the denominator to be 0 because of the internal rust implementation, and we treat this - // special case as 0 progress ratio in the receiver [ScanProgress] class. - require(denominator >= 0L) { - "Denominator $denominator is outside of allowed range [0, Long.MAX_VALUE]" + require(denominator >= 1L) { + "Denominator $denominator is outside of allowed range [1, Long.MAX_VALUE]" } require(numerator.toFloat().div(denominator) >= 0f) { "Result of ${numerator.toFloat()}/$denominator is outside of allowed range" From 5e85c39b78f5ac0b9d466c8cdc7d9a6af47b0cb8 Mon Sep 17 00:00:00 2001 From: Honza Date: Sat, 9 Sep 2023 19:32:35 +0200 Subject: [PATCH 58/76] Silent rust compile error mismatched types: expected `&u64`, found integer --- backend-lib/src/main/rust/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend-lib/src/main/rust/lib.rs b/backend-lib/src/main/rust/lib.rs index 4b165b34..3bb40e9f 100644 --- a/backend-lib/src/main/rust/lib.rs +++ b/backend-lib/src/main/rust/lib.rs @@ -1173,7 +1173,7 @@ pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_ge match db_data .get_wallet_summary(0) .map_err(|e| format_err!("Error while fetching scan progress: {}", e))? - .and_then(|summary| summary.scan_progress().filter(|r| r.denominator() > 0)) + .and_then(|summary| summary.scan_progress().filter(|r| r.denominator() > &0)) { Some(progress) => encode_scan_progress(&env, progress), None => Ok(ptr::null_mut()), From 95da0f25ab2ab4e935626670c1ee099c49b204cc Mon Sep 17 00:00:00 2001 From: Honza Date: Sat, 9 Sep 2023 19:56:56 +0200 Subject: [PATCH 59/76] BlockHeight subtraction fix --- .../java/cash/z/ecc/android/sdk/model/BlockHeight.kt | 2 +- .../cash/z/ecc/android/sdk/model/BlockHeightTest.kt | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/model/BlockHeight.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/model/BlockHeight.kt index b15a3c1b..ed4b42ef 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/model/BlockHeight.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/model/BlockHeight.kt @@ -54,7 +54,7 @@ data class BlockHeight internal constructor(val value: Long) : Comparable Date: Sun, 10 Sep 2023 10:19:08 +0200 Subject: [PATCH 60/76] retryUpToAndContinue exception wrapper trigger fix --- .../java/cash/z/ecc/android/sdk/internal/ext/WalletService.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/WalletService.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/WalletService.kt index 3f470546..a729ec54 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/WalletService.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/WalletService.kt @@ -66,7 +66,7 @@ suspend inline fun retryUpToAndContinue( return } catch (t: Throwable) { failedAttempts++ - if (failedAttempts > retries) { + if (failedAttempts == retries) { exceptionWrapper(t) } val duration = (initialDelayMillis.toDouble() * 2.0.pow(failedAttempts.toDouble() - 1)).toLong() From 4c8c32587b6968bd64fb38597fb123d7ca0989f2 Mon Sep 17 00:00:00 2001 From: Honza Date: Sun, 10 Sep 2023 10:31:56 +0200 Subject: [PATCH 61/76] Add note on scan continuity error texts --- .../java/cash/z/ecc/android/sdk/internal/ext/ExceptionExt.kt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/ExceptionExt.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/ExceptionExt.kt index 92b40975..b87426c2 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/ExceptionExt.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/ExceptionExt.kt @@ -45,6 +45,8 @@ internal inline fun tryWarn( } } +// Note: Do NOT change these texts as they match the ones from ScanError in +// librustzcash/zcash_client_backend/src/scanning.rs internal const val PREV_HASH_MISMATCH = "The parent hash of proposed block does not correspond to the block hash at " + "height" // $NON-NLS internal const val BLOCK_HEIGHT_DISCONTINUITY = "Block height discontinuity at height" // $NON-NLS From fe7edf3ec581e975d420208df2bf451108dedc30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Honza=20Rychnovsk=C3=BD?= Date: Sun, 10 Sep 2023 11:45:53 +0200 Subject: [PATCH 62/76] Fix Changelog typos Co-authored-by: Daira Emma Hopwood --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 01350cc7..3e2d2750 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,8 +17,8 @@ effect on the current rewind functionality result. - Internally, we removed access to the shared block table from the Kotlin layer, which resulted in eliminating these APIs: - - `SdkSynchornizer.findBlockHash()` - - `SdkSynchornizer.findBlockHashAsHex()` + - `SdkSynchronizer.findBlockHash()` + - `SdkSynchronizer.findBlockHashAsHex()` ### Changed - `CompactBlockProcessor.quickRewind()` and `CompactBlockProcessor.rewindToNearestHeight()` now might fail due to From c9d3085001b8a4079721640d2208199974b8db9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Honza=20Rychnovsk=C3=BD?= Date: Sun, 10 Sep 2023 11:47:46 +0200 Subject: [PATCH 63/76] Update Changelog Co-authored-by: Daira Emma Hopwood --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e2d2750..c964a5e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,8 +13,8 @@ case of older Linear synchronization algorithm. - `CompactBlockProcessor.ProcessorInfo.isSyncing`. Use `Synchronizer.status` instead. - `CompactBlockProcessor.ProcessorInfo.syncProgress`. Use `Synchronizer.progress` instead. -- `alsoClearBlockCache` parameter from rewind functions of `Synchronizer` and `CompactBlockProcessor` as it take no - effect on the current rewind functionality result. +- `alsoClearBlockCache` parameter from rewind functions of `Synchronizer` and `CompactBlockProcessor`, as it has no + effect on the current behaviour of these functions. - Internally, we removed access to the shared block table from the Kotlin layer, which resulted in eliminating these APIs: - `SdkSynchronizer.findBlockHash()` From 4c38354f019bf063b87c9c46c49c781ec46533fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Honza=20Rychnovsk=C3=BD?= Date: Sun, 10 Sep 2023 11:50:59 +0200 Subject: [PATCH 64/76] Update Changelog Co-authored-by: Daira Emma Hopwood --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c964a5e7..91f381fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,7 +29,7 @@ ### Fixed - `Synchronizer.getMemos()` now correctly returns a flow of strings for sent and received transactions. Issue **#1154**. - `CompactBlockProcessor` now triggers transaction polling while block synchronization is in progress as expected. - Clients will be notified briefly after every new transaction is discovered via `Synchronizer.transactions` API. + Clients will be notified shortly after every new transaction is discovered via `Synchronizer.transactions` API. Issue **#1170**. ## 1.21.0-beta01 From a082fe9034117ca7b5336ad65eb9dc90e45cc5f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Honza=20Rychnovsk=C3=BD?= Date: Mon, 11 Sep 2023 09:11:01 +0200 Subject: [PATCH 65/76] [#1231] GetConsensusBranchId works with chain tip (#1232) * [#1231] GetConsensusBranchId works with chain tip Closes #1231 --- .../cash/z/ecc/android/sdk/SdkSynchronizer.kt | 27 +++++++++++++++++-- .../transaction/TransactionEncoder.kt | 7 ++++- .../transaction/TransactionEncoderImpl.kt | 15 ++++++++--- 3 files changed, 43 insertions(+), 6 deletions(-) diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt index 0f0dd65d..41e36c15 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt @@ -27,6 +27,7 @@ import cash.z.ecc.android.sdk.internal.ext.isNullOrEmpty import cash.z.ecc.android.sdk.internal.ext.tryNull import cash.z.ecc.android.sdk.internal.jni.RustBackend import cash.z.ecc.android.sdk.internal.model.Checkpoint +import cash.z.ecc.android.sdk.internal.model.ext.toBlockHeight import cash.z.ecc.android.sdk.internal.repository.CompactBlockRepository import cash.z.ecc.android.sdk.internal.repository.DerivedDataRepository import cash.z.ecc.android.sdk.internal.storage.block.FileCompactBlockRepository @@ -50,6 +51,7 @@ import cash.z.ecc.android.sdk.type.AddressType.Unified import cash.z.ecc.android.sdk.type.ConsensusMatchType import co.electriccoin.lightwallet.client.LightWalletClient import co.electriccoin.lightwallet.client.model.LightWalletEndpoint +import co.electriccoin.lightwallet.client.model.Response import co.electriccoin.lightwallet.client.new import kotlinx.coroutines.CoroutineExceptionHandler import kotlinx.coroutines.CoroutineScope @@ -623,9 +625,30 @@ class SdkSynchronizer private constructor( override suspend fun validateConsensusBranch(): ConsensusMatchType { val serverBranchId = tryNull { processor.downloader.getServerInfo()?.consensusBranchId } - val sdkBranchId = tryNull { - (txManager as OutboundTransactionManagerImpl).encoder.getConsensusBranchId() + + val currentChainTip = when ( + val response = + processor.downloader.getLatestBlockHeight() + ) { + is Response.Success -> { + Twig.info { "Chain tip for validate consensus branch action fetched: ${response.result.value}" } + runCatching { response.result.toBlockHeight(network) }.getOrNull() + } + is Response.Failure -> { + Twig.error { + "Chain tip fetch failed for validate consensus branch action with:" + + " ${response.toThrowable()}" + } + null + } } + + val sdkBranchId = currentChainTip?.let { + tryNull { + (txManager as OutboundTransactionManagerImpl).encoder.getConsensusBranchId(currentChainTip) + } + } + return ConsensusMatchType( sdkBranchId?.let { ConsensusBranchId.fromId(it) }, serverBranchId?.let { ConsensusBranchId.fromHex(it) } diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/transaction/TransactionEncoder.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/transaction/TransactionEncoder.kt index 97bd5185..4a5383e9 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/transaction/TransactionEncoder.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/transaction/TransactionEncoder.kt @@ -1,6 +1,7 @@ package cash.z.ecc.android.sdk.internal.transaction import cash.z.ecc.android.sdk.internal.model.EncodedTransaction +import cash.z.ecc.android.sdk.model.BlockHeight import cash.z.ecc.android.sdk.model.TransactionRecipient import cash.z.ecc.android.sdk.model.UnifiedSpendingKey import cash.z.ecc.android.sdk.model.Zatoshi @@ -69,6 +70,10 @@ internal interface TransactionEncoder { /** * Return the consensus branch that the encoder is using when making transactions. + * + * @param height the height at which we want to get the consensus branch + * + * @return id of consensus branch */ - suspend fun getConsensusBranchId(): Long + suspend fun getConsensusBranchId(height: BlockHeight): Long } diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/transaction/TransactionEncoderImpl.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/transaction/TransactionEncoderImpl.kt index 8a4c626e..e174a20d 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/transaction/TransactionEncoderImpl.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/transaction/TransactionEncoderImpl.kt @@ -7,6 +7,7 @@ import cash.z.ecc.android.sdk.internal.Twig import cash.z.ecc.android.sdk.internal.TypesafeBackend import cash.z.ecc.android.sdk.internal.model.EncodedTransaction import cash.z.ecc.android.sdk.internal.repository.DerivedDataRepository +import cash.z.ecc.android.sdk.model.BlockHeight import cash.z.ecc.android.sdk.model.FirstClassByteArray import cash.z.ecc.android.sdk.model.TransactionRecipient import cash.z.ecc.android.sdk.model.UnifiedSpendingKey @@ -97,9 +98,17 @@ internal class TransactionEncoderImpl( override suspend fun isValidUnifiedAddress(address: String): Boolean = backend.isValidUnifiedAddr(address) - override suspend fun getConsensusBranchId(): Long { - val height = backend.getMaxScannedHeight() - if (height == null || height < backend.network.saplingActivationHeight) { + /** + * Return the consensus branch that the encoder is using when making transactions. + * + * @param height the height at which we want to get the consensus branch + * + * @return id of consensus branch + * + * @throws TransactionEncoderException.IncompleteScanException if the [height] is less than activation height + */ + override suspend fun getConsensusBranchId(height: BlockHeight): Long { + if (height < backend.network.saplingActivationHeight) { throw TransactionEncoderException.IncompleteScanException(height) } return backend.getBranchIdForHeight(height) From 916f9e957b14bcf00e122e2477f46e86fd1dd862 Mon Sep 17 00:00:00 2001 From: Honza Date: Mon, 11 Sep 2023 13:37:12 +0200 Subject: [PATCH 66/76] Update PersistableWallet API These required changes came from the latest SDK acceptance testing with Zashi wallet. --- CHANGELOG.md | 4 ++-- .../screen/home/viewmodel/WalletViewModel.kt | 8 +++++--- .../ui/screen/seed/view/ConfigureSeedView.kt | 7 +++++-- .../sdk/fixture/PersistableWalletFixture.kt | 8 ++++++-- .../z/ecc/android/sdk/WalletCoordinator.kt | 2 +- .../android/sdk/model/PersistableWallet.kt | 20 ++++++++++++++----- 6 files changed, 34 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 91f381fc..b55b1eb8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,8 +23,8 @@ ### Changed - `CompactBlockProcessor.quickRewind()` and `CompactBlockProcessor.rewindToNearestHeight()` now might fail due to internal changes in getting scanned height. Thus, these functions return `Boolean` results. -- `Synchronizer.new()` requires a new `walletInitMode` parameter of type `WalletInitMode`, which describes wallet - initialization mode. See related function and sealed class documentation. +- `Synchronizer.new()` and `PersistableWallet` APIs require a new `walletInitMode` parameter of type `WalletInitMode`, + which describes wallet initialization mode. See related function and sealed class documentation. ### Fixed - `Synchronizer.getMemos()` now correctly returns a flow of strings for sent and received transactions. Issue **#1154**. diff --git a/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/ui/screen/home/viewmodel/WalletViewModel.kt b/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/ui/screen/home/viewmodel/WalletViewModel.kt index dd2c3a3e..54376f75 100644 --- a/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/ui/screen/home/viewmodel/WalletViewModel.kt +++ b/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/ui/screen/home/viewmodel/WalletViewModel.kt @@ -141,10 +141,13 @@ class WalletViewModel(application: Application) : AndroidViewModel(application) */ fun persistNewWallet() { val application = getApplication() - PersistableWallet.walletInitMode = WalletInitMode.NewWallet viewModelScope.launch { - val newWallet = PersistableWallet.new(application, ZcashNetwork.fromResources(application)) + val newWallet = PersistableWallet.new( + application, + ZcashNetwork.fromResources(application), + WalletInitMode.NewWallet + ) persistWallet(newWallet) } } @@ -154,7 +157,6 @@ class WalletViewModel(application: Application) : AndroidViewModel(application) * to see the side effects. This would be used for a user restoring a wallet from a backup. */ fun persistExistingWallet(persistableWallet: PersistableWallet) { - PersistableWallet.walletInitMode = WalletInitMode.RestoreWallet persistWallet(persistableWallet) } diff --git a/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/ui/screen/seed/view/ConfigureSeedView.kt b/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/ui/screen/seed/view/ConfigureSeedView.kt index 17e0cf9a..9341579e 100644 --- a/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/ui/screen/seed/view/ConfigureSeedView.kt +++ b/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/ui/screen/seed/view/ConfigureSeedView.kt @@ -15,6 +15,7 @@ import androidx.compose.runtime.Composable import androidx.compose.ui.Modifier import androidx.compose.ui.res.stringResource import androidx.compose.ui.tooling.preview.Preview +import cash.z.ecc.android.sdk.WalletInitMode import cash.z.ecc.android.sdk.demoapp.R import cash.z.ecc.android.sdk.fixture.WalletFixture import cash.z.ecc.android.sdk.model.PersistableWallet @@ -76,7 +77,8 @@ private fun ConfigureSeedMainContent( val newWallet = PersistableWallet( zcashNetwork, WalletFixture.Alice.getBirthday(zcashNetwork), - SeedPhrase.new(WalletFixture.Alice.seedPhrase) + SeedPhrase.new(WalletFixture.Alice.seedPhrase), + WalletInitMode.RestoreWallet ) onExistingWallet(newWallet) } @@ -88,7 +90,8 @@ private fun ConfigureSeedMainContent( val newWallet = PersistableWallet( zcashNetwork, WalletFixture.Ben.getBirthday(zcashNetwork), - SeedPhrase.new(WalletFixture.Ben.seedPhrase) + SeedPhrase.new(WalletFixture.Ben.seedPhrase), + WalletInitMode.RestoreWallet ) onExistingWallet(newWallet) } diff --git a/sdk-incubator-lib/src/androidTest/kotlin/cash/z/ecc/android/sdk/fixture/PersistableWalletFixture.kt b/sdk-incubator-lib/src/androidTest/kotlin/cash/z/ecc/android/sdk/fixture/PersistableWalletFixture.kt index cf69700e..c3351529 100644 --- a/sdk-incubator-lib/src/androidTest/kotlin/cash/z/ecc/android/sdk/fixture/PersistableWalletFixture.kt +++ b/sdk-incubator-lib/src/androidTest/kotlin/cash/z/ecc/android/sdk/fixture/PersistableWalletFixture.kt @@ -1,5 +1,6 @@ package cash.z.ecc.android.sdk.fixture +import cash.z.ecc.android.sdk.WalletInitMode import cash.z.ecc.android.sdk.model.BlockHeight import cash.z.ecc.android.sdk.model.PersistableWallet import cash.z.ecc.android.sdk.model.SeedPhrase @@ -15,9 +16,12 @@ object PersistableWalletFixture { val SEED_PHRASE = SeedPhraseFixture.new() + val WALLET_INIT_MODE = WalletInitMode.ExistingWallet + fun new( network: ZcashNetwork = NETWORK, birthday: BlockHeight = BIRTHDAY, - seedPhrase: SeedPhrase = SEED_PHRASE - ) = PersistableWallet(network, birthday, seedPhrase) + seedPhrase: SeedPhrase = SEED_PHRASE, + walletInitMode: WalletInitMode = WALLET_INIT_MODE + ) = PersistableWallet(network, birthday, seedPhrase, walletInitMode) } diff --git a/sdk-incubator-lib/src/main/java/cash/z/ecc/android/sdk/WalletCoordinator.kt b/sdk-incubator-lib/src/main/java/cash/z/ecc/android/sdk/WalletCoordinator.kt index 5f1f9e52..ecad251f 100644 --- a/sdk-incubator-lib/src/main/java/cash/z/ecc/android/sdk/WalletCoordinator.kt +++ b/sdk-incubator-lib/src/main/java/cash/z/ecc/android/sdk/WalletCoordinator.kt @@ -80,7 +80,7 @@ class WalletCoordinator( lightWalletEndpoint = LightWalletEndpoint.defaultForNetwork(persistableWallet.network), birthday = persistableWallet.birthday, seed = persistableWallet.seedPhrase.toByteArray(), - walletInitMode = PersistableWallet.walletInitMode, + walletInitMode = persistableWallet.walletInitMode, ) trySend(InternalSynchronizerStatus.Available(closeableSynchronizer)) diff --git a/sdk-incubator-lib/src/main/java/cash/z/ecc/android/sdk/model/PersistableWallet.kt b/sdk-incubator-lib/src/main/java/cash/z/ecc/android/sdk/model/PersistableWallet.kt index 370431c7..41c96835 100644 --- a/sdk-incubator-lib/src/main/java/cash/z/ecc/android/sdk/model/PersistableWallet.kt +++ b/sdk-incubator-lib/src/main/java/cash/z/ecc/android/sdk/model/PersistableWallet.kt @@ -14,8 +14,12 @@ import org.json.JSONObject data class PersistableWallet( val network: ZcashNetwork, val birthday: BlockHeight?, - val seedPhrase: SeedPhrase + val seedPhrase: SeedPhrase, + val walletInitMode: WalletInitMode ) { + init { + _walletInitMode = walletInitMode + } /** * @return Wallet serialized to JSON format, suitable for long-term encrypted storage. @@ -44,7 +48,7 @@ data class PersistableWallet( // Note: This is not the ideal way to hold such a value. But we also want to avoid persisting the wallet // initialization mode with the persistable wallet. - var walletInitMode: WalletInitMode = WalletInitMode.ExistingWallet + private var _walletInitMode: WalletInitMode = WalletInitMode.ExistingWallet fun from(jsonObject: JSONObject): PersistableWallet { when (val version = jsonObject.getInt(KEY_VERSION)) { @@ -64,7 +68,8 @@ data class PersistableWallet( return PersistableWallet( network = network, birthday = birthday, - seedPhrase = SeedPhrase.new(seedPhrase) + seedPhrase = SeedPhrase.new(seedPhrase), + walletInitMode = _walletInitMode ) } else -> { @@ -76,7 +81,11 @@ data class PersistableWallet( /** * @return A new PersistableWallet with a random seed phrase. */ - suspend fun new(application: Application, zcashNetwork: ZcashNetwork): PersistableWallet { + suspend fun new( + application: Application, + zcashNetwork: ZcashNetwork, + walletInitMode: WalletInitMode + ): PersistableWallet { val birthday = BlockHeight.ofLatestCheckpoint(application, zcashNetwork) val seedPhrase = newSeedPhrase() @@ -84,7 +93,8 @@ data class PersistableWallet( return PersistableWallet( zcashNetwork, birthday, - seedPhrase + seedPhrase, + walletInitMode ) } } From c7125f4183ae872bc9366c58c7f5d3c38df494e2 Mon Sep 17 00:00:00 2001 From: Honza Date: Mon, 11 Sep 2023 19:11:03 +0200 Subject: [PATCH 67/76] Update createAccountAndGetSpendingKey API - TypesafeBackend.createAccountAndGetSpendingKey now works with a type-safe TreeState model class instead of ByteArray. - New type-safe TreeState added. Once we get the TreeState object from the lightwalletd server, potential validation comes into this object. --- .../cash/z/ecc/android/sdk/SdkSynchronizer.kt | 3 ++- .../android/sdk/internal/TypesafeBackend.kt | 3 ++- .../sdk/internal/TypesafeBackendImpl.kt | 5 ++-- .../sdk/internal/db/derived/DerivedDataDb.kt | 2 +- .../android/sdk/internal/model/Checkpoint.kt | 4 +-- .../android/sdk/internal/model/TreeState.kt | 26 +++++++++++++++++++ 6 files changed, 36 insertions(+), 7 deletions(-) create mode 100644 sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/TreeState.kt diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt index 41e36c15..7fedfcd4 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt @@ -27,6 +27,7 @@ import cash.z.ecc.android.sdk.internal.ext.isNullOrEmpty import cash.z.ecc.android.sdk.internal.ext.tryNull import cash.z.ecc.android.sdk.internal.jni.RustBackend import cash.z.ecc.android.sdk.internal.model.Checkpoint +import cash.z.ecc.android.sdk.internal.model.TreeState import cash.z.ecc.android.sdk.internal.model.ext.toBlockHeight import cash.z.ecc.android.sdk.internal.repository.CompactBlockRepository import cash.z.ecc.android.sdk.internal.repository.DerivedDataRepository @@ -511,7 +512,7 @@ class SdkSynchronizer private constructor( // Not ready to be a public API; internal for testing only internal suspend fun createAccount( seed: ByteArray, - treeState: ByteArray, + treeState: TreeState, recoverUntil: BlockHeight? ): UnifiedSpendingKey? { return runCatching { diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackend.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackend.kt index 0b12c44c..23c19f1c 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackend.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackend.kt @@ -4,6 +4,7 @@ import cash.z.ecc.android.sdk.internal.model.JniBlockMeta import cash.z.ecc.android.sdk.internal.model.ScanProgress import cash.z.ecc.android.sdk.internal.model.ScanRange import cash.z.ecc.android.sdk.internal.model.SubtreeRoot +import cash.z.ecc.android.sdk.internal.model.TreeState import cash.z.ecc.android.sdk.model.Account import cash.z.ecc.android.sdk.model.BlockHeight import cash.z.ecc.android.sdk.model.FirstClassByteArray @@ -19,7 +20,7 @@ internal interface TypesafeBackend { suspend fun createAccountAndGetSpendingKey( seed: ByteArray, - treeState: ByteArray, + treeState: TreeState, recoverUntil: BlockHeight? ): UnifiedSpendingKey diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackendImpl.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackendImpl.kt index 9063e5e6..41801599 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackendImpl.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/TypesafeBackendImpl.kt @@ -5,6 +5,7 @@ import cash.z.ecc.android.sdk.internal.model.JniSubtreeRoot import cash.z.ecc.android.sdk.internal.model.ScanProgress import cash.z.ecc.android.sdk.internal.model.ScanRange import cash.z.ecc.android.sdk.internal.model.SubtreeRoot +import cash.z.ecc.android.sdk.internal.model.TreeState import cash.z.ecc.android.sdk.model.Account import cash.z.ecc.android.sdk.model.BlockHeight import cash.z.ecc.android.sdk.model.FirstClassByteArray @@ -22,13 +23,13 @@ internal class TypesafeBackendImpl(private val backend: Backend) : TypesafeBacke override suspend fun createAccountAndGetSpendingKey( seed: ByteArray, - treeState: ByteArray, + treeState: TreeState, recoverUntil: BlockHeight? ): UnifiedSpendingKey { return UnifiedSpendingKey( backend.createAccount( seed = seed, - treeState = treeState, + treeState = treeState.encoded, recoverUntil = recoverUntil?.value ) ) diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/DerivedDataDb.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/DerivedDataDb.kt index 32c65f78..945a18cb 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/DerivedDataDb.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/db/derived/DerivedDataDb.kt @@ -79,7 +79,7 @@ internal class DerivedDataDb private constructor( runCatching { backend.createAccountAndGetSpendingKey( seed = checkedSeed, - treeState = checkpoint.treeState().encoded, + treeState = checkpoint.treeState(), recoverUntil = recoverUntil ) }.onFailure { diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/Checkpoint.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/Checkpoint.kt index 6173c6c1..a06c1ab9 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/Checkpoint.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/Checkpoint.kt @@ -20,11 +20,11 @@ internal data class Checkpoint( // Note: this field does NOT match the name of the JSON, so will break with field-based JSON parsing val tree: String ) { - fun treeState(): TreeStateUnsafe { + fun treeState(): TreeState { require(epochSeconds.isInUIntRange()) { "epochSeconds $epochSeconds is outside of allowed UInt range" } - return TreeStateUnsafe.fromParts(height.value, hash, epochSeconds.toInt(), tree) + return TreeState.fromParts(height.value, hash, epochSeconds.toInt(), tree) } internal companion object diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/TreeState.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/TreeState.kt new file mode 100644 index 00000000..927db67c --- /dev/null +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/TreeState.kt @@ -0,0 +1,26 @@ +package cash.z.ecc.android.sdk.internal.model + +import co.electriccoin.lightwallet.client.model.TreeStateUnsafe + +class TreeState( + val encoded: ByteArray +) { + companion object { + fun new(unsafe: TreeStateUnsafe): TreeState { + // Potential validation comes here + return TreeState( + encoded = unsafe.encoded + ) + } + + fun fromParts( + height: Long, + hash: String, + time: Int, + tree: String + ): TreeState { + val unsafeTreeState = TreeStateUnsafe.fromParts(height, hash, time, tree) + return TreeState.new(unsafeTreeState) + } + } +} From 39ba2b48b0cf4f42c728423fcef45425af1edfd7 Mon Sep 17 00:00:00 2001 From: Honza Date: Mon, 11 Sep 2023 20:22:41 +0200 Subject: [PATCH 68/76] Fix ktlint warning --- .../java/cash/z/ecc/android/sdk/internal/model/Checkpoint.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/Checkpoint.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/Checkpoint.kt index a06c1ab9..ee3a72de 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/Checkpoint.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/Checkpoint.kt @@ -2,7 +2,6 @@ package cash.z.ecc.android.sdk.internal.model import cash.z.ecc.android.sdk.internal.ext.isInUIntRange import cash.z.ecc.android.sdk.model.BlockHeight -import co.electriccoin.lightwallet.client.model.TreeStateUnsafe /** * Represents a checkpoint, which is used to speed sync times. From 01e9b058cbf8260a3c5c9a6f2fe1e4a8b4a63e44 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Tue, 12 Sep 2023 14:26:32 +0000 Subject: [PATCH 69/76] rust: Handle zero-conf transparent funds the same way as iOS --- backend-lib/src/main/rust/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend-lib/src/main/rust/lib.rs b/backend-lib/src/main/rust/lib.rs index 3bb40e9f..c632ae1e 100644 --- a/backend-lib/src/main/rust/lib.rs +++ b/backend-lib/src/main/rust/lib.rs @@ -734,7 +734,7 @@ pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_ge .map_err(|e| format_err!("Error while fetching anchor height: {}", e)) .and_then(|opt_anchor| { opt_anchor - .map(|(_, a)| a) + .map(|(target, _)| target) // Include unconfirmed funds. .ok_or(format_err!("Anchor height not available; scan required.")) }) .and_then(|anchor| { @@ -1488,7 +1488,7 @@ pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_sh .map_err(|e| format_err!("Error while fetching anchor height: {}", e)) .and_then(|opt_anchor| { opt_anchor - .map(|(_, a)| a) + .map(|(target, _)| target) // Include unconfirmed funds. .ok_or(format_err!("Anchor height not available; scan required.")) }) .and_then(|anchor| { From d162cdf35f46b7a50e7bf26707de13f3dc42b4bc Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Tue, 12 Sep 2023 14:29:35 +0000 Subject: [PATCH 70/76] rust: Migrate to `hdwallet 0.4` to de-duplicate dependencies --- backend-lib/Cargo.lock | 140 +++++++---------------------------------- backend-lib/Cargo.toml | 6 +- 2 files changed, 27 insertions(+), 119 deletions(-) diff --git a/backend-lib/Cargo.lock b/backend-lib/Cargo.lock index acc36f5f..c84c06ff 100644 --- a/backend-lib/Cargo.lock +++ b/backend-lib/Cargo.lock @@ -24,7 +24,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" dependencies = [ "crypto-common", - "generic-array 0.14.7", + "generic-array", ] [[package]] @@ -105,9 +105,9 @@ dependencies = [ [[package]] name = "base58" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5024ee8015f02155eee35c711107ddd9a9bf3cb689cf2a9089c97e79b6e1ae83" +checksum = "6107fe1be6682a68940da878d9e9f5e90ca5745b3dec9fd1bb393c8777d4f581" [[package]] name = "base64" @@ -208,34 +208,13 @@ dependencies = [ "constant_time_eq", ] -[[package]] -name = "block-buffer" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" -dependencies = [ - "block-padding", - "byte-tools", - "byteorder", - "generic-array 0.12.4", -] - [[package]] name = "block-buffer" version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ - "generic-array 0.14.7", -] - -[[package]] -name = "block-padding" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" -dependencies = [ - "byte-tools", + "generic-array", ] [[package]] @@ -267,12 +246,6 @@ version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" -[[package]] -name = "byte-tools" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" - [[package]] name = "byteorder" version = "1.4.3" @@ -430,7 +403,7 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ - "generic-array 0.14.7", + "generic-array", "typenum", ] @@ -449,22 +422,13 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" -[[package]] -name = "digest" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" -dependencies = [ - "generic-array 0.12.4", -] - [[package]] name = "digest" version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer 0.10.4", + "block-buffer", "crypto-common", "subtle", ] @@ -621,15 +585,6 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" -[[package]] -name = "generic-array" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffdf9f34f1447443d37393cc6c2b8313aebddcd96906caf34e54c68d8e57d7bd" -dependencies = [ - "typenum", -] - [[package]] name = "generic-array" version = "0.14.7" @@ -728,18 +683,6 @@ dependencies = [ "hashbrown", ] -[[package]] -name = "hdwallet" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cd89bf343be18dbe1e505100e48168bbd084760e842a8fed0317d2361470193" -dependencies = [ - "lazy_static", - "rand_core", - "ring", - "secp256k1 0.21.3", -] - [[package]] name = "hdwallet" version = "0.4.1" @@ -749,20 +692,20 @@ dependencies = [ "lazy_static", "rand_core", "ring", - "secp256k1 0.26.0", + "secp256k1", "thiserror", ] [[package]] name = "hdwallet-bitcoin" -version = "0.3.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "969c513e03167e65d4bb59f5c51ec3820210975044ad7f218ab801fc169760fa" +checksum = "4412333586deae44def90f2627065d95928decda2315de17df4551ac32059542" dependencies = [ "base58", - "hdwallet 0.3.1", + "hdwallet", "hex", - "ripemd160", + "ripemd", ] [[package]] @@ -789,7 +732,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest 0.10.7", + "digest", ] [[package]] @@ -826,7 +769,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" dependencies = [ - "generic-array 0.14.7", + "generic-array", ] [[package]] @@ -1094,12 +1037,6 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" -[[package]] -name = "opaque-debug" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" - [[package]] name = "opaque-debug" version = "0.3.0" @@ -1195,7 +1132,7 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271779f35b581956db91a3e55737327a03aa051e90b1c47aeb189508533adfd7" dependencies = [ - "digest 0.10.7", + "digest", "password-hash", ] @@ -1234,7 +1171,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf" dependencies = [ "cpufeatures", - "opaque-debug 0.3.0", + "opaque-debug", "universal-hash", ] @@ -1474,18 +1411,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd124222d17ad93a644ed9d011a40f4fb64aa54275c08cc216524a9ea82fb09f" dependencies = [ - "digest 0.10.7", -] - -[[package]] -name = "ripemd160" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad5112e0dbbb87577bfbc56c42450235e3012ce336e29c5befd7807bd626da4a" -dependencies = [ - "block-buffer 0.7.3", - "digest 0.8.1", - "opaque-debug 0.2.3", + "digest", ] [[package]] @@ -1560,31 +1486,13 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "secp256k1" -version = "0.21.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c42e6f1735c5f00f51e43e28d6634141f2bcad10931b2609ddd74a86d751260" -dependencies = [ - "secp256k1-sys 0.4.2", -] - [[package]] name = "secp256k1" version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4124a35fe33ae14259c490fd70fa199a32b9ce9502f2ee6bc4f81ec06fa65894" dependencies = [ - "secp256k1-sys 0.8.1", -] - -[[package]] -name = "secp256k1-sys" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "957da2573cde917463ece3570eab4a0b3f19de6f1646cde62e6fd3868f566036" -dependencies = [ - "cc", + "secp256k1-sys", ] [[package]] @@ -1633,7 +1541,7 @@ checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" dependencies = [ "cfg-if", "cpufeatures", - "digest 0.10.7", + "digest", ] [[package]] @@ -2167,7 +2075,7 @@ version = "0.0.4" dependencies = [ "dlopen2", "failure", - "hdwallet 0.3.1", + "hdwallet", "hdwallet-bitcoin", "hex", "jni", @@ -2179,7 +2087,7 @@ dependencies = [ "rayon", "rusqlite", "schemer", - "secp256k1 0.21.3", + "secp256k1", "secrecy", "tracing", "tracing-subscriber", @@ -2215,7 +2123,7 @@ dependencies = [ "byteorder", "crossbeam-channel", "group", - "hdwallet 0.4.1", + "hdwallet", "hex", "incrementalmerkletree", "memuse", @@ -2246,7 +2154,7 @@ dependencies = [ "bs58", "byteorder", "group", - "hdwallet 0.4.1", + "hdwallet", "incrementalmerkletree", "jubjub", "maybe-rayon", @@ -2304,7 +2212,7 @@ dependencies = [ "ff", "fpe", "group", - "hdwallet 0.4.1", + "hdwallet", "hex", "incrementalmerkletree", "jubjub", @@ -2315,7 +2223,7 @@ dependencies = [ "rand", "rand_core", "ripemd", - "secp256k1 0.26.0", + "secp256k1", "sha2", "subtle", "zcash_address", diff --git a/backend-lib/Cargo.toml b/backend-lib/Cargo.toml index c87cff94..dacfb005 100644 --- a/backend-lib/Cargo.toml +++ b/backend-lib/Cargo.toml @@ -12,14 +12,14 @@ rust-version = "1.60" [dependencies] failure = "0.1" -hdwallet = "0.3.1" -hdwallet-bitcoin = "0.3" +hdwallet = "0.4" +hdwallet-bitcoin = "0.4" hex = "0.4" jni = { version = "0.20", default-features = false } prost = "0.12" rusqlite = "0.29" schemer = "0.2" -secp256k1 = "0.21" +secp256k1 = "0.26" secrecy = "0.8" zcash_address = "0.3" zcash_client_backend = { version = "=0.10.0-rc.1", features = ["transparent-inputs", "unstable"] } From d775421f5a41ead7822758f641e5d8d8755720b2 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Tue, 12 Sep 2023 15:24:45 +0000 Subject: [PATCH 71/76] rust: Migrate to `zcash_client_backend 0.10.0-rc.2` etc --- backend-lib/Cargo.lock | 8 ++++---- backend-lib/Cargo.toml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/backend-lib/Cargo.lock b/backend-lib/Cargo.lock index c84c06ff..c912f596 100644 --- a/backend-lib/Cargo.lock +++ b/backend-lib/Cargo.lock @@ -2112,9 +2112,9 @@ dependencies = [ [[package]] name = "zcash_client_backend" -version = "0.10.0-rc.1" +version = "0.10.0-rc.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dc7204683d2ef58f87a78168f7135edd2612a888d1663812e642d5835b32607" +checksum = "432f902a7308d10435351275a33629c61905f5b4289dc8b89259b1b8c1a87a86" dependencies = [ "base64", "bech32", @@ -2147,9 +2147,9 @@ dependencies = [ [[package]] name = "zcash_client_sqlite" -version = "0.8.0-rc.1" +version = "0.8.0-rc.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8095730f4fcc20c5eeacee2a9edec10f1e600770e254fcdba6bc6abb2ebd320" +checksum = "9746289b9650b14f5d24446c01e8398739e82a8bb8350eafa6a057e3a7f08c7e" dependencies = [ "bs58", "byteorder", diff --git a/backend-lib/Cargo.toml b/backend-lib/Cargo.toml index dacfb005..8c999231 100644 --- a/backend-lib/Cargo.toml +++ b/backend-lib/Cargo.toml @@ -22,8 +22,8 @@ schemer = "0.2" secp256k1 = "0.26" secrecy = "0.8" zcash_address = "0.3" -zcash_client_backend = { version = "=0.10.0-rc.1", features = ["transparent-inputs", "unstable"] } -zcash_client_sqlite = { version = "=0.8.0-rc.1", features = ["transparent-inputs", "unstable"] } +zcash_client_backend = { version = "=0.10.0-rc.2", features = ["transparent-inputs", "unstable", "unstable-serialization"] } +zcash_client_sqlite = { version = "=0.8.0-rc.2", features = ["transparent-inputs", "unstable"] } zcash_primitives = "=0.13.0-rc.1" zcash_proofs = "=0.13.0-rc.1" orchard = { version = "0.6", default-features = false } From 11b031a6afdfc3c54c7de144aec5ae32f0144f3e Mon Sep 17 00:00:00 2001 From: Honza Date: Tue, 12 Sep 2023 19:27:20 +0200 Subject: [PATCH 72/76] Exit execution when retry limit reached --- .../java/cash/z/ecc/android/sdk/internal/ext/WalletService.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/WalletService.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/WalletService.kt index a729ec54..75aca6e5 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/WalletService.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/ext/WalletService.kt @@ -68,6 +68,7 @@ suspend inline fun retryUpToAndContinue( failedAttempts++ if (failedAttempts == retries) { exceptionWrapper(t) + return } val duration = (initialDelayMillis.toDouble() * 2.0.pow(failedAttempts.toDouble() - 1)).toLong() Twig.warn(t) { "Retrying ($failedAttempts/$retries) in ${duration}s..." } From 96aeec2e9d9a086a1f84af41247f222da1a54bdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Honza=20Rychnovsk=C3=BD?= Date: Tue, 12 Sep 2023 19:29:39 +0200 Subject: [PATCH 73/76] Add throws documentation Co-authored-by: Daira Emma Hopwood --- .../cash/z/ecc/android/sdk/internal/model/JniScanProgress.kt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniScanProgress.kt b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniScanProgress.kt index ce4a9c57..f0fec273 100644 --- a/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniScanProgress.kt +++ b/backend-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/JniScanProgress.kt @@ -5,6 +5,8 @@ import androidx.annotation.Keep /** * Serves as cross layer (Kotlin, Rust) communication class. * + * @throws IllegalArgumentException unless (numerator is nonnegative, denominator is + * positive, and the represented ratio is in the range 0.0 to 1.0 inclusive). * @param numerator the numerator of the progress ratio * @param denominator the denominator of the progress ratio */ From 9b18fe12d9b8cc301ce460eedeb28b6d83ad5794 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Honza=20Rychnovsk=C3=BD?= Date: Tue, 12 Sep 2023 19:36:39 +0200 Subject: [PATCH 74/76] Add encode_scan_progress documentation Co-authored-by: Daira Emma Hopwood --- backend-lib/src/main/rust/lib.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend-lib/src/main/rust/lib.rs b/backend-lib/src/main/rust/lib.rs index c632ae1e..d16a23ff 100644 --- a/backend-lib/src/main/rust/lib.rs +++ b/backend-lib/src/main/rust/lib.rs @@ -1147,6 +1147,8 @@ pub unsafe extern "C" fn Java_cash_z_ecc_android_sdk_internal_jni_RustBackend_ge unwrap_exc_or(&env, res, -1) } +/// Returns a `JniScanProgress` object, provided that numerator is nonnegative, denominator +/// is positive, and the represented ratio is in the range 0.0 to 1.0 inclusive. fn encode_scan_progress(env: &JNIEnv<'_>, progress: Ratio) -> Result { let output = env.new_object( "cash/z/ecc/android/sdk/internal/model/JniScanProgress", From 5037102fafac757be418fc72617ca8e1532b8af7 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Tue, 12 Sep 2023 12:15:53 -0600 Subject: [PATCH 75/76] Update to zcash_client_sqlite-0.8.0-rc.3 --- backend-lib/Cargo.lock | 4 ++-- backend-lib/Cargo.toml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/backend-lib/Cargo.lock b/backend-lib/Cargo.lock index c912f596..96697766 100644 --- a/backend-lib/Cargo.lock +++ b/backend-lib/Cargo.lock @@ -2147,9 +2147,9 @@ dependencies = [ [[package]] name = "zcash_client_sqlite" -version = "0.8.0-rc.2" +version = "0.8.0-rc.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9746289b9650b14f5d24446c01e8398739e82a8bb8350eafa6a057e3a7f08c7e" +checksum = "a7f5ac4a1ff9258f215b090d97df506c0bcb3ada3f13aa388cdceeb69f67492d" dependencies = [ "bs58", "byteorder", diff --git a/backend-lib/Cargo.toml b/backend-lib/Cargo.toml index 8c999231..6b634645 100644 --- a/backend-lib/Cargo.toml +++ b/backend-lib/Cargo.toml @@ -22,8 +22,8 @@ schemer = "0.2" secp256k1 = "0.26" secrecy = "0.8" zcash_address = "0.3" -zcash_client_backend = { version = "=0.10.0-rc.2", features = ["transparent-inputs", "unstable", "unstable-serialization"] } -zcash_client_sqlite = { version = "=0.8.0-rc.2", features = ["transparent-inputs", "unstable"] } +zcash_client_backend = { version = "=0.10.0-rc.2", features = ["transparent-inputs", "unstable"] } +zcash_client_sqlite = { version = "=0.8.0-rc.3", features = ["transparent-inputs", "unstable"] } zcash_primitives = "=0.13.0-rc.1" zcash_proofs = "=0.13.0-rc.1" orchard = { version = "0.6", default-features = false } From 82757353f6435db3721adac78f1e3ee64d026ffb Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Tue, 12 Sep 2023 13:06:53 -0600 Subject: [PATCH 76/76] Update version & changelog for 2.0.0-rc.1 release. Fixes #1237 --- CHANGELOG.md | 94 ++++++++++++++++++++++++++++------------------- gradle.properties | 2 +- 2 files changed, 58 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b55b1eb8..799d7831 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,38 +1,58 @@ -# Change Log +# Changelog +All notable changes to this library will be documented in this file. -## Unreleased -- `CompactBlockProcessor` now processes compact blocks from the lightwalletd server with **Spend-before-Sync** algorithm - (i.e. non-linear order). This feature shortens the time after which a wallet's spendable balance can be used. -- The block synchronization mechanism is about one-third faster thanks to the optimized -`CompactBlockProcessor.SYNC_BATCH_SIZE`. Issue **#1206**. +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this library adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +## [2.0.0-rc.1] - 2023-09-12 + +### Notable Changes + +- `CompactBlockProcessor` now processes compact blocks from the lightwalletd + server using the **Spend-before-Sync** algorithm, which allows scanning of + wallet blocks to be performed in arbitrary order and optimized to make it + possible to spend received notes without waiting for synchronization to be + complete. This feature shortens the time until a wallet's spendable balance + can be used. +- The block synchronization mechanism is additionally about one-third faster + thanks to an optimized `CompactBlockProcessor.SYNC_BATCH_SIZE` (issue **#1206**). ### Removed -- `CompactBlockProcessor.ProcessorInfo.lastSyncHeight` which the SDK is no longer able to provide because of the new - **SpendBeforeSync** synchronization algorithm adoption. Use `CompactBlockProcessor.ProcessorInfo.overallSyncRange` - which contains all blocks in case of `SpendBeforeSync` synchronization algorithm. No internal change was made in - case of older Linear synchronization algorithm. +- `CompactBlockProcessor.ProcessorInfo.lastSyncHeight` no longer had a + well-defined meaning after implementation of the **SpendBeforeSync** + synchronization algorithm and has been removed. + `CompactBlockProcessor.ProcessorInfo.overallSyncRange` provides related + information. - `CompactBlockProcessor.ProcessorInfo.isSyncing`. Use `Synchronizer.status` instead. - `CompactBlockProcessor.ProcessorInfo.syncProgress`. Use `Synchronizer.progress` instead. -- `alsoClearBlockCache` parameter from rewind functions of `Synchronizer` and `CompactBlockProcessor`, as it has no - effect on the current behaviour of these functions. -- Internally, we removed access to the shared block table from the Kotlin layer, which resulted in eliminating these - APIs: +- `alsoClearBlockCache` parameter from rewind functions of `Synchronizer` and + `CompactBlockProcessor`, as it has no effect on the current behaviour of + these functions. +- Internally, we removed access to the shared block table from the Kotlin + layer, which resulted in eliminating these APIs: - `SdkSynchronizer.findBlockHash()` - `SdkSynchronizer.findBlockHashAsHex()` ### Changed -- `CompactBlockProcessor.quickRewind()` and `CompactBlockProcessor.rewindToNearestHeight()` now might fail due to - internal changes in getting scanned height. Thus, these functions return `Boolean` results. -- `Synchronizer.new()` and `PersistableWallet` APIs require a new `walletInitMode` parameter of type `WalletInitMode`, - which describes wallet initialization mode. See related function and sealed class documentation. +- `CompactBlockProcessor.quickRewind()` and `CompactBlockProcessor.rewindToNearestHeight()` + now might fail due to internal changes in getting scanned height. Thus, these + functions now return `Boolean` results. +- `Synchronizer.new()` and `PersistableWallet` APIs require a new + `walletInitMode` parameter of type `WalletInitMode`, which describes wallet + initialization mode. See related function and sealed class documentation. ### Fixed -- `Synchronizer.getMemos()` now correctly returns a flow of strings for sent and received transactions. Issue **#1154**. -- `CompactBlockProcessor` now triggers transaction polling while block synchronization is in progress as expected. - Clients will be notified shortly after every new transaction is discovered via `Synchronizer.transactions` API. - Issue **#1170**. +- `Synchronizer.getMemos()` now correctly returns a flow of strings for sent + and received transactions. Issue **#1154**. +- `CompactBlockProcessor` now triggers transaction polling while block + synchronization is in progress as expected. Clients will be notified shortly + after every new transaction is discovered via `Synchronizer.transactions` + API. Issue **#1170**. + +## [1.21.0-beta01] -## 1.21.0-beta01 Note: This is the last _1.x_ version release. The upcoming version _2.0_ brings the **Spend-before-Sync** feature, which speeds up discovering the wallet's spendable balance. @@ -50,7 +70,7 @@ which speeds up discovering the wallet's spendable balance. ## 1.20.0-beta01 - The SDK internally migrated from `BackendExt` rust backend extension functions to more type-safe `TypesafeBackend`. -- `Synchronizer.getMemos()` now internally handles expected `RuntimeException` from the rust layer and transforms it +- `Synchronizer.getMemos()` now internally handles expected `RuntimeException` from the rust layer and transforms it in an empty string. ## 1.19.0-beta01 @@ -60,16 +80,16 @@ which speeds up discovering the wallet's spendable balance. ### Fixed - `TransactionOverview` object returned with `SdkSynchronizer.transactions` now contains a correct `TransactionState. Pending` in case of the transaction is mined,but not fully confirmed. -- When the SDK internally works with a recently created transaction there was a moment in which could the transaction +- When the SDK internally works with a recently created transaction there was a moment in which could the transaction causes the SDK to crash, because of its invalid mined height. Fixed now. ## 1.18.0-beta01 -- Synchronizer's functions `getUnifiedAddress`, `getSaplingAddress`, `getTransparentAddress`, and `refreshUtxos` now - do not provide `Account.DEFAULT` value for the account argument. As accounts are not fully supported by the SDK - yet, the caller should explicitly set Account.DEFAULT as the account argument to keep the same behavior. +- Synchronizer's functions `getUnifiedAddress`, `getSaplingAddress`, `getTransparentAddress`, and `refreshUtxos` now + do not provide `Account.DEFAULT` value for the account argument. As accounts are not fully supported by the SDK + yet, the caller should explicitly set Account.DEFAULT as the account argument to keep the same behavior. - Gradle 8.1.1 - AGP 8.0.2 - + ## 1.17.0-beta01 - Transparent fund balances are now displayed almost immediately - Synchronization of shielded balances and transaction history is about 30% faster @@ -78,7 +98,7 @@ which speeds up discovering the wallet's spendable balance. - `Synchronizer.progress` now returns `Flow` instead of `Flow`. PercentDecimal is a type-safe model. Use `PercentDecimal.toPercentage()` to get a number within 0-100% scale. - `Synchronizer.clearedTransactions` has been renamed to `Synchronizer.transactions` and includes sent, received, and pending transactions. Synchronizer APIs for listing sent, received, and pending transactions have been removed. Clients can determine whether a transaction is sent, received, or pending by filtering the `TransactionOverview` objects returned by `Synchronizer.transactions` - `Synchronizer.send()` and `shieldFunds()` are now `suspend` functions with `Long` return values representing the ID of the newly created transaction. Errors are reported by thrown exceptions. - - `DerivationTool` is now an interface, rather than an `object`, which makes it easier to inject alternative implementations into tests. To adapt to the new API, replace calls to `DerivationTool.methodName()` with `DerivationTool.getInstance().methodName()`. + - `DerivationTool` is now an interface, rather than an `object`, which makes it easier to inject alternative implementations into tests. To adapt to the new API, replace calls to `DerivationTool.methodName()` with `DerivationTool.getInstance().methodName()`. - `DerivationTool` methods are no longer suspending, which should make it easier to call them in various situations. Obtaining a `DerivationTool` instance via `DerivationTool.getInstance()` frontloads the need for a suspending call. - `DerivationTool.deriveUnifiedFullViewingKeys()` no longer has a default argument for `numberOfAccounts`. Clients should now pass `DerivationTool.DEFAULT_NUMBER_OF_ACCOUNTS` as the value. Note that the SDK does not currently have proper support for multiple accounts. - The SDK's internals for connecting with librustzcash have been refactored to a separate Gradle module `backend-lib` (and therefore a separate artifact) which is a transitive dependency of the Zcash Android SDK. SDK consumers that use Gradle dependency locks may notice this difference, but otherwise it should be mostly an invisible change. @@ -91,7 +111,7 @@ which speeds up discovering the wallet's spendable balance. ## 1.15.0-beta01 ### Changed - A new package `sdk-incubator-lib` is now available as a public API. This package contains experimental APIs that may be promoted to the SDK in the future. The APIs in this package are not guaranteed to be stable, and may change at any time. -- `Synchronizer.refreshUtxos` now takes `Account` type as first parameter instead of transparent address of type +- `Synchronizer.refreshUtxos` now takes `Account` type as first parameter instead of transparent address of type `String`, and thus it downloads all UTXOs for the given account addresses. The Account object provides a default `0` index Account with `Account.DEFAULT`. ## 1.14.0-beta01 @@ -100,11 +120,11 @@ which speeds up discovering the wallet's spendable balance. ## 1.13.0-beta01 ### Changed -- The SDK's internal networking has been refactored to a separate Gradle module `lightwallet-client-lib` (and +- The SDK's internal networking has been refactored to a separate Gradle module `lightwallet-client-lib` (and therefore a separate artifact) which is a transitive dependency of the Zcash Android SDK. - The `z.cash.ecc.android.sdk.model.LightWalletEndpoint` class has been moved to `co.electriccoin.lightwallet.client.model.LightWalletEndpoint` - The new networking module now provides a `LightWalletClient` for asynchronous calls. - - Most unary calls respond with the new `Response` class and its subclasses. Streaming calls will be updated + - Most unary calls respond with the new `Response` class and its subclasses. Streaming calls will be updated with the Response class later. - SDK clients should avoid using generated GRPC objects, as these are an internal implementation detail and are in process of being removed from the public API. Any clients using GRPC objects will find these have been repackaged from `cash.z.wallet.sdk.rpc` to `cash.z.wallet.sdk.internal.rpc` to signal they are not a public API. @@ -161,7 +181,7 @@ which speeds up discovering the wallet's spendable balance. - `Synchronizer.sendToAddress()` and `Synchronizer.shieldFunds()` return flows that can now be collected multiple times. Prior versions of the SDK had a bug that could submit transactions multiple times if the flow was collected more than once. - Updated dependencies: - Kotlin 1.7.21 - - AndroidX + - AndroidX - etc. - Updated checkpoints @@ -186,14 +206,14 @@ which speeds up discovering the wallet's spendable balance. - `DerivationTool.deriveTransparentSecretKey` (use `DerivationTool.deriveUnifiedSpendingKey` instead). - `DerivationTool.deriveShieldedAddress` - `DerivationTool.deriveUnifiedViewingKeys` (use `DerivationTool.deriveUnifiedFullViewingKey` instead) - - `DerivationTool.validateUnifiedViewingKey` + - `DerivationTool.validateUnifiedViewingKey` ## Version 1.9.0-beta05 - The minimum version of Android supported is now API 21 - Fixed R8/ProGuard consumer rule, which eliminates a runtime crash for minified apps ## Version 1.9.0-beta04 -- The SDK now stores sapling param files in `no_backup/co.electricoin.zcash` folder instead of the `cache/params` +- The SDK now stores sapling param files in `no_backup/co.electricoin.zcash` folder instead of the `cache/params` folder. Besides that, `SaplingParamTool` also does validation of downloaded sapling param file hash and size. **No action required from client app**. @@ -209,7 +229,7 @@ which speeds up discovering the wallet's spendable balance. - Updated checkpoints ## Version 1.8.0-beta01 -- Enabled automated unit tests run on the CI server +- Enabled automated unit tests run on the CI server - Added `BlockHeight` typesafe object to represent block heights - Significantly reduced memory usage, fixing potential OutOfMemoryError during block download - Kotlin 1.7.10 diff --git a/gradle.properties b/gradle.properties index a5456236..27fbff09 100644 --- a/gradle.properties +++ b/gradle.properties @@ -26,7 +26,7 @@ ZCASH_ASCII_GPG_KEY= # Configures whether release is an unstable snapshot, therefore published to the snapshot repository. IS_SNAPSHOT=true -LIBRARY_VERSION=1.21.0-beta01 +LIBRARY_VERSION=2.0.0-rc.1 # Kotlin compiler warnings can be considered errors, failing the build. ZCASH_IS_TREAT_WARNINGS_AS_ERRORS=true