Download and decrypt test

.gitignore

@@ -0,0 +1,4 @@
+/target
+.env
+.idea/
+src/generated/*.rs

Cargo.toml

@@ -0,0 +1,43 @@
+[package]
+name = "sync"
+version = "0.1.0"
+authors = ["Hanh <hanh425@gmail.com>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+dotenv = "0.15.0"
+anyhow = "1.0.40"
+log = "0.4.14"
+flexi_logger = {version="0.17.1", features = ["compress"]}
+serde = {version = "1.0.126", features = ["derive"]}
+serde_json = "1.0.64"
+tokio = { version = "^1.6", features = ["macros", "rt-multi-thread"] }
+protobuf = "2.23.0"
+jubjub = "0.6"
+bls12_381 = "0.4.0"
+ff = "0.9"
+group = "0.9"
+hex = "0.4.3"
+bytes = "1.0.1"
+futures = "0.3.15"
+tonic = {version = "0.4.3", features = ["tls", "tls-roots"]}
+prost = "0.7"
+rayon = "1.5.1"
+
+[dependencies.zcash_client_backend]
+git = "https://github.com/zcash/librustzcash.git"
+rev = "d50bb12a97da768dc8f3ee39b81f84262103e6eb"
+
+[dependencies.zcash_primitives]
+git = "https://github.com/zcash/librustzcash.git"
+features = [ "transparent-inputs" ]
+rev = "d50bb12a97da768dc8f3ee39b81f84262103e6eb"
+
+[dependencies.zcash_proofs]
+git = "https://github.com/zcash/librustzcash.git"
+rev = "d50bb12a97da768dc8f3ee39b81f84262103e6eb"
+
+[build-dependencies]
+tonic-build = "0.4.2"

build.rs

@@ -0,0 +1,9 @@
+fn main() {
+    tonic_build::configure()
+        .out_dir("src/generated")
+        .compile(
+            &["proto/service.proto", "proto/compact_formats.proto"],
+            &["proto"],
+        )
+        .unwrap();
+}

proto/compact_formats.proto

@@ -0,0 +1,56 @@
+// Copyright (c) 2019-2020 The Zcash developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or https://www.opensource.org/licenses/mit-license.php .
+
+syntax = "proto3";
+package cash.z.wallet.sdk.rpc;
+option go_package = "lightwalletd/walletrpc";
+option swift_prefix = "";
+// Remember that proto3 fields are all optional. A field that is not present will be set to its zero value.
+// bytes fields of hashes are in canonical little-endian format.
+
+// CompactBlock is a packaging of ONLY the data from a block that's needed to:
+//   1. Detect a payment to your shielded Sapling address
+//   2. Detect a spend of your shielded Sapling notes
+//   3. Update your witnesses to generate new Sapling spend proofs.
+message CompactBlock {
+    uint32 protoVersion = 1;    // the version of this wire format, for storage
+    uint64 height = 2;          // the height of this block
+    bytes hash = 3;             // the ID (hash) of this block, same as in block explorers
+    bytes prevHash = 4;         // the ID (hash) of this block's predecessor
+    uint32 time = 5;            // Unix epoch time when the block was mined
+    bytes header = 6;           // (hash, prevHash, and time) OR (full header)
+    repeated CompactTx vtx = 7; // zero or more compact transactions from this block
+}
+
+// CompactTx contains the minimum information for a wallet to know if this transaction
+// is relevant to it (either pays to it or spends from it) via shielded elements
+// only. This message will not encode a transparent-to-transparent transaction.
+message CompactTx {
+    uint64 index = 1;   // the index within the full block
+    bytes hash = 2;     // the ID (hash) of this transaction, same as in block explorers
+
+    // The transaction fee: present if server can provide. In the case of a
+    // stateless server and a transaction with transparent inputs, this will be
+    // unset because the calculation requires reference to prior transactions.
+    // in a pure-Sapling context, the fee will be calculable as:
+    //    valueBalance + (sum(vPubNew) - sum(vPubOld) - sum(tOut))
+    uint32 fee = 3;
+
+    repeated CompactSpend spends = 4;   // inputs
+    repeated CompactOutput outputs = 5; // outputs
+}
+
+// CompactSpend is a Sapling Spend Description as described in 7.3 of the Zcash
+// protocol specification.
+message CompactSpend {
+    bytes nf = 1;   // nullifier (see the Zcash protocol specification)
+}
+
+// output is a Sapling Output Description as described in section 7.4 of the
+// Zcash protocol spec. Total size is 948.
+message CompactOutput {
+    bytes cmu = 1;          // note commitment u-coordinate
+    bytes epk = 2;          // ephemeral public key
+    bytes ciphertext = 3;   // ciphertext and zkproof
+}

proto/service.proto

@@ -0,0 +1,209 @@
+// Copyright (c) 2019-2020 The Zcash developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or https://www.opensource.org/licenses/mit-license.php .
+
+syntax = "proto3";
+package cash.z.wallet.sdk.rpc;
+option go_package = "lightwalletd/walletrpc";
+option swift_prefix = "";
+import "compact_formats.proto";
+
+// A BlockID message contains identifiers to select a block: a height or a
+// hash. Specification by hash is not implemented, but may be in the future.
+message BlockID {
+     uint64 height = 1;
+     bytes hash = 2;
+}
+
+// BlockRange specifies a series of blocks from start to end inclusive.
+// Both BlockIDs must be heights; specification by hash is not yet supported.
+message BlockRange {
+    BlockID start = 1;
+    BlockID end = 2;
+}
+
+// A TxFilter contains the information needed to identify a particular
+// transaction: either a block and an index, or a direct transaction hash.
+// Currently, only specification by hash is supported.
+message TxFilter {
+     BlockID block = 1;     // block identifier, height or hash
+     uint64 index = 2;      // index within the block
+     bytes hash = 3;        // transaction ID (hash, txid)
+}
+
+// RawTransaction contains the complete transaction data. It also optionally includes 
+// the block height in which the transaction was included.
+message RawTransaction {
+    bytes data = 1;     // exact data returned by Zcash 'getrawtransaction'
+    uint64 height = 2;  // height that the transaction was mined (or -1)
+}
+
+// A SendResponse encodes an error code and a string. It is currently used
+// only by SendTransaction(). If error code is zero, the operation was
+// successful; if non-zero, it and the message specify the failure.
+message SendResponse {
+    int32 errorCode = 1;
+    string errorMessage = 2;
+}
+
+// Chainspec is a placeholder to allow specification of a particular chain fork.
+message ChainSpec {}
+
+// Empty is for gRPCs that take no arguments, currently only GetLightdInfo.
+message Empty {}
+
+// LightdInfo returns various information about this lightwalletd instance
+// and the state of the blockchain.
+message LightdInfo {
+    string version = 1;
+    string vendor = 2;
+    bool   taddrSupport = 3;            // true
+    string chainName = 4;               // either "main" or "test"
+    uint64 saplingActivationHeight = 5; // depends on mainnet or testnet
+    string consensusBranchId = 6;       // protocol identifier, see consensus/upgrades.cpp
+    uint64 blockHeight = 7;             // latest block on the best chain
+    string gitCommit = 8;
+    string branch = 9;
+    string buildDate = 10;
+    string buildUser = 11;
+    uint64 estimatedHeight = 12;        // less than tip height if zcashd is syncing
+    string zcashdBuild = 13;            // example: "v4.1.1-877212414"
+    string zcashdSubversion = 14;       // example: "/MagicBean:4.1.1/"
+}
+
+// TransparentAddressBlockFilter restricts the results to the given address
+// or block range.
+message TransparentAddressBlockFilter {
+    string address = 1;     // t-address
+    BlockRange range = 2;   // start, end heights
+}
+
+// Duration is currently used only for testing, so that the Ping rpc
+// can simulate a delay, to create many simultaneous connections. Units
+// are microseconds.
+message Duration {
+    int64 intervalUs = 1;
+}
+
+// PingResponse is used to indicate concurrency, how many Ping rpcs
+// are executing upon entry and upon exit (after the delay).
+// This rpc is used for testing only.
+message PingResponse {
+    int64 entry = 1;
+    int64 exit = 2;
+}
+
+message Address {
+    string address = 1;
+}
+message AddressList {
+    repeated string addresses = 1;
+}
+message Balance {
+    int64 valueZat = 1;
+}
+
+message Exclude {
+    repeated bytes txid = 1;
+}
+
+// The TreeState is derived from the Zcash z_gettreestate rpc.
+message TreeState {
+    string network = 1; // "main" or "test"
+    uint64 height = 2;
+    string hash = 3;    // block id
+    uint32 time = 4;    // Unix epoch time when the block was mined
+    string tree = 5;    // sapling commitment tree state
+}
+
+// Results are sorted by height, which makes it easy to issue another
+// request that picks up from where the previous left off.
+message GetAddressUtxosArg {
+    repeated string addresses = 1;
+    uint64 startHeight = 2;
+    uint32 maxEntries = 3; // zero means unlimited
+}
+message GetAddressUtxosReply {
+    string address = 6;
+    bytes txid = 1;
+    int32 index = 2;
+    bytes script = 3;
+    int64 valueZat = 4;
+    uint64 height = 5;
+}
+message GetAddressUtxosReplyList {
+    repeated GetAddressUtxosReply addressUtxos = 1;
+}
+
+
+message PriceRequest {
+    // List of timestamps(in sec) at which the price is being requested
+    uint64 timestamp = 1;
+
+    // 3 letter currency-code
+    string currency = 2;
+}
+
+message PriceResponse {
+    // Timestamp at which this price quote was fetched. Note, this may not be the same
+    // as the request timestamp, but the server will respond with the closest timestamp that it has/can fetch    
+    int64 timestamp = 1;  
+    
+    // 3-letter currency code, matching the request
+    string currency = 2;  
+
+    // price of ZEC
+    double price = 3;
+}
+
+service CompactTxStreamer {
+    // Return the height of the tip of the best chain
+    rpc GetLatestBlock(ChainSpec) returns (BlockID) {}
+    // Return the compact block corresponding to the given block identifier
+    rpc GetBlock(BlockID) returns (CompactBlock) {}
+    // Return a list of consecutive compact blocks
+    rpc GetBlockRange(BlockRange) returns (stream CompactBlock) {}
+    
+    // Get the historical and current prices 
+    rpc GetZECPrice(PriceRequest) returns (PriceResponse) {}
+    rpc GetCurrentZECPrice(Empty) returns (PriceResponse) {}
+
+    // Return the requested full (not compact) transaction (as from zcashd)
+    rpc GetTransaction(TxFilter) returns (RawTransaction) {}
+    // Submit the given transaction to the Zcash network
+    rpc SendTransaction(RawTransaction) returns (SendResponse) {}
+
+    // Return the txids corresponding to the given t-address within the given block range
+    rpc GetTaddressTxids(TransparentAddressBlockFilter) returns (stream RawTransaction) {}
+
+    // Legacy API that is used as a fallback for t-Address support, if the server is running the old version (lwdv2)
+    rpc GetAddressTxids(TransparentAddressBlockFilter) returns (stream RawTransaction) {}
+
+    rpc GetTaddressBalance(AddressList) returns (Balance) {}
+    rpc GetTaddressBalanceStream(stream Address) returns (Balance) {}
+
+    // Return the compact transactions currently in the mempool; the results
+    // can be a few seconds out of date. If the Exclude list is empty, return
+    // all transactions; otherwise return all *except* those in the Exclude list
+    // (if any); this allows the client to avoid receiving transactions that it
+    // already has (from an earlier call to this rpc). The transaction IDs in the
+    // Exclude list can be shortened to any number of bytes to make the request
+    // more bandwidth-efficient; if two or more transactions in the mempool
+    // match a shortened txid, they are all sent (none is excluded). Transactions
+    // in the exclude list that don't exist in the mempool are ignored.
+    rpc GetMempoolTx(Exclude) returns (stream CompactTx) {}
+
+    // GetTreeState returns the note commitment tree state corresponding to the given block.
+    // See section 3.7 of the Zcash protocol specification. It returns several other useful
+    // values also (even though they can be obtained using GetBlock).
+    // The block can be specified by either height or hash.
+    rpc GetTreeState(BlockID) returns (TreeState) {}
+
+    rpc GetAddressUtxos(GetAddressUtxosArg) returns (GetAddressUtxosReplyList) {}
+    rpc GetAddressUtxosStream(GetAddressUtxosArg) returns (stream GetAddressUtxosReply) {}
+
+    // Return information about this lightwalletd instance and the blockchain
+    rpc GetLightdInfo(Empty) returns (LightdInfo) {}
+    // Testing-only, requires lightwalletd --ping-very-insecure (do not enable in production)
+    rpc Ping(Duration) returns (PingResponse) {}
+}

src/chain.rs

@@ -0,0 +1,137 @@
+use crate::lw_rpc::compact_tx_streamer_client::CompactTxStreamerClient;
+use crate::lw_rpc::*;
+use tonic::transport::Channel;
+use tonic::Request;
+use zcash_primitives::sapling::note_encryption::try_sapling_compact_note_decryption;
+use crate::NETWORK;
+use zcash_primitives::consensus::BlockHeight;
+use zcash_primitives::sapling::SaplingIvk;
+use zcash_primitives::transaction::components::OutputDescription;
+use jubjub::Scalar;
+use group::GroupEncoding;
+use ff::PrimeField;
+use zcash_primitives::transaction::components::sapling::CompactOutputDescription;
+use tokio::runtime::Runtime;
+use std::sync::{Arc, Mutex};
+use tokio::task::JoinHandle;
+use futures::future::JoinAll;
+use rayon::prelude::*;
+
+const MAX_CHUNK: u32 = 50000;
+
+pub async fn get_latest_height(
+    client: &mut CompactTxStreamerClient<Channel>,
+) -> anyhow::Result<u32> {
+    let chainspec = ChainSpec {};
+    let rep = client.get_latest_block(Request::new(chainspec)).await?;
+    let block_id = rep.into_inner();
+    Ok(block_id.height as u32)
+}
+
+/* download [start_height+1, end_height] inclusive */
+pub async fn download_chain(
+    client: &mut CompactTxStreamerClient<Channel>,
+    start_height: u32,
+    end_height: u32
+) -> anyhow::Result<Vec<CompactBlock>> {
+    let mut cbs: Vec<CompactBlock> = Vec::new();
+    let mut s = start_height + 1;
+    while s < end_height {
+        eprintln!("{}", s);
+        let e = (s + MAX_CHUNK).min(end_height);
+        let range = BlockRange {
+            start: Some(BlockId { height: s as u64, hash: vec![] }),
+            end: Some(BlockId { height: e as u64, hash: vec![] })
+        };
+        let mut block_stream = client.get_block_range(Request::new(range)).await?.into_inner();
+        while let Some(block) = block_stream.message().await? {
+            cbs.push(block);
+        }
+        s = e + 1;
+    }
+    Ok(cbs)
+}
+
+struct DecryptNode {
+    ivks: Vec<SaplingIvk>,
+}
+
+fn decrypt_notes(block: &CompactBlock, ivks: &[SaplingIvk]) {
+    let height = BlockHeight::from_u32(block.height as u32);
+    for vtx in block.vtx.iter() {
+        for co in vtx.outputs.iter() {
+            let mut cmu = [0u8; 32];
+            cmu.copy_from_slice(&co.cmu);
+            let cmu = bls12_381::Scalar::from_repr(cmu).unwrap();
+            let mut epk = [0u8; 32];
+            epk.copy_from_slice(&co.epk);
+            let epk = jubjub::ExtendedPoint::from_bytes(&epk).unwrap();
+            let od = CompactOutputDescription {
+                epk,
+                cmu,
+                enc_ciphertext: co.ciphertext.to_vec(),
+            };
+            for ivk in ivks.iter() {
+                if let Some((note, pa)) = try_sapling_compact_note_decryption(&NETWORK, height, ivk, &od) {
+                    println!("{:?} {:?}", note, pa);
+                }
+            }
+        }
+    }
+}
+
+impl DecryptNode {
+    pub fn new(ivks: Vec<SaplingIvk>) -> DecryptNode {
+        DecryptNode {
+            ivks,
+        }
+    }
+    pub fn decrypt_blocks(&self, blocks: &[CompactBlock]) {
+        blocks.par_iter().for_each(|b| {
+            decrypt_notes(b, &self.ivks);
+        });
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::chain::{get_latest_height, download_chain, DecryptNode};
+    use crate::lw_rpc::compact_tx_streamer_client::CompactTxStreamerClient;
+    use zcash_primitives::consensus::{Parameters, NetworkUpgrade};
+    use zcash_client_backend::encoding::decode_extended_full_viewing_key;
+    use crate::NETWORK;
+    use dotenv;
+    use tokio::runtime::Runtime;
+    use std::time::Instant;
+
+    #[tokio::test]
+    async fn test_get_latest_height() -> anyhow::Result<()> {
+        let mut client = CompactTxStreamerClient::connect("http://127.0.0.1:9067").await?;
+        let height = get_latest_height(&mut client).await?;
+        assert!(height > 1288000);
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_download_chain() -> anyhow::Result<()> {
+        dotenv::dotenv().unwrap();
+        let ivk = dotenv::var("IVK").unwrap();
+
+        let fvk = decode_extended_full_viewing_key(NETWORK.hrp_sapling_extended_full_viewing_key(), &ivk).unwrap().unwrap();
+        let ivk = fvk.fvk.vk.ivk();
+        let decrypter = DecryptNode::new(vec![ivk]);
+        let mut client = CompactTxStreamerClient::connect("http://127.0.0.1:9067").await?;
+        let start_height: u32 = crate::NETWORK.activation_height(NetworkUpgrade::Sapling).unwrap().into();
+        let end_height = get_latest_height(&mut client).await?;
+
+        let start = Instant::now();
+        let cbs = download_chain(&mut client, start_height, end_height).await?;
+        eprintln!("Download chain: {} ms", start.elapsed().as_millis());
+
+        let start = Instant::now();
+        decrypter.decrypt_blocks(&cbs);
+        eprintln!("Decrypt Notes: {} ms", start.elapsed().as_millis());
+
+        Ok(())
+    }
+}

src/lib.rs

@@ -0,0 +1,8 @@
+use zcash_primitives::consensus::Network;
+
+#[path = "generated/cash.z.wallet.sdk.rpc.rs"]
+pub mod lw_rpc;
+
+pub const NETWORK: Network = Network::MainNetwork;
+
+mod chain;

src/main.rs

@@ -0,0 +1,5 @@
+#[tokio::main]
+async fn main() -> anyhow::Result<()> {
+
+    Ok(())
+}