GetTransactionInfo performance optimizations

src/chain.rs

@@ -1,4 +1,5 @@
 use crate::commitment::{CTree, Witness};
+use crate::db::AccountViewKey;
 use crate::lw_rpc::compact_tx_streamer_client::CompactTxStreamerClient;
 use crate::lw_rpc::*;
 use crate::{advance_tree, NETWORK};
@@ -17,7 +18,6 @@ use zcash_primitives::sapling::note_encryption::try_sapling_compact_note_decrypt
 use zcash_primitives::sapling::{Node, Note, PaymentAddress};
 use zcash_primitives::transaction::components::sapling::CompactOutputDescription;
 use zcash_primitives::zip32::ExtendedFullViewingKey;
-use crate::db::AccountViewKey;
 
 const MAX_CHUNK: u32 = 50000;
 
@@ -130,7 +130,7 @@ pub fn to_output_description(co: &CompactOutput) -> CompactOutputDescription {
 
 fn decrypt_notes<'a>(
     block: &'a CompactBlock,
-    vks: &HashMap<u32, AccountViewKey>
+    vks: &HashMap<u32, AccountViewKey>,
 ) -> DecryptedBlock<'a> {
     let height = BlockHeight::from_u32(block.height as u32);
     let mut count_outputs = 0u32;
@@ -366,6 +366,7 @@ mod tests {
         calculate_tree_state_v1, calculate_tree_state_v2, download_chain, get_latest_height,
         get_tree_state, DecryptNode,
     };
+    use crate::db::AccountViewKey;
     use crate::lw_rpc::compact_tx_streamer_client::CompactTxStreamerClient;
     use crate::LWD_URL;
     use crate::NETWORK;
@@ -374,7 +375,6 @@ mod tests {
     use std::time::Instant;
     use zcash_client_backend::encoding::decode_extended_full_viewing_key;
     use zcash_primitives::consensus::{NetworkUpgrade, Parameters};
-    use crate::db::AccountViewKey;
 
     #[tokio::test]
     async fn test_get_latest_height() -> anyhow::Result<()> {

src/db.rs

@@ -1,15 +1,15 @@
 use crate::chain::{Nf, NfRef};
 use crate::db::migration::{get_schema_version, update_schema_version};
 use crate::taddr::{derive_tkeys, BIP44_PATH};
-use crate::transaction::{TransactionInfo, Contact};
+use crate::transaction::{Contact, TransactionInfo};
 use crate::{CTree, Witness, NETWORK};
 use rusqlite::{params, Connection, OptionalExtension, NO_PARAMS};
 use std::collections::HashMap;
+use zcash_client_backend::encoding::decode_extended_full_viewing_key;
 use zcash_primitives::consensus::{NetworkUpgrade, Parameters};
 use zcash_primitives::merkle_tree::IncrementalWitness;
 use zcash_primitives::sapling::{Diversifier, Node, Note, Rseed, SaplingIvk};
 use zcash_primitives::zip32::{DiversifierIndex, ExtendedFullViewingKey};
-use zcash_client_backend::encoding::decode_extended_full_viewing_key;
 
 mod migration;
 
@@ -57,14 +57,10 @@ impl AccountViewKey {
 impl DbAdapter {
     pub fn new(db_path: &str) -> anyhow::Result<DbAdapter> {
         let connection = Connection::open(db_path)?;
+        connection.execute("PRAGMA synchronous = off", NO_PARAMS)?;
         Ok(DbAdapter { connection })
     }
 
-    pub fn synchronous(&self, flag: bool) -> anyhow::Result<()> {
-        self.connection.execute(&format!("PRAGMA synchronous = {}", if flag { "on" } else { "off" }), NO_PARAMS)?;
-        Ok(())
-    }
-
     pub fn begin_transaction(&self) -> anyhow::Result<()> {
         self.connection.execute("BEGIN TRANSACTION", NO_PARAMS)?;
         Ok(())
@@ -175,7 +171,8 @@ impl DbAdapter {
                 account INTEGER NOT NULL,
                 name TEXT NOT NULL,
                 address TEXT NOT NULL,
-                PRIMARY KEY (account, address))", NO_PARAMS
+                PRIMARY KEY (account, address))",
+                NO_PARAMS,
             )?;
         }
 
@@ -213,9 +210,21 @@ impl DbAdapter {
             let account: u32 = row.get(0)?;
             let ivk: String = row.get(1)?;
             let sk: Option<String> = row.get(2)?;
-            let fvk = decode_extended_full_viewing_key(NETWORK.hrp_sapling_extended_full_viewing_key(), &ivk).unwrap().unwrap();
+            let fvk = decode_extended_full_viewing_key(
+                NETWORK.hrp_sapling_extended_full_viewing_key(),
+                &ivk,
+            )
+            .unwrap()
+            .unwrap();
             let ivk = fvk.fvk.vk.ivk();
-            Ok((account, AccountViewKey { fvk, ivk, viewonly: sk.is_none() }))
+            Ok((
+                account,
+                AccountViewKey {
+                    fvk,
+                    ivk,
+                    viewonly: sk.is_none(),
+                },
+            ))
         })?;
         let mut fvks: HashMap<u32, AccountViewKey> = HashMap::new();
         for r in rows {
@@ -245,18 +254,19 @@ impl DbAdapter {
         Ok(())
     }
 
-    pub fn get_txhash(&self, id_tx: u32) -> anyhow::Result<(u32, u32, Vec<u8>)> {
+    pub fn get_txhash(&self, id_tx: u32) -> anyhow::Result<(u32, u32, Vec<u8>, String)> {
-        let (account, height, tx_hash) = self.connection.query_row(
+        let (account, height, tx_hash, ivk) = self.connection.query_row(
-            "SELECT account, height, txid FROM transactions WHERE id_tx = ?1",
+            "SELECT account, height, txid, ivk FROM transactions t, accounts a WHERE id_tx = ?1 AND t.account = a.id_account",
             params![id_tx],
             |row| {
                 let account: u32 = row.get(0)?;
                 let height: u32 = row.get(1)?;
                 let tx_hash: Vec<u8> = row.get(2)?;
-                Ok((account, height, tx_hash))
+                let ivk: String = row.get(3)?;
+                Ok((account, height, tx_hash, ivk))
             },
         )?;
-        Ok((account, height, tx_hash))
+        Ok((account, height, tx_hash, ivk))
     }
 
     pub fn store_block(
@@ -574,13 +584,17 @@ impl DbAdapter {
     pub fn store_contact(&self, account: u32, contact: &Contact) -> anyhow::Result<()> {
         log::info!("{:?}", contact);
         if contact.name.is_empty() {
-            self.connection.execute("DELETE FROM contacts WHERE account = ?1 AND address = ?2",
+            self.connection.execute(
-            params![account, contact.address])?;
+                "DELETE FROM contacts WHERE account = ?1 AND address = ?2",
-        }
+                params![account, contact.address],
-        else {
+            )?;
-            self.connection.execute("INSERT INTO contacts(account, name, address)
+        } else {
+            self.connection.execute(
+                "INSERT INTO contacts(account, name, address)
         VALUES (?1, ?2, ?3) ON CONFLICT (account, address) DO UPDATE SET
-        name = excluded.name", params![account, &contact.name, &contact.address])?;
+        name = excluded.name",
+                params![account, &contact.name, &contact.address],
+            )?;
         }
         Ok(())
     }
@@ -606,16 +620,14 @@ impl DbAdapter {
 
     pub fn get_seed(&self, account: u32) -> anyhow::Result<Option<String>> {
         log::info!("+get_seed");
-        let seed = self
+        let seed = self.connection.query_row(
-            .connection
+            "SELECT seed FROM accounts WHERE id_account = ?1",
-            .query_row(
+            params![account],
-                "SELECT seed FROM accounts WHERE id_account = ?1",
+            |row| {
-                params![account],
+                let sk: Option<String> = row.get(0)?;
-                |row| {
+                Ok(sk)
-                    let sk: Option<String> = row.get(0)?;
+            },
-                    Ok(sk)
+        )?;
-                },
-            )?;
         log::info!("-get_seed");
         Ok(seed)
     }

src/main/cli.rs

@@ -1,6 +1,6 @@
-use rustyline::Editor;
-use rustyline::error::ReadlineError;
 use clap::{AppSettings, Clap};
+use rustyline::error::ReadlineError;
+use rustyline::Editor;
 
 #[derive(Clap, Debug)]
 #[clap(setting = AppSettings::NoBinaryName)]
@@ -32,7 +32,7 @@ fn main() {
                 match Commands::try_parse_from(line.split_whitespace()) {
                     Ok(cmd) => {
                         run_cmd(&cmd.cmd);
-                    },
+                    }
                     Err(err) => {
                         eprintln!("{}", err);
                     }

src/main/warp_cli.rs

@@ -2,9 +2,7 @@ use bip39::{Language, Mnemonic};
 use rand::rngs::OsRng;
 use rand::{thread_rng, RngCore};
 use rusqlite::NO_PARAMS;
-use sync::{
+use sync::{pedersen_hash, print_witness2, ChainError, DbAdapter, Wallet, Witness, LWD_URL};
-    pedersen_hash, print_witness2, ChainError, DbAdapter, Wallet, Witness, LWD_URL,
-};
 use zcash_client_backend::data_api::wallet::ANCHOR_OFFSET;
 use zcash_primitives::merkle_tree::Hashable;
 use zcash_primitives::sapling::Node;

src/scan.rs

@@ -1,6 +1,6 @@
 use crate::builder::BlockProcessor;
 use crate::chain::{Nf, NfRef};
-use crate::db::{DbAdapter, ReceivedNote, AccountViewKey};
+use crate::db::{AccountViewKey, DbAdapter, ReceivedNote};
 use crate::lw_rpc::compact_tx_streamer_client::CompactTxStreamerClient;
 use crate::transaction::retrieve_tx_info;
 use crate::{
@@ -130,7 +130,6 @@ pub async fn sync_async(
 
     let processor = tokio::spawn(async move {
         let db = DbAdapter::new(&db_path)?;
-        db.synchronous(false)?;
         let mut nfs = db.get_nullifiers()?;
 
         let (mut tree, mut witnesses) = db.get_tree()?;
@@ -268,7 +267,9 @@ pub async fn sync_async(
 
             let start = Instant::now();
             if get_tx && !new_tx_ids.is_empty() {
-                retrieve_tx_info(&mut client, &db_path2, &new_tx_ids).await.unwrap();
+                retrieve_tx_info(&mut client, &db_path2, &new_tx_ids)
+                    .await
+                    .unwrap();
             }
             log::info!("Transaction Details : {}", start.elapsed().as_millis());
 

src/transaction.rs

@@ -12,6 +12,7 @@ use zcash_primitives::sapling::note_encryption::{
     try_sapling_note_decryption, try_sapling_output_recovery,
 };
 use zcash_primitives::transaction::Transaction;
+use zcash_primitives::zip32::ExtendedFullViewingKey;
 
 #[derive(Debug)]
 pub struct TransactionInfo {
@@ -31,13 +32,10 @@ pub async fn decode_transaction(
     client: &mut CompactTxStreamerClient<Channel>,
     nfs: &HashMap<(u32, Vec<u8>), u64>,
     account: u32,
-    fvk: &str,
+    fvk: &ExtendedFullViewingKey,
     tx_hash: &[u8],
     height: u32,
 ) -> anyhow::Result<TransactionInfo> {
-    let fvk =
-        decode_extended_full_viewing_key(NETWORK.hrp_sapling_extended_full_viewing_key(), &fvk)?
-            .unwrap();
     let ivk = fvk.fvk.vk.ivk();
     let ovk = fvk.fvk.ovk;
 
@@ -82,7 +80,7 @@ pub async fn decode_transaction(
                 tx_memo = memo;
             }
         } else if let Some((_note, pa, memo)) =
-        try_sapling_output_recovery(&NETWORK, height, &ovk, &output)
+            try_sapling_output_recovery(&NETWORK, height, &ovk, &output)
         {
             address = encode_payment_address(NETWORK.hrp_sapling_payment_address(), &pa);
             tx_memo = memo;
@@ -107,21 +105,33 @@ pub async fn decode_transaction(
     Ok(tx_info)
 }
 
-pub async fn retrieve_tx_info(client: &mut CompactTxStreamerClient<Channel>, db_path: &str, tx_ids: &[u32]) -> anyhow::Result<()> {
+pub async fn retrieve_tx_info(
+    client: &mut CompactTxStreamerClient<Channel>,
+    db_path: &str,
+    tx_ids: &[u32],
+) -> anyhow::Result<()> {
     let db = DbAdapter::new(db_path)?;
-    let mut tx_ids_set: HashSet<u32> = HashSet::new();
+    db.begin_transaction()?;
     let nfs = db.get_nullifiers_raw()?;
     let mut nf_map: HashMap<(u32, Vec<u8>), u64> = HashMap::new();
     for nf in nfs.iter() {
         nf_map.insert((nf.0, nf.2.clone()), nf.1);
     }
-    for &id_tx in tx_ids.iter() { // need to keep tx order
+    let mut tx_ids_set: HashSet<u32> = HashSet::new();
-        if tx_ids_set.contains(&id_tx) { continue }
+    let mut fvk_cache: HashMap<u32, ExtendedFullViewingKey> = HashMap::new();
+    for &id_tx in tx_ids.iter() {
+        // need to keep tx order
+        if tx_ids_set.contains(&id_tx) {
+            continue;
+        }
         tx_ids_set.insert(id_tx);
-        let (account, height, tx_hash) = db.get_txhash(id_tx)?;
+        let (account, height, tx_hash, ivk) = db.get_txhash(id_tx)?;
-        let fvk = db.get_ivk(account)?;
+        let fvk = fvk_cache.entry(account).or_insert_with(|| {
-        let tx_info =
+            decode_extended_full_viewing_key(NETWORK.hrp_sapling_extended_full_viewing_key(), &ivk)
-            decode_transaction(client, &nf_map, account, &fvk, &tx_hash, height).await?;
+                .unwrap()
+                .unwrap()
+        });
+        let tx_info = decode_transaction(client, &nf_map, account, fvk, &tx_hash, height).await?;
         if !tx_info.address.is_empty() && !tx_info.memo.is_empty() {
             if let Some(contact) = decode_contact(&tx_info.address, &tx_info.memo)? {
                 db.store_contact(account, &contact)?;
@@ -129,6 +139,7 @@ pub async fn retrieve_tx_info(client: &mut CompactTxStreamerClient<Channel>, db_
         }
         db.store_tx_metadata(id_tx, &tx_info)?;
     }
+    db.commit()?;
 
     Ok(())
 }
@@ -140,15 +151,19 @@ fn decode_contact(address: &str, memo: &str) -> anyhow::Result<Option<Contact>>
             name: name.trim().to_string(),
             address: address.to_string(),
         })
-    } else { None };
+    } else {
+        None
+    };
     Ok(res)
 }
 
 #[cfg(test)]
 mod tests {
     use crate::transaction::decode_transaction;
-    use crate::{connect_lightwalletd, DbAdapter, LWD_URL};
+    use crate::{connect_lightwalletd, DbAdapter, LWD_URL, NETWORK};
     use std::collections::HashMap;
+    use zcash_client_backend::encoding::decode_extended_full_viewing_key;
+    use zcash_primitives::consensus::Parameters;
 
     #[tokio::test]
     async fn test_decode_transaction() {
@@ -166,6 +181,10 @@ mod tests {
             }
         }
         let fvk = db.get_ivk(account).unwrap();
+        let fvk =
+            decode_extended_full_viewing_key(NETWORK.hrp_sapling_extended_full_viewing_key(), &fvk)
+                .unwrap()
+                .unwrap();
         let tx_info = decode_transaction(&mut client, &nf_map, account, &fvk, &tx_hash, 1313212)
             .await
             .unwrap();