zecfoundation
/
zcash-sync
mirror of https://github.com/ZcashFoundation/zcash-sync.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

GetTransactionInfo performance optimizations

This commit is contained in:
Hanh 2021-07-30 16:11:44 +08:00
parent 632792ffba
commit 7590bf179b
6 changed files with 88 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/chain.rs
View File

@ -1,4 +1,5 @@
use crate::commitment::{CTree, Witness};
use crate::db::AccountViewKey;
use crate::lw_rpc::compact_tx_streamer_client::CompactTxStreamerClient;
use crate::lw_rpc::*;
use crate::{advance_tree, NETWORK};
@ -17,7 +18,6 @@ use zcash_primitives::sapling::note_encryption::try_sapling_compact_note_decrypt
use zcash_primitives::sapling::{Node, Note, PaymentAddress};
use zcash_primitives::transaction::components::sapling::CompactOutputDescription;
use zcash_primitives::zip32::ExtendedFullViewingKey;
use crate::db::AccountViewKey;
const MAX_CHUNK: u32 = 50000;
@ -130,7 +130,7 @@ pub fn to_output_description(co: &CompactOutput) -> CompactOutputDescription {
fn decrypt_notes<'a>(
block: &'a CompactBlock,
vks: &HashMap<u32, AccountViewKey>
vks: &HashMap<u32, AccountViewKey>,
) -> DecryptedBlock<'a> {
let height = BlockHeight::from_u32(block.height as u32);
let mut count_outputs = 0u32;
@ -366,6 +366,7 @@ mod tests {
calculate_tree_state_v1, calculate_tree_state_v2, download_chain, get_latest_height,
get_tree_state, DecryptNode,
};
use crate::db::AccountViewKey;
use crate::lw_rpc::compact_tx_streamer_client::CompactTxStreamerClient;
use crate::LWD_URL;
use crate::NETWORK;
@ -374,7 +375,6 @@ mod tests {
use std::time::Instant;
use zcash_client_backend::encoding::decode_extended_full_viewing_key;
use zcash_primitives::consensus::{NetworkUpgrade, Parameters};
use crate::db::AccountViewKey;
#[tokio::test]
async fn test_get_latest_height() -> anyhow::Result<()> {

74
src/db.rs
View File

@ -1,15 +1,15 @@
use crate::chain::{Nf, NfRef};
use crate::db::migration::{get_schema_version, update_schema_version};
use crate::taddr::{derive_tkeys, BIP44_PATH};
use crate::transaction::{TransactionInfo, Contact};
use crate::transaction::{Contact, TransactionInfo};
use crate::{CTree, Witness, NETWORK};
use rusqlite::{params, Connection, OptionalExtension, NO_PARAMS};
use std::collections::HashMap;
use zcash_client_backend::encoding::decode_extended_full_viewing_key;
use zcash_primitives::consensus::{NetworkUpgrade, Parameters};
use zcash_primitives::merkle_tree::IncrementalWitness;
use zcash_primitives::sapling::{Diversifier, Node, Note, Rseed, SaplingIvk};
use zcash_primitives::zip32::{DiversifierIndex, ExtendedFullViewingKey};
use zcash_client_backend::encoding::decode_extended_full_viewing_key;
mod migration;
@ -57,14 +57,10 @@ impl AccountViewKey {
impl DbAdapter {
pub fn new(db_path: &str) -> anyhow::Result<DbAdapter> {
let connection = Connection::open(db_path)?;
connection.execute("PRAGMA synchronous = off", NO_PARAMS)?;
Ok(DbAdapter { connection })
}
pub fn synchronous(&self, flag: bool) -> anyhow::Result<()> {
self.connection.execute(&format!("PRAGMA synchronous = {}", if flag { "on" } else { "off" }), NO_PARAMS)?;
Ok(())
}
pub fn begin_transaction(&self) -> anyhow::Result<()> {
self.connection.execute("BEGIN TRANSACTION", NO_PARAMS)?;
Ok(())
@ -175,7 +171,8 @@ impl DbAdapter {
account INTEGER NOT NULL,
name TEXT NOT NULL,
address TEXT NOT NULL,
PRIMARY KEY (account, address))", NO_PARAMS
PRIMARY KEY (account, address))",
NO_PARAMS,
)?;
}
@ -213,9 +210,21 @@ impl DbAdapter {
let account: u32 = row.get(0)?;
let ivk: String = row.get(1)?;
let sk: Option<String> = row.get(2)?;
let fvk = decode_extended_full_viewing_key(NETWORK.hrp_sapling_extended_full_viewing_key(), &ivk).unwrap().unwrap();
let fvk = decode_extended_full_viewing_key(
NETWORK.hrp_sapling_extended_full_viewing_key(),
&ivk,
)
.unwrap()
.unwrap();
let ivk = fvk.fvk.vk.ivk();
Ok((account, AccountViewKey { fvk, ivk, viewonly: sk.is_none() }))
Ok((
account,
AccountViewKey {
fvk,
ivk,
viewonly: sk.is_none(),
},
))
})?;
let mut fvks: HashMap<u32, AccountViewKey> = HashMap::new();
for r in rows {
@ -245,18 +254,19 @@ impl DbAdapter {
Ok(())
}
pub fn get_txhash(&self, id_tx: u32) -> anyhow::Result<(u32, u32, Vec<u8>)> {
let (account, height, tx_hash) = self.connection.query_row(
"SELECT account, height, txid FROM transactions WHERE id_tx = ?1",
pub fn get_txhash(&self, id_tx: u32) -> anyhow::Result<(u32, u32, Vec<u8>, String)> {
let (account, height, tx_hash, ivk) = self.connection.query_row(
"SELECT account, height, txid, ivk FROM transactions t, accounts a WHERE id_tx = ?1 AND t.account = a.id_account",
params![id_tx],
|row| {
let account: u32 = row.get(0)?;
let height: u32 = row.get(1)?;
let tx_hash: Vec<u8> = row.get(2)?;
Ok((account, height, tx_hash))
let ivk: String = row.get(3)?;
Ok((account, height, tx_hash, ivk))
},
)?;
Ok((account, height, tx_hash))
Ok((account, height, tx_hash, ivk))
}
pub fn store_block(
@ -574,13 +584,17 @@ impl DbAdapter {
pub fn store_contact(&self, account: u32, contact: &Contact) -> anyhow::Result<()> {
log::info!("{:?}", contact);
if contact.name.is_empty() {
self.connection.execute("DELETE FROM contacts WHERE account = ?1 AND address = ?2",
params![account, contact.address])?;
}
else {
self.connection.execute("INSERT INTO contacts(account, name, address)
self.connection.execute(
"DELETE FROM contacts WHERE account = ?1 AND address = ?2",
params![account, contact.address],
)?;
} else {
self.connection.execute(
"INSERT INTO contacts(account, name, address)
VALUES (?1, ?2, ?3) ON CONFLICT (account, address) DO UPDATE SET
name = excluded.name", params![account, &contact.name, &contact.address])?;
name = excluded.name",
params![account, &contact.name, &contact.address],
)?;
}
Ok(())
}
@ -606,16 +620,14 @@ impl DbAdapter {
pub fn get_seed(&self, account: u32) -> anyhow::Result<Option<String>> {
log::info!("+get_seed");
let seed = self
.connection
.query_row(
"SELECT seed FROM accounts WHERE id_account = ?1",
params![account],
|row| {
let sk: Option<String> = row.get(0)?;
Ok(sk)
},
)?;
let seed = self.connection.query_row(
"SELECT seed FROM accounts WHERE id_account = ?1",
params![account],
|row| {
let sk: Option<String> = row.get(0)?;
Ok(sk)
},
)?;
log::info!("-get_seed");
Ok(seed)
}

6
src/main/cli.rs
View File

@ -1,6 +1,6 @@
use rustyline::Editor;
use rustyline::error::ReadlineError;
use clap::{AppSettings, Clap};
use rustyline::error::ReadlineError;
use rustyline::Editor;
#[derive(Clap, Debug)]
#[clap(setting = AppSettings::NoBinaryName)]
@ -32,7 +32,7 @@ fn main() {
match Commands::try_parse_from(line.split_whitespace()) {
Ok(cmd) => {
run_cmd(&cmd.cmd);
},
}
Err(err) => {
eprintln!("{}", err);
}

4
src/main/warp_cli.rs
View File

@ -2,9 +2,7 @@ use bip39::{Language, Mnemonic};
use rand::rngs::OsRng;
use rand::{thread_rng, RngCore};
use rusqlite::NO_PARAMS;
use sync::{
pedersen_hash, print_witness2, ChainError, DbAdapter, Wallet, Witness, LWD_URL,
};
use sync::{pedersen_hash, print_witness2, ChainError, DbAdapter, Wallet, Witness, LWD_URL};
use zcash_client_backend::data_api::wallet::ANCHOR_OFFSET;
use zcash_primitives::merkle_tree::Hashable;
use zcash_primitives::sapling::Node;

7
src/scan.rs
View File

@ -1,6 +1,6 @@
use crate::builder::BlockProcessor;
use crate::chain::{Nf, NfRef};
use crate::db::{DbAdapter, ReceivedNote, AccountViewKey};
use crate::db::{AccountViewKey, DbAdapter, ReceivedNote};
use crate::lw_rpc::compact_tx_streamer_client::CompactTxStreamerClient;
use crate::transaction::retrieve_tx_info;
use crate::{
@ -130,7 +130,6 @@ pub async fn sync_async(
let processor = tokio::spawn(async move {
let db = DbAdapter::new(&db_path)?;
db.synchronous(false)?;
let mut nfs = db.get_nullifiers()?;
let (mut tree, mut witnesses) = db.get_tree()?;
@ -268,7 +267,9 @@ pub async fn sync_async(
let start = Instant::now();
if get_tx && !new_tx_ids.is_empty() {
retrieve_tx_info(&mut client, &db_path2, &new_tx_ids).await.unwrap();
retrieve_tx_info(&mut client, &db_path2, &new_tx_ids)
.await
.unwrap();
}
log::info!("Transaction Details : {}", start.elapsed().as_millis());

49
src/transaction.rs
View File

@ -12,6 +12,7 @@ use zcash_primitives::sapling::note_encryption::{
try_sapling_note_decryption, try_sapling_output_recovery,
};
use zcash_primitives::transaction::Transaction;
use zcash_primitives::zip32::ExtendedFullViewingKey;
#[derive(Debug)]
pub struct TransactionInfo {
@ -31,13 +32,10 @@ pub async fn decode_transaction(
client: &mut CompactTxStreamerClient<Channel>,
nfs: &HashMap<(u32, Vec<u8>), u64>,
account: u32,
fvk: &str,
fvk: &ExtendedFullViewingKey,
tx_hash: &[u8],
height: u32,
) -> anyhow::Result<TransactionInfo> {
let fvk =
decode_extended_full_viewing_key(NETWORK.hrp_sapling_extended_full_viewing_key(), &fvk)?
.unwrap();
let ivk = fvk.fvk.vk.ivk();
let ovk = fvk.fvk.ovk;
@ -82,7 +80,7 @@ pub async fn decode_transaction(
tx_memo = memo;
}
} else if let Some((_note, pa, memo)) =
try_sapling_output_recovery(&NETWORK, height, &ovk, &output)
try_sapling_output_recovery(&NETWORK, height, &ovk, &output)
{
address = encode_payment_address(NETWORK.hrp_sapling_payment_address(), &pa);
tx_memo = memo;
@ -107,21 +105,33 @@ pub async fn decode_transaction(
Ok(tx_info)
}
pub async fn retrieve_tx_info(client: &mut CompactTxStreamerClient<Channel>, db_path: &str, tx_ids: &[u32]) -> anyhow::Result<()> {
pub async fn retrieve_tx_info(
client: &mut CompactTxStreamerClient<Channel>,
db_path: &str,
tx_ids: &[u32],
) -> anyhow::Result<()> {
let db = DbAdapter::new(db_path)?;
let mut tx_ids_set: HashSet<u32> = HashSet::new();
db.begin_transaction()?;
let nfs = db.get_nullifiers_raw()?;
let mut nf_map: HashMap<(u32, Vec<u8>), u64> = HashMap::new();
for nf in nfs.iter() {
nf_map.insert((nf.0, nf.2.clone()), nf.1);
}
for &id_tx in tx_ids.iter() { // need to keep tx order
if tx_ids_set.contains(&id_tx) { continue }
let mut tx_ids_set: HashSet<u32> = HashSet::new();
let mut fvk_cache: HashMap<u32, ExtendedFullViewingKey> = HashMap::new();
for &id_tx in tx_ids.iter() {
// need to keep tx order
if tx_ids_set.contains(&id_tx) {
continue;
}
tx_ids_set.insert(id_tx);
let (account, height, tx_hash) = db.get_txhash(id_tx)?;
let fvk = db.get_ivk(account)?;
let tx_info =
decode_transaction(client, &nf_map, account, &fvk, &tx_hash, height).await?;
let (account, height, tx_hash, ivk) = db.get_txhash(id_tx)?;
let fvk = fvk_cache.entry(account).or_insert_with(|| {
decode_extended_full_viewing_key(NETWORK.hrp_sapling_extended_full_viewing_key(), &ivk)
.unwrap()
.unwrap()
});
let tx_info = decode_transaction(client, &nf_map, account, fvk, &tx_hash, height).await?;
if !tx_info.address.is_empty() && !tx_info.memo.is_empty() {
if let Some(contact) = decode_contact(&tx_info.address, &tx_info.memo)? {
db.store_contact(account, &contact)?;
@ -129,6 +139,7 @@ pub async fn retrieve_tx_info(client: &mut CompactTxStreamerClient<Channel>, db_
}
db.store_tx_metadata(id_tx, &tx_info)?;
}
db.commit()?;
Ok(())
}
@ -140,15 +151,19 @@ fn decode_contact(address: &str, memo: &str) -> anyhow::Result<Option<Contact>>
name: name.trim().to_string(),
address: address.to_string(),
})
} else { None };
} else {
None
};
Ok(res)
}
#[cfg(test)]
mod tests {
use crate::transaction::decode_transaction;
use crate::{connect_lightwalletd, DbAdapter, LWD_URL};
use crate::{connect_lightwalletd, DbAdapter, LWD_URL, NETWORK};
use std::collections::HashMap;
use zcash_client_backend::encoding::decode_extended_full_viewing_key;
use zcash_primitives::consensus::Parameters;
#[tokio::test]
async fn test_decode_transaction() {
@ -166,6 +181,10 @@ mod tests {
}
}
let fvk = db.get_ivk(account).unwrap();
let fvk =
decode_extended_full_viewing_key(NETWORK.hrp_sapling_extended_full_viewing_key(), &fvk)
.unwrap()
.unwrap();
let tx_info = decode_transaction(&mut client, &nf_map, account, &fvk, &tx_hash, 1313212)
.await
.unwrap();