diff --git a/Cargo.toml b/Cargo.toml index 7679d04..783ac8c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,7 @@ log = "0.4.14" flexi_logger = {version="0.17.1", features = ["compress"]} serde = {version = "1.0.126", features = ["derive"]} serde_json = "1.0.64" +bincode = "1.3.3" tokio = { version = "^1.6", features = ["macros", "rt-multi-thread"] } tokio-stream = "0.1.7" protobuf = "2.23.0" diff --git a/src/contact.rs b/src/contact.rs new file mode 100644 index 0000000..056697e --- /dev/null +++ b/src/contact.rs @@ -0,0 +1,115 @@ +use prost::bytes::{BufMut, Buf}; +use serde::{Deserialize, Serialize}; +use zcash_primitives::memo::{Memo, MemoBytes}; +use std::convert::TryFrom; + +const CONTACT_COOKIE: u32 = 0x434E5440; + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct Contact { + pub id: u32, + pub name: String, + pub address: String, +} + +pub fn serialize_contacts(contacts: &[Contact]) -> anyhow::Result> { + let cs_bin = bincode::serialize(&contacts)?; + let chunks = cs_bin.chunks(500); + let memos: Vec<_> = chunks.enumerate().map(|(i, c)| { + let n = i as u8; + let mut bytes = [0u8; 511]; + let mut bb: Vec = vec![]; + bb.put_u32(CONTACT_COOKIE); + bb.put_u8(n); + bb.put_u16(c.len() as u16); + bb.put_slice(c); + bytes[0..bb.len()].copy_from_slice(&bb); + Memo::Arbitrary(Box::new(bytes)) + }).collect(); + + Ok(memos) +} + +pub struct ContactDecoder { + has_contacts: bool, + chunks: Vec>, +} + +impl ContactDecoder { + pub fn new(n: usize) -> ContactDecoder { + let mut chunks = vec![]; + chunks.resize(n, vec![]); + ContactDecoder { + has_contacts: false, + chunks + } + } + + pub fn add_memo(&mut self, memo: &MemoBytes) -> anyhow::Result<()> { + let memo = Memo::try_from(memo.clone())?; + if let Memo::Arbitrary(bytes) = memo { + let (n, data) = ContactDecoder::_decode_box(&bytes)?; + self.has_contacts = true; + self.chunks[n as usize] = data; + } + + Ok(()) + } + + pub fn finalize(&self) -> anyhow::Result> { + if !self.has_contacts { + return Ok(Vec::new()) + } + let data: Vec<_> = self.chunks.iter().cloned().flatten().collect(); + let contacts = bincode::deserialize::>(&data)?; + Ok(contacts) + } + + fn _decode_box(bb: &[u8; 511]) -> anyhow::Result<(u8, Vec)> { + let mut bb: &[u8] = bb; + let magic = bb.get_u32(); + if magic != CONTACT_COOKIE { + anyhow::bail!("Not a contact record"); + } + let n = bb.get_u8(); + let len = bb.get_u16() as usize; + if len > bb.len() { + anyhow::bail!("Buffer overflow"); + } + + let data = &bb[0..len]; + Ok((n, data.to_vec())) + } +} + +#[cfg(test)] +mod tests { + use crate::{DbAdapter, LWD_URL, Wallet}; + use crate::contact::{serialize_contacts, Contact}; + use crate::db::DEFAULT_DB_PATH; + + #[test] + fn test_contacts() { + let db = DbAdapter::new(DEFAULT_DB_PATH).unwrap(); + let contact = Contact { + id: 0, + name: "hanh".to_string(), + address: "zs1lvzgfzzwl9n85446j292zg0valw2p47hmxnw42wnqsehsmyuvjk0mhxktcs0pqrplacm2vchh35".to_string(), + }; + db.store_contact(&contact, true).unwrap(); + } + + #[tokio::test] + async fn test_serialize() { + let db = DbAdapter::new(DEFAULT_DB_PATH).unwrap(); + let contacts = db.get_unsaved_contacts().unwrap(); + let memos = serialize_contacts(&contacts).unwrap(); + for m in memos.iter() { + println!("{:?}", m); + } + + let wallet = Wallet::new("zec.db", LWD_URL); + let tx_id = wallet.save_contacts_tx(&memos, 1, 3).await.unwrap(); + println!("{}", tx_id); + } +} diff --git a/src/db.rs b/src/db.rs index 21592e6..0728649 100644 --- a/src/db.rs +++ b/src/db.rs @@ -1,7 +1,7 @@ use crate::chain::{Nf, NfRef}; use crate::prices::Quote; use crate::taddr::derive_tkeys; -use crate::transaction::{Contact, TransactionInfo}; +use crate::transaction::TransactionInfo; use crate::{CTree, Witness, NETWORK}; use chrono::NaiveDateTime; use rusqlite::{params, Connection, OptionalExtension, NO_PARAMS, Transaction}; @@ -11,6 +11,7 @@ use zcash_primitives::consensus::{NetworkUpgrade, Parameters}; use zcash_primitives::merkle_tree::IncrementalWitness; use zcash_primitives::sapling::{Diversifier, Node, Note, Rseed, SaplingIvk}; use zcash_primitives::zip32::{DiversifierIndex, ExtendedFullViewingKey}; +use crate::contact::Contact; mod migration; @@ -482,24 +483,46 @@ impl DbAdapter { Ok(()) } - pub fn store_contact(&self, contact: &Contact) -> anyhow::Result<()> { - log::info!("{:?}", contact); - if contact.name.is_empty() { + pub fn store_contact(&self, contact: &Contact, dirty: bool) -> anyhow::Result<()> { + if contact.id == 0 { self.connection.execute( - "DELETE FROM contacts WHERE account = ?1 AND address = ?2", - params![contact.account, contact.address], + "INSERT INTO contacts(name, address, dirty) + VALUES (?1, ?2, ?3)", + params![&contact.name, &contact.address, dirty], )?; - } else { + } + else { self.connection.execute( - "INSERT INTO contacts(account, name, address) - VALUES (?1, ?2, ?3) ON CONFLICT (account, address) DO UPDATE SET - name = excluded.name", - params![contact.account, &contact.name, &contact.address], + "INSERT INTO contacts(id, name, address, dirty) + VALUES (?1, ?2, ?3, ?4) ON CONFLICT (id) DO UPDATE SET + name = excluded.name, address = excluded.address, dirty = excluded.dirty", + params![contact.id, &contact.name, &contact.address, dirty], )?; } Ok(()) } + pub fn get_unsaved_contacts(&self) -> anyhow::Result> { + let mut statement = self.connection.prepare("SELECT id, name, address FROM contacts WHERE dirty = TRUE")?; + let rows = statement.query_map(NO_PARAMS, |row| { + let id: u32 = row.get(0)?; + let name: String = row.get(1)?; + let address: String = row.get(2)?; + let contact = Contact { + id, + name, + address, + }; + Ok(contact) + })?; + let mut contacts: Vec = vec![]; + for r in rows { + contacts.push(r?); + } + + Ok(contacts) + } + pub fn get_backup( &self, account: u32, @@ -696,6 +719,17 @@ impl DbAdapter { }).optional()?; Ok(quote) } + + pub fn truncate_data(&self) -> anyhow::Result<()> { + self.connection.execute("DELETE FROM blocks", NO_PARAMS)?; + self.connection.execute("DELETE FROM contacts", NO_PARAMS)?; + self.connection.execute("DELETE FROM diversifiers", NO_PARAMS)?; + self.connection.execute("DELETE FROM historical_prices", NO_PARAMS)?; + self.connection.execute("DELETE FROM received_notes", NO_PARAMS)?; + self.connection.execute("DELETE FROM sapling_witnesses", NO_PARAMS)?; + self.connection.execute("DELETE FROM transactions", NO_PARAMS)?; + Ok(()) + } } #[cfg(test)] diff --git a/src/db/migration.rs b/src/db/migration.rs index d2c9468..99608fb 100644 --- a/src/db/migration.rs +++ b/src/db/migration.rs @@ -142,7 +142,22 @@ pub fn init_db(connection: &Connection) -> anyhow::Result<()> { )?; } - update_schema_version(&connection, 5)?; + if version < 6 { + connection.execute( + "DROP TABLE contacts", + NO_PARAMS, + )?; + connection.execute( + "CREATE TABLE IF NOT EXISTS contacts ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + address TEXT NOT NULL, + dirty BOOL NOT NULL)", + NO_PARAMS, + )?; + } + + update_schema_version(&connection, 6)?; Ok(()) } diff --git a/src/lib.rs b/src/lib.rs index b54b295..a9dd124 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -8,12 +8,12 @@ pub use coin::{get_branch, NETWORK, TICKER}; // pub const LWD_URL: &str = "https://mainnet.lightwalletd.com:9067"; // pub const LWD_URL: &str = "https://lwdv3.zecwallet.co"; // pub const LWD_URL: &str = "http://lwd.hanh.me:9067"; -pub const LWD_URL: &str = "http://127.0.0.1:9067"; +// pub const LWD_URL: &str = "http://127.0.0.1:9067"; // Testnet // pub const LWD_URL: &str = "https://testnet.lightwalletd.com:9067"; // pub const LWD_URL: &str = "http://lwd.hanh.me:9067"; -// pub const LWD_URL: &str = "http://127.0.0.1:9067"; +pub const LWD_URL: &str = "http://127.0.0.1:9067"; mod builder; mod chain; @@ -29,6 +29,7 @@ mod print; mod scan; mod taddr; mod transaction; +mod contact; mod wallet; pub use crate::builder::advance_tree; diff --git a/src/pay.rs b/src/pay.rs index c874faa..f5e3302 100644 --- a/src/pay.rs +++ b/src/pay.rs @@ -1,5 +1,5 @@ use crate::db::SpendableNote; -use crate::wallet::Recipient; +use crate::wallet::RecipientMemo; use crate::{connect_lightwalletd, get_latest_height, RawTransaction, NETWORK}; use jubjub::Fr; use rand::rngs::OsRng; @@ -19,8 +19,6 @@ use zcash_primitives::transaction::components::amount::DEFAULT_FEE; use zcash_primitives::transaction::components::Amount; use zcash_primitives::zip32::{ExtendedFullViewingKey, ExtendedSpendingKey}; use zcash_proofs::prover::LocalTxProver; -use std::str::FromStr; -use std::convert::TryFrom; #[derive(Serialize, Deserialize, Debug)] pub struct Tx { @@ -72,7 +70,7 @@ pub trait TxBuilder { address: &str, ovk: &OutgoingViewingKey, amount: Amount, - memo: &MemoBytes, + memo: &Memo, ) -> anyhow::Result<()>; } @@ -128,13 +126,13 @@ impl TxBuilder for ColdTxBuilder { address: &str, ovk: &OutgoingViewingKey, amount: Amount, - memo: &MemoBytes, + memo: &Memo, ) -> anyhow::Result<()> { let tx_out = TxOut { addr: address.to_string(), amount: u64::from(amount), ovk: hex::encode(ovk.0), - memo: hex::encode(memo.as_slice()), + memo: hex::encode(MemoBytes::from(memo).as_slice()), }; self.tx.outputs.push(tx_out); Ok(()) @@ -178,12 +176,13 @@ impl TxBuilder for Builder<'_, Network, OsRng> { address: &str, ovk: &OutgoingViewingKey, amount: Amount, - memo: &MemoBytes, + memo: &Memo, ) -> anyhow::Result<()> { let to_addr = RecipientAddress::decode(&NETWORK, address) .ok_or(anyhow::anyhow!("Not a valid address"))?; if let RecipientAddress::Shielded(pa) = to_addr { - self.add_sapling_output(Some(ovk.clone()), pa.clone(), amount, Some(memo.clone()))?; + let memo_bytes = MemoBytes::from(memo); + self.add_sapling_output(Some(ovk.clone()), pa.clone(), amount, Some(memo_bytes))?; } Ok(()) } @@ -195,7 +194,7 @@ pub fn prepare_tx( notes: &[SpendableNote], target_amount: Amount, fvk: &ExtendedFullViewingKey, - recipients: &[Recipient], + recipients: &[RecipientMemo], ) -> anyhow::Result> { let mut amount = target_amount; amount += DEFAULT_FEE; @@ -242,9 +241,7 @@ pub fn prepare_tx( match &to_addr { RecipientAddress::Shielded(_pa) => { log::info!("Sapling output: {}", r.amount); - let memo = Memo::from_str(&r.memo)?; - let memo = MemoBytes::try_from(memo)?; - builder.add_z_output(&r.address, ovk, amount, &memo) + builder.add_z_output(&r.address, ovk, amount, &r.memo) } RecipientAddress::Transparent(_address) => builder.add_t_output(&r.address, amount), }?; diff --git a/src/transaction.rs b/src/transaction.rs index 7406e49..6ad4084 100644 --- a/src/transaction.rs +++ b/src/transaction.rs @@ -16,9 +16,11 @@ use zcash_primitives::sapling::note_encryption::{ }; use zcash_primitives::transaction::Transaction; use zcash_primitives::zip32::ExtendedFullViewingKey; +use crate::contact::{ContactDecoder, Contact}; #[derive(Debug)] pub struct TransactionInfo { + height: u32, index: u32, // index of tx in block id_tx: u32, // id of tx in db account: u32, @@ -26,14 +28,14 @@ pub struct TransactionInfo { pub memo: String, amount: i64, pub fee: u64, + pub contacts: Vec, } #[derive(Debug)] -pub struct Contact { - pub account: u32, - index: u32, - pub name: String, - pub address: String, +pub struct ContactRef { + pub height: u32, + pub index: u32, + pub contact: Contact, } pub async fn decode_transaction( @@ -70,6 +72,8 @@ pub async fn decode_transaction( } } + let mut contact_decoder = ContactDecoder::new(tx.shielded_outputs.len()); + let mut tx_memo = MemoBytes::empty(); for output in tx.vout.iter() { if let Some(t_address) = output.script_pubkey.address() { @@ -85,6 +89,7 @@ pub async fn decode_transaction( if let Some((note, pa, memo)) = try_sapling_note_decryption(&NETWORK, height, &ivk, output) { amount += note.value as i64; // change or self transfer + contact_decoder.add_memo(&memo)?; if address.is_empty() { address = encode_payment_address(NETWORK.hrp_sapling_payment_address(), &pa); tx_memo = memo; @@ -105,7 +110,9 @@ pub async fn decode_transaction( Memo::Future(_) => "Unrecognized".to_string(), Memo::Arbitrary(_) => "Unrecognized".to_string(), }; + let contacts = contact_decoder.finalize()?; let tx_info = TransactionInfo { + height: u32::from(height), index, id_tx, account, @@ -113,6 +120,7 @@ pub async fn decode_transaction( memo, amount, fee, + contacts, }; Ok(tx_info) @@ -185,23 +193,20 @@ pub async fn retrieve_tx_info( }); let f = tokio::spawn(async move { - let mut contacts: Vec = vec![]; + let mut contacts: Vec = vec![]; while let Ok(tx_info) = rx.recv() { - if !tx_info.address.is_empty() && !tx_info.memo.is_empty() { - if let Some(contact) = decode_contact( - tx_info.account, - tx_info.index, - &tx_info.address, - &tx_info.memo, - )? { - contacts.push(contact); - } + for c in tx_info.contacts.iter() { + contacts.push(ContactRef { + height: tx_info.height, + index: tx_info.index, + contact: c.clone(), + }); } db.store_tx_metadata(tx_info.id_tx, &tx_info)?; } contacts.sort_by(|a, b| a.index.cmp(&b.index)); - for c in contacts.iter() { - db.store_contact(c)?; + for cref in contacts.iter() { + db.store_contact(&cref.contact, false)?; } Ok::<_, anyhow::Error>(()) @@ -214,26 +219,6 @@ pub async fn retrieve_tx_info( Ok(()) } -fn decode_contact( - account: u32, - index: u32, - address: &str, - memo: &str, -) -> anyhow::Result> { - let res = if let Some(memo_line) = memo.lines().next() { - let name = memo_line.strip_prefix("Contact:"); - name.map(|name| Contact { - account, - index, - name: name.trim().to_string(), - address: address.to_string(), - }) - } else { - None - }; - Ok(res) -} - #[cfg(test)] mod tests { use crate::transaction::decode_transaction; diff --git a/src/wallet.rs b/src/wallet.rs index 85f8bad..eca5614 100644 --- a/src/wallet.rs +++ b/src/wallet.rs @@ -24,10 +24,13 @@ use zcash_client_backend::encoding::{ use zcash_params::{OUTPUT_PARAMS, SPEND_PARAMS}; use zcash_primitives::consensus::{BlockHeight, Parameters}; use zcash_primitives::transaction::builder::{Builder, Progress}; -use zcash_primitives::transaction::components::amount::MAX_MONEY; +use zcash_primitives::transaction::components::amount::{MAX_MONEY, DEFAULT_FEE}; use zcash_primitives::transaction::components::Amount; use zcash_primitives::zip32::ExtendedFullViewingKey; use zcash_proofs::prover::LocalTxProver; +use zcash_primitives::memo::Memo; +use std::str::FromStr; +use crate::contact::{Contact, serialize_contacts}; const DEFAULT_CHUNK_SIZE: u32 = 100_000; @@ -62,6 +65,22 @@ pub struct Recipient { pub memo: String, } +pub struct RecipientMemo { + pub address: String, + pub amount: u64, + pub memo: Memo, +} + +impl From<&Recipient> for RecipientMemo { + fn from(r: &Recipient) -> Self { + RecipientMemo { + address: r.address.clone(), + amount: r.amount, + memo: Memo::from_str(&r.memo).unwrap(), + } + } +} + impl Wallet { pub fn new(db_path: &str, ld_url: &str) -> Wallet { let prover = LocalTxProver::from_bytes(SPEND_PARAMS, OUTPUT_PARAMS); @@ -212,6 +231,7 @@ impl Wallet { progress_callback: impl Fn(Progress) + Send + 'static, ) -> anyhow::Result { let recipients: Vec = serde_json::from_str(recipients_json)?; + let recipients: Vec<_> = recipients.iter().map(|r| RecipientMemo::from(r)).collect(); self._send_payment(account, &recipients, anchor_offset, false, progress_callback) .await } @@ -237,7 +257,7 @@ impl Wallet { account: u32, amount: u64, last_height: u32, - recipients: &Vec, + recipients: &[RecipientMemo], anchor_offset: u32, ) -> anyhow::Result { let amount = Amount::from_u64(amount).unwrap(); @@ -247,7 +267,7 @@ impl Wallet { &ivk, )? .unwrap(); - let notes = self._get_spendable_notes(account, &extfvk, last_height, anchor_offset)?; + let notes = self.get_spendable_notes(account, &extfvk, last_height, anchor_offset)?; let mut builder = ColdTxBuilder::new(last_height); prepare_tx(&mut builder, None, ¬es, amount, &extfvk, recipients)?; Ok(builder.tx) @@ -272,7 +292,7 @@ impl Wallet { async fn _send_payment( &mut self, account: u32, - recipients: &[Recipient], + recipients: &[RecipientMemo], anchor_offset: u32, shield_transparent_balance: bool, progress_callback: impl Fn(Progress) + Send + 'static, @@ -284,7 +304,7 @@ impl Wallet { .unwrap(); let extfvk = ExtendedFullViewingKey::from(&skey); let last_height = self.get_latest_height().await?; - let notes = self._get_spendable_notes(account, &extfvk, last_height, anchor_offset)?; + let notes = self.get_spendable_notes(account, &extfvk, last_height, anchor_offset)?; log::info!("Spendable notes = {}", notes.len()); let mut builder = Builder::new(NETWORK, BlockHeight::from_u32(last_height)); @@ -369,26 +389,72 @@ impl Wallet { shield_taddr(&self.db, account, &self.prover, &self.ld_url).await } + pub fn store_contact(&self, id: u32, name: &str, address: &str, dirty: bool) -> anyhow::Result<()> { + let contact = Contact { + id, + name: name.to_string(), + address: address.to_string(), + }; + self.db.store_contact(&contact, dirty)?; + Ok(()) + } + + pub async fn commit_unsaved_contacts(&self, account: u32, anchor_offset: u32) -> anyhow::Result { + let contacts = self.db.get_unsaved_contacts()?; + let memos = serialize_contacts(&contacts)?; + let tx_id = self.save_contacts_tx(&memos, account, anchor_offset).await.unwrap(); + Ok(tx_id) + } + + pub async fn save_contacts_tx(&self, memos: &[Memo], account: u32, anchor_offset: u32) -> anyhow::Result { + let mut client = connect_lightwalletd(&self.ld_url).await?; + let last_height = get_latest_height(&mut client).await?; + + let secret_key = self.db.get_sk(account)?; + let address = self.db.get_address(account)?; + let skey = decode_extended_spending_key(NETWORK.hrp_sapling_extended_spending_key(), &secret_key)?.unwrap(); + let extfvk = ExtendedFullViewingKey::from(&skey); + let notes = self.get_spendable_notes(account, &extfvk, last_height, anchor_offset)?; + + let mut builder = Builder::new(NETWORK, BlockHeight::from_u32(last_height)); + + let recipients: Vec<_> = memos.iter().map(|m| { + RecipientMemo { + address: address.clone(), + amount: 0, + memo: m.clone(), + } + }).collect(); + prepare_tx(&mut builder, Some(skey), ¬es, DEFAULT_FEE, &extfvk, &recipients)?; + + let consensus_branch_id = get_branch(last_height); + let (tx, _) = builder.build(consensus_branch_id, &self.prover)?; + let mut raw_tx: Vec = vec![]; + tx.write(&mut raw_tx)?; + + let tx_id = send_transaction(&mut client, &raw_tx, last_height).await?; + log::info!("Tx ID = {}", tx_id); + Ok(tx_id) + } + pub fn set_lwd_url(&mut self, ld_url: &str) -> anyhow::Result<()> { self.ld_url = ld_url.to_string(); Ok(()) } - fn _get_spendable_notes( + pub fn get_spendable_notes( &self, account: u32, extfvk: &ExtendedFullViewingKey, last_height: u32, anchor_offset: u32, ) -> anyhow::Result> { - let anchor_height = self - .db + let anchor_height = self.db .get_last_sync_height()? .ok_or_else(|| anyhow::anyhow!("No spendable notes"))?; let anchor_height = anchor_height.min(last_height - anchor_offset); log::info!("Anchor = {}", anchor_height); - let mut notes = self - .db + let mut notes = self.db .get_spendable_notes(account, anchor_height, extfvk)?; notes.shuffle(&mut OsRng); log::info!("Spendable notes = {}", notes.len()); @@ -401,8 +467,8 @@ impl Wallet { amount: u64, max_amount_per_note: u64, memo: &str, - ) -> anyhow::Result> { - let mut recipients: Vec = vec![]; + ) -> anyhow::Result> { + let mut recipients: Vec = vec![]; let target_amount = Amount::from_u64(amount).unwrap(); let max_amount_per_note = if max_amount_per_note != 0 { Amount::from_u64(max_amount_per_note).unwrap() @@ -412,10 +478,10 @@ impl Wallet { let mut remaining_amount = target_amount; while remaining_amount.is_positive() { let note_amount = remaining_amount.min(max_amount_per_note); - let recipient = Recipient { + let recipient = RecipientMemo { address: to_address.to_string(), amount: u64::from(note_amount), - memo: memo.to_string(), + memo: Memo::from_str(memo)?, }; recipients.push(recipient); remaining_amount -= note_amount; @@ -435,6 +501,10 @@ impl Wallet { self.db.store_historical_prices("es, currency).unwrap(); Ok(quotes.len() as u32) } + + pub fn truncate_data(&self) -> anyhow::Result<()> { + self.db.truncate_data() + } } #[cfg(test)]