Add a scanning results reader (#8104)
* Add a scanning results reader * Simplify intro & titles in `zebra-utils/README.md` * Add a tutorial for the scanning results reader * Reformat `zebra-utils/Cargo.toml` Co-authored-by: teor <teor@riseup.net> * Update `zebra-utils/README.md` Co-authored-by: teor <teor@riseup.net> * standard grouping Co-authored-by: teor <teor@riseup.net> * Add a comment on a first empty memo Co-authored-by: teor <teor@riseup.net> * Use `exactly_one()` instead of `next()` * Simplify various type conversions --------- Co-authored-by: teor <teor@riseup.net>
This commit is contained in:
Marek
GitHub
parent
a14cb40c1f
commit
8f090e605d
15
Cargo.lock
15
Cargo.lock
|
|
@ -2192,6 +2192,17 @@ dependencies = [
|
|||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jsonrpc"
|
||||
version = "0.16.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "34efde8d2422fb79ed56db1d3aea8fa5b583351d15a26770cdee2f88813dd702"
|
||||
dependencies = [
|
||||
"base64 0.13.1",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jsonrpc-core"
|
||||
version = "18.0.0"
|
||||
|
|
@ -5996,6 +6007,7 @@ dependencies = [
|
|||
"color-eyre",
|
||||
"hex",
|
||||
"itertools 0.12.0",
|
||||
"jsonrpc",
|
||||
"regex",
|
||||
"reqwest",
|
||||
"serde_json",
|
||||
|
|
@ -6005,9 +6017,12 @@ dependencies = [
|
|||
"tokio",
|
||||
"tracing-error",
|
||||
"tracing-subscriber",
|
||||
"zcash_client_backend",
|
||||
"zcash_primitives",
|
||||
"zebra-chain",
|
||||
"zebra-node-services",
|
||||
"zebra-rpc",
|
||||
"zebra-scan",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
use std::ops::{Add, Sub};
|
||||
use thiserror::Error;
|
||||
use zcash_primitives::consensus::BlockHeight;
|
||||
|
||||
use crate::{serialization::SerializationError, BoxError};
|
||||
|
||||
|
|
@ -105,6 +106,12 @@ impl Height {
|
|||
}
|
||||
}
|
||||
|
||||
impl From<Height> for BlockHeight {
|
||||
fn from(height: Height) -> Self {
|
||||
BlockHeight::from_u32(height.0)
|
||||
}
|
||||
}
|
||||
|
||||
/// A difference between two [`Height`]s, possibly negative.
|
||||
///
|
||||
/// This can represent the difference between any height values,
|
||||
|
|
|
|||
|
|
@ -345,3 +345,12 @@ impl From<Network> for zcash_primitives::consensus::Network {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<zcash_primitives::consensus::Network> for Network {
|
||||
fn from(network: zcash_primitives::consensus::Network) -> Self {
|
||||
match network {
|
||||
zcash_primitives::consensus::Network::MainNetwork => Network::Mainnet,
|
||||
zcash_primitives::consensus::Network::TestNetwork => Network::Testnet,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -36,6 +36,11 @@ name = "block-template-to-proposal"
|
|||
path = "src/bin/block-template-to-proposal/main.rs"
|
||||
required-features = ["getblocktemplate-rpcs"]
|
||||
|
||||
[[bin]]
|
||||
name = "scanning-results-reader"
|
||||
path = "src/bin/scanning-results-reader/main.rs"
|
||||
required-features = ["shielded-scan"]
|
||||
|
||||
[features]
|
||||
default = []
|
||||
|
||||
|
|
@ -61,6 +66,14 @@ getblocktemplate-rpcs = [
|
|||
"zebra-chain/getblocktemplate-rpcs",
|
||||
]
|
||||
|
||||
shielded-scan = [
|
||||
"itertools",
|
||||
"jsonrpc",
|
||||
"zcash_primitives",
|
||||
"zcash_client_backend",
|
||||
"zebra-scan"
|
||||
]
|
||||
|
||||
[dependencies]
|
||||
color-eyre = "0.6.2"
|
||||
# This is a transitive dependency via color-eyre.
|
||||
|
|
@ -76,6 +89,7 @@ thiserror = "1.0.48"
|
|||
|
||||
zebra-node-services = { path = "../zebra-node-services", version = "1.0.0-beta.32" }
|
||||
zebra-chain = { path = "../zebra-chain", version = "1.0.0-beta.32" }
|
||||
zebra-scan = { path = "../zebra-scan", version = "0.1.0-alpha.1", optional = true }
|
||||
|
||||
# These crates are needed for the block-template-to-proposal binary
|
||||
zebra-rpc = { path = "../zebra-rpc", version = "1.0.0-beta.32", optional = true }
|
||||
|
|
@ -90,3 +104,8 @@ reqwest = { version = "0.11.22", default-features = false, features = ["rustls-t
|
|||
|
||||
# These crates are needed for the zebra-checkpoints and search-issue-refs binaries
|
||||
tokio = { version = "1.34.0", features = ["full"], optional = true }
|
||||
|
||||
jsonrpc = { version = "0.16.0", optional = true }
|
||||
|
||||
zcash_primitives = { version = "0.13.0-rc.1", optional = true }
|
||||
zcash_client_backend = {version = "0.10.0-rc.1", optional = true}
|
||||
|
|
|
|||
|
|
@ -1,17 +1,16 @@
|
|||
# Zebra Utilities
|
||||
|
||||
This crate contains tools for zebra maintainers.
|
||||
|
||||
## Programs
|
||||
Tools for maintaining and testing Zebra:
|
||||
|
||||
- [zebra-checkpoints](#zebra-checkpoints)
|
||||
- [zebrad-hash-lookup](#zebrad-hash-lookup)
|
||||
- [zebrad-log-filter](#zebrad-log-filter)
|
||||
- [zcash-rpc-diff](#zcash-rpc-diff)
|
||||
- [scanning-results-reader](#scanning-results-reader)
|
||||
|
||||
Binaries are easier to use if they are located in your system execution path.
|
||||
|
||||
### zebra-checkpoints
|
||||
## zebra-checkpoints
|
||||
|
||||
This command generates a list of zebra checkpoints, and writes them to standard output. Each checkpoint consists of a block height and hash.
|
||||
|
||||
|
|
@ -93,7 +92,7 @@ Then use the commands above to regenerate the checkpoints.
|
|||
- Open a pull request with the updated Mainnet and Testnet lists at:
|
||||
https://github.com/ZcashFoundation/zebra/pulls
|
||||
|
||||
### zebrad-hash-lookup
|
||||
## zebrad-hash-lookup
|
||||
|
||||
Given a block hash the script will get additional information using `zcash-cli`.
|
||||
|
||||
|
|
@ -108,7 +107,7 @@ $
|
|||
```
|
||||
This program is commonly used as part of `zebrad-log-filter` where hashes will be captured from `zebrad` output.
|
||||
|
||||
### zebrad-log-filter
|
||||
## zebrad-log-filter
|
||||
|
||||
The program is designed to filter the output from the zebra terminal or log file. Each time a hash is seen the script will capture it and get the additional information using `zebrad-hash-lookup`.
|
||||
|
||||
|
|
@ -127,7 +126,7 @@ next: 00000001436277884eef900772f0fcec9566becccebaab4713fd665b60fab309
|
|||
...
|
||||
```
|
||||
|
||||
### zcash-rpc-diff
|
||||
## zcash-rpc-diff
|
||||
|
||||
This program compares `zebrad` and `zcashd` RPC responses.
|
||||
|
||||
|
|
@ -188,3 +187,52 @@ You can override the binaries the script calls using these environmental variabl
|
|||
- `$ZCASH_CLI`
|
||||
- `$DIFF`
|
||||
- `$JQ`
|
||||
|
||||
## Scanning Results Reader
|
||||
|
||||
A utility for displaying Zebra's scanning results.
|
||||
|
||||
### How It Works
|
||||
|
||||
1. Opens Zebra's scanning storage and reads the results containing scanning keys
|
||||
and TXIDs.
|
||||
2. Fetches the transactions by their TXIDs from Zebra using the
|
||||
`getrawtransaction` RPC.
|
||||
3. Decrypts the tx outputs using the corresponding scanning key.
|
||||
4. Prints the memos in the outputs.
|
||||
|
||||
### How to Try It
|
||||
|
||||
#### Scan the Block Chain with Zebra
|
||||
|
||||
1. Add a viewing key to your Zebra config file. For example:
|
||||
|
||||
``` toml
|
||||
[shielded_scan.sapling_keys_to_scan]
|
||||
"zxviews1q0duytgcqqqqpqre26wkl45gvwwwd706xw608hucmvfalr759ejwf7qshjf5r9aa7323zulvz6plhttp5mltqcgs9t039cx2d09mgq05ts63n8u35hyv6h9nc9ctqqtue2u7cer2mqegunuulq2luhq3ywjcz35yyljewa4mgkgjzyfwh6fr6jd0dzd44ghk0nxdv2hnv4j5nxfwv24rwdmgllhe0p8568sgqt9ckt02v2kxf5ahtql6s0ltjpkckw8gtymxtxuu9gcr0swvz" = 1
|
||||
```
|
||||
This key is from [ZECpages](https://zecpages.com/boardinfo).
|
||||
|
||||
2. Make sure Zebra runs on Mainnet and listens on the default RPC port by having
|
||||
the following in the same config file:
|
||||
|
||||
``` toml
|
||||
[network]
|
||||
network = 'Mainnet'
|
||||
|
||||
[rpc]
|
||||
listen_addr = "127.0.0.1:8232"
|
||||
```
|
||||
|
||||
3. Compile and run Zebra with `--features "shielded-scan"` and your config file.
|
||||
Zebra will start scanning the block chain and inform you about its progress
|
||||
each 10 000 blocks in the log.
|
||||
|
||||
#### Run the Reader
|
||||
|
||||
4. To print the memos in outputs decryptable by the provided scanning key, run
|
||||
the reader while also running Zebra. For example:
|
||||
|
||||
``` bash
|
||||
cargo run --release --features shielded-scan --bin scanning-results-reader
|
||||
```
|
||||
|
|
|
|||
|
|
@ -0,0 +1,118 @@
|
|||
//! Displays Zebra's scanning results:
|
||||
//!
|
||||
//! 1. Opens Zebra's scanning storage and reads the results containing scanning keys and TXIDs.
|
||||
//! 2. Fetches the transactions by their TXIDs from Zebra using the `getrawtransaction` RPC.
|
||||
//! 3. Decrypts the tx outputs using the corresponding scanning key.
|
||||
//! 4. Prints the memos in the outputs.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use hex::ToHex;
|
||||
use itertools::Itertools;
|
||||
use jsonrpc::simple_http::SimpleHttpTransport;
|
||||
use jsonrpc::Client;
|
||||
use serde_json::value::RawValue;
|
||||
|
||||
use zcash_client_backend::decrypt_transaction;
|
||||
use zcash_client_backend::keys::UnifiedFullViewingKey;
|
||||
use zcash_primitives::consensus::{BlockHeight, BranchId};
|
||||
use zcash_primitives::transaction::Transaction;
|
||||
use zcash_primitives::zip32::AccountId;
|
||||
|
||||
use zebra_scan::scan::sapling_key_to_scan_block_keys;
|
||||
use zebra_scan::{storage::Storage, Config};
|
||||
|
||||
/// Prints the memos of transactions from Zebra's scanning results storage.
|
||||
///
|
||||
/// Reads the results storage, iterates through all decrypted memos, and prints the them to standard
|
||||
/// output. Filters out some frequent and uninteresting memos typically associated with ZECPages.
|
||||
///
|
||||
/// Notes:
|
||||
///
|
||||
/// - `#[allow(clippy::print_stdout)]` is set to allow usage of `println!` for displaying the memos.
|
||||
/// - This function expects Zebra's RPC server to be available.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// When:
|
||||
///
|
||||
/// - The Sapling key from the storage is not valid.
|
||||
/// - There is no diversifiable full viewing key (dfvk) available.
|
||||
/// - The RPC response cannot be decoded from a hex string to bytes.
|
||||
/// - The transaction fetched via RPC cannot be deserialized from raw bytes.
|
||||
#[allow(clippy::print_stdout)]
|
||||
pub fn main() {
|
||||
let network = zcash_primitives::consensus::Network::MainNetwork;
|
||||
let storage = Storage::new(&Config::default(), network.into(), true);
|
||||
// If the first memo is empty, it doesn't get printed. But we never print empty memos anyway.
|
||||
let mut prev_memo = "".to_owned();
|
||||
|
||||
for (key, _) in storage.sapling_keys_last_heights().iter() {
|
||||
let dfvk = sapling_key_to_scan_block_keys(key, network.into())
|
||||
.expect("Scanning key from the storage should be valid")
|
||||
.0
|
||||
.into_iter()
|
||||
.exactly_one()
|
||||
.expect("There should be exactly one dfvk");
|
||||
|
||||
let ufvk_with_acc_id = HashMap::from([(
|
||||
AccountId::from(1),
|
||||
UnifiedFullViewingKey::new(Some(dfvk), None).expect("`dfvk` should be `Some`"),
|
||||
)]);
|
||||
|
||||
for (height, txids) in storage.sapling_results(key) {
|
||||
let height = BlockHeight::from(height);
|
||||
|
||||
for txid in txids.iter() {
|
||||
let tx = Transaction::read(
|
||||
&hex::decode(&get_tx_via_rpc(txid.encode_hex()))
|
||||
.expect("RPC response should be decodable from hex string to bytes")[..],
|
||||
BranchId::for_height(&network, height),
|
||||
)
|
||||
.expect("TX fetched via RPC should be deserializable from raw bytes");
|
||||
|
||||
for output in decrypt_transaction(&network, height, &tx, &ufvk_with_acc_id) {
|
||||
let memo = memo_bytes_to_string(output.memo.as_array());
|
||||
|
||||
if !memo.is_empty()
|
||||
// Filter out some uninteresting and repeating memos from ZECPages.
|
||||
&& !memo.contains("LIKE:")
|
||||
&& !memo.contains("VOTE:")
|
||||
&& memo != prev_memo
|
||||
{
|
||||
println!("{memo}\n");
|
||||
prev_memo = memo;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Trims trailing zeroes from a memo, and returns the memo as a [`String`].
|
||||
fn memo_bytes_to_string(memo: &[u8; 512]) -> String {
|
||||
match memo.iter().rposition(|&byte| byte != 0) {
|
||||
Some(i) => String::from_utf8_lossy(&memo[..=i]).into_owned(),
|
||||
None => "".to_owned(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Uses the `getrawtransaction` RPC to retrieve a transaction by its TXID.
|
||||
fn get_tx_via_rpc(txid: String) -> String {
|
||||
// Wrap the TXID with `"` so that [`RawValue::from_string`] eats it.
|
||||
let txid = format!("\"{}\"", txid);
|
||||
let transport = SimpleHttpTransport::builder()
|
||||
.url("127.0.0.1:8232")
|
||||
.expect("URL should be valid")
|
||||
.build();
|
||||
let client = Client::with_transport(transport);
|
||||
let params = [RawValue::from_string(txid).expect("Provided TXID should be a valid JSON")];
|
||||
let request = client.build_request("getrawtransaction", ¶ms);
|
||||
let response = client
|
||||
.send_request(request)
|
||||
.expect("Sending the `getrawtransaction` request should succeed");
|
||||
|
||||
response
|
||||
.result()
|
||||
.expect("Zebra's RPC response should contain a valid result")
|
||||
}
|
||||
Loading…
Reference in New Issue