feat(script): Add binary for finding references to closed issues (#6347)
* adds search-todos to zebra-utils * updates example output * updates example usage. * search for full issue url refs too * renames search-todos to search-issue-refs * Makes url in doc comment a hyperlink * fixes spelling * adds tokio to run a bit quicker * Update zebra-utils/src/bin/search-issue-refs/main.rs * replaces `if let .. else` with `let .. else` * address suggestions from PR review * removes println from debugging and adds comment * updates column in issue refs * adds location of references triggering github api calls that fail * Applies suggestions from code review * Add remote file refs * Update example output * Update zebra-utils/src/bin/search-issue-refs/main.rs Co-authored-by: Alfredo Garcia <oxarbitrage@gmail.com> --------- Co-authored-by: Alfredo Garcia <oxarbitrage@gmail.com>
This commit is contained in:
Arya
GitHub
parent
d104a25688
commit
aa1e4077a3
|
|
@ -5836,10 +5836,13 @@ version = "1.0.0-beta.22"
|
|||
dependencies = [
|
||||
"color-eyre",
|
||||
"hex",
|
||||
"regex",
|
||||
"reqwest",
|
||||
"serde_json",
|
||||
"structopt",
|
||||
"thiserror",
|
||||
"tinyvec",
|
||||
"tokio",
|
||||
"tracing-error",
|
||||
"tracing-subscriber 0.3.16",
|
||||
"zebra-chain",
|
||||
|
|
|
|||
|
|
@ -16,9 +16,16 @@ name = "block-template-to-proposal"
|
|||
path = "src/bin/block-template-to-proposal/main.rs"
|
||||
required-features = ["getblocktemplate-rpcs"]
|
||||
|
||||
[[bin]]
|
||||
name = "search-issue-refs"
|
||||
path = "src/bin/search-issue-refs/main.rs"
|
||||
required-features = ["search-issue-refs"]
|
||||
|
||||
[features]
|
||||
default = []
|
||||
|
||||
search-issue-refs = ["regex", "reqwest", "tokio"]
|
||||
|
||||
# Production features that activate extra dependencies, or extra features in dependencies
|
||||
|
||||
# Experimental mining RPC support
|
||||
|
|
@ -41,6 +48,11 @@ tracing-error = "0.2.0"
|
|||
tracing-subscriber = "0.3.16"
|
||||
thiserror = "1.0.40"
|
||||
|
||||
# These crates are needed for the search-issue-refs binary
|
||||
regex = { version = "1.7.1", optional = true }
|
||||
reqwest = { version = "0.11.14", optional = true }
|
||||
tokio = { version = "1.26.0", features = ["full"], optional = true }
|
||||
|
||||
zebra-node-services = { path = "../zebra-node-services" }
|
||||
zebra-chain = { path = "../zebra-chain" }
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,265 @@
|
|||
//! Recursively searches local directory for references to issues that are closed.
|
||||
//!
|
||||
//! Requires a Github access token as this program will make queries to the GitHub API where authentication is needed.
|
||||
//!
|
||||
//! See <https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token>
|
||||
//!
|
||||
//! Example usage:
|
||||
//!
|
||||
//! (from the root directory of the Zebra repo)
|
||||
//! ```console
|
||||
//! GITHUB_TOKEN={valid_github_access_token} search-issue-refs
|
||||
//! ```
|
||||
//!
|
||||
//! Example output:
|
||||
//!
|
||||
//! > Found 3 possible issue refs, checking Github issue statuses..
|
||||
//! >
|
||||
//! > --------------------------------------
|
||||
//! > Found reference to closed issue #4794: ./zebra-rpc/src/methods/get_block_template_rpcs.rs:114:19
|
||||
//! > <https://github.com/ZcashFoundation/zebra/blob/main/zebra-rpc/src/methods/get_block_template_rpcs.rs#L114>
|
||||
//! > <https://github.com/ZcashFoundation/zebra/issues/4794>
|
||||
//! >
|
||||
//! > --------------------------------------
|
||||
//! > Found reference to closed issue #2379: ./zebra-consensus/src/transaction.rs:717:49
|
||||
//! > <https://github.com/ZcashFoundation/zebra/blob/main/zebra-consensus/src/transaction.rs#L717>
|
||||
//! > <https://github.com/ZcashFoundation/zebra/issues/2379>
|
||||
//! >
|
||||
//! > --------------------------------------
|
||||
//! > Found reference to closed issue #3027: ./zebra-consensus/src/transaction/check.rs:319:6
|
||||
//! > <https://github.com/ZcashFoundation/zebra/blob/main/zebra-consensus/src/transaction/check.rs#L319>
|
||||
//! > <https://github.com/ZcashFoundation/zebra/issues/3027>
|
||||
//! >
|
||||
//! > Found 3 references to closed issues.
|
||||
|
||||
use std::{
|
||||
env,
|
||||
ffi::OsStr,
|
||||
fs::{self, File},
|
||||
io::{self, BufRead},
|
||||
path::PathBuf,
|
||||
};
|
||||
|
||||
use color_eyre::eyre::Result;
|
||||
use regex::Regex;
|
||||
use reqwest::{
|
||||
header::{self, HeaderMap, HeaderValue},
|
||||
ClientBuilder,
|
||||
};
|
||||
|
||||
use tokio::task::JoinSet;
|
||||
use zebra_utils::init_tracing;
|
||||
|
||||
const GITHUB_TOKEN_ENV_KEY: &str = "GITHUB_TOKEN";
|
||||
|
||||
const VALID_EXTENSIONS: [&str; 4] = ["rs", "yml", "yaml", "toml"];
|
||||
|
||||
fn check_file_ext(ext: &OsStr) -> bool {
|
||||
VALID_EXTENSIONS
|
||||
.into_iter()
|
||||
.any(|valid_extension| valid_extension == ext)
|
||||
}
|
||||
|
||||
fn search_directory(path: &PathBuf) -> Result<Vec<PathBuf>> {
|
||||
if path.starts_with("/target/") {
|
||||
return Ok(vec![]);
|
||||
}
|
||||
|
||||
Ok(fs::read_dir(path)?
|
||||
.filter_map(|entry| {
|
||||
let path = entry.ok()?.path();
|
||||
|
||||
if path.is_dir() {
|
||||
search_directory(&path).ok()
|
||||
} else if path.is_file() {
|
||||
match path.extension() {
|
||||
Some(ext) if check_file_ext(ext) => Some(vec![path]),
|
||||
_ => None,
|
||||
}
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.flatten()
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn github_issue_url(issue_id: &str) -> String {
|
||||
format!("https://github.com/ZcashFoundation/zebra/issues/{issue_id}")
|
||||
}
|
||||
|
||||
fn github_remote_file_ref(file_path: &str, line: usize) -> String {
|
||||
let file_path = &file_path[2..];
|
||||
format!("https://github.com/ZcashFoundation/zebra/blob/main/{file_path}#L{line}")
|
||||
}
|
||||
|
||||
fn github_issue_api_url(issue_id: &str) -> String {
|
||||
format!("https://api.github.com/repos/ZcashFoundation/zebra/issues/{issue_id}")
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct PossibleIssueRef {
|
||||
file_path: String,
|
||||
line_number: usize,
|
||||
column: usize,
|
||||
id: String,
|
||||
}
|
||||
|
||||
/// Process entry point for `search-issue-refs`
|
||||
#[allow(clippy::print_stdout, clippy::print_stderr)]
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
init_tracing();
|
||||
color_eyre::install()?;
|
||||
|
||||
let file_paths = search_directory(&".".into())?;
|
||||
|
||||
// Zebra's github issue numbers could be up to 4 digits
|
||||
let issue_regex =
|
||||
Regex::new(r"(https://github.com/ZcashFoundation/zebra/issues/|#)(\d{1,4})").unwrap();
|
||||
|
||||
let mut possible_issue_refs: Vec<PossibleIssueRef> = vec![];
|
||||
|
||||
for file_path in file_paths {
|
||||
let file = File::open(&file_path)?;
|
||||
let lines = io::BufReader::new(file).lines();
|
||||
|
||||
for (line_idx, line) in lines.into_iter().enumerate() {
|
||||
let line = line?;
|
||||
|
||||
possible_issue_refs.extend(issue_regex.captures_iter(&line).map(|captures| {
|
||||
let file_path = file_path
|
||||
.to_str()
|
||||
.expect("paths from read_dir should be valid unicode")
|
||||
.to_string();
|
||||
|
||||
let potential_issue_ref = captures.get(2).expect("matches should have 2 captures");
|
||||
let matching_text = potential_issue_ref.as_str();
|
||||
|
||||
let id =
|
||||
matching_text[matching_text.len().checked_sub(4).unwrap_or(1)..].to_string();
|
||||
|
||||
PossibleIssueRef {
|
||||
file_path,
|
||||
line_number: line_idx + 1,
|
||||
column: captures
|
||||
.get(1)
|
||||
.expect("matches should have 2 captures")
|
||||
.start()
|
||||
+ 1,
|
||||
id,
|
||||
}
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
let num_possible_issue_refs = possible_issue_refs.len();
|
||||
|
||||
println!(
|
||||
"\nFound {num_possible_issue_refs} possible issue refs, checking Github issue statuses..\n"
|
||||
);
|
||||
|
||||
// check if issues are closed on Github
|
||||
|
||||
let github_token = match env::vars().find(|(key, _)| key == GITHUB_TOKEN_ENV_KEY) {
|
||||
Some((_, github_token)) => github_token,
|
||||
_ => {
|
||||
println!(
|
||||
"Can't find {GITHUB_TOKEN_ENV_KEY} in env vars, printing all found possible issue refs, \
|
||||
see https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token \
|
||||
to create a github token."
|
||||
);
|
||||
|
||||
for PossibleIssueRef {
|
||||
file_path,
|
||||
line_number,
|
||||
column,
|
||||
id,
|
||||
} in possible_issue_refs
|
||||
{
|
||||
let github_url = github_issue_url(&id);
|
||||
let github_file_ref = github_remote_file_ref(&file_path, line_number);
|
||||
|
||||
println!("\n--------------------------------------");
|
||||
println!("Found possible reference to closed issue #{id}: {file_path}:{line_number}:{column}");
|
||||
println!("{github_file_ref}");
|
||||
println!("{github_url}");
|
||||
}
|
||||
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
|
||||
let mut headers = HeaderMap::new();
|
||||
let mut auth_value = HeaderValue::from_str(&format!("Bearer {github_token}"))?;
|
||||
let accept_value = HeaderValue::from_static("application/vnd.github+json");
|
||||
let github_api_version_value = HeaderValue::from_static("2022-11-28");
|
||||
let user_agent_value = HeaderValue::from_static("search-issue-refs");
|
||||
|
||||
auth_value.set_sensitive(true);
|
||||
|
||||
headers.insert(header::AUTHORIZATION, auth_value);
|
||||
headers.insert(header::ACCEPT, accept_value);
|
||||
headers.insert("X-GitHub-Api-Version", github_api_version_value);
|
||||
headers.insert(header::USER_AGENT, user_agent_value);
|
||||
|
||||
let client = ClientBuilder::new().default_headers(headers).build()?;
|
||||
|
||||
let mut github_api_requests = JoinSet::new();
|
||||
let mut num_possible_issue_refs = 0;
|
||||
|
||||
for possible_issue_ref in possible_issue_refs {
|
||||
let request = client
|
||||
.get(github_issue_api_url(&possible_issue_ref.id))
|
||||
.send();
|
||||
github_api_requests.spawn(async move { (request.await, possible_issue_ref) });
|
||||
}
|
||||
|
||||
while let Some(res) = github_api_requests.join_next().await {
|
||||
let Ok((
|
||||
res,
|
||||
PossibleIssueRef {
|
||||
file_path,
|
||||
line_number,
|
||||
column,
|
||||
id,
|
||||
},
|
||||
)) = res else {
|
||||
println!("warning: failed to join api request thread/task");
|
||||
continue;
|
||||
};
|
||||
|
||||
let Ok(res) = res else {
|
||||
println!("warning: no response from github api about issue #{id}, {file_path}:{line_number}:{column}");
|
||||
continue;
|
||||
};
|
||||
|
||||
let Ok(text) = res.text().await else {
|
||||
println!("warning: failed to get text from response about issue #{id}, {file_path}:{line_number}:{column}");
|
||||
continue;
|
||||
};
|
||||
|
||||
let Ok(json): Result<serde_json::Value, _> = serde_json::from_str(&text) else {
|
||||
println!("warning: failed to get serde_json::Value from response for issue #{id}, {file_path}:{line_number}:{column}");
|
||||
continue;
|
||||
};
|
||||
|
||||
if json["closed_at"] == serde_json::Value::Null {
|
||||
continue;
|
||||
};
|
||||
|
||||
num_possible_issue_refs += 1;
|
||||
|
||||
let github_url = github_issue_url(&id);
|
||||
let github_file_ref = github_remote_file_ref(&file_path, line_number);
|
||||
|
||||
println!("\n--------------------------------------");
|
||||
println!("Found reference to closed issue #{id}: {file_path}:{line_number}:{column}");
|
||||
println!("{github_file_ref}");
|
||||
println!("{github_url}");
|
||||
}
|
||||
|
||||
println!("\nFound {num_possible_issue_refs} references to closed issues.\n");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
Loading…
Reference in New Issue