From 8d951776abfc7fc919ab40fa6bececf5ae862128 Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Fri, 3 Jul 2020 01:46:29 -0600 Subject: [PATCH] Rpc: add filter to getProgramAccounts (#10888) * Add RpcFilterType, and implement CompareBytes for getProgramAccounts * Accept bytes in bs58 * Rename to memcmp * Add Memcmp optional encoding field * Add dataSize filter * Update docs * Clippy * Simplify tests that don't need to test account contents; add multiple-filter tests --- client/src/lib.rs | 1 + client/src/rpc_config.rs | 9 ++ client/src/rpc_filter.rs | 143 +++++++++++++++++++++++ core/src/rpc.rs | 219 +++++++++++++++++++++++++++++++++-- core/tests/rpc.rs | 4 +- docs/src/apps/jsonrpc-api.md | 14 +++ 6 files changed, 378 insertions(+), 12 deletions(-) create mode 100644 client/src/rpc_filter.rs diff --git a/client/src/lib.rs b/client/src/lib.rs index 205058a92..6a4bce877 100644 --- a/client/src/lib.rs +++ b/client/src/lib.rs @@ -8,6 +8,7 @@ pub mod perf_utils; pub mod pubsub_client; pub mod rpc_client; pub mod rpc_config; +pub mod rpc_filter; pub mod rpc_request; pub mod rpc_response; pub mod rpc_sender; diff --git a/client/src/rpc_config.rs b/client/src/rpc_config.rs index 91220cbc5..091e9921e 100644 --- a/client/src/rpc_config.rs +++ b/client/src/rpc_config.rs @@ -1,3 +1,4 @@ +use crate::rpc_filter::RpcFilterType; use solana_account_decoder::UiAccountEncoding; use solana_sdk::{clock::Epoch, commitment_config::CommitmentConfig}; @@ -49,3 +50,11 @@ pub struct RpcAccountInfoConfig { #[serde(flatten)] pub commitment: Option, } + +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RpcProgramAccountsConfig { + pub filters: Option>, + #[serde(flatten)] + pub account_config: RpcAccountInfoConfig, +} diff --git a/client/src/rpc_filter.rs b/client/src/rpc_filter.rs new file mode 100644 index 000000000..416742d59 --- /dev/null +++ b/client/src/rpc_filter.rs @@ -0,0 +1,143 @@ +use thiserror::Error; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum RpcFilterType { + DataSize(u64), + Memcmp(Memcmp), +} + +impl RpcFilterType { + pub fn verify(&self) -> Result<(), RpcFilterError> { + match self { + RpcFilterType::DataSize(_) => Ok(()), + RpcFilterType::Memcmp(compare) => { + let encoding = compare.encoding.as_ref().unwrap_or(&MemcmpEncoding::Binary); + match encoding { + MemcmpEncoding::Binary => { + let MemcmpEncodedBytes::Binary(bytes) = &compare.bytes; + bs58::decode(&bytes) + .into_vec() + .map(|_| ()) + .map_err(|e| e.into()) + } + } + } + } + } +} + +#[derive(Error, Debug)] +pub enum RpcFilterError { + #[error("bs58 decode error")] + DecodeError(#[from] bs58::decode::Error), +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum MemcmpEncoding { + Binary, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", untagged)] +pub enum MemcmpEncodedBytes { + Binary(String), +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Memcmp { + /// Data offset to begin match + pub offset: usize, + /// Bytes, encoded with specified encoding, or default Binary + pub bytes: MemcmpEncodedBytes, + /// Optional encoding specification + pub encoding: Option, +} + +impl Memcmp { + pub fn bytes_match(&self, data: &[u8]) -> bool { + match &self.bytes { + MemcmpEncodedBytes::Binary(bytes) => { + let bytes = bs58::decode(bytes).into_vec(); + if bytes.is_err() { + return false; + } + let bytes = bytes.unwrap(); + if self.offset > data.len() { + return false; + } + if data[self.offset..].len() < bytes.len() { + return false; + } + data[self.offset..self.offset + bytes.len()] == bytes[..] + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_bytes_match() { + let data = vec![1, 2, 3, 4, 5]; + + // Exact match of data succeeds + assert!(Memcmp { + offset: 0, + bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![1, 2, 3, 4, 5]).into_string()), + encoding: None, + } + .bytes_match(&data)); + + // Partial match of data succeeds + assert!(Memcmp { + offset: 0, + bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![1, 2]).into_string()), + encoding: None, + } + .bytes_match(&data)); + + // Offset partial match of data succeeds + assert!(Memcmp { + offset: 2, + bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![3, 4]).into_string()), + encoding: None, + } + .bytes_match(&data)); + + // Incorrect partial match of data fails + assert!(!Memcmp { + offset: 0, + bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![2]).into_string()), + encoding: None, + } + .bytes_match(&data)); + + // Bytes overrun data fails + assert!(!Memcmp { + offset: 2, + bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![3, 4, 5, 6]).into_string()), + encoding: None, + } + .bytes_match(&data)); + + // Offset outside data fails + assert!(!Memcmp { + offset: 6, + bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![5]).into_string()), + encoding: None, + } + .bytes_match(&data)); + + // Invalid base-58 fails + assert!(!Memcmp { + offset: 0, + bytes: MemcmpEncodedBytes::Binary("III".to_string()), + encoding: None, + } + .bytes_match(&data)); + } +} diff --git a/core/src/rpc.rs b/core/src/rpc.rs index 04a7e0535..0a55e0ee0 100644 --- a/core/src/rpc.rs +++ b/core/src/rpc.rs @@ -11,6 +11,7 @@ use jsonrpc_derive::rpc; use solana_account_decoder::{UiAccount, UiAccountEncoding}; use solana_client::{ rpc_config::*, + rpc_filter::RpcFilterType, rpc_request::{ DELINQUENT_VALIDATOR_SLOT_DISTANCE, MAX_GET_CONFIRMED_SIGNATURES_FOR_ADDRESS_SLOT_RANGE, MAX_GET_SIGNATURE_STATUSES_QUERY_ITEMS, NUM_LARGEST_ACCOUNTS, @@ -232,6 +233,7 @@ impl JsonRpcRequestProcessor { &self, program_id: &Pubkey, config: Option, + filters: Vec, ) -> Result> { let config = config.unwrap_or_default(); let bank = self.bank(config.commitment)?; @@ -239,6 +241,12 @@ impl JsonRpcRequestProcessor { Ok(bank .get_program_accounts(Some(&program_id)) .into_iter() + .filter(|(_, account)| { + filters.iter().all(|filter_type| match filter_type { + RpcFilterType::DataSize(size) => account.data.len() as u64 == *size, + RpcFilterType::Memcmp(compare) => compare.bytes_match(&account.data), + }) + }) .map(|(pubkey, account)| RpcKeyedAccount { pubkey: pubkey.to_string(), account: UiAccount::encode(account, encoding.clone()), @@ -760,16 +768,22 @@ impl JsonRpcRequestProcessor { } } +fn verify_filter(input: &RpcFilterType) -> Result<()> { + input + .verify() + .map_err(|e| Error::invalid_params(format!("Invalid param: {:?}", e))) +} + fn verify_pubkey(input: String) -> Result { input .parse() - .map_err(|e| Error::invalid_params(format!("{:?}", e))) + .map_err(|e| Error::invalid_params(format!("Invalid param: {:?}", e))) } fn verify_signature(input: &str) -> Result { input .parse() - .map_err(|e| Error::invalid_params(format!("{:?}", e))) + .map_err(|e| Error::invalid_params(format!("Invalid param: {:?}", e))) } /// Run transactions against a frozen bank without committing the results @@ -839,7 +853,7 @@ pub trait RpcSol { &self, meta: Self::Metadata, program_id_str: String, - config: Option, + config: Option, ) -> Result>; #[rpc(meta, name = "getMinimumBalanceForRentExemption")] @@ -1104,14 +1118,25 @@ impl RpcSol for RpcSolImpl { &self, meta: Self::Metadata, program_id_str: String, - config: Option, + config: Option, ) -> Result> { debug!( "get_program_accounts rpc request received: {:?}", program_id_str ); let program_id = verify_pubkey(program_id_str)?; - meta.get_program_accounts(&program_id, config) + let (config, filters) = if let Some(config) = config { + ( + Some(config.account_config), + config.filters.unwrap_or_default(), + ) + } else { + (None, vec![]) + }; + for filter in &filters { + verify_filter(filter)?; + } + meta.get_program_accounts(&program_id, config, filters) } fn get_inflation_governor( @@ -1620,6 +1645,7 @@ pub mod tests { futures::future::Future, ErrorCode, MetaIoHandler, Output, Response, Value, }; use jsonrpc_core_client::transports::local; + use solana_client::rpc_filter::{Memcmp, MemcmpEncodedBytes}; use solana_ledger::{ blockstore::entries_to_test_shreds, blockstore_processor::fill_blockstore_slot_with_ticks, @@ -1633,9 +1659,9 @@ pub mod tests { hash::{hash, Hash}, instruction::InstructionError, message::Message, - rpc_port, + nonce, rpc_port, signature::{Keypair, Signer}, - system_transaction, + system_instruction, system_program, system_transaction, transaction::{self, TransactionError}, }; use solana_transaction_status::{EncodedTransaction, TransactionWithStatusMeta, UiMessage}; @@ -2272,6 +2298,7 @@ pub mod tests { meta, bank, blockhash, + alice, .. } = start_rpc_handler_with_tx(&bob.pubkey()); @@ -2282,7 +2309,7 @@ pub mod tests { r#"{{"jsonrpc":"2.0","id":1,"method":"getProgramAccounts","params":["{}"]}}"#, new_program_id ); - let res = io.handle_request_sync(&req, meta); + let res = io.handle_request_sync(&req, meta.clone()); let expected = format!( r#"{{ "jsonrpc":"2.0", @@ -2308,6 +2335,159 @@ pub mod tests { let result: Response = serde_json::from_str(&res.expect("actual response")) .expect("actual response deserialization"); assert_eq!(expected, result); + + // Set up nonce accounts to test filters + let nonce_keypair0 = Keypair::new(); + let instruction = system_instruction::create_nonce_account( + &alice.pubkey(), + &nonce_keypair0.pubkey(), + &bob.pubkey(), + 100_000, + ); + let message = Message::new(&instruction, Some(&alice.pubkey())); + let tx = Transaction::new(&[&alice, &nonce_keypair0], message, blockhash); + bank.process_transaction(&tx).unwrap(); + + let nonce_keypair1 = Keypair::new(); + let authority = Pubkey::new_rand(); + let instruction = system_instruction::create_nonce_account( + &alice.pubkey(), + &nonce_keypair1.pubkey(), + &authority, + 100_000, + ); + let message = Message::new(&instruction, Some(&alice.pubkey())); + let tx = Transaction::new(&[&alice, &nonce_keypair1], message, blockhash); + bank.process_transaction(&tx).unwrap(); + + // Test memcmp filter; filter on Initialized state + let req = format!( + r#"{{ + "jsonrpc":"2.0", + "id":1, + "method":"getProgramAccounts", + "params":["{}",{{"filters": [ + {{ + "memcmp": {{"offset": 4,"bytes": "{}"}} + }} + ]}}] + }}"#, + system_program::id(), + bs58::encode(vec![1]).into_string(), + ); + let res = io.handle_request_sync(&req, meta.clone()); + let json: Value = serde_json::from_str(&res.unwrap()).unwrap(); + let accounts: Vec = serde_json::from_value(json["result"].clone()) + .expect("actual response deserialization"); + assert_eq!(accounts.len(), 2); + + let req = format!( + r#"{{ + "jsonrpc":"2.0", + "id":1, + "method":"getProgramAccounts", + "params":["{}",{{"filters": [ + {{ + "memcmp": {{"offset": 0,"bytes": "{}"}} + }} + ]}}] + }}"#, + system_program::id(), + bs58::encode(vec![1]).into_string(), + ); + let res = io.handle_request_sync(&req, meta.clone()); + let json: Value = serde_json::from_str(&res.unwrap()).unwrap(); + let accounts: Vec = serde_json::from_value(json["result"].clone()) + .expect("actual response deserialization"); + assert_eq!(accounts.len(), 0); + + // Test dataSize filter + let req = format!( + r#"{{ + "jsonrpc":"2.0", + "id":1, + "method":"getProgramAccounts", + "params":["{}",{{"filters": [ + {{ + "dataSize": {} + }} + ]}}] + }}"#, + system_program::id(), + nonce::State::size(), + ); + let res = io.handle_request_sync(&req, meta.clone()); + let json: Value = serde_json::from_str(&res.unwrap()).unwrap(); + let accounts: Vec = serde_json::from_value(json["result"].clone()) + .expect("actual response deserialization"); + assert_eq!(accounts.len(), 2); + + let req = format!( + r#"{{ + "jsonrpc":"2.0", + "id":1, + "method":"getProgramAccounts", + "params":["{}",{{"filters": [ + {{ + "dataSize": 1 + }} + ]}}] + }}"#, + system_program::id(), + ); + let res = io.handle_request_sync(&req, meta.clone()); + let json: Value = serde_json::from_str(&res.unwrap()).unwrap(); + let accounts: Vec = serde_json::from_value(json["result"].clone()) + .expect("actual response deserialization"); + assert_eq!(accounts.len(), 0); + + // Test multiple filters + let req = format!( + r#"{{ + "jsonrpc":"2.0", + "id":1, + "method":"getProgramAccounts", + "params":["{}",{{"filters": [ + {{ + "memcmp": {{"offset": 4,"bytes": "{}"}} + }}, + {{ + "memcmp": {{"offset": 8,"bytes": "{}"}} + }} + ]}}] + }}"#, + system_program::id(), + bs58::encode(vec![1]).into_string(), + authority, + ); // Filter on Initialized and Nonce authority + let res = io.handle_request_sync(&req, meta.clone()); + let json: Value = serde_json::from_str(&res.unwrap()).unwrap(); + let accounts: Vec = serde_json::from_value(json["result"].clone()) + .expect("actual response deserialization"); + assert_eq!(accounts.len(), 1); + + let req = format!( + r#"{{ + "jsonrpc":"2.0", + "id":1, + "method":"getProgramAccounts", + "params":["{}",{{"filters": [ + {{ + "memcmp": {{"offset": 4,"bytes": "{}"}} + }}, + {{ + "dataSize": 1 + }} + ]}}] + }}"#, + system_program::id(), + bs58::encode(vec![1]).into_string(), + ); // Filter on Initialized and non-matching data size + let res = io.handle_request_sync(&req, meta); + let json: Value = serde_json::from_str(&res.unwrap()).unwrap(); + let accounts: Vec = serde_json::from_value(json["result"].clone()) + .expect("actual response deserialization"); + assert_eq!(accounts.len(), 0); } #[test] @@ -2887,6 +3067,25 @@ pub mod tests { ); } + #[test] + fn test_rpc_verify_filter() { + let filter = RpcFilterType::Memcmp(Memcmp { + offset: 0, + bytes: MemcmpEncodedBytes::Binary( + "13LeFbG6m2EP1fqCj9k66fcXsoTHMMtgr7c78AivUrYD".to_string(), + ), + encoding: None, + }); + assert_eq!(verify_filter(&filter), Ok(())); + // Invalid base-58 + let filter = RpcFilterType::Memcmp(Memcmp { + offset: 0, + bytes: MemcmpEncodedBytes::Binary("III".to_string()), + encoding: None, + }); + assert!(verify_filter(&filter).is_err()); + } + #[test] fn test_rpc_verify_pubkey() { let pubkey = Pubkey::new_rand(); @@ -2894,7 +3093,7 @@ pub mod tests { let bad_pubkey = "a1b2c3d4"; assert_eq!( verify_pubkey(bad_pubkey.to_string()), - Err(Error::invalid_params("WrongSize")) + Err(Error::invalid_params("Invalid param: WrongSize")) ); } @@ -2908,7 +3107,7 @@ pub mod tests { let bad_signature = "a1b2c3d4"; assert_eq!( verify_signature(&bad_signature.to_string()), - Err(Error::invalid_params("WrongSize")) + Err(Error::invalid_params("Invalid param: WrongSize")) ); } diff --git a/core/tests/rpc.rs b/core/tests/rpc.rs index e94a2e2cf..37bb955e4 100644 --- a/core/tests/rpc.rs +++ b/core/tests/rpc.rs @@ -121,14 +121,14 @@ fn test_rpc_invalid_requests() { let json = post_rpc(req, &leader_data); let the_error = json["error"]["message"].as_str().unwrap(); - assert_eq!(the_error, "Invalid"); + assert_eq!(the_error, "Invalid param: Invalid"); // test invalid get_account_info request let req = json_req!("getAccountInfo", json!(["invalid9999"])); let json = post_rpc(req, &leader_data); let the_error = json["error"]["message"].as_str().unwrap(); - assert_eq!(the_error, "Invalid"); + assert_eq!(the_error, "Invalid param: Invalid"); // test invalid get_account_info request let req = json_req!("getAccountInfo", json!([bob_pubkey.to_string()])); diff --git a/docs/src/apps/jsonrpc-api.md b/docs/src/apps/jsonrpc-api.md index 93d5a935f..c345531d0 100644 --- a/docs/src/apps/jsonrpc-api.md +++ b/docs/src/apps/jsonrpc-api.md @@ -790,6 +790,14 @@ Returns all accounts owned by the provided program Pubkey * (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment) * (optional) `encoding: ` - encoding for Account data, either "binary" or jsonParsed". If parameter not provided, the default encoding is binary. Parsed-JSON encoding attempts to use program-specific state parsers to return more human-readable and explicit account state data. If parsed-JSON is requested but a parser cannot be found, the field falls back to binary encoding, detectable when the `data` field is type ``. + * (optional) `filters: ` - filter results using various [filter objects](jsonrpc-api.md#filters); account must meet all filter criteria to be included in results + +##### Filters: +* `memcmp: ` - compares a provided series of bytes with program account data at a particular offset. Fields: + * `offset: ` - offset into program account data to start comparison + * `bytes: ` - data to match, as base-58 encoded string + +* `dataSize: ` - compares the program account data length with the provided data size #### Results: @@ -809,6 +817,12 @@ The result field will be an array of JSON objects, which will contain: // Request curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0", "id":1, "method":"getProgramAccounts", "params":["4Nd1mBQtrMJVYVfKf2PJy9NZUZdTAsp7D4xWLs4gDB4T"]}' http://localhost:8899 +// Result +{"jsonrpc":"2.0","result":[{"account":{"data":"2R9jLfiAQ9bgdcw6h8s44439","executable":false,"lamports":15298080,"owner":"4Nd1mBQtrMJVYVfKf2PJy9NZUZdTAsp7D4xWLs4gDB4T","rentEpoch":28},"pubkey":"CxELquR1gPP8wHe33gZ4QxqGB3sZ9RSwsJ2KshVewkFY"}],"id":1} + +// Request with Filters +curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0", "id":1, "method":"getProgramAccounts", "params":["4Nd1mBQtrMJVYVfKf2PJy9NZUZdTAsp7D4xWLs4gDB4T", {"filters":[{"dataSize": 17},{"memcmp": {"offset": 4, "bytes": "3Mc6vR"}}]}]}' http://localhost:8899 + // Result {"jsonrpc":"2.0","result":[{"account":{"data":"2R9jLfiAQ9bgdcw6h8s44439","executable":false,"lamports":15298080,"owner":"4Nd1mBQtrMJVYVfKf2PJy9NZUZdTAsp7D4xWLs4gDB4T","rentEpoch":28},"pubkey":"CxELquR1gPP8wHe33gZ4QxqGB3sZ9RSwsJ2KshVewkFY"}],"id":1} ```