blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Rpc: add filter to getProgramAccounts (#10888) 

* Add RpcFilterType, and implement CompareBytes for getProgramAccounts

* Accept bytes in bs58

* Rename to memcmp

* Add Memcmp optional encoding field

* Add dataSize filter

* Update docs

* Clippy

* Simplify tests that don't need to test account contents; add multiple-filter tests
This commit is contained in:
Tyera Eulberg 2020-07-03 01:46:29 -06:00 committed by GitHub
parent 832d47317e
commit 8d951776ab
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 378 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
client/src/lib.rs
View File

@ -8,6 +8,7 @@ pub mod perf_utils;
pub mod pubsub_client;
pub mod rpc_client;
pub mod rpc_config;
pub mod rpc_filter;
pub mod rpc_request;
pub mod rpc_response;
pub mod rpc_sender;

9
client/src/rpc_config.rs
View File

@ -1,3 +1,4 @@
use crate::rpc_filter::RpcFilterType;
use solana_account_decoder::UiAccountEncoding;
use solana_sdk::{clock::Epoch, commitment_config::CommitmentConfig};
@ -49,3 +50,11 @@ pub struct RpcAccountInfoConfig {
#[serde(flatten)]
pub commitment: Option<CommitmentConfig>,
}
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct RpcProgramAccountsConfig {
pub filters: Option<Vec<RpcFilterType>>,
#[serde(flatten)]
pub account_config: RpcAccountInfoConfig,
}

143
client/src/rpc_filter.rs Normal file
View File

@ -0,0 +1,143 @@
use thiserror::Error;
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub enum RpcFilterType {
DataSize(u64),
Memcmp(Memcmp),
}
impl RpcFilterType {
pub fn verify(&self) -> Result<(), RpcFilterError> {
match self {
RpcFilterType::DataSize(_) => Ok(()),
RpcFilterType::Memcmp(compare) => {
let encoding = compare.encoding.as_ref().unwrap_or(&MemcmpEncoding::Binary);
match encoding {
MemcmpEncoding::Binary => {
let MemcmpEncodedBytes::Binary(bytes) = &compare.bytes;
bs58::decode(&bytes)
.into_vec()
.map(|_| ())
.map_err(|e| e.into())
}
}
}
}
}
}
#[derive(Error, Debug)]
pub enum RpcFilterError {
#[error("bs58 decode error")]
DecodeError(#[from] bs58::decode::Error),
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub enum MemcmpEncoding {
Binary,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", untagged)]
pub enum MemcmpEncodedBytes {
Binary(String),
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Memcmp {
/// Data offset to begin match
pub offset: usize,
/// Bytes, encoded with specified encoding, or default Binary
pub bytes: MemcmpEncodedBytes,
/// Optional encoding specification
pub encoding: Option<MemcmpEncoding>,
}
impl Memcmp {
pub fn bytes_match(&self, data: &[u8]) -> bool {
match &self.bytes {
MemcmpEncodedBytes::Binary(bytes) => {
let bytes = bs58::decode(bytes).into_vec();
if bytes.is_err() {
return false;
}
let bytes = bytes.unwrap();
if self.offset > data.len() {
return false;
}
if data[self.offset..].len() < bytes.len() {
return false;
}
data[self.offset..self.offset + bytes.len()] == bytes[..]
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_bytes_match() {
let data = vec![1, 2, 3, 4, 5];
// Exact match of data succeeds
assert!(Memcmp {
offset: 0,
bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![1, 2, 3, 4, 5]).into_string()),
encoding: None,
}
.bytes_match(&data));
// Partial match of data succeeds
assert!(Memcmp {
offset: 0,
bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![1, 2]).into_string()),
encoding: None,
}
.bytes_match(&data));
// Offset partial match of data succeeds
assert!(Memcmp {
offset: 2,
bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![3, 4]).into_string()),
encoding: None,
}
.bytes_match(&data));
// Incorrect partial match of data fails
assert!(!Memcmp {
offset: 0,
bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![2]).into_string()),
encoding: None,
}
.bytes_match(&data));
// Bytes overrun data fails
assert!(!Memcmp {
offset: 2,
bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![3, 4, 5, 6]).into_string()),
encoding: None,
}
.bytes_match(&data));
// Offset outside data fails
assert!(!Memcmp {
offset: 6,
bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![5]).into_string()),
encoding: None,
}
.bytes_match(&data));
// Invalid base-58 fails
assert!(!Memcmp {
offset: 0,
bytes: MemcmpEncodedBytes::Binary("III".to_string()),
encoding: None,
}
.bytes_match(&data));
}
}

219
core/src/rpc.rs
View File

@ -11,6 +11,7 @@ use jsonrpc_derive::rpc;
use solana_account_decoder::{UiAccount, UiAccountEncoding};
use solana_client::{
rpc_config::*,
rpc_filter::RpcFilterType,
rpc_request::{
DELINQUENT_VALIDATOR_SLOT_DISTANCE, MAX_GET_CONFIRMED_SIGNATURES_FOR_ADDRESS_SLOT_RANGE,
MAX_GET_SIGNATURE_STATUSES_QUERY_ITEMS, NUM_LARGEST_ACCOUNTS,
@ -232,6 +233,7 @@ impl JsonRpcRequestProcessor {
&self,
program_id: &Pubkey,
config: Option<RpcAccountInfoConfig>,
filters: Vec<RpcFilterType>,
) -> Result<Vec<RpcKeyedAccount>> {
let config = config.unwrap_or_default();
let bank = self.bank(config.commitment)?;
@ -239,6 +241,12 @@ impl JsonRpcRequestProcessor {
Ok(bank
.get_program_accounts(Some(&program_id))
.into_iter()
.filter(|(_, account)| {
filters.iter().all(|filter_type| match filter_type {
RpcFilterType::DataSize(size) => account.data.len() as u64 == *size,
RpcFilterType::Memcmp(compare) => compare.bytes_match(&account.data),
})
})
.map(|(pubkey, account)| RpcKeyedAccount {
pubkey: pubkey.to_string(),
account: UiAccount::encode(account, encoding.clone()),
@ -760,16 +768,22 @@ impl JsonRpcRequestProcessor {
}
}
fn verify_filter(input: &RpcFilterType) -> Result<()> {
input
.verify()
.map_err(|e| Error::invalid_params(format!("Invalid param: {:?}", e)))
}
fn verify_pubkey(input: String) -> Result<Pubkey> {
input
.parse()
.map_err(|e| Error::invalid_params(format!("{:?}", e)))
.map_err(|e| Error::invalid_params(format!("Invalid param: {:?}", e)))
}
fn verify_signature(input: &str) -> Result<Signature> {
input
.parse()
.map_err(|e| Error::invalid_params(format!("{:?}", e)))
.map_err(|e| Error::invalid_params(format!("Invalid param: {:?}", e)))
}
/// Run transactions against a frozen bank without committing the results
@ -839,7 +853,7 @@ pub trait RpcSol {
&self,
meta: Self::Metadata,
program_id_str: String,
config: Option<RpcAccountInfoConfig>,
config: Option<RpcProgramAccountsConfig>,
) -> Result<Vec<RpcKeyedAccount>>;
#[rpc(meta, name = "getMinimumBalanceForRentExemption")]
@ -1104,14 +1118,25 @@ impl RpcSol for RpcSolImpl {
&self,
meta: Self::Metadata,
program_id_str: String,
config: Option<RpcAccountInfoConfig>,
config: Option<RpcProgramAccountsConfig>,
) -> Result<Vec<RpcKeyedAccount>> {
debug!(
"get_program_accounts rpc request received: {:?}",
program_id_str
);
let program_id = verify_pubkey(program_id_str)?;
meta.get_program_accounts(&program_id, config)
let (config, filters) = if let Some(config) = config {
(
Some(config.account_config),
config.filters.unwrap_or_default(),
)
} else {
(None, vec![])
};
for filter in &filters {
verify_filter(filter)?;
}
meta.get_program_accounts(&program_id, config, filters)
}
fn get_inflation_governor(
@ -1620,6 +1645,7 @@ pub mod tests {
futures::future::Future, ErrorCode, MetaIoHandler, Output, Response, Value,
};
use jsonrpc_core_client::transports::local;
use solana_client::rpc_filter::{Memcmp, MemcmpEncodedBytes};
use solana_ledger::{
blockstore::entries_to_test_shreds,
blockstore_processor::fill_blockstore_slot_with_ticks,
@ -1633,9 +1659,9 @@ pub mod tests {
hash::{hash, Hash},
instruction::InstructionError,
message::Message,
rpc_port,
nonce, rpc_port,
signature::{Keypair, Signer},
system_transaction,
system_instruction, system_program, system_transaction,
transaction::{self, TransactionError},
};
use solana_transaction_status::{EncodedTransaction, TransactionWithStatusMeta, UiMessage};
@ -2272,6 +2298,7 @@ pub mod tests {
meta,
bank,
blockhash,
alice,
..
} = start_rpc_handler_with_tx(&bob.pubkey());
@ -2282,7 +2309,7 @@ pub mod tests {
r#"{{"jsonrpc":"2.0","id":1,"method":"getProgramAccounts","params":["{}"]}}"#,
new_program_id
);
let res = io.handle_request_sync(&req, meta);
let res = io.handle_request_sync(&req, meta.clone());
let expected = format!(
r#"{{
"jsonrpc":"2.0",
@ -2308,6 +2335,159 @@ pub mod tests {
let result: Response = serde_json::from_str(&res.expect("actual response"))
.expect("actual response deserialization");
assert_eq!(expected, result);
// Set up nonce accounts to test filters
let nonce_keypair0 = Keypair::new();
let instruction = system_instruction::create_nonce_account(
&alice.pubkey(),
&nonce_keypair0.pubkey(),
&bob.pubkey(),
100_000,
);
let message = Message::new(&instruction, Some(&alice.pubkey()));
let tx = Transaction::new(&[&alice, &nonce_keypair0], message, blockhash);
bank.process_transaction(&tx).unwrap();
let nonce_keypair1 = Keypair::new();
let authority = Pubkey::new_rand();
let instruction = system_instruction::create_nonce_account(
&alice.pubkey(),
&nonce_keypair1.pubkey(),
&authority,
100_000,
);
let message = Message::new(&instruction, Some(&alice.pubkey()));
let tx = Transaction::new(&[&alice, &nonce_keypair1], message, blockhash);
bank.process_transaction(&tx).unwrap();
// Test memcmp filter; filter on Initialized state
let req = format!(
r#"{{
"jsonrpc":"2.0",
"id":1,
"method":"getProgramAccounts",
"params":["{}",{{"filters": [
{{
"memcmp": {{"offset": 4,"bytes": "{}"}}
}}
]}}]
}}"#,
system_program::id(),
bs58::encode(vec![1]).into_string(),
);
let res = io.handle_request_sync(&req, meta.clone());
let json: Value = serde_json::from_str(&res.unwrap()).unwrap();
let accounts: Vec<RpcKeyedAccount> = serde_json::from_value(json["result"].clone())
.expect("actual response deserialization");
assert_eq!(accounts.len(), 2);
let req = format!(
r#"{{
"jsonrpc":"2.0",
"id":1,
"method":"getProgramAccounts",
"params":["{}",{{"filters": [
{{
"memcmp": {{"offset": 0,"bytes": "{}"}}
}}
]}}]
}}"#,
system_program::id(),
bs58::encode(vec![1]).into_string(),
);
let res = io.handle_request_sync(&req, meta.clone());
let json: Value = serde_json::from_str(&res.unwrap()).unwrap();
let accounts: Vec<RpcKeyedAccount> = serde_json::from_value(json["result"].clone())
.expect("actual response deserialization");
assert_eq!(accounts.len(), 0);
// Test dataSize filter
let req = format!(
r#"{{
"jsonrpc":"2.0",
"id":1,
"method":"getProgramAccounts",
"params":["{}",{{"filters": [
{{
"dataSize": {}
}}
]}}]
}}"#,
system_program::id(),
nonce::State::size(),
);
let res = io.handle_request_sync(&req, meta.clone());
let json: Value = serde_json::from_str(&res.unwrap()).unwrap();
let accounts: Vec<RpcKeyedAccount> = serde_json::from_value(json["result"].clone())
.expect("actual response deserialization");
assert_eq!(accounts.len(), 2);
let req = format!(
r#"{{
"jsonrpc":"2.0",
"id":1,
"method":"getProgramAccounts",
"params":["{}",{{"filters": [
{{
"dataSize": 1
}}
]}}]
}}"#,
system_program::id(),
);
let res = io.handle_request_sync(&req, meta.clone());
let json: Value = serde_json::from_str(&res.unwrap()).unwrap();
let accounts: Vec<RpcKeyedAccount> = serde_json::from_value(json["result"].clone())
.expect("actual response deserialization");
assert_eq!(accounts.len(), 0);
// Test multiple filters
let req = format!(
r#"{{
"jsonrpc":"2.0",
"id":1,
"method":"getProgramAccounts",
"params":["{}",{{"filters": [
{{
"memcmp": {{"offset": 4,"bytes": "{}"}}
}},
{{
"memcmp": {{"offset": 8,"bytes": "{}"}}
}}
]}}]
}}"#,
system_program::id(),
bs58::encode(vec![1]).into_string(),
authority,
); // Filter on Initialized and Nonce authority
let res = io.handle_request_sync(&req, meta.clone());
let json: Value = serde_json::from_str(&res.unwrap()).unwrap();
let accounts: Vec<RpcKeyedAccount> = serde_json::from_value(json["result"].clone())
.expect("actual response deserialization");
assert_eq!(accounts.len(), 1);
let req = format!(
r#"{{
"jsonrpc":"2.0",
"id":1,
"method":"getProgramAccounts",
"params":["{}",{{"filters": [
{{
"memcmp": {{"offset": 4,"bytes": "{}"}}
}},
{{
"dataSize": 1
}}
]}}]
}}"#,
system_program::id(),
bs58::encode(vec![1]).into_string(),
); // Filter on Initialized and non-matching data size
let res = io.handle_request_sync(&req, meta);
let json: Value = serde_json::from_str(&res.unwrap()).unwrap();
let accounts: Vec<RpcKeyedAccount> = serde_json::from_value(json["result"].clone())
.expect("actual response deserialization");
assert_eq!(accounts.len(), 0);
}
#[test]
@ -2887,6 +3067,25 @@ pub mod tests {
);
}
#[test]
fn test_rpc_verify_filter() {
let filter = RpcFilterType::Memcmp(Memcmp {
offset: 0,
bytes: MemcmpEncodedBytes::Binary(
"13LeFbG6m2EP1fqCj9k66fcXsoTHMMtgr7c78AivUrYD".to_string(),
),
encoding: None,
});
assert_eq!(verify_filter(&filter), Ok(()));
// Invalid base-58
let filter = RpcFilterType::Memcmp(Memcmp {
offset: 0,
bytes: MemcmpEncodedBytes::Binary("III".to_string()),
encoding: None,
});
assert!(verify_filter(&filter).is_err());
}
#[test]
fn test_rpc_verify_pubkey() {
let pubkey = Pubkey::new_rand();
@ -2894,7 +3093,7 @@ pub mod tests {
let bad_pubkey = "a1b2c3d4";
assert_eq!(
verify_pubkey(bad_pubkey.to_string()),
Err(Error::invalid_params("WrongSize"))
Err(Error::invalid_params("Invalid param: WrongSize"))
);
}
@ -2908,7 +3107,7 @@ pub mod tests {
let bad_signature = "a1b2c3d4";
assert_eq!(
verify_signature(&bad_signature.to_string()),
Err(Error::invalid_params("WrongSize"))
Err(Error::invalid_params("Invalid param: WrongSize"))
);
}

4
core/tests/rpc.rs
View File

@ -121,14 +121,14 @@ fn test_rpc_invalid_requests() {
let json = post_rpc(req, &leader_data);
let the_error = json["error"]["message"].as_str().unwrap();
assert_eq!(the_error, "Invalid");
assert_eq!(the_error, "Invalid param: Invalid");
// test invalid get_account_info request
let req = json_req!("getAccountInfo", json!(["invalid9999"]));
let json = post_rpc(req, &leader_data);
let the_error = json["error"]["message"].as_str().unwrap();
assert_eq!(the_error, "Invalid");
assert_eq!(the_error, "Invalid param: Invalid");
// test invalid get_account_info request
let req = json_req!("getAccountInfo", json!([bob_pubkey.to_string()]));

14
docs/src/apps/jsonrpc-api.md
View File

@ -790,6 +790,14 @@ Returns all accounts owned by the provided program Pubkey
* (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment)
* (optional) `encoding: <string>` - encoding for Account data, either "binary" or jsonParsed". If parameter not provided, the default encoding is binary.
Parsed-JSON encoding attempts to use program-specific state parsers to return more human-readable and explicit account state data. If parsed-JSON is requested but a parser cannot be found, the field falls back to binary encoding, detectable when the `data` field is type `<string>`.
* (optional) `filters: <array>` - filter results using various [filter objects](jsonrpc-api.md#filters); account must meet all filter criteria to be included in results
##### Filters:
* `memcmp: <object>` - compares a provided series of bytes with program account data at a particular offset. Fields:
* `offset: <usize>` - offset into program account data to start comparison
* `bytes: <string>` - data to match, as base-58 encoded string
* `dataSize: <u64>` - compares the program account data length with the provided data size
#### Results:
@ -809,6 +817,12 @@ The result field will be an array of JSON objects, which will contain:
// Request
curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0", "id":1, "method":"getProgramAccounts", "params":["4Nd1mBQtrMJVYVfKf2PJy9NZUZdTAsp7D4xWLs4gDB4T"]}' http://localhost:8899
// Result
{"jsonrpc":"2.0","result":[{"account":{"data":"2R9jLfiAQ9bgdcw6h8s44439","executable":false,"lamports":15298080,"owner":"4Nd1mBQtrMJVYVfKf2PJy9NZUZdTAsp7D4xWLs4gDB4T","rentEpoch":28},"pubkey":"CxELquR1gPP8wHe33gZ4QxqGB3sZ9RSwsJ2KshVewkFY"}],"id":1}
// Request with Filters
curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0", "id":1, "method":"getProgramAccounts", "params":["4Nd1mBQtrMJVYVfKf2PJy9NZUZdTAsp7D4xWLs4gDB4T", {"filters":[{"dataSize": 17},{"memcmp": {"offset": 4, "bytes": "3Mc6vR"}}]}]}' http://localhost:8899
// Result
{"jsonrpc":"2.0","result":[{"account":{"data":"2R9jLfiAQ9bgdcw6h8s44439","executable":false,"lamports":15298080,"owner":"4Nd1mBQtrMJVYVfKf2PJy9NZUZdTAsp7D4xWLs4gDB4T","rentEpoch":28},"pubkey":"CxELquR1gPP8wHe33gZ4QxqGB3sZ9RSwsJ2KshVewkFY"}],"id":1}
```