From 88ca04dbdb93b3099a7270be38078ecac0cfae59 Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Mon, 10 Aug 2020 16:35:29 -0600 Subject: [PATCH] Add config param to specify offset/length for single and program account info (#11515) * Add config param to specify dataSlice for account info and program accounts * Use match instead of if --- Cargo.lock | 1 + account-decoder/src/lib.rs | 68 ++++++++++++-- cli/src/cli.rs | 8 +- cli/src/offline/blockhash_query.rs | 1 + client/src/rpc_client.rs | 1 + client/src/rpc_config.rs | 3 +- core/Cargo.toml | 1 + core/src/rpc.rs | 137 +++++++++++++++++++++++++---- core/src/rpc_pubsub.rs | 4 + core/src/rpc_subscriptions.rs | 7 +- core/tests/rpc.rs | 1 + docs/src/apps/jsonrpc-api.md | 4 + 12 files changed, 210 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 25be241e4b..98f0afdbd1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3509,6 +3509,7 @@ dependencies = [ name = "solana-core" version = "1.4.0" dependencies = [ + "base64 0.12.3", "bincode", "bs58", "bv", diff --git a/account-decoder/src/lib.rs b/account-decoder/src/lib.rs index 82caf04b83..a4d32e5e01 100644 --- a/account-decoder/src/lib.rs +++ b/account-decoder/src/lib.rs @@ -51,19 +51,23 @@ impl UiAccount { account: Account, encoding: UiAccountEncoding, additional_data: Option, + data_slice_config: Option, ) -> Self { let data = match encoding { - UiAccountEncoding::Binary => { - UiAccountData::Binary(bs58::encode(account.data).into_string()) - } - UiAccountEncoding::Binary64 => UiAccountData::Binary64(base64::encode(account.data)), + UiAccountEncoding::Binary => UiAccountData::Binary( + bs58::encode(slice_data(&account.data, data_slice_config)).into_string(), + ), + UiAccountEncoding::Binary64 => UiAccountData::Binary64(base64::encode(slice_data( + &account.data, + data_slice_config, + ))), UiAccountEncoding::JsonParsed => { if let Ok(parsed_data) = parse_account_data(pubkey, &account.owner, &account.data, additional_data) { UiAccountData::Json(parsed_data) } else { - UiAccountData::Binary64(base64::encode(account.data)) + UiAccountData::Binary64(base64::encode(&account.data)) } } }; @@ -113,3 +117,57 @@ impl Default for UiFeeCalculator { } } } + +#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct UiDataSliceConfig { + pub offset: usize, + pub length: usize, +} + +fn slice_data(data: &[u8], data_slice_config: Option) -> &[u8] { + if let Some(UiDataSliceConfig { offset, length }) = data_slice_config { + if offset >= data.len() { + &[] + } else if length > data.len() - offset { + &data[offset..] + } else { + &data[offset..offset + length] + } + } else { + data + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_slice_data() { + let data = vec![1, 2, 3, 4, 5]; + let slice_config = Some(UiDataSliceConfig { + offset: 0, + length: 5, + }); + assert_eq!(slice_data(&data, slice_config), &data[..]); + + let slice_config = Some(UiDataSliceConfig { + offset: 0, + length: 10, + }); + assert_eq!(slice_data(&data, slice_config), &data[..]); + + let slice_config = Some(UiDataSliceConfig { + offset: 1, + length: 2, + }); + assert_eq!(slice_data(&data, slice_config), &data[1..3]); + + let slice_config = Some(UiDataSliceConfig { + offset: 10, + length: 2, + }); + assert_eq!(slice_data(&data, slice_config), &[] as &[u8]); + } +} diff --git a/cli/src/cli.rs b/cli/src/cli.rs index 231694c7e9..ebf5fa1ced 100644 --- a/cli/src/cli.rs +++ b/cli/src/cli.rs @@ -1120,7 +1120,13 @@ fn process_show_account( let cli_account = CliAccount { keyed_account: RpcKeyedAccount { pubkey: account_pubkey.to_string(), - account: UiAccount::encode(account_pubkey, account, UiAccountEncoding::Binary64, None), + account: UiAccount::encode( + account_pubkey, + account, + UiAccountEncoding::Binary64, + None, + None, + ), }, use_lamports_unit, }; diff --git a/cli/src/offline/blockhash_query.rs b/cli/src/offline/blockhash_query.rs index c15de0b241..049c833e88 100644 --- a/cli/src/offline/blockhash_query.rs +++ b/cli/src/offline/blockhash_query.rs @@ -355,6 +355,7 @@ mod tests { nonce_account, UiAccountEncoding::Binary64, None, + None, ); let get_account_response = json!(Response { context: RpcResponseContext { slot: 1 }, diff --git a/client/src/rpc_client.rs b/client/src/rpc_client.rs index 2806f6dc1c..f3549bf645 100644 --- a/client/src/rpc_client.rs +++ b/client/src/rpc_client.rs @@ -472,6 +472,7 @@ impl RpcClient { let config = RpcAccountInfoConfig { encoding: Some(UiAccountEncoding::Binary64), commitment: Some(commitment_config), + data_slice: None, }; let response = self.sender.send( RpcRequest::GetAccountInfo, diff --git a/client/src/rpc_config.rs b/client/src/rpc_config.rs index 5722f11857..868c6c7540 100644 --- a/client/src/rpc_config.rs +++ b/client/src/rpc_config.rs @@ -1,5 +1,5 @@ use crate::rpc_filter::RpcFilterType; -use solana_account_decoder::UiAccountEncoding; +use solana_account_decoder::{UiAccountEncoding, UiDataSliceConfig}; use solana_sdk::{clock::Epoch, commitment_config::CommitmentConfig}; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -47,6 +47,7 @@ pub struct RpcStakeConfig { #[serde(rename_all = "camelCase")] pub struct RpcAccountInfoConfig { pub encoding: Option, + pub data_slice: Option, #[serde(flatten)] pub commitment: Option, } diff --git a/core/Cargo.toml b/core/Cargo.toml index 2821926a85..c57ddb8df6 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -79,6 +79,7 @@ solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.4.0" } trees = "0.2.1" [dev-dependencies] +base64 = "0.12.3" matches = "0.1.6" reqwest = { version = "0.10.6", default-features = false, features = ["blocking", "rustls-tls", "json"] } serial_test = "0.4.0" diff --git a/core/src/rpc.rs b/core/src/rpc.rs index 25f0e90e53..2b10a24948 100644 --- a/core/src/rpc.rs +++ b/core/src/rpc.rs @@ -244,6 +244,7 @@ impl JsonRpcRequestProcessor { let config = config.unwrap_or_default(); let bank = self.bank(config.commitment); let encoding = config.encoding.unwrap_or(UiAccountEncoding::Binary); + check_slice_and_encoding(&encoding, config.data_slice.is_some())?; let mut response = None; if let Some(account) = bank.get_account(pubkey) { if account.owner == spl_token_id_v1_0() && encoding == UiAccountEncoding::JsonParsed { @@ -256,7 +257,13 @@ impl JsonRpcRequestProcessor { data: None, }); } else { - response = Some(UiAccount::encode(pubkey, account, encoding, None)); + response = Some(UiAccount::encode( + pubkey, + account, + encoding, + None, + config.data_slice, + )); } } @@ -277,21 +284,31 @@ impl JsonRpcRequestProcessor { program_id: &Pubkey, config: Option, filters: Vec, - ) -> Vec { + ) -> Result> { let config = config.unwrap_or_default(); let bank = self.bank(config.commitment); let encoding = config.encoding.unwrap_or(UiAccountEncoding::Binary); + let data_slice_config = config.data_slice; + check_slice_and_encoding(&encoding, data_slice_config.is_some())?; let keyed_accounts = get_filtered_program_accounts(&bank, program_id, filters); - if program_id == &spl_token_id_v1_0() && encoding == UiAccountEncoding::JsonParsed { - get_parsed_token_accounts(bank, keyed_accounts).collect() - } else { - keyed_accounts - .map(|(pubkey, account)| RpcKeyedAccount { - pubkey: pubkey.to_string(), - account: UiAccount::encode(&pubkey, account, encoding.clone(), None), - }) - .collect() - } + let result = + if program_id == &spl_token_id_v1_0() && encoding == UiAccountEncoding::JsonParsed { + get_parsed_token_accounts(bank, keyed_accounts).collect() + } else { + keyed_accounts + .map(|(pubkey, account)| RpcKeyedAccount { + pubkey: pubkey.to_string(), + account: UiAccount::encode( + &pubkey, + account, + encoding.clone(), + None, + data_slice_config, + ), + }) + .collect() + }; + Ok(result) } pub fn get_inflation_governor( @@ -1107,6 +1124,8 @@ impl JsonRpcRequestProcessor { let config = config.unwrap_or_default(); let bank = self.bank(config.commitment); let encoding = config.encoding.unwrap_or(UiAccountEncoding::Binary); + let data_slice_config = config.data_slice; + check_slice_and_encoding(&encoding, data_slice_config.is_some())?; let (token_program_id, mint) = get_token_program_id_and_mint(&bank, token_account_filter)?; let mut filters = vec![ @@ -1134,7 +1153,13 @@ impl JsonRpcRequestProcessor { keyed_accounts .map(|(pubkey, account)| RpcKeyedAccount { pubkey: pubkey.to_string(), - account: UiAccount::encode(&pubkey, account, encoding.clone(), None), + account: UiAccount::encode( + &pubkey, + account, + encoding.clone(), + None, + data_slice_config, + ), }) .collect() }; @@ -1150,6 +1175,8 @@ impl JsonRpcRequestProcessor { let config = config.unwrap_or_default(); let bank = self.bank(config.commitment); let encoding = config.encoding.unwrap_or(UiAccountEncoding::Binary); + let data_slice_config = config.data_slice; + check_slice_and_encoding(&encoding, data_slice_config.is_some())?; let (token_program_id, mint) = get_token_program_id_and_mint(&bank, token_account_filter)?; let mut filters = vec![ @@ -1185,7 +1212,13 @@ impl JsonRpcRequestProcessor { keyed_accounts .map(|(pubkey, account)| RpcKeyedAccount { pubkey: pubkey.to_string(), - account: UiAccount::encode(&pubkey, account, encoding.clone(), None), + account: UiAccount::encode( + &pubkey, + account, + encoding.clone(), + None, + data_slice_config, + ), }) .collect() }; @@ -1226,6 +1259,26 @@ fn verify_token_account_filter( } } +fn check_slice_and_encoding(encoding: &UiAccountEncoding, data_slice_is_some: bool) -> Result<()> { + match encoding { + UiAccountEncoding::JsonParsed => { + if data_slice_is_some { + let message = + "Sliced account data can only be encoded using binary (base 58) or binary64 encoding." + .to_string(); + Err(error::Error { + code: error::ErrorCode::InvalidRequest, + message, + data: None, + }) + } else { + Ok(()) + } + } + UiAccountEncoding::Binary | UiAccountEncoding::Binary64 => Ok(()), + } +} + /// Use a set of filters to get an iterator of keyed program accounts from a bank fn get_filtered_program_accounts( bank: &Arc, @@ -1258,6 +1311,7 @@ pub(crate) fn get_parsed_token_account( account, UiAccountEncoding::JsonParsed, additional_data, + None, ) } @@ -1286,6 +1340,7 @@ where account, UiAccountEncoding::JsonParsed, additional_data, + None, ), } }) @@ -1753,7 +1808,7 @@ impl RpcSol for RpcSolImpl { for filter in &filters { verify_filter(filter)?; } - Ok(meta.get_program_accounts(&program_id, config, filters)) + meta.get_program_accounts(&program_id, config, filters) } fn get_inflation_governor( @@ -3028,13 +3083,13 @@ pub mod tests { #[test] fn test_rpc_get_account_info() { let bob_pubkey = Pubkey::new_rand(); - let RpcHandler { io, meta, .. } = start_rpc_handler_with_tx(&bob_pubkey); + let RpcHandler { io, meta, bank, .. } = start_rpc_handler_with_tx(&bob_pubkey); let req = format!( r#"{{"jsonrpc":"2.0","id":1,"method":"getAccountInfo","params":["{}"]}}"#, bob_pubkey ); - let res = io.handle_request_sync(&req, meta); + let res = io.handle_request_sync(&req, meta.clone()); let expected = json!({ "jsonrpc": "2.0", "result": { @@ -3054,6 +3109,54 @@ pub mod tests { let result: Response = serde_json::from_str(&res.expect("actual response")) .expect("actual response deserialization"); assert_eq!(expected, result); + + let address = Pubkey::new_rand(); + let data = vec![1, 2, 3, 4, 5]; + let mut account = Account::new(42, 5, &Pubkey::default()); + account.data = data.clone(); + bank.store_account(&address, &account); + + let req = format!( + r#"{{"jsonrpc":"2.0","id":1,"method":"getAccountInfo","params":["{}", {{"encoding":"binary64"}}]}}"#, + address + ); + let res = io.handle_request_sync(&req, meta.clone()); + let result: Value = serde_json::from_str(&res.expect("actual response")) + .expect("actual response deserialization"); + assert_eq!(result["result"]["value"]["data"], base64::encode(&data)); + + let req = format!( + r#"{{"jsonrpc":"2.0","id":1,"method":"getAccountInfo","params":["{}", {{"encoding":"binary64", "dataSlice": {{"length": 2, "offset": 1}}}}]}}"#, + address + ); + let res = io.handle_request_sync(&req, meta.clone()); + let result: Value = serde_json::from_str(&res.expect("actual response")) + .expect("actual response deserialization"); + assert_eq!( + result["result"]["value"]["data"], + base64::encode(&data[1..3]), + ); + + let req = format!( + r#"{{"jsonrpc":"2.0","id":1,"method":"getAccountInfo","params":["{}", {{"encoding":"binary", "dataSlice": {{"length": 2, "offset": 1}}}}]}}"#, + address + ); + let res = io.handle_request_sync(&req, meta.clone()); + let result: Value = serde_json::from_str(&res.expect("actual response")) + .expect("actual response deserialization"); + assert_eq!( + result["result"]["value"]["data"], + bs58::encode(&data[1..3]).into_string(), + ); + + let req = format!( + r#"{{"jsonrpc":"2.0","id":1,"method":"getAccountInfo","params":["{}", {{"encoding":"jsonParsed", "dataSlice": {{"length": 2, "offset": 1}}}}]}}"#, + address + ); + let res = io.handle_request_sync(&req, meta); + let result: Value = serde_json::from_str(&res.expect("actual response")) + .expect("actual response deserialization"); + result["error"].as_object().unwrap(); } #[test] diff --git a/core/src/rpc_pubsub.rs b/core/src/rpc_pubsub.rs index 3787f4527e..2c4300dad7 100644 --- a/core/src/rpc_pubsub.rs +++ b/core/src/rpc_pubsub.rs @@ -542,6 +542,7 @@ mod tests { Some(RpcAccountInfoConfig { commitment: Some(CommitmentConfig::recent()), encoding: None, + data_slice: None, }), ); @@ -649,6 +650,7 @@ mod tests { Some(RpcAccountInfoConfig { commitment: Some(CommitmentConfig::recent()), encoding: Some(UiAccountEncoding::JsonParsed), + data_slice: None, }), ); @@ -769,6 +771,7 @@ mod tests { Some(RpcAccountInfoConfig { commitment: Some(CommitmentConfig::root()), encoding: None, + data_slice: None, }), ); @@ -818,6 +821,7 @@ mod tests { Some(RpcAccountInfoConfig { commitment: Some(CommitmentConfig::root()), encoding: None, + data_slice: None, }), ); diff --git a/core/src/rpc_subscriptions.rs b/core/src/rpc_subscriptions.rs index 4874b93247..685a459cf2 100644 --- a/core/src/rpc_subscriptions.rs +++ b/core/src/rpc_subscriptions.rs @@ -265,7 +265,7 @@ fn filter_account_result( } else { return ( Box::new(iter::once(UiAccount::encode( - pubkey, account, encoding, None, + pubkey, account, encoding, None, None, ))), fork, ); @@ -316,7 +316,7 @@ fn filter_program_results( Box::new( keyed_accounts.map(move |(pubkey, account)| RpcKeyedAccount { pubkey: pubkey.to_string(), - account: UiAccount::encode(&pubkey, account, encoding.clone(), None), + account: UiAccount::encode(&pubkey, account, encoding.clone(), None, None), }), ) }; @@ -1033,6 +1033,7 @@ pub(crate) mod tests { Some(RpcAccountInfoConfig { commitment: Some(CommitmentConfig::recent()), encoding: None, + data_slice: None, }), sub_id.clone(), subscriber, @@ -1517,6 +1518,7 @@ pub(crate) mod tests { Some(RpcAccountInfoConfig { commitment: Some(CommitmentConfig::single_gossip()), encoding: None, + data_slice: None, }), sub_id0.clone(), subscriber0, @@ -1585,6 +1587,7 @@ pub(crate) mod tests { Some(RpcAccountInfoConfig { commitment: Some(CommitmentConfig::single_gossip()), encoding: None, + data_slice: None, }), sub_id1.clone(), subscriber1, diff --git a/core/tests/rpc.rs b/core/tests/rpc.rs index fa59d3b7fe..d60a7cad07 100644 --- a/core/tests/rpc.rs +++ b/core/tests/rpc.rs @@ -105,6 +105,7 @@ fn test_rpc_send_tx() { let config = RpcAccountInfoConfig { encoding: Some(UiAccountEncoding::Binary64), commitment: None, + data_slice: None, }; let req = json_req!( "getAccountInfo", diff --git a/docs/src/apps/jsonrpc-api.md b/docs/src/apps/jsonrpc-api.md index 8c330d7c0a..a13c9e01ba 100644 --- a/docs/src/apps/jsonrpc-api.md +++ b/docs/src/apps/jsonrpc-api.md @@ -158,6 +158,7 @@ Returns all information associated with the account of provided Pubkey - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment) - (optional) `encoding: ` - encoding for Account data, either "binary", "binary64", or jsonParsed". If parameter not provided, the default encoding is "binary". "binary" is base-58 encoded and limited to Account data of less than 128 bytes. "binary64" will return base64 encoded data for Account data of any size. Parsed-JSON encoding attempts to use program-specific state parsers to return more human-readable and explicit account state data. If parsed-JSON is requested but a parser cannot be found, the field falls back to binary encoding, detectable when the `data` field is type ``. **jsonParsed encoding is UNSTABLE** + - (optional) `dataSlice: ` - limit the returned account data using the provided `offset: ` and `length: ` fields; only available for "binary" or "binary64" encoding. #### Results: @@ -845,6 +846,7 @@ Returns all accounts owned by the provided program Pubkey - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment) - (optional) `encoding: ` - encoding for Account data, either "binary" or jsonParsed". If parameter not provided, the default encoding is binary. Parsed-JSON encoding attempts to use program-specific state parsers to return more human-readable and explicit account state data. If parsed-JSON is requested but a parser cannot be found, the field falls back to binary encoding, detectable when the `data` field is type ``. **jsonParsed encoding is UNSTABLE** + - (optional) `dataSlice: ` - limit the returned account data using the provided `offset: ` and `length: ` fields; only available for "binary" or "binary64" encoding. - (optional) `filters: ` - filter results using various [filter objects](jsonrpc-api.md#filters); account must meet all filter criteria to be included in results ##### Filters: @@ -1099,6 +1101,7 @@ Returns all SPL Token accounts by approved Delegate. **UNSTABLE** - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment) - (optional) `encoding: ` - encoding for Account data, either "binary" or jsonParsed". If parameter not provided, the default encoding is binary. Parsed-JSON encoding attempts to use program-specific state parsers to return more human-readable and explicit account state data. If parsed-JSON is requested but a parser cannot be found, the field falls back to binary encoding, detectable when the `data` field is type ``. **jsonParsed encoding is UNSTABLE** + - (optional) `dataSlice: ` - limit the returned account data using the provided `offset: ` and `length: ` fields; only available for "binary" or "binary64" encoding. #### Results: @@ -1135,6 +1138,7 @@ Returns all SPL Token accounts by token owner. **UNSTABLE** - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment) - (optional) `encoding: ` - encoding for Account data, either "binary" or jsonParsed". If parameter not provided, the default encoding is binary. Parsed-JSON encoding attempts to use program-specific state parsers to return more human-readable and explicit account state data. If parsed-JSON is requested but a parser cannot be found, the field falls back to binary encoding, detectable when the `data` field is type ``. **jsonParsed encoding is UNSTABLE** + - (optional) `dataSlice: ` - limit the returned account data using the provided `offset: ` and `length: ` fields; only available for "binary" or "binary64" encoding. #### Results: