From 215ddecaa5daca5becc0388198dfc1e7be291966 Mon Sep 17 00:00:00 2001 From: Michael Vines Date: Tue, 24 Nov 2020 12:52:01 -0800 Subject: [PATCH] Add base64+zstd encoding for RPC account data --- Cargo.lock | 1 + account-decoder/Cargo.toml | 1 + account-decoder/src/lib.rs | 57 ++++++++++++++++++++-- core/src/rpc.rs | 5 +- docs/src/developing/clients/jsonrpc-api.md | 49 ++++++++++--------- 5 files changed, 86 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bd0282bca..8f51c124e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3601,6 +3601,7 @@ dependencies = [ "solana-vote-program", "spl-token", "thiserror", + "zstd", ] [[package]] diff --git a/account-decoder/Cargo.toml b/account-decoder/Cargo.toml index 74b5adf87..c53b7a6ce 100644 --- a/account-decoder/Cargo.toml +++ b/account-decoder/Cargo.toml @@ -24,6 +24,7 @@ solana-stake-program = { path = "../programs/stake", version = "1.5.0" } solana-vote-program = { path = "../programs/vote", version = "1.5.0" } spl-token-v2-0 = { package = "spl-token", version = "=3.0.0", features = ["no-entrypoint"] } thiserror = "1.0" +zstd = "0.5.1" [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] diff --git a/account-decoder/src/lib.rs b/account-decoder/src/lib.rs index 8c1f11caf..de06324c3 100644 --- a/account-decoder/src/lib.rs +++ b/account-decoder/src/lib.rs @@ -12,9 +12,14 @@ pub mod parse_token; pub mod parse_vote; pub mod validator_info; -use crate::parse_account_data::{parse_account_data, AccountAdditionalData, ParsedAccount}; -use solana_sdk::{account::Account, clock::Epoch, fee_calculator::FeeCalculator, pubkey::Pubkey}; -use std::str::FromStr; +use { + crate::parse_account_data::{parse_account_data, AccountAdditionalData, ParsedAccount}, + solana_sdk::{account::Account, clock::Epoch, fee_calculator::FeeCalculator, pubkey::Pubkey}, + std::{ + io::{Read, Write}, + str::FromStr, + }, +}; pub type StringAmount = String; @@ -44,6 +49,8 @@ pub enum UiAccountEncoding { Base58, Base64, JsonParsed, + #[serde(rename = "base64+zstd")] + Base64Zstd, } impl UiAccount { @@ -66,6 +73,19 @@ impl UiAccount { base64::encode(slice_data(&account.data, data_slice_config)), encoding, ), + UiAccountEncoding::Base64Zstd => { + let mut encoder = zstd::stream::write::Encoder::new(Vec::new(), 0).unwrap(); + match encoder + .write_all(slice_data(&account.data, data_slice_config)) + .and_then(|()| encoder.finish()) + { + Ok(zstd_data) => UiAccountData::Binary(base64::encode(zstd_data), encoding), + Err(_) => UiAccountData::Binary( + base64::encode(slice_data(&account.data, data_slice_config)), + UiAccountEncoding::Base64, + ), + } + } UiAccountEncoding::JsonParsed => { if let Ok(parsed_data) = parse_account_data(pubkey, &account.owner, &account.data, additional_data) @@ -92,6 +112,16 @@ impl UiAccount { UiAccountData::Binary(blob, encoding) => match encoding { UiAccountEncoding::Base58 => bs58::decode(blob).into_vec().ok(), UiAccountEncoding::Base64 => base64::decode(blob).ok(), + UiAccountEncoding::Base64Zstd => base64::decode(blob) + .ok() + .map(|zstd_data| { + let mut data = vec![]; + zstd::stream::read::Decoder::new(zstd_data.as_slice()) + .and_then(|mut reader| reader.read_to_end(&mut data)) + .map(|_| data) + .ok() + }) + .flatten(), UiAccountEncoding::Binary | UiAccountEncoding::JsonParsed => None, }, }?; @@ -179,4 +209,25 @@ mod test { }); assert_eq!(slice_data(&data, slice_config), &[] as &[u8]); } + + #[test] + fn test_base64_zstd() { + let encoded_account = UiAccount::encode( + &Pubkey::default(), + Account { + data: vec![0; 1024], + ..Account::default() + }, + UiAccountEncoding::Base64Zstd, + None, + None, + ); + assert!(matches!( + encoded_account.data, + UiAccountData::Binary(_, UiAccountEncoding::Base64Zstd) + )); + + let decoded_account = encoded_account.decode().unwrap(); + assert_eq!(decoded_account.data, vec![0; 1024]); + } } diff --git a/core/src/rpc.rs b/core/src/rpc.rs index 587deeeb8..9fe42daa8 100644 --- a/core/src/rpc.rs +++ b/core/src/rpc.rs @@ -1335,7 +1335,10 @@ fn check_slice_and_encoding(encoding: &UiAccountEncoding, data_slice_is_some: bo Ok(()) } } - UiAccountEncoding::Binary | UiAccountEncoding::Base58 | UiAccountEncoding::Base64 => Ok(()), + UiAccountEncoding::Binary + | UiAccountEncoding::Base58 + | UiAccountEncoding::Base64 + | UiAccountEncoding::Base64Zstd => Ok(()), } } diff --git a/docs/src/developing/clients/jsonrpc-api.md b/docs/src/developing/clients/jsonrpc-api.md index 19f308202..a2f7cb7a4 100644 --- a/docs/src/developing/clients/jsonrpc-api.md +++ b/docs/src/developing/clients/jsonrpc-api.md @@ -205,16 +205,12 @@ Returns all information associated with the account of provided Pubkey - `` - (optional) Configuration object containing the following optional fields: - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment) - - `encoding: ` - encoding for Account data, either "base58" (*slow*), -"base64", or "jsonParsed". "base58" is limited to Account data of less than 128 -bytes. "base64" will return base64 encoded data for Account data of any size. -"jsonParsed" encoding attempts to use program-specific state parsers to return -more human-readable and explicit account state data. If "jsonParsed" is requested -but a parser cannot be found, the field falls back to "base64" encoding, detectable -when the `data` field is type ``. - - (optional) `dataSlice: ` - limit the returned account data using the -provided `offset: ` and `length: ` fields; only available for -"base58" or "base64" encoding. + - `encoding: ` - encoding for Account data, either "base58" (*slow*), "base64", "base64+zstd", or "jsonParsed". + "base58" is limited to Account data of less than 128 bytes. + "base64" will return base64 encoded data for Account data of any size. + "base64+zstd" compresses the Account data using [Zstandard](https://facebook.github.io/zstd/) and base64-encodes the result. + "jsonParsed" encoding attempts to use program-specific state parsers to return more human-readable and explicit account state data. If "jsonParsed" is requested but a parser cannot be found, the field falls back to "base64" encoding, detectable when the `data` field is type ``. + - (optional) `dataSlice: ` - limit the returned account data using the provided `offset: ` and `length: ` fields; only available for "base58", "base64" or "base64+zstd" encodings. #### Results: @@ -469,7 +465,7 @@ Returns identity and transaction information about a confirmed block in the ledg #### Parameters: - `` - slot, as u64 integer -- `` - encoding for each returned Transaction, either "json", "jsonParsed", "base58" (*slow*), or "base64". If parameter not provided, the default encoding is JSON. +- `` - encoding for each returned Transaction, either "json", "jsonParsed", "base58" (*slow*), "base64". If parameter not provided, the default encoding is "json". "jsonParsed" encoding attempts to use program-specific instruction parsers to return more human-readable and explicit data in the `transaction.message.instructions` list. If "jsonParsed" is requested but a parser cannot be found, the instruction falls back to regular JSON encoding (`accounts`, `data`, and `programIdIndex` fields). #### Results: @@ -1565,9 +1561,13 @@ Returns the account information for a list of Pubkeys - `` - An array of Pubkeys to query, as base-58 encoded strings - `` - (optional) Configuration object containing the following optional fields: - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment) - - `encoding: ` - encoding for Account data, either "base58" (*slow*), "base64", or "jsonParsed". "base58" is limited to Account data of less than 128 bytes. "base64" will return base64 encoded data for Account data of any size. - "jsonParsed" encoding attempts to use program-specific state parsers to return more human-readable and explicit account state data. If "jsonParsed" is requested but a parser cannot be found, the field falls back to base64 encoding, detectable when the `data` field is type ``. - - (optional) `dataSlice: ` - limit the returned account data using the provided `offset: ` and `length: ` fields; only available for "base58" or "base64" encoding. + - `encoding: ` - encoding for Account data, either "base58" (*slow*), "base64", "base64+zstd", or "jsonParsed". + "base58" is limited to Account data of less than 128 bytes. + "base64" will return base64 encoded data for Account data of any size. + "base64+zstd" compresses the Account data using [Zstandard](https://facebook.github.io/zstd/) and base64-encodes the result. + "jsonParsed" encoding attempts to use program-specific state parsers to return more human-readable and explicit account state data. If "jsonParsed" is requested but a parser cannot be found, the field falls back to "base64" encoding, detectable when the `data` field is type ``. + - (optional) `dataSlice: ` - limit the returned account data using the provided `offset: ` and `length: ` fields; only available for "base58", "base64" or "base64+zstd" encodings. + #### Results: @@ -1708,9 +1708,12 @@ Returns all accounts owned by the provided program Pubkey - `` - Pubkey of program, as base-58 encoded string - `` - (optional) Configuration object containing the following optional fields: - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment) - - `encoding: ` - encoding for Account data, either "base58" (*slow*), "base64" or "jsonParsed". - "jsonParsed" encoding attempts to use program-specific state parsers to return more human-readable and explicit account state data. If "jsonParsed" is requested but a parser cannot be found, the field falls back to base64 encoding, detectable when the `data` field is type ``. If "jsonParsed" is requested for the SPL Token program, when a valid mint cannot be found for a particular account, that account will be filtered out from results. - - (optional) `dataSlice: ` - limit the returned account data using the provided `offset: ` and `length: ` fields; only available for "base58" or "base64" encoding. + - `encoding: ` - encoding for Account data, either "base58" (*slow*), "base64", "base64+zstd", or "jsonParsed". + "base58" is limited to Account data of less than 128 bytes. + "base64" will return base64 encoded data for Account data of any size. + "base64+zstd" compresses the Account data using [Zstandard](https://facebook.github.io/zstd/) and base64-encodes the result. + "jsonParsed" encoding attempts to use program-specific state parsers to return more human-readable and explicit account state data. If "jsonParsed" is requested but a parser cannot be found, the field falls back to "base64" encoding, detectable when the `data` field is type ``. + - (optional) `dataSlice: ` - limit the returned account data using the provided `offset: ` and `length: ` fields; only available for "base58", "base64" or "base64+zstd" encodings. - (optional) `filters: ` - filter results using various [filter objects](jsonrpc-api.md#filters); account must meet all filter criteria to be included in results ##### Filters: @@ -2252,9 +2255,9 @@ Returns all SPL Token accounts by approved Delegate. **UNSTABLE** * `programId: ` - Pubkey of the Token program ID that owns the accounts, as base-58 encoded string - `` - (optional) Configuration object containing the following optional fields: - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment) - - `encoding: ` - encoding for Account data, either "base58" (*slow*), "base64" or "jsonParsed". + - `encoding: ` - encoding for Account data, either "base58" (*slow*), "base64", "base64+zstd" or "jsonParsed". "jsonParsed" encoding attempts to use program-specific state parsers to return more human-readable and explicit account state data. If "jsonParsed" is requested but a valid mint cannot be found for a particular account, that account will be filtered out from results. - - (optional) `dataSlice: ` - limit the returned account data using the provided `offset: ` and `length: ` fields; only available for "base58" or "base64" encoding. + - (optional) `dataSlice: ` - limit the returned account data using the provided `offset: ` and `length: ` fields; only available for "base58", "base64" or "base64+zstd" encodings. #### Results: @@ -2341,9 +2344,9 @@ Returns all SPL Token accounts by token owner. **UNSTABLE** * `programId: ` - Pubkey of the Token program ID that owns the accounts, as base-58 encoded string - `` - (optional) Configuration object containing the following optional fields: - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment) - - `encoding: ` - encoding for Account data, either "base58" (*slow*), "base64" or "jsonParsed". + - `encoding: ` - encoding for Account data, either "base58" (*slow*), "base64", "base64+zstd" or "jsonParsed". "jsonParsed" encoding attempts to use program-specific state parsers to return more human-readable and explicit account state data. If "jsonParsed" is requested but a valid mint cannot be found for a particular account, that account will be filtered out from results. - - (optional) `dataSlice: ` - limit the returned account data using the provided `offset: ` and `length: ` fields; only available for "base58" or "base64" encoding. + - (optional) `dataSlice: ` - limit the returned account data using the provided `offset: ` and `length: ` fields; only available for "base58", "base64" or "base64+zstd" encodings. #### Results: @@ -2854,7 +2857,7 @@ Subscribe to an account to receive notifications when the lamports or data for a - `` - account Pubkey, as base-58 encoded string - `` - (optional) Configuration object containing the following optional fields: - `` - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment) - - `encoding: ` - encoding for Account data, either "base58" (*slow*), "base64" or "jsonParsed". + - `encoding: ` - encoding for Account data, either "base58" (*slow*), "base64", "base64+zstd" or "jsonParsed". "jsonParsed" encoding attempts to use program-specific state parsers to return more human-readable and explicit account state data. If "jsonParsed" is requested but a parser cannot be found, the field falls back to binary encoding, detectable when the `data` field is type ``. #### Results: @@ -2989,7 +2992,7 @@ Subscribe to a program to receive notifications when the lamports or data for a - `` - program_id Pubkey, as base-58 encoded string - `` - (optional) Configuration object containing the following optional fields: - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment) - - `encoding: ` - encoding for Account data, either "base58" (*slow*), "base64" or "jsonParsed". + - `encoding: ` - encoding for Account data, either "base58" (*slow*), "base64", "base64+zstd" or "jsonParsed". "jsonParsed" encoding attempts to use program-specific state parsers to return more human-readable and explicit account state data. If "jsonParsed" is requested but a parser cannot be found, the field falls back to base64 encoding, detectable when the `data` field is type ``. - (optional) `filters: ` - filter results using various [filter objects](jsonrpc-api.md#filters); account must meet all filter criteria to be included in results