Add base64+zstd encoding for RPC account data

Cargo.lock

@@ -3601,6 +3601,7 @@ dependencies = [
  "solana-vote-program",
  "spl-token",
  "thiserror",
+ "zstd",
 ]
 
 [[package]]

account-decoder/Cargo.toml

@@ -24,6 +24,7 @@ solana-stake-program = { path = "../programs/stake", version = "1.5.0" }
 solana-vote-program = { path = "../programs/vote", version = "1.5.0" }
 spl-token-v2-0 = { package = "spl-token", version = "=3.0.0", features = ["no-entrypoint"] }
 thiserror = "1.0"
+zstd = "0.5.1"
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

account-decoder/src/lib.rs

@@ -12,9 +12,14 @@ pub mod parse_token;
 pub mod parse_vote;
 pub mod validator_info;
 
-use crate::parse_account_data::{parse_account_data, AccountAdditionalData, ParsedAccount};
+use {
-use solana_sdk::{account::Account, clock::Epoch, fee_calculator::FeeCalculator, pubkey::Pubkey};
+    crate::parse_account_data::{parse_account_data, AccountAdditionalData, ParsedAccount},
-use std::str::FromStr;
+    solana_sdk::{account::Account, clock::Epoch, fee_calculator::FeeCalculator, pubkey::Pubkey},
+    std::{
+        io::{Read, Write},
+        str::FromStr,
+    },
+};
 
 pub type StringAmount = String;
 
@@ -44,6 +49,8 @@ pub enum UiAccountEncoding {
     Base58,
     Base64,
     JsonParsed,
+    #[serde(rename = "base64+zstd")]
+    Base64Zstd,
 }
 
 impl UiAccount {
@@ -66,6 +73,19 @@ impl UiAccount {
                 base64::encode(slice_data(&account.data, data_slice_config)),
                 encoding,
             ),
+            UiAccountEncoding::Base64Zstd => {
+                let mut encoder = zstd::stream::write::Encoder::new(Vec::new(), 0).unwrap();
+                match encoder
+                    .write_all(slice_data(&account.data, data_slice_config))
+                    .and_then(|()| encoder.finish())
+                {
+                    Ok(zstd_data) => UiAccountData::Binary(base64::encode(zstd_data), encoding),
+                    Err(_) => UiAccountData::Binary(
+                        base64::encode(slice_data(&account.data, data_slice_config)),
+                        UiAccountEncoding::Base64,
+                    ),
+                }
+            }
             UiAccountEncoding::JsonParsed => {
                 if let Ok(parsed_data) =
                     parse_account_data(pubkey, &account.owner, &account.data, additional_data)
@@ -92,6 +112,16 @@ impl UiAccount {
             UiAccountData::Binary(blob, encoding) => match encoding {
                 UiAccountEncoding::Base58 => bs58::decode(blob).into_vec().ok(),
                 UiAccountEncoding::Base64 => base64::decode(blob).ok(),
+                UiAccountEncoding::Base64Zstd => base64::decode(blob)
+                    .ok()
+                    .map(|zstd_data| {
+                        let mut data = vec![];
+                        zstd::stream::read::Decoder::new(zstd_data.as_slice())
+                            .and_then(|mut reader| reader.read_to_end(&mut data))
+                            .map(|_| data)
+                            .ok()
+                    })
+                    .flatten(),
                 UiAccountEncoding::Binary | UiAccountEncoding::JsonParsed => None,
             },
         }?;
@@ -179,4 +209,25 @@ mod test {
         });
         assert_eq!(slice_data(&data, slice_config), &[] as &[u8]);
     }
+
+    #[test]
+    fn test_base64_zstd() {
+        let encoded_account = UiAccount::encode(
+            &Pubkey::default(),
+            Account {
+                data: vec![0; 1024],
+                ..Account::default()
+            },
+            UiAccountEncoding::Base64Zstd,
+            None,
+            None,
+        );
+        assert!(matches!(
+            encoded_account.data,
+            UiAccountData::Binary(_, UiAccountEncoding::Base64Zstd)
+        ));
+
+        let decoded_account = encoded_account.decode().unwrap();
+        assert_eq!(decoded_account.data, vec![0; 1024]);
+    }
 }

core/src/rpc.rs

@@ -1335,7 +1335,10 @@ fn check_slice_and_encoding(encoding: &UiAccountEncoding, data_slice_is_some: bo
                 Ok(())
             }
         }
-        UiAccountEncoding::Binary | UiAccountEncoding::Base58 | UiAccountEncoding::Base64 => Ok(()),
+        UiAccountEncoding::Binary
+        | UiAccountEncoding::Base58
+        | UiAccountEncoding::Base64
+        | UiAccountEncoding::Base64Zstd => Ok(()),
     }
 }
 

docs/src/developing/clients/jsonrpc-api.md

@@ -205,16 +205,12 @@ Returns all information associated with the account of provided Pubkey
 - `<object>` - (optional) Configuration object containing the following optional
 fields:
   - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment)
-  - `encoding: <string>` - encoding for Account data, either "base58" (*slow*),
+  - `encoding: <string>` - encoding for Account data, either "base58" (*slow*), "base64", "base64+zstd", or "jsonParsed".
-"base64", or "jsonParsed". "base58" is limited to Account data of less than 128
+    "base58" is limited to Account data of less than 128 bytes.
-bytes. "base64" will return base64 encoded data for Account data of any size.
+    "base64" will return base64 encoded data for Account data of any size.
-"jsonParsed" encoding attempts to use program-specific state parsers to return
+    "base64+zstd" compresses the Account data using [Zstandard](https://facebook.github.io/zstd/) and base64-encodes the result.
-more human-readable and explicit account state data. If "jsonParsed" is requested
+    "jsonParsed" encoding attempts to use program-specific state parsers to return more human-readable and explicit account state data. If "jsonParsed" is requested but a parser cannot be found, the field falls back to "base64" encoding, detectable when the `data` field is type `<string>`.
-but a parser cannot be found, the field falls back to "base64" encoding, detectable
+  - (optional) `dataSlice: <object>` - limit the returned account data using the provided `offset: <usize>` and `length: <usize>` fields; only available for "base58", "base64" or "base64+zstd" encodings.
-when the `data` field is type `<string>`.
-  - (optional) `dataSlice: <object>` - limit the returned account data using the
-provided `offset: <usize>` and `length: <usize>` fields; only available for
-"base58" or "base64" encoding.
 
 #### Results:
 
@@ -469,7 +465,7 @@ Returns identity and transaction information about a confirmed block in the ledg
 #### Parameters:
 
 - `<u64>` - slot, as u64 integer
-- `<string>` - encoding for each returned Transaction, either "json", "jsonParsed", "base58" (*slow*), or "base64". If parameter not provided, the default encoding is JSON.
+- `<string>` - encoding for each returned Transaction, either "json", "jsonParsed", "base58" (*slow*), "base64". If parameter not provided, the default encoding is "json".
   "jsonParsed" encoding attempts to use program-specific instruction parsers to return more human-readable and explicit data in the `transaction.message.instructions` list. If "jsonParsed" is requested but a parser cannot be found, the instruction falls back to regular JSON encoding (`accounts`, `data`, and `programIdIndex` fields).
 
 #### Results:
@@ -1565,9 +1561,13 @@ Returns the account information for a list of Pubkeys
 - `<array>` - An array of Pubkeys to query, as base-58 encoded strings
 - `<object>` - (optional) Configuration object containing the following optional fields:
   - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment)
-  - `encoding: <string>` - encoding for Account data, either "base58" (*slow*), "base64", or "jsonParsed". "base58" is limited to Account data of less than 128 bytes. "base64" will return base64 encoded data for Account data of any size.
+  - `encoding: <string>` - encoding for Account data, either "base58" (*slow*), "base64", "base64+zstd", or "jsonParsed".
-    "jsonParsed" encoding attempts to use program-specific state parsers to return more human-readable and explicit account state data. If "jsonParsed" is requested but a parser cannot be found, the field falls back to base64 encoding, detectable when the `data` field is type `<string>`.
+    "base58" is limited to Account data of less than 128 bytes.
-  - (optional) `dataSlice: <object>` - limit the returned account data using the provided `offset: <usize>` and `length: <usize>` fields; only available for "base58" or "base64" encoding.
+    "base64" will return base64 encoded data for Account data of any size.
+    "base64+zstd" compresses the Account data using [Zstandard](https://facebook.github.io/zstd/) and base64-encodes the result.
+    "jsonParsed" encoding attempts to use program-specific state parsers to return more human-readable and explicit account state data. If "jsonParsed" is requested but a parser cannot be found, the field falls back to "base64" encoding, detectable when the `data` field is type `<string>`.
+  - (optional) `dataSlice: <object>` - limit the returned account data using the provided `offset: <usize>` and `length: <usize>` fields; only available for "base58", "base64" or "base64+zstd" encodings.
+
 
 #### Results:
 
@@ -1708,9 +1708,12 @@ Returns all accounts owned by the provided program Pubkey
 - `<string>` - Pubkey of program, as base-58 encoded string
 - `<object>` - (optional) Configuration object containing the following optional fields:
   - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment)
-  - `encoding: <string>` - encoding for Account data, either "base58" (*slow*), "base64" or "jsonParsed".
+  - `encoding: <string>` - encoding for Account data, either "base58" (*slow*), "base64", "base64+zstd", or "jsonParsed".
-    "jsonParsed" encoding attempts to use program-specific state parsers to return more human-readable and explicit account state data. If "jsonParsed" is requested but a parser cannot be found, the field falls back to base64 encoding, detectable when the `data` field is type `<string>`. If "jsonParsed" is requested for the SPL Token program, when a valid mint cannot be found for a particular account, that account will be filtered out from results.
+    "base58" is limited to Account data of less than 128 bytes.
-  - (optional) `dataSlice: <object>` - limit the returned account data using the provided `offset: <usize>` and `length: <usize>` fields; only available for "base58" or "base64" encoding.
+    "base64" will return base64 encoded data for Account data of any size.
+    "base64+zstd" compresses the Account data using [Zstandard](https://facebook.github.io/zstd/) and base64-encodes the result.
+    "jsonParsed" encoding attempts to use program-specific state parsers to return more human-readable and explicit account state data. If "jsonParsed" is requested but a parser cannot be found, the field falls back to "base64" encoding, detectable when the `data` field is type `<string>`.
+  - (optional) `dataSlice: <object>` - limit the returned account data using the provided `offset: <usize>` and `length: <usize>` fields; only available for "base58", "base64" or "base64+zstd" encodings.
   - (optional) `filters: <array>` - filter results using various [filter objects](jsonrpc-api.md#filters); account must meet all filter criteria to be included in results
 
 ##### Filters:
@@ -2252,9 +2255,9 @@ Returns all SPL Token accounts by approved Delegate. **UNSTABLE**
   * `programId: <string>` - Pubkey of the Token program ID that owns the accounts, as base-58 encoded string
 - `<object>` - (optional) Configuration object containing the following optional fields:
   - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment)
-  - `encoding: <string>` - encoding for Account data, either "base58" (*slow*), "base64" or "jsonParsed".
+  - `encoding: <string>` - encoding for Account data, either "base58" (*slow*), "base64", "base64+zstd" or "jsonParsed".
     "jsonParsed" encoding attempts to use program-specific state parsers to return more human-readable and explicit account state data. If "jsonParsed" is requested but a valid mint cannot be found for a particular account, that account will be filtered out from results.
-  - (optional) `dataSlice: <object>` - limit the returned account data using the provided `offset: <usize>` and `length: <usize>` fields; only available for "base58" or "base64" encoding.
+  - (optional) `dataSlice: <object>` - limit the returned account data using the provided `offset: <usize>` and `length: <usize>` fields; only available for "base58", "base64" or "base64+zstd" encodings.
 
 #### Results:
 
@@ -2341,9 +2344,9 @@ Returns all SPL Token accounts by token owner. **UNSTABLE**
   * `programId: <string>` - Pubkey of the Token program ID that owns the accounts, as base-58 encoded string
 - `<object>` - (optional) Configuration object containing the following optional fields:
   - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment)
-  - `encoding: <string>` - encoding for Account data, either "base58" (*slow*), "base64" or "jsonParsed".
+  - `encoding: <string>` - encoding for Account data, either "base58" (*slow*), "base64", "base64+zstd" or "jsonParsed".
     "jsonParsed" encoding attempts to use program-specific state parsers to return more human-readable and explicit account state data. If "jsonParsed" is requested but a valid mint cannot be found for a particular account, that account will be filtered out from results.
-  - (optional) `dataSlice: <object>` - limit the returned account data using the provided `offset: <usize>` and `length: <usize>` fields; only available for "base58" or "base64" encoding.
+  - (optional) `dataSlice: <object>` - limit the returned account data using the provided `offset: <usize>` and `length: <usize>` fields; only available for "base58", "base64" or "base64+zstd" encodings.
 
 #### Results:
 
@@ -2854,7 +2857,7 @@ Subscribe to an account to receive notifications when the lamports or data for a
 - `<string>` - account Pubkey, as base-58 encoded string
 - `<object>` - (optional) Configuration object containing the following optional fields:
   - `<object>` - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment)
-  - `encoding: <string>` - encoding for Account data, either "base58" (*slow*), "base64" or "jsonParsed".
+  - `encoding: <string>` - encoding for Account data, either "base58" (*slow*), "base64", "base64+zstd" or "jsonParsed".
     "jsonParsed" encoding attempts to use program-specific state parsers to return more human-readable and explicit account state data. If "jsonParsed" is requested but a parser cannot be found, the field falls back to binary encoding, detectable when the `data` field is type `<string>`.
 
 #### Results:
@@ -2989,7 +2992,7 @@ Subscribe to a program to receive notifications when the lamports or data for a
 - `<string>` - program_id Pubkey, as base-58 encoded string
 - `<object>` - (optional) Configuration object containing the following optional fields:
   - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment)
-  - `encoding: <string>` - encoding for Account data, either "base58" (*slow*), "base64" or "jsonParsed".
+  - `encoding: <string>` - encoding for Account data, either "base58" (*slow*), "base64", "base64+zstd" or "jsonParsed".
     "jsonParsed" encoding attempts to use program-specific state parsers to return more human-readable and explicit account state data. If "jsonParsed" is requested but a parser cannot be found, the field falls back to base64 encoding, detectable when the `data` field is type `<string>`.
   - (optional) `filters: <array>` - filter results using various [filter objects](jsonrpc-api.md#filters); account must meet all filter criteria to be included in results