From 8f29e2bb1ec4439098ec51ce4c79cca4c051c17a Mon Sep 17 00:00:00 2001 From: aniketfuryrocks Date: Mon, 3 Jul 2023 05:52:14 +0530 Subject: [PATCH] size fixed --- configure/general/accounts.ts | 117 +++++++++++++----------- configure/main.ts | 35 +------ configure/output_file.ts | 22 ++--- run_bench.sh | 59 ++++++++++++ src/rpc_client.rs | 16 ++-- src/solana_runtime/accounts_fetching.rs | 39 ++------ 6 files changed, 155 insertions(+), 133 deletions(-) create mode 100755 run_bench.sh diff --git a/configure/general/accounts.ts b/configure/general/accounts.ts index 50a2ac2..36e36ac 100644 --- a/configure/general/accounts.ts +++ b/configure/general/accounts.ts @@ -1,62 +1,77 @@ import { - Connection, - Keypair, - SystemProgram, - PublicKey, - Transaction, - sendAndConfirmTransaction, + Connection, + Keypair, + SystemProgram, + PublicKey, + Transaction, + sendAndConfirmTransaction, } from "@solana/web3.js"; -export async function configure_accounts( - connection: Connection, - authority: Keypair, - count: number, - programs: PublicKey[] -): Promise { - let all_accounts: PublicKey[] = []; - // create accounts in batches of 16 - for (let i = 0; i < count; i += 16) { - let end = Math.min(i + 16, count); - let nbOfAccs = end - i; - let accounts = await Promise.all( - Array.from(Array(nbOfAccs).keys()).map(async (_) => { - let size = Math.random() * 10_000_000; - if (size < 100) { - size = 100; +export class AccountGenerator { + private connection: Connection; + private feePayer: Keypair; + + public STABLE_SIZE = 7340032; // 7 MB + + constructor(connection: Connection, feePayer: Keypair) { + this.connection = connection; + this.feePayer = feePayer; + } + + + async find_max_size_fetchable(): Promise { + // create Solana accounts till a size of 5 mega bytes + let good_size = 0; + + for (let i = 1; i < 10; i++) { + const size = i * 1024 * 1024; + + const account = await this.createSolanaAccount(size); + // get account info + try { + const accountInfo = await this.connection.getAccountInfo(account.publicKey); + good_size = size; + + console.log(`account size possible ${accountInfo?.data.length} Bytes`); + } catch (err) { + console.log(`maximum possible size is ${i - 1} MB or ${good_size} Bytes`, err); + break; + } } - size = Math.floor(size); - const lamports = await connection.getMinimumBalanceForRentExemption( - size - ); - let kp = Keypair.generate(); - const program = programs[Math.floor(Math.random() * programs.length)]; + return good_size; + } + async generate_fetchable_accounts(amount: number): Promise { + + return await Promise.all(Array.from(Array(amount).keys()).map(async (i) => { + const size = this.STABLE_SIZE + (i * 1024); // add a KB to each account + return await this.createSolanaAccount(size); + })); + } + + async createSolanaAccount(space: number): Promise { + // Generate a new keypair for the account + const accountKeyPair = Keypair.generate(); + + // Fetch the minimum required balance for creating an account + const minimumBalance = await this.connection.getMinimumBalanceForRentExemption(space); + + // Build the transaction to create the account const transaction = new Transaction().add( - SystemProgram.createAccount({ - fromPubkey: authority.publicKey, - newAccountPubkey: kp.publicKey, - lamports, - space: size, - programId: program, - }) + SystemProgram.createAccount({ + fromPubkey: this.feePayer.publicKey, + newAccountPubkey: accountKeyPair.publicKey, + lamports: minimumBalance, + space, + programId: new PublicKey('11111111111111111111111111111111'), // Replace with the desired program ID + }) ); - transaction.feePayer = authority.publicKey; - let hash = await connection.getRecentBlockhash(); - transaction.recentBlockhash = hash.blockhash; - // Sign transaction, broadcast, and confirm - await sendAndConfirmTransaction( - connection, - transaction, - [authority, kp], - { commitment: "confirmed" } - ); + // send and confirm transaction + await sendAndConfirmTransaction(this.connection, transaction, [this.feePayer, accountKeyPair]); - return kp.publicKey; - }) - ); - all_accounts = all_accounts.concat(accounts); - } - return all_accounts; + return accountKeyPair; + } } + diff --git a/configure/main.ts b/configure/main.ts index 1cac989..7e2fb66 100644 --- a/configure/main.ts +++ b/configure/main.ts @@ -12,7 +12,7 @@ import { import { getKeypairFromFile } from "./common_utils"; import { deploy_programs } from "./deploy_programs"; import { User, createUser, mintUser } from "./general/create_users"; -import { configure_accounts } from "./general/accounts"; +import { AccountGenerator } from "./general/accounts"; import { Command, OutputFile } from "./output_file"; import { MintUtils } from "./general/mint_utils"; import { OpenbookConfigurator } from "./openbook-v2/configure_openbook"; @@ -257,42 +257,15 @@ async function configure( } console.log("Creating accounts"); - let accounts = await configure_accounts( - connection, - authority, - numberOfAccountsToBeCreated, - programIds - ); - // adding known accounts - const marketAccountsList = markets - .map((market) => [ - market.asks, - market.bids, - market.market_pk, - market.oracle, - market.quote_vault, - market.base_vault, - market.base_mint, - market.quote_mint, - ]) - .flat(); - const userAccountsList = userData - .map((user) => { - const allOpenOrdersAccounts = user.open_orders - .map((x) => x.open_orders) - .flat(); - const allTokenAccounts = user.token_data.map((x) => x.token_account); - return allOpenOrdersAccounts.concat(allTokenAccounts); - }) - .flat(); - accounts = accounts.concat(marketAccountsList).concat(userAccountsList); + const accounts = await (new AccountGenerator(connection, authority)).generate_fetchable_accounts(numberOfAccountsToBeCreated); + const known_accounts = accounts.map((x) => x.publicKey); console.log("Accounts created"); let outputFile: OutputFile = { programs: programOutputData, - known_accounts: accounts, + known_accounts, users: userData, mints, markets, diff --git a/configure/output_file.ts b/configure/output_file.ts index 83a21d3..82be889 100644 --- a/configure/output_file.ts +++ b/configure/output_file.ts @@ -3,21 +3,21 @@ import { Market } from "./openbook-v2/create_markets"; import { User } from "./general/create_users"; export interface Command { - name: String; - instruction: number[]; - argument_sizes: number[]; + name: String; + instruction: number[]; + argument_sizes: number[]; } export interface ProgramOutputData { - name: String; - program_id: PublicKey; - commands: Command[]; + name: String; + program_id: PublicKey; + commands: Command[]; } export interface OutputFile { - users: User[]; - programs: ProgramOutputData[]; - known_accounts: PublicKey[]; - mints: PublicKey[]; - markets: Market[]; + users: User[]; + programs: ProgramOutputData[]; + known_accounts: PublicKey[]; + mints: PublicKey[]; + markets: Market[]; } diff --git a/run_bench.sh b/run_bench.sh new file mode 100755 index 0000000..3270273 --- /dev/null +++ b/run_bench.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +# exit on error +set -e + +# directory and url +directory="metrics/$1" +url="$2" + +# write a function to print error message in red color and exit +function error_exit() { + echo -e "\e[31m$1\e[0m" + exit 1 +} + +# write a function to print message in green color +function info() { + echo -e "\e[32m$1\e[0m" +} + +# exit error if $1 is not provided +if [ -z "$1" ]; then + echo "Error: Please provide a directory name" + exit 1 +fi + +# set url to localhost if not provided +if [ -z "$url" ]; then + url="http://localhost:8899" +fi + +# Fetch hard limit +hard_limit=$(ulimit -Hn) + +# Soft limit 9/10th of hard limit +soft_limit=$((hard_limit * 9 / 10)) +# set +ulimit -Sn "$soft_limit" + +info "Hard limit: $hard_limit" +info "Soft limit: $(ulimit -Sn)" + +# check for directory +if [ ! -d "$directoy" ]; then + info "Creating directory: $directory" + mkdir -p "$directory" +else + error_exit "Directory already exists: $directory" +fi + +# airdrop +info "Airdropping..." +solana airdrop 100000 -u "$url" +# configure +info "Configuring..." +yarn configure -u "$url" +# run bench +info "Running bench..." +./bench-rpc-by-threads.sh "$directory" -r "$url" diff --git a/src/rpc_client.rs b/src/rpc_client.rs index 5c00214..8cf30df 100644 --- a/src/rpc_client.rs +++ b/src/rpc_client.rs @@ -4,9 +4,7 @@ use bytes::Bytes; use reqwest::header::CONTENT_TYPE; use serde_json::{json, Value}; use solana_client::rpc_request::RpcRequest; -use solana_program::pubkey::Pubkey; -use solana_rpc_client::nonblocking::rpc_client::RpcClient; -use solana_rpc_client::rpc_client::SerializableTransaction; +use solana_rpc_client::{nonblocking::rpc_client::RpcClient, rpc_client::SerializableTransaction}; use crate::bencher::Run; @@ -58,13 +56,13 @@ impl CustomRpcClient { self.send(RpcRequest::GetBlock, json! {[slot.into()]}).await } - pub async fn raw_get_multiple_accounts(&mut self, accounts: Vec) { - let accounts: Vec = accounts - .into_iter() - .map(|pubkey| pubkey.to_string()) - .collect(); + //pub async fn raw_get_multiple_accounts(&mut self, accounts: &[&str]) { + // self.send(RpcRequest::GetMultipleAccounts, json!([accounts])) + // .await + //} - self.send(RpcRequest::GetMultipleAccounts, json!([accounts])) + pub async fn raw_get_account_info(&mut self, account: &str) { + self.send(RpcRequest::GetAccountInfo, json!([account, { "encoding" : "base64" }])) .await } diff --git a/src/solana_runtime/accounts_fetching.rs b/src/solana_runtime/accounts_fetching.rs index c8d64e1..3820158 100644 --- a/src/solana_runtime/accounts_fetching.rs +++ b/src/solana_runtime/accounts_fetching.rs @@ -1,8 +1,6 @@ use async_trait::async_trait; -use const_env::from_env; use rand::{seq::IteratorRandom, SeedableRng}; -use solana_sdk::{pubkey::Pubkey, signature::Keypair, signer::Signer}; -use std::{str::FromStr, sync::Arc}; +use std::sync::Arc; use tokio::time::Instant; use crate::{ @@ -13,36 +11,19 @@ use crate::{ test_registry::TestingTask, }; -#[from_env] -const NB_OF_ACCOUNTS_FETCHED_PER_TASK: usize = 100; - pub struct AccountsFetchingTests; -impl AccountsFetchingTests { - pub fn create_random_address(count: usize) -> Vec { - (0..count).map(|_| Keypair::new().pubkey()).collect() - } -} - #[async_trait] impl TestingTask for AccountsFetchingTests { async fn test(&self, args: &Args, config: &Config) -> anyhow::Result { - let accounts = config - .known_accounts - .iter() - .map(|x| Pubkey::from_str(x.as_str()).unwrap()) - .collect::>(); - let unknown_accounts: Vec = - AccountsFetchingTests::create_random_address(accounts.len()); - let instant = GetAccountsBench { - accounts_list: Arc::new([accounts, unknown_accounts].concat()), + accounts_list: Arc::new(config.known_accounts.clone()), }; + let metric = Bencher::bench::(instant, args).await?; Ok(metric) } - fn get_name(&self) -> String { "Accounts Fetching".to_string() } @@ -50,7 +31,7 @@ impl TestingTask for AccountsFetchingTests { #[derive(Clone)] pub struct GetAccountsBench { - accounts_list: Arc>, + accounts_list: Arc>, } #[async_trait::async_trait] @@ -62,18 +43,14 @@ impl Benchmark for GetAccountsBench { random_number: u64, ) -> anyhow::Result<()> { let mut rng = rand_chacha::ChaCha8Rng::seed_from_u64(random_number); - let number_of_fetched_accounts = - NB_OF_ACCOUNTS_FETCHED_PER_TASK.min(self.accounts_list.len()); + let start = Instant::now(); while start.elapsed() < duration { - let accounts = self - .accounts_list - .iter() - .copied() - .choose_multiple(&mut rng, number_of_fetched_accounts); + // get single random account from accounts_list + let account = self.accounts_list.iter().choose(&mut rng).unwrap(); - rpc_client.raw_get_multiple_accounts(accounts).await + rpc_client.raw_get_account_info(account).await } Ok(())