Load multiple markets simultaneously during startup (#16)

* load multiple markets at once. For large amounts of markets lots and lots of getAccountInfo's are being called simultaneously. Condense this into smaller GMA calls.

The associated account tokens are passed to the dex instructions but as it happens they don't actually need to be created for the market to be cranked. We can just get the address without calling the rpc to check if they exist or create them.

before: each market had to call getAccountInfo to get the account Info then 2 more calls to getAccountInfo to get the decimals for the baseMint and QuoteMint - even if we had already fetched these baseMints and quoteMints we would keep re-fetching them for each market that had them. There was also a further call to getAccountInfo to check if the associated token account existed for each baseMint token which would then create the account if it does not exist.

At least in my test environment, loading many markets was causing me RPC errors due to the spraying of all these getAccountInfo calls at the same time. This would happen a relatively small number of markets (around 25) but now with these changes I am able to load up to 325 markets at once (still testing limits)

---------

Co-authored-by: solpkr <solpkr1@gmail.com>

src/scripts/crank.ts

@@ -13,13 +13,9 @@ import {
   BlockhashWithExpiryBlockHeight,
   TransactionInstruction,
 } from '@solana/web3.js';
-import { getMultipleAccounts, sleep, chunk } from '../utils/utils';
+import {getMultipleAccounts,loadMultipleOpenbookMarkets,getMultipleAssociatedTokenAddresses,  sleep,  chunk} from '../utils/utils';
 import BN from 'bn.js';
-import {
-  decodeEventQueue,
-  DexInstructions,
-  Market,
-} from '@project-serum/serum';
+import {decodeEventQueue, DexInstructions} from '@project-serum/serum';
 import { Token, TOKEN_PROGRAM_ID } from '@solana/spl-token';
 import { Logger } from 'tslog';
 import axios  from "axios"
@@ -125,20 +121,9 @@ async function run() {
   } else {
     marketsList = markets[cluster];
   }
+
   // load selected markets
-  const spotMarkets = await Promise.all(
-    marketsList.map((m) => {
-      return Market.load(
-        connection,
-        new PublicKey(m.address),
-        {
-          skipPreflight: true,
-          commitment: 'processed' as Commitment,
-        },
-        serumProgramId,
-      );
-    }),
-  );
+  let spotMarkets = await loadMultipleOpenbookMarkets(connection,serumProgramId,marketsList);
 
   log.info("Cranking the following markets");
   marketsList.forEach(m => log.info(`${m.name}: ${m.address}`));
@@ -153,20 +138,11 @@ async function run() {
     .getOrCreateAssociatedAccountInfo(payer.publicKey)
     .then((a) => a.address);
 
-  const baseWallets = await Promise.all(
-    spotMarkets.map((m) => {
-      const token = new Token(
-        connection,
-        m.baseMintAddress,
-        TOKEN_PROGRAM_ID,
-        payer,
-      );
-
-      return token
-        .getOrCreateAssociatedAccountInfo(payer.publicKey)
-        .then((a) => a.address);
-    }),
+  //get the token accounts for each basemint & the associated account address for our token account
+  const tokenAccounts = spotMarkets.map(
+    (market) => market['_decoded'].baseMint
   );
+  const baseWallets = await getMultipleAssociatedTokenAddresses(connection,payer,tokenAccounts);
 
   const eventQueuePks = spotMarkets.map(
     (market) => market['_decoded'].eventQueue,

src/utils/utils.ts

@@ -6,6 +6,9 @@ import {
 } from '@solana/web3.js';
 
 import * as fzstd from 'fzstd';
+import {Market} from "@project-serum/serum";
+import * as buffer_layout from "buffer-layout";
+import { Token, TOKEN_PROGRAM_ID } from '@solana/spl-token';
 
 export async function sleep(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
@@ -66,4 +69,66 @@ export async function getMultipleAccounts(
     );
   }))).flat();
 
+}
+
+//load multiple markets at once instead of calling getAccountInfo for each market 3 times
+//by default it is 1 call to get the market and 2 calls to get the decimals for baseMint and quoteMint
+//this can be condensed into 2 calls total per 100 markets
+export async function loadMultipleOpenbookMarkets(connection,programId,marketsList){
+
+  let marketsMap = new Map();
+  let decimalMap = new Map();
+  let uniqueMints = new Set();
+
+  //get all the market data for an openbook market
+  let pubKeys = marketsList.map((item) => new PublicKey(item.address));
+  let marketsAccountInfos = await getMultipleAccounts(connection, pubKeys, 'processed');
+  marketsAccountInfos.forEach(function (result) {
+    let layout = Market.getLayout(programId);
+    let decoded = layout.decode(result.accountInfo.data);
+    uniqueMints.add(decoded.baseMint);
+    uniqueMints.add(decoded.quoteMint);
+    marketsMap.set(result.publicKey.toString(), {
+      decoded: decoded,
+      baseMint: decoded.baseMint,
+      quoteMint: decoded.quoteMint,
+      programId: programId
+    });
+  });
+
+  //get all the token's decimal values
+  const MINT_LAYOUT = buffer_layout.struct([buffer_layout.blob(44), buffer_layout.u8('decimals'), buffer_layout.blob(37)]);
+  let uniqueMintsPubKeys: any[] = Array.from(uniqueMints);
+  let uniqueMintsAccountInfos = await getMultipleAccounts(connection, uniqueMintsPubKeys, 'processed');
+  uniqueMintsAccountInfos.forEach(function (result) {
+    const {decimals} = MINT_LAYOUT.decode(result.accountInfo.data);
+    decimalMap.set(result.publicKey.toString(), decimals);
+  });
+
+  //loop back through the markets and load the market with the decoded data and the base/quote decimals
+  let spotMarkets: Market[] = [];
+  marketsMap.forEach(function (market) {
+    let baseMint = market.baseMint.toString();
+    let quoteMint = market.quoteMint.toString();
+    let openbookMarket = new Market(market.decoded, decimalMap.get(baseMint), decimalMap.get(quoteMint), {}, programId, null);
+    spotMarkets.push(openbookMarket);
+  });
+
+  return spotMarkets;
+}
+
+//get the associated accounts but don't check if they exist.
+//connection is passed to constructor but no RPC calls are made.
+export async function getMultipleAssociatedTokenAddresses(connection,owner,tokenAccounts){
+
+  //token.associatedProgramId & token.programId will be the same for each token
+  const token = new Token(connection, tokenAccounts[0].toString(), TOKEN_PROGRAM_ID, owner);
+  let associatedAccounts: PublicKey[] = [];
+
+  for (const tokenAccount of tokenAccounts) {
+    const associatedAddress = await Token.getAssociatedTokenAddress(token.associatedProgramId, token.programId, tokenAccount, owner.publicKey);
+    associatedAccounts.push(associatedAddress);
+  }
+
+  return associatedAccounts;
 }