drop freezes on processing data (#481)

* feat: endpoint from query

* chore: cleanup

* feat: processing meta by batch

* drop unused data

* fix: missed await

js/packages/common/src/contexts/connection.tsx

@@ -16,6 +16,7 @@ import {
 import React, { useContext, useEffect, useMemo, useState } from 'react';
 import { notify } from '../utils/notifications';
 import { ExplorerLink } from '../components/ExplorerLink';
+import { useQuerySearch } from '../hooks';
 import {
   TokenInfo,
   TokenListProvider,
@@ -87,10 +88,16 @@ const ConnectionContext = React.createContext<ConnectionConfig>({
 });
 
 export function ConnectionProvider({ children = undefined as any }) {
-  const [endpoint, setEndpoint] = useLocalStorageState(
+  const searchParams = useQuerySearch();
+  const network = searchParams.get('network');
+  const queryEndpoint =
+    network && ENDPOINTS.find(({ name }) => name.startsWith(network))?.endpoint;
+
+  const [savedEndpoint, setEndpoint] = useLocalStorageState(
     'connectionEndpoint',
     ENDPOINTS[0].endpoint,
   );
+  const endpoint = queryEndpoint || savedEndpoint;
 
   const connection = useMemo(
     () => new Connection(endpoint, 'recent'),

js/packages/common/src/contexts/meta/createPipelineExecutor.ts

@@ -0,0 +1,55 @@
+export async function createPipelineExecutor<T>(
+  data: IterableIterator<T>,
+  executor: (d: T) => void,
+  {
+    delay = 0,
+    jobsCount = 1,
+    sequence = 1,
+  }: {
+    delay?: number;
+    jobsCount?: number;
+    sequence?: number;
+  } = {},
+) {
+  function execute<T>(iter: IteratorResult<T, any>) {
+    // TODO: wait for async executor
+    executor(iter.value);
+  }
+
+  async function next() {
+    if (sequence <= 1) {
+      const iter = data.next();
+      if (iter.done) {
+        return;
+      }
+      await execute(iter);
+    } else {
+      const promises: any[] = [];
+      let isDone = false;
+      for (let i = 0; i < sequence; i++) {
+        const iter = data.next();
+        if (!iter.done) {
+          promises.push(execute(iter));
+        } else {
+          isDone = true;
+          break;
+        }
+      }
+      await Promise.all(promises);
+      if (isDone) {
+        return;
+      }
+    }
+    if (delay > 0) {
+      await new Promise(resolve => setTimeout(resolve, delay));
+    } else {
+      await Promise.resolve();
+    }
+    await next();
+  }
+  const result = new Array<Promise<void>>(jobsCount);
+  for (let i = 0; i < jobsCount; i++) {
+    result[i] = next();
+  }
+  await Promise.all(result);
+}

js/packages/common/src/contexts/meta/getEmptyMetaState.ts

@@ -22,6 +22,4 @@ export const getEmptyMetaState = (): MetaState => ({
   prizeTrackingTickets: {},
   safetyDepositConfigsByAuctionManagerAndIndex: {},
   bidRedemptionV2sByAuctionManagerAndWinningIndex: {},
-  stores: {},
-  creators: {},
 });

js/packages/common/src/contexts/meta/loadAccounts.ts

@@ -19,6 +19,7 @@ import {
   decodeMetadata,
   getAuctionExtended,
 } from '../../actions';
+import { WhitelistedCreator } from '../../models/metaplex';
 import { AccountInfo, Connection, PublicKey } from '@solana/web3.js';
 import {
   AccountAndPubkey,
@@ -34,6 +35,8 @@ import { processVaultData } from './processVaultData';
 import { ParsedAccount } from '../accounts/types';
 import { getEmptyMetaState } from './getEmptyMetaState';
 import { getMultipleAccounts } from '../accounts/getMultipleAccounts';
+import { getProgramAccounts } from './web3';
+import { createPipelineExecutor } from './createPipelineExecutor';
 
 export const USE_SPEED_RUN = false;
 const WHITELISTED_METADATA = ['98vYFjBYS9TguUMWQRPjy2SZuxKuUMcqR4vnQiLjZbte'];
@@ -52,60 +55,6 @@ const WHITELISTED_AUCTION_MANAGER = [
 ];
 const WHITELISTED_VAULT = ['3wHCBd3fYRPWjd5GqzrXanLJUKRyU3nECKbTPKfVwcFX'];
 
-async function getProgramAccounts(
-  connection: Connection,
-  programId: StringPublicKey,
-  configOrCommitment?: any,
-): Promise<Array<AccountAndPubkey>> {
-  const extra: any = {};
-  let commitment;
-  //let encoding;
-
-  if (configOrCommitment) {
-    if (typeof configOrCommitment === 'string') {
-      commitment = configOrCommitment;
-    } else {
-      commitment = configOrCommitment.commitment;
-      //encoding = configOrCommitment.encoding;
-
-      if (configOrCommitment.dataSlice) {
-        extra.dataSlice = configOrCommitment.dataSlice;
-      }
-
-      if (configOrCommitment.filters) {
-        extra.filters = configOrCommitment.filters;
-      }
-    }
-  }
-
-  const args = connection._buildArgs([programId], commitment, 'base64', extra);
-  const unsafeRes = await (connection as any)._rpcRequest(
-    'getProgramAccounts',
-    args,
-  );
-
-  const data = (
-    unsafeRes.result as Array<{
-      account: AccountInfo<[string, string]>;
-      pubkey: string;
-    }>
-  ).map(item => {
-    return {
-      account: {
-        // TODO: possible delay parsing could be added here
-        data: Buffer.from(item.account.data[0], 'base64'),
-        executable: item.account.executable,
-        lamports: item.account.lamports,
-        // TODO: maybe we can do it in lazy way? or just use string
-        owner: item.account.owner,
-      } as AccountInfo<Buffer>,
-      pubkey: item.pubkey,
-    };
-  });
-
-  return data;
-}
-
 export const limitedLoadAccounts = async (connection: Connection) => {
   const tempCache: MetaState = getEmptyMetaState();
   const updateTemp = makeSetter(tempCache);
@@ -321,40 +270,39 @@ export const limitedLoadAccounts = async (connection: Connection) => {
   return tempCache;
 };
 
-export const loadAccounts = async (connection: Connection, all: boolean) => {
+export const loadAccounts = async (connection: Connection, all = false) => {
   const tempCache: MetaState = getEmptyMetaState();
   const updateTemp = makeSetter(tempCache);
-
-  const forEach =
-    (fn: ProcessAccountsFunc) => async (accounts: AccountAndPubkey[]) => {
-      for (const account of accounts) {
-        await fn(account, updateTemp, all);
-      }
-    };
+  const forEachAccount = processingAccounts(updateTemp, all);
 
   const pullMetadata = async (creators: AccountAndPubkey[]) => {
-    await forEach(processMetaplexAccounts)(creators);
+    await forEachAccount(processMetaplexAccounts)(creators);
   };
 
   const basePromises = [
-    getProgramAccounts(connection, VAULT_ID).then(forEach(processVaultData)),
-    getProgramAccounts(connection, AUCTION_ID).then(forEach(processAuctions)),
-    getProgramAccounts(connection, METAPLEX_ID).then(
-      forEach(processMetaplexAccounts),
+    getProgramAccounts(connection, VAULT_ID).then(
+      forEachAccount(processVaultData),
     ),
+    getProgramAccounts(connection, AUCTION_ID).then(
+      forEachAccount(processAuctions),
+    ),
+    getProgramAccounts(connection, METAPLEX_ID).then(
+      forEachAccount(processMetaplexAccounts),
+    ), // ???
     getProgramAccounts(connection, METAPLEX_ID, {
       filters: [
         {
           dataSize: MAX_WHITELISTED_CREATOR_SIZE,
         },
       ],
-    }).then(pullMetadata),
+    }).then(pullMetadata), // ???
   ];
+
   await Promise.all(basePromises);
   const additionalPromises: Promise<void>[] = getAdditionalPromises(
     connection,
-    tempCache,
-    forEach,
+    Object.values(tempCache.whitelistedCreatorsByCreator),
+    forEachAccount,
   );
 
   await Promise.all(additionalPromises);
@@ -374,6 +322,7 @@ const pullEditions = async (
   all: boolean,
 ) => {
   console.log('Pulling editions for optimized metadata');
+
   let setOf100MetadataEditionKeys: string[] = [];
   const editionPromises: Promise<{
     keys: string[];
@@ -435,43 +384,41 @@ const pullEditions = async (
     Object.keys(tempCache.masterEditions).length,
   );
 };
+
 const getAdditionalPromises = (
   connection: Connection,
-  tempCache: MetaState,
-  forEach: any,
+  whitelistedCreators: ParsedAccount<WhitelistedCreator>[],
+  forEach: ReturnType<typeof processingAccounts>,
 ): Promise<void>[] => {
   console.log('pulling optimized nfts');
-  const whitelistedCreators = Object.values(
-    tempCache.whitelistedCreatorsByCreator,
-  );
+
   const additionalPromises: Promise<void>[] = [];
-  for (let i = 0; i < MAX_CREATOR_LIMIT; i++) {
-    for (let j = 0; j < whitelistedCreators.length; j++) {
-      additionalPromises.push(
-        getProgramAccounts(connection, METADATA_PROGRAM_ID, {
-          filters: [
-            {
-              memcmp: {
-                offset:
-                  1 + // key
-                  32 + // update auth
-                  32 + // mint
-                  4 + // name string length
-                  MAX_NAME_LENGTH + // name
-                  4 + // uri string length
-                  MAX_URI_LENGTH + // uri
-                  4 + // symbol string length
-                  MAX_SYMBOL_LENGTH + // symbol
-                  2 + // seller fee basis points
-                  1 + // whether or not there is a creators vec
-                  4 + // creators vec length
-                  i * MAX_CREATOR_LEN,
-                bytes: whitelistedCreators[j].info.address,
-              },
+  for (const creator of whitelistedCreators) {
+    for (let i = 0; i < MAX_CREATOR_LIMIT; i++) {
+      const promise = getProgramAccounts(connection, METADATA_PROGRAM_ID, {
+        filters: [
+          {
+            memcmp: {
+              offset:
+                1 + // key
+                32 + // update auth
+                32 + // mint
+                4 + // name string length
+                MAX_NAME_LENGTH + // name
+                4 + // uri string length
+                MAX_URI_LENGTH + // uri
+                4 + // symbol string length
+                MAX_SYMBOL_LENGTH + // symbol
+                2 + // seller fee basis points
+                1 + // whether or not there is a creators vec
+                4 + // creators vec length
+                i * MAX_CREATOR_LEN,
+              bytes: creator.info.address,
             },
-          ],
-        }).then(forEach(processMetaData)),
-      );
+          },
+        ],
+      }).then(forEach(processMetaData));
+      additionalPromises.push(promise);
     }
   }
 
@@ -489,7 +436,21 @@ export const makeSetter =
     return state;
   };
 
-const postProcessMetadata = async (tempCache: MetaState, all: boolean) => {
+export const processingAccounts =
+  (updater: ReturnType<typeof makeSetter>, all = false) =>
+  (fn: ProcessAccountsFunc) =>
+  async (accounts: AccountAndPubkey[]) => {
+    await createPipelineExecutor(
+      accounts.values(),
+      account => fn(account, updater, all),
+      {
+        sequence: 20,
+        delay: 1,
+      },
+    );
+  };
+
+const postProcessMetadata = async (tempCache: MetaState, all = false) => {
   const values = Object.values(tempCache.metadataByMint);
 
   for (const metadata of values) {

js/packages/common/src/contexts/meta/processAuctions.ts

@@ -11,7 +11,7 @@ import {
   BIDDER_POT_LEN,
   MAX_AUCTION_DATA_EXTENDED_SIZE,
 } from '../../actions';
-import { AUCTION_ID } from '../../utils';
+import { AUCTION_ID, pubkeyToString } from '../../utils';
 import { ParsedAccount } from '../accounts';
 import { cache } from '../accounts';
 import { CheckAccountFunc, ProcessAccountsFunc } from './types';
@@ -92,7 +92,7 @@ export const processAuctions: ProcessAccountsFunc = (
 };
 
 const isAuctionAccount: CheckAccountFunc = account =>
-  (account.owner as unknown as any) === AUCTION_ID;
+  pubkeyToString(account.owner) === AUCTION_ID;
 
 const isExtendedAuctionAccount: CheckAccountFunc = account =>
   account.data.length === MAX_AUCTION_DATA_EXTENDED_SIZE;

js/packages/common/src/contexts/meta/processMetaData.ts

@@ -12,7 +12,7 @@ import {
   MetadataKey,
 } from '../../actions';
 import { ParsedAccount } from '../accounts/types';
-import { METADATA_PROGRAM_ID } from '../../utils';
+import { METADATA_PROGRAM_ID, pubkeyToString } from '../../utils';
 
 export const processMetaData: ProcessAccountsFunc = (
   { account, pubkey },
@@ -83,9 +83,8 @@ export const processMetaData: ProcessAccountsFunc = (
   }
 };
 
-const isMetadataAccount = (account: AccountInfo<Buffer>) => {
-  return (account.owner as unknown as any) === METADATA_PROGRAM_ID;
-};
+const isMetadataAccount = (account: AccountInfo<Buffer>) =>
+  pubkeyToString(account.owner) === METADATA_PROGRAM_ID;
 
 const isMetadataV1Account = (account: AccountInfo<Buffer>) =>
   account.data[0] === MetadataKey.MetadataV1;

js/packages/common/src/contexts/meta/processMetaplexAccounts.ts

@@ -20,7 +20,7 @@ import {
   SafetyDepositConfig,
 } from '../../models';
 import { ProcessAccountsFunc } from './types';
-import { METAPLEX_ID, programIds } from '../../utils';
+import { METAPLEX_ID, programIds, pubkeyToString } from '../../utils';
 import { ParsedAccount } from '../accounts';
 import { cache } from '../accounts';
 
@@ -112,7 +112,6 @@ export const processMetaplexAccounts: ProcessAccountsFunc = async (
       if (STORE_ID && pubkey === STORE_ID.toBase58()) {
         setter('store', pubkey, parsedAccount);
       }
-      setter('stores', pubkey, parsedAccount);
     }
 
     if (isSafetyDepositConfigV1Account(account)) {
@@ -152,14 +151,6 @@ export const processMetaplexAccounts: ProcessAccountsFunc = async (
           );
         }
       }
-
-      if (useAll) {
-        setter(
-          'creators',
-          parsedAccount.info.address + '-' + pubkey,
-          parsedAccount,
-        );
-      }
     }
   } catch {
     // ignore errors
@@ -168,7 +159,7 @@ export const processMetaplexAccounts: ProcessAccountsFunc = async (
 };
 
 const isMetaplexAccount = (account: AccountInfo<Buffer>) =>
-  (account.owner as unknown as any) === METAPLEX_ID;
+  pubkeyToString(account.owner) === METAPLEX_ID;
 
 const isAuctionManagerV1Account = (account: AccountInfo<Buffer>) =>
   account.data[0] === MetaplexKey.AuctionManagerV1;

js/packages/common/src/contexts/meta/processVaultData.ts

@@ -6,7 +6,7 @@ import {
   Vault,
   VaultKey,
 } from '../../actions';
-import { VAULT_ID } from '../../utils';
+import { VAULT_ID, pubkeyToString } from '../../utils';
 import { ParsedAccount } from '../accounts/types';
 import { ProcessAccountsFunc } from './types';
 
@@ -47,7 +47,7 @@ export const processVaultData: ProcessAccountsFunc = (
 };
 
 const isVaultAccount = (account: AccountInfo<Buffer>) =>
-  (account.owner as unknown as any) === VAULT_ID;
+  pubkeyToString(account.owner) === VAULT_ID;
 
 const isSafetyDepositBoxV1Account = (account: AccountInfo<Buffer>) =>
   account.data[0] === VaultKey.SafetyDepositBoxV1;

js/packages/common/src/contexts/meta/types.ts

@@ -71,8 +71,6 @@ export interface MetaState {
     ParsedAccount<WhitelistedCreator>
   >;
   payoutTickets: Record<string, ParsedAccount<PayoutTicket>>;
-  stores: Record<string, ParsedAccount<Store>>;
-  creators: Record<string, ParsedAccount<WhitelistedCreator>>;
 }
 
 export interface MetaContextState extends MetaState {

js/packages/common/src/contexts/meta/web3.ts

@@ -0,0 +1,61 @@
+import { AccountInfo, Connection } from '@solana/web3.js';
+import { StringPublicKey } from '../../utils/ids';
+import { AccountAndPubkey } from './types';
+
+export async function getProgramAccounts(
+  connection: Connection,
+  programId: StringPublicKey,
+  configOrCommitment?: any,
+): Promise<Array<AccountAndPubkey>> {
+  const extra: any = {};
+  let commitment;
+  //let encoding;
+
+  if (configOrCommitment) {
+    if (typeof configOrCommitment === 'string') {
+      commitment = configOrCommitment;
+    } else {
+      commitment = configOrCommitment.commitment;
+      //encoding = configOrCommitment.encoding;
+
+      if (configOrCommitment.dataSlice) {
+        extra.dataSlice = configOrCommitment.dataSlice;
+      }
+
+      if (configOrCommitment.filters) {
+        extra.filters = configOrCommitment.filters;
+      }
+    }
+  }
+
+  const args = connection._buildArgs([programId], commitment, 'base64', extra);
+  const unsafeRes = await (connection as any)._rpcRequest(
+    'getProgramAccounts',
+    args,
+  );
+
+  return unsafeResAccounts(unsafeRes.result);
+}
+
+export function unsafeAccount(account: AccountInfo<[string, string]>) {
+  return {
+    // TODO: possible delay parsing could be added here
+    data: Buffer.from(account.data[0], 'base64'),
+    executable: account.executable,
+    lamports: account.lamports,
+    // TODO: maybe we can do it in lazy way? or just use string
+    owner: account.owner,
+  } as AccountInfo<Buffer>;
+}
+
+export function unsafeResAccounts(
+  data: Array<{
+    account: AccountInfo<[string, string]>;
+    pubkey: string;
+  }>,
+) {
+  return data.map(item => ({
+    account: unsafeAccount(item.account),
+    pubkey: item.pubkey,
+  }));
+}