feat: optimize loading order

js/packages/common/src/actions/metadata.ts

@@ -250,12 +250,12 @@ export class Metadata {
     this.data = args.data;
     this.primarySaleHappened = args.primarySaleHappened;
     this.isMutable = args.isMutable;
-    this.editionNonce = args.editionNonce;
+    this.editionNonce = args.editionNonce ?? null;
   }
 
   public async init() {
     const metadata = toPublicKey(programIds().metadata);
-    if (this.editionNonce != null) {
+    if (this.editionNonce !== null) {
       this.edition = (
         await PublicKey.createProgramAddress(
           [

js/packages/common/src/contexts/meta/isMetadataPartOfStore.ts

@@ -4,20 +4,20 @@ import { ParsedAccount } from '../accounts/types';
 
 export const isMetadataPartOfStore = (
   m: ParsedAccount<Metadata>,
-  store: ParsedAccount<Store> | null,
   whitelistedCreatorsByCreator: Record<
     string,
     ParsedAccount<WhitelistedCreator>
   >,
+  store?: ParsedAccount<Store> | null,
 ) => {
-  if (!m?.info?.data?.creators || !store?.info) {
+  if (!m?.info?.data?.creators) {
     return false;
   }
 
   return m.info.data.creators.some(
     c =>
       c.verified &&
-      (store.info.public ||
+      (store?.info.public ||
         whitelistedCreatorsByCreator[c.address]?.info?.activated),
   );
 };

js/packages/common/src/contexts/meta/loadAccounts.ts

@@ -20,12 +20,13 @@ import {
   getAuctionExtended,
 } from '../../actions';
 import { WhitelistedCreator } from '../../models/metaplex';
-import { AccountInfo, Connection, PublicKey } from '@solana/web3.js';
+import { Connection, PublicKey } from '@solana/web3.js';
 import {
   AccountAndPubkey,
   MetaState,
   ProcessAccountsFunc,
   UpdateStateValueFunc,
+  UnPromise,
 } from './types';
 import { isMetadataPartOfStore } from './isMetadataPartOfStore';
 import { processAuctions } from './processAuctions';
@@ -266,125 +267,138 @@ export const limitedLoadAccounts = async (connection: Connection) => {
 };
 
 export const loadAccounts = async (connection: Connection) => {
-  const tempCache: MetaState = getEmptyMetaState();
-  const updateTemp = makeSetter(tempCache);
-  const forEachAccount = processingAccounts(updateTemp);
+  const state: MetaState = getEmptyMetaState();
+  const updateState = makeSetter(state);
+  const forEachAccount = processingAccounts(updateState);
 
-  const pullMetadata = async (creators: AccountAndPubkey[]) => {
-    await forEachAccount(processMetaplexAccounts)(creators);
-  };
-
-  const basePromises = [
+  const loadVaults = () =>
     getProgramAccounts(connection, VAULT_ID).then(
       forEachAccount(processVaultData),
-    ),
+    );
+  const loadAuctions = () =>
     getProgramAccounts(connection, AUCTION_ID).then(
       forEachAccount(processAuctions),
-    ),
+    );
+  const loadMetaplex = () =>
     getProgramAccounts(connection, METAPLEX_ID).then(
       forEachAccount(processMetaplexAccounts),
-    ), // ???
+    );
+  const loadCreators = () =>
     getProgramAccounts(connection, METAPLEX_ID, {
       filters: [
         {
           dataSize: MAX_WHITELISTED_CREATOR_SIZE,
         },
       ],
-    }).then(pullMetadata), // ???
+    }).then(forEachAccount(processMetaplexAccounts));
+  const loadMetadata = () =>
+    pullMetadataByCreators(connection, state, updateState);
+  const loadEditions = () => pullEditions(connection, updateState, state);
+
+  const loading = [
+    loadCreators().then(loadMetadata).then(loadEditions),
+    loadVaults(),
+    loadAuctions(),
+    loadMetaplex(),
   ];
 
-  await Promise.all(basePromises);
-  const additionalPromises: Promise<void>[] = getAdditionalPromises(
-    connection,
-    Object.values(tempCache.whitelistedCreatorsByCreator),
-    forEachAccount,
-  );
+  await Promise.all(loading);
 
-  await Promise.all(additionalPromises);
+  console.log('Metadata size', state.metadata.length);
 
-  await postProcessMetadata(tempCache);
-  console.log('Metadata size', tempCache.metadata.length);
-
-  await pullEditions(connection, updateTemp, tempCache);
-
-  return tempCache;
+  return state;
 };
 
 const pullEditions = async (
   connection: Connection,
-  updateTemp: UpdateStateValueFunc,
-  tempCache: MetaState,
+  updater: UpdateStateValueFunc,
+  state: MetaState,
 ) => {
   console.log('Pulling editions for optimized metadata');
 
+  type MultipleAccounts = UnPromise<ReturnType<typeof getMultipleAccounts>>;
   let setOf100MetadataEditionKeys: string[] = [];
-  const editionPromises: Promise<{
-    keys: string[];
-    array: AccountInfo<Buffer>[];
-  }>[] = [];
+  const editionPromises: Promise<void>[] = [];
 
-  for (let i = 0; i < tempCache.metadata.length; i++) {
-    let edition: StringPublicKey;
-    if (tempCache.metadata[i].info.editionNonce != null) {
-      edition = (
-        await PublicKey.createProgramAddress(
-          [
-            Buffer.from(METADATA_PREFIX),
-            toPublicKey(METADATA_PROGRAM_ID).toBuffer(),
-            toPublicKey(tempCache.metadata[i].info.mint).toBuffer(),
-            new Uint8Array([tempCache.metadata[i].info.editionNonce || 0]),
-          ],
-          toPublicKey(METADATA_PROGRAM_ID),
-        )
-      ).toBase58();
-    } else {
-      edition = await getEdition(tempCache.metadata[i].info.mint);
-    }
-
-    setOf100MetadataEditionKeys.push(edition);
-
-    if (setOf100MetadataEditionKeys.length >= 100) {
+  const loadBatch = () => {
     editionPromises.push(
-        getMultipleAccounts(connection, setOf100MetadataEditionKeys, 'recent'),
+      getMultipleAccounts(
+        connection,
+        setOf100MetadataEditionKeys,
+        'recent',
+      ).then(processEditions),
     );
     setOf100MetadataEditionKeys = [];
-    }
-  }
+  };
 
-  if (setOf100MetadataEditionKeys.length >= 0) {
-    editionPromises.push(
-      getMultipleAccounts(connection, setOf100MetadataEditionKeys, 'recent'),
-    );
-    setOf100MetadataEditionKeys = [];
-  }
-
-  const responses = await Promise.all(editionPromises);
-  for (let i = 0; i < responses.length; i++) {
-    const returnedAccounts = responses[i];
+  const processEditions = (returnedAccounts: MultipleAccounts) => {
     for (let j = 0; j < returnedAccounts.array.length; j++) {
       processMetaData(
         {
           pubkey: returnedAccounts.keys[j],
           account: returnedAccounts.array[j],
         },
-        updateTemp,
+        updater,
       );
     }
+  };
+
+  for (const metadata of state.metadata) {
+    let editionKey: StringPublicKey;
+    if (metadata.info.editionNonce === null) {
+      editionKey = await getEdition(metadata.info.mint);
+    } else {
+      editionKey = (
+        await PublicKey.createProgramAddress(
+          [
+            Buffer.from(METADATA_PREFIX),
+            toPublicKey(METADATA_PROGRAM_ID).toBuffer(),
+            toPublicKey(metadata.info.mint).toBuffer(),
+            new Uint8Array([metadata.info.editionNonce || 0]),
+          ],
+          toPublicKey(METADATA_PROGRAM_ID),
+        )
+      ).toBase58();
     }
+
+    setOf100MetadataEditionKeys.push(editionKey);
+
+    if (setOf100MetadataEditionKeys.length >= 100) {
+      loadBatch();
+    }
+  }
+
+  if (setOf100MetadataEditionKeys.length >= 0) {
+    loadBatch();
+  }
+
+  await Promise.all(editionPromises);
+
   console.log(
     'Edition size',
-    Object.keys(tempCache.editions).length,
-    Object.keys(tempCache.masterEditions).length,
+    Object.keys(state.editions).length,
+    Object.keys(state.masterEditions).length,
   );
 };
 
-const getAdditionalPromises = (
+const pullMetadataByCreators = (
   connection: Connection,
-  whitelistedCreators: ParsedAccount<WhitelistedCreator>[],
-  forEach: ReturnType<typeof processingAccounts>,
-): Promise<void>[] => {
+  state: MetaState,
+  updater: UpdateStateValueFunc,
+): Promise<any> => {
   console.log('pulling optimized nfts');
 
+  const whitelistedCreators = Object.values(state.whitelistedCreatorsByCreator);
+
+  const setter: UpdateStateValueFunc = async (prop, key, value) => {
+    if (prop === 'metadataByMint') {
+      await initMetadata(value, state.whitelistedCreatorsByCreator, updater);
+    } else {
+      updater(prop, key, value);
+    }
+  };
+  const forEachAccount = processingAccounts(setter);
+
   const additionalPromises: Promise<void>[] = [];
   for (const creator of whitelistedCreators) {
     for (let i = 0; i < MAX_CREATOR_LIMIT; i++) {
@@ -410,44 +424,47 @@ const getAdditionalPromises = (
             },
           },
         ],
-      }).then(forEach(processMetaData));
+      }).then(forEachAccount(processMetaData));
       additionalPromises.push(promise);
     }
   }
 
-  return additionalPromises;
+  return Promise.all(additionalPromises);
 };
 
 export const makeSetter =
-  (state: MetaState) =>
-  (prop: keyof MetaState, key: string, value: ParsedAccount<any>) => {
+  (state: MetaState): UpdateStateValueFunc<MetaState> =>
+  (prop, key, value) => {
     if (prop === 'store') {
       state[prop] = value;
-    } else if (prop !== 'metadata') {
+    } else if (prop === 'metadata') {
+      state.metadata.push(value);
+    } else {
       state[prop][key] = value;
     }
     return state;
   };
 
 export const processingAccounts =
-  (updater: ReturnType<typeof makeSetter>) =>
+  (updater: UpdateStateValueFunc) =>
   (fn: ProcessAccountsFunc) =>
   async (accounts: AccountAndPubkey[]) => {
     await createPipelineExecutor(
       accounts.values(),
       account => fn(account, updater),
       {
-        sequence: 20,
+        sequence: 10,
         delay: 1,
+        jobsCount: 3,
       },
     );
   };
 
-const postProcessMetadata = async (tempCache: MetaState) => {
-  const values = Object.values(tempCache.metadataByMint);
+const postProcessMetadata = async (state: MetaState) => {
+  const values = Object.values(state.metadataByMint);
 
   for (const metadata of values) {
-    await metadataByMintUpdater(metadata, tempCache);
+    await metadataByMintUpdater(metadata, state);
   }
 };
 
@@ -456,13 +473,7 @@ export const metadataByMintUpdater = async (
   state: MetaState,
 ) => {
   const key = metadata.info.mint;
-  if (
-    isMetadataPartOfStore(
-      metadata,
-      state.store,
-      state.whitelistedCreatorsByCreator,
-    )
-  ) {
+  if (isMetadataPartOfStore(metadata, state.whitelistedCreatorsByCreator)) {
     await metadata.info.init();
     const masterEditionKey = metadata.info?.masterEdition;
     if (masterEditionKey) {
@@ -475,3 +486,19 @@ export const metadataByMintUpdater = async (
   }
   return state;
 };
+
+export const initMetadata = async (
+  metadata: ParsedAccount<Metadata>,
+  whitelistedCreators: Record<string, ParsedAccount<WhitelistedCreator>>,
+  setter: UpdateStateValueFunc,
+) => {
+  if (isMetadataPartOfStore(metadata, whitelistedCreators)) {
+    await metadata.info.init();
+    setter('metadataByMint', metadata.info.mint, metadata);
+    setter('metadata', '', metadata);
+    const masterEditionKey = metadata.info?.masterEdition;
+    if (masterEditionKey) {
+      setter('metadataByMasterEdition', masterEditionKey, metadata);
+    }
+  }
+};

js/packages/common/src/contexts/meta/processMetaData.ts

@@ -14,7 +14,7 @@ import {
 import { ParsedAccount } from '../accounts/types';
 import { METADATA_PROGRAM_ID, pubkeyToString } from '../../utils';
 
-export const processMetaData: ProcessAccountsFunc = (
+export const processMetaData: ProcessAccountsFunc = async (
   { account, pubkey },
   setter,
 ) => {
@@ -32,7 +32,7 @@ export const processMetaData: ProcessAccountsFunc = (
           account,
           info: metadata,
         };
-        setter('metadataByMint', metadata.mint, parsedAccount);
+        await setter('metadataByMint', metadata.mint, parsedAccount);
       }
     }
 

js/packages/common/src/contexts/meta/subscribeAccountsChange.ts

@@ -6,7 +6,7 @@ import {
   toPublicKey,
   VAULT_ID,
 } from '../../utils';
-import { makeSetter, metadataByMintUpdater } from './loadAccounts';
+import { makeSetter, initMetadata } from './loadAccounts';
 import { onChangeAccount } from './onChangeAccount';
 import { processAuctions } from './processAuctions';
 import { processMetaData } from './processMetaData';
@@ -52,12 +52,25 @@ export const subscribeAccountsChange = (
     connection.onProgramAccountChange(
       toPublicKey(METADATA_PROGRAM_ID),
       onChangeAccount(processMetaData, async (prop, key, value) => {
+        const state = { ...getState() };
+        const setter = makeSetter(state);
+        let hasChanges = false;
+        const updater: UpdateStateValueFunc = (...args) => {
+          hasChanges = true;
+          setter(...args);
+        };
+
         if (prop === 'metadataByMint') {
-          const state = getState();
-          const nextState = await metadataByMintUpdater(value, state);
-          setState(nextState);
+          await initMetadata(
+            value,
+            state.whitelistedCreatorsByCreator,
+            updater,
+          );
         } else {
-          updateStateValue(prop, key, value);
+          updater(prop, key, value);
+        }
+        if (hasChanges) {
+          setState(state);
         }
       }),
     ),

js/packages/common/src/contexts/meta/types.ts

@@ -90,11 +90,11 @@ export type AccountAndPubkey = {
   account: AccountInfo<Buffer>;
 };
 
-export type UpdateStateValueFunc = (
+export type UpdateStateValueFunc<T = void> = (
   prop: keyof MetaState,
   key: string,
   value: ParsedAccount<any>,
-) => void;
+) => T;
 
 export type ProcessAccountsFunc = (
   account: PublicKeyStringAndAccount<Buffer>,
@@ -102,3 +102,7 @@ export type ProcessAccountsFunc = (
 ) => void;
 
 export type CheckAccountFunc = (account: AccountInfo<Buffer>) => boolean;
+
+export type UnPromise<T extends Promise<any>> = T extends Promise<infer U>
+  ? U
+  : never;