From 314c4ec54332f4cc12e2d067f2fa1e2ed927099d Mon Sep 17 00:00:00 2001 From: exromany Date: Fri, 24 Sep 2021 23:25:06 +0300 Subject: [PATCH] feat: optimize loading order --- js/packages/common/src/actions/metadata.ts | 4 +- .../contexts/meta/isMetadataPartOfStore.ts | 6 +- .../common/src/contexts/meta/loadAccounts.ts | 207 ++++++++++-------- .../src/contexts/meta/processMetaData.ts | 4 +- .../contexts/meta/subscribeAccountsChange.ts | 23 +- js/packages/common/src/contexts/meta/types.ts | 8 +- 6 files changed, 148 insertions(+), 104 deletions(-) diff --git a/js/packages/common/src/actions/metadata.ts b/js/packages/common/src/actions/metadata.ts index 642e974..7fb2b51 100644 --- a/js/packages/common/src/actions/metadata.ts +++ b/js/packages/common/src/actions/metadata.ts @@ -250,12 +250,12 @@ export class Metadata { this.data = args.data; this.primarySaleHappened = args.primarySaleHappened; this.isMutable = args.isMutable; - this.editionNonce = args.editionNonce; + this.editionNonce = args.editionNonce ?? null; } public async init() { const metadata = toPublicKey(programIds().metadata); - if (this.editionNonce != null) { + if (this.editionNonce !== null) { this.edition = ( await PublicKey.createProgramAddress( [ diff --git a/js/packages/common/src/contexts/meta/isMetadataPartOfStore.ts b/js/packages/common/src/contexts/meta/isMetadataPartOfStore.ts index 2bcf158..455cb18 100644 --- a/js/packages/common/src/contexts/meta/isMetadataPartOfStore.ts +++ b/js/packages/common/src/contexts/meta/isMetadataPartOfStore.ts @@ -4,20 +4,20 @@ import { ParsedAccount } from '../accounts/types'; export const isMetadataPartOfStore = ( m: ParsedAccount, - store: ParsedAccount | null, whitelistedCreatorsByCreator: Record< string, ParsedAccount >, + store?: ParsedAccount | null, ) => { - if (!m?.info?.data?.creators || !store?.info) { + if (!m?.info?.data?.creators) { return false; } return m.info.data.creators.some( c => c.verified && - (store.info.public || + (store?.info.public || whitelistedCreatorsByCreator[c.address]?.info?.activated), ); }; diff --git a/js/packages/common/src/contexts/meta/loadAccounts.ts b/js/packages/common/src/contexts/meta/loadAccounts.ts index 8f0ea88..0954f51 100644 --- a/js/packages/common/src/contexts/meta/loadAccounts.ts +++ b/js/packages/common/src/contexts/meta/loadAccounts.ts @@ -20,12 +20,13 @@ import { getAuctionExtended, } from '../../actions'; import { WhitelistedCreator } from '../../models/metaplex'; -import { AccountInfo, Connection, PublicKey } from '@solana/web3.js'; +import { Connection, PublicKey } from '@solana/web3.js'; import { AccountAndPubkey, MetaState, ProcessAccountsFunc, UpdateStateValueFunc, + UnPromise, } from './types'; import { isMetadataPartOfStore } from './isMetadataPartOfStore'; import { processAuctions } from './processAuctions'; @@ -266,125 +267,138 @@ export const limitedLoadAccounts = async (connection: Connection) => { }; export const loadAccounts = async (connection: Connection) => { - const tempCache: MetaState = getEmptyMetaState(); - const updateTemp = makeSetter(tempCache); - const forEachAccount = processingAccounts(updateTemp); + const state: MetaState = getEmptyMetaState(); + const updateState = makeSetter(state); + const forEachAccount = processingAccounts(updateState); - const pullMetadata = async (creators: AccountAndPubkey[]) => { - await forEachAccount(processMetaplexAccounts)(creators); - }; - - const basePromises = [ + const loadVaults = () => getProgramAccounts(connection, VAULT_ID).then( forEachAccount(processVaultData), - ), + ); + const loadAuctions = () => getProgramAccounts(connection, AUCTION_ID).then( forEachAccount(processAuctions), - ), + ); + const loadMetaplex = () => getProgramAccounts(connection, METAPLEX_ID).then( forEachAccount(processMetaplexAccounts), - ), // ??? + ); + const loadCreators = () => getProgramAccounts(connection, METAPLEX_ID, { filters: [ { dataSize: MAX_WHITELISTED_CREATOR_SIZE, }, ], - }).then(pullMetadata), // ??? + }).then(forEachAccount(processMetaplexAccounts)); + const loadMetadata = () => + pullMetadataByCreators(connection, state, updateState); + const loadEditions = () => pullEditions(connection, updateState, state); + + const loading = [ + loadCreators().then(loadMetadata).then(loadEditions), + loadVaults(), + loadAuctions(), + loadMetaplex(), ]; - await Promise.all(basePromises); - const additionalPromises: Promise[] = getAdditionalPromises( - connection, - Object.values(tempCache.whitelistedCreatorsByCreator), - forEachAccount, - ); + await Promise.all(loading); - await Promise.all(additionalPromises); + console.log('Metadata size', state.metadata.length); - await postProcessMetadata(tempCache); - console.log('Metadata size', tempCache.metadata.length); - - await pullEditions(connection, updateTemp, tempCache); - - return tempCache; + return state; }; const pullEditions = async ( connection: Connection, - updateTemp: UpdateStateValueFunc, - tempCache: MetaState, + updater: UpdateStateValueFunc, + state: MetaState, ) => { console.log('Pulling editions for optimized metadata'); + type MultipleAccounts = UnPromise>; let setOf100MetadataEditionKeys: string[] = []; - const editionPromises: Promise<{ - keys: string[]; - array: AccountInfo[]; - }>[] = []; + const editionPromises: Promise[] = []; - for (let i = 0; i < tempCache.metadata.length; i++) { - let edition: StringPublicKey; - if (tempCache.metadata[i].info.editionNonce != null) { - edition = ( - await PublicKey.createProgramAddress( - [ - Buffer.from(METADATA_PREFIX), - toPublicKey(METADATA_PROGRAM_ID).toBuffer(), - toPublicKey(tempCache.metadata[i].info.mint).toBuffer(), - new Uint8Array([tempCache.metadata[i].info.editionNonce || 0]), - ], - toPublicKey(METADATA_PROGRAM_ID), - ) - ).toBase58(); - } else { - edition = await getEdition(tempCache.metadata[i].info.mint); - } - - setOf100MetadataEditionKeys.push(edition); - - if (setOf100MetadataEditionKeys.length >= 100) { - editionPromises.push( - getMultipleAccounts(connection, setOf100MetadataEditionKeys, 'recent'), - ); - setOf100MetadataEditionKeys = []; - } - } - - if (setOf100MetadataEditionKeys.length >= 0) { + const loadBatch = () => { editionPromises.push( - getMultipleAccounts(connection, setOf100MetadataEditionKeys, 'recent'), + getMultipleAccounts( + connection, + setOf100MetadataEditionKeys, + 'recent', + ).then(processEditions), ); setOf100MetadataEditionKeys = []; - } + }; - const responses = await Promise.all(editionPromises); - for (let i = 0; i < responses.length; i++) { - const returnedAccounts = responses[i]; + const processEditions = (returnedAccounts: MultipleAccounts) => { for (let j = 0; j < returnedAccounts.array.length; j++) { processMetaData( { pubkey: returnedAccounts.keys[j], account: returnedAccounts.array[j], }, - updateTemp, + updater, ); } + }; + + for (const metadata of state.metadata) { + let editionKey: StringPublicKey; + if (metadata.info.editionNonce === null) { + editionKey = await getEdition(metadata.info.mint); + } else { + editionKey = ( + await PublicKey.createProgramAddress( + [ + Buffer.from(METADATA_PREFIX), + toPublicKey(METADATA_PROGRAM_ID).toBuffer(), + toPublicKey(metadata.info.mint).toBuffer(), + new Uint8Array([metadata.info.editionNonce || 0]), + ], + toPublicKey(METADATA_PROGRAM_ID), + ) + ).toBase58(); + } + + setOf100MetadataEditionKeys.push(editionKey); + + if (setOf100MetadataEditionKeys.length >= 100) { + loadBatch(); + } } + + if (setOf100MetadataEditionKeys.length >= 0) { + loadBatch(); + } + + await Promise.all(editionPromises); + console.log( 'Edition size', - Object.keys(tempCache.editions).length, - Object.keys(tempCache.masterEditions).length, + Object.keys(state.editions).length, + Object.keys(state.masterEditions).length, ); }; -const getAdditionalPromises = ( +const pullMetadataByCreators = ( connection: Connection, - whitelistedCreators: ParsedAccount[], - forEach: ReturnType, -): Promise[] => { + state: MetaState, + updater: UpdateStateValueFunc, +): Promise => { console.log('pulling optimized nfts'); + const whitelistedCreators = Object.values(state.whitelistedCreatorsByCreator); + + const setter: UpdateStateValueFunc = async (prop, key, value) => { + if (prop === 'metadataByMint') { + await initMetadata(value, state.whitelistedCreatorsByCreator, updater); + } else { + updater(prop, key, value); + } + }; + const forEachAccount = processingAccounts(setter); + const additionalPromises: Promise[] = []; for (const creator of whitelistedCreators) { for (let i = 0; i < MAX_CREATOR_LIMIT; i++) { @@ -410,44 +424,47 @@ const getAdditionalPromises = ( }, }, ], - }).then(forEach(processMetaData)); + }).then(forEachAccount(processMetaData)); additionalPromises.push(promise); } } - return additionalPromises; + return Promise.all(additionalPromises); }; export const makeSetter = - (state: MetaState) => - (prop: keyof MetaState, key: string, value: ParsedAccount) => { + (state: MetaState): UpdateStateValueFunc => + (prop, key, value) => { if (prop === 'store') { state[prop] = value; - } else if (prop !== 'metadata') { + } else if (prop === 'metadata') { + state.metadata.push(value); + } else { state[prop][key] = value; } return state; }; export const processingAccounts = - (updater: ReturnType) => + (updater: UpdateStateValueFunc) => (fn: ProcessAccountsFunc) => async (accounts: AccountAndPubkey[]) => { await createPipelineExecutor( accounts.values(), account => fn(account, updater), { - sequence: 20, + sequence: 10, delay: 1, + jobsCount: 3, }, ); }; -const postProcessMetadata = async (tempCache: MetaState) => { - const values = Object.values(tempCache.metadataByMint); +const postProcessMetadata = async (state: MetaState) => { + const values = Object.values(state.metadataByMint); for (const metadata of values) { - await metadataByMintUpdater(metadata, tempCache); + await metadataByMintUpdater(metadata, state); } }; @@ -456,13 +473,7 @@ export const metadataByMintUpdater = async ( state: MetaState, ) => { const key = metadata.info.mint; - if ( - isMetadataPartOfStore( - metadata, - state.store, - state.whitelistedCreatorsByCreator, - ) - ) { + if (isMetadataPartOfStore(metadata, state.whitelistedCreatorsByCreator)) { await metadata.info.init(); const masterEditionKey = metadata.info?.masterEdition; if (masterEditionKey) { @@ -475,3 +486,19 @@ export const metadataByMintUpdater = async ( } return state; }; + +export const initMetadata = async ( + metadata: ParsedAccount, + whitelistedCreators: Record>, + setter: UpdateStateValueFunc, +) => { + if (isMetadataPartOfStore(metadata, whitelistedCreators)) { + await metadata.info.init(); + setter('metadataByMint', metadata.info.mint, metadata); + setter('metadata', '', metadata); + const masterEditionKey = metadata.info?.masterEdition; + if (masterEditionKey) { + setter('metadataByMasterEdition', masterEditionKey, metadata); + } + } +}; diff --git a/js/packages/common/src/contexts/meta/processMetaData.ts b/js/packages/common/src/contexts/meta/processMetaData.ts index d922d4b..93e7636 100644 --- a/js/packages/common/src/contexts/meta/processMetaData.ts +++ b/js/packages/common/src/contexts/meta/processMetaData.ts @@ -14,7 +14,7 @@ import { import { ParsedAccount } from '../accounts/types'; import { METADATA_PROGRAM_ID, pubkeyToString } from '../../utils'; -export const processMetaData: ProcessAccountsFunc = ( +export const processMetaData: ProcessAccountsFunc = async ( { account, pubkey }, setter, ) => { @@ -32,7 +32,7 @@ export const processMetaData: ProcessAccountsFunc = ( account, info: metadata, }; - setter('metadataByMint', metadata.mint, parsedAccount); + await setter('metadataByMint', metadata.mint, parsedAccount); } } diff --git a/js/packages/common/src/contexts/meta/subscribeAccountsChange.ts b/js/packages/common/src/contexts/meta/subscribeAccountsChange.ts index f93595d..d9ad9f2 100644 --- a/js/packages/common/src/contexts/meta/subscribeAccountsChange.ts +++ b/js/packages/common/src/contexts/meta/subscribeAccountsChange.ts @@ -6,7 +6,7 @@ import { toPublicKey, VAULT_ID, } from '../../utils'; -import { makeSetter, metadataByMintUpdater } from './loadAccounts'; +import { makeSetter, initMetadata } from './loadAccounts'; import { onChangeAccount } from './onChangeAccount'; import { processAuctions } from './processAuctions'; import { processMetaData } from './processMetaData'; @@ -52,12 +52,25 @@ export const subscribeAccountsChange = ( connection.onProgramAccountChange( toPublicKey(METADATA_PROGRAM_ID), onChangeAccount(processMetaData, async (prop, key, value) => { + const state = { ...getState() }; + const setter = makeSetter(state); + let hasChanges = false; + const updater: UpdateStateValueFunc = (...args) => { + hasChanges = true; + setter(...args); + }; + if (prop === 'metadataByMint') { - const state = getState(); - const nextState = await metadataByMintUpdater(value, state); - setState(nextState); + await initMetadata( + value, + state.whitelistedCreatorsByCreator, + updater, + ); } else { - updateStateValue(prop, key, value); + updater(prop, key, value); + } + if (hasChanges) { + setState(state); } }), ), diff --git a/js/packages/common/src/contexts/meta/types.ts b/js/packages/common/src/contexts/meta/types.ts index f37eb8d..fe1c992 100644 --- a/js/packages/common/src/contexts/meta/types.ts +++ b/js/packages/common/src/contexts/meta/types.ts @@ -90,11 +90,11 @@ export type AccountAndPubkey = { account: AccountInfo; }; -export type UpdateStateValueFunc = ( +export type UpdateStateValueFunc = ( prop: keyof MetaState, key: string, value: ParsedAccount, -) => void; +) => T; export type ProcessAccountsFunc = ( account: PublicKeyStringAndAccount, @@ -102,3 +102,7 @@ export type ProcessAccountsFunc = ( ) => void; export type CheckAccountFunc = (account: AccountInfo) => boolean; + +export type UnPromise> = T extends Promise + ? U + : never;