feat: optimize loading order
This commit is contained in:
parent
d785fd3052
commit
314c4ec543
|
@ -250,12 +250,12 @@ export class Metadata {
|
|||
this.data = args.data;
|
||||
this.primarySaleHappened = args.primarySaleHappened;
|
||||
this.isMutable = args.isMutable;
|
||||
this.editionNonce = args.editionNonce;
|
||||
this.editionNonce = args.editionNonce ?? null;
|
||||
}
|
||||
|
||||
public async init() {
|
||||
const metadata = toPublicKey(programIds().metadata);
|
||||
if (this.editionNonce != null) {
|
||||
if (this.editionNonce !== null) {
|
||||
this.edition = (
|
||||
await PublicKey.createProgramAddress(
|
||||
[
|
||||
|
|
|
@ -4,20 +4,20 @@ import { ParsedAccount } from '../accounts/types';
|
|||
|
||||
export const isMetadataPartOfStore = (
|
||||
m: ParsedAccount<Metadata>,
|
||||
store: ParsedAccount<Store> | null,
|
||||
whitelistedCreatorsByCreator: Record<
|
||||
string,
|
||||
ParsedAccount<WhitelistedCreator>
|
||||
>,
|
||||
store?: ParsedAccount<Store> | null,
|
||||
) => {
|
||||
if (!m?.info?.data?.creators || !store?.info) {
|
||||
if (!m?.info?.data?.creators) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return m.info.data.creators.some(
|
||||
c =>
|
||||
c.verified &&
|
||||
(store.info.public ||
|
||||
(store?.info.public ||
|
||||
whitelistedCreatorsByCreator[c.address]?.info?.activated),
|
||||
);
|
||||
};
|
||||
|
|
|
@ -20,12 +20,13 @@ import {
|
|||
getAuctionExtended,
|
||||
} from '../../actions';
|
||||
import { WhitelistedCreator } from '../../models/metaplex';
|
||||
import { AccountInfo, Connection, PublicKey } from '@solana/web3.js';
|
||||
import { Connection, PublicKey } from '@solana/web3.js';
|
||||
import {
|
||||
AccountAndPubkey,
|
||||
MetaState,
|
||||
ProcessAccountsFunc,
|
||||
UpdateStateValueFunc,
|
||||
UnPromise,
|
||||
} from './types';
|
||||
import { isMetadataPartOfStore } from './isMetadataPartOfStore';
|
||||
import { processAuctions } from './processAuctions';
|
||||
|
@ -266,125 +267,138 @@ export const limitedLoadAccounts = async (connection: Connection) => {
|
|||
};
|
||||
|
||||
export const loadAccounts = async (connection: Connection) => {
|
||||
const tempCache: MetaState = getEmptyMetaState();
|
||||
const updateTemp = makeSetter(tempCache);
|
||||
const forEachAccount = processingAccounts(updateTemp);
|
||||
const state: MetaState = getEmptyMetaState();
|
||||
const updateState = makeSetter(state);
|
||||
const forEachAccount = processingAccounts(updateState);
|
||||
|
||||
const pullMetadata = async (creators: AccountAndPubkey[]) => {
|
||||
await forEachAccount(processMetaplexAccounts)(creators);
|
||||
};
|
||||
|
||||
const basePromises = [
|
||||
const loadVaults = () =>
|
||||
getProgramAccounts(connection, VAULT_ID).then(
|
||||
forEachAccount(processVaultData),
|
||||
),
|
||||
);
|
||||
const loadAuctions = () =>
|
||||
getProgramAccounts(connection, AUCTION_ID).then(
|
||||
forEachAccount(processAuctions),
|
||||
),
|
||||
);
|
||||
const loadMetaplex = () =>
|
||||
getProgramAccounts(connection, METAPLEX_ID).then(
|
||||
forEachAccount(processMetaplexAccounts),
|
||||
), // ???
|
||||
);
|
||||
const loadCreators = () =>
|
||||
getProgramAccounts(connection, METAPLEX_ID, {
|
||||
filters: [
|
||||
{
|
||||
dataSize: MAX_WHITELISTED_CREATOR_SIZE,
|
||||
},
|
||||
],
|
||||
}).then(pullMetadata), // ???
|
||||
}).then(forEachAccount(processMetaplexAccounts));
|
||||
const loadMetadata = () =>
|
||||
pullMetadataByCreators(connection, state, updateState);
|
||||
const loadEditions = () => pullEditions(connection, updateState, state);
|
||||
|
||||
const loading = [
|
||||
loadCreators().then(loadMetadata).then(loadEditions),
|
||||
loadVaults(),
|
||||
loadAuctions(),
|
||||
loadMetaplex(),
|
||||
];
|
||||
|
||||
await Promise.all(basePromises);
|
||||
const additionalPromises: Promise<void>[] = getAdditionalPromises(
|
||||
connection,
|
||||
Object.values(tempCache.whitelistedCreatorsByCreator),
|
||||
forEachAccount,
|
||||
);
|
||||
await Promise.all(loading);
|
||||
|
||||
await Promise.all(additionalPromises);
|
||||
console.log('Metadata size', state.metadata.length);
|
||||
|
||||
await postProcessMetadata(tempCache);
|
||||
console.log('Metadata size', tempCache.metadata.length);
|
||||
|
||||
await pullEditions(connection, updateTemp, tempCache);
|
||||
|
||||
return tempCache;
|
||||
return state;
|
||||
};
|
||||
|
||||
const pullEditions = async (
|
||||
connection: Connection,
|
||||
updateTemp: UpdateStateValueFunc,
|
||||
tempCache: MetaState,
|
||||
updater: UpdateStateValueFunc,
|
||||
state: MetaState,
|
||||
) => {
|
||||
console.log('Pulling editions for optimized metadata');
|
||||
|
||||
type MultipleAccounts = UnPromise<ReturnType<typeof getMultipleAccounts>>;
|
||||
let setOf100MetadataEditionKeys: string[] = [];
|
||||
const editionPromises: Promise<{
|
||||
keys: string[];
|
||||
array: AccountInfo<Buffer>[];
|
||||
}>[] = [];
|
||||
const editionPromises: Promise<void>[] = [];
|
||||
|
||||
for (let i = 0; i < tempCache.metadata.length; i++) {
|
||||
let edition: StringPublicKey;
|
||||
if (tempCache.metadata[i].info.editionNonce != null) {
|
||||
edition = (
|
||||
await PublicKey.createProgramAddress(
|
||||
[
|
||||
Buffer.from(METADATA_PREFIX),
|
||||
toPublicKey(METADATA_PROGRAM_ID).toBuffer(),
|
||||
toPublicKey(tempCache.metadata[i].info.mint).toBuffer(),
|
||||
new Uint8Array([tempCache.metadata[i].info.editionNonce || 0]),
|
||||
],
|
||||
toPublicKey(METADATA_PROGRAM_ID),
|
||||
)
|
||||
).toBase58();
|
||||
} else {
|
||||
edition = await getEdition(tempCache.metadata[i].info.mint);
|
||||
}
|
||||
|
||||
setOf100MetadataEditionKeys.push(edition);
|
||||
|
||||
if (setOf100MetadataEditionKeys.length >= 100) {
|
||||
editionPromises.push(
|
||||
getMultipleAccounts(connection, setOf100MetadataEditionKeys, 'recent'),
|
||||
);
|
||||
setOf100MetadataEditionKeys = [];
|
||||
}
|
||||
}
|
||||
|
||||
if (setOf100MetadataEditionKeys.length >= 0) {
|
||||
const loadBatch = () => {
|
||||
editionPromises.push(
|
||||
getMultipleAccounts(connection, setOf100MetadataEditionKeys, 'recent'),
|
||||
getMultipleAccounts(
|
||||
connection,
|
||||
setOf100MetadataEditionKeys,
|
||||
'recent',
|
||||
).then(processEditions),
|
||||
);
|
||||
setOf100MetadataEditionKeys = [];
|
||||
}
|
||||
};
|
||||
|
||||
const responses = await Promise.all(editionPromises);
|
||||
for (let i = 0; i < responses.length; i++) {
|
||||
const returnedAccounts = responses[i];
|
||||
const processEditions = (returnedAccounts: MultipleAccounts) => {
|
||||
for (let j = 0; j < returnedAccounts.array.length; j++) {
|
||||
processMetaData(
|
||||
{
|
||||
pubkey: returnedAccounts.keys[j],
|
||||
account: returnedAccounts.array[j],
|
||||
},
|
||||
updateTemp,
|
||||
updater,
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
for (const metadata of state.metadata) {
|
||||
let editionKey: StringPublicKey;
|
||||
if (metadata.info.editionNonce === null) {
|
||||
editionKey = await getEdition(metadata.info.mint);
|
||||
} else {
|
||||
editionKey = (
|
||||
await PublicKey.createProgramAddress(
|
||||
[
|
||||
Buffer.from(METADATA_PREFIX),
|
||||
toPublicKey(METADATA_PROGRAM_ID).toBuffer(),
|
||||
toPublicKey(metadata.info.mint).toBuffer(),
|
||||
new Uint8Array([metadata.info.editionNonce || 0]),
|
||||
],
|
||||
toPublicKey(METADATA_PROGRAM_ID),
|
||||
)
|
||||
).toBase58();
|
||||
}
|
||||
|
||||
setOf100MetadataEditionKeys.push(editionKey);
|
||||
|
||||
if (setOf100MetadataEditionKeys.length >= 100) {
|
||||
loadBatch();
|
||||
}
|
||||
}
|
||||
|
||||
if (setOf100MetadataEditionKeys.length >= 0) {
|
||||
loadBatch();
|
||||
}
|
||||
|
||||
await Promise.all(editionPromises);
|
||||
|
||||
console.log(
|
||||
'Edition size',
|
||||
Object.keys(tempCache.editions).length,
|
||||
Object.keys(tempCache.masterEditions).length,
|
||||
Object.keys(state.editions).length,
|
||||
Object.keys(state.masterEditions).length,
|
||||
);
|
||||
};
|
||||
|
||||
const getAdditionalPromises = (
|
||||
const pullMetadataByCreators = (
|
||||
connection: Connection,
|
||||
whitelistedCreators: ParsedAccount<WhitelistedCreator>[],
|
||||
forEach: ReturnType<typeof processingAccounts>,
|
||||
): Promise<void>[] => {
|
||||
state: MetaState,
|
||||
updater: UpdateStateValueFunc,
|
||||
): Promise<any> => {
|
||||
console.log('pulling optimized nfts');
|
||||
|
||||
const whitelistedCreators = Object.values(state.whitelistedCreatorsByCreator);
|
||||
|
||||
const setter: UpdateStateValueFunc = async (prop, key, value) => {
|
||||
if (prop === 'metadataByMint') {
|
||||
await initMetadata(value, state.whitelistedCreatorsByCreator, updater);
|
||||
} else {
|
||||
updater(prop, key, value);
|
||||
}
|
||||
};
|
||||
const forEachAccount = processingAccounts(setter);
|
||||
|
||||
const additionalPromises: Promise<void>[] = [];
|
||||
for (const creator of whitelistedCreators) {
|
||||
for (let i = 0; i < MAX_CREATOR_LIMIT; i++) {
|
||||
|
@ -410,44 +424,47 @@ const getAdditionalPromises = (
|
|||
},
|
||||
},
|
||||
],
|
||||
}).then(forEach(processMetaData));
|
||||
}).then(forEachAccount(processMetaData));
|
||||
additionalPromises.push(promise);
|
||||
}
|
||||
}
|
||||
|
||||
return additionalPromises;
|
||||
return Promise.all(additionalPromises);
|
||||
};
|
||||
|
||||
export const makeSetter =
|
||||
(state: MetaState) =>
|
||||
(prop: keyof MetaState, key: string, value: ParsedAccount<any>) => {
|
||||
(state: MetaState): UpdateStateValueFunc<MetaState> =>
|
||||
(prop, key, value) => {
|
||||
if (prop === 'store') {
|
||||
state[prop] = value;
|
||||
} else if (prop !== 'metadata') {
|
||||
} else if (prop === 'metadata') {
|
||||
state.metadata.push(value);
|
||||
} else {
|
||||
state[prop][key] = value;
|
||||
}
|
||||
return state;
|
||||
};
|
||||
|
||||
export const processingAccounts =
|
||||
(updater: ReturnType<typeof makeSetter>) =>
|
||||
(updater: UpdateStateValueFunc) =>
|
||||
(fn: ProcessAccountsFunc) =>
|
||||
async (accounts: AccountAndPubkey[]) => {
|
||||
await createPipelineExecutor(
|
||||
accounts.values(),
|
||||
account => fn(account, updater),
|
||||
{
|
||||
sequence: 20,
|
||||
sequence: 10,
|
||||
delay: 1,
|
||||
jobsCount: 3,
|
||||
},
|
||||
);
|
||||
};
|
||||
|
||||
const postProcessMetadata = async (tempCache: MetaState) => {
|
||||
const values = Object.values(tempCache.metadataByMint);
|
||||
const postProcessMetadata = async (state: MetaState) => {
|
||||
const values = Object.values(state.metadataByMint);
|
||||
|
||||
for (const metadata of values) {
|
||||
await metadataByMintUpdater(metadata, tempCache);
|
||||
await metadataByMintUpdater(metadata, state);
|
||||
}
|
||||
};
|
||||
|
||||
|
@ -456,13 +473,7 @@ export const metadataByMintUpdater = async (
|
|||
state: MetaState,
|
||||
) => {
|
||||
const key = metadata.info.mint;
|
||||
if (
|
||||
isMetadataPartOfStore(
|
||||
metadata,
|
||||
state.store,
|
||||
state.whitelistedCreatorsByCreator,
|
||||
)
|
||||
) {
|
||||
if (isMetadataPartOfStore(metadata, state.whitelistedCreatorsByCreator)) {
|
||||
await metadata.info.init();
|
||||
const masterEditionKey = metadata.info?.masterEdition;
|
||||
if (masterEditionKey) {
|
||||
|
@ -475,3 +486,19 @@ export const metadataByMintUpdater = async (
|
|||
}
|
||||
return state;
|
||||
};
|
||||
|
||||
export const initMetadata = async (
|
||||
metadata: ParsedAccount<Metadata>,
|
||||
whitelistedCreators: Record<string, ParsedAccount<WhitelistedCreator>>,
|
||||
setter: UpdateStateValueFunc,
|
||||
) => {
|
||||
if (isMetadataPartOfStore(metadata, whitelistedCreators)) {
|
||||
await metadata.info.init();
|
||||
setter('metadataByMint', metadata.info.mint, metadata);
|
||||
setter('metadata', '', metadata);
|
||||
const masterEditionKey = metadata.info?.masterEdition;
|
||||
if (masterEditionKey) {
|
||||
setter('metadataByMasterEdition', masterEditionKey, metadata);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
|
|
@ -14,7 +14,7 @@ import {
|
|||
import { ParsedAccount } from '../accounts/types';
|
||||
import { METADATA_PROGRAM_ID, pubkeyToString } from '../../utils';
|
||||
|
||||
export const processMetaData: ProcessAccountsFunc = (
|
||||
export const processMetaData: ProcessAccountsFunc = async (
|
||||
{ account, pubkey },
|
||||
setter,
|
||||
) => {
|
||||
|
@ -32,7 +32,7 @@ export const processMetaData: ProcessAccountsFunc = (
|
|||
account,
|
||||
info: metadata,
|
||||
};
|
||||
setter('metadataByMint', metadata.mint, parsedAccount);
|
||||
await setter('metadataByMint', metadata.mint, parsedAccount);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -6,7 +6,7 @@ import {
|
|||
toPublicKey,
|
||||
VAULT_ID,
|
||||
} from '../../utils';
|
||||
import { makeSetter, metadataByMintUpdater } from './loadAccounts';
|
||||
import { makeSetter, initMetadata } from './loadAccounts';
|
||||
import { onChangeAccount } from './onChangeAccount';
|
||||
import { processAuctions } from './processAuctions';
|
||||
import { processMetaData } from './processMetaData';
|
||||
|
@ -52,12 +52,25 @@ export const subscribeAccountsChange = (
|
|||
connection.onProgramAccountChange(
|
||||
toPublicKey(METADATA_PROGRAM_ID),
|
||||
onChangeAccount(processMetaData, async (prop, key, value) => {
|
||||
const state = { ...getState() };
|
||||
const setter = makeSetter(state);
|
||||
let hasChanges = false;
|
||||
const updater: UpdateStateValueFunc = (...args) => {
|
||||
hasChanges = true;
|
||||
setter(...args);
|
||||
};
|
||||
|
||||
if (prop === 'metadataByMint') {
|
||||
const state = getState();
|
||||
const nextState = await metadataByMintUpdater(value, state);
|
||||
setState(nextState);
|
||||
await initMetadata(
|
||||
value,
|
||||
state.whitelistedCreatorsByCreator,
|
||||
updater,
|
||||
);
|
||||
} else {
|
||||
updateStateValue(prop, key, value);
|
||||
updater(prop, key, value);
|
||||
}
|
||||
if (hasChanges) {
|
||||
setState(state);
|
||||
}
|
||||
}),
|
||||
),
|
||||
|
|
|
@ -90,11 +90,11 @@ export type AccountAndPubkey = {
|
|||
account: AccountInfo<Buffer>;
|
||||
};
|
||||
|
||||
export type UpdateStateValueFunc = (
|
||||
export type UpdateStateValueFunc<T = void> = (
|
||||
prop: keyof MetaState,
|
||||
key: string,
|
||||
value: ParsedAccount<any>,
|
||||
) => void;
|
||||
) => T;
|
||||
|
||||
export type ProcessAccountsFunc = (
|
||||
account: PublicKeyStringAndAccount<Buffer>,
|
||||
|
@ -102,3 +102,7 @@ export type ProcessAccountsFunc = (
|
|||
) => void;
|
||||
|
||||
export type CheckAccountFunc = (account: AccountInfo<Buffer>) => boolean;
|
||||
|
||||
export type UnPromise<T extends Promise<any>> = T extends Promise<infer U>
|
||||
? U
|
||||
: never;
|
||||
|
|
Loading…
Reference in New Issue