diff --git a/js/packages/web/src/contexts/meta/loadAccounts.ts b/js/packages/web/src/contexts/meta/loadAccounts.ts index 0f6e6be..5212aa0 100644 --- a/js/packages/web/src/contexts/meta/loadAccounts.ts +++ b/js/packages/web/src/contexts/meta/loadAccounts.ts @@ -14,6 +14,9 @@ import { processVaultData } from './processVaultData'; import { MAX_CREATOR_LEN, MAX_CREATOR_LIMIT, + MAX_NAME_LENGTH, + MAX_SYMBOL_LENGTH, + MAX_URI_LENGTH, Metadata, ParsedAccount, } from '../../../../common/dist/lib'; @@ -56,22 +59,6 @@ export const loadAccounts = async (connection: Connection, all: boolean) => { } }; - await connection - .getProgramAccounts(METAPLEX_ID, { - filters: [ - { - dataSize: MAX_WHITELISTED_CREATOR_SIZE, - }, - { - memcmp: { - offset: 0, - bytes: MetaplexKey.WhitelistedCreatorV1.toString(), - }, - }, - ], - }) - .then(forEach(processMetaplexAccounts)); - const promises = [ connection.getProgramAccounts(VAULT_ID).then(forEach(processVaultData)), connection.getProgramAccounts(AUCTION_ID).then(forEach(processAuctions)), @@ -79,27 +66,58 @@ export const loadAccounts = async (connection: Connection, all: boolean) => { connection .getProgramAccounts(METAPLEX_ID) .then(forEach(processMetaplexAccounts)), - ]; - for (let i = 0; i < MAX_CREATOR_LIMIT; i++) { - promises.push( - connection - .getProgramAccounts(METADATA_PROGRAM_ID, { - filters: [ - { - memcmp: { - offset: 0, - bytes: MetaplexKey.WhitelistedCreatorV1.toString(), - }, - }, - ], - }) - .then(forEach(processMetaData)), - ); - } + connection + .getProgramAccounts(METAPLEX_ID, { + filters: [ + { + dataSize: MAX_WHITELISTED_CREATOR_SIZE, + }, + ], + }) + .then(async creators => { + await forEach(processMetaplexAccounts)(creators); + const whitelistedCreators = Object.values( + tempCache.whitelistedCreatorsByCreator, + ); + + for (let i = 0; i < MAX_CREATOR_LIMIT; i++) { + for (let j = 0; j < whitelistedCreators.length; j++) { + promises.push( + connection + .getProgramAccounts(METADATA_PROGRAM_ID, { + filters: [ + { + memcmp: { + offset: + 1 + // key + 32 + // update auth + 32 + // mint + 4 + // name string length + MAX_NAME_LENGTH + // name + 4 + // uri string length + MAX_URI_LENGTH + // uri + 4 + // symbol string length + MAX_SYMBOL_LENGTH + // symbol + 2 + // seller fee basis points + 1 + // whether or not there is a creators vec + 4 + // creators vec length + i * MAX_CREATOR_LEN, + bytes: whitelistedCreators[j].info.address.toBase58(), + }, + }, + ], + }) + .then(forEach(processMetaData)), + ); + } + } + }), + ]; await Promise.all(promises); await postProcessMetadata(tempCache, all); + console.log('Metadata size', tempCache.metadata.length); return tempCache; }; diff --git a/js/packages/web/src/contexts/meta/processMetaplexAccounts.ts b/js/packages/web/src/contexts/meta/processMetaplexAccounts.ts index 9f018b8..6bdf225 100644 --- a/js/packages/web/src/contexts/meta/processMetaplexAccounts.ts +++ b/js/packages/web/src/contexts/meta/processMetaplexAccounts.ts @@ -131,7 +131,6 @@ export const processMetaplexAccounts: ProcessAccountsFunc = async ( if (isWhitelistedCreatorV1Account(account)) { const whitelistedCreator = decodeWhitelistedCreator(account.data); - // TODO: figure out a way to avoid generating creator addresses during parsing // should we store store id inside creator? const creatorKeyIfCreatorWasPartOfThisStore = await getWhitelistedCreator( diff --git a/rust/token-metadata/program/src/instruction.rs b/rust/token-metadata/program/src/instruction.rs index bd1f95d..82fe0a7 100644 --- a/rust/token-metadata/program/src/instruction.rs +++ b/rust/token-metadata/program/src/instruction.rs @@ -238,6 +238,11 @@ pub enum MetadataInstruction { /// 15. `[]` System program /// 16. `[]` Rent info MintNewEditionFromMasterEditionViaVaultProxy(MintNewEditionFromMasterEditionViaTokenArgs), + + /// Puff a Metadata - make all of it's variable length fields (name/uri/symbol) a fixed length using a null character + /// so that it can be found using offset searches by the RPC to make client lookups cheaper. + /// 0. `[writable]` Metadata account + PuffMetadata, } /// Creates an CreateMetadataAccounts instruction @@ -308,6 +313,22 @@ pub fn update_metadata_accounts( } } +/// puff metadata account instruction +pub fn puff_metadata_account( + program_id: Pubkey, + metadata_account: Pubkey, +) -> Instruction { + Instruction { + program_id, + accounts: vec![ + AccountMeta::new(metadata_account, false), + ], + data: MetadataInstruction::PuffMetadata + .try_to_vec() + .unwrap(), + } +} + /// creates a update_primary_sale_happened_via_token instruction #[allow(clippy::too_many_arguments)] pub fn update_primary_sale_happened_via_token( diff --git a/rust/token-metadata/program/src/processor.rs b/rust/token-metadata/program/src/processor.rs index 656493e..f977683 100644 --- a/rust/token-metadata/program/src/processor.rs +++ b/rust/token-metadata/program/src/processor.rs @@ -130,6 +130,13 @@ pub fn process_instruction<'a>( args.edition, ) } + MetadataInstruction::PuffMetadata => { + msg!("Instruction: Puff Metadata"); + process_puff_metadata_account( + program_id, + accounts + ) + } } } @@ -570,3 +577,22 @@ pub fn process_mint_new_edition_from_master_edition_via_vault_proxy<'a>( process_mint_new_edition_from_master_edition_via_token_logic(program_id, args, edition, true) } + +/// Puff out the variable length fields to a fixed length on a metadata +/// account in a permissionless way. +pub fn process_puff_metadata_account( + program_id: &Pubkey, + accounts: &[AccountInfo], +) -> ProgramResult { + let account_info_iter = &mut accounts.iter(); + + let metadata_account_info = next_account_info(account_info_iter)?; + let mut metadata = Metadata::from_account_info(metadata_account_info)?; + + assert_owned_by(metadata_account_info, program_id)?; + + puff_out_data_fields(&mut metadata); + + metadata.serialize(&mut *metadata_account_info.data.borrow_mut())?; + Ok(()) +} \ No newline at end of file diff --git a/rust/token-metadata/test/src/main.rs b/rust/token-metadata/test/src/main.rs index 638d689..f69e29d 100644 --- a/rust/token-metadata/test/src/main.rs +++ b/rust/token-metadata/test/src/main.rs @@ -23,17 +23,63 @@ use { spl_token_metadata::{ instruction::{ create_master_edition, create_metadata_accounts, - mint_new_edition_from_master_edition_via_token, update_metadata_accounts, + mint_new_edition_from_master_edition_via_token, update_metadata_accounts,puff_metadata_account }, state::{ get_reservation_list, Data, Edition, Key, MasterEditionV1, MasterEditionV2, Metadata, - EDITION, PREFIX, + EDITION, PREFIX,MAX_NAME_LENGTH, MAX_URI_LENGTH, MAX_SYMBOL_LENGTH }, }, std::str::FromStr, }; const TOKEN_PROGRAM_PUBKEY: &str = "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA"; +fn puff_unpuffed_metadata(_app_matches: &ArgMatches, payer: Keypair, client: RpcClient) { + let metadata_accounts = client.get_program_accounts(&spl_token_metadata::id()).unwrap(); + let mut needing_puffing = vec![]; + for acct in metadata_accounts { + if acct.1.data[0] == Key::MetadataV1 as u8 { + let account: Metadata = try_from_slice_unchecked(&acct.1.data).unwrap(); + if account.data.name.len() < MAX_NAME_LENGTH || account.data.uri.len() < MAX_URI_LENGTH || account.data.symbol.len() < MAX_SYMBOL_LENGTH { + needing_puffing.push(acct.0); + } + } + } + println!("Found {} accounts needing puffing", needing_puffing.len()); + + let mut instructions = vec![]; + let mut i = 0; + while i < needing_puffing.len() { + let pubkey = needing_puffing[i]; + instructions.push(puff_metadata_account(spl_token_metadata::id(), pubkey)); + if instructions.len() == 20 { + let mut transaction = Transaction::new_with_payer(&instructions, Some(&payer.pubkey())); + let recent_blockhash = client.get_recent_blockhash().unwrap().0; + + transaction.sign(&[&payer], recent_blockhash); + match client.send_and_confirm_transaction(&transaction) { + Ok(_) => { + println!("Another 20 down. At {} / {}", i, needing_puffing.len()); + instructions = vec![]; + i += 1; + }, + Err(_) => { + println!("Txn failed. Retry."); + std::thread::sleep(std::time::Duration::from_millis(1000)); + }, + } + } else { + i += 1; + } + } + + if instructions.len() > 0 { + let mut transaction = Transaction::new_with_payer(&instructions, Some(&payer.pubkey())); + let recent_blockhash = client.get_recent_blockhash().unwrap().0; + transaction.sign(&[&payer], recent_blockhash); + client.send_and_confirm_transaction(&transaction).unwrap(); + } +} fn mint_coins(app_matches: &ArgMatches, payer: Keypair, client: RpcClient) { let token_key = Pubkey::from_str(TOKEN_PROGRAM_PUBKEY).unwrap(); @@ -780,7 +826,10 @@ fn main() { .takes_value(true) .help("Account's authority, defaults to you"), ) - ).get_matches(); + + ).subcommand( + SubCommand::with_name("puff_unpuffed_metadata") + .about("Take metadata that still have variable length name, symbol, and uri fields and stretch them out with null symbols so they can be searched more easily by RPC.")).get_matches(); let client = RpcClient::new( app_matches @@ -831,6 +880,9 @@ fn main() { ("mint_coins", Some(arg_matches)) => { mint_coins(arg_matches, payer, client); } + ("puff_unpuffed_metadata", Some(arg_matches)) => { + puff_unpuffed_metadata(arg_matches, payer, client); + } _ => unreachable!(), } }