Move to global address lookup tables

programs/mango-v4/src/address_lookup_table/mod.rs

@@ -1,3 +1,4 @@
+use anchor_lang::prelude::*;
 mod solana_address_lookup_table_instruction;
 pub use solana_address_lookup_table_instruction::*;
 use solana_program::pubkey::{Pubkey, PUBKEY_BYTES};
@@ -26,3 +27,24 @@ pub fn contains(table: &[u8], pubkey: &Pubkey) -> bool {
         .find(|&addr| addr == pubkey)
         .is_some()
 }
+
+pub fn extend<'info>(
+    lookup_table_ai: AccountInfo<'info>,
+    authority_ai: AccountInfo<'info>,
+    payer_ai: AccountInfo<'info>,
+    signer_seeds: &[&[&[u8]]],
+    new_addresses: Vec<Pubkey>,
+) -> std::result::Result<(), ProgramError> {
+    let instruction = extend_lookup_table(
+        lookup_table_ai.key(),
+        authority_ai.key(),
+        payer_ai.key(),
+        new_addresses,
+    );
+    let account_infos = [
+        lookup_table_ai,
+        authority_ai,
+        payer_ai,
+    ];
+    solana_program::program::invoke_signed(&instruction, &account_infos, signer_seeds)
+}

programs/mango-v4/src/instructions/create_account.rs

@@ -1,6 +1,5 @@
 use anchor_lang::prelude::*;
 
-use crate::address_lookup_table;
 use crate::error::*;
 use crate::state::*;
 
@@ -20,80 +19,19 @@ pub struct CreateAccount<'info> {
 
     pub owner: Signer<'info>,
 
-    // We can't use anchor's `init` here because the create_lookup_table instruction
-    // expects an unallocated table.
-    // Even though this is a PDA, we can't use anchor's `seeds` here because the
-    // address must be based on a recent slot hash, and create_lookup_table() will
-    // validate in anyway.
-    #[account(mut)]
-    pub address_lookup_table: UncheckedAccount<'info>, // TODO: wrapper?
-
     #[account(mut)]
     pub payer: Signer<'info>,
 
     pub system_program: Program<'info, System>,
     pub rent: Sysvar<'info, Rent>,
-    pub address_lookup_table_program: UncheckedAccount<'info>, // TODO: force address?
 }
 
-pub fn create_account(
-    ctx: Context<CreateAccount>,
-    account_num: u8,
-    address_lookup_table_recent_slot: u64,
-) -> Result<()> {
-    {
-        let mut account = ctx.accounts.account.load_init()?;
-        account.group = ctx.accounts.group.key();
-        account.owner = ctx.accounts.owner.key();
-        account.address_lookup_table = ctx.accounts.address_lookup_table.key();
-        account.account_num = account_num;
-        account.bump = *ctx.bumps.get("account").ok_or(MangoError::SomeError)?;
-    }
-
-    // Setup the address lookup table
-    //
-    // First: Pre-pay for max-length address lookup table -- otherwise extending it
-    // (later, in deposit etc) will need a payer!
-    let rent = Rent::get()?;
-    let required_lamports = rent
-        .minimum_balance(address_lookup_table::LOOKUP_TABLE_MAX_ACCOUNT_SIZE)
-        .max(1)
-        .saturating_sub(ctx.accounts.address_lookup_table.lamports());
-    if required_lamports > 0 {
-        solana_program::program::invoke(
-            &solana_program::system_instruction::transfer(
-                &ctx.accounts.payer.key(),
-                &ctx.accounts.address_lookup_table.key(),
-                required_lamports,
-            ),
-            &[
-                ctx.accounts.payer.to_account_info(),
-                ctx.accounts.address_lookup_table.to_account_info(),
-                ctx.accounts.system_program.to_account_info(),
-            ],
-        )?;
-    }
-    // Now create the account.
-    // TODO: We could save some CU here by not using create_lookup_table():
-    //       it - unnecessarily - derives the lookup table address again.
-    let (instruction, _expected_adress_map_address) = address_lookup_table::create_lookup_table(
-        ctx.accounts.account.key(),
-        ctx.accounts.payer.key(),
-        address_lookup_table_recent_slot,
-    );
-    let account_infos = [
-        ctx.accounts.address_lookup_table.to_account_info(),
-        ctx.accounts.account.to_account_info(),
-        ctx.accounts.payer.to_account_info(),
-        ctx.accounts.system_program.to_account_info(),
-    ];
-    // Anchor only sets the discriminator after this function finishes,
-    // calling load() right now would cause an error. But we _do_ need an immutable borrow
-    // so hack it by calling exit() early (which only sets the discriminator)
-    ctx.accounts.account.exit(&crate::id())?;
-    let account = ctx.accounts.account.load()?;
-    let seeds = account_seeds!(account);
-    solana_program::program::invoke_signed(&instruction, &account_infos, &[seeds])?;
+pub fn create_account(ctx: Context<CreateAccount>, account_num: u8) -> Result<()> {
+    let mut account = ctx.accounts.account.load_init()?;
+    account.group = ctx.accounts.group.key();
+    account.owner = ctx.accounts.owner.key();
+    account.account_num = account_num;
+    account.bump = *ctx.bumps.get("account").ok_or(MangoError::SomeError)?;
 
     Ok(())
 }

programs/mango-v4/src/instructions/deposit.rs

@@ -3,7 +3,6 @@ use anchor_spl::token;
 use anchor_spl::token::Token;
 use anchor_spl::token::TokenAccount;
 
-use crate::address_lookup_table;
 use crate::state::*;
 
 #[derive(Accounts)]
@@ -13,7 +12,6 @@ pub struct Deposit<'info> {
     #[account(
         mut,
         has_one = group,
-        has_one = address_lookup_table,
     )]
     pub account: AccountLoader<'info, MangoAccount>,
 
@@ -32,11 +30,7 @@ pub struct Deposit<'info> {
     pub token_account: Box<Account<'info, TokenAccount>>,
     pub token_authority: Signer<'info>,
 
-    #[account(mut)]
-    pub address_lookup_table: UncheckedAccount<'info>, // TODO: wrapper?
-
     pub token_program: Program<'info, Token>,
-    pub address_lookup_table_program: UncheckedAccount<'info>, // TODO: force address?
 }
 
 impl<'info> Deposit<'info> {
@@ -55,72 +49,21 @@ impl<'info> Deposit<'info> {
 //       That would save a lot of computation that needs to go into finding the
 //       right index for the mint.
 pub fn deposit(ctx: Context<Deposit>, amount: u64) -> Result<()> {
-    let (is_new_position, oracle) = {
-        // Find the mint's token index
-        let group = ctx.accounts.group.load()?;
-        let mint = ctx.accounts.token_account.mint;
-        let token_index = group.tokens.index_for_mint(&mint)?;
+    // Find the mint's token index
+    let group = ctx.accounts.group.load()?;
+    let mint = ctx.accounts.token_account.mint;
+    let token_index = group.tokens.index_for_mint(&mint)?;
 
-        // Get the account's position for that token index
-        let mut account = ctx.accounts.account.load_mut()?;
-        let position = account.indexed_positions.get_mut_or_create(token_index)?;
-        let is_new_position = !position.is_active();
+    // Get the account's position for that token index
+    let mut account = ctx.accounts.account.load_mut()?;
+    let position = account.indexed_positions.get_mut_or_create(token_index)?;
 
-        // Update the bank and position
-        let mut bank = ctx.accounts.bank.load_mut()?;
-        bank.deposit(position, amount);
+    // Update the bank and position
+    let mut bank = ctx.accounts.bank.load_mut()?;
+    bank.deposit(position, amount);
 
-        // Transfer the actual tokens
-        token::transfer(ctx.accounts.transfer_ctx(), amount)?;
-
-        (is_new_position, bank.oracle)
-    };
-
-    // Do we need to add (oracle, bank) to the user's address lookup table?
-    //
-    // Since they are always added as a pair, checking for one is sufficient.
-    let add_to_lookup_table = is_new_position
-        && !address_lookup_table::contains(
-            &ctx.accounts.address_lookup_table.try_borrow_data()?,
-            &oracle,
-        );
-    if add_to_lookup_table {
-        // NOTE: Unfortunately extend() _requires_ a payer, even though we've already
-        // fully funded the address lookup table. No further transfer will be necessary.
-        // We'll pass the account as payer.
-        let mut instruction = address_lookup_table::extend_lookup_table(
-            ctx.accounts.address_lookup_table.key(),
-            ctx.accounts.account.key(),
-            ctx.accounts.account.key(),
-            vec![ctx.accounts.bank.key(), oracle],
-        );
-        // Sneakily remove the system_program account: that way any attempted transfer would error.
-        instruction.accounts.pop();
-        let account_infos = [
-            ctx.accounts.address_lookup_table.to_account_info(),
-            ctx.accounts.account.to_account_info(),
-            ctx.accounts.account.to_account_info(),
-        ];
-        // Signing for the account is complicated because it must work as a payer which means
-        // a mutable borrow. Thus we must make copies of the values in the seed.
-        struct AccountSeedValues {
-            group: Pubkey,
-            owner: Pubkey,
-            account_num: u8,
-            bump: u8,
-        }
-        let account_seed_values = {
-            let account = ctx.accounts.account.load()?;
-            AccountSeedValues {
-                group: account.group,
-                owner: account.owner,
-                account_num: account.account_num,
-                bump: account.bump,
-            }
-        };
-        let account_seeds = account_seeds!(account_seed_values);
-        solana_program::program::invoke_signed(&instruction, &account_infos, &[account_seeds])?;
-    }
+    // Transfer the actual tokens
+    token::transfer(ctx.accounts.transfer_ctx(), amount)?;
 
     Ok(())
 }

programs/mango-v4/src/instructions/register_token.rs

@@ -5,6 +5,7 @@ use anchor_spl::token::TokenAccount;
 use fixed::types::I80F48;
 use fixed_macro::types::I80F48;
 
+use crate::address_lookup_table;
 use crate::error::*;
 use crate::state::*;
 
@@ -40,13 +41,32 @@ pub struct RegisterToken<'info> {
     )]
     pub vault: Account<'info, TokenAccount>,
 
+    #[account(
+        init,
+        seeds = [group.key().as_ref(), b"mintinfo".as_ref(), mint.key().as_ref()],
+        bump,
+        payer = payer,
+        space = 8 + std::mem::size_of::<MintInfo>(),
+    )]
+    pub mint_info: AccountLoader<'info, MintInfo>,
+
     pub oracle: UncheckedAccount<'info>,
 
+    // Creating an address lookup table needs a recent valid slot as an
+    // input argument. That makes creating ALTs from governance instructions
+    // impossible. Hence the ALT that this instruction uses must be created
+    // externally and the admin is responsible for placing banks/oracles into
+    // sensible address lookup tables.
+    // constraint: must be created, have the admin authority and have free space
+    #[account(mut)]
+    pub address_lookup_table: UncheckedAccount<'info>, // TODO: wrapper?
+
     #[account(mut)]
     pub payer: Signer<'info>,
 
     pub token_program: Program<'info, Token>,
     pub system_program: Program<'info, System>,
+    pub address_lookup_table_program: UncheckedAccount<'info>, // TODO: force address?
     pub rent: Sysvar<'info, Rent>,
 }
 
@@ -95,5 +115,29 @@ pub fn register_token(
         token_index: token_index as TokenIndex,
     };
 
+    let alt_previous_size =
+        address_lookup_table::addresses(&ctx.accounts.address_lookup_table.try_borrow_data()?)
+            .iter()
+            .count();
+    let mut mint_info = ctx.accounts.mint_info.load_init()?;
+    *mint_info = MintInfo {
+        mint: ctx.accounts.mint.key(),
+        vault: ctx.accounts.vault.key(),
+        oracle: ctx.accounts.oracle.key(),
+        bank: ctx.accounts.bank.key(),
+        address_lookup_table: ctx.accounts.address_lookup_table.key(),
+        address_lookup_table_bank_index: alt_previous_size as u8,
+        address_lookup_table_oracle_index: alt_previous_size as u8 + 1,
+    };
+
+    address_lookup_table::extend(
+        ctx.accounts.address_lookup_table.to_account_info(),
+        // TODO: is using the admin as ALT authority a good idea?
+        ctx.accounts.admin.to_account_info(),
+        ctx.accounts.payer.to_account_info(),
+        &[],
+        vec![ctx.accounts.bank.key(), ctx.accounts.oracle.key()],
+    )?;
+
     Ok(())
 }

programs/mango-v4/src/lib.rs

@@ -40,12 +40,8 @@ pub mod mango_v4 {
         )
     }
 
-    pub fn create_account(
-        ctx: Context<CreateAccount>,
-        account_num: u8,
-        address_lookup_table_recent_slot: u64,
-    ) -> Result<()> {
-        instructions::create_account(ctx, account_num, address_lookup_table_recent_slot)
+    pub fn create_account(ctx: Context<CreateAccount>, account_num: u8) -> Result<()> {
+        instructions::create_account(ctx, account_num)
     }
 
     // todo:

programs/mango-v4/src/state/mint_info.rs

@@ -0,0 +1,19 @@
+use anchor_lang::prelude::*;
+
+// This struct describes which address lookup table can be used to pass
+// the accounts that are relevant for this mint. The idea is that clients
+// can load this account to figure out which address maps to use when calling
+// instructions that need banks/oracles for all active positions.
+#[account(zero_copy)]
+pub struct MintInfo {
+    // TODO: none of these pubkeys are needed, remove?
+    pub mint: Pubkey,
+    pub bank: Pubkey,
+    pub vault: Pubkey,
+    pub oracle: Pubkey,
+
+    // describe what address map relevant accounts are found on
+    pub address_lookup_table: Pubkey,
+    pub address_lookup_table_bank_index: u8,
+    pub address_lookup_table_oracle_index: u8,
+}

programs/mango-v4/src/state/mod.rs

@@ -1,6 +1,7 @@
 pub use health::*;
 pub use mango_account::*;
 pub use mango_group::*;
+pub use mint_info::*;
 pub use oracle::*;
 pub use token_bank::*;
 
@@ -9,6 +10,7 @@ pub use token_bank::*;
 mod health;
 mod mango_account;
 mod mango_group;
+mod mint_info;
 mod oracle;
 mod token_bank;
 // mod order_book_state_header;

programs/mango-v4/tests/program_test/mango_client.rs

@@ -92,13 +92,10 @@ impl<'keypair> ClientInstruction for WithdrawInstruction<'keypair> {
             allow_borrow: self.allow_borrow,
         };
 
-        // load account so we know its mint
+        // load accounts, find PDAs, find remainingAccounts
         let token_account: TokenAccount = account_loader.load(&self.token_account).await.unwrap();
         let account: MangoAccount = account_loader.load(&self.account).await.unwrap();
-        let lookup_table = account_loader
-            .load_bytes(&account.address_lookup_table)
-            .await
-            .unwrap();
+        let group: MangoGroup = account_loader.load(&account.group).await.unwrap();
 
         let bank = Pubkey::find_program_address(
             &[
@@ -119,6 +116,30 @@ impl<'keypair> ClientInstruction for WithdrawInstruction<'keypair> {
         )
         .0;
 
+        // figure out all the banks/oracles that need to be passed for the health check
+        let mut banks = vec![];
+        let mut oracles = vec![];
+        for position in account.indexed_positions.iter_active() {
+            let mint_pk = group.tokens.infos[position.token_index as usize].mint;
+            let mint_info_pk = Pubkey::find_program_address(
+                &[
+                    account.group.as_ref(),
+                    b"mintinfo".as_ref(),
+                    mint_pk.as_ref(),
+                ],
+                &program_id,
+            )
+            .0;
+            let mint_info: MintInfo = account_loader.load(&mint_info_pk).await.unwrap();
+            let lookup_table = account_loader
+                .load_bytes(&mint_info.address_lookup_table)
+                .await
+                .unwrap();
+            let addresses = mango_v4::address_lookup_table::addresses(&lookup_table);
+            banks.push(addresses[mint_info.address_lookup_table_bank_index as usize]);
+            oracles.push(addresses[mint_info.address_lookup_table_oracle_index as usize]);
+        }
+
         let accounts = Self::Accounts {
             group: account.group,
             account: self.account,
@@ -130,15 +151,18 @@ impl<'keypair> ClientInstruction for WithdrawInstruction<'keypair> {
         };
 
         let mut instruction = make_instruction(program_id, &accounts, instruction);
-        instruction.accounts.extend(
-            mango_v4::address_lookup_table::addresses(&lookup_table)
-                .iter()
-                .map(|&pubkey| AccountMeta {
-                    pubkey,
-                    is_writable: false,
-                    is_signer: false,
-                }),
-        );
+        instruction
+            .accounts
+            .extend(
+                banks
+                    .iter()
+                    .chain(oracles.iter())
+                    .map(|&pubkey| AccountMeta {
+                        pubkey,
+                        is_writable: false,
+                        is_signer: false,
+                    }),
+            );
 
         (accounts, instruction)
     }
@@ -196,11 +220,9 @@ impl<'keypair> ClientInstruction for DepositInstruction<'keypair> {
             account: self.account,
             bank,
             vault,
-            address_lookup_table: account.address_lookup_table,
             token_account: self.token_account,
             token_authority: self.token_authority.pubkey(),
             token_program: Token::id(),
-            address_lookup_table_program: mango_v4::address_lookup_table::id(),
         };
 
         let instruction = make_instruction(program_id, &accounts, instruction);
@@ -222,6 +244,7 @@ pub struct RegisterTokenInstruction<'keypair> {
     pub group: Pubkey,
     pub admin: &'keypair Keypair,
     pub mint: Pubkey,
+    pub address_lookup_table: Pubkey,
     pub payer: &'keypair Keypair,
 }
 #[async_trait::async_trait(?Send)]
@@ -259,6 +282,15 @@ impl<'keypair> ClientInstruction for RegisterTokenInstruction<'keypair> {
             &program_id,
         )
         .0;
+        let mint_info = Pubkey::find_program_address(
+            &[
+                self.group.as_ref(),
+                b"mintinfo".as_ref(),
+                self.mint.as_ref(),
+            ],
+            &program_id,
+        )
+        .0;
         let oracle = Pubkey::find_program_address(
             &[b"stub_oracle".as_ref(), self.mint.as_ref()],
             &program_id,
@@ -271,10 +303,13 @@ impl<'keypair> ClientInstruction for RegisterTokenInstruction<'keypair> {
             mint: self.mint,
             bank,
             vault,
+            mint_info,
             oracle,
+            address_lookup_table: self.address_lookup_table,
             payer: self.payer.pubkey(),
             token_program: Token::id(),
             system_program: System::id(),
+            address_lookup_table_program: mango_v4::address_lookup_table::id(),
             rent: sysvar::rent::Rent::id(),
         };
 
@@ -403,7 +438,6 @@ impl<'keypair> ClientInstruction for CreateGroupInstruction<'keypair> {
 
 pub struct CreateAccountInstruction<'keypair> {
     pub account_num: u8,
-    pub recent_slot: u64,
 
     pub group: Pubkey,
     pub owner: &'keypair Keypair,
@@ -420,7 +454,6 @@ impl<'keypair> ClientInstruction for CreateAccountInstruction<'keypair> {
         let program_id = mango_v4::id();
         let instruction = mango_v4::instruction::CreateAccount {
             account_num: self.account_num,
-            address_lookup_table_recent_slot: self.recent_slot,
         };
 
         let account = Pubkey::find_program_address(
@@ -433,19 +466,14 @@ impl<'keypair> ClientInstruction for CreateAccountInstruction<'keypair> {
             &program_id,
         )
         .0;
-        let address_lookup_table =
-            mango_v4::address_lookup_table::derive_lookup_table_address(&account, self.recent_slot)
-                .0;
 
         let accounts = mango_v4::accounts::CreateAccount {
             group: self.group,
             owner: self.owner.pubkey(),
             account,
-            address_lookup_table,
             payer: self.payer.pubkey(),
             system_program: System::id(),
             rent: sysvar::rent::Rent::id(),
-            address_lookup_table_program: mango_v4::address_lookup_table::id(),
         };
 
         let instruction = make_instruction(program_id, &accounts, instruction);

programs/mango-v4/tests/program_test/solana.rs

@@ -101,6 +101,23 @@ impl SolanaCookie {
         return keypair.pubkey();
     }
 
+    #[allow(dead_code)]
+    pub async fn create_address_lookup_table(
+        &self,
+        authority: &Keypair,
+        payer: &Keypair,
+    ) -> Pubkey {
+        let (instruction, alt_address) = mango_v4::address_lookup_table::create_lookup_table(
+            authority.pubkey(),
+            payer.pubkey(),
+            0, // TODO: get a good recent slot value from a sysvar
+        );
+        self.process_transaction(&[instruction], Some(&[authority, payer]))
+            .await
+            .unwrap();
+        alt_address
+    }
+
     #[allow(dead_code)]
     pub async fn get_account_data(&self, address: Pubkey) -> Option<Vec<u8>> {
         Some(

programs/mango-v4/tests/test_basic.rs

@@ -36,7 +36,6 @@ async fn test_basic() -> Result<(), TransportError> {
         solana,
         CreateAccountInstruction {
             account_num: 0,
-            recent_slot: 0, // TODO: get a real recent_slot, probably from SlotHistory
             group,
             owner,
             payer,
@@ -68,6 +67,8 @@ async fn test_basic() -> Result<(), TransportError> {
     .await
     .unwrap();
 
+    let address_lookup_table = solana.create_address_lookup_table(admin, payer).await;
+
     let register_token_accounts = send_tx(
         solana,
         RegisterTokenInstruction {
@@ -79,6 +80,7 @@ async fn test_basic() -> Result<(), TransportError> {
             group,
             admin,
             mint: mint0.pubkey,
+            address_lookup_table,
             payer,
         },
     )