mc/ts and keeper improvements (#127)

* ts and keeper improvements

Signed-off-by: microwavedcola1 <microwavedcola@gmail.com>

* remove stray heroku experiments

Signed-off-by: microwavedcola1 <microwavedcola@gmail.com>

* Fixes from reviews

Signed-off-by: microwavedcola1 <microwavedcola@gmail.com>

Cargo.lock

@@ -1242,6 +1242,7 @@ dependencies = [
  "mango-v4",
  "pyth-sdk-solana",
  "serum_dex 0.4.0 (git+https://github.com/blockworks-foundation/serum-dex.git)",
+ "shellexpand",
  "solana-account-decoder",
  "solana-client",
  "solana-sdk",
@@ -2891,7 +2892,6 @@ dependencies = [
  "mango-v4",
  "pyth-sdk-solana",
  "serum_dex 0.4.0 (git+https://github.com/blockworks-foundation/serum-dex.git)",
- "shellexpand",
  "solana-client",
  "solana-sdk",
  "tokio",

client/Cargo.toml

@@ -17,6 +17,7 @@ itertools = "0.10.3"
 mango-v4 = { path = "../programs/mango-v4" }
 pyth-sdk-solana = "0.1.0"
 serum_dex = { version = "0.4.0", git = "https://github.com/blockworks-foundation/serum-dex.git", default-features=false, features = ["no-entrypoint", "program"] }
+shellexpand = "2.1.0"
 solana-account-decoder = "~1.10.29"
 solana-client = "~1.10.29"
 solana-sdk = "~1.10.29"

client/src/client.rs

@@ -1,3 +1,4 @@
+use std::str::FromStr;
 use std::sync::Arc;
 
 use anchor_client::{Client, ClientError, Cluster, Program};
@@ -14,6 +15,7 @@ use mango_v4::instructions::{Serum3OrderType, Serum3SelfTradeBehavior, Serum3Sid
 use mango_v4::state::{AccountSize, Bank, Group, MangoAccountValue, Serum3MarketIndex, TokenIndex};
 
 use solana_client::rpc_client::RpcClient;
+use solana_sdk::signer::keypair;
 
 use crate::account_fetcher::*;
 use crate::context::{MangoGroupContext, Serum3MarketContext, TokenContext};
@@ -146,6 +148,7 @@ impl MangoClient {
                     ),
                 })
                 .send()
+                .map_err(prettify_client_error)
                 .context("Failed to create account...")?;
         }
         let mango_account_tuples = fetch_mango_accounts(&program, group, payer.pubkey())?;
@@ -753,7 +756,7 @@ pub enum MangoClientError {
 /// Unfortunately solana's RpcResponseError will very unhelpfully print [N log messages]
 /// instead of showing the actual log messages. This unpacks the error to provide more useful
 /// output.
-fn prettify_client_error(err: anchor_client::ClientError) -> anyhow::Error {
+pub fn prettify_client_error(err: anchor_client::ClientError) -> anyhow::Error {
     use solana_client::client_error::ClientErrorKind;
     use solana_client::rpc_request::{RpcError, RpcResponseErrorData};
     match &err {
@@ -777,3 +780,15 @@ fn prettify_client_error(err: anchor_client::ClientError) -> anyhow::Error {
     };
     err.into()
 }
+
+pub fn keypair_from_cli(keypair: &str) -> Keypair {
+    let maybe_keypair = keypair::read_keypair(&mut keypair.as_bytes());
+    match maybe_keypair {
+        Ok(keypair) => keypair,
+        Err(_) => {
+            let path = std::path::PathBuf::from_str(&*shellexpand::tilde(keypair)).unwrap();
+            keypair::read_keypair_file(path)
+                .unwrap_or_else(|_| panic!("Failed to read keypair from {}", keypair))
+        }
+    }
+}

keeper/Cargo.toml

@@ -21,7 +21,6 @@ log = "0.4.0"
 mango-v4 = { path = "../programs/mango-v4" }
 pyth-sdk-solana = "0.1.0"
 serum_dex = { version = "0.4.0", git = "https://github.com/blockworks-foundation/serum-dex.git", default-features=false, features = ["no-entrypoint", "program"] }
-shellexpand = "2.1.0"
 solana-client = "~1.10.29"
 solana-sdk = "~1.10.29"
 tokio = { version = "1.14.1", features = ["rt-multi-thread", "time", "macros", "sync"] }

keeper/src/crank.rs

@@ -3,6 +3,7 @@ use std::{sync::Arc, time::Duration};
 use crate::MangoClient;
 
 use anchor_lang::{__private::bytemuck::cast_ref, solana_program};
+use client::prettify_client_error;
 use futures::Future;
 use mango_v4::state::{EventQueue, EventType, FillEvent, OutEvent, PerpMarket, TokenIndex};
 use solana_sdk::{
@@ -11,6 +12,8 @@ use solana_sdk::{
 };
 use tokio::time;
 
+// TODO: move instructions into the client proper
+
 pub async fn runner(
     mango_client: Arc<MangoClient>,
     debugging_handle: impl Future,
@@ -88,7 +91,8 @@ pub async fn loop_update_index_and_rate(mango_client: Arc<MangoClient>, token_in
                     ix.accounts.append(&mut banks);
                     ix
                 })
-                .send();
+                .send()
+                .map_err(prettify_client_error);
 
             if let Err(e) = sig_result {
                 log::error!("{:?}", e)
@@ -188,7 +192,8 @@ pub async fn loop_consume_events(
                         &mango_v4::instruction::PerpConsumeEvents { limit: 10 },
                     ),
                 })
-                .send();
+                .send()
+                .map_err(prettify_client_error);
 
             if let Err(e) = sig_result {
                 log::error!("{:?}", e)
@@ -246,7 +251,8 @@ pub async fn loop_update_funding(
                         &mango_v4::instruction::PerpUpdateFunding {},
                     ),
                 })
-                .send();
+                .send()
+                .map_err(prettify_client_error);
             if let Err(e) = sig_result {
                 log::error!("{:?}", e)
             } else {

keeper/src/main.rs

@@ -1,19 +1,14 @@
 mod crank;
 mod taker;
 
-use std::str::FromStr;
 use std::sync::Arc;
 
 use anchor_client::Cluster;
 
 use clap::{Parser, Subcommand};
-use client::MangoClient;
+use client::{keypair_from_cli, MangoClient};
 use solana_sdk::pubkey::Pubkey;
-use solana_sdk::{
-    commitment_config::CommitmentConfig,
-    signature::Signer,
-    signer::{keypair, keypair::Keypair},
-};
+use solana_sdk::{commitment_config::CommitmentConfig, signature::Signer};
 use tokio::time;
 
 // TODO
@@ -33,21 +28,22 @@ struct CliDotenv {
     remaining_args: Vec<std::ffi::OsString>,
 }
 
-#[derive(Parser)]
+#[derive(Parser, Debug, Clone)]
 #[clap()]
 struct Cli {
     #[clap(short, long, env)]
     rpc_url: String,
 
     #[clap(short, long, env = "PAYER_KEYPAIR")]
-    payer: std::path::PathBuf,
+    payer: String,
 
     #[clap(short, long, env)]
     group: Option<Pubkey>,
 
     // These exist only as a shorthand to make testing easier. Normal users would provide the group.
     #[clap(long, env)]
-    group_from_admin_keypair: Option<std::path::PathBuf>,
+    group_from_admin_keypair: Option<String>,
+
     #[clap(long, env, default_value = "0")]
     group_from_admin_num: u32,
 
@@ -58,13 +54,7 @@ struct Cli {
     command: Command,
 }
 
-fn keypair_from_path(p: &std::path::PathBuf) -> Keypair {
-    let path = std::path::PathBuf::from_str(&*shellexpand::tilde(p.to_str().unwrap())).unwrap();
-    keypair::read_keypair_file(path)
-        .unwrap_or_else(|_| panic!("Failed to read keypair from {}", p.to_string_lossy()))
-}
-
-#[derive(Subcommand)]
+#[derive(Subcommand, Debug, Clone)]
 enum Command {
     Crank {},
     Taker {},
@@ -83,7 +73,7 @@ fn main() -> Result<(), anyhow::Error> {
     };
     let cli = Cli::parse_from(args);
 
-    let payer = keypair_from_path(&cli.payer);
+    let payer = keypair_from_cli(&cli.payer);
 
     let rpc_url = cli.rpc_url;
     let ws_url = rpc_url.replace("https", "wss");
@@ -97,7 +87,7 @@ fn main() -> Result<(), anyhow::Error> {
     let group = if let Some(group) = cli.group {
         group
     } else if let Some(p) = cli.group_from_admin_keypair {
-        let admin = keypair_from_path(&p);
+        let admin = keypair_from_cli(&p);
         MangoClient::group_for_admin(admin.pubkey(), cli.group_from_admin_num)
     } else {
         panic!("Must provide either group or group_from_admin_keypair");

liquidator/src/main.rs

@@ -4,7 +4,7 @@ use std::time::Duration;
 
 use anchor_client::Cluster;
 use clap::Parser;
-use client::{MangoClient, MangoGroupContext};
+use client::{keypair_from_cli, MangoClient, MangoGroupContext};
 use log::*;
 use mango_v4::state::{PerpMarketIndex, TokenIndex};
 
@@ -13,12 +13,8 @@ use once_cell::sync::OnceCell;
 use solana_client::rpc_client::RpcClient;
 use solana_sdk::commitment_config::CommitmentConfig;
 use solana_sdk::pubkey::Pubkey;
-use solana_sdk::signer::{
-    keypair::{self, Keypair},
-    Signer,
-};
+use solana_sdk::signer::Signer;
 use std::collections::HashSet;
-use std::str::FromStr;
 
 pub mod account_shared_data;
 pub mod chain_data;
@@ -74,7 +70,8 @@ struct Cli {
 
     // These exist only as a shorthand to make testing easier. Normal users would provide the group.
     #[clap(long, env)]
-    group_from_admin_keypair: Option<std::path::PathBuf>,
+    group_from_admin_keypair: Option<String>,
+
     #[clap(long, env, default_value = "0")]
     group_from_admin_num: u32,
 
@@ -87,7 +84,7 @@ struct Cli {
     serum_program: Pubkey,
 
     #[clap(long, env)]
-    liqor_owner: std::path::PathBuf,
+    liqor_owner: String,
 
     #[clap(long, env)]
     liqor_mango_account_name: String,
@@ -104,12 +101,6 @@ struct Cli {
     get_multiple_accounts_count: usize,
 }
 
-fn keypair_from_path(p: &std::path::PathBuf) -> Keypair {
-    let path = std::path::PathBuf::from_str(&*shellexpand::tilde(p.to_str().unwrap())).unwrap();
-    keypair::read_keypair_file(path)
-        .unwrap_or_else(|_| panic!("Failed to read keypair from {}", p.to_string_lossy()))
-}
-
 pub fn encode_address(addr: &Pubkey) -> String {
     bs58::encode(&addr.to_bytes()).into_string()
 }
@@ -125,12 +116,12 @@ async fn main() -> anyhow::Result<()> {
     };
     let cli = Cli::parse_from(args);
 
-    let liqor_owner = keypair_from_path(&cli.liqor_owner);
+    let liqor_owner = keypair_from_cli(&cli.liqor_owner);
 
     let mango_group = if let Some(group) = cli.group {
         group
     } else if let Some(p) = cli.group_from_admin_keypair {
-        let admin = keypair_from_path(&p);
+        let admin = keypair_from_cli(&p);
         MangoClient::group_for_admin(admin.pubkey(), cli.group_from_admin_num)
     } else {
         panic!("Must provide either group or group_from_admin_keypair");

programs/mango-v4/src/state/health.rs

@@ -48,7 +48,7 @@ pub trait AccountRetriever {
 
 /// Assumes the account infos needed for the health computation follow a strict order.
 ///
-/// 1. n_banks Bank account, in the order of account.tokens.iter_active()
+/// 1. n_banks Bank account, in the order of account.token_iter_active()
 /// 2. n_banks oracle accounts, one for each bank in the same order
 /// 3. PerpMarket accounts, in the order of account.perps.iter_active_accounts()
 /// 4. serum3 OpenOrders accounts, in the order of account.serum3.iter_active()

ts/client/ids.json

@@ -106,4 +106,4 @@
       "perpMarkets": []
     }
   ]
-}
+}

ts/client/src/accounts/bank.ts

@@ -16,6 +16,8 @@ export class Bank {
   public borrowIndex: I80F48;
   public cachedIndexedTotalDeposits: I80F48;
   public cachedIndexedTotalBorrows: I80F48;
+  public avgUtilization: I80F48;
+  public adjustmentFactor: I80F48;
   public maxRate: I80F48;
   public rate0: I80F48;
   public rate1: I80F48;
@@ -41,7 +43,10 @@ export class Bank {
       borrowIndex: I80F48Dto;
       cachedIndexedTotalDeposits: I80F48Dto;
       cachedIndexedTotalBorrows: I80F48Dto;
-      lastUpdated: BN;
+      indexLastUpdated: BN;
+      bankRateLastUpdated: BN;
+      avgUtilization: I80F48Dto;
+      adjustmentFactor: I80F48Dto;
       util0: I80F48Dto;
       rate0: I80F48Dto;
       util1: I80F48Dto;
@@ -72,7 +77,10 @@ export class Bank {
       obj.borrowIndex,
       obj.cachedIndexedTotalDeposits,
       obj.cachedIndexedTotalBorrows,
-      obj.lastUpdated,
+      obj.indexLastUpdated,
+      obj.bankRateLastUpdated,
+      obj.avgUtilization,
+      obj.adjustmentFactor,
       obj.util0,
       obj.rate0,
       obj.util1,
@@ -104,7 +112,10 @@ export class Bank {
     borrowIndex: I80F48Dto,
     indexedTotalDeposits: I80F48Dto,
     indexedTotalBorrows: I80F48Dto,
-    lastUpdated: BN,
+    public indexLastUpdated: BN,
+    public bankRateLastUpdated: BN,
+    avgUtilization: I80F48Dto,
+    adjustmentFactor: I80F48Dto,
     util0: I80F48Dto,
     rate0: I80F48Dto,
     util1: I80F48Dto,
@@ -127,6 +138,8 @@ export class Bank {
     this.borrowIndex = I80F48.from(borrowIndex);
     this.cachedIndexedTotalDeposits = I80F48.from(indexedTotalDeposits);
     this.cachedIndexedTotalBorrows = I80F48.from(indexedTotalBorrows);
+    this.avgUtilization = I80F48.from(avgUtilization);
+    this.adjustmentFactor = I80F48.from(adjustmentFactor);
     this.maxRate = I80F48.from(maxRate);
     this.util0 = I80F48.from(util0);
     this.rate0 = I80F48.from(rate0);
@@ -155,6 +168,14 @@ export class Bank {
       this.cachedIndexedTotalDeposits.toNumber() +
       '\n cachedIndexedTotalBorrows - ' +
       this.cachedIndexedTotalBorrows.toNumber() +
+      '\n indexLastUpdated - ' +
+      new Date(this.indexLastUpdated * 1000) +
+      '\n bankRateLastUpdated - ' +
+      new Date(this.bankRateLastUpdated * 1000) +
+      '\n avgUtilization - ' +
+      this.avgUtilization.toNumber() +
+      '\n adjustmentFactor - ' +
+      this.adjustmentFactor.toNumber() +
       '\n maxRate - ' +
       this.maxRate.toNumber() +
       '\n util0 - ' +
@@ -174,7 +195,15 @@ export class Bank {
       '\n initLiabWeight - ' +
       this.initLiabWeight.toNumber() +
       '\n liquidationFee - ' +
-      this.liquidationFee.toNumber()
+      this.liquidationFee.toNumber() +
+      '\n uiDeposits() - ' +
+      this.uiDeposits() +
+      '\n uiBorrows() - ' +
+      this.uiBorrows() +
+      '\n getDepositRate() - ' +
+      this.getDepositRate().toNumber() +
+      '\n getBorrowRate() - ' +
+      this.getBorrowRate().toNumber()
     );
   }
 

ts/client/src/accounts/mangoAccount.ts

@@ -99,11 +99,12 @@ export class MangoAccount {
     sourceBank: Bank,
     nativeTokenPosition: TokenPosition,
   ): I80F48 {
-    return nativeTokenPosition ?
-      nativeTokenPosition.native(sourceBank)
-      .mul(I80F48.fromNumber(Math.pow(10, QUOTE_DECIMALS)))
-      .div(I80F48.fromNumber(Math.pow(10, sourceBank.mintDecimals)))
-      .mul(sourceBank.price)
+    return nativeTokenPosition
+      ? nativeTokenPosition
+          .native(sourceBank)
+          .mul(I80F48.fromNumber(Math.pow(10, QUOTE_DECIMALS)))
+          .div(I80F48.fromNumber(Math.pow(10, sourceBank.mintDecimals)))
+          .mul(sourceBank.price)
       : ZERO_I80F48;
   }
 

ts/client/src/client.ts

@@ -34,6 +34,7 @@ import {
   AccountSize,
   MangoAccount,
   MangoAccountData,
+  TokenPosition,
 } from './accounts/mangoAccount';
 import { StubOracle } from './accounts/oracle';
 import { OrderType, PerpMarket, Side } from './accounts/perp';
@@ -1734,19 +1735,21 @@ export class MangoClient {
   ) {
     const healthRemainingAccounts: PublicKey[] = [];
 
-    const tokenIndices = mangoAccount.tokens
-      .filter((token) => token.tokenIndex !== 65535)
-      .map((token) => token.tokenIndex);
-
-    if (banks?.length) {
+    // allTokenIndices will contain tokenIndices from existing token positions, and,
+    // tokenIndices from possibly new token positons which will take earliest free slot available
+    let allTokenIndices = mangoAccount.tokens.map((token) => token.tokenIndex);
+    if (banks) {
       for (const bank of banks) {
-        tokenIndices.push(bank.tokenIndex);
+        if (allTokenIndices.indexOf(bank.tokenIndex) == -1) {
+          allTokenIndices[
+            mangoAccount.tokens.findIndex((token) => !token.isActive())
+          ] = bank.tokenIndex;
+        }
       }
     }
-
-    const mintInfos = [...new Set(tokenIndices)].map(
-      (tokenIndex) => group.mintInfosMap.get(tokenIndex)!,
-    );
+    const mintInfos = allTokenIndices
+      .filter((index) => index != TokenPosition.TokenIndexUnset)
+      .map((tokenIndex) => group.mintInfosMap.get(tokenIndex)!);
     healthRemainingAccounts.push(
       ...mintInfos.map((mintInfo) => mintInfo.firstBank()),
     );
@@ -1755,12 +1758,12 @@ export class MangoClient {
     );
     healthRemainingAccounts.push(
       ...mangoAccount.serum3
-        .filter((serum3Account) => serum3Account.marketIndex !== 65535)
+        .filter((serum3Account) => serum3Account.isActive())
         .map((serum3Account) => serum3Account.openOrders),
     );
     healthRemainingAccounts.push(
       ...mangoAccount.perps
-        .filter((perp) => perp.marketIndex !== 65535)
+        .filter((perp) => perp.isActive())
         .map(
           (perp) =>
             Array.from(group.perpMarketsMap.values()).filter(

ts/client/src/scripts/example1-admin.ts

@@ -288,7 +288,7 @@ async function main() {
     );
     console.log(`https://explorer.solana.com/tx/${sig}?cluster=devnet`);
     await group.reloadAll(client);
-    console.log(group.banksMap.get('USDC').toString());
+    console.log(group.banksMap.get('USDC')!.toString());
   } catch (error) {
     throw error;
   }

ts/client/src/scripts/example1-user2.ts

@@ -0,0 +1,134 @@
+import { AnchorProvider, Wallet } from '@project-serum/anchor';
+import { Connection, Keypair } from '@solana/web3.js';
+import fs from 'fs';
+import { AccountSize, HealthType } from '../accounts/mangoAccount';
+import { MangoClient } from '../client';
+import { MANGO_V4_ID } from '../constants';
+import { toUiDecimals } from '../utils';
+
+const GROUP_NUM = Number(process.env.GROUP_NUM || 0);
+
+async function main() {
+  const options = AnchorProvider.defaultOptions();
+  const connection = new Connection(
+    'https://mango.devnet.rpcpool.com',
+    options,
+  );
+
+  const user = Keypair.fromSecretKey(
+    Buffer.from(
+      JSON.parse(fs.readFileSync(process.env.USER2_KEYPAIR!, 'utf-8')),
+    ),
+  );
+  const userWallet = new Wallet(user);
+  const userProvider = new AnchorProvider(connection, userWallet, options);
+  const client = await MangoClient.connect(
+    userProvider,
+    'devnet',
+    MANGO_V4_ID['devnet'],
+  );
+  console.log(`User ${userWallet.publicKey.toBase58()}`);
+
+  // fetch group
+  const admin = Keypair.fromSecretKey(
+    Buffer.from(
+      JSON.parse(fs.readFileSync(process.env.ADMIN_KEYPAIR!, 'utf-8')),
+    ),
+  );
+  const group = await client.getGroupForAdmin(admin.publicKey, GROUP_NUM);
+  console.log(group.toString());
+
+  // create + fetch account
+  console.log(`Creating mangoaccount...`);
+  const mangoAccount = await client.getOrCreateMangoAccount(
+    group,
+    user.publicKey,
+    0,
+    AccountSize.small,
+    'my_mango_account',
+  );
+  console.log(`...created/found mangoAccount ${mangoAccount.publicKey}`);
+  console.log(mangoAccount.toString());
+
+  if (true) {
+    await group.reloadAll(client);
+    console.log(group.banksMap.get('USDC')!.toString());
+    console.log(group.banksMap.get('BTC')!.toString());
+  }
+
+  if (false) {
+    // deposit and withdraw
+    try {
+      console.log(`...depositing 0.0005 BTC`);
+      await client.tokenDeposit(group, mangoAccount, 'BTC', 0.0005);
+      await mangoAccount.reload(client, group);
+      console.log(`...withdrawing 5 USDC`);
+      await client.tokenWithdraw(group, mangoAccount, 'USDC', 50, true);
+      await mangoAccount.reload(client, group);
+    } catch (error) {
+      console.log(error);
+    }
+  }
+
+  if (true) {
+    await mangoAccount.reload(client, group);
+    console.log(
+      '...mangoAccount.getEquity() ' +
+        toUiDecimals(mangoAccount.getEquity().toNumber()),
+    );
+    console.log(
+      '...mangoAccount.getCollateralValue() ' +
+        toUiDecimals(mangoAccount.getCollateralValue().toNumber()),
+    );
+    console.log(
+      '...mangoAccount.accountData["healthCache"].health(HealthType.init) ' +
+        toUiDecimals(
+          mangoAccount.accountData['healthCache']
+            .health(HealthType.init)
+            .toNumber(),
+        ),
+    );
+    console.log(
+      '...mangoAccount.getAssetsVal() ' +
+        toUiDecimals(mangoAccount.getAssetsVal().toNumber()),
+    );
+    console.log(
+      '...mangoAccount.getLiabsVal() ' +
+        toUiDecimals(mangoAccount.getLiabsVal().toNumber()),
+    );
+    console.log(
+      '...mangoAccount.getMaxWithdrawWithBorrowForToken(group, "SOL") ' +
+        toUiDecimals(
+          (
+            await mangoAccount.getMaxWithdrawWithBorrowForToken(group, 'SOL')
+          ).toNumber(),
+        ),
+    );
+    console.log(
+      "...mangoAccount.getSerum3MarketMarginAvailable(group, 'BTC/USDC') " +
+        toUiDecimals(
+          mangoAccount
+            .getSerum3MarketMarginAvailable(group, 'BTC/USDC')
+            .toNumber(),
+        ),
+    );
+    console.log(
+      "...mangoAccount.getPerpMarketMarginAvailable(group, 'BTC-PERP') " +
+        toUiDecimals(
+          mangoAccount
+            .getPerpMarketMarginAvailable(group, 'BTC-PERP')
+            .toNumber(),
+        ),
+    );
+  }
+
+  if (true) {
+    await group.reloadAll(client);
+    console.log(group.banksMap.get('USDC')!.toString());
+    console.log(group.banksMap.get('BTC')!.toString());
+  }
+
+  process.exit();
+}
+
+main();

ts/client/src/scripts/scratch/example1-create-liquidation-candidate.ts

@@ -99,7 +99,9 @@ async function main() {
   await user2MangoAccount.reload(user2Client, group);
   console.log(`${user2MangoAccount.toString(group)}`);
 
-  const maxNative = await (await user2MangoAccount.getMaxWithdrawWithBorrowForToken(group, token)).toNumber();
+  const maxNative = await (
+    await user2MangoAccount.getMaxWithdrawWithBorrowForToken(group, token)
+  ).toNumber();
   amount = 0.9 * maxNative;
   console.log(`Withdrawing...${amount} native BTC'`);
   await user2Client.tokenWithdraw2(