Merge pull request #9 from blockworks-foundation/pan/improvements

Bug Fixes & Performance Enhancements

.eslintrc.json

@@ -0,0 +1,40 @@
+{
+    "parser": "@typescript-eslint/parser",
+    "plugins": ["@typescript-eslint"],
+    "extends": [
+      "eslint:recommended",
+      "plugin:@typescript-eslint/recommended",
+      "prettier"
+    ],
+    "env": {
+      "es6": true,
+      "browser": true,
+      "jest": true,
+      "node": true
+    },
+    "rules": {
+      "@typescript-eslint/explicit-module-boundary-types": 0,
+      "@typescript-eslint/ban-ts-comment": 0,
+      "@typescript-eslint/explicit-function-return-type": 0,
+      "@typescript-eslint/explicit-member-accessibility": 0,
+      "@typescript-eslint/indent": 0,
+      "@typescript-eslint/member-delimiter-style": 0,
+      "@typescript-eslint/no-empty-function": 0,
+      "@typescript-eslint/no-explicit-any": 0,
+      "@typescript-eslint/no-var-requires": 0,
+      "@typescript-eslint/no-use-before-define": 0,
+      "@typescript-eslint/no-unused-vars": [
+        2,
+        {
+          "argsIgnorePattern": "^_"
+        }
+      ]
+      // "no-console": [
+      //   0,
+      //   {
+      //     "allow": ["warn", "error"]
+      //   }
+      // ]
+    }
+  }
+  

README.md

@@ -14,17 +14,19 @@ To run the liquidator you will need:
 | Variable | Default | Description |
 | -------- | ------- | ----------- |
 | `CLUSTER` | `mainnet` | The Solana cluster to use |
-| `ENDPOINT_URL` | `https://solana-api.projectserum.com` | Your RPC node endpoint |
+| `ENDPOINT_URL` | `https://mango.rpcpool.com/946ef7337da3f5b8d3e4a34e7f88` | Your RPC node endpoint |
 | `KEYPAIR` | `${HOME}/.config/solana/id.json` | The location of your wallet keypair |
 | `GROUP` | `mainnet.1` | Name of the group in ids.json to run the Liquidator against |
 | `TARGETS` | `0 0 0 0 0 0 0 0` | Space separated list of the amount of each asset to maintain when rebalancing |
 | `INTERVAL` | `3500` | Milliseconds to wait before checking for sick accounts |
-| `INTERVAL_ACCOUNTS` | `120000` | Milliseconds to wait before reloading all Mango accounts |
+| `INTERVAL_ACCOUNTS` | `600000` | Milliseconds to wait before reloading all Mango accounts |
 | `INTERVAL_WEBSOCKET` | `300000` | Milliseconds to wait before reconnecting to the websocket |
-| `LIQOR_PK` | N/A | Liqor Mango account Public Key, by default uses the largest value account owned by the keypair |
+| `INTERVAL_REBALANCE` | `10000` | Milliseconds to wait before reconnecting to the websocket |
+| `LIQOR_PK` | N/A | Liqor Mango Account Public Key, by default uses the largest value account owned by the keypair |
 | `WEBHOOK_URL` | N/A | Discord webhook URL to post liquidation events and errors to |
+| `LIAB_LIMIT` | `0.9` | Percentage of your available margin to use when taking on liabs |
 
-You can add these varibles to a `.env` file in the project root to load automatically on liquidator startup. For example:
+You can add these variables to a `.env` file in the project root to load automatically on liquidator startup. For example:
 ```bash
 ENDPOINT_URL=https://solana-api.projectserum.com
 KEYPAIR=${HOME}/.config/solana/my-keypair.json 
@@ -33,6 +35,9 @@ TARGETS=500 0.1 0.75 0 0 0 0 0
 ## Rebalancing
 The liquidator will attempt to close all perp positions, and balance the tokens in the liqor account after each liquidation. By default it will sell all token assets into USDC. You can choose to maintain a certain amount of each asset through this process by editing the value in the `TARGETS` environment variable at the position of the asset. You can find the order of the assets in the 'oracles' property of the group in [ids.json](https://github.com/blockworks-foundation/mango-client-v3/blob/main/src/ids.json#L81) The program will attempt to make buy/sell orders during balancing to maintain this level.
 
+## Advanced Orders Triggering
+The liquidator triggers advanced orders for users when their trigger condition is met. Upon successfully triggering the order, the liquidator wallet will receive 100x the transaction fee as a reward.
+
 ## Run
 ```
 yarn liquidator

package.json

@@ -25,7 +25,6 @@
     "eslint-config-prettier": "^7.2.0",
     "mocha": "^8.4.0",
     "prettier": "^2.0.5",
-    "ts-node": "^9.1.1",
     "typedoc": "^0.22.5",
     "typescript": "^4.1.3"
   },
@@ -49,6 +48,7 @@
     "bn.js": "^5.2.0",
     "buffer-layout": "^1.2.1",
     "dotenv": "^10.0.0",
-    "dotenv-expand": "^5.1.0"
+    "dotenv-expand": "^5.1.0",
+    "ts-node": "^9.1.1"
   }
 }

src/liquidator.ts

@@ -34,14 +34,20 @@ envExpand(Env.config());
 
 const interval = parseInt(process.env.INTERVAL || '3500');
 const refreshAccountsInterval = parseInt(
-  process.env.INTERVAL_ACCOUNTS || '120000',
+  process.env.INTERVAL_ACCOUNTS || '600000',
 );
 const refreshWebsocketInterval = parseInt(
   process.env.INTERVAL_WEBSOCKET || '300000',
 );
+const rebalanceInterval = parseInt(process.env.INTERVAL_REBALANCE || '10000');
 const checkTriggers = process.env.CHECK_TRIGGERS
   ? process.env.CHECK_TRIGGERS === 'true'
   : true;
+const liabLimit = I80F48.fromNumber(
+  Math.min(parseFloat(process.env.LIAB_LIMIT || '0.9'), 1),
+);
+let lastRebalance = Date.now();
+
 const config = new Config(IDS);
 
 const cluster = (process.env.CLUSTER || 'mainnet') as Cluster;
@@ -68,70 +74,25 @@ const payer = new Account(
   ),
 );
 console.log(`Payer: ${payer.publicKey.toBase58()}`);
-
+const rpcEndpoint = process.env.ENDPOINT_URL || config.cluster_urls[cluster];
-const connection = new Connection(
+const connection = new Connection(rpcEndpoint, 'processed' as Commitment);
-  process.env.ENDPOINT_URL || config.cluster_urls[cluster],
-  'processed' as Commitment,
-);
 const client = new MangoClient(connection, mangoProgramId);
 
 let mangoSubscriptionId = -1;
 let dexSubscriptionId = -1;
 
-/**
- * Process trigger orders for one mango account
- */
-async function processTriggerOrders(
-  mangoGroup: MangoGroup,
-  cache: MangoCache,
-  perpMarkets: PerpMarket[],
-  mangoAccount: MangoAccount,
-) {
-  if (!groupIds) {
-    throw new Error(`Group ${groupName} not found`);
-  }
-
-  for (let i = 0; i < mangoAccount.advancedOrders.length; i++) {
-    const order = mangoAccount.advancedOrders[i];
-    if (!(order.perpTrigger && order.perpTrigger.isActive)) {
-      continue;
-    }
-
-    const trigger = order.perpTrigger;
-    const currentPrice = cache.priceCache[trigger.marketIndex].price;
-    const configMarketIndex = groupIds.perpMarkets.findIndex(
-      (pm) => pm.marketIndex === trigger.marketIndex,
-    );
-    if (
-      (trigger.triggerCondition == 'above' &&
-        currentPrice.gt(trigger.triggerPrice)) ||
-      (trigger.triggerCondition == 'below' &&
-        currentPrice.lt(trigger.triggerPrice))
-    ) {
-      console.log(
-        `Executing order for account ${mangoAccount.publicKey.toBase58()}`,
-      );
-      await client.executePerpTriggerOrder(
-        mangoGroup,
-        mangoAccount,
-        cache,
-        perpMarkets[configMarketIndex],
-        payer,
-        i,
-      );
-    }
-  }
-}
-
 async function main() {
   if (!groupIds) {
     throw new Error(`Group ${groupName} not found`);
   }
   console.log(`Starting liquidator for ${groupName}...`);
+  console.log(`RPC Endpoint: ${rpcEndpoint}`);
+
   const mangoGroup = await client.getMangoGroup(mangoGroupKey);
   let cache = await mangoGroup.loadCache(connection);
   let liqorMangoAccount: MangoAccount;
 
+  try {
     if (process.env.LIQOR_PK) {
       liqorMangoAccount = await client.getMangoAccount(
         new PublicKey(process.env.LIQOR_PK),
@@ -158,11 +119,17 @@ async function main() {
         throw new Error('No Mango Account found for this Keypair');
       }
     }
+  } catch (err: any) {
+    console.error(`Error loading liqor Mango Account: ${err}`);
+    return;
+  }
 
   console.log(`Liqor Public Key: ${liqorMangoAccount.publicKey.toBase58()}`);
+
   let mangoAccounts: MangoAccount[] = [];
   await refreshAccounts(mangoGroup, mangoAccounts);
   watchAccounts(groupIds.mangoProgramId, mangoGroup, mangoAccounts);
+
   const perpMarkets = await Promise.all(
     groupIds.perpMarkets.map((perpMarket) => {
       return mangoGroup.loadPerpMarket(
@@ -199,7 +166,7 @@ async function main() {
         const advancedOrders = await getMultipleAccounts(connection, allAOs);
         [cache, liqorMangoAccount] = await Promise.all([
           mangoGroup.loadCache(connection),
-          liqorMangoAccount.reload(connection),
+          liqorMangoAccount.reload(connection, mangoGroup.dexProgramId),
         ]);
 
         mangoAccountsWithAOs.forEach((ma, i) => {
@@ -211,7 +178,7 @@ async function main() {
       } else {
         [cache, liqorMangoAccount] = await Promise.all([
           mangoGroup.loadCache(connection),
-          liqorMangoAccount.reload(connection),
+          liqorMangoAccount.reload(connection, mangoGroup.dexProgramId),
         ]);
       }
 
@@ -227,11 +194,22 @@ async function main() {
               perpMarkets,
               mangoAccount,
             );
-          } catch (err) {
+          } catch (err: any) {
+            if (err.message.includes('MangoErrorCode::InvalidParam')) {
               console.error(
-              `Failed to execute trigger order for ${mangoAccountKeyString}`,
+                'Failed to execute trigger order, order already executed',
-              err,
               );
+            } else if (
+              err.message.includes('MangoErrorCode::TriggerConditionFalse')
+            ) {
+              console.error(
+                'Failed to execute trigger order, trigger condition was false',
+              );
+            } else {
+              console.error(
+                `Failed to execute trigger order for ${mangoAccountKeyString}: ${err}`,
+              );
+            }
           }
         }
 
@@ -250,13 +228,15 @@ async function main() {
         }
 
         const health = mangoAccount.getHealthRatio(mangoGroup, cache, 'Maint');
+        const accountInfoString = mangoAccount.toPrettyString(
+          groupIds,
+          mangoGroup,
+          cache,
+        );
         console.log(
-          `Sick account ${mangoAccountKeyString} health ratio: ${health.toString()}`,
+          `Sick account ${mangoAccountKeyString} health ratio: ${health.toString()}\n${accountInfoString}`,
         );
-        notify(
+        notify(`Sick account\n${accountInfoString}`);
-          `Sick account ${mangoAccountKeyString} health ratio: ${health.toString()}`,
-        );
-        console.log(mangoAccount.toPrettyString(groupIds, mangoGroup, cache));
         try {
           await liquidateAccount(
             mangoGroup,
@@ -270,11 +250,9 @@ async function main() {
 
           console.log('Liquidated account', mangoAccountKeyString);
           notify(`Liquidated account ${mangoAccountKeyString}`);
-        } catch (err) {
+        } catch (err: any) {
           console.error(
-            'Failed to liquidate account',
+            `Failed to liquidate account ${mangoAccountKeyString}: ${err}`,
-            mangoAccountKeyString,
-            err,
           );
           notify(
             `Failed to liquidate account ${mangoAccountKeyString}: ${err}`,
@@ -331,7 +309,7 @@ function watchAccounts(
 
     mangoSubscriptionId = connection.onProgramAccountChange(
       mangoProgramId,
-      ({ accountId, accountInfo }) => {
+      async ({ accountId, accountInfo }) => {
         const index = mangoAccounts.findIndex((account) =>
           account.publicKey.equals(accountId),
         );
@@ -341,14 +319,17 @@ function watchAccounts(
           MangoAccountLayout.decode(accountInfo.data),
         );
         if (index == -1) {
-          //console.log('New Account');
           mangoAccounts.push(mangoAccount);
         } else {
           const spotOpenOrdersAccounts =
             mangoAccounts[index].spotOpenOrdersAccounts;
           mangoAccount.spotOpenOrdersAccounts = spotOpenOrdersAccounts;
           mangoAccounts[index] = mangoAccount;
-          //console.log('Updated account ' + accountId.toBase58());
+          await mangoAccount.loadOpenOrders(
+            connection,
+            mangoGroup.dexProgramId,
+          );
+          console.log('updated account', mangoAccount.publicKey.toBase58())
         }
       },
       'processed',
@@ -381,7 +362,6 @@ function watchAccounts(
           );
           mangoAccounts[ownerIndex].spotOpenOrdersAccounts[openOrdersIndex] =
             openOrders;
-          //console.log('Updated OpenOrders for account ' + mangoAccounts[ownerIndex].publicKey.toBase58());
         } else {
           console.error('Could not match OpenOrdersAccount to MangoAccount');
         }
@@ -417,15 +397,18 @@ async function refreshAccounts(
   try {
     console.log('Refreshing accounts...');
     console.time('getAllMangoAccounts');
-    mangoAccounts.splice(0, mangoAccounts.length, ...(await client.getAllMangoAccounts(
+
-      mangoGroup,
+    mangoAccounts.splice(
-      undefined,
+      0,
-      true,
+      mangoAccounts.length,
-    )));
+      ...(await client.getAllMangoAccounts(mangoGroup, undefined, true)),
+    );
+    shuffleArray(mangoAccounts);
+
     console.timeEnd('getAllMangoAccounts');
     console.log(`Fetched ${mangoAccounts.length} accounts`);
-  } catch (err) {
+  } catch (err: any) {
-    console.error('Error reloading accounts', err);
+    console.error(`Error reloading accounts: ${err}`);
   } finally {
     setTimeout(
       refreshAccounts,
@@ -436,6 +419,51 @@ async function refreshAccounts(
   }
 }
 
+/**
+ * Process trigger orders for one mango account
+ */
+async function processTriggerOrders(
+  mangoGroup: MangoGroup,
+  cache: MangoCache,
+  perpMarkets: PerpMarket[],
+  mangoAccount: MangoAccount,
+) {
+  if (!groupIds) {
+    throw new Error(`Group ${groupName} not found`);
+  }
+
+  for (let i = 0; i < mangoAccount.advancedOrders.length; i++) {
+    const order = mangoAccount.advancedOrders[i];
+    if (!(order.perpTrigger && order.perpTrigger.isActive)) {
+      continue;
+    }
+
+    const trigger = order.perpTrigger;
+    const currentPrice = cache.priceCache[trigger.marketIndex].price;
+    const configMarketIndex = groupIds.perpMarkets.findIndex(
+      (pm) => pm.marketIndex === trigger.marketIndex,
+    );
+    if (
+      (trigger.triggerCondition == 'above' &&
+        currentPrice.gt(trigger.triggerPrice)) ||
+      (trigger.triggerCondition == 'below' &&
+        currentPrice.lt(trigger.triggerPrice))
+    ) {
+      console.log(
+        `Executing order for account ${mangoAccount.publicKey.toBase58()}`,
+      );
+      await client.executePerpTriggerOrder(
+        mangoGroup,
+        mangoAccount,
+        cache,
+        perpMarkets[configMarketIndex],
+        payer,
+        i,
+      );
+    }
+  }
+}
+
 async function liquidateAccount(
   mangoGroup: MangoGroup,
   cache: MangoCache,
@@ -462,21 +490,20 @@ async function liquidateAccount(
         );
       }),
     );
-    await sleep(interval * 2);
-  }
     await liqee.reload(connection, mangoGroup.dexProgramId);
     if (!liqee.isLiquidatable(mangoGroup, cache)) {
       throw new Error('Account no longer liquidatable');
     }
+  }
 
-  while (liqee.hasAnySpotOrders()) {
+  for (let r = 0; r < 5 && liqee.hasAnySpotOrders(); r++) {
     for (let i = 0; i < mangoGroup.spotMarkets.length; i++) {
+      if (liqee.inMarginBasket[i]) {
         const spotMarket = spotMarkets[i];
         const baseRootBank = rootBanks[i];
         const quoteRootBank = rootBanks[QUOTE_INDEX];
 
         if (baseRootBank && quoteRootBank) {
-        if (liqee.inMarginBasket[i]) {
           console.log('forceCancelOrders ', i);
           await client.forceCancelSpotOrders(
             mangoGroup,
@@ -506,14 +533,6 @@ async function liquidateAccount(
     healthComponents.quote,
     'Maint',
   );
-  const initHealths = liqee.getHealthsFromComponents(
-    mangoGroup,
-    cache,
-    healthComponents.spot,
-    healthComponents.perps,
-    healthComponents.quote,
-    'Init',
-  );
 
   let shouldLiquidateSpot = false;
   for (let i = 0; i < mangoGroup.tokens.length; i++) {
@@ -522,32 +541,23 @@ async function liquidateAccount(
       break;
     }
   }
-  const shouldLiquidatePerps =
-    maintHealths.perp.lt(ZERO_I80F48) ||
-    (initHealths.perp.lt(ZERO_I80F48) && liqee.beingLiquidated);
 
   if (shouldLiquidateSpot) {
     await liquidateSpot(
       mangoGroup,
       cache,
-      spotMarkets,
       perpMarkets,
       rootBanks,
       liqee,
       liqor,
     );
+    await liqee.reload(connection, mangoGroup.dexProgramId);
+    if (!liqee.isLiquidatable(mangoGroup, cache)) {
+      return;
+    }
   }
 
-  if (shouldLiquidatePerps) {
+  await liquidatePerps(mangoGroup, cache, perpMarkets, rootBanks, liqee, liqor);
-    await liquidatePerps(
-      mangoGroup,
-      cache,
-      perpMarkets,
-      rootBanks,
-      liqee,
-      liqor,
-    );
-  }
 
   if (
     !shouldLiquidateSpot &&
@@ -555,6 +565,7 @@ async function liquidateAccount(
     liqee.beingLiquidated
   ) {
     // Send a ForceCancelPerp to reset the being_liquidated flag
+    console.log('forceCancelAllPerpOrdersInMarket');
     await client.forceCancelAllPerpOrdersInMarket(
       mangoGroup,
       liqee,
@@ -568,7 +579,6 @@ async function liquidateAccount(
 async function liquidateSpot(
   mangoGroup: MangoGroup,
   cache: MangoCache,
-  spotMarkets: Market[],
   perpMarkets: PerpMarket[],
   rootBanks: (RootBank | undefined)[],
   liqee: MangoAccount,
@@ -616,11 +626,13 @@ async function liquidateSpot(
       ? mangoGroup.spotMarkets[maxNetIndex].initAssetWeight
       : ONE_I80F48;
 
-    const maxLiabTransfer = liqorInitHealth.div(
+    const maxLiabTransfer = liqorInitHealth
+      .div(
         mangoGroup
           .getPriceNative(minNetIndex, cache)
           .mul(liabInitLiabWeight.sub(assetInitAssetWeight).abs()),
-    );
+      )
+      .mul(liabLimit);
 
     if (liqee.isBankrupt) {
       console.log('Bankrupt account', liqee.publicKey.toBase58());
@@ -638,14 +650,7 @@ async function liquidateSpot(
         await liqee.reload(connection, mangoGroup.dexProgramId);
       }
     } else {
-      console.log(
+      if (maxNet.lt(ZERO_I80F48) || maxNetIndex == -1) {
-        `Liquidating max ${maxLiabTransfer.toString()}/${liqee.getNativeBorrow(
-          liabRootBank,
-          minNetIndex,
-        )} of liab ${minNetIndex}, asset ${maxNetIndex}`,
-      );
-      console.log(maxNet.toString());
-      if (maxNet.lt(ONE_I80F48) || maxNetIndex == -1) {
         const highestHealthMarket = perpMarkets
           .map((perpMarket, i) => {
             const marketIndex = mangoGroup.getPerpMarketIndex(
@@ -669,14 +674,14 @@ async function liquidateSpot(
             return b.perpHealth.sub(a.perpHealth).toNumber();
           })[0];
 
-        let maxLiabTransfer = liqorInitHealth;
+        let maxLiabTransfer = liqorInitHealth.mul(liabLimit);
         if (maxNetIndex !== QUOTE_INDEX) {
-          maxLiabTransfer = liqorInitHealth.div(
+          maxLiabTransfer = liqorInitHealth
-            ONE_I80F48.sub(assetInitAssetWeight),
+            .div(ONE_I80F48.sub(assetInitAssetWeight))
-          );
+            .mul(liabLimit);
         }
 
-        console.log('liquidateTokenAndPerp ' + highestHealthMarket.marketIndex);
+        console.log('liquidateTokenAndPerp', highestHealthMarket.marketIndex);
         await client.liquidateTokenAndPerp(
           mangoGroup,
           liqee,
@@ -687,9 +692,10 @@ async function liquidateSpot(
           highestHealthMarket.marketIndex,
           AssetType.Token,
           minNetIndex,
-          liqee.perpAccounts[highestHealthMarket.marketIndex].quotePosition,
+          maxLiabTransfer,
         );
       } else {
+        console.log('liquidateTokenAndToken', maxNetIndex, minNetIndex);
         await client.liquidateTokenAndToken(
           mangoGroup,
           liqee,
@@ -759,18 +765,14 @@ async function liquidatePerps(
   const marketIndex = lowestHealthMarket.marketIndex;
   const perpAccount = liqee.perpAccounts[marketIndex];
   const perpMarket = perpMarkets[lowestHealthMarket.i];
-  // const baseRootBank = rootBanks[marketIndex];
-  //
-  // if (!baseRootBank) {
-  //   throw new Error(`Base root bank not found for ${marketIndex}`);
-  // }
 
   if (!perpMarket) {
     throw new Error(`Perp market not found for ${marketIndex}`);
   }
 
+  const liqorInitHealth = liqor.getHealth(mangoGroup, cache, 'Init');
+  let maxLiabTransfer = liqorInitHealth.mul(liabLimit);
   if (liqee.isBankrupt) {
-    const maxLiabTransfer = perpAccount.quotePosition.abs();
     const quoteRootBank = rootBanks[QUOTE_INDEX];
     if (quoteRootBank) {
       // don't do anything it if quote position is zero
@@ -808,13 +810,15 @@ async function liquidatePerps(
     if (perpAccount.basePosition.isZero()) {
       if (assetRootBank) {
         // we know that since sum of perp healths is negative, lowest perp market must be negative
-        console.log('liquidateTokenAndPerp ' + marketIndex);
+        console.log('liquidateTokenAndPerp', marketIndex);
-        // maxLiabTransfer
-        let maxLiabTransfer = liqorInitHealth;
         if (maxNetIndex !== QUOTE_INDEX) {
-          maxLiabTransfer = liqorInitHealth.div(
+          maxLiabTransfer = liqorInitHealth
-            ONE_I80F48.sub(mangoGroup.spotMarkets[maxNetIndex].initAssetWeight),
+            .div(
-          );
+              ONE_I80F48.sub(
+                mangoGroup.spotMarkets[maxNetIndex].initAssetWeight,
+              ),
+            )
+            .mul(liabLimit);
         }
         await client.liquidateTokenAndPerp(
           mangoGroup,
@@ -830,7 +834,7 @@ async function liquidatePerps(
         );
       }
     } else {
-      console.log('liquidatePerpMarket ' + marketIndex);
+      console.log('liquidatePerpMarket', marketIndex);
 
       // technically can be higher because of liquidation fee, but
       // let's just give ourselves extra room
@@ -846,6 +850,7 @@ async function liquidatePerps(
             .div(mangoGroup.getPriceNative(marketIndex, cache))
             .div(I80F48.fromI64(perpMarketInfo.baseLotSize))
             .floor()
+            .mul(liabLimit)
             .toNumber(),
         );
       } else {
@@ -855,6 +860,7 @@ async function liquidatePerps(
             .div(mangoGroup.getPriceNative(marketIndex, cache))
             .div(I80F48.fromI64(perpMarketInfo.baseLotSize))
             .floor()
+            .mul(liabLimit)
             .toNumber(),
         ).neg();
       }
@@ -869,10 +875,9 @@ async function liquidatePerps(
       );
     }
 
-    await sleep(interval);
     await liqee.reload(connection, mangoGroup.dexProgramId);
     if (liqee.isBankrupt) {
-      const maxLiabTransfer = perpAccount.quotePosition.abs();
+      const maxLiabTransfer = liqorInitHealth.mul(liabLimit);
       const quoteRootBank = rootBanks[QUOTE_INDEX];
       if (quoteRootBank) {
         console.log('resolvePerpBankruptcy', maxLiabTransfer.toString());
@@ -914,6 +919,46 @@ function getDiffsAndNet(
   return { diffs, netValues };
 }
 
+async function balanceAccount(
+  mangoGroup: MangoGroup,
+  mangoAccount: MangoAccount,
+  mangoCache: MangoCache,
+  spotMarkets: Market[],
+  perpMarkets: PerpMarket[],
+) {
+  if (Date.now() < lastRebalance + rebalanceInterval) {
+    return;
+  }
+
+  const { diffs, netValues } = getDiffsAndNet(
+    mangoGroup,
+    mangoAccount,
+    mangoCache,
+  );
+  const tokensUnbalanced = netValues.some(
+    (nv) => Math.abs(diffs[nv[0]].toNumber()) > spotMarkets[nv[0]].minOrderSize,
+  );
+  const positionsUnbalanced = perpMarkets.some((pm) => {
+    const index = mangoGroup.getPerpMarketIndex(pm.publicKey);
+    const perpAccount = mangoAccount.perpAccounts[index];
+    const basePositionSize = Math.abs(
+      pm.baseLotsToNumber(perpAccount.basePosition),
+    );
+
+    return basePositionSize != 0 || perpAccount.quotePosition.gt(ZERO_I80F48);
+  });
+
+  if (tokensUnbalanced) {
+    await balanceTokens(mangoGroup, mangoAccount, spotMarkets);
+  }
+
+  if (positionsUnbalanced) {
+    await closePositions(mangoGroup, mangoAccount, perpMarkets);
+  }
+
+  lastRebalance = Date.now();
+}
+
 async function balanceTokens(
   mangoGroup: MangoGroup,
   mangoAccount: MangoAccount,
@@ -956,7 +1001,7 @@ async function balanceTokens(
         );
       }
     }
-    console.log('Cancelling ' + cancelOrdersPromises.length + ' orders');
+    console.log(`Cancelling ${cancelOrdersPromises.length} orders`);
     await Promise.all(cancelOrdersPromises);
 
     const openOrders = await mangoAccount.loadOpenOrders(
@@ -976,7 +1021,7 @@ async function balanceTokens(
         );
       }
     }
-    console.log('Settling on ' + settlePromises.length + ' markets');
+    console.log(`Settling on ${settlePromises.length} markets`);
     await Promise.all(settlePromises);
 
     const { diffs, netValues } = getDiffsAndNet(
@@ -989,44 +1034,18 @@ async function balanceTokens(
     for (let i = 0; i < groupIds!.spotMarkets.length; i++) {
       const marketIndex = netValues[i][0];
       const market = markets[marketIndex];
+      const liquidationFee = mangoGroup.spotMarkets[marketIndex].liquidationFee;
       if (Math.abs(diffs[marketIndex].toNumber()) > market.minOrderSize) {
-        if (netValues[i][1].gt(ZERO_I80F48)) {
+        const side = netValues[i][1].gt(ZERO_I80F48) ? 'sell' : 'buy';
-          // sell to close
         const price = mangoGroup
           .getPrice(marketIndex, cache)
-            .mul(I80F48.fromNumber(0.95));
+          .mul(ONE_I80F48.sub(liquidationFee))
-          console.log(
+          .toNumber();
-            `Sell to close ${marketIndex} ${Math.abs(
+        const quantity = Math.abs(diffs[marketIndex].toNumber());
-              diffs[marketIndex].toNumber(),
-            )} @ ${price.toString()}`,
-          );
-          await client.placeSpotOrder(
-            mangoGroup,
-            mangoAccount,
-            mangoGroup.mangoCache,
-            markets[marketIndex],
-            payer,
-            'sell',
-            price.toNumber(),
-            Math.abs(diffs[marketIndex].toNumber()),
-            'limit',
-          );
-          await client.settleFunds(
-            mangoGroup,
-            mangoAccount,
-            payer,
-            markets[marketIndex],
-          );
-        } else if (netValues[i][1].lt(ZERO_I80F48)) {
-          //buy to close
-          const price = mangoGroup
-            .getPrice(marketIndex, cache)
-            .mul(I80F48.fromNumber(1.05));
 
         console.log(
-            `Buy to close ${marketIndex} ${Math.abs(
+          `${side}ing ${quantity} of ${groupIds?.spotMarkets[marketIndex].baseSymbol} for $${price}`,
-              diffs[marketIndex].toNumber(),
+          ONE_I80F48.sub(liquidationFee).toString(),
-            )} @ ${price.toString()}`,
         );
         await client.placeSpotOrder(
           mangoGroup,
@@ -1034,8 +1053,8 @@ async function balanceTokens(
           mangoGroup.mangoCache,
           markets[marketIndex],
           payer,
-            'buy',
+          side,
-            price.toNumber(),
+          price,
           Math.abs(diffs[marketIndex].toNumber()),
           'limit',
         );
@@ -1047,46 +1066,11 @@ async function balanceTokens(
         );
       }
     }
-    }
   } catch (err) {
     console.error('Error rebalancing tokens', err);
   }
 }
 
-async function balanceAccount(
-  mangoGroup: MangoGroup,
-  mangoAccount: MangoAccount,
-  mangoCache: MangoCache,
-  spotMarkets: Market[],
-  perpMarkets: PerpMarket[],
-) {
-  const { diffs, netValues } = getDiffsAndNet(
-    mangoGroup,
-    mangoAccount,
-    mangoCache,
-  );
-  const tokensUnbalanced = netValues.some(
-    (nv) => Math.abs(diffs[nv[0]].toNumber()) > spotMarkets[nv[0]].minOrderSize,
-  );
-  const positionsUnbalanced = perpMarkets.some((pm) => {
-    const index = mangoGroup.getPerpMarketIndex(pm.publicKey);
-    const perpAccount = mangoAccount.perpAccounts[index];
-    const basePositionSize = Math.abs(
-      pm.baseLotsToNumber(perpAccount.basePosition),
-    );
-
-    return basePositionSize != 0 || perpAccount.quotePosition.gt(ZERO_I80F48);
-  });
-
-  if (tokensUnbalanced) {
-    await balanceTokens(mangoGroup, mangoAccount, spotMarkets);
-  }
-
-  if (positionsUnbalanced) {
-    await closePositions(mangoGroup, mangoAccount, perpMarkets);
-  }
-}
-
 async function closePositions(
   mangoGroup: MangoGroup,
   mangoAccount: MangoAccount,
@@ -1125,21 +1109,20 @@ async function closePositions(
 
         if (basePositionSize != 0) {
           const side = perpAccount.basePosition.gt(ZERO_BN) ? 'sell' : 'buy';
-          // const liquidationFee =
+          const liquidationFee = mangoGroup.perpMarkets[index].liquidationFee;
-          //   mangoGroup.perpMarkets[index].liquidationFee.toNumber();
-
           const orderPrice =
-            side == 'sell' ? price.toNumber() * 0.95 : price.toNumber() * 1.05; // TODO: base this on liquidation fee
+            side == 'sell'
+              ? price.mul(ONE_I80F48.sub(liquidationFee)).toNumber()
+              : price.mul(ONE_I80F48.add(liquidationFee)).toNumber();
+          const bookSideInfo =
+            side == 'sell'
+              ? await connection.getAccountInfo(perpMarket.bids)
+              : await connection.getAccountInfo(perpMarket.asks);
 
           console.log(
-            side +
+            `${side}ing ${basePositionSize} of ${groupIds?.perpMarkets[i].baseSymbol}-PERP for $${orderPrice}`,
-              'ing ' +
-              basePositionSize +
-              ' of perp ' +
-              i +
-              ' for $' +
-              orderPrice,
           );
+
           await client.placePerpOrder(
             mangoGroup,
             mangoAccount,
@@ -1150,8 +1133,8 @@ async function closePositions(
             orderPrice,
             basePositionSize,
             'ioc',
-            undefined,
+            0,
-            undefined,
+            bookSideInfo ? bookSideInfo : undefined,
             true,
           );
         }
@@ -1180,6 +1163,13 @@ async function closePositions(
   }
 }
 
+function shuffleArray(array) {
+  for (let i = array.length - 1; i > 0; i--) {
+    const j = Math.floor(Math.random() * (i + 1));
+    [array[i], array[j]] = [array[j], array[i]];
+  }
+}
+
 function notify(content: string) {
   if (content && process.env.WEBHOOK_URL) {
     try {
@@ -1190,4 +1180,8 @@ function notify(content: string) {
   }
 }
 
+process.on('unhandledRejection', (err) => {
+  console.error(`Unhandled rejection: ${err})`);
+});
+
 main();

yarn.lock

@@ -31,8 +31,8 @@
     regenerator-runtime "^0.13.4"
 
 "@blockworks-foundation/mango-client@git+https://github.com/blockworks-foundation/mango-client-v3.git":
-  version "3.2.9"
+  version "3.2.14"
-  resolved "git+https://github.com/blockworks-foundation/mango-client-v3.git#a24f41584cff5a7548e3f9aafb3357036c55317c"
+  resolved "git+https://github.com/blockworks-foundation/mango-client-v3.git#7fb0f294a6c7cec98348cefbbfc3a725bc10c232"
   dependencies:
     "@project-serum/anchor" "^0.16.2"
     "@project-serum/serum" "0.13.55"