Merge branch 'dev'

.dockerignore

@@ -0,0 +1,19 @@
+.DS_Store
+
+.anchor
+test-ledger
+target
+
+**/*.rs.bk
+
+dist
+node_modules
+yarn-error.log
+
+.idea
+.vscode
+
+programs/margin-trade/src/lib-expanded.rs
+programs/mango-v4/src/lib-expanded.rs
+
+keeper/.env

.github/workflows/ci-docker-heroku-deploy.yml

@@ -0,0 +1,35 @@
+name: Deploy Docker Image to Heroku
+
+on:
+  workflow_dispatch:
+    inputs:
+      appName:
+        description: 'Heroku App Name'
+        required: true
+        type: string
+      imageName:
+        description: 'Docker Image Name'
+        required: true
+        type: string
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout 
+        uses: actions/checkout@v2
+
+      - name: Login
+        env: 
+          HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}
+        run: heroku container:login
+
+      - name: Push
+        env:
+          HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}
+        run: heroku container:push ${{ inputs.imageName }} -a ${{ inputs.appName }} --recursive 
+
+      - name: Release
+        env:
+          HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}
+        run: heroku container:release ${{ inputs.imageName }} -a ${{ inputs.appName }}

.github/workflows/ci-docker-publish.yml

@@ -0,0 +1,60 @@
+name: Publish Docker Image to GCR
+
+on:
+  push:
+    branches: [dev]
+  workflow_call:
+    secrets:
+      GCR_PROJECT:
+        required: true
+      GCR_SA_KEY:
+        required: true
+
+env:
+  PROJECT_ID: ${{ secrets.GCR_PROJECT }}
+  IMAGE: mango-v4
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          submodules: recursive
+      
+      # Use docker buildx
+      - name: Use docker buildx
+        uses: docker/setup-buildx-action@v2
+        id: buildx
+        with:
+          install: true
+          buildkitd-flags: --debug
+
+      # Login to Google Cloud
+      - name: 'Login to Google Cloud'
+        uses: 'google-github-actions/auth@v0'
+        id: auth
+        with:
+          token_format: 'access_token'
+          credentials_json: '${{ secrets.GCR_SA_KEY }}'
+
+      # Login to GCR
+      - name: Login to GCR
+        uses: docker/login-action@v2
+        with:
+          registry: us-docker.pkg.dev
+          username: oauth2accesstoken
+          password: ${{ steps.auth.outputs.access_token }}
+
+      # Build and push the image, leveraging layer caching
+      - name: Build and Push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: |
+            us-docker.pkg.dev/${{ env.PROJECT_ID }}/gcr.io/${{ env.IMAGE }}:${{ github.sha }}
+            us-docker.pkg.dev/${{ env.PROJECT_ID }}/gcr.io/${{ env.IMAGE }}:latest
+          cache-from: type=registry,ref=us-docker.pkg.dev/${{ env.PROJECT_ID }}/gcr.io/${{ env.IMAGE }}:buildcache
+          cache-to: type=registry,ref=us-docker.pkg.dev/${{ env.PROJECT_ID }}/gcr.io/${{ env.IMAGE }}:buildcache,mode=max

Dockerfile

@@ -0,0 +1,21 @@
+# Base image containing all binaries, deployed to gcr.io/mango-markets/mango-v4:latest
+FROM rust:1.60 as build
+RUN apt-get update && apt-get -y install clang cmake
+
+WORKDIR /app
+COPY ./ .
+
+# Hack to prevent a ghost member lib/init
+RUN sed -i 's|lib/\*|lib/checked_math|' Cargo.toml
+
+# Mount cache for downloaded and compiled dependencies
+RUN --mount=type=cache,target=/usr/local/cargo,from=rust,source=/usr/local/cargo \
+    --mount=type=cache,target=target \
+    cargo build --release --bins
+
+# Copy bins out of cache
+RUN --mount=type=cache,target=target mkdir .bin && cp target/release/keeper target/release/liquidator .bin/
+
+FROM debian:buster-slim as run
+RUN apt-get update && apt-get -y install ca-certificates
+COPY --from=build /app/.bin/* /usr/local/bin/

keeper/Dockerfile.keeper

@@ -0,0 +1,6 @@
+# Dockerfile for keeper service in Heroku
+# heroku container:push keeper -R -a HEROKU_APP_NAME
+# heroku container:release -a HEROKU_APP_NAME
+FROM us-docker.pkg.dev/mango-markets/gcr.io/mango-v4:latest
+ENTRYPOINT ["keeper"] 
+CMD ["crank"]

keeper/src/crank.rs

@@ -64,7 +64,7 @@ pub async fn loop_update_index_and_rate(
     mango_client: Arc<MangoClient>,
     token_indices: Vec<TokenIndex>,
 ) {
-    let mut interval = time::interval(Duration::from_secs(5));
+    let mut interval = time::interval(Duration::from_secs(60));
     loop {
         interval.tick().await;
 

liquidator/Dockerfile.liquidator

@@ -0,0 +1,5 @@
+# Dockerfile for keeper service in Heroku
+# heroku container:push keeper -R -a HEROKU_APP_NAME
+# heroku container:release -a HEROKU_APP_NAME
+FROM us-docker.pkg.dev/mango-markets/gcr.io/mango-v4:latest
+ENTRYPOINT ["liquidator"] 

ts/client/src/accounts/bank.ts

@@ -14,6 +14,8 @@ export class Bank {
   public name: string;
   public depositIndex: I80F48;
   public borrowIndex: I80F48;
+  public indexedDeposits: I80F48;
+  public indexedBorrows: I80F48;
   public cachedIndexedTotalDeposits: I80F48;
   public cachedIndexedTotalBorrows: I80F48;
   public avgUtilization: I80F48;
@@ -24,6 +26,8 @@ export class Bank {
   public util0: I80F48;
   public util1: I80F48;
   public price: I80F48;
+  public loanFeeRate: I80F48;
+  public loanOriginationFeeRate: I80F48;
   public initAssetWeight: I80F48;
   public maintAssetWeight: I80F48;
   public initLiabWeight: I80F48;
@@ -55,8 +59,8 @@ export class Bank {
       rate1: I80F48Dto;
       maxRate: I80F48Dto;
       collectedFeesNative: I80F48Dto;
-      loanOriginationFeeRate: I80F48Dto;
       loanFeeRate: I80F48Dto;
+      loanOriginationFeeRate: I80F48Dto;
       maintAssetWeight: I80F48Dto;
       initAssetWeight: I80F48Dto;
       maintLiabWeight: I80F48Dto;
@@ -94,8 +98,8 @@ export class Bank {
       obj.rate1,
       obj.maxRate,
       obj.collectedFeesNative,
-      obj.loanOriginationFeeRate,
       obj.loanFeeRate,
+      obj.loanOriginationFeeRate,
       obj.maintAssetWeight,
       obj.initAssetWeight,
       obj.maintLiabWeight,
@@ -134,8 +138,8 @@ export class Bank {
     rate1: I80F48Dto,
     maxRate: I80F48Dto,
     collectedFeesNative: I80F48Dto,
-    loanOriginationFeeRate: I80F48Dto,
     loanFeeRate: I80F48Dto,
+    loanOriginationFeeRate: I80F48Dto,
     maintAssetWeight: I80F48Dto,
     initAssetWeight: I80F48Dto,
     maintLiabWeight: I80F48Dto,
@@ -151,6 +155,8 @@ export class Bank {
     this.name = utf8.decode(new Uint8Array(name)).split('\x00')[0];
     this.depositIndex = I80F48.from(depositIndex);
     this.borrowIndex = I80F48.from(borrowIndex);
+    this.indexedDeposits = I80F48.from(indexedDeposits);
+    this.indexedBorrows = I80F48.from(indexedBorrows);
     this.cachedIndexedTotalDeposits = I80F48.from(indexedTotalDeposits);
     this.cachedIndexedTotalBorrows = I80F48.from(indexedTotalBorrows);
     this.avgUtilization = I80F48.from(avgUtilization);
@@ -160,6 +166,8 @@ export class Bank {
     this.rate0 = I80F48.from(rate0);
     this.util1 = I80F48.from(util1);
     this.rate1 = I80F48.from(rate1);
+    this.loanFeeRate = I80F48.from(loanFeeRate);
+    this.loanOriginationFeeRate = I80F48.from(loanOriginationFeeRate);
     this.maintAssetWeight = I80F48.from(maintAssetWeight);
     this.initAssetWeight = I80F48.from(initAssetWeight);
     this.maintLiabWeight = I80F48.from(maintLiabWeight);
@@ -171,14 +179,22 @@ export class Bank {
   toString(): string {
     return (
       'Bank ' +
+      '\n public key - ' +
+      this.publicKey.toBase58() +
       '\n token index - ' +
       this.tokenIndex +
       '\n token name - ' +
       this.name +
+      '\n vault - ' +
+      this.vault.toBase58() +
       '\n deposit index - ' +
       this.depositIndex.toNumber() +
       '\n borrow index - ' +
       this.borrowIndex.toNumber() +
+      '\n indexedDeposits - ' +
+      this.indexedDeposits.toNumber() +
+      '\n indexedBorrows - ' +
+      this.indexedBorrows.toNumber() +
       '\n cachedIndexedTotalDeposits - ' +
       this.cachedIndexedTotalDeposits.toNumber() +
       '\n cachedIndexedTotalBorrows - ' +
@@ -201,6 +217,10 @@ export class Bank {
       this.util1.toNumber() +
       '\n rate1 - ' +
       this.rate1.toNumber() +
+      '\n loanFeeRate - ' +
+      this.loanFeeRate.toNumber() +
+      '\n loanOriginationFeeRate - ' +
+      this.loanOriginationFeeRate.toNumber() +
       '\n maintAssetWeight - ' +
       this.maintAssetWeight.toNumber() +
       '\n initAssetWeight - ' +

ts/client/src/client.ts

@@ -137,7 +137,7 @@ export class MangoClient {
       },
     ];
 
-    if (groupNum) {
+    if (groupNum !== undefined) {
       const bbuf = Buffer.alloc(4);
       bbuf.writeUInt32LE(groupNum);
       filters.push({

ts/client/src/debug-scripts/mb-debug-user.ts

@@ -0,0 +1,134 @@
+import { AnchorProvider, Wallet } from '@project-serum/anchor';
+import { Connection, Keypair } from '@solana/web3.js';
+import fs from 'fs';
+import { HealthType } from '../accounts/mangoAccount';
+import { MangoClient } from '../client';
+import { MANGO_V4_ID } from '../constants';
+import { toUiDecimals } from '../utils';
+
+async function debugUser(client, group, mangoAccount) {
+  console.log(mangoAccount.toString(group));
+  await mangoAccount.reload(client, group);
+
+  console.log(
+    'mangoAccount.getEquity() ' +
+      toUiDecimals(mangoAccount.getEquity().toNumber()),
+  );
+  console.log(
+    'mangoAccount.getHealth(HealthType.init) ' +
+      toUiDecimals(mangoAccount.getHealth(HealthType.init).toNumber()),
+  );
+  console.log(
+    'mangoAccount.getHealthRatio(HealthType.init) ' +
+      mangoAccount.getHealthRatio(HealthType.init).toNumber(),
+  );
+  console.log(
+    'mangoAccount.getCollateralValue() ' +
+      toUiDecimals(mangoAccount.getCollateralValue().toNumber()),
+  );
+  console.log(
+    'mangoAccount.getAssetsVal() ' +
+      toUiDecimals(mangoAccount.getAssetsVal().toNumber()),
+  );
+  console.log(
+    'mangoAccount.getLiabsVal() ' +
+      toUiDecimals(mangoAccount.getLiabsVal().toNumber()),
+  );
+
+  console.log(
+    "mangoAccount.getMaxWithdrawWithBorrowForToken(group, 'SOL') " +
+      toUiDecimals(
+        (
+          await mangoAccount.getMaxWithdrawWithBorrowForToken(group, 'SOL')
+        ).toNumber(),
+      ),
+  );
+
+  console.log(
+    "mangoAccount.getMaxSourceForTokenSwap(group, 'USDC', 'BTC') " +
+      toUiDecimals(
+        (
+          await mangoAccount.getMaxSourceForTokenSwap(
+            group,
+            'USDC',
+            'BTC',
+            0.94,
+          )
+        ).toNumber(),
+      ),
+  );
+
+  console.log(
+    'mangoAccount.simHealthWithTokenPositionChanges ' +
+      toUiDecimals(
+        (
+          await mangoAccount.simHealthWithTokenPositionChanges(group, [
+            {
+              tokenName: 'USDC',
+              tokenAmount:
+                -20_000 *
+                Math.pow(10, group.banksMap.get('BTC')!.mintDecimals!),
+            },
+            {
+              tokenName: 'BTC',
+              tokenAmount:
+                1 * Math.pow(10, group.banksMap.get('BTC')!.mintDecimals!),
+            },
+          ])
+        ).toNumber(),
+      ),
+  );
+}
+
+async function main() {
+  const options = AnchorProvider.defaultOptions();
+  const connection = new Connection(process.env.MB_CLUSTER_URL!, options);
+
+  const admin = Keypair.fromSecretKey(
+    Buffer.from(
+      JSON.parse(fs.readFileSync(process.env.MB_PAYER_KEYPAIR!, 'utf-8')),
+    ),
+  );
+  console.log(`Admin ${admin.publicKey.toBase58()}`);
+
+  const adminWallet = new Wallet(admin);
+  const adminProvider = new AnchorProvider(connection, adminWallet, options);
+  const client = MangoClient.connect(
+    adminProvider,
+    'mainnet-beta',
+    MANGO_V4_ID['mainnet-beta'],
+  );
+
+  const group = await client.getGroupForCreator(admin.publicKey, 0);
+  console.log(`${group.toString()}`);
+
+  for (const bank of await group.banksMap.values()) {
+    console.log(`${bank.toString()}`);
+  }
+
+  for (const keypair of [
+    process.env.MB_PAYER_KEYPAIR,
+    process.env.MB_USER2_KEYPAIR,
+  ]) {
+    console.log();
+    const user = Keypair.fromSecretKey(
+      Buffer.from(JSON.parse(fs.readFileSync(keypair, 'utf-8'))),
+    );
+    const userWallet = new Wallet(user);
+    console.log(`User ${userWallet.publicKey.toBase58()}`);
+    const mangoAccount = (
+      await client.getMangoAccountsForOwner(group, user.publicKey)
+    )[0];
+    console.log(`MangoAccount ${mangoAccount.publicKey}`);
+
+    await debugUser(client, group, mangoAccount);
+  }
+
+  process.exit();
+}
+
+try {
+  main();
+} catch (error) {
+  console.log(error);
+}

ts/client/src/scripts/example1-user-delegate.ts

@@ -43,7 +43,9 @@ async function main() {
 
   // delegate
   const delegate = Keypair.fromSecretKey(
-    Buffer.from(JSON.parse(fs.readFileSync(process.env.DELEGATE!, 'utf-8'))),
+    Buffer.from(
+      JSON.parse(fs.readFileSync(process.env.USER3_KEYPAIR!, 'utf-8')),
+    ),
   );
   const delegateWallet = new Wallet(delegate);
   const delegateProvider = new AnchorProvider(

ts/client/src/scripts/mb-add-group-to-ids-json.ts

@@ -25,7 +25,7 @@ async function main() {
 
   // build client and fetch group for admin
   const options = AnchorProvider.defaultOptions();
-  const connection = new Connection(process.env.CLUSTER_URL, options);
+  const connection = new Connection(process.env.MB_CLUSTER_URL, options);
   const user = Keypair.fromSecretKey(
     Buffer.from(
       JSON.parse(fs.readFileSync(process.env.USER_KEYPAIR!, 'utf-8')),
@@ -40,9 +40,7 @@ async function main() {
   );
   const admin = Keypair.fromSecretKey(
     Buffer.from(
-      JSON.parse(
-        fs.readFileSync(process.env.MANGO_MAINNET_PAYER_KEYPAIR!, 'utf-8'),
-      ),
+      JSON.parse(fs.readFileSync(process.env.MB_PAYER_KEYPAIR!, 'utf-8')),
     ),
   );
   console.log(`Admin ${admin.publicKey.toBase58()}`);

ts/client/src/scripts/mb-example1-admin-close.ts

@@ -16,13 +16,11 @@ const MAINNET_MINTS = new Map([
 
 async function main() {
   const options = AnchorProvider.defaultOptions();
-  const connection = new Connection(process.env.CLUSTER_URL, options);
+  const connection = new Connection(process.env.MB_CLUSTER_URL, options);
 
   const admin = Keypair.fromSecretKey(
     Buffer.from(
-      JSON.parse(
-        fs.readFileSync(process.env.MANGO_MAINNET_PAYER_KEYPAIR!, 'utf-8'),
-      ),
+      JSON.parse(fs.readFileSync(process.env.MB_PAYER_KEYPAIR!, 'utf-8')),
     ),
   );
   const adminWallet = new Wallet(admin);

ts/client/src/scripts/mb-example1-admin.ts

@@ -20,13 +20,11 @@ const MAINNET_ORACLES = new Map([
 
 async function main() {
   const options = AnchorProvider.defaultOptions();
-  const connection = new Connection(process.env.CLUSTER_URL, options);
+  const connection = new Connection(process.env.MB_CLUSTER_URL, options);
 
   const admin = Keypair.fromSecretKey(
     Buffer.from(
-      JSON.parse(
-        fs.readFileSync(process.env.MANGO_MAINNET_PAYER_KEYPAIR!, 'utf-8'),
-      ),
+      JSON.parse(fs.readFileSync(process.env.MB_PAYER_KEYPAIR!, 'utf-8')),
     ),
   );
   const adminWallet = new Wallet(admin);

ts/client/src/scripts/mb-example1-close-account.ts

@@ -9,14 +9,12 @@ import { MangoClient } from '../client';
 //
 async function main() {
   const options = AnchorProvider.defaultOptions();
-  const connection = new Connection(process.env.CLUSTER_URL!, options);
+  const connection = new Connection(process.env.MB_CLUSTER_URL!, options);
 
   // user
   const user = Keypair.fromSecretKey(
     Buffer.from(
-      JSON.parse(
-        fs.readFileSync(process.env.MANGO_MAINNET_PAYER_KEYPAIR!, 'utf-8'),
-      ),
+      JSON.parse(fs.readFileSync(process.env.MB_PAYER_KEYPAIR!, 'utf-8')),
     ),
   );
   const userWallet = new Wallet(user);
@@ -30,9 +28,7 @@ async function main() {
   // admin
   const admin = Keypair.fromSecretKey(
     Buffer.from(
-      JSON.parse(
-        fs.readFileSync(process.env.MANGO_MAINNET_PAYER_KEYPAIR!, 'utf-8'),
-      ),
+      JSON.parse(fs.readFileSync(process.env.MB_PAYER_KEYPAIR!, 'utf-8')),
     ),
   );
   console.log(`Admin ${admin.publicKey.toBase58()}`);

ts/client/src/scripts/mb-example1-ids-json.ts

@@ -2,7 +2,7 @@ import { AnchorProvider, Wallet } from '@project-serum/anchor';
 import { Connection, Keypair } from '@solana/web3.js';
 import fs from 'fs';
 import { HealthType } from '../accounts/mangoAccount';
-import { AccountSize, MangoClient } from '../index';
+import { MangoClient } from '../index';
 import { toUiDecimals } from '../utils';
 
 //
@@ -11,13 +11,11 @@ import { toUiDecimals } from '../utils';
 
 async function main() {
   const options = AnchorProvider.defaultOptions();
-  const connection = new Connection(process.env.CLUSTER_URL!, options);
+  const connection = new Connection(process.env.MB_CLUSTER_URL!, options);
 
   const user = Keypair.fromSecretKey(
     Buffer.from(
-      JSON.parse(
-        fs.readFileSync(process.env.MANGO_MAINNET_PAYER_KEYPAIR!, 'utf-8'),
-      ),
+      JSON.parse(fs.readFileSync(process.env.MB_PAYER_KEYPAIR!, 'utf-8')),
     ),
   );
   const userWallet = new Wallet(user);
@@ -30,9 +28,7 @@ async function main() {
 
   const admin = Keypair.fromSecretKey(
     Buffer.from(
-      JSON.parse(
-        fs.readFileSync(process.env.MANGO_MAINNET_PAYER_KEYPAIR!, 'utf-8'),
-      ),
+      JSON.parse(fs.readFileSync(process.env.MB_PAYER_KEYPAIR!, 'utf-8')),
     ),
   );
   console.log(`Admin ${admin.publicKey.toBase58()}`);

ts/client/src/scripts/mb-flash-loan.ts

@@ -10,14 +10,14 @@ import {
 import BN from 'bn.js';
 import fs from 'fs';
 import { QUOTE_DECIMALS } from '../accounts/bank';
-import { AccountSize, MangoClient } from '../index';
+import { MangoClient } from '../index';
 import { getAssociatedTokenAddress } from '../utils';
 
-const CLUSTER_URL =
-  process.env.CLUSTER_URL ||
+const MB_CLUSTER_URL =
+  process.env.MB_CLUSTER_URL ||
   'https://mango.rpcpool.com/946ef7337da3f5b8d3e4a34e7f88';
-const MANGO_MAINNET_PAYER_KEYPAIR =
-  process.env.MANGO_MAINNET_PAYER_KEYPAIR ||
+const MB_PAYER_KEYPAIR =
+  process.env.MB_PAYER_KEYPAIR ||
   '/Users/tylershipe/.config/solana/deploy.json';
 
 //
@@ -26,13 +26,11 @@ const MANGO_MAINNET_PAYER_KEYPAIR =
 // NOTE: we assume that ATA for source and target already exist for wallet
 async function main() {
   const options = AnchorProvider.defaultOptions();
-  const connection = new Connection(CLUSTER_URL, options);
+  const connection = new Connection(MB_CLUSTER_URL, options);
 
   // load user key
   const user = Keypair.fromSecretKey(
-    Buffer.from(
-      JSON.parse(fs.readFileSync(MANGO_MAINNET_PAYER_KEYPAIR!, 'utf-8')),
-    ),
+    Buffer.from(JSON.parse(fs.readFileSync(MB_PAYER_KEYPAIR!, 'utf-8'))),
   );
   const userWallet = new Wallet(user);
   const userProvider = new AnchorProvider(connection, userWallet, options);
@@ -44,9 +42,7 @@ async function main() {
 
   // load admin key
   const admin = Keypair.fromSecretKey(
-    Buffer.from(
-      JSON.parse(fs.readFileSync(MANGO_MAINNET_PAYER_KEYPAIR!, 'utf-8')),
-    ),
+    Buffer.from(JSON.parse(fs.readFileSync(MB_PAYER_KEYPAIR!, 'utf-8'))),
   );
   console.log(`Admin ${admin.publicKey.toBase58()}`);