Merge pull request #273 from blockworks-foundation/silas/ci-updates

CI job updates
2022-11-01 12:45:18 +00:00 · 2022-11-01 12:45:18 +00:00 · c9ae812b47
parent d75e38ac87 c22deb9c25
commit c9ae812b47
9 changed files with 257 additions and 107 deletions
--- a/.cargo/audit.toml
+++ b/.cargo/audit.toml
@ -1,32 +0,0 @@
-# All of the options which can be passed via CLI arguments can also be
-# permanently specified in this file.
-
-# RUSTSEC-2020-0071 and RUSTSEC-2020-0159 are low severity vulnerable upstream Solana crates. Ignored for now.
-
-[advisories]
-ignore = ["RUSTSEC-2020-0016", "RUSTSEC-2020-0036",  "RUSTSEC-2020-0056", "RUSTSEC-2020-0071", "RUSTSEC-2020-0159", "RUSTSEC-2021-0078", "RUSTSEC-2021-0079"] # advisory IDs to ignore e.g. ["RUSTSEC-2019-0001", ...]
-informational_warnings = ["unmaintained"] # warn for categories of informational advisories
-severity_threshold = "medium" # CVSS severity ("none", "low", "medium", "high", "critical")
-
-# Advisory Database Configuration
-[database]
-path = "~/.cargo/advisory-db" # Path where advisory git repo will be cloned
-url = "https://github.com/RustSec/advisory-db.git" # URL to git repo
-fetch = true # Perform a `git fetch` before auditing (default: true)
-stale = false # Allow stale advisory DB (i.e. no commits for 90 days, default: false)
-
-# Output Configuration
-[output]
-deny = [] # exit on error if unmaintained dependencies are found
-format = "terminal" # "terminal" (human readable report) or "json"
-quiet = false # Only print information on error
-show_tree = true # Show inverse dependency trees along with advisories (default: true)
-
-# Target Configuration
-[target]
-# arch = "x86_64" # Ignore advisories for CPU architectures other than this one
-# os = "linux" # Ignore advisories for operating systems other than this one
-
-[yanked]
-enabled = false # Warn for yanked crates in Cargo.lock (default: true)
-update_index = true # Auto-update the crates.io index (default: true)
--- a/.github/workflows/ci-cargo-audit.yml
+++ b/.github/workflows/ci-cargo-audit.yml
@ -1,29 +0,0 @@
-name: Cargo Audit
-
-on:
-  push:
-    branches:
-      - main
-      - dev
-  pull_request:
-  workflow_dispatch:
-
-jobs:
-  cargo-audit:
-    name: Cargo Vulnerability Scanner
-    if: github.actor != 'github-actions[bot]'
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-      - name: Checkout submodules
-        run: git submodule update --init
-      # Install cargo audit
-      - name: Install Cargo Audit
-        uses: actions-rs/install@v0.1
-        with:
-          crate: cargo-audit
-          version: latest
-      # Run cargo audit using args from .cargo/audit.toml (ignores, etc.)
-      - name: Run Cargo Audit
-        run: cargo audit -c always
--- a/.github/workflows/ci-code-review-rust.yml
+++ b/.github/workflows/ci-code-review-rust.yml
@ -1,16 +1,28 @@
-name: Lint and Test
+name: Code Review - Rust
 on:
  push:
-    branches:
-      - main
-      - dev
-  pull_request:
-  workflow_dispatch:
+    paths: ['cli/**',
+            'client/**',
+            'programs/**',
+            'keeper/**',
+            'lib/**',
+            'liquidator/**',
+            'anchor/cli/**']
+  pull_request: 
+    branches: ['main', 'dev']
+    paths: ['cli/**',
+            'client/**',
+            'programs/**',
+            'keeper/**',
+            'lib/**',
+            'liquidator/**',
+            'anchor/cli/**']
+  workflow_dispatch: # Pick branch manually

 env:
  CARGO_TERM_COLOR: always
-  SOLANA_VERSION: '1.10.29'
-  RUST_TOOLCHAIN: 1.60.0
+  SOLANA_VERSION: '1.13.3'
+  RUST_TOOLCHAIN: '1.60.0'
  LOG_PROGRAM: 'm43thNJ58XCjL798ZSq6JGAG1BnWskhdq5or6kcnfsD'

 defaults:
@ -20,15 +32,18 @@ defaults:
 jobs:
  lint:
    name: Lint
-    if: github.actor != 'github-actions[bot]'
    runs-on: ubuntu-latest
+    if: (github.actor != 'dependabot[bot]')
    steps:
      - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
+
      - name: Checkout submodules
        run: git submodule update --init
+
      - name: Install Linux dependencies
        run: sudo apt-get update && sudo apt-get install -y pkg-config build-essential libudev-dev
+
      - name: Install Rust
        uses: actions-rs/toolchain@v1
        with:
@ -36,30 +51,37 @@ jobs:
          profile: minimal
          toolchain: ${{ env.RUST_TOOLCHAIN }}
          components: rustfmt, clippy
+
      - name: Cache dependencies
-        uses: Swatinem/rust-cache@v1
+        uses: Swatinem/rust-cache@v2
+        
      - name: Run fmt
        run: cargo fmt -- --check
+
      - name: Run clippy
        run: cargo clippy -- --deny=warnings --allow=clippy::style --allow=clippy::complexity

  tests:
-    name: Test
-    if: github.actor != 'github-actions[bot]'
+    name: Run tests
    runs-on: ubuntu-latest
+    if: (github.actor != 'dependabot[bot]')
    steps:
      - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
+
      - name: Checkout submodules
        run: git submodule update --init
+
      - name: Install Linux dependencies
        run: sudo apt-get update && sudo apt-get install -y pkg-config build-essential libudev-dev
+
      - name: Install Rust
        uses: actions-rs/toolchain@v1
        with:
          override: true
          profile: minimal
          toolchain: ${{ env.RUST_TOOLCHAIN }}
+
      - name: Install Solana
        run: |
          sh -c "$(curl -sSfL https://release.solana.com/v${{ env.SOLANA_VERSION }}/install)"
@ -68,13 +90,20 @@ jobs:
          solana --version
          echo "Generating keypair..."
          solana-keygen new -o "$HOME/.config/solana/id.json" --no-passphrase --silent
+
      - name: Build all deps
        run: |
          cargo build-bpf || true
          cargo +bpf build-bpf
+
+      - name: Run unit tests
+        continue-on-error: true
+        run: cargo test --lib
+
        # Run bpf tests and output to runner and log
-      - name: Run tests
+      - name: Run bpf tests
        run: cargo +bpf test-bpf 2> >(tee raw-test-bpf.log >&2)
+
      - name: Save raw log
        uses: actions/upload-artifact@v3
        with:
@ -84,34 +113,39 @@ jobs:
  # Download logs and process them
  process-logs:
    name: Process logs
-    if: github.actor != 'github-actions[bot]'
    runs-on: ubuntu-latest
+    if: (github.actor != 'dependabot[bot]')
    needs: [lint, tests]
    steps:
      - name: Download raw log
        uses: actions/download-artifact@v3
        with:
          name: raw-test-bpf
+
      - name: Install deps
        run: |
          sudo apt-get install ripgrep
          curl -Lo xsv.tar.gz "https://github.com/BurntSushi/xsv/releases/latest/download/xsv-0.13.0-x86_64-unknown-linux-musl.tar.gz"
          sudo tar xf xsv.tar.gz -C /usr/local/bin
+
      - name: Setup date input
        id: date
        run: echo "::set-output name=today::$(date +'%Y-%m-%d')"
+
      - name: Process raw log
        run: |
          rg -oNI "(Instruction: |Program ${{ env.LOG_PROGRAM }} consumed).*$" raw-test-bpf.log \
            | rg -U 'Instruction:.*\nProgram ${{ env.LOG_PROGRAM }}.*' \
            | awk 'NR % 2 == 1 { o=$0 ; next } { print o " " $0 }' \
            | sort | uniq -u | sort > cu-per-ix.log
+
      - name: Clean up log
        run: |
          rg -N 'Instruction: (\w+) .* consumed (\d+) .*' cu-per-ix.log -r '${{ steps.date.outputs.today }},$1,$2' \
            | uniq | xsv sort -s 2 -N -R \
            | sort -t ',' -k 2,3 -u \
            | sort > cu-per-ix-clean.log
+
      - name: Save clean log
        uses: actions/upload-artifact@v3
        with:
--- a/.github/workflows/ci-code-review-ts.yml
+++ b/.github/workflows/ci-code-review-ts.yml
@ -0,0 +1,85 @@
+name: Code Review - TypeScript
+
+on:
+  pull_request: 
+    branches: ['main', 'dev']
+    paths: ['ts/**']
+  push:
+    paths: ['ts/**']
+
+jobs:
+  format:
+    name: Format
+    runs-on: ubuntu-latest
+    if: (github.actor != 'dependabot[bot]')
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+
+    - name: Setup Node
+      uses: actions/setup-node@v3
+      with:
+        node-version: '16'
+        cache: 'yarn'
+
+    - name: Install dependencies
+      run: yarn install --frozen-lockfile
+
+    - name: Format
+      run: yarn format
+
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    if: (github.actor != 'dependabot[bot]')
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+
+    - name: Setup Node
+      uses: actions/setup-node@v3
+      with:
+        node-version: '16'
+        cache: 'yarn'
+
+    - name: Install dependencies
+      run: yarn install --frozen-lockfile
+
+    - name: Lint
+      run: yarn lint
+
+  unit-test:
+    name: Unit Test
+    runs-on: ubuntu-latest
+    if: (github.actor != 'dependabot[bot]')
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+
+    - name: Setup Node
+      uses: actions/setup-node@v3
+      with:
+        node-version: '16'
+        cache: 'yarn'
+
+    - name: Install dependencies
+      run: yarn install --frozen-lockfile
+
+    - name: Run Test
+      run: yarn test
+
+  semgrep:
+    name: Security Scan
+    runs-on: ubuntu-latest
+    if: (github.actor != 'dependabot[bot]')
+    container:
+      image: returntocorp/semgrep
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Run semgrep
+        run: semgrep ci
+        env:
+           SEMGREP_RULES: p/typescript
--- a/.github/workflows/ci-dependency-scan-cargo.yml
+++ b/.github/workflows/ci-dependency-scan-cargo.yml
@ -0,0 +1,53 @@
+name: Dependency Security Scan - Cargo
+
+on:
+  pull_request: 
+    branches: ['main', 'dev']
+    paths: ['cli/**',
+            'client/**',
+            'programs/**',
+            'keeper/**',
+            'lib/**',
+            'liquidator/**',
+            'anchor/cli/**',
+            'Cargo.lock']
+  push:
+    paths: ['cli/**',
+            'client/**',
+            'programs/**',
+            'keeper/**',
+            'lib/**',
+            'liquidator/**',
+            'anchor/cli/**',
+            'Cargo.lock']
+
+jobs:
+  trivy:
+    name: Dependency Scan
+    runs-on: ubuntu-latest
+    if: (github.actor != 'dependabot[bot]')
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      # Report all vulnerabilities in CI output
+      - name: Report on all vulnerabilities
+        uses: aquasecurity/trivy-action@master
+        with:
+          scan-type: 'fs'
+          scan-ref: 'Cargo.lock'
+          ignore-unfixed: true
+          hide-progress: true
+          format: 'table'
+          
+      # Fail the job on critical vulnerabiliies with fix available
+      - name: Fail on critical vulnerabilities
+        uses: aquasecurity/trivy-action@master
+        with:
+          scan-type: 'fs'
+          scan-ref: 'Cargo.lock'
+          ignore-unfixed: true
+          hide-progress: true
+          format: 'table'
+          severity: 'CRITICAL'
+          exit-code: '1'
--- a/.github/workflows/ci-dependency-scan-yarn.yml
+++ b/.github/workflows/ci-dependency-scan-yarn.yml
@ -0,0 +1,39 @@
+name: Dependency Security Scan - Yarn
+
+on:
+  pull_request: 
+    branches: ['main', 'dev']
+    paths: ['ts/**', 'yarn.lock']
+  push:
+    paths: ['ts/**', 'yarn.lock']
+
+jobs:
+  trivy:
+    name: Dependency Scan
+    runs-on: ubuntu-latest
+    if: (github.actor != 'dependabot[bot]')
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      # Report all vulnerabilities in CI output
+      - name: Report on all vulnerabilities
+        uses: aquasecurity/trivy-action@master
+        with:
+          scan-type: 'fs'
+          scan-ref: 'yarn.lock'
+          ignore-unfixed: true
+          hide-progress: true
+          format: 'table'
+          
+      # Fail the job on critical vulnerabiliies with fix available
+      - name: Fail on critical vulnerabilities
+        uses: aquasecurity/trivy-action@master
+        with:
+          scan-type: 'fs'
+          scan-ref: 'yarn.lock'
+          ignore-unfixed: true
+          hide-progress: true
+          format: 'table'
+          severity: 'CRITICAL'
+          exit-code: '1'
--- a/.github/workflows/ci-soteria.yml
+++ b/.github/workflows/ci-soteria.yml
@ -2,31 +2,32 @@ name: Soteria Scan

 on:
  push:
-    branches:
-      - main
-      - dev
+    paths: ['programs/**']
  pull_request:
-  workflow_dispatch: #pick branch to manually run on
+    branches: ['main', 'dev']
+    paths: ['programs/**']
+  workflow_dispatch: # Pick branch manually

 env:
  CARGO_TERM_COLOR: always
-  SOLANA_VERSION: "1.10.29"
+  SOLANA_VERSION: "1.13.3"

 jobs:
  build:
    name: Soteria
-    if: github.actor != 'github-actions[bot]'
    runs-on: ubuntu-latest
+    if: (github.actor != 'dependabot[bot]')
    strategy:
      fail-fast: false
      matrix:
-        program: ["programs/mango-v4", "programs/margin-trade"]
+        program: ['programs/mango-v4/',
+                  'programs/margin-trade/']
    env:
      PROGRAM_PATH: ${{ matrix.program }}

    steps:
-      - name: Check-out repo
-        uses: actions/checkout@v2
+      - name: Checkout code
+        uses: actions/checkout@v3

      - name: Cache Solana binaries
        uses: actions/cache@v2
@ -39,9 +40,9 @@ jobs:
          key: solana-${{ env.SOLANA_VERSION }}

      - name: Cache build dependencies
-        uses: Swatinem/rust-cache@v1
+        uses: Swatinem/rust-cache@v2
        with:
-          target-dir: .coderrect/build
+          target-dir: ${{ matrix.program }}.coderrect/build

      - name: Install Solana
        if: steps.solana-cache.outputs.cache-hit != 'true'
--- a/ts/client/src/client.ts
+++ b/ts/client/src/client.ts
@ -1742,7 +1742,7 @@ export class MangoClient {
        [group.getFirstBankByTokenIndex(0 as TokenIndex)],
        [perpMarket],
      );
-    const bank = group.banksMapByTokenIndex.get(0 as TokenIndex)![0]
+    const bank = group.banksMapByTokenIndex.get(0 as TokenIndex)![0];
    const ix = await this.program.methods
      .perpSettlePnl()
      .accounts({
@ -1764,14 +1764,14 @@ export class MangoClient {
      )
      .instruction();

-      return await sendTransaction(
-        this.program.provider as AnchorProvider,
-        [ix],
-        group.addressLookupTablesList,
-        {
-          postSendTxCallback: this.postSendTxCallback,
-        },
-      );
+    return await sendTransaction(
+      this.program.provider as AnchorProvider,
+      [ix],
+      group.addressLookupTablesList,
+      {
+        postSendTxCallback: this.postSendTxCallback,
+      },
+    );
  }

  async perpSettleFees(
@ -1789,7 +1789,7 @@ export class MangoClient {
        [group.getFirstBankByTokenIndex(0 as TokenIndex)],
        [perpMarket],
      );
-    const bank = group.banksMapByTokenIndex.get(0 as TokenIndex)![0]
+    const bank = group.banksMapByTokenIndex.get(0 as TokenIndex)![0];
    const ix = await this.program.methods
      .perpSettleFees(maxSettleAmount)
      .accounts({
@ -1808,14 +1808,14 @@ export class MangoClient {
      )
      .instruction();

-      return await sendTransaction(
-        this.program.provider as AnchorProvider,
-        [ix],
-        group.addressLookupTablesList,
-        {
-          postSendTxCallback: this.postSendTxCallback,
-        },
-      );
+    return await sendTransaction(
+      this.program.provider as AnchorProvider,
+      [ix],
+      group.addressLookupTablesList,
+      {
+        postSendTxCallback: this.postSendTxCallback,
+      },
+    );
  }

  public async perpConsumeEvents(
--- a/ts/client/src/scripts/mm/market-maker.ts
+++ b/ts/client/src/scripts/mm/market-maker.ts
@ -348,7 +348,6 @@ async function makeMarketUpdateInstructions(
  const fairValue = (aggBid + aggAsk) / 2;
  const aggSpread = (aggAsk - aggBid) / fairValue;

-  
  const requoteThresh = mc.params.requoteThresh;
  const equity = toUiDecimalsForQuote(mangoAccount.getEquity(group));
  const sizePerc = mc.params.sizePerc;