Initial work on BitGo integration. Functions as expected.

backend/grant/commands.py

@@ -8,6 +8,7 @@ import click
 from flask import current_app
 from flask.cli import with_appcontext
 from werkzeug.exceptions import MethodNotAllowed, NotFound
+from sqlalchemy import text
 
 HERE = os.path.abspath(os.path.dirname(__file__))
 PROJECT_ROOT = os.path.join(HERE, os.pardir)
@@ -167,6 +168,15 @@ def reset_db_chain_data():
 
     # Commit state
     db.session.commit()
+
+    # Attempt to reset contribution ID sequence, psql specific. Don't fail out
+    # if this messes up though, just warn them.
+    try:
+        db.engine.execute(text('ALTER SEQUENCE proposal_contribution_id_seq RESTART WITH 1'))
+    except e:
+        print(e)
+        print('Failed to reset contribution id sequence, see above error. Continuing anyway.')
+
     print('Successfully wiped chain-dependent db state!')
     print(f'* Deleted {p_count} proposals and their linked entities')
     print(f'* Deleted {t_count} tasks')

blockchain/.env.example

@@ -14,13 +14,26 @@ MINIMUM_BLOCK_CONFIRMATIONS="6"
 API_SECRET_HASH=""
 API_SECRET_KEY=""
 
+############################ ADDRESS DERIVATION ############################
+# You should only set one OR the other. The former will generate addresses #
+# using Bip32 address derivation. The latter uses BitGo's API. If you set  #
+# both, BitGo takes precedence. API key should ONLY HAVE VIEW ACCESS!      #
+############################################################################
+
+# BITGO_WALLET_ID=""
+# BITGO_ACCESS_TOKEN=""
+
+### OR ###
+
+# BIP32_XPUB=""
+
+############################################################################
+
+
 # Addresses, run `yarn genaddress` to get sprout information
 SPROUT_ADDRESS=""
 SPROUT_VIEWKEY=""
 
-# extended public seed
-BIP32_XPUB=""
-
 # Block heights to fall back on for starting our scan
 MAINNET_START_BLOCK="464000"
 TESTNET_START_BLOCK="390000"

blockchain/package.json

@@ -35,6 +35,7 @@
     "@types/dotenv": "^6.1.0",
     "@types/ws": "^6.0.1",
     "axios": "0.18.0",
+    "bitgo": "4.48.1",
     "body-parser": "1.18.3",
     "cors": "2.8.5",
     "dotenv": "^6.1.0",

blockchain/src/bin/genaddress.ts

@@ -1,4 +1,5 @@
 import node from '../node';
+import { extractErrMessage } from '../util';
 
 async function printAddressAndKey() {
   try {
@@ -9,11 +10,7 @@ async function printAddressAndKey() {
     console.log(`SPROUT_ADDRESS="${address}"`);
     console.log(`SPROUT_VIEWKEY="${viewkey}"\n`);
   } catch(err) {
-    if (err.response && err.response.data) {
-      console.error(err.response.data);
-    } else {
-      console.error(err);
-    }
+    console.error(extractErrMessage(err));
     process.exit(1);
   }
 }

blockchain/src/bitgo.ts

@@ -0,0 +1,52 @@
+import { BitGo, Wallet } from 'bitgo';
+import bitcore from "zcash-bitcore-lib";
+import env from './env';
+import log from './log';
+import { getNetwork } from './node';
+
+let bitgoWallet: Wallet;
+
+export async function initBitGo() {
+  if (!env.BITGO_ACCESS_TOKEN || !env.BITGO_WALLET_ID) {
+    log.info('BITGO environment variables not set, nooping initBitGo');
+    return;
+  }
+
+  // Assert that we're on mainnet
+  const network = getNetwork();
+  if (network !== bitcore.Networks.mainnet) {
+    throw new Error(`BitGo cannot be used on anything but mainnet, connected node is ${network}`);
+  }
+
+  const bitgo = new BitGo({
+    env: 'prod', // Non-prod ZEC is not supported
+    accessToken: env.BITGO_ACCESS_TOKEN,
+  });
+  bitgoWallet = await bitgo.coin('zec').wallets().get({ id: env.BITGO_WALLET_ID });
+  log.info(`Initialized BitGo wallet "${bitgoWallet.label()}"`);
+}
+
+export async function getContributionAddress(id: number) {
+  if (!bitgoWallet) {
+    throw new Error('Must run initBitGo before getContributionAddress');
+  }
+
+  // Attempt to fetch first
+  const label = `Contribution #${id}`;
+  const res = await bitgoWallet.addresses({ labelContains: label });
+  if (res.addresses.length) {
+    if (res.addresses.length > 1) {
+      log.warn(`Contribution ${id} has ${res.addresses.length} associated with it. Using the first one (${res.addresses[0].address})`);
+    }
+    return res.addresses[0].address;
+  }
+
+  // Create a new one otherwise
+  const createRes = await bitgoWallet.createAddress({ label });
+  log.info(`Generate new address for contribution ${id}`);
+  return createRes.address;
+}
+
+function generateLabel(id: number) {
+  return `Contribution #${id}`;
+}

blockchain/src/env.ts

@@ -18,9 +18,13 @@ const DEFAULTS = {
   ZCASH_NODE_PASSWORD: "",
   MINIMUM_BLOCK_CONFIRMATIONS: "6",
 
+  BITGO_WALLET_ID: "",
+  BITGO_ACCESS_TOKEN: "",
+
+  BIP32_XPUB: "",
+
   SPROUT_ADDRESS: "",
   SPROUT_VIEWKEY: "",
-  BIP32_XPUB: "",
 
   MAINNET_START_BLOCK: "464000",
   TESTNET_START_BLOCK: "390000",
@@ -28,21 +32,39 @@ const DEFAULTS = {
   SENTRY_DSN: "",
 };
 
+const OPTIONAL: { [key: string]: undefined | boolean } = {
+  BITGO_WALLET_ID: true,
+  BITGO_ACCESS_TOKEN: true,
+  BIP32_XPUB: true,
+  // NOTE: Remove these from optional when sapling is ready
+  SPROUT_ADDRESS: true,
+  SPROUT_VIEWKEY: true,
+}
+
 type CustomEnvironment = typeof DEFAULTS;
 
 // ignore when testing
 if (process.env.NODE_ENV !== "test") {
+  // Set environment variables, throw on missing required ones
   Object.entries(DEFAULTS).forEach(([k, v]) => {
     if (!process.env[k]) {
       const defVal = (DEFAULTS as any)[k];
       if (defVal) {
         console.info(`Using default environment variable ${k}="${defVal}"`);
         process.env[k] = defVal;
-      } else {
+      } else if (!OPTIONAL[k]) {
         throw new Error(`Missing required environment variable ${k}`);
       }
     }
   });
+
+  // Ensure we have either xpub or bitgo, and warn if we have both
+  if (!process.env.BIP32_XPUB && (!process.env.BITGO_WALLET_ID || !process.env.BITGO_ACCESS_TOKEN)) {
+    throw new Error('Either BIP32_XPUB or BITGO_* environment variables required, missing both');
+  }
+  if (process.env.BIP32_XPUB && process.env.BITGO_WALLET_ID) {
+    console.info('BIP32_XPUB and BITGO environment variables set, BIP32_XPUB will be ignored');
+  }
 }
 
 export default (process.env as any) as CustomEnvironment;

blockchain/src/index.ts

@@ -2,6 +2,8 @@ import * as Sentry from "@sentry/node";
 import * as Webhooks from "./webhooks";
 import * as RestServer from "./server";
 import { initNode } from "./node";
+import { initBitGo } from "./bitgo";
+import { extractErrMessage } from "./util";
 import env from "./env";
 import log from "./log";
 
@@ -15,6 +17,7 @@ async function start() {
 
   log.info("============== Starting services ==============");
   await initNode();
+  await initBitGo();
   await RestServer.start();
   Webhooks.start();
   log.info("===============================================");
@@ -28,4 +31,8 @@ process.on("SIGINT", () => {
   process.exit();
 });
 
-start();
+start().catch(err => {
+  Sentry.captureException(err);
+  log.error(`Unexpected error while starting blockchain watcher: ${extractErrMessage(err)}`);
+  process.exit(1);
+});

blockchain/src/node.ts

@@ -3,6 +3,7 @@ import bitcore from "zcash-bitcore-lib";
 import { captureException } from "@sentry/node";
 import env from "./env";
 import log from "./log";
+import { extractErrMessage } from "./util";
 
 export interface BlockChainInfo {
   chain: string;
@@ -166,31 +167,29 @@ export async function initNode() {
     }
   } catch (err) {
     captureException(err);
-    log.error(err.response ? err.response.data : err);
-    log.error(
-      "Failed to connect to zcash node with the following credentials:\r\n",
-      rpcOptions
-    );
+    log.error(extractErrMessage(err));
+    log.error(`Failed to connect to zcash node with the following credentials: ${JSON.stringify(rpcOptions, null, 2)}`);
     process.exit(1);
   }
 
   // Check if sprout address is readable
-  try {
-    if (!env.SPROUT_ADDRESS) {
-      console.error("Missing SPROUT_ADDRESS environment variable, exiting");
-      process.exit(1);
-    }
-    await node.z_getbalance(env.SPROUT_ADDRESS as string);
-  } catch (err) {
-    if (!env.SPROUT_VIEWKEY) {
-      log.error(
-        "Unable to view SPROUT_ADDRESS and missing SPROUT_VIEWKEY environment variable, exiting"
-      );
-      process.exit(1);
-    }
-    await node.z_importviewingkey(env.SPROUT_VIEWKEY as string);
-    await node.z_getbalance(env.SPROUT_ADDRESS as string);
-  }
+  // NOTE: Replace with sapling when ready
+  // try {
+  //   if (!env.SPROUT_ADDRESS) {
+  //     console.error("Missing SPROUT_ADDRESS environment variable, exiting");
+  //     process.exit(1);
+  //   }
+  //   await node.z_getbalance(env.SPROUT_ADDRESS as string);
+  // } catch (err) {
+  //   if (!env.SPROUT_VIEWKEY) {
+  //     log.error(
+  //       "Unable to view SPROUT_ADDRESS and missing SPROUT_VIEWKEY environment variable, exiting"
+  //     );
+  //     process.exit(1);
+  //   }
+  //   await node.z_importviewingkey(env.SPROUT_VIEWKEY as string);
+  //   await node.z_getbalance(env.SPROUT_ADDRESS as string);
+  // }
 }
 
 export function getNetwork() {
@@ -210,23 +209,21 @@ export async function getBootstrapBlockHeight(txid: string | undefined) {
         block.height - parseInt(env.MINIMUM_BLOCK_CONFIRMATIONS, 10);
       return height.toString();
     } catch (err) {
-      console.warn(
-        `Attempted to get block height for tx ${txid} but failed with the following error:\n`,
-        err
-      );
-      console.warn("Falling back to hard-coded starter blocks");
+      log.warn(`Attempted to get block height for tx ${txid} but failed with the following error: ${extractErrMessage(err)}`);
     }
   }
 
   // If we can't find the latest tx block, fall back to when the grant
-  // system first launched, and scan from there.
+  // system first launched, and scan from there. Regtest or unknown networks
+  // start from the bottom.
   const net = getNetwork();
+  let height = "0";
   if (net === bitcore.Networks.mainnet) {
-    return env.MAINNET_START_BLOCK;
+    height = env.MAINNET_START_BLOCK;
   } else if (net === bitcore.Networks.testnet && !net.regtestEnabled) {
-    return env.TESTNET_START_BLOCK;
+    height = env.TESTNET_START_BLOCK;
   }
 
-  // Regtest or otherwise unknown networks should start at the bottom
-  return "0";
+  log.info(`Falling back to hard-coded starter block height ${height}`);
+  return height;
 }

blockchain/src/server/index.ts

@@ -14,7 +14,7 @@ import {
 } from '../store';
 import env from '../env';
 import node, { getBootstrapBlockHeight } from '../node';
-import { makeContributionMemo } from '../util';
+import { makeContributionMemo, extractErrMessage } from '../util';
 import log from '../log';
 
 // Configure server
@@ -29,8 +29,16 @@ app.use(authMiddleware);
 // Routes
 app.post('/bootstrap', async (req, res) => {
   const { pendingContributions, latestTxId } = req.body;
-  const info = await node.getblockchaininfo();
-  const startHeight = await getBootstrapBlockHeight(latestTxId);
+
+  let info;
+  let startHeight;
+  try {
+    info = await node.getblockchaininfo();
+    startHeight = await getBootstrapBlockHeight(latestTxId);
+  } catch(err) {
+    log.error(`Unknown node error during bootstrap: ${extractErrMessage(err)}`);
+    return res.status(500).json({ error: 'Unknown zcash node error' });
+  }
 
   console.info('Bootstrapping watcher!');
   console.info(' * Start height:', startHeight);
@@ -39,11 +47,18 @@ app.post('/bootstrap', async (req, res) => {
   console.info('Generating addresses to watch for each contribution...');
 
   // Running generate address on each will add each contribution to redux state
-  pendingContributions.forEach((c: any) => {
-    store.dispatch(generateAddresses(c.id));
-  });
-  console.info(`Done! Generated ${pendingContributions.length} addresses.`);
-  store.dispatch(setStartingBlockHeight(startHeight));
+  try {
+    const dispatchers = pendingContributions.map(async (c: any) => {
+      const action = await generateAddresses(c.id);
+      store.dispatch(action);
+    });
+    await Promise.all(dispatchers);
+    console.info(`Done! Generated ${pendingContributions.length} addresses.`);
+    store.dispatch(setStartingBlockHeight(startHeight));
+  } catch(err) {
+    log.error(`Unknown error during bootstrap address generation: ${extractErrMessage(err)}`);
+    return res.status(500).json({ error: 'Failed to generate addresses for contributions' });
+  }
 
   // Send back some basic info about where the chain is at
   res.json({
@@ -54,20 +69,29 @@ app.post('/bootstrap', async (req, res) => {
   });
 });
 
-app.get('/contribution/addresses', (req, res) => {
+app.get('/contribution/addresses', async (req, res) => {
   const { contributionId } = req.query;
   let addresses = getAddressesByContributionId(store.getState(), contributionId)
   if (!addresses) {
-    const action = generateAddresses(req.query.contributionId);
-    addresses = action.payload.addresses;
-    store.dispatch(action);
+    try {
+      const action = await generateAddresses(contributionId);
+      addresses = action.payload.addresses;
+      store.dispatch(action);
+    } catch(err) {
+      log.error(`Unknown error during address generation for contribution ${contributionId}: ${extractErrMessage(err)}`);
+    }
+  }
+
+  if (addresses) {
+    res.json({
+      data: {
+        ...addresses,
+        memo: makeContributionMemo(contributionId),
+      },
+    });
+  } else {
+    res.status(500).json({ error: 'Failed to generate addresses' });
   }
-  res.json({
-    data: {
-      ...addresses,
-      memo: makeContributionMemo(contributionId),
-    },
-  });
 });
 
 
@@ -96,7 +120,7 @@ app.post('/contribution/disclosure', async (req, res) => {
       return res.status(400).json({ error: err.response.data.error.message });
     }
     else {
-      log.error('Unknown node error:', err.response ? err.response.data : err);
+      log.error(`Unknown node error during disclosure: ${extractErrMessage(err)}`);
       return res.status(500).json({ error: 'Unknown zcash node error' });
     }
   }

blockchain/src/server/middleware/errorHandler.ts

@@ -1,6 +1,7 @@
 import { captureException } from "@sentry/node";
 import { Request, Response, NextFunction } from 'express';
 import log from "../../log";
+import { extractErrMessage } from "../../util";
 
 export default function errorHandler(err: Error, req: Request, res: Response, next: NextFunction) {
   // Non-error responses, or something else handled & responded
@@ -9,7 +10,7 @@ export default function errorHandler(err: Error, req: Request, res: Response, ne
   }
 
   captureException(err);
-  log.error(`Uncaught ${err.name} exception at ${req.method} ${req.path}: ${err.message}`);
+  log.error(`Uncaught ${err.name} exception at ${req.method} ${req.path}: ${extractErrMessage(err)}`);
   log.debug(`Query: ${JSON.stringify(req.query, null, 2)}`);
   log.debug(`Body: ${JSON.stringify(req.body, null, 2)}`);
   log.debug(`Full stacktrace:\n${err.stack}`);

blockchain/src/store/actions.ts

@@ -1,6 +1,7 @@
 import type, { AddressCollection } from './types';
 import { deriveTransparentAddress } from '../util';
 import { getNetwork } from '../node';
+import { getContributionAddress } from '../bitgo';
 import env from '../env';
 
 export function setStartingBlockHeight(height: string | number) {
@@ -10,10 +11,16 @@ export function setStartingBlockHeight(height: string | number) {
   }
 }
 
-export function generateAddresses(contributionId: number) {
-  // 2^31 is the maximum number of BIP32 addresses
+export async function generateAddresses(contributionId: number) {
+  let transparent;
+  if (env.BITGO_WALLET_ID) {
+    transparent = await getContributionAddress(contributionId);
+  } else {
+    transparent = deriveTransparentAddress(contributionId, getNetwork());
+  }
+
   const addresses: AddressCollection = {
-    transparent: deriveTransparentAddress(contributionId, getNetwork()),
+    transparent,
     sprout: env.SPROUT_ADDRESS,
   };
   return {

blockchain/src/util.ts

@@ -73,3 +73,15 @@ export function sleep(ms: number) {
     setTimeout(resolve, ms);
   });
 }
+
+// They come in all shapes and sizes, and nested data can get truncated as
+// [object Object], so try to extract the best parts available.
+export function extractErrMessage(err: any) {
+  if (err.response && err.response.data) {
+    if (err.response.data.error && err.response.data.error.message) {
+      return err.response.data.error.message;
+    }
+    return JSON.stringify(err.response.data, null, 2);
+  }
+  return err.message || err.toString();
+}

blockchain/src/webhooks/index.ts

@@ -5,7 +5,7 @@ import { Notifier } from "./notifiers/notifier";
 import node from "../node";
 import env from "../env";
 import { store } from "../store";
-import { sleep } from "../util";
+import { sleep, extractErrMessage } from "../util";
 import log from "../log";
 
 let blockScanTimeout: any = null;
@@ -64,8 +64,7 @@ async function scanBlock(height: number) {
     notifiers.forEach(n => n.onNewBlock && n.onNewBlock(block));
     consecutiveBlockFailures = 0;
   } catch(err) {
-    log.warn(err.response ? err.response.data : err);
-    log.warn(`Failed to fetch block ${height}, see above error`);
+    log.warn(`Failed to fetch block ${height}: ${extractErrMessage(err)}`);
     consecutiveBlockFailures++;
     // If we fail a certain number of times, it's reasonable to
     // assume that the blockchain is down, and we should just quit.
@@ -94,8 +93,7 @@ async function requestBootstrap() {
     log.debug('Requesting bootstrap from backend...');
     await send('/blockchain/bootstrap', 'GET');
   } catch(err) {
-    log.error(err.response ? err.response.data : err);
-    log.error('Request for bootstrap failed, see above for details');
+    log.error(`Request for bootstrap failed: ${extractErrMessage(err)}`);
   }
 }
 
@@ -126,7 +124,6 @@ const send: Send = (route, method, payload) => {
       return;
     }
     captureException(err);
-    const errMsg = err.response ? `Response: ${JSON.stringify(err.response.data, null, 2)}` : err.message;
-    log.error(`Webhook server request to ${method} ${route} failed: ${errMsg}`);
+    log.error(`Webhook server request to ${method} ${route} failed: ${extractErrMessage(err)}`);
   });
 };

blockchain/src/webhooks/notifiers/contribution/index.ts

@@ -10,7 +10,7 @@ import {
 } from "../../../store";
 import env from "../../../env";
 import log from "../../../log";
-import { getContributionIdFromMemo, decodeHexMemo, toBaseUnit } from "../../../util";
+import { getContributionIdFromMemo, decodeHexMemo, toBaseUnit, extractErrMessage } from "../../../util";
 
 interface ContributionConfirmationPayload {
   to: string;
@@ -26,8 +26,9 @@ export default class ContributionNotifier implements Notifier {
 
   onNewBlock = (block: BlockWithTransactions) => {
     this.checkBlockForTransparentPayments(block);
-    this.checkForMemoPayments();
-    this.checkDisclosuresForPayment(block);
+    // NOTE: Re-enable when sapling is ready
+    // this.checkForMemoPayments();
+    // this.checkDisclosuresForPayment(block);
   };
 
   registerSend = (sm: Send) => (this.send = sm);
@@ -91,7 +92,7 @@ export default class ContributionNotifier implements Notifier {
       captureException(err);
       log.error(
         'Failed to check sprout address for memo payments:\n',
-        err.response ? err.response.data : err,
+        extractErrMessage(err),
       );
     }
   };
@@ -131,9 +132,9 @@ export default class ContributionNotifier implements Notifier {
       store.dispatch(confirmPaymentDisclosure(contributionId, disclosure));
     } catch(err) {
       captureException(err);
-      log.error(
+      log.warn(
         'Encountered an error while checking disclosure:\n',
-        err.response ? err.response.data : err,
+        extractErrMessage(err),
       );
     }
   };

blockchain/tsconfig.json

@@ -50,7 +50,8 @@
     // https://www.npmjs.com/package/ts-node#help-my-types-are-missing
     "paths": {
       "stdrpc": ["types/stdrpc"],
-      "zcash-bitcore-lib": ["types/zcash-bitcore-lib"]
+      "zcash-bitcore-lib": ["types/zcash-bitcore-lib"],
+      "bitgo": ["types/bitgo"]
     },
 
     /* Source Map Options */

blockchain/types/bitgo.d.ts

@@ -0,0 +1,70 @@
+// Adapted from documentation here: https://www.bitgo.com/api/v2/?javascript
+// Far from exhaustive, only functions used are properly typed.
+
+declare module 'bitgo' {
+  // Wallet
+  interface CreateAddressOptions {
+    label: string;
+  }
+
+  interface GetAddressesOptions {
+    labelContains?: string;
+    limit?: number;
+    mine?: boolean;
+    prevId?: string;
+    chains?: number[];
+    sort?: 1 | -1;
+  }
+
+  interface AddressInfo {
+    id: string;
+    address: string;
+    chain: number;
+    index: number;
+    coin: string;
+    lastNonce: number;
+    wallet: string;
+    label: string;
+    addressType: string;
+  }
+
+  interface GetAddressResponse {
+    coin: string;
+    totalAddressCount: number;
+    pendingAddressCount: number;
+    addresses: AddressInfo[];
+    nextBatchPrevId: string;
+  }
+
+  export class Wallet {
+    id(): string;
+    label(): string;
+    createAddress(options?: CreateAddressOptions): Promise<AddressInfo>;
+    addresses(options?: GetAddressesOptions): Promise<GetAddressResponse>;
+  }
+
+  // Wallets
+  interface GetWalletOptions {
+    id: string;
+  }
+
+  export class Wallets {
+    get(options: GetWalletOptions): Promise<Wallet>;
+  }
+
+  // BaseCoin
+  export class BaseCoin {
+    wallets(): Wallets;
+  }
+
+  // BitGo
+  interface BitGoOptions {
+    env: 'test' | 'prod';
+    accessToken: string;
+  }
+
+  export class BitGo {
+    constructor(options: BitGoOptions);
+    coin(coin: string): BaseCoin;
+  }
+}