Merge pull request #406 from grant-project/bitgo

Bitgo integration for mainnet
This commit is contained in:
Daniel Ternyak 2019-03-21 15:57:03 -05:00 committed by GitHub
commit 7f2d2c9490
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 1774 additions and 102 deletions

View File

@ -158,6 +158,7 @@ def register_commands(app):
app.cli.add_command(commands.lint)
app.cli.add_command(commands.clean)
app.cli.add_command(commands.urls)
app.cli.add_command(commands.reset_db_chain_data)
app.cli.add_command(proposal.commands.create_proposal)
app.cli.add_command(proposal.commands.create_proposals)
app.cli.add_command(user.commands.set_admin)

View File

@ -8,6 +8,7 @@ import click
from flask import current_app
from flask.cli import with_appcontext
from werkzeug.exceptions import MethodNotAllowed, NotFound
from sqlalchemy import text
HERE = os.path.abspath(os.path.dirname(__file__))
PROJECT_ROOT = os.path.join(HERE, os.pardir)
@ -137,3 +138,46 @@ def urls(url, order):
for row in rows:
click.echo(str_template.format(*row[:column_length]))
@click.command()
@with_appcontext
def reset_db_chain_data():
"""Removes chain-state dependent entities from the database. Cannot be undone!"""
from grant.extensions import db
from grant.proposal.models import Proposal
from grant.user.models import UserSettings
from grant.task.models import Task
# Delete all proposals. Should cascade to contributions, comments etc.
p_count = 0
for proposal in Proposal.query.all():
db.session.delete(proposal)
p_count = p_count + 1
# Delete all outstanding tasks
t_count = Task.query.delete()
# Delete refund address from settings
s_count = 0
for settings in UserSettings.query.all():
if settings.refund_address:
settings.refund_address = None
db.session.add(settings)
s_count = s_count + 1
# Commit state
db.session.commit()
# Attempt to reset contribution ID sequence, psql specific. Don't fail out
# if this messes up though, just warn them.
try:
db.engine.execute(text('ALTER SEQUENCE proposal_contribution_id_seq RESTART WITH 1'))
except e:
print(e)
print('Failed to reset contribution id sequence, see above error. Continuing anyway.')
print('Successfully wiped chain-dependent db state!')
print(f'* Deleted {p_count} proposals and their linked entities')
print(f'* Deleted {t_count} tasks')
print(f'* Removed refund address from {s_count} user settings')

View File

@ -14,13 +14,26 @@ MINIMUM_BLOCK_CONFIRMATIONS="6"
API_SECRET_HASH=""
API_SECRET_KEY=""
############################ ADDRESS DERIVATION ############################
# You should only set one OR the other. The former will generate addresses #
# using Bip32 address derivation. The latter uses BitGo's API. If you set #
# both, BitGo takes precedence. API key should ONLY HAVE VIEW ACCESS! #
############################################################################
# BITGO_WALLET_ID=""
# BITGO_ACCESS_TOKEN=""
### OR ###
# BIP32_XPUB=""
############################################################################
# Addresses, run `yarn genaddress` to get sprout information
SPROUT_ADDRESS=""
SPROUT_VIEWKEY=""
# extended public seed
BIP32_XPUB=""
# Block heights to fall back on for starting our scan
MAINNET_START_BLOCK="464000"
TESTNET_START_BLOCK="390000"
@ -30,3 +43,6 @@ SENTRY_DSN=""
# Logging level
LOG_LEVEL="debug"
# Fixie proxy URL for BitGo requests (optional)
# FIXIE_URL=""

View File

@ -35,6 +35,7 @@
"@types/dotenv": "^6.1.0",
"@types/ws": "^6.0.1",
"axios": "0.18.0",
"bitgo": "4.48.1",
"body-parser": "1.18.3",
"cors": "2.8.5",
"dotenv": "^6.1.0",

View File

@ -1,4 +1,5 @@
import node from '../node';
import { extractErrMessage } from '../util';
async function printAddressAndKey() {
try {
@ -9,11 +10,7 @@ async function printAddressAndKey() {
console.log(`SPROUT_ADDRESS="${address}"`);
console.log(`SPROUT_VIEWKEY="${viewkey}"\n`);
} catch(err) {
if (err.response && err.response.data) {
console.error(err.response.data);
} else {
console.error(err);
}
console.error(extractErrMessage(err));
process.exit(1);
}
}

57
blockchain/src/bitgo.ts Normal file
View File

@ -0,0 +1,57 @@
import { BitGo, Wallet } from 'bitgo';
import bitcore from "zcash-bitcore-lib";
import env from './env';
import log from './log';
import { getNetwork } from './node';
let bitgoWallet: Wallet;
export async function initBitGo() {
if (!env.BITGO_ACCESS_TOKEN || !env.BITGO_WALLET_ID) {
log.info('BITGO environment variables not set, nooping initBitGo');
return;
}
// Assert that we're on mainnet
const network = getNetwork();
if (network !== bitcore.Networks.mainnet) {
throw new Error(`BitGo cannot be used on anything but mainnet, connected node is ${network}`);
}
const proxy = env.FIXIE_URL || undefined;
const bitgo = new BitGo({
env: 'prod', // Non-prod ZEC is not supported
accessToken: env.BITGO_ACCESS_TOKEN,
proxy,
});
bitgoWallet = await bitgo.coin('zec').wallets().get({ id: env.BITGO_WALLET_ID });
log.info(`Initialized BitGo wallet "${bitgoWallet.label()}"`);
if (proxy) {
log.info(`Proxying BitGo requests through ${proxy}`);
}
}
export async function getContributionAddress(id: number) {
if (!bitgoWallet) {
throw new Error('Must run initBitGo before getContributionAddress');
}
// Attempt to fetch first
const label = `Contribution #${id}`;
const res = await bitgoWallet.addresses({ labelContains: label });
if (res.addresses.length) {
if (res.addresses.length > 1) {
log.warn(`Contribution ${id} has ${res.addresses.length} associated with it. Using the first one (${res.addresses[0].address})`);
}
return res.addresses[0].address;
}
// Create a new one otherwise
const createRes = await bitgoWallet.createAddress({ label });
log.info(`Generate new address for contribution ${id}`);
return createRes.address;
}
function generateLabel(id: number) {
return `Contribution #${id}`;
}

View File

@ -18,31 +18,55 @@ const DEFAULTS = {
ZCASH_NODE_PASSWORD: "",
MINIMUM_BLOCK_CONFIRMATIONS: "6",
BITGO_WALLET_ID: "",
BITGO_ACCESS_TOKEN: "",
BIP32_XPUB: "",
SPROUT_ADDRESS: "",
SPROUT_VIEWKEY: "",
BIP32_XPUB: "",
MAINNET_START_BLOCK: "464000",
TESTNET_START_BLOCK: "390000",
SENTRY_DSN: "",
FIXIE_URL: "",
};
const OPTIONAL: { [key: string]: undefined | boolean } = {
BITGO_WALLET_ID: true,
BITGO_ACCESS_TOKEN: true,
BIP32_XPUB: true,
FIXIE_URL: true,
// NOTE: Remove these from optional when sapling is ready
SPROUT_ADDRESS: true,
SPROUT_VIEWKEY: true,
}
type CustomEnvironment = typeof DEFAULTS;
// ignore when testing
if (process.env.NODE_ENV !== "test") {
// Set environment variables, throw on missing required ones
Object.entries(DEFAULTS).forEach(([k, v]) => {
if (!process.env[k]) {
const defVal = (DEFAULTS as any)[k];
if (defVal) {
console.info(`Using default environment variable ${k}="${defVal}"`);
process.env[k] = defVal;
} else {
} else if (!OPTIONAL[k]) {
throw new Error(`Missing required environment variable ${k}`);
}
}
});
// Ensure we have either xpub or bitgo, and warn if we have both
if (!process.env.BIP32_XPUB && (!process.env.BITGO_WALLET_ID || !process.env.BITGO_ACCESS_TOKEN)) {
throw new Error('Either BIP32_XPUB or BITGO_* environment variables required, missing both');
}
if (process.env.BIP32_XPUB && process.env.BITGO_WALLET_ID) {
console.info('BIP32_XPUB and BITGO environment variables set, BIP32_XPUB will be ignored');
}
}
export default (process.env as any) as CustomEnvironment;

View File

@ -2,6 +2,8 @@ import * as Sentry from "@sentry/node";
import * as Webhooks from "./webhooks";
import * as RestServer from "./server";
import { initNode } from "./node";
import { initBitGo } from "./bitgo";
import { extractErrMessage } from "./util";
import env from "./env";
import log from "./log";
@ -15,6 +17,7 @@ async function start() {
log.info("============== Starting services ==============");
await initNode();
await initBitGo();
await RestServer.start();
Webhooks.start();
log.info("===============================================");
@ -28,4 +31,8 @@ process.on("SIGINT", () => {
process.exit();
});
start();
start().catch(err => {
Sentry.captureException(err);
log.error(`Unexpected error while starting blockchain watcher: ${extractErrMessage(err)}`);
process.exit(1);
});

View File

@ -3,6 +3,7 @@ import bitcore from "zcash-bitcore-lib";
import { captureException } from "@sentry/node";
import env from "./env";
import log from "./log";
import { extractErrMessage } from "./util";
export interface BlockChainInfo {
chain: string;
@ -49,6 +50,27 @@ export interface Transaction {
vjoinsplit: any[];
}
export interface RawTransaction {
txid: string;
hex: string;
overwintered: boolean;
version: number;
versiongroupid: number;
locktime: number;
expiryheight: string;
vin: VIn[];
vout: VOut[];
valueBalance: string;
blockhash: string;
blocktime: number;
confirmations: number;
time: number;
// unclear what these are
vjoinsplit: any[];
vShieldedSpend: any[];
vShieldedOutput: any[];
}
export interface Block {
hash: string;
confirmations: number;
@ -119,6 +141,10 @@ interface ZCashNode {
(numberOrHash: string | number, verbosity: 0): Promise<string>;
};
gettransaction: (txid: string) => Promise<Transaction>;
getrawtransaction: {
(numberOrHash: string | number, verbosity: 1): Promise<RawTransaction>;
(numberOrHash: string | number, verbosity?: 0): Promise<string>;
};
validateaddress: (address: string) => Promise<ValidationResponse>;
z_getbalance: (address: string, minConf?: number) => Promise<number>;
z_getnewaddress: (type?: "sprout" | "sapling") => Promise<string>;
@ -166,31 +192,29 @@ export async function initNode() {
}
} catch (err) {
captureException(err);
log.error(err.response ? err.response.data : err);
log.error(
"Failed to connect to zcash node with the following credentials:\r\n",
rpcOptions
);
log.error(extractErrMessage(err));
log.error(`Failed to connect to zcash node with the following credentials: ${JSON.stringify(rpcOptions, null, 2)}`);
process.exit(1);
}
// Check if sprout address is readable
try {
if (!env.SPROUT_ADDRESS) {
console.error("Missing SPROUT_ADDRESS environment variable, exiting");
process.exit(1);
}
await node.z_getbalance(env.SPROUT_ADDRESS as string);
} catch (err) {
if (!env.SPROUT_VIEWKEY) {
log.error(
"Unable to view SPROUT_ADDRESS and missing SPROUT_VIEWKEY environment variable, exiting"
);
process.exit(1);
}
await node.z_importviewingkey(env.SPROUT_VIEWKEY as string);
await node.z_getbalance(env.SPROUT_ADDRESS as string);
}
// NOTE: Replace with sapling when ready
// try {
// if (!env.SPROUT_ADDRESS) {
// console.error("Missing SPROUT_ADDRESS environment variable, exiting");
// process.exit(1);
// }
// await node.z_getbalance(env.SPROUT_ADDRESS as string);
// } catch (err) {
// if (!env.SPROUT_VIEWKEY) {
// log.error(
// "Unable to view SPROUT_ADDRESS and missing SPROUT_VIEWKEY environment variable, exiting"
// );
// process.exit(1);
// }
// await node.z_importviewingkey(env.SPROUT_VIEWKEY as string);
// await node.z_getbalance(env.SPROUT_ADDRESS as string);
// }
}
export function getNetwork() {
@ -204,29 +228,27 @@ export function getNetwork() {
export async function getBootstrapBlockHeight(txid: string | undefined) {
if (txid) {
try {
const tx = await node.gettransaction(txid);
const tx = await node.getrawtransaction(txid, 1);
const block = await node.getblock(tx.blockhash);
const height =
block.height - parseInt(env.MINIMUM_BLOCK_CONFIRMATIONS, 10);
return height.toString();
} catch (err) {
console.warn(
`Attempted to get block height for tx ${txid} but failed with the following error:\n`,
err
);
console.warn("Falling back to hard-coded starter blocks");
log.warn(`Attempted to get block height for tx ${txid} but failed with the following error: ${extractErrMessage(err)}`);
}
}
// If we can't find the latest tx block, fall back to when the grant
// system first launched, and scan from there.
// system first launched, and scan from there. Regtest or unknown networks
// start from the bottom.
const net = getNetwork();
let height = "0";
if (net === bitcore.Networks.mainnet) {
return env.MAINNET_START_BLOCK;
height = env.MAINNET_START_BLOCK;
} else if (net === bitcore.Networks.testnet && !net.regtestEnabled) {
return env.TESTNET_START_BLOCK;
height = env.TESTNET_START_BLOCK;
}
// Regtest or otherwise unknown networks should start at the bottom
return "0";
log.info(`Falling back to hard-coded starter block height ${height}`);
return height;
}

View File

@ -14,7 +14,7 @@ import {
} from '../store';
import env from '../env';
import node, { getBootstrapBlockHeight } from '../node';
import { makeContributionMemo } from '../util';
import { makeContributionMemo, extractErrMessage } from '../util';
import log from '../log';
// Configure server
@ -29,8 +29,16 @@ app.use(authMiddleware);
// Routes
app.post('/bootstrap', async (req, res) => {
const { pendingContributions, latestTxId } = req.body;
const info = await node.getblockchaininfo();
const startHeight = await getBootstrapBlockHeight(latestTxId);
let info;
let startHeight;
try {
info = await node.getblockchaininfo();
startHeight = await getBootstrapBlockHeight(latestTxId);
} catch(err) {
log.error(`Unknown node error during bootstrap: ${extractErrMessage(err)}`);
return res.status(500).json({ error: 'Unknown zcash node error' });
}
console.info('Bootstrapping watcher!');
console.info(' * Start height:', startHeight);
@ -39,11 +47,18 @@ app.post('/bootstrap', async (req, res) => {
console.info('Generating addresses to watch for each contribution...');
// Running generate address on each will add each contribution to redux state
pendingContributions.forEach((c: any) => {
store.dispatch(generateAddresses(c.id));
});
console.info(`Done! Generated ${pendingContributions.length} addresses.`);
store.dispatch(setStartingBlockHeight(startHeight));
try {
const dispatchers = pendingContributions.map(async (c: any) => {
const action = await generateAddresses(c.id);
store.dispatch(action);
});
await Promise.all(dispatchers);
console.info(`Done! Generated ${pendingContributions.length} addresses.`);
store.dispatch(setStartingBlockHeight(startHeight));
} catch(err) {
log.error(`Unknown error during bootstrap address generation: ${extractErrMessage(err)}`);
return res.status(500).json({ error: 'Failed to generate addresses for contributions' });
}
// Send back some basic info about where the chain is at
res.json({
@ -54,20 +69,29 @@ app.post('/bootstrap', async (req, res) => {
});
});
app.get('/contribution/addresses', (req, res) => {
app.get('/contribution/addresses', async (req, res) => {
const { contributionId } = req.query;
let addresses = getAddressesByContributionId(store.getState(), contributionId)
if (!addresses) {
const action = generateAddresses(req.query.contributionId);
addresses = action.payload.addresses;
store.dispatch(action);
try {
const action = await generateAddresses(contributionId);
addresses = action.payload.addresses;
store.dispatch(action);
} catch(err) {
log.error(`Unknown error during address generation for contribution ${contributionId}: ${extractErrMessage(err)}`);
}
}
if (addresses) {
res.json({
data: {
...addresses,
memo: makeContributionMemo(contributionId),
},
});
} else {
res.status(500).json({ error: 'Failed to generate addresses' });
}
res.json({
data: {
...addresses,
memo: makeContributionMemo(contributionId),
},
});
});
@ -96,7 +120,7 @@ app.post('/contribution/disclosure', async (req, res) => {
return res.status(400).json({ error: err.response.data.error.message });
}
else {
log.error('Unknown node error:', err.response ? err.response.data : err);
log.error(`Unknown node error during disclosure: ${extractErrMessage(err)}`);
return res.status(500).json({ error: 'Unknown zcash node error' });
}
}

View File

@ -1,6 +1,7 @@
import { captureException } from "@sentry/node";
import { Request, Response, NextFunction } from 'express';
import log from "../../log";
import { extractErrMessage } from "../../util";
export default function errorHandler(err: Error, req: Request, res: Response, next: NextFunction) {
// Non-error responses, or something else handled & responded
@ -9,7 +10,7 @@ export default function errorHandler(err: Error, req: Request, res: Response, ne
}
captureException(err);
log.error(`Uncaught ${err.name} exception at ${req.method} ${req.path}: ${err.message}`);
log.error(`Uncaught ${err.name} exception at ${req.method} ${req.path}: ${extractErrMessage(err)}`);
log.debug(`Query: ${JSON.stringify(req.query, null, 2)}`);
log.debug(`Body: ${JSON.stringify(req.body, null, 2)}`);
log.debug(`Full stacktrace:\n${err.stack}`);

View File

@ -1,6 +1,7 @@
import type, { AddressCollection } from './types';
import { deriveTransparentAddress } from '../util';
import { getNetwork } from '../node';
import { getContributionAddress } from '../bitgo';
import env from '../env';
export function setStartingBlockHeight(height: string | number) {
@ -10,10 +11,16 @@ export function setStartingBlockHeight(height: string | number) {
}
}
export function generateAddresses(contributionId: number) {
// 2^31 is the maximum number of BIP32 addresses
export async function generateAddresses(contributionId: number) {
let transparent;
if (env.BITGO_WALLET_ID) {
transparent = await getContributionAddress(contributionId);
} else {
transparent = deriveTransparentAddress(contributionId, getNetwork());
}
const addresses: AddressCollection = {
transparent: deriveTransparentAddress(contributionId, getNetwork()),
transparent,
sprout: env.SPROUT_ADDRESS,
};
return {
@ -45,8 +52,9 @@ export function confirmPaymentDisclosure(contributionId: number, disclosure: str
};
}
type UnwrapPromise<T> = T extends Promise<infer U> ? U : T;
export type ActionTypes =
| ReturnType<typeof setStartingBlockHeight>
| ReturnType<typeof generateAddresses>
| UnwrapPromise<ReturnType<typeof generateAddresses>>
| ReturnType<typeof addPaymentDisclosure>
| ReturnType<typeof confirmPaymentDisclosure>;

View File

@ -73,3 +73,15 @@ export function sleep(ms: number) {
setTimeout(resolve, ms);
});
}
// They come in all shapes and sizes, and nested data can get truncated as
// [object Object], so try to extract the best parts available.
export function extractErrMessage(err: any) {
if (err.response && err.response.data) {
if (err.response.data.error && err.response.data.error.message) {
return err.response.data.error.message;
}
return JSON.stringify(err.response.data, null, 2);
}
return err.message || err.toString();
}

View File

@ -5,7 +5,7 @@ import { Notifier } from "./notifiers/notifier";
import node from "../node";
import env from "../env";
import { store } from "../store";
import { sleep } from "../util";
import { sleep, extractErrMessage } from "../util";
import log from "../log";
let blockScanTimeout: any = null;
@ -64,8 +64,7 @@ async function scanBlock(height: number) {
notifiers.forEach(n => n.onNewBlock && n.onNewBlock(block));
consecutiveBlockFailures = 0;
} catch(err) {
log.warn(err.response ? err.response.data : err);
log.warn(`Failed to fetch block ${height}, see above error`);
log.warn(`Failed to fetch block ${height}: ${extractErrMessage(err)}`);
consecutiveBlockFailures++;
// If we fail a certain number of times, it's reasonable to
// assume that the blockchain is down, and we should just quit.
@ -94,8 +93,7 @@ async function requestBootstrap() {
log.debug('Requesting bootstrap from backend...');
await send('/blockchain/bootstrap', 'GET');
} catch(err) {
log.error(err.response ? err.response.data : err);
log.error('Request for bootstrap failed, see above for details');
log.error(`Request for bootstrap failed: ${extractErrMessage(err)}`);
}
}
@ -126,7 +124,6 @@ const send: Send = (route, method, payload) => {
return;
}
captureException(err);
const errMsg = err.response ? `Response: ${JSON.stringify(err.response.data, null, 2)}` : err.message;
log.error(`Webhook server request to ${method} ${route} failed: ${errMsg}`);
log.error(`Webhook server request to ${method} ${route} failed: ${extractErrMessage(err)}`);
});
};

View File

@ -10,7 +10,7 @@ import {
} from "../../../store";
import env from "../../../env";
import log from "../../../log";
import { getContributionIdFromMemo, decodeHexMemo, toBaseUnit } from "../../../util";
import { getContributionIdFromMemo, decodeHexMemo, toBaseUnit, extractErrMessage } from "../../../util";
interface ContributionConfirmationPayload {
to: string;
@ -26,8 +26,9 @@ export default class ContributionNotifier implements Notifier {
onNewBlock = (block: BlockWithTransactions) => {
this.checkBlockForTransparentPayments(block);
this.checkForMemoPayments();
this.checkDisclosuresForPayment(block);
// NOTE: Re-enable when sapling is ready
// this.checkForMemoPayments();
// this.checkDisclosuresForPayment(block);
};
registerSend = (sm: Send) => (this.send = sm);
@ -91,7 +92,7 @@ export default class ContributionNotifier implements Notifier {
captureException(err);
log.error(
'Failed to check sprout address for memo payments:\n',
err.response ? err.response.data : err,
extractErrMessage(err),
);
}
};
@ -131,9 +132,9 @@ export default class ContributionNotifier implements Notifier {
store.dispatch(confirmPaymentDisclosure(contributionId, disclosure));
} catch(err) {
captureException(err);
log.error(
log.warn(
'Encountered an error while checking disclosure:\n',
err.response ? err.response.data : err,
extractErrMessage(err),
);
}
};

View File

@ -50,7 +50,8 @@
// https://www.npmjs.com/package/ts-node#help-my-types-are-missing
"paths": {
"stdrpc": ["types/stdrpc"],
"zcash-bitcore-lib": ["types/zcash-bitcore-lib"]
"zcash-bitcore-lib": ["types/zcash-bitcore-lib"],
"bitgo": ["types/bitgo"]
},
/* Source Map Options */

71
blockchain/types/bitgo.d.ts vendored Normal file
View File

@ -0,0 +1,71 @@
// Adapted from documentation here: https://www.bitgo.com/api/v2/?javascript
// Far from exhaustive, only functions used are properly typed.
declare module 'bitgo' {
// Wallet
interface CreateAddressOptions {
label: string;
}
interface GetAddressesOptions {
labelContains?: string;
limit?: number;
mine?: boolean;
prevId?: string;
chains?: number[];
sort?: 1 | -1;
}
interface AddressInfo {
id: string;
address: string;
chain: number;
index: number;
coin: string;
lastNonce: number;
wallet: string;
label: string;
addressType: string;
}
interface GetAddressResponse {
coin: string;
totalAddressCount: number;
pendingAddressCount: number;
addresses: AddressInfo[];
nextBatchPrevId: string;
}
export class Wallet {
id(): string;
label(): string;
createAddress(options?: CreateAddressOptions): Promise<AddressInfo>;
addresses(options?: GetAddressesOptions): Promise<GetAddressResponse>;
}
// Wallets
interface GetWalletOptions {
id: string;
}
export class Wallets {
get(options: GetWalletOptions): Promise<Wallet>;
}
// BaseCoin
export class BaseCoin {
wallets(): Wallets;
}
// BitGo
interface BitGoOptions {
env: 'test' | 'prod';
accessToken: string;
proxy?: string;
}
export class BitGo {
constructor(options: BitGoOptions);
coin(coin: string): BaseCoin;
}
}

File diff suppressed because it is too large Load Diff