Keygen automatic restart possibility

2019-11-25 23:41:36 +03:00 · 2019-11-25 23:41:36 +03:00 · 11e22b3bc0
parent dc5b415b97
commit 11e22b3bc0
12 changed files with 146 additions and 19 deletions
--- a/demo/validator1/.env.development
+++ b/demo/validator1/.env.development
@ -9,7 +9,7 @@ SIDE_SHARED_DB_ADDRESS=0xd5fE0D28e058D375b0b038fFbB446Da37E85fFdc

 FOREIGN_URL=http://http-api:8000
 FOREIGN_CHAIN_ID=Binance-Dev
-FOREIGN_ASSET=DEV-BA8
+FOREIGN_ASSET=DEV-9BA
 FOREIGN_FETCH_MAX_TIME_INTERVAL=60000
 FOREIGN_FETCH_INTERVAL=5000
 FOREIGN_FETCH_BLOCK_TIME_OFFSET=10000
@ -18,6 +18,9 @@ SIGN_ATTEMPT_TIMEOUT=120000
 SIGN_NONCE_CHECK_INTERVAL=10000
 SEND_TIMEOUT=60000

+KEYGEN_ATTEMPT_TIMEOUT=120000
+KEYGEN_EPOCH_CHECK_INTERVAL=10000
+
 VALIDATOR_PRIVATE_KEY=2be3f252e16541bf1bb2d4a517d2bf173e6d09f2d765d32c64dc50515aec63ea

 VOTES_PROXY_PORT=5001
--- a/demo/validator1/.env.staging
+++ b/demo/validator1/.env.staging
@ -18,6 +18,9 @@ SIGN_ATTEMPT_TIMEOUT=120000
 SIGN_NONCE_CHECK_INTERVAL=10000
 SEND_TIMEOUT=60000

+KEYGEN_ATTEMPT_TIMEOUT=120000
+KEYGEN_EPOCH_CHECK_INTERVAL=10000
+
 #VALIDATOR_PRIVATE_KEY is taken from .keys

 VOTES_PROXY_PORT=5001
--- a/demo/validator2/.env.development
+++ b/demo/validator2/.env.development
@ -9,7 +9,7 @@ SIDE_SHARED_DB_ADDRESS=0xd5fE0D28e058D375b0b038fFbB446Da37E85fFdc

 FOREIGN_URL=http://http-api:8000
 FOREIGN_CHAIN_ID=Binance-Dev
-FOREIGN_ASSET=DEV-BA8
+FOREIGN_ASSET=DEV-9BA
 FOREIGN_FETCH_MAX_TIME_INTERVAL=60000
 FOREIGN_FETCH_INTERVAL=5000
 FOREIGN_FETCH_BLOCK_TIME_OFFSET=10000
@ -18,6 +18,9 @@ SIGN_ATTEMPT_TIMEOUT=120000
 SIGN_NONCE_CHECK_INTERVAL=10000
 SEND_TIMEOUT=60000

+KEYGEN_ATTEMPT_TIMEOUT=120000
+KEYGEN_EPOCH_CHECK_INTERVAL=10000
+
 VALIDATOR_PRIVATE_KEY=e59d58c77b791f98f10187117374ae9c589d48a62720ec6a5e142b0cc134f685

 VOTES_PROXY_PORT=5002
--- a/demo/validator2/.env.staging
+++ b/demo/validator2/.env.staging
@ -18,6 +18,9 @@ SIGN_ATTEMPT_TIMEOUT=120000
 SIGN_NONCE_CHECK_INTERVAL=10000
 SEND_TIMEOUT=60000

+KEYGEN_ATTEMPT_TIMEOUT=120000
+KEYGEN_EPOCH_CHECK_INTERVAL=10000
+
 #VALIDATOR_PRIVATE_KEY is taken from .keys

 VOTES_PROXY_PORT=5002
--- a/demo/validator3/.env.development
+++ b/demo/validator3/.env.development
@ -9,7 +9,7 @@ SIDE_SHARED_DB_ADDRESS=0xd5fE0D28e058D375b0b038fFbB446Da37E85fFdc

 FOREIGN_URL=http://http-api:8000
 FOREIGN_CHAIN_ID=Binance-Dev
-FOREIGN_ASSET=DEV-BA8
+FOREIGN_ASSET=DEV-9BA
 FOREIGN_FETCH_MAX_TIME_INTERVAL=60000
 FOREIGN_FETCH_INTERVAL=5000
 FOREIGN_FETCH_BLOCK_TIME_OFFSET=10000
@ -18,6 +18,9 @@ SIGN_ATTEMPT_TIMEOUT=120000
 SIGN_NONCE_CHECK_INTERVAL=10000
 SEND_TIMEOUT=60000

+KEYGEN_ATTEMPT_TIMEOUT=120000
+KEYGEN_EPOCH_CHECK_INTERVAL=10000
+
 VALIDATOR_PRIVATE_KEY=afaa4d4d6e54d25b0bf0361e3fd6cef562f6311bf6200de2dd0aa4cab63ae3b5

 VOTES_PROXY_PORT=5003
--- a/demo/validator3/.env.staging
+++ b/demo/validator3/.env.staging
@ -18,6 +18,9 @@ SIGN_ATTEMPT_TIMEOUT=120000
 SIGN_NONCE_CHECK_INTERVAL=10000
 SEND_TIMEOUT=60000

+KEYGEN_ATTEMPT_TIMEOUT=120000
+KEYGEN_EPOCH_CHECK_INTERVAL=10000
+
 #VALIDATOR_PRIVATE_KEY is taken from .keys

 VOTES_PROXY_PORT=5003
--- a/src/oracle/docker-compose-test.yml
+++ b/src/oracle/docker-compose-test.yml
@ -34,6 +34,8 @@ services:
    environment:
      - 'RABBITMQ_URL=amqp://rabbitmq:5672'
      - 'PROXY_URL=http://local_proxy:8001'
+      - KEYGEN_ATTEMPT_TIMEOUT
+      - KEYGEN_EPOCH_CHECK_INTERVAL
      - LOG_LEVEL
    volumes:
      - '${PWD}/${TARGET_NETWORK}/keys:/keys'
--- a/src/oracle/docker-compose.yml
+++ b/src/oracle/docker-compose.yml
@ -31,6 +31,8 @@ services:
    environment:
      - 'RABBITMQ_URL=amqp://rabbitmq:5672'
      - 'PROXY_URL=http://proxy:8001'
+      - KEYGEN_ATTEMPT_TIMEOUT
+      - KEYGEN_EPOCH_CHECK_INTERVAL
      - LOG_LEVEL
    volumes:
      - '${PWD}/${TARGET_NETWORK}/keys:/keys'
--- a/src/oracle/proxy/index.js
+++ b/src/oracle/proxy/index.js
@ -73,6 +73,19 @@ function sideSendQuery(query) {
  })
 }

+async function status(req, res) {
+  logger.debug('Status call')
+  const [bridgeEpoch, bridgeStatus] = await Promise.all([
+    bridge.epoch(),
+    bridge.status()
+  ])
+  res.send({
+    bridgeEpoch,
+    bridgeStatus
+  })
+  logger.debug('Status end')
+}
+
 async function get(req, res) {
  logger.debug('Get call, %o', req.body.key)
  const round = req.body.key.second
@ -131,12 +144,26 @@ async function signupKeygen(req, res) {
  const epoch = await bridge.nextEpoch()
  const partyId = await bridge.getNextPartyId(validatorAddress)

+  logger.debug('Checking previous attempts')
+  let attempt = 1
+  let uuid
+  while (true) {
+    uuid = `k${epoch}_${attempt}`
+    const data = await sharedDb.getData(validatorAddress, ethers.utils.id(uuid), ethers.utils.id('round1_0'))
+    if (data.length === 2) {
+      break
+    }
+    logger.trace(`Attempt ${attempt} is already used`)
+    attempt += 1
+  }
+  logger.debug(`Using attempt ${attempt}`)
+
  if (partyId === 0) {
    res.send(Err({ message: 'Not a validator' }))
    logger.debug('Not a validator')
  } else {
    res.send(Ok({
-      uuid: `k${epoch}`,
+      uuid,
      number: partyId
    }))
    logger.debug('SignupKeygen end')
@ -287,6 +314,22 @@ async function voteChangeThreshold(req, res) {
  }
 }

+async function voteChangeRangeSize(req, res) {
+  if (/^[0-9]+$/.test(req.params.rangeSize)) {
+    logger.info('Voting for changing range size')
+    const epoch = await bridge.epoch()
+    const message = buildMessage(
+      Action.VOTE_CHANGE_RANGE_SIZE,
+      epoch,
+      parseInt(req.params.rangeSize, 10),
+      padZeros(req.attempt, 54)
+    )
+    await processMessage(message)
+    res.send('Voted\n')
+    logger.info('Voted successfully')
+  }
+}
+
 async function voteChangeCloseEpoch(req, res) {
  if (req.params.closeEpoch === 'true' || req.params.closeEpoch === 'false') {
    logger.info('Voting for changing close epoch')
@ -350,7 +393,7 @@ async function info(req, res) {
  try {
    const [
      x, y, epoch, rangeSize, nextRangeSize, closeEpoch, nextCloseEpoch, epochStartBlock,
-      foreignNonce, nextEpoch, threshold, nextThreshold, validators, nextValidators, status,
+      foreignNonce, nextEpoch, threshold, nextThreshold, validators, nextValidators, bridgeStatus,
      homeBalance
    ] = await Promise.all([
      bridge.getX().then((value) => new BN(value).toString(16)),
@ -394,7 +437,7 @@ async function info(req, res) {
      homeBalance,
      foreignBalanceTokens: parseFloat(balances[FOREIGN_ASSET]) || 0,
      foreignBalanceNative: parseFloat(balances.BNB) || 0,
-      bridgeStatus: decodeStatus(status)
+      bridgeStatus: decodeStatus(bridgeStatus)
    }
    logger.trace('%o', msg)
    res.send(msg)
@ -408,6 +451,8 @@ async function info(req, res) {
  logger.debug('Info end')
 }

+app.get('/status', status)
+
 app.post('/get', get)
 app.post('/set', set)
 app.post('/signupkeygen', signupKeygen)
@ -437,6 +482,7 @@ votesProxyApp.use('/vote', (req, res, next) => {
 votesProxyApp.get('/vote/addValidator/:validator', voteAddValidator)
 votesProxyApp.get('/vote/removeValidator/:validator', voteRemoveValidator)
 votesProxyApp.get('/vote/changeThreshold/:threshold', voteChangeThreshold)
+votesProxyApp.get('/vote/changeRangeSize/:rangeSize', voteChangeRangeSize)
 votesProxyApp.get('/vote/changeCloseEpoch/:closeEpoch', voteChangeCloseEpoch)
 votesProxyApp.get('/info', info)

--- a/src/oracle/tss-keygen/keygen.js
+++ b/src/oracle/tss-keygen/keygen.js
@ -9,6 +9,12 @@ const { publicKeyToAddress } = require('./crypto')
 const { delay } = require('./wait')

 const { RABBITMQ_URL, PROXY_URL } = process.env
+const KEYGEN_ATTEMPT_TIMEOUT = parseInt(process.env.KEYGEN_ATTEMPT_TIMEOUT, 10)
+const KEYGEN_EPOCH_CHECK_INTERVAL = parseInt(process.env.KEYGEN_EPOCH_CHECK_INTERVAL, 10)
+
+const KEYGEN_OK = 0
+const KEYGEN_EPOCH_INTERRUPT = 1
+const KEYGEN_FAILED = 2

 const app = express()

@ -34,6 +40,54 @@ function writeParams(parties, threshold) {
  }))
 }

+function killKeygen() {
+  exec.execSync('pkill gg18_keygen || true')
+}
+
+function keygen(keysFile, epoch) {
+  let restartTimeoutId
+  let epochDaemonIntervalId
+  let epochInterrupt
+
+  return new Promise((resolve) => {
+    const cmd = exec.execFile('./keygen-entrypoint.sh', [PROXY_URL, keysFile], (error) => {
+      logger.trace('Keygen entrypoint exited, %o', error)
+      clearTimeout(restartTimeoutId)
+      clearInterval(epochDaemonIntervalId)
+      currentKeygenEpoch = null
+      if (fs.existsSync(keysFile)) {
+        logger.info(`Finished keygen for epoch ${epoch}`)
+        resolve(KEYGEN_OK)
+      } else {
+        logger.warn(`Keygen for epoch ${epoch} failed, will start new attempt`)
+        resolve(epochInterrupt ? KEYGEN_EPOCH_INTERRUPT : KEYGEN_FAILED)
+      }
+    })
+    cmd.stdout.on('data', (data) => {
+      const str = data.toString()
+      if (str.includes('Got all party signups')) {
+        restartTimeoutId = setTimeout(killKeygen, KEYGEN_ATTEMPT_TIMEOUT)
+      }
+      logger.debug(str)
+    })
+    cmd.stderr.on('data', (data) => logger.debug(data.toString()))
+
+    // Kill keygen if keygen for current epoch is already confirmed
+    epochDaemonIntervalId = setInterval(async () => {
+      logger.info(`Checking if bridge has confirmations keygen for epoch ${epoch}`)
+      const { bridgeEpoch, bridgeStatus } = (await proxyClient.get('/status')).data
+      logger.trace(`Current bridge epoch: ${bridgeEpoch}, current bridge status: ${bridgeStatus}`)
+      if (bridgeEpoch > epoch || bridgeStatus > 3) {
+        logger.info(`Bridge has already confirmed keygen for epoch ${epoch}`)
+        epochInterrupt = true
+        // Additional delay, maybe keygen will eventually finish
+        await delay(5000)
+        killKeygen()
+      }
+    }, KEYGEN_EPOCH_CHECK_INTERVAL)
+  })
+}
+
 async function keygenConsumer(msg) {
  const { epoch, parties, threshold } = JSON.parse(msg.content)
  logger.info(`Consumed new epoch event, starting keygen for epoch ${epoch}`)
@ -44,23 +98,26 @@ async function keygenConsumer(msg) {
  currentKeygenEpoch = epoch

  writeParams(parties, threshold)
-  const cmd = exec.execFile('./keygen-entrypoint.sh', [PROXY_URL, keysFile], async () => {
-    currentKeygenEpoch = null
-    if (fs.existsSync(keysFile)) {
-      logger.info(`Finished keygen for epoch ${epoch}`)
+
+  while (true) {
+    const keygenResult = await keygen(keysFile, epoch)
+
+    if (keygenResult === KEYGEN_OK) {
      const publicKey = JSON.parse(fs.readFileSync(keysFile))[5]
      logger.warn(`Generated multisig account in binance chain: ${publicKeyToAddress(publicKey)}`)

      logger.info('Sending keys confirmation')
      await confirmKeygen(publicKey, epoch)
-    } else {
-      logger.warn(`Keygen for epoch ${epoch} failed`)
+      break
+    } else if (keygenResult === KEYGEN_EPOCH_INTERRUPT) {
+      logger.warn('Keygen was interrupted by epoch daemon')
+      break
    }
-    logger.debug('Ack for keygen message')
-    channel.ack(msg)
-  })
-  cmd.stdout.on('data', (data) => logger.debug(data.toString()))
-  cmd.stderr.on('data', (data) => logger.debug(data.toString()))
+
+    await delay(1000)
+  }
+  logger.info('Acking message')
+  channel.ack(msg)
 }

 async function main() {
@ -81,7 +138,7 @@ async function main() {
    logger.info(`Consumed new cancel event for epoch ${epoch} keygen`)
    if (currentKeygenEpoch === epoch) {
      logger.info('Cancelling current keygen')
-      exec.execSync('pkill gg18_keygen || true')
+      killKeygen()
    }
    channel.ack(msg)
  })
--- a/src/oracle/tss-sign/signer.js
+++ b/src/oracle/tss-sign/signer.js
@ -213,6 +213,8 @@ function sign(keysFile, tx, publicKey, signerAddress) {
      if (sequence > tx.tx.sequence) {
        logger.info('Account already has needed nonce, cancelling current sign process')
        nonceInterrupt = true
+        // Additional delay, maybe signer will eventually finish
+        await delay(5000)
        killSigner()
      }
    }, SIGN_NONCE_CHECK_INTERVAL)
--- a/src/tss/multi-party-ecdsa
+++ b/src/tss/multi-party-ecdsa
@ -1 +1 @@
-Subproject commit f67cc7ffab0f460807f6e112ad93b5ce58e47ac6
+Subproject commit 1eb620d9a37f9839d3c391a3765062846e07039f