move guardian-set-init.sh to docker/tilt

.gitignore

@@ -8,6 +8,8 @@ target
 /mutagen.sh
 venv
 .env
+.env.hex
+.env.0x
 bigtable-admin.json
 bigtable-writer.json
 **/cert.pem

Dockerfile.const

@@ -0,0 +1,50 @@
+# syntax=docker.io/docker/dockerfile:1.3@sha256:42399d4635eddd7a9b8a24be879d2f9a930d0ed040a61324cfdf59ef1357b3b2
+FROM docker.io/fedora:34 AS const-build
+
+ARG num_guardians
+ENV NUM_GUARDIANS=$num_guardians
+
+# add additional root CAs
+COPY cert.pem* /certs/
+RUN if [ -e /certs/cert.pem ]; then cp /certs/cert.pem /etc/pki/tls/certs/ca-bundle.crt; fi
+
+# fetch scripts/guardian-set-init.sh deps
+RUN dnf -y install jq
+
+# fetch clients/** deps
+RUN curl -fsSL https://rpm.nodesource.com/setup_16.x | bash - && dnf -y install nodejs
+
+# configure node & npm to work with custom root CAs
+ENV NODE_EXTRA_CA_CERTS=/certs/cert.pem
+ENV NODE_OPTIONS=--use-openssl-ca
+RUN if [ -e /certs/cert.pem ]; then npm config set cafile /certs/cert.pem; fi
+
+# install token_bridge deps & build
+WORKDIR /clients/token_bridge
+# copy package.json & package-lock.json by themselves to create a cache layer
+COPY clients/token_bridge/package.json clients/token_bridge/package-lock.json ./
+# mount the buildkit cache on npm's cache dir, install dependencies
+RUN --mount=type=cache,target=/root/.npm npm ci
+# copy the rest of the source files, as a layer on top of the deps
+COPY clients/token_bridge ./
+RUN npm run build
+
+# install nft_bridge deps & build
+WORKDIR /clients/nft_bridge
+COPY clients/nft_bridge/package.json clients/nft_bridge/package-lock.json ./
+RUN --mount=type=cache,target=/root/.npm npm ci
+COPY clients/nft_bridge ./
+RUN npm run build
+
+WORKDIR /
+
+COPY scripts ./scripts
+COPY ethereum/.env.test ./ethereum/.env.test
+
+# run guardian-set-init.sh to create env files with the init state for NUM_GUARDIANS
+RUN ./scripts/guardian-set-init.sh $NUM_GUARDIANS
+
+FROM scratch AS const-export
+COPY --from=const-build /scripts/.env.0x ethereum/.env
+COPY --from=const-build /scripts/.env.hex solana/.env
+COPY --from=const-build /scripts/.env.hex terra/tools/.env

Tiltfile

@@ -104,6 +104,15 @@ local_resource(
     trigger_mode = trigger_mode,
 )
 
+local_resource(
+    name = "const-gen",
+    deps = ["scripts", "clients", "ethereum/.env.test"],
+    cmd = 'tilt docker build -- --target const-export -f Dockerfile.const -o type=local,dest=. --build-arg num_guardians=%s .' % (num_guardians),
+    env = {"DOCKER_BUILDKIT": "1"},
+    allow_parallel = True,
+    trigger_mode = trigger_mode,
+)
+
 if algorand:
     local_resource(
         name = "teal-gen",
@@ -210,14 +219,8 @@ k8s_resource(
     trigger_mode = trigger_mode,
 )
 
-local_resource(
+# guardian set update - triggered by "tilt args" changes
-    name = "guardian-set-init",
+if num_guardians >= 2 and ci == False:
-    deps = ["scripts", "ethereum", "clients", "solana", "terra"],
-    cmd = './scripts/guardian-set-init.sh %s' % (num_guardians),
-    allow_parallel = True,
-    trigger_mode = trigger_mode,
-)
-if num_guardians >= 2:
     local_resource(
         name = "guardian-set-update",
         resource_deps = guardian_resource_deps + ["guardian"],
@@ -274,7 +277,7 @@ if solana:
             port_forward(8900, name = "Solana WS [:8900]", host = webHost),
             port_forward(9000, name = "Solana PubSub [:9000]", host = webHost),
         ],
-        resource_deps = ["guardian-set-init"],
+        resource_deps = ["const-gen"],
         labels = ["solana"],
         trigger_mode = trigger_mode,
     )
@@ -360,7 +363,7 @@ k8s_resource(
     port_forwards = [
         port_forward(8545, name = "Ganache RPC [:8545]", host = webHost),
     ],
-    resource_deps = ["guardian-set-init"],
+    resource_deps = ["const-gen"],
     labels = ["evm"],
     trigger_mode = trigger_mode,
 )
@@ -370,7 +373,7 @@ k8s_resource(
     port_forwards = [
         port_forward(8546, name = "Ganache RPC [:8546]", host = webHost),
     ],
-    resource_deps = ["guardian-set-init"],
+    resource_deps = ["const-gen"],
     labels = ["evm"],
     trigger_mode = trigger_mode,
 )
@@ -549,7 +552,7 @@ k8s_resource(
         port_forward(26657, name = "Terra RPC [:26657]", host = webHost),
         port_forward(1317, name = "Terra LCD [:1317]", host = webHost),
     ],
-    resource_deps = ["guardian-set-init"],
+    resource_deps = ["const-gen"],
     labels = ["terra"],
     trigger_mode = trigger_mode,
 )

docs/devnet.md

@@ -29,6 +29,6 @@
 
 The terra testnet can be used just like a normal localterra network (can be selected in finder and station).
 
-### devent guardian addresses
+### devnet guardian addresses
 
 see [../scripts/devnet-consts.json](../scripts/devnet-consts.json)

scripts/guardian-set-init.sh

@@ -1,40 +1,75 @@
 #!/usr/bin/env bash
 # This script allows devnet initalization with more than one guardian.
 # First argument is the number of guardians for the initial guardian set.
-set -e
+set -exuo pipefail
 
 numGuardians=$1
 echo "number of guardians to initialize: ${numGuardians}"
 
 addressesJson="./scripts/devnet-consts.json"
 
+# working files for accumulating state
+envFile="./scripts/.env.hex" # for generic hex data, for solana, terra, etc
+ethFile="./scripts/.env.0x" # for "0x" prefixed data, for ethereum scripts
 
-# create an array of strings containing the ECDSA public keys of the devnet guardians in the guardianset
+# copy the eth defaults so we can override just the things we need
+cp ./ethereum/.env.test $ethFile
 
+
+# function for updating or inserting a KEY=value pair in a file.
+function upsert_env_file {
+    file=${1} # file will be created if it does not exist.
+    key=${2}  # line must start with the key.
+    new_value=${3}
+
+    # replace the value if it exists, else, append it to the file
+    if [[ -f $file ]] && grep -q "^$key=" $file; then
+        # file has the key, update it:
+        sed -i "/^$key=/s/=.*/=$new_value/" $file
+    else
+        # file does not have the key, add it:
+        echo "$key=$new_value" >> $file
+    fi
+}
+
+# assert jq exists before trying to use it
+if ! type -p jq; then
+    echo "ERROR: jq is not installed"! >&2
+    exit 1
+fi
+
+
+# 1) guardian public keys - used as the inital guardian set when initializing contracts.
+echo "generating guardian set addresses"
+# create an array of strings containing the ECDSA public keys of the devnet guardians in the guardianset:
 # guardiansPublicEth has the leading "0x" that Eth scripts expect.
 guardiansPublicEth=$(jq -c --argjson lastIndex $numGuardians '.devnetGuardians[:$lastIndex] | [.[].public]' $addressesJson)
 # guardiansPublicHex does not have a leading "0x", just hex strings.
 guardiansPublicHex=$(jq -c --argjson lastIndex $numGuardians '.devnetGuardians[:$lastIndex] | [.[].public[2:]]' $addressesJson)
+# also make a CSV string of the hex addresses, so the client scripts that need that format don't have to.
+guardiansPublicHexCSV=$(echo ${guardiansPublicHex} | jq --raw-output -c  '. | join(",")')
+
+# write the lists of addresses to the env files
+initSigners="INIT_SIGNERS"
+upsert_env_file $ethFile $initSigners $guardiansPublicEth
+upsert_env_file $envFile $initSigners $guardiansPublicHex
+upsert_env_file $envFile "INIT_SIGNERS_CSV" $guardiansPublicHexCSV
 
 
-# copy the eth defaults to a new file so we can override just the things we need
+# 2) guardian private keys - used for generating the initial governance VAAs (register token bridge & nft bridge contracts on each chain).
-cp ./ethereum/.env.test ./ethereum/.env
+echo "generating guardian set keys"
-
-# override the default INIT_SIGNERS with the list created above
-sed -i "/INIT_SIGNERS=/c\INIT_SIGNERS=$guardiansPublicEth" ./ethereum/.env
-
-# create a local .env file, to be used by solana & terra
-echo "INIT_SIGNERS=$guardiansPublicHex" > ./scripts/.env
-
-
-
 # create an array of strings containing the private keys of the devnet guardians in the guardianset
 guardiansPrivate=$(jq -c --argjson lastIndex $numGuardians '.devnetGuardians[:$lastIndex] | [.[].private]' $addressesJson)
-
 # create a CSV string with the private keys of the guardians in the guardianset, that will be used to create registration VAAs
 guardiansPrivateCSV=$( echo ${guardiansPrivate} | jq --raw-output -c  '. | join(",")')
 
-echo "getting contract addresses from $addressesJson"
+# write the lists of keys to the env files
+upsert_env_file $ethFile "INIT_SIGNERS_KEYS_JSON" $guardiansPrivate
+upsert_env_file $envFile "INIT_SIGNERS_KEYS_CSV"  $guardiansPrivateCSV
+
+
+# 3) fetch and store the contract addresses that we need to make contract registration governance VAAs for:
+echo "getting contract addresses for chain registrations from $addressesJson"
 # get addresses from the constants file
 solTokenBridge=$(jq --raw-output '.chains."1".contracts.tokenBridgeEmitterAddress' $addressesJson)
 ethTokenBridge=$(jq --raw-output '.chains."2".contracts.tokenBridgeEmitterAddress' $addressesJson)
@@ -45,68 +80,91 @@ solNFTBridge=$(jq --raw-output '.chains."1".contracts.nftBridgeEmitterAddress' $
 ethNFTBridge=$(jq --raw-output '.chains."2".contracts.nftBridgeEmitterAddress' $addressesJson)
 terraNFTBridge=$(jq --raw-output '.chains."3".contracts.nftBridgeEmitterAddress' $addressesJson)
 
-# generate the registration VAAs
 
+# 4) create token bridge registration VAAs
+echo "generating contract registration VAAs for token bridges"
 # fetch dependencies for the clients/token_bridge script that generates token bridge registration VAAs
 if [[ ! -d ./clients/token_bridge/node_modules ]]; then
     echo "going to install node modules in clients/token_bridge"
     npm ci --prefix clients/token_bridge && npm run build --prefix clients/token_bridge
 fi
-# create token bridge registration VAAs
+# invoke clients/token_bridge commands to create registration VAAs
-echo "generating VAAs for token bridges"
 solTokenBridgeVAA=$(npm --prefix clients/token_bridge run --silent main -- generate_register_chain_vaa 1 0x${solTokenBridge} --guardian_secret ${guardiansPrivateCSV})
 ethTokenBridgeVAA=$(npm --prefix clients/token_bridge run --silent main -- generate_register_chain_vaa 2 0x${ethTokenBridge} --guardian_secret ${guardiansPrivateCSV} )
 terraTokenBridgeVAA=$(npm --prefix clients/token_bridge run --silent main -- generate_register_chain_vaa 3 0x${terraTokenBridge} --guardian_secret ${guardiansPrivateCSV})
 bscTokenBridgeVAA=$(npm --prefix clients/token_bridge run --silent main -- generate_register_chain_vaa 4 0x${bscTokenBridge} --guardian_secret ${guardiansPrivateCSV})
 
 
+# 5) create nft bridge registration VAAs
 # fetch dependencies for the clients/nft_bridge script that generates nft bridge registration VAAs
 if [[ ! -d ./clients/nft_bridge/node_modules ]]; then
     echo "going to install node modules in clients/nft_bridge"
     npm ci --prefix clients/nft_bridge && npm run build --prefix clients/nft_bridge
 fi
-# create nft bridge registration VAAs
+echo "generating contract registration VAAs for nft bridges"
-echo "generating VAAs for nft bridges"
 solNFTBridgeVAA=$(npm --prefix clients/nft_bridge run --silent main -- generate_register_chain_vaa 1 0x${solNFTBridge} --guardian_secret ${guardiansPrivateCSV})
 ethNFTBridgeVAA=$(npm --prefix clients/nft_bridge run --silent main -- generate_register_chain_vaa 2 0x${ethNFTBridge} --guardian_secret ${guardiansPrivateCSV})
 terraNFTBridgeVAA=$(npm --prefix clients/nft_bridge run --silent main -- generate_register_chain_vaa 3 0x${terraNFTBridge} --guardian_secret ${guardiansPrivateCSV})
 
 
-# write the registration VAAs to env files
+
-echo "updating .env files"
+# 6) write the registration VAAs to env files
+echo "writing VAAs to .env files"
+# define the keys that will hold the chain registration governance VAAs
+solTokenBridge="REGISTER_SOL_TOKEN_BRIDGE_VAA"
+ethTokenBridge="REGISTER_ETH_TOKEN_BRIDGE_VAA"
+terraTokenBridge="REGISTER_TERRA_TOKEN_BRIDGE_VAA"
+bscTokenBridge="REGISTER_BSC_TOKEN_BRIDGE_VAA"
+
+solNFTBridge="REGISTER_SOL_NFT_BRIDGE_VAA"
+ethNFTBridge="REGISTER_ETH_NFT_BRIDGE_VAA"
+terraNFTBridge="REGISTER_TERRA_NFT_BRIDGE_VAA"
+
 
 # solana token bridge
-sed -i "/REGISTER_SOL_TOKEN_BRIDGE_VAA=/c\REGISTER_SOL_TOKEN_BRIDGE_VAA=$solTokenBridgeVAA" ./ethereum/.env
+upsert_env_file $ethFile $solTokenBridge $solTokenBridgeVAA
-echo "REGISTER_SOL_TOKEN_BRIDGE_VAA=$solTokenBridgeVAA" >> ./scripts/.env
+upsert_env_file $envFile $solTokenBridge $solTokenBridgeVAA
 # solana nft bridge
-sed -i "/REGISTER_SOL_NFT_BRIDGE_VAA=/c\REGISTER_SOL_NFT_BRIDGE_VAA=$solNFTBridgeVAA" ./ethereum/.env
+upsert_env_file $ethFile $solNFTBridge $solNFTBridgeVAA
-echo "REGISTER_SOL_NFT_BRIDGE_VAA=$solNFTBridgeVAA" >> ./scripts/.env
+upsert_env_file $envFile $solNFTBridge $solNFTBridgeVAA
 
 
 # ethereum token bridge
-sed -i "/REGISTER_ETH_TOKEN_BRIDGE_VAA=/c\REGISTER_ETH_TOKEN_BRIDGE_VAA=$ethTokenBridgeVAA" ./ethereum/.env
+upsert_env_file $ethFile $ethTokenBridge $ethTokenBridgeVAA
-echo "REGISTER_ETH_TOKEN_BRIDGE_VAA=$ethTokenBridgeVAA" >> ./scripts/.env
+upsert_env_file $envFile $ethTokenBridge $ethTokenBridgeVAA
 # ethereum nft bridge
-sed -i "/REGISTER_ETH_NFT_BRIDGE_VAA=/c\REGISTER_ETH_NFT_BRIDGE_VAA=$ethNFTBridgeVAA" ./ethereum/.env
+upsert_env_file $ethFile $ethNFTBridge $ethNFTBridgeVAA
-echo "REGISTER_ETH_NFT_BRIDGE_VAA=$ethNFTBridgeVAA" >> ./scripts/.env
+upsert_env_file $envFile $ethNFTBridge $ethNFTBridgeVAA
 
 
 # terra token bridge
-sed -i "/REGISTER_TERRA_TOKEN_BRIDGE_VAA=/c\REGISTER_TERRA_TOKEN_BRIDGE_VAA=$terraTokenBridgeVAA" ./ethereum/.env
+upsert_env_file $ethFile $terraTokenBridge $terraTokenBridgeVAA
-echo "REGISTER_TERRA_TOKEN_BRIDGE_VAA=$terraTokenBridgeVAA" >> ./scripts/.env
+upsert_env_file $envFile $terraTokenBridge $terraTokenBridgeVAA
 # terra nft bridge
-sed -i "/REGISTER_TERRA_NFT_BRIDGE_VAA=/c\REGISTER_TERRA_NFT_BRIDGE_VAA=$terraNFTBridgeVAA" ./ethereum/.env
+upsert_env_file $ethFile $terraNFTBridge $terraNFTBridgeVAA
-echo "REGISTER_TERRA_NFT_BRIDGE_VAA=$terraNFTBridgeVAA" >> ./scripts/.env
+upsert_env_file $envFile $terraNFTBridge $terraNFTBridgeVAA
 
 
 # bsc token bridge
-sed -i "/REGISTER_BSC_TOKEN_BRIDGE_VAA=/c\REGISTER_BSC_TOKEN_BRIDGE_VAA=$bscTokenBridgeVAA" ./ethereum/.env
+upsert_env_file $ethFile $bscTokenBridge $bscTokenBridgeVAA
-echo "REGISTER_BSC_TOKEN_BRIDGE_VAA=$bscTokenBridgeVAA" >> ./scripts/.env
+upsert_env_file $envFile $bscTokenBridge $bscTokenBridgeVAA
 
 
-# copy the local .env file to the solana & terra dirs
+# 7) copy the local .env file to the solana & terra dirs, if the script is running on the host machine
-cp ./scripts/.env ./solana
+# chain dirs will not exist if running in docker for Tilt, only if running locally. check before copying.
-cp ./scripts/.env ./terra/tools
+# copy ethFile to ethereum
+if [[ -d ./ethereum ]]; then
+    echo "copying $ethFile to /etherum/.env"
+    cp $ethFile ./ethereum/.env
+fi
 
+# copy the hex envFile to each of the non-EVM chains
+for envDest in ./solana/.env ./terra/tools/.env; do
+    dirname=$(dirname $envDest)
+    if [[ -d "$dirname" ]]; then
+        echo "copying $envFile to $envDest"
+        cp $envFile $envDest
+    fi
+done
 
 echo "guardian set init complete!"

solana/devnet_setup.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 # This script configures the devnet for test transfers with hardcoded addresses.
-set -x
+set -xu
 
 # Configure CLI (works the same as upstream Solana CLI)
 mkdir -p ~/.config/solana/cli
@@ -35,8 +35,6 @@ chain_id_ethereum=2
 
 # load the .env file with the devent init data
 source .env
-# remove brackets and double quotes, transform json list of strings to CSV with no quotes.
-guardians=$(echo $INIT_SIGNERS | sed -r 's/(\[|\]|\")//g')
 
 retry () {
   while ! $@; do
@@ -87,7 +85,7 @@ token-bridge-client create-meta "$nft" "Not a PUNK 2🎸" "PUNK2🎸" "https://w
 
 # Create the bridge contract at a known address
 # OK to fail on subsequent attempts (already created).
-retry client create-bridge "$bridge_address" "$guardians" 86400 100
+retry client create-bridge "$bridge_address" "$INIT_SIGNERS_CSV" 86400 100
 
 # Initialize the token bridge
 retry token-bridge-client create-bridge "$token_bridge_address" "$bridge_address"