diff --git a/.gitignore b/.gitignore index 24cf00620..c30bfe054 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,8 @@ target /mutagen.sh venv .env +.env.hex +.env.0x bigtable-admin.json bigtable-writer.json **/cert.pem diff --git a/Dockerfile.const b/Dockerfile.const new file mode 100644 index 000000000..61a75c212 --- /dev/null +++ b/Dockerfile.const @@ -0,0 +1,50 @@ +# syntax=docker.io/docker/dockerfile:1.3@sha256:42399d4635eddd7a9b8a24be879d2f9a930d0ed040a61324cfdf59ef1357b3b2 +FROM docker.io/fedora:34 AS const-build + +ARG num_guardians +ENV NUM_GUARDIANS=$num_guardians + +# add additional root CAs +COPY cert.pem* /certs/ +RUN if [ -e /certs/cert.pem ]; then cp /certs/cert.pem /etc/pki/tls/certs/ca-bundle.crt; fi + +# fetch scripts/guardian-set-init.sh deps +RUN dnf -y install jq + +# fetch clients/** deps +RUN curl -fsSL https://rpm.nodesource.com/setup_16.x | bash - && dnf -y install nodejs + +# configure node & npm to work with custom root CAs +ENV NODE_EXTRA_CA_CERTS=/certs/cert.pem +ENV NODE_OPTIONS=--use-openssl-ca +RUN if [ -e /certs/cert.pem ]; then npm config set cafile /certs/cert.pem; fi + +# install token_bridge deps & build +WORKDIR /clients/token_bridge +# copy package.json & package-lock.json by themselves to create a cache layer +COPY clients/token_bridge/package.json clients/token_bridge/package-lock.json ./ +# mount the buildkit cache on npm's cache dir, install dependencies +RUN --mount=type=cache,target=/root/.npm npm ci +# copy the rest of the source files, as a layer on top of the deps +COPY clients/token_bridge ./ +RUN npm run build + +# install nft_bridge deps & build +WORKDIR /clients/nft_bridge +COPY clients/nft_bridge/package.json clients/nft_bridge/package-lock.json ./ +RUN --mount=type=cache,target=/root/.npm npm ci +COPY clients/nft_bridge ./ +RUN npm run build + +WORKDIR / + +COPY scripts ./scripts +COPY ethereum/.env.test ./ethereum/.env.test + +# run guardian-set-init.sh to create env files with the init state for NUM_GUARDIANS +RUN ./scripts/guardian-set-init.sh $NUM_GUARDIANS + +FROM scratch AS const-export +COPY --from=const-build /scripts/.env.0x ethereum/.env +COPY --from=const-build /scripts/.env.hex solana/.env +COPY --from=const-build /scripts/.env.hex terra/tools/.env diff --git a/Tiltfile b/Tiltfile index 3109e0fc5..3de488d7e 100644 --- a/Tiltfile +++ b/Tiltfile @@ -104,6 +104,15 @@ local_resource( trigger_mode = trigger_mode, ) +local_resource( + name = "const-gen", + deps = ["scripts", "clients", "ethereum/.env.test"], + cmd = 'tilt docker build -- --target const-export -f Dockerfile.const -o type=local,dest=. --build-arg num_guardians=%s .' % (num_guardians), + env = {"DOCKER_BUILDKIT": "1"}, + allow_parallel = True, + trigger_mode = trigger_mode, +) + if algorand: local_resource( name = "teal-gen", @@ -210,14 +219,8 @@ k8s_resource( trigger_mode = trigger_mode, ) -local_resource( - name = "guardian-set-init", - deps = ["scripts", "ethereum", "clients", "solana", "terra"], - cmd = './scripts/guardian-set-init.sh %s' % (num_guardians), - allow_parallel = True, - trigger_mode = trigger_mode, -) -if num_guardians >= 2: +# guardian set update - triggered by "tilt args" changes +if num_guardians >= 2 and ci == False: local_resource( name = "guardian-set-update", resource_deps = guardian_resource_deps + ["guardian"], @@ -274,7 +277,7 @@ if solana: port_forward(8900, name = "Solana WS [:8900]", host = webHost), port_forward(9000, name = "Solana PubSub [:9000]", host = webHost), ], - resource_deps = ["guardian-set-init"], + resource_deps = ["const-gen"], labels = ["solana"], trigger_mode = trigger_mode, ) @@ -360,7 +363,7 @@ k8s_resource( port_forwards = [ port_forward(8545, name = "Ganache RPC [:8545]", host = webHost), ], - resource_deps = ["guardian-set-init"], + resource_deps = ["const-gen"], labels = ["evm"], trigger_mode = trigger_mode, ) @@ -370,7 +373,7 @@ k8s_resource( port_forwards = [ port_forward(8546, name = "Ganache RPC [:8546]", host = webHost), ], - resource_deps = ["guardian-set-init"], + resource_deps = ["const-gen"], labels = ["evm"], trigger_mode = trigger_mode, ) @@ -549,7 +552,7 @@ k8s_resource( port_forward(26657, name = "Terra RPC [:26657]", host = webHost), port_forward(1317, name = "Terra LCD [:1317]", host = webHost), ], - resource_deps = ["guardian-set-init"], + resource_deps = ["const-gen"], labels = ["terra"], trigger_mode = trigger_mode, ) diff --git a/docs/devnet.md b/docs/devnet.md index abb72f2e3..334f13cfc 100644 --- a/docs/devnet.md +++ b/docs/devnet.md @@ -29,6 +29,6 @@ The terra testnet can be used just like a normal localterra network (can be selected in finder and station). -### devent guardian addresses +### devnet guardian addresses see [../scripts/devnet-consts.json](../scripts/devnet-consts.json) diff --git a/scripts/guardian-set-init.sh b/scripts/guardian-set-init.sh index 833ea0141..f41b590a5 100755 --- a/scripts/guardian-set-init.sh +++ b/scripts/guardian-set-init.sh @@ -1,40 +1,75 @@ #!/usr/bin/env bash # This script allows devnet initalization with more than one guardian. # First argument is the number of guardians for the initial guardian set. -set -e +set -exuo pipefail numGuardians=$1 echo "number of guardians to initialize: ${numGuardians}" addressesJson="./scripts/devnet-consts.json" +# working files for accumulating state +envFile="./scripts/.env.hex" # for generic hex data, for solana, terra, etc +ethFile="./scripts/.env.0x" # for "0x" prefixed data, for ethereum scripts -# create an array of strings containing the ECDSA public keys of the devnet guardians in the guardianset +# copy the eth defaults so we can override just the things we need +cp ./ethereum/.env.test $ethFile + +# function for updating or inserting a KEY=value pair in a file. +function upsert_env_file { + file=${1} # file will be created if it does not exist. + key=${2} # line must start with the key. + new_value=${3} + + # replace the value if it exists, else, append it to the file + if [[ -f $file ]] && grep -q "^$key=" $file; then + # file has the key, update it: + sed -i "/^$key=/s/=.*/=$new_value/" $file + else + # file does not have the key, add it: + echo "$key=$new_value" >> $file + fi +} + +# assert jq exists before trying to use it +if ! type -p jq; then + echo "ERROR: jq is not installed"! >&2 + exit 1 +fi + + +# 1) guardian public keys - used as the inital guardian set when initializing contracts. +echo "generating guardian set addresses" +# create an array of strings containing the ECDSA public keys of the devnet guardians in the guardianset: # guardiansPublicEth has the leading "0x" that Eth scripts expect. guardiansPublicEth=$(jq -c --argjson lastIndex $numGuardians '.devnetGuardians[:$lastIndex] | [.[].public]' $addressesJson) # guardiansPublicHex does not have a leading "0x", just hex strings. guardiansPublicHex=$(jq -c --argjson lastIndex $numGuardians '.devnetGuardians[:$lastIndex] | [.[].public[2:]]' $addressesJson) +# also make a CSV string of the hex addresses, so the client scripts that need that format don't have to. +guardiansPublicHexCSV=$(echo ${guardiansPublicHex} | jq --raw-output -c '. | join(",")') + +# write the lists of addresses to the env files +initSigners="INIT_SIGNERS" +upsert_env_file $ethFile $initSigners $guardiansPublicEth +upsert_env_file $envFile $initSigners $guardiansPublicHex +upsert_env_file $envFile "INIT_SIGNERS_CSV" $guardiansPublicHexCSV -# copy the eth defaults to a new file so we can override just the things we need -cp ./ethereum/.env.test ./ethereum/.env - -# override the default INIT_SIGNERS with the list created above -sed -i "/INIT_SIGNERS=/c\INIT_SIGNERS=$guardiansPublicEth" ./ethereum/.env - -# create a local .env file, to be used by solana & terra -echo "INIT_SIGNERS=$guardiansPublicHex" > ./scripts/.env - - - +# 2) guardian private keys - used for generating the initial governance VAAs (register token bridge & nft bridge contracts on each chain). +echo "generating guardian set keys" # create an array of strings containing the private keys of the devnet guardians in the guardianset guardiansPrivate=$(jq -c --argjson lastIndex $numGuardians '.devnetGuardians[:$lastIndex] | [.[].private]' $addressesJson) - # create a CSV string with the private keys of the guardians in the guardianset, that will be used to create registration VAAs guardiansPrivateCSV=$( echo ${guardiansPrivate} | jq --raw-output -c '. | join(",")') -echo "getting contract addresses from $addressesJson" +# write the lists of keys to the env files +upsert_env_file $ethFile "INIT_SIGNERS_KEYS_JSON" $guardiansPrivate +upsert_env_file $envFile "INIT_SIGNERS_KEYS_CSV" $guardiansPrivateCSV + + +# 3) fetch and store the contract addresses that we need to make contract registration governance VAAs for: +echo "getting contract addresses for chain registrations from $addressesJson" # get addresses from the constants file solTokenBridge=$(jq --raw-output '.chains."1".contracts.tokenBridgeEmitterAddress' $addressesJson) ethTokenBridge=$(jq --raw-output '.chains."2".contracts.tokenBridgeEmitterAddress' $addressesJson) @@ -45,68 +80,91 @@ solNFTBridge=$(jq --raw-output '.chains."1".contracts.nftBridgeEmitterAddress' $ ethNFTBridge=$(jq --raw-output '.chains."2".contracts.nftBridgeEmitterAddress' $addressesJson) terraNFTBridge=$(jq --raw-output '.chains."3".contracts.nftBridgeEmitterAddress' $addressesJson) -# generate the registration VAAs +# 4) create token bridge registration VAAs +echo "generating contract registration VAAs for token bridges" # fetch dependencies for the clients/token_bridge script that generates token bridge registration VAAs if [[ ! -d ./clients/token_bridge/node_modules ]]; then echo "going to install node modules in clients/token_bridge" npm ci --prefix clients/token_bridge && npm run build --prefix clients/token_bridge fi -# create token bridge registration VAAs -echo "generating VAAs for token bridges" +# invoke clients/token_bridge commands to create registration VAAs solTokenBridgeVAA=$(npm --prefix clients/token_bridge run --silent main -- generate_register_chain_vaa 1 0x${solTokenBridge} --guardian_secret ${guardiansPrivateCSV}) ethTokenBridgeVAA=$(npm --prefix clients/token_bridge run --silent main -- generate_register_chain_vaa 2 0x${ethTokenBridge} --guardian_secret ${guardiansPrivateCSV} ) terraTokenBridgeVAA=$(npm --prefix clients/token_bridge run --silent main -- generate_register_chain_vaa 3 0x${terraTokenBridge} --guardian_secret ${guardiansPrivateCSV}) bscTokenBridgeVAA=$(npm --prefix clients/token_bridge run --silent main -- generate_register_chain_vaa 4 0x${bscTokenBridge} --guardian_secret ${guardiansPrivateCSV}) +# 5) create nft bridge registration VAAs # fetch dependencies for the clients/nft_bridge script that generates nft bridge registration VAAs if [[ ! -d ./clients/nft_bridge/node_modules ]]; then echo "going to install node modules in clients/nft_bridge" npm ci --prefix clients/nft_bridge && npm run build --prefix clients/nft_bridge fi -# create nft bridge registration VAAs -echo "generating VAAs for nft bridges" +echo "generating contract registration VAAs for nft bridges" solNFTBridgeVAA=$(npm --prefix clients/nft_bridge run --silent main -- generate_register_chain_vaa 1 0x${solNFTBridge} --guardian_secret ${guardiansPrivateCSV}) ethNFTBridgeVAA=$(npm --prefix clients/nft_bridge run --silent main -- generate_register_chain_vaa 2 0x${ethNFTBridge} --guardian_secret ${guardiansPrivateCSV}) terraNFTBridgeVAA=$(npm --prefix clients/nft_bridge run --silent main -- generate_register_chain_vaa 3 0x${terraNFTBridge} --guardian_secret ${guardiansPrivateCSV}) -# write the registration VAAs to env files -echo "updating .env files" + +# 6) write the registration VAAs to env files +echo "writing VAAs to .env files" +# define the keys that will hold the chain registration governance VAAs +solTokenBridge="REGISTER_SOL_TOKEN_BRIDGE_VAA" +ethTokenBridge="REGISTER_ETH_TOKEN_BRIDGE_VAA" +terraTokenBridge="REGISTER_TERRA_TOKEN_BRIDGE_VAA" +bscTokenBridge="REGISTER_BSC_TOKEN_BRIDGE_VAA" + +solNFTBridge="REGISTER_SOL_NFT_BRIDGE_VAA" +ethNFTBridge="REGISTER_ETH_NFT_BRIDGE_VAA" +terraNFTBridge="REGISTER_TERRA_NFT_BRIDGE_VAA" + # solana token bridge -sed -i "/REGISTER_SOL_TOKEN_BRIDGE_VAA=/c\REGISTER_SOL_TOKEN_BRIDGE_VAA=$solTokenBridgeVAA" ./ethereum/.env -echo "REGISTER_SOL_TOKEN_BRIDGE_VAA=$solTokenBridgeVAA" >> ./scripts/.env +upsert_env_file $ethFile $solTokenBridge $solTokenBridgeVAA +upsert_env_file $envFile $solTokenBridge $solTokenBridgeVAA # solana nft bridge -sed -i "/REGISTER_SOL_NFT_BRIDGE_VAA=/c\REGISTER_SOL_NFT_BRIDGE_VAA=$solNFTBridgeVAA" ./ethereum/.env -echo "REGISTER_SOL_NFT_BRIDGE_VAA=$solNFTBridgeVAA" >> ./scripts/.env +upsert_env_file $ethFile $solNFTBridge $solNFTBridgeVAA +upsert_env_file $envFile $solNFTBridge $solNFTBridgeVAA # ethereum token bridge -sed -i "/REGISTER_ETH_TOKEN_BRIDGE_VAA=/c\REGISTER_ETH_TOKEN_BRIDGE_VAA=$ethTokenBridgeVAA" ./ethereum/.env -echo "REGISTER_ETH_TOKEN_BRIDGE_VAA=$ethTokenBridgeVAA" >> ./scripts/.env +upsert_env_file $ethFile $ethTokenBridge $ethTokenBridgeVAA +upsert_env_file $envFile $ethTokenBridge $ethTokenBridgeVAA # ethereum nft bridge -sed -i "/REGISTER_ETH_NFT_BRIDGE_VAA=/c\REGISTER_ETH_NFT_BRIDGE_VAA=$ethNFTBridgeVAA" ./ethereum/.env -echo "REGISTER_ETH_NFT_BRIDGE_VAA=$ethNFTBridgeVAA" >> ./scripts/.env +upsert_env_file $ethFile $ethNFTBridge $ethNFTBridgeVAA +upsert_env_file $envFile $ethNFTBridge $ethNFTBridgeVAA # terra token bridge -sed -i "/REGISTER_TERRA_TOKEN_BRIDGE_VAA=/c\REGISTER_TERRA_TOKEN_BRIDGE_VAA=$terraTokenBridgeVAA" ./ethereum/.env -echo "REGISTER_TERRA_TOKEN_BRIDGE_VAA=$terraTokenBridgeVAA" >> ./scripts/.env +upsert_env_file $ethFile $terraTokenBridge $terraTokenBridgeVAA +upsert_env_file $envFile $terraTokenBridge $terraTokenBridgeVAA # terra nft bridge -sed -i "/REGISTER_TERRA_NFT_BRIDGE_VAA=/c\REGISTER_TERRA_NFT_BRIDGE_VAA=$terraNFTBridgeVAA" ./ethereum/.env -echo "REGISTER_TERRA_NFT_BRIDGE_VAA=$terraNFTBridgeVAA" >> ./scripts/.env +upsert_env_file $ethFile $terraNFTBridge $terraNFTBridgeVAA +upsert_env_file $envFile $terraNFTBridge $terraNFTBridgeVAA # bsc token bridge -sed -i "/REGISTER_BSC_TOKEN_BRIDGE_VAA=/c\REGISTER_BSC_TOKEN_BRIDGE_VAA=$bscTokenBridgeVAA" ./ethereum/.env -echo "REGISTER_BSC_TOKEN_BRIDGE_VAA=$bscTokenBridgeVAA" >> ./scripts/.env +upsert_env_file $ethFile $bscTokenBridge $bscTokenBridgeVAA +upsert_env_file $envFile $bscTokenBridge $bscTokenBridgeVAA -# copy the local .env file to the solana & terra dirs -cp ./scripts/.env ./solana -cp ./scripts/.env ./terra/tools +# 7) copy the local .env file to the solana & terra dirs, if the script is running on the host machine +# chain dirs will not exist if running in docker for Tilt, only if running locally. check before copying. +# copy ethFile to ethereum +if [[ -d ./ethereum ]]; then + echo "copying $ethFile to /etherum/.env" + cp $ethFile ./ethereum/.env +fi +# copy the hex envFile to each of the non-EVM chains +for envDest in ./solana/.env ./terra/tools/.env; do + dirname=$(dirname $envDest) + if [[ -d "$dirname" ]]; then + echo "copying $envFile to $envDest" + cp $envFile $envDest + fi +done echo "guardian set init complete!" diff --git a/solana/devnet_setup.sh b/solana/devnet_setup.sh index 283a2f669..ce78c4367 100755 --- a/solana/devnet_setup.sh +++ b/solana/devnet_setup.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # This script configures the devnet for test transfers with hardcoded addresses. -set -x +set -xu # Configure CLI (works the same as upstream Solana CLI) mkdir -p ~/.config/solana/cli @@ -35,8 +35,6 @@ chain_id_ethereum=2 # load the .env file with the devent init data source .env -# remove brackets and double quotes, transform json list of strings to CSV with no quotes. -guardians=$(echo $INIT_SIGNERS | sed -r 's/(\[|\]|\")//g') retry () { while ! $@; do @@ -87,7 +85,7 @@ token-bridge-client create-meta "$nft" "Not a PUNK 2🎸" "PUNK2🎸" "https://w # Create the bridge contract at a known address # OK to fail on subsequent attempts (already created). -retry client create-bridge "$bridge_address" "$guardians" 86400 100 +retry client create-bridge "$bridge_address" "$INIT_SIGNERS_CSV" 86400 100 # Initialize the token bridge retry token-bridge-client create-bridge "$token_bridge_address" "$bridge_address"