From a97a34e1740e54616252b5aca1f4aaf4beb1d167 Mon Sep 17 00:00:00 2001 From: Stan Drozd Date: Wed, 4 Aug 2021 16:33:14 +0200 Subject: [PATCH] Add a test pyth instance Change-Id: Ifa5b50fb80f01f386fc8079eec3a0564df8072e1 --- Tiltfile | 9 ++++ devnet/pyth.yaml | 55 +++++++++++++++++++++++++ devnet/solana-devnet.yaml | 3 ++ solana/Dockerfile | 31 ++++++++++++-- solana/keys/pyth_program.json | 1 + solana/keys/pyth_publisher.json | 1 + solana/pyth2wormhole/Cargo.lock | 2 + third_party/pyth/Dockerfile | 44 ++++++++++++++++++++ third_party/pyth/pyth_publisher.py | 66 ++++++++++++++++++++++++++++++ third_party/pyth/pyth_utils.py | 59 ++++++++++++++++++++++++++ 10 files changed, 268 insertions(+), 3 deletions(-) create mode 100644 devnet/pyth.yaml create mode 100644 solana/keys/pyth_program.json create mode 100644 solana/keys/pyth_publisher.json create mode 100644 third_party/pyth/Dockerfile create mode 100644 third_party/pyth/pyth_publisher.py create mode 100644 third_party/pyth/pyth_utils.py diff --git a/Tiltfile b/Tiltfile index dcc485758..73d9ccb2d 100644 --- a/Tiltfile +++ b/Tiltfile @@ -98,6 +98,15 @@ k8s_resource("guardian", resource_deps = ["proto-gen", "solana-devnet"], port_fo port_forward(7071, name = "Public REST [:7071]"), ]) +docker_build( + ref = "pyth", + context = ".", + dockerfile = "third_party/pyth/Dockerfile" +) +k8s_yaml_with_ns("./devnet/pyth.yaml") + +k8s_resource("pyth", resource_deps = ["solana-devnet"]) + # publicRPC proxy that allows grpc over http1, for local development k8s_yaml_with_ns("./devnet/envoy-proxy.yaml") diff --git a/devnet/pyth.yaml b/devnet/pyth.yaml new file mode 100644 index 000000000..267cbe4c6 --- /dev/null +++ b/devnet/pyth.yaml @@ -0,0 +1,55 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: pyth + labels: + app: pyth +spec: + clusterIP: None + selector: + app: pyth + ports: + - port: 8898 + name: pyth-tx + protocol: TCP +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: pyth +spec: + selector: + matchLabels: + app: pyth + serviceName: pyth + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + app: pyth + spec: + restartPolicy: Always + terminationGracePeriodSeconds: 0 + containers: + - name: pyth-publisher + image: pyth + command: + - python3 + - /opt/pyth/pyth_publisher.py + readinessProbe: + tcpSocket: + port: 2000 + periodSeconds: 1 + failureThreshold: 300 + - name: pyth-tx + image: pyth + command: + - ./pyth_tx + - -r + - solana-devnet + ports: + - containerPort: 8898 + name: pyth-tx + protocol: TCP diff --git a/devnet/solana-devnet.yaml b/devnet/solana-devnet.yaml index a06aab15d..aad2110be 100644 --- a/devnet/solana-devnet.yaml +++ b/devnet/solana-devnet.yaml @@ -50,6 +50,9 @@ spec: - --bpf-program - metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s - /opt/solana/deps/spl_token_metadata.so + - --bpf-program + - gMYYig2utAxVoXnM9UhtTWrt8e7x2SVBZqsWZJeT5Gw # Derived from pyth_program.json + - /opt/solana/deps/pyth_oracle.so - --log ports: - containerPort: 8001 diff --git a/solana/Dockerfile b/solana/Dockerfile index a2ecbbcf4..ba9ecc821 100644 --- a/solana/Dockerfile +++ b/solana/Dockerfile @@ -1,9 +1,18 @@ #syntax=docker/dockerfile:1.2@sha256:e2a8561e419ab1ba6b2fe6cbdf49fd92b95912df1cf7d313c3e2230a333fdbcc FROM docker.io/library/rust:1.49@sha256:a50165ea96983c21832578afb1c8c028674c965bc1ed43b607871b1f362e06a5 -RUN apt-get update && apt-get install -y libssl-dev libudev-dev pkg-config zlib1g-dev llvm clang -RUN rustup component add rustfmt -RUN rustup default nightly-2021-08-01 +RUN apt-get update && \ + apt-get install -y \ + clang \ + libssl-dev \ + libudev-dev \ + llvm \ + pkg-config \ + zlib1g-dev \ + && \ + rm -rf /var/lib/apt/lists/* && \ + rustup component add rustfmt && \ + rustup default nightly-2021-08-01 RUN sh -c "$(curl -sSfL https://release.solana.com/v1.7.8/install)" @@ -14,6 +23,17 @@ RUN cargo init --lib /tmp/decoy-crate && \ cd /tmp/decoy-crate && cargo build-bpf && \ rm -rf /tmp/decoy-crate +# Cache Pyth sources +# This comes soon after mainnet-v2.1 +ENV PYTH_SRC_REV=31e3188bbf52ec1a25f71e4ab969378b27415b0a +ENV PYTH_DIR=/usr/src/pyth/pyth-client + +WORKDIR $PYTH_DIR +ADD https://github.com/pyth-network/pyth-client/archive/$PYTH_SRC_REV.tar.gz . + +# GitHub appends revision to dir in archive +RUN tar -xvf *.tar.gz && rm -rf *.tar.gz && mv pyth-client-$PYTH_SRC_REV pyth-client + # Add bridge contract sources WORKDIR /usr/src/bridge @@ -36,5 +56,10 @@ RUN --mount=type=cache,target=bridge/target \ cp modules/token_bridge/target/deploy/token_bridge.so /opt/solana/deps/token_bridge.so && \ cp modules/token_bridge/token-metadata/spl_token_metadata.so /opt/solana/deps/spl_token_metadata.so +# Build the Pyth Solana program +WORKDIR $PYTH_DIR/pyth-client/program +RUN make SOLANA=~/.local/share/solana/install/active_release/bin OUT_DIR=../target && \ + cp ../target/oracle.so /opt/solana/deps/pyth_oracle.so + ENV RUST_LOG="solana_runtime::system_instruction_processor=trace,solana_runtime::message_processor=trace,solana_bpf_loader=debug,solana_rbpf=debug" ENV RUST_BACKTRACE=1 diff --git a/solana/keys/pyth_program.json b/solana/keys/pyth_program.json new file mode 100644 index 000000000..e46168f63 --- /dev/null +++ b/solana/keys/pyth_program.json @@ -0,0 +1 @@ +[151,156,152,229,131,186,5,254,107,42,234,87,191,209,182,237,170,57,174,150,37,14,5,58,100,237,114,141,46,22,155,104,10,20,225,112,227,95,250,0,102,170,119,34,187,74,144,163,181,123,233,253,191,6,2,70,127,227,138,51,98,209,205,172] \ No newline at end of file diff --git a/solana/keys/pyth_publisher.json b/solana/keys/pyth_publisher.json new file mode 100644 index 000000000..60eb8789f --- /dev/null +++ b/solana/keys/pyth_publisher.json @@ -0,0 +1 @@ +[62,189,176,181,215,49,125,17,130,43,109,83,115,112,151,110,117,239,235,54,205,209,6,255,76,27,210,115,206,166,217,165,250,48,211,191,77,246,195,18,170,246,162,103,141,129,14,143,127,4,243,114,79,112,11,46,90,174,215,2,63,42,134,56] \ No newline at end of file diff --git a/solana/pyth2wormhole/Cargo.lock b/solana/pyth2wormhole/Cargo.lock index d8fa2d76a..313b7ff59 100644 --- a/solana/pyth2wormhole/Cargo.lock +++ b/solana/pyth2wormhole/Cargo.lock @@ -251,9 +251,11 @@ dependencies = [ "borsh", "byteorder", "primitive-types", + "serde", "sha3", "solana-program", "solitaire", + "wasm-bindgen", ] [[package]] diff --git a/third_party/pyth/Dockerfile b/third_party/pyth/Dockerfile new file mode 100644 index 000000000..5daf6afac --- /dev/null +++ b/third_party/pyth/Dockerfile @@ -0,0 +1,44 @@ +# syntax=docker/dockerfile:1.2 +# Wormhole-specific setup for pyth +FROM pythfoundation/pyth-client:devnet-v2.2@sha256:2ce8de6a43b2fafafd15ebdb723c530a0319860dc40c9fdb97149d5aa270fdde + +USER root + +# At the time of this writing, debian is fussy about performing an +# apt-get update. Please add one if repos go stale +RUN apt-get install -y netcat-openbsd python3 && \ + rm -rf /var/lib/apt/lists/* + +ADD solana/keys /opt/solana/keys + +ENV PYTH_KEY_STORE=/home/pyth/.pythd + +# Prepare keys +WORKDIR $PYTH_KEY_STORE + +RUN cp /opt/solana/keys/pyth_publisher.json publish_key_pair.json && \ + cp /opt/solana/keys/pyth_program.json program_key_pair.json && \ + chown pyth:pyth -R . && \ + chmod go-rwx -R . + +# Rebuild Pyth sources from scratch +# This comes soon after mainnet-v2.1 +ENV PYTH_SRC_REV=31e3188bbf52ec1a25f71e4ab969378b27415b0a +ENV PYTH_SRC_ROOT=/home/pyth/pyth-client + +ADD https://github.com/drozdziak1/pyth-client/archive/$PYTH_SRC_REV.tar.gz . +RUN tar -xvf *.tar.gz && \ + rm -rf $PYTH_SRC_ROOT *.tar.gz && \ + mv pyth-client-$PYTH_SRC_REV $PYTH_SRC_ROOT/ + +WORKDIR $PYTH_SRC_ROOT/build + +RUN cmake .. && make + +# Prepare setup script +ADD third_party/pyth/ /opt/pyth/ +RUN chmod a+rx /opt/pyth/*.py +USER pyth + +ENV PYTH=$PYTH_SRC_ROOT/build/pyth +ENV READINESS_PORT=2000 diff --git a/third_party/pyth/pyth_publisher.py b/third_party/pyth/pyth_publisher.py new file mode 100644 index 000000000..1a761dc96 --- /dev/null +++ b/third_party/pyth/pyth_publisher.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 + +from pyth_utils import * + +import random +import sys +import threading +import time + +# Accept connections from readiness probe +def publisher_readiness(): + run_or_die(["nc", "-k", "-l", "-p", READINESS_PORT]) + +# Update the specified price with random values +def publisher_random_update(price_pubkey): + value = random.randrange(1024) + confidence = 1 + pyth_run_or_die("upd_price_val", args=[price_pubkey, str(value), str(confidence), "trading"]) + print("Price updated!") + +# Fund the publisher +sol_run_or_die("airdrop", [str(SOL_AIRDROP_AMT), + "--keypair", PYTH_PUBLISHER_KEYPAIR, + "--commitment", "finalized", + ]) + +# Create a mapping +pyth_run_or_die("init_mapping") + +# Add a product +prod_pubkey = pyth_run_or_die("add_product", capture_output=True).stdout.strip() +print(f"Added product {prod_pubkey}") + +# Add a price +price_pubkey = pyth_run_or_die( + "add_price", + args=[prod_pubkey, "price"], + confirm=False, + capture_output=True +).stdout.strip() + +print(f"Added price {price_pubkey}") + +publisher_pubkey = sol_run_or_die("address", args=["--keypair", PYTH_PUBLISHER_KEYPAIR], capture_output=True).stdout.strip() + +# Become a publisher +pyth_run_or_die("add_publisher", args=[publisher_pubkey, price_pubkey], confirm=False, debug=True, capture_output=True) +print(f"Added publisher {publisher_pubkey}") + +# Update the price as the newly added publisher +publisher_random_update(price_pubkey) + +print(f"Updated price {price_pubkey}. Mock updates ready to roll. Updating every {str(PYTH_PUBLISHER_INTERVAL)} seconds") + +# Spin off the readiness probe endpoint into a separate thread +readiness_thread = threading.Thread(target=publisher_readiness) + +readiness_thread.start() + +while True: + print(f"Updating price {price_pubkey}") + publisher_random_update(price_pubkey) + time.sleep(PYTH_PUBLISHER_INTERVAL) + sys.stdout.flush() + +readiness_thread.join() diff --git a/third_party/pyth/pyth_utils.py b/third_party/pyth/pyth_utils.py new file mode 100644 index 000000000..bc82d6f4b --- /dev/null +++ b/third_party/pyth/pyth_utils.py @@ -0,0 +1,59 @@ +import os +import sys +import subprocess + +PYTH=os.environ.get("PYTH", "./pyth") +PYTH_KEY_STORE = os.environ.get("PYTH_KEY_STORE", "/home/pyth/.pythd") +PYTH_PROGRAM_KEYPAIR = f"{PYTH_KEY_STORE}/program_key_pair.json" +PYTH_PROGRAM_SO_PATH=os.environ.get("PYTH_PROGRAM_SO", "../target/oracle.so") +PYTH_PUBLISHER_KEYPAIR = f"{PYTH_KEY_STORE}/publish_key_pair.json" +PYTH_PUBLISHER_INTERVAL = float(os.environ.get("PYTH_PUBLISHER_INTERVAL", "5")) + +SOL_AIRDROP_AMT = 100 +SOL_RPC_HOST = "solana-devnet" +SOL_RPC_PORT = 8899 +SOL_RPC_URL = f"http://{SOL_RPC_HOST}:{str(SOL_RPC_PORT)}" + +READINESS_PORT=os.environ.get("READINESS_PORT", "2000") + +# pretend we're set -e +def run_or_die(args, die=True, **kwargs): + args_readable = ' '.join(args) + print(f"CMD RUN\t{args_readable}", file=sys.stderr) + sys.stderr.flush() + ret = subprocess.run(args, text=True, **kwargs) + + if ret.returncode is not 0: + print(f"CMD FAIL {ret.returncode}\t{args_readable}", file=sys.stderr) + + out = ret.stdout if ret.stdout is not None else "" + err = ret.stderr if ret.stderr is not None else "" + + print(f"CMD STDOUT\n{out}", file=sys.stderr) + print(f"CMD STDERR\n{err}", file=sys.stderr) + + if die: + sys.exit(ret.returncode) + else: + print(f"CMD DIE FALSE", file=sys.stderr) + + else: + print(f"CMD OK\t{args_readable}", file=sys.stderr) + sys.stderr.flush() + return ret + +# Pyth boilerplate in front of run_or_die +def pyth_run_or_die(subcommand, args=[], debug=False, confirm=True, **kwargs): + return run_or_die([PYTH, subcommand] + + args + + (["-d"] if debug else []) + + ([] if confirm else ["-n"]) # Note: not all pyth subcommands accept -n + + ["-k", PYTH_KEY_STORE] + + ["-r", SOL_RPC_HOST] + + ["-c", "finalized"], **kwargs) + +# Solana boilerplate in front of run_or_die +def sol_run_or_die(subcommand, args=[], **kwargs): + return run_or_die(["solana", subcommand] + + args + + ["--url", SOL_RPC_URL], **kwargs)