From 5b61f3d949ababe8e4fa8e9dfc7c69903875ca7f Mon Sep 17 00:00:00 2001 From: Riordan Panayides Date: Wed, 17 May 2023 17:11:17 +0100 Subject: [PATCH] wip: many changes for deployment - split out postgres config and read from env - make ssl optional - revert to separate dockerfiles - give tokio enough workers - allow custom server bind address - fix warnings - cargo fmt --- Cargo.lock | 224 +++++++++++++++++++++++++++---------- Cargo.toml | 6 +- Dockerfile | 24 ---- Dockerfile.server | 4 +- Dockerfile.worker | 10 +- cd/server.toml | 19 ++++ cd/worker.toml | 9 ++ src/backfill/main.rs | 118 +++++++++---------- src/database/initialize.rs | 40 +++---- src/database/insert.rs | 2 - src/server/coingecko.rs | 2 +- src/server/main.rs | 18 +-- src/utils/mod.rs | 23 +++- src/worker/main.rs | 17 +-- 14 files changed, 315 insertions(+), 201 deletions(-) delete mode 100644 Dockerfile create mode 100644 cd/server.toml create mode 100644 cd/worker.toml diff --git a/Cargo.lock b/Cargo.lock index 3afaf23..99f3efb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -73,7 +73,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "465a6172cf69b960917811022d8f29bc0b7fa1398bc4f78b3c466673db1213b6" dependencies = [ "quote 1.0.26", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -188,7 +188,7 @@ dependencies = [ "actix-router", "proc-macro2 1.0.56", "quote 1.0.26", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -291,7 +291,7 @@ dependencies = [ "proc-macro2 1.0.56", "quote 1.0.26", "regex", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -306,7 +306,7 @@ dependencies = [ "proc-macro2 1.0.56", "quote 1.0.26", "rustversion", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -317,7 +317,7 @@ checksum = "e1be64a48e395fe00b8217287f226078be2cf32dae42fdf8a885b997945c3d28" dependencies = [ "anchor-syn", "proc-macro2 1.0.56", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -329,7 +329,7 @@ dependencies = [ "anchor-syn", "proc-macro2 1.0.56", "quote 1.0.26", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -342,7 +342,7 @@ dependencies = [ "anyhow", "proc-macro2 1.0.56", "quote 1.0.26", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -356,7 +356,7 @@ dependencies = [ "heck 0.3.3", "proc-macro2 1.0.56", "quote 1.0.26", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -369,7 +369,7 @@ dependencies = [ "anyhow", "proc-macro2 1.0.56", "quote 1.0.26", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -382,7 +382,7 @@ dependencies = [ "anyhow", "proc-macro2 1.0.56", "quote 1.0.26", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -412,7 +412,7 @@ dependencies = [ "anyhow", "proc-macro2 1.0.56", "quote 1.0.26", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -454,7 +454,7 @@ dependencies = [ "serde", "serde_json", "sha2 0.9.9", - "syn 1.0.107", + "syn 1.0.109", "thiserror", ] @@ -530,7 +530,7 @@ checksum = "726535892e8eae7e70657b4c8ea93d26b8553afb1ce617caee529ef96d7dee6c" dependencies = [ "proc-macro2 1.0.56", "quote 1.0.26", - "syn 1.0.107", + "syn 1.0.109", "synstructure", ] @@ -542,7 +542,7 @@ checksum = "2777730b2039ac0f95f093556e61b6d26cebed5393ca6f152717777cec3a42ed" dependencies = [ "proc-macro2 1.0.56", "quote 1.0.26", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -592,7 +592,7 @@ checksum = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27" dependencies = [ "proc-macro2 1.0.56", "quote 1.0.26", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -603,7 +603,7 @@ checksum = "705339e0e4a9690e2908d2b3d049d85682cf19fbd5782494498fbf7003a6a282" dependencies = [ "proc-macro2 1.0.56", "quote 1.0.26", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -846,7 +846,7 @@ dependencies = [ "borsh-schema-derive-internal", "proc-macro-crate 0.1.5", "proc-macro2 1.0.56", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -857,7 +857,7 @@ checksum = "5449c28a7b352f2d1e592a8a28bf139bc71afb0764a14f3c02500935d8c44065" dependencies = [ "proc-macro2 1.0.56", "quote 1.0.26", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -868,7 +868,7 @@ checksum = "cdbd5696d8bfa21d53d9fe39a714a18538bad11492a42d066dbbc395fb1951c0" dependencies = [ "proc-macro2 1.0.56", "quote 1.0.26", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -947,7 +947,7 @@ checksum = "1aca418a974d83d40a0c1f0c5cba6ff4bc28d8df099109ca459a2118d40b6322" dependencies = [ "proc-macro2 1.0.56", "quote 1.0.26", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -1169,6 +1169,25 @@ dependencies = [ "unreachable", ] +[[package]] +name = "config" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d379af7f68bfc21714c6c7dea883544201741d2ce8274bb12fa54f89507f52a7" +dependencies = [ + "async-trait", + "json5", + "lazy_static", + "nom", + "pathdiff", + "ron", + "rust-ini", + "serde", + "serde_json", + "toml", + "yaml-rust", +] + [[package]] name = "console" version = "0.15.5" @@ -1404,7 +1423,7 @@ dependencies = [ "proc-macro2 1.0.56", "quote 1.0.26", "scratch", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -1421,7 +1440,7 @@ checksum = "65e07508b90551e610910fa648a1878991d367064997a596135b86df30daf07e" dependencies = [ "proc-macro2 1.0.56", "quote 1.0.26", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -1451,6 +1470,7 @@ dependencies = [ "deadpool-runtime", "num_cpus", "retain_mut", + "serde", "tokio", ] @@ -1462,6 +1482,7 @@ checksum = "836a24a9d49deefe610b8b60c767a7412e9a931d79a89415cd2d2d71630ca8d7" dependencies = [ "deadpool", "log 0.4.17", + "serde", "tokio", "tokio-postgres", ] @@ -1525,7 +1546,7 @@ dependencies = [ "proc-macro2 1.0.56", "quote 1.0.26", "rustc_version 0.4.0", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -1598,7 +1619,7 @@ checksum = "3bf95dc3f046b9da4f2d51833c0d3547d8564ef6910f5c1ed130306a75b92886" dependencies = [ "proc-macro2 1.0.56", "quote 1.0.26", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -1624,6 +1645,12 @@ dependencies = [ "syn 0.15.44", ] +[[package]] +name = "dlv-list" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0688c2a7f92e427f44895cd63841bff7b29f8d7a1648b9e7e07a4a365b2e1257" + [[package]] name = "dotenv" version = "0.15.0" @@ -1709,7 +1736,7 @@ checksum = "8958699f9359f0b04e691a13850d48b7de329138023876d07cbd024c2c820598" dependencies = [ "proc-macro2 1.0.56", "quote 1.0.26", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -1721,7 +1748,7 @@ dependencies = [ "once_cell", "proc-macro2 1.0.56", "quote 1.0.26", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -1741,7 +1768,7 @@ checksum = "946ee94e3dbf58fdd324f9ce245c7b238d46a66f00e86a020b71996349e46cce" dependencies = [ "proc-macro2 1.0.56", "quote 1.0.26", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -1986,7 +2013,7 @@ checksum = "3eb14ed937631bd8b8b8977f2c198443447a8355b6e3ca599f38c975e5a963b6" dependencies = [ "proc-macro2 1.0.56", "quote 1.0.26", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -2600,6 +2627,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "json5" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96b0db21af676c1ce64250b5f40f3ce2cf27e4e47cb91ed91eb6fe9350b430c1" +dependencies = [ + "pest", + "pest_derive", + "serde", +] + [[package]] name = "jsonrpc-client-transports" version = "18.0.0" @@ -2653,7 +2691,7 @@ dependencies = [ "proc-macro-crate 0.1.5", "proc-macro2 1.0.56", "quote 1.0.26", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -3133,7 +3171,7 @@ checksum = "5a7d5f7076603ebc68de2dc6a650ec331a062a13abaa346975be747bbfa4b789" dependencies = [ "proc-macro2 1.0.56", "quote 1.0.26", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -3256,7 +3294,7 @@ checksum = "876a53fff98e03a936a674b29568b0e605f06b29372c2489ff4de23f1949743d" dependencies = [ "proc-macro2 1.0.56", "quote 1.0.26", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -3338,7 +3376,7 @@ dependencies = [ "proc-macro-crate 1.2.1", "proc-macro2 1.0.56", "quote 1.0.26", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -3393,6 +3431,7 @@ dependencies = [ "borsh", "bytemuck", "chrono", + "config", "deadpool-postgres", "derive_more", "dotenv", @@ -3443,7 +3482,7 @@ checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" dependencies = [ "proc-macro2 1.0.56", "quote 1.0.26", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -3475,6 +3514,16 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "ordered-multimap" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccd746e37177e1711c20dd619a1620f34f5c8b569c53590a72dedd5344d8924a" +dependencies = [ + "dlv-list", + "hashbrown 0.12.3", +] + [[package]] name = "os_str_bytes" version = "6.4.1" @@ -3501,7 +3550,7 @@ dependencies = [ "proc-macro-error", "proc-macro2 1.0.56", "quote 1.0.26", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -3584,6 +3633,12 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d01a5bd0424d00070b0098dd17ebca6f961a959dead1dbcbbbc1d1cd8d3deeba" +[[package]] +name = "pathdiff" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd" + [[package]] name = "pbkdf2" version = "0.4.0" @@ -3640,14 +3695,48 @@ dependencies = [ [[package]] name = "pest" -version = "2.5.3" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4257b4a04d91f7e9e6290be5d3da4804dd5784fafde3a497d73eb2b4a158c30a" +checksum = "e68e84bfb01f0507134eac1e9b410a12ba379d064eab48c50ba4ce329a527b70" dependencies = [ "thiserror", "ucd-trie", ] +[[package]] +name = "pest_derive" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b79d4c71c865a25a4322296122e3924d30bc8ee0834c8bfc8b95f7f054afbfb" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c435bf1076437b851ebc8edc3a18442796b30f1728ffea6262d59bbe28b077e" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2 1.0.56", + "quote 1.0.26", + "syn 2.0.15", +] + +[[package]] +name = "pest_meta" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "745a452f8eb71e39ffd8ee32b3c5f51d03845f99786fa9b68db6ff509c505411" +dependencies = [ + "once_cell", + "pest", + "sha2 0.10.6", +] + [[package]] name = "petgraph" version = "0.6.2" @@ -3693,7 +3782,7 @@ checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" dependencies = [ "proc-macro2 1.0.56", "quote 1.0.26", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -3799,7 +3888,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e97e3215779627f01ee256d2fad52f3d95e8e1c11e9fc6fd08f7cd455d5d5c78" dependencies = [ "proc-macro2 1.0.56", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -3831,7 +3920,7 @@ dependencies = [ "proc-macro-error-attr", "proc-macro2 1.0.56", "quote 1.0.26", - "syn 1.0.107", + "syn 1.0.109", "version_check 0.9.4", ] @@ -3872,7 +3961,7 @@ checksum = "4bf29726d67464d49fa6224a1d07936a8c08bb3fba727c7493f6cf1616fdaada" dependencies = [ "proc-macro2 1.0.56", "quote 1.0.26", - "syn 1.0.107", + "syn 1.0.109", "version_check 0.9.4", "yansi", ] @@ -3904,7 +3993,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 1.0.107", + "syn 1.0.109", "tempfile", "which", ] @@ -3919,7 +4008,7 @@ dependencies = [ "itertools", "proc-macro2 1.0.56", "quote 1.0.26", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -4392,6 +4481,17 @@ dependencies = [ "librocksdb-sys", ] +[[package]] +name = "ron" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88073939a61e5b7680558e6be56b419e208420c2adb92be54921fa6b72283f1a" +dependencies = [ + "base64 0.13.1", + "bitflags", + "serde", +] + [[package]] name = "rpassword" version = "6.0.1" @@ -4404,6 +4504,16 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "rust-ini" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6d5f2436026b4f6e79dc829837d467cc7e9a55ee40e750d716713540715a2df" +dependencies = [ + "cfg-if 1.0.0", + "ordered-multimap", +] + [[package]] name = "rustc-demangle" version = "0.1.21" @@ -4579,7 +4689,7 @@ checksum = "bdbda6ac5cd1321e724fa9cee216f3a61885889b896f073b8f82322789c5250e" dependencies = [ "proc-macro2 1.0.56", "quote 1.0.26", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -4680,7 +4790,7 @@ checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" dependencies = [ "proc-macro2 1.0.56", "quote 1.0.26", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -5255,7 +5365,7 @@ dependencies = [ "proc-macro2 1.0.56", "quote 1.0.26", "rustc_version 0.4.0", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -5757,7 +5867,7 @@ dependencies = [ "proc-macro2 1.0.56", "quote 1.0.26", "rustversion", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -6169,7 +6279,7 @@ dependencies = [ "proc-macro2 1.0.56", "quote 1.0.26", "rustversion", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -6197,9 +6307,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.107" +version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ "proc-macro2 1.0.56", "quote 1.0.26", @@ -6231,7 +6341,7 @@ checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" dependencies = [ "proc-macro2 1.0.56", "quote 1.0.26", - "syn 1.0.107", + "syn 1.0.109", "unicode-xid 0.2.4", ] @@ -6314,7 +6424,7 @@ checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" dependencies = [ "proc-macro2 1.0.56", "quote 1.0.26", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -6459,7 +6569,7 @@ checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" dependencies = [ "proc-macro2 1.0.56", "quote 1.0.26", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -6670,7 +6780,7 @@ dependencies = [ "proc-macro2 1.0.56", "prost-build", "quote 1.0.26", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -6745,7 +6855,7 @@ checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" dependencies = [ "proc-macro2 1.0.56", "quote 1.0.26", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -7069,7 +7179,7 @@ dependencies = [ "once_cell", "proc-macro2 1.0.56", "quote 1.0.26", - "syn 1.0.107", + "syn 1.0.109", "wasm-bindgen-shared", ] @@ -7103,7 +7213,7 @@ checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" dependencies = [ "proc-macro2 1.0.56", "quote 1.0.26", - "syn 1.0.107", + "syn 1.0.109", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -7416,7 +7526,7 @@ checksum = "44bf07cb3e50ea2003396695d58bf46bc9887a1f362260446fad6bc4e79bd36c" dependencies = [ "proc-macro2 1.0.56", "quote 1.0.26", - "syn 1.0.107", + "syn 1.0.109", "synstructure", ] diff --git a/Cargo.toml b/Cargo.toml index 4f3fa56..50a543e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,7 +26,7 @@ futures = "0.3.27" jsonrpc-core-client = { version = "18.0.0", features = ["ws", "http"] } -deadpool-postgres = { version = "0.10.5", features = [ "rt_tokio_1" ] } +deadpool-postgres = { version = "0.10.5", features = [ "rt_tokio_1", "serde" ] } tokio-postgres = { version = "0.7", features = ["with-chrono-0_4"] } postgres-native-tls = "0.5.0" native-tls = "0.2.11" @@ -61,4 +61,6 @@ actix-web = "4" arrayref = "0.3.6" bytemuck = "1.12.3" -num_enum = "0.6.1" \ No newline at end of file +num_enum = "0.6.1" + +config = "0.13.1" \ No newline at end of file diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 7e863e7..0000000 --- a/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -FROM lukemathwalker/cargo-chef:latest-rust-1.67.1-slim AS chef - -FROM chef AS planner -COPY . . -RUN cargo chef prepare --recipe-path recipe.json - -FROM chef AS builder -COPY --from=planner recipe.json recipe.json -RUN apt-get update && apt-get install -y libudev-dev clang pkg-config libssl-dev build-essential cmake -RUN rustup component add rustfmt && update-ca-certificates -RUN cargo chef cook --release --recipe-path recipe.json -# Build application -COPY . . -RUN cargo build --release --bins - -FROM debian:bullseye-slim as base_image -RUN apt-get update && apt-get -y install ca-certificates libssl1.1 - -# We do not need the Rust toolchain to run the binary! -FROM base_image AS runtime -COPY --from=builder /target/release/server /usr/local/bin -COPY --from=builder /target/release/worker /usr/local/bin -COPY --from=builder markets.json . -COPY --from=builder ca.cer . diff --git a/Dockerfile.server b/Dockerfile.server index e313d38..95dbae7 100644 --- a/Dockerfile.server +++ b/Dockerfile.server @@ -19,4 +19,6 @@ RUN apt-get update && apt-get -y install ca-certificates libssl1.1 # We do not need the Rust toolchain to run the binary! FROM base_image AS runtime COPY --from=builder /target/release/server /usr/local/bin -ENTRYPOINT ["/usr/local/bin/server"] \ No newline at end of file +COPY --from=builder markets.json . +COPY --from=builder ca.cer . +COPY --from=builder client.pks . \ No newline at end of file diff --git a/Dockerfile.worker b/Dockerfile.worker index 4922bdf..98899cd 100644 --- a/Dockerfile.worker +++ b/Dockerfile.worker @@ -2,13 +2,13 @@ FROM lukemathwalker/cargo-chef:latest-rust-1.67.1-slim AS chef FROM chef AS planner COPY . . -RUN cargo chef prepare --recipe-path worker-recipe.json +RUN cargo chef prepare --recipe-path recipe.json FROM chef AS builder -COPY --from=planner worker-recipe.json worker-recipe.json +COPY --from=planner recipe.json recipe.json RUN apt-get update && apt-get install -y libudev-dev clang pkg-config libssl-dev build-essential cmake RUN rustup component add rustfmt && update-ca-certificates -RUN cargo chef cook --release --recipe-path worker-recipe.json +RUN cargo chef cook --release --recipe-path recipe.json # Build application COPY . . RUN cargo build --release --bin worker @@ -19,4 +19,6 @@ RUN apt-get update && apt-get -y install ca-certificates libssl1.1 # We do not need the Rust toolchain to run the binary! FROM base_image AS runtime COPY --from=builder /target/release/worker /usr/local/bin -ENTRYPOINT ["/usr/local/bin/worker"] \ No newline at end of file +COPY --from=builder markets.json . +COPY --from=builder ca.cer . +COPY --from=builder client.pks . \ No newline at end of file diff --git a/cd/server.toml b/cd/server.toml new file mode 100644 index 0000000..a4b1bb1 --- /dev/null +++ b/cd/server.toml @@ -0,0 +1,19 @@ +app = "openbook-candles" +kill_signal = "SIGTERM" +kill_timeout = 30 + +[build] + dockerfile = "../Dockerfile.server" + +[experimental] + cmd = ["server", "markets.json"] + +[[services]] + internal_port = 8080 + processes = ["app"] + protocol = "tcp" + + [services.concurrency] + hard_limit = 1024 + soft_limit = 1024 + type = "connections" diff --git a/cd/worker.toml b/cd/worker.toml new file mode 100644 index 0000000..8f6da45 --- /dev/null +++ b/cd/worker.toml @@ -0,0 +1,9 @@ +app = "openbook-candles-worker" +kill_signal = "SIGTERM" +kill_timeout = 30 + +[build] + dockerfile = "../Dockerfile.worker" + +[experimental] + cmd = ["worker", "markets.json"] diff --git a/src/backfill/main.rs b/src/backfill/main.rs index 663c708..f772ff2 100644 --- a/src/backfill/main.rs +++ b/src/backfill/main.rs @@ -1,36 +1,35 @@ -use std::{collections::HashMap, str::FromStr, env}; use anchor_lang::prelude::Pubkey; -use chrono::{NaiveDateTime, DateTime, Utc, Duration}; +use chrono::{DateTime, Duration, NaiveDateTime, Utc}; use futures::future::join_all; -use openbook_candles::{structs::{openbook::OpenBookFillEventLog, markets::{load_markets, fetch_market_infos}}, worker::trade_fetching::{scrape::scrape_transactions, parsing::parse_trades_from_openbook_txns}, database::{initialize::connect_to_database, insert::persist_fill_events}, utils::Config}; -use solana_client::{rpc_config::RpcTransactionConfig, nonblocking::rpc_client::RpcClient, rpc_client::GetConfirmedSignaturesForAddress2Config, rpc_response::RpcConfirmedTransactionStatusWithSignature}; +use openbook_candles::{ + database::{initialize::connect_to_database, insert::persist_fill_events}, + structs::{ + markets::{fetch_market_infos, load_markets}, + openbook::OpenBookFillEventLog, + }, + utils::Config, + worker::trade_fetching::parsing::parse_trades_from_openbook_txns, +}; +use solana_client::{ + nonblocking::rpc_client::RpcClient, rpc_client::GetConfirmedSignaturesForAddress2Config, + rpc_config::RpcTransactionConfig, rpc_response::RpcConfirmedTransactionStatusWithSignature, +}; use solana_sdk::{commitment_config::CommitmentConfig, signature::Signature}; use solana_transaction_status::UiTransactionEncoding; -use tokio::sync::mpsc::{Sender, self}; +use std::{collections::HashMap, env, str::FromStr}; +use tokio::sync::mpsc::{self, Sender}; #[tokio::main] -async fn main() -> anyhow::Result<()> { +async fn main() -> anyhow::Result<()> { dotenv::dotenv().ok(); let args: Vec = env::args().collect(); assert!(args.len() == 2); - + let path_to_markets_json = &args[1]; let rpc_url: String = dotenv::var("RPC_URL").unwrap(); - let database_url: String = dotenv::var("DATABASE_URL").unwrap(); - let ca_cert_path: String = dotenv::var("CA_CERT_PATH").unwrap(); - let client_key_path: String = dotenv::var("CLIENT_KEY_PATH").unwrap(); - let max_pg_pool_connections: usize = dotenv::var("MAX_PG_POOL_CONNS_WORKER") - .unwrap() - .parse::() - .unwrap(); let config = Config { rpc_url: rpc_url.clone(), - database_url, - max_pg_pool_connections, - use_ssl: true, - ca_cert_path, - client_key_path, }; let markets = load_markets(&path_to_markets_json); let market_infos = fetch_market_infos(&config, markets.clone()).await?; @@ -40,10 +39,10 @@ async fn main() -> anyhow::Result<()> { } println!("{:?}", target_markets); - let pool = connect_to_database(&config).await?; + let pool = connect_to_database().await?; let (fill_sender, mut fill_receiver) = mpsc::channel::(1000); - tokio::spawn(async move { + tokio::spawn(async move { loop { persist_fill_events(&pool, &mut fill_receiver) .await @@ -60,7 +59,6 @@ pub async fn backfill( fill_sender: &Sender, target_markets: &HashMap, ) -> anyhow::Result<()> { - println!("backfill started"); let mut before_sig: Option = None; let mut now_time = Utc::now().timestamp(); @@ -69,7 +67,8 @@ pub async fn backfill( let mut handles = vec![]; while now_time > end_time { - let rpc_client = RpcClient::new_with_commitment(rpc_url.clone(), CommitmentConfig::confirmed()); + let rpc_client = + RpcClient::new_with_commitment(rpc_url.clone(), CommitmentConfig::confirmed()); let maybe_r = get_signatures(&rpc_client, before_sig).await; match maybe_r { @@ -90,10 +89,10 @@ pub async fn backfill( get_transactions(&rpc_client, sigs, &cloned_sender, &cloned_markets).await; }); handles.push(handle); - }, - None => {}, + } + None => {} } - }; + } futures::future::join_all(handles).await; @@ -101,39 +100,46 @@ pub async fn backfill( Ok(()) } +pub async fn get_signatures( + rpc_client: &RpcClient, + before_sig: Option, +) -> Option<( + Signature, + i64, + Vec, +)> { + let rpc_config = GetConfirmedSignaturesForAddress2Config { + before: before_sig, + until: None, + limit: None, + commitment: Some(CommitmentConfig::confirmed()), + }; -pub async fn get_signatures(rpc_client: &RpcClient, - before_sig: Option) -> Option<(Signature, i64, Vec)> { - let rpc_config = GetConfirmedSignaturesForAddress2Config { - before: before_sig, - until: None, - limit: None, - commitment: Some(CommitmentConfig::confirmed()), - }; - - let sigs = match rpc_client - .get_signatures_for_address_with_config( - &Pubkey::from_str("srmqPvymJeFKQ4zGQed1GFppgkRHL9kaELCbyksJtPX").unwrap(), - rpc_config, - ) - .await - { - Ok(s) => s, - Err(e) => { - println!("Error in get_signatures_for_address_with_config: {}", e); - return None; - } - }; - - if sigs.len() == 0 { - println!("No signatures found"); + let sigs = match rpc_client + .get_signatures_for_address_with_config( + &Pubkey::from_str("srmqPvymJeFKQ4zGQed1GFppgkRHL9kaELCbyksJtPX").unwrap(), + rpc_config, + ) + .await + { + Ok(s) => s, + Err(e) => { + println!("Error in get_signatures_for_address_with_config: {}", e); return None; } - let last = sigs.last().unwrap(); - return Some((Signature::from_str(&last.signature).unwrap(), last.block_time.unwrap(), sigs)); + }; + + if sigs.len() == 0 { + println!("No signatures found"); + return None; } - - + let last = sigs.last().unwrap(); + return Some(( + Signature::from_str(&last.signature).unwrap(), + last.block_time.unwrap(), + sigs, + )); +} pub async fn get_transactions( rpc_client: &RpcClient, @@ -180,4 +186,4 @@ fn backfill_time_left(current_time: i64, backfill_end: i64) -> Duration { let cur_date = DateTime::::from_utc(naive_cur, Utc); let bf_date = DateTime::::from_utc(naive_bf, Utc); cur_date - bf_date -} \ No newline at end of file +} diff --git a/src/database/initialize.rs b/src/database/initialize.rs index 30ec312..65ab604 100644 --- a/src/database/initialize.rs +++ b/src/database/initialize.rs @@ -1,28 +1,21 @@ use std::{fs, time::Duration}; use deadpool_postgres::{ - Config as PgConfig, ManagerConfig, Pool, PoolConfig, RecyclingMethod, Runtime, SslMode, - Timeouts, + ManagerConfig, Pool, PoolConfig, RecyclingMethod, Runtime, SslMode, Timeouts, }; use native_tls::{Certificate, Identity, TlsConnector}; use postgres_native_tls::MakeTlsConnector; -use crate::utils::Config; +use crate::utils::PgConfig; -pub async fn connect_to_database(config: &Config) -> anyhow::Result { - let mut x = PgConfig::new(); +pub async fn connect_to_database() -> anyhow::Result { + let mut pg_config = PgConfig::from_env()?; - // TODO: fix - x.host = Some("".to_owned()); - x.user = Some("".to_owned()); - x.password = Some("".to_owned()); - x.dbname = Some("postgres".to_owned()); - - x.manager = Some(ManagerConfig { + pg_config.pg.manager = Some(ManagerConfig { recycling_method: RecyclingMethod::Fast, }); - x.pool = Some(PoolConfig { - max_size: config.max_pg_pool_connections, + pg_config.pg.pool = Some(PoolConfig { + max_size: pg_config.pg_max_pool_connections, timeouts: Timeouts::default(), }); @@ -30,10 +23,16 @@ pub async fn connect_to_database(config: &Config) -> anyhow::Result { // base64 -i ca.cer -o ca.cer.b64 && base64 -i client.pks -o client.pks.b64 // fly secrets set PG_CA_CERT=- < ./ca.cer.b64 -a mango-fills // fly secrets set PG_CLIENT_KEY=- < ./client.pks.b64 -a mango-fills - let tls = if config.use_ssl { - x.ssl_mode = Some(SslMode::Require); - let ca_cert = fs::read(&config.ca_cert_path).expect("reading client cert from file"); - let client_key = fs::read(&config.client_key_path).expect("reading client key from file"); + let tls = if pg_config.pg_use_ssl { + pg_config.pg.ssl_mode = Some(SslMode::Require); + let ca_cert = fs::read(&pg_config.pg_ca_cert_path.expect("reading ca cert from env")) + .expect("reading ca cert from file"); + let client_key = fs::read( + &pg_config + .pg_client_key_path + .expect("reading client key from env"), + ) + .expect("reading client key from file"); MakeTlsConnector::new( TlsConnector::builder() .add_root_certificate(Certificate::from_pem(&ca_cert)?) @@ -50,7 +49,10 @@ pub async fn connect_to_database(config: &Config) -> anyhow::Result { ) }; - let pool = x.create_pool(Some(Runtime::Tokio1), tls).unwrap(); + let pool = pg_config + .pg + .create_pool(Some(Runtime::Tokio1), tls) + .unwrap(); match pool.get().await { Ok(_) => println!("Database connected"), Err(e) => { diff --git a/src/database/insert.rs b/src/database/insert.rs index 410c5b2..414c35b 100644 --- a/src/database/insert.rs +++ b/src/database/insert.rs @@ -57,7 +57,6 @@ pub async fn persist_fill_events( // } } } - Ok(()) } pub async fn persist_candles( @@ -93,7 +92,6 @@ pub async fn persist_candles( // } // }; } - Ok(()) } fn build_fills_upsert_statement(events: HashMap) -> String { diff --git a/src/server/coingecko.rs b/src/server/coingecko.rs index c381db8..12149da 100644 --- a/src/server/coingecko.rs +++ b/src/server/coingecko.rs @@ -10,7 +10,7 @@ use openbook_candles::{ CoinGecko24HourVolume, CoinGeckoOrderBook, CoinGeckoPair, CoinGeckoTicker, PgCoinGecko24HighLow, }, - slab::{get_best_bids_and_asks, get_orderbooks_with_depth}, + slab::get_orderbooks_with_depth, }, utils::WebContext, }; diff --git a/src/server/main.rs b/src/server/main.rs index 8c563d9..3e71738 100644 --- a/src/server/main.rs +++ b/src/server/main.rs @@ -29,27 +29,15 @@ async fn main() -> std::io::Result<()> { assert!(args.len() == 2); let path_to_markets_json = &args[1]; let rpc_url: String = dotenv::var("RPC_URL").unwrap(); - let database_url: String = dotenv::var("DATABASE_URL").unwrap(); - let use_ssl: bool = dotenv::var("USE_SSL").unwrap().parse::().unwrap(); - let ca_cert_path: String = dotenv::var("CA_CERT_PATH").unwrap(); - let client_key_path: String = dotenv::var("CLIENT_KEY_PATH").unwrap(); - let max_pg_pool_connections: usize = dotenv::var("MAX_PG_POOL_CONNS_WORKER") - .unwrap() - .parse::() - .unwrap(); + let bind_addr: String = dotenv::var("SERVER_BIND_ADDR").expect("reading bind addr from env"); let config = Config { rpc_url: rpc_url.clone(), - database_url, - max_pg_pool_connections, - use_ssl, - ca_cert_path, - client_key_path, }; let markets = load_markets(path_to_markets_json); let market_infos = fetch_market_infos(&config, markets).await.unwrap(); - let pool = connect_to_database(&config).await.unwrap(); + let pool = connect_to_database().await.unwrap(); let context = Data::new(WebContext { rpc_url, @@ -71,7 +59,7 @@ async fn main() -> std::io::Result<()> { .service(coingecko::service()), ) }) - .bind(("127.0.0.1", 8080))? + .bind(&bind_addr)? .run() .await } diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 333cfcc..f71bce3 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -16,14 +16,27 @@ impl AnyhowWrap for Result { } } +#[derive(Debug, serde::Deserialize)] +pub struct PgConfig { + pub pg: deadpool_postgres::Config, + pub pg_max_pool_connections: usize, + pub pg_use_ssl: bool, + pub pg_ca_cert_path: Option, + pub pg_client_key_path: Option, +} + +impl PgConfig { + pub fn from_env() -> Result { + config::Config::builder() + .add_source(config::Environment::default().separator("_")) + .build()? + .try_deserialize() + } +} + #[derive(Clone, Debug, Deserialize)] pub struct Config { pub rpc_url: String, - pub database_url: String, - pub max_pg_pool_connections: usize, - pub use_ssl: bool, - pub ca_cert_path: String, - pub client_key_path: String, } pub struct WebContext { diff --git a/src/worker/main.rs b/src/worker/main.rs index 22efbfd..e8c774f 100644 --- a/src/worker/main.rs +++ b/src/worker/main.rs @@ -16,7 +16,7 @@ use std::env; use std::{collections::HashMap, str::FromStr}; use tokio::sync::mpsc; -#[tokio::main] +#[tokio::main(flavor = "multi_thread", worker_threads = 10)] async fn main() -> anyhow::Result<()> { dotenv::dotenv().ok(); @@ -24,22 +24,9 @@ async fn main() -> anyhow::Result<()> { assert!(args.len() == 2); let path_to_markets_json = &args[1]; let rpc_url: String = dotenv::var("RPC_URL").unwrap(); - let database_url: String = dotenv::var("DATABASE_URL").unwrap(); - let use_ssl: bool = dotenv::var("USE_SSL").unwrap().parse::().unwrap(); - let ca_cert_path: String = dotenv::var("CA_CERT_PATH").unwrap(); - let client_key_path: String = dotenv::var("CLIENT_KEY_PATH").unwrap(); - let max_pg_pool_connections: usize = dotenv::var("MAX_PG_POOL_CONNS_WORKER") - .unwrap() - .parse::() - .unwrap(); let config = Config { rpc_url: rpc_url.clone(), - database_url, - max_pg_pool_connections, - use_ssl, - ca_cert_path, - client_key_path, }; let markets = load_markets(&path_to_markets_json); @@ -50,7 +37,7 @@ async fn main() -> anyhow::Result<()> { } println!("{:?}", target_markets); - let pool = connect_to_database(&config).await?; + let pool = connect_to_database().await?; setup_database(&pool).await?; let mut handles = vec![];