wip: many changes for deployment

- split out postgres config and read from env
- make ssl optional
- revert to separate dockerfiles
- give tokio enough workers
- allow custom server bind address
- fix warnings
- cargo fmt
This commit is contained in:
Riordan Panayides 2023-05-17 17:11:17 +01:00
parent 697a20d9ff
commit 5b61f3d949
14 changed files with 315 additions and 201 deletions

224
Cargo.lock generated
View File

@ -73,7 +73,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "465a6172cf69b960917811022d8f29bc0b7fa1398bc4f78b3c466673db1213b6"
dependencies = [
"quote 1.0.26",
"syn 1.0.107",
"syn 1.0.109",
]
[[package]]
@ -188,7 +188,7 @@ dependencies = [
"actix-router",
"proc-macro2 1.0.56",
"quote 1.0.26",
"syn 1.0.107",
"syn 1.0.109",
]
[[package]]
@ -291,7 +291,7 @@ dependencies = [
"proc-macro2 1.0.56",
"quote 1.0.26",
"regex",
"syn 1.0.107",
"syn 1.0.109",
]
[[package]]
@ -306,7 +306,7 @@ dependencies = [
"proc-macro2 1.0.56",
"quote 1.0.26",
"rustversion",
"syn 1.0.107",
"syn 1.0.109",
]
[[package]]
@ -317,7 +317,7 @@ checksum = "e1be64a48e395fe00b8217287f226078be2cf32dae42fdf8a885b997945c3d28"
dependencies = [
"anchor-syn",
"proc-macro2 1.0.56",
"syn 1.0.107",
"syn 1.0.109",
]
[[package]]
@ -329,7 +329,7 @@ dependencies = [
"anchor-syn",
"proc-macro2 1.0.56",
"quote 1.0.26",
"syn 1.0.107",
"syn 1.0.109",
]
[[package]]
@ -342,7 +342,7 @@ dependencies = [
"anyhow",
"proc-macro2 1.0.56",
"quote 1.0.26",
"syn 1.0.107",
"syn 1.0.109",
]
[[package]]
@ -356,7 +356,7 @@ dependencies = [
"heck 0.3.3",
"proc-macro2 1.0.56",
"quote 1.0.26",
"syn 1.0.107",
"syn 1.0.109",
]
[[package]]
@ -369,7 +369,7 @@ dependencies = [
"anyhow",
"proc-macro2 1.0.56",
"quote 1.0.26",
"syn 1.0.107",
"syn 1.0.109",
]
[[package]]
@ -382,7 +382,7 @@ dependencies = [
"anyhow",
"proc-macro2 1.0.56",
"quote 1.0.26",
"syn 1.0.107",
"syn 1.0.109",
]
[[package]]
@ -412,7 +412,7 @@ dependencies = [
"anyhow",
"proc-macro2 1.0.56",
"quote 1.0.26",
"syn 1.0.107",
"syn 1.0.109",
]
[[package]]
@ -454,7 +454,7 @@ dependencies = [
"serde",
"serde_json",
"sha2 0.9.9",
"syn 1.0.107",
"syn 1.0.109",
"thiserror",
]
@ -530,7 +530,7 @@ checksum = "726535892e8eae7e70657b4c8ea93d26b8553afb1ce617caee529ef96d7dee6c"
dependencies = [
"proc-macro2 1.0.56",
"quote 1.0.26",
"syn 1.0.107",
"syn 1.0.109",
"synstructure",
]
@ -542,7 +542,7 @@ checksum = "2777730b2039ac0f95f093556e61b6d26cebed5393ca6f152717777cec3a42ed"
dependencies = [
"proc-macro2 1.0.56",
"quote 1.0.26",
"syn 1.0.107",
"syn 1.0.109",
]
[[package]]
@ -592,7 +592,7 @@ checksum = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27"
dependencies = [
"proc-macro2 1.0.56",
"quote 1.0.26",
"syn 1.0.107",
"syn 1.0.109",
]
[[package]]
@ -603,7 +603,7 @@ checksum = "705339e0e4a9690e2908d2b3d049d85682cf19fbd5782494498fbf7003a6a282"
dependencies = [
"proc-macro2 1.0.56",
"quote 1.0.26",
"syn 1.0.107",
"syn 1.0.109",
]
[[package]]
@ -846,7 +846,7 @@ dependencies = [
"borsh-schema-derive-internal",
"proc-macro-crate 0.1.5",
"proc-macro2 1.0.56",
"syn 1.0.107",
"syn 1.0.109",
]
[[package]]
@ -857,7 +857,7 @@ checksum = "5449c28a7b352f2d1e592a8a28bf139bc71afb0764a14f3c02500935d8c44065"
dependencies = [
"proc-macro2 1.0.56",
"quote 1.0.26",
"syn 1.0.107",
"syn 1.0.109",
]
[[package]]
@ -868,7 +868,7 @@ checksum = "cdbd5696d8bfa21d53d9fe39a714a18538bad11492a42d066dbbc395fb1951c0"
dependencies = [
"proc-macro2 1.0.56",
"quote 1.0.26",
"syn 1.0.107",
"syn 1.0.109",
]
[[package]]
@ -947,7 +947,7 @@ checksum = "1aca418a974d83d40a0c1f0c5cba6ff4bc28d8df099109ca459a2118d40b6322"
dependencies = [
"proc-macro2 1.0.56",
"quote 1.0.26",
"syn 1.0.107",
"syn 1.0.109",
]
[[package]]
@ -1169,6 +1169,25 @@ dependencies = [
"unreachable",
]
[[package]]
name = "config"
version = "0.13.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d379af7f68bfc21714c6c7dea883544201741d2ce8274bb12fa54f89507f52a7"
dependencies = [
"async-trait",
"json5",
"lazy_static",
"nom",
"pathdiff",
"ron",
"rust-ini",
"serde",
"serde_json",
"toml",
"yaml-rust",
]
[[package]]
name = "console"
version = "0.15.5"
@ -1404,7 +1423,7 @@ dependencies = [
"proc-macro2 1.0.56",
"quote 1.0.26",
"scratch",
"syn 1.0.107",
"syn 1.0.109",
]
[[package]]
@ -1421,7 +1440,7 @@ checksum = "65e07508b90551e610910fa648a1878991d367064997a596135b86df30daf07e"
dependencies = [
"proc-macro2 1.0.56",
"quote 1.0.26",
"syn 1.0.107",
"syn 1.0.109",
]
[[package]]
@ -1451,6 +1470,7 @@ dependencies = [
"deadpool-runtime",
"num_cpus",
"retain_mut",
"serde",
"tokio",
]
@ -1462,6 +1482,7 @@ checksum = "836a24a9d49deefe610b8b60c767a7412e9a931d79a89415cd2d2d71630ca8d7"
dependencies = [
"deadpool",
"log 0.4.17",
"serde",
"tokio",
"tokio-postgres",
]
@ -1525,7 +1546,7 @@ dependencies = [
"proc-macro2 1.0.56",
"quote 1.0.26",
"rustc_version 0.4.0",
"syn 1.0.107",
"syn 1.0.109",
]
[[package]]
@ -1598,7 +1619,7 @@ checksum = "3bf95dc3f046b9da4f2d51833c0d3547d8564ef6910f5c1ed130306a75b92886"
dependencies = [
"proc-macro2 1.0.56",
"quote 1.0.26",
"syn 1.0.107",
"syn 1.0.109",
]
[[package]]
@ -1624,6 +1645,12 @@ dependencies = [
"syn 0.15.44",
]
[[package]]
name = "dlv-list"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0688c2a7f92e427f44895cd63841bff7b29f8d7a1648b9e7e07a4a365b2e1257"
[[package]]
name = "dotenv"
version = "0.15.0"
@ -1709,7 +1736,7 @@ checksum = "8958699f9359f0b04e691a13850d48b7de329138023876d07cbd024c2c820598"
dependencies = [
"proc-macro2 1.0.56",
"quote 1.0.26",
"syn 1.0.107",
"syn 1.0.109",
]
[[package]]
@ -1721,7 +1748,7 @@ dependencies = [
"once_cell",
"proc-macro2 1.0.56",
"quote 1.0.26",
"syn 1.0.107",
"syn 1.0.109",
]
[[package]]
@ -1741,7 +1768,7 @@ checksum = "946ee94e3dbf58fdd324f9ce245c7b238d46a66f00e86a020b71996349e46cce"
dependencies = [
"proc-macro2 1.0.56",
"quote 1.0.26",
"syn 1.0.107",
"syn 1.0.109",
]
[[package]]
@ -1986,7 +2013,7 @@ checksum = "3eb14ed937631bd8b8b8977f2c198443447a8355b6e3ca599f38c975e5a963b6"
dependencies = [
"proc-macro2 1.0.56",
"quote 1.0.26",
"syn 1.0.107",
"syn 1.0.109",
]
[[package]]
@ -2600,6 +2627,17 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "json5"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96b0db21af676c1ce64250b5f40f3ce2cf27e4e47cb91ed91eb6fe9350b430c1"
dependencies = [
"pest",
"pest_derive",
"serde",
]
[[package]]
name = "jsonrpc-client-transports"
version = "18.0.0"
@ -2653,7 +2691,7 @@ dependencies = [
"proc-macro-crate 0.1.5",
"proc-macro2 1.0.56",
"quote 1.0.26",
"syn 1.0.107",
"syn 1.0.109",
]
[[package]]
@ -3133,7 +3171,7 @@ checksum = "5a7d5f7076603ebc68de2dc6a650ec331a062a13abaa346975be747bbfa4b789"
dependencies = [
"proc-macro2 1.0.56",
"quote 1.0.26",
"syn 1.0.107",
"syn 1.0.109",
]
[[package]]
@ -3256,7 +3294,7 @@ checksum = "876a53fff98e03a936a674b29568b0e605f06b29372c2489ff4de23f1949743d"
dependencies = [
"proc-macro2 1.0.56",
"quote 1.0.26",
"syn 1.0.107",
"syn 1.0.109",
]
[[package]]
@ -3338,7 +3376,7 @@ dependencies = [
"proc-macro-crate 1.2.1",
"proc-macro2 1.0.56",
"quote 1.0.26",
"syn 1.0.107",
"syn 1.0.109",
]
[[package]]
@ -3393,6 +3431,7 @@ dependencies = [
"borsh",
"bytemuck",
"chrono",
"config",
"deadpool-postgres",
"derive_more",
"dotenv",
@ -3443,7 +3482,7 @@ checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c"
dependencies = [
"proc-macro2 1.0.56",
"quote 1.0.26",
"syn 1.0.107",
"syn 1.0.109",
]
[[package]]
@ -3475,6 +3514,16 @@ dependencies = [
"vcpkg",
]
[[package]]
name = "ordered-multimap"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ccd746e37177e1711c20dd619a1620f34f5c8b569c53590a72dedd5344d8924a"
dependencies = [
"dlv-list",
"hashbrown 0.12.3",
]
[[package]]
name = "os_str_bytes"
version = "6.4.1"
@ -3501,7 +3550,7 @@ dependencies = [
"proc-macro-error",
"proc-macro2 1.0.56",
"quote 1.0.26",
"syn 1.0.107",
"syn 1.0.109",
]
[[package]]
@ -3584,6 +3633,12 @@ version = "1.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d01a5bd0424d00070b0098dd17ebca6f961a959dead1dbcbbbc1d1cd8d3deeba"
[[package]]
name = "pathdiff"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd"
[[package]]
name = "pbkdf2"
version = "0.4.0"
@ -3640,14 +3695,48 @@ dependencies = [
[[package]]
name = "pest"
version = "2.5.3"
version = "2.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4257b4a04d91f7e9e6290be5d3da4804dd5784fafde3a497d73eb2b4a158c30a"
checksum = "e68e84bfb01f0507134eac1e9b410a12ba379d064eab48c50ba4ce329a527b70"
dependencies = [
"thiserror",
"ucd-trie",
]
[[package]]
name = "pest_derive"
version = "2.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b79d4c71c865a25a4322296122e3924d30bc8ee0834c8bfc8b95f7f054afbfb"
dependencies = [
"pest",
"pest_generator",
]
[[package]]
name = "pest_generator"
version = "2.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c435bf1076437b851ebc8edc3a18442796b30f1728ffea6262d59bbe28b077e"
dependencies = [
"pest",
"pest_meta",
"proc-macro2 1.0.56",
"quote 1.0.26",
"syn 2.0.15",
]
[[package]]
name = "pest_meta"
version = "2.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "745a452f8eb71e39ffd8ee32b3c5f51d03845f99786fa9b68db6ff509c505411"
dependencies = [
"once_cell",
"pest",
"sha2 0.10.6",
]
[[package]]
name = "petgraph"
version = "0.6.2"
@ -3693,7 +3782,7 @@ checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55"
dependencies = [
"proc-macro2 1.0.56",
"quote 1.0.26",
"syn 1.0.107",
"syn 1.0.109",
]
[[package]]
@ -3799,7 +3888,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e97e3215779627f01ee256d2fad52f3d95e8e1c11e9fc6fd08f7cd455d5d5c78"
dependencies = [
"proc-macro2 1.0.56",
"syn 1.0.107",
"syn 1.0.109",
]
[[package]]
@ -3831,7 +3920,7 @@ dependencies = [
"proc-macro-error-attr",
"proc-macro2 1.0.56",
"quote 1.0.26",
"syn 1.0.107",
"syn 1.0.109",
"version_check 0.9.4",
]
@ -3872,7 +3961,7 @@ checksum = "4bf29726d67464d49fa6224a1d07936a8c08bb3fba727c7493f6cf1616fdaada"
dependencies = [
"proc-macro2 1.0.56",
"quote 1.0.26",
"syn 1.0.107",
"syn 1.0.109",
"version_check 0.9.4",
"yansi",
]
@ -3904,7 +3993,7 @@ dependencies = [
"prost",
"prost-types",
"regex",
"syn 1.0.107",
"syn 1.0.109",
"tempfile",
"which",
]
@ -3919,7 +4008,7 @@ dependencies = [
"itertools",
"proc-macro2 1.0.56",
"quote 1.0.26",
"syn 1.0.107",
"syn 1.0.109",
]
[[package]]
@ -4392,6 +4481,17 @@ dependencies = [
"librocksdb-sys",
]
[[package]]
name = "ron"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "88073939a61e5b7680558e6be56b419e208420c2adb92be54921fa6b72283f1a"
dependencies = [
"base64 0.13.1",
"bitflags",
"serde",
]
[[package]]
name = "rpassword"
version = "6.0.1"
@ -4404,6 +4504,16 @@ dependencies = [
"winapi 0.3.9",
]
[[package]]
name = "rust-ini"
version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6d5f2436026b4f6e79dc829837d467cc7e9a55ee40e750d716713540715a2df"
dependencies = [
"cfg-if 1.0.0",
"ordered-multimap",
]
[[package]]
name = "rustc-demangle"
version = "0.1.21"
@ -4579,7 +4689,7 @@ checksum = "bdbda6ac5cd1321e724fa9cee216f3a61885889b896f073b8f82322789c5250e"
dependencies = [
"proc-macro2 1.0.56",
"quote 1.0.26",
"syn 1.0.107",
"syn 1.0.109",
]
[[package]]
@ -4680,7 +4790,7 @@ checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e"
dependencies = [
"proc-macro2 1.0.56",
"quote 1.0.26",
"syn 1.0.107",
"syn 1.0.109",
]
[[package]]
@ -5255,7 +5365,7 @@ dependencies = [
"proc-macro2 1.0.56",
"quote 1.0.26",
"rustc_version 0.4.0",
"syn 1.0.107",
"syn 1.0.109",
]
[[package]]
@ -5757,7 +5867,7 @@ dependencies = [
"proc-macro2 1.0.56",
"quote 1.0.26",
"rustversion",
"syn 1.0.107",
"syn 1.0.109",
]
[[package]]
@ -6169,7 +6279,7 @@ dependencies = [
"proc-macro2 1.0.56",
"quote 1.0.26",
"rustversion",
"syn 1.0.107",
"syn 1.0.109",
]
[[package]]
@ -6197,9 +6307,9 @@ dependencies = [
[[package]]
name = "syn"
version = "1.0.107"
version = "1.0.109"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5"
checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
dependencies = [
"proc-macro2 1.0.56",
"quote 1.0.26",
@ -6231,7 +6341,7 @@ checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f"
dependencies = [
"proc-macro2 1.0.56",
"quote 1.0.26",
"syn 1.0.107",
"syn 1.0.109",
"unicode-xid 0.2.4",
]
@ -6314,7 +6424,7 @@ checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f"
dependencies = [
"proc-macro2 1.0.56",
"quote 1.0.26",
"syn 1.0.107",
"syn 1.0.109",
]
[[package]]
@ -6459,7 +6569,7 @@ checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8"
dependencies = [
"proc-macro2 1.0.56",
"quote 1.0.26",
"syn 1.0.107",
"syn 1.0.109",
]
[[package]]
@ -6670,7 +6780,7 @@ dependencies = [
"proc-macro2 1.0.56",
"prost-build",
"quote 1.0.26",
"syn 1.0.107",
"syn 1.0.109",
]
[[package]]
@ -6745,7 +6855,7 @@ checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a"
dependencies = [
"proc-macro2 1.0.56",
"quote 1.0.26",
"syn 1.0.107",
"syn 1.0.109",
]
[[package]]
@ -7069,7 +7179,7 @@ dependencies = [
"once_cell",
"proc-macro2 1.0.56",
"quote 1.0.26",
"syn 1.0.107",
"syn 1.0.109",
"wasm-bindgen-shared",
]
@ -7103,7 +7213,7 @@ checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c"
dependencies = [
"proc-macro2 1.0.56",
"quote 1.0.26",
"syn 1.0.107",
"syn 1.0.109",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
@ -7416,7 +7526,7 @@ checksum = "44bf07cb3e50ea2003396695d58bf46bc9887a1f362260446fad6bc4e79bd36c"
dependencies = [
"proc-macro2 1.0.56",
"quote 1.0.26",
"syn 1.0.107",
"syn 1.0.109",
"synstructure",
]

View File

@ -26,7 +26,7 @@ futures = "0.3.27"
jsonrpc-core-client = { version = "18.0.0", features = ["ws", "http"] }
deadpool-postgres = { version = "0.10.5", features = [ "rt_tokio_1" ] }
deadpool-postgres = { version = "0.10.5", features = [ "rt_tokio_1", "serde" ] }
tokio-postgres = { version = "0.7", features = ["with-chrono-0_4"] }
postgres-native-tls = "0.5.0"
native-tls = "0.2.11"
@ -61,4 +61,6 @@ actix-web = "4"
arrayref = "0.3.6"
bytemuck = "1.12.3"
num_enum = "0.6.1"
num_enum = "0.6.1"
config = "0.13.1"

View File

@ -1,24 +0,0 @@
FROM lukemathwalker/cargo-chef:latest-rust-1.67.1-slim AS chef
FROM chef AS planner
COPY . .
RUN cargo chef prepare --recipe-path recipe.json
FROM chef AS builder
COPY --from=planner recipe.json recipe.json
RUN apt-get update && apt-get install -y libudev-dev clang pkg-config libssl-dev build-essential cmake
RUN rustup component add rustfmt && update-ca-certificates
RUN cargo chef cook --release --recipe-path recipe.json
# Build application
COPY . .
RUN cargo build --release --bins
FROM debian:bullseye-slim as base_image
RUN apt-get update && apt-get -y install ca-certificates libssl1.1
# We do not need the Rust toolchain to run the binary!
FROM base_image AS runtime
COPY --from=builder /target/release/server /usr/local/bin
COPY --from=builder /target/release/worker /usr/local/bin
COPY --from=builder markets.json .
COPY --from=builder ca.cer .

View File

@ -19,4 +19,6 @@ RUN apt-get update && apt-get -y install ca-certificates libssl1.1
# We do not need the Rust toolchain to run the binary!
FROM base_image AS runtime
COPY --from=builder /target/release/server /usr/local/bin
ENTRYPOINT ["/usr/local/bin/server"]
COPY --from=builder markets.json .
COPY --from=builder ca.cer .
COPY --from=builder client.pks .

View File

@ -2,13 +2,13 @@ FROM lukemathwalker/cargo-chef:latest-rust-1.67.1-slim AS chef
FROM chef AS planner
COPY . .
RUN cargo chef prepare --recipe-path worker-recipe.json
RUN cargo chef prepare --recipe-path recipe.json
FROM chef AS builder
COPY --from=planner worker-recipe.json worker-recipe.json
COPY --from=planner recipe.json recipe.json
RUN apt-get update && apt-get install -y libudev-dev clang pkg-config libssl-dev build-essential cmake
RUN rustup component add rustfmt && update-ca-certificates
RUN cargo chef cook --release --recipe-path worker-recipe.json
RUN cargo chef cook --release --recipe-path recipe.json
# Build application
COPY . .
RUN cargo build --release --bin worker
@ -19,4 +19,6 @@ RUN apt-get update && apt-get -y install ca-certificates libssl1.1
# We do not need the Rust toolchain to run the binary!
FROM base_image AS runtime
COPY --from=builder /target/release/worker /usr/local/bin
ENTRYPOINT ["/usr/local/bin/worker"]
COPY --from=builder markets.json .
COPY --from=builder ca.cer .
COPY --from=builder client.pks .

19
cd/server.toml Normal file
View File

@ -0,0 +1,19 @@
app = "openbook-candles"
kill_signal = "SIGTERM"
kill_timeout = 30
[build]
dockerfile = "../Dockerfile.server"
[experimental]
cmd = ["server", "markets.json"]
[[services]]
internal_port = 8080
processes = ["app"]
protocol = "tcp"
[services.concurrency]
hard_limit = 1024
soft_limit = 1024
type = "connections"

9
cd/worker.toml Normal file
View File

@ -0,0 +1,9 @@
app = "openbook-candles-worker"
kill_signal = "SIGTERM"
kill_timeout = 30
[build]
dockerfile = "../Dockerfile.worker"
[experimental]
cmd = ["worker", "markets.json"]

View File

@ -1,36 +1,35 @@
use std::{collections::HashMap, str::FromStr, env};
use anchor_lang::prelude::Pubkey;
use chrono::{NaiveDateTime, DateTime, Utc, Duration};
use chrono::{DateTime, Duration, NaiveDateTime, Utc};
use futures::future::join_all;
use openbook_candles::{structs::{openbook::OpenBookFillEventLog, markets::{load_markets, fetch_market_infos}}, worker::trade_fetching::{scrape::scrape_transactions, parsing::parse_trades_from_openbook_txns}, database::{initialize::connect_to_database, insert::persist_fill_events}, utils::Config};
use solana_client::{rpc_config::RpcTransactionConfig, nonblocking::rpc_client::RpcClient, rpc_client::GetConfirmedSignaturesForAddress2Config, rpc_response::RpcConfirmedTransactionStatusWithSignature};
use openbook_candles::{
database::{initialize::connect_to_database, insert::persist_fill_events},
structs::{
markets::{fetch_market_infos, load_markets},
openbook::OpenBookFillEventLog,
},
utils::Config,
worker::trade_fetching::parsing::parse_trades_from_openbook_txns,
};
use solana_client::{
nonblocking::rpc_client::RpcClient, rpc_client::GetConfirmedSignaturesForAddress2Config,
rpc_config::RpcTransactionConfig, rpc_response::RpcConfirmedTransactionStatusWithSignature,
};
use solana_sdk::{commitment_config::CommitmentConfig, signature::Signature};
use solana_transaction_status::UiTransactionEncoding;
use tokio::sync::mpsc::{Sender, self};
use std::{collections::HashMap, env, str::FromStr};
use tokio::sync::mpsc::{self, Sender};
#[tokio::main]
async fn main() -> anyhow::Result<()> {
async fn main() -> anyhow::Result<()> {
dotenv::dotenv().ok();
let args: Vec<String> = env::args().collect();
assert!(args.len() == 2);
let path_to_markets_json = &args[1];
let rpc_url: String = dotenv::var("RPC_URL").unwrap();
let database_url: String = dotenv::var("DATABASE_URL").unwrap();
let ca_cert_path: String = dotenv::var("CA_CERT_PATH").unwrap();
let client_key_path: String = dotenv::var("CLIENT_KEY_PATH").unwrap();
let max_pg_pool_connections: usize = dotenv::var("MAX_PG_POOL_CONNS_WORKER")
.unwrap()
.parse::<usize>()
.unwrap();
let config = Config {
rpc_url: rpc_url.clone(),
database_url,
max_pg_pool_connections,
use_ssl: true,
ca_cert_path,
client_key_path,
};
let markets = load_markets(&path_to_markets_json);
let market_infos = fetch_market_infos(&config, markets.clone()).await?;
@ -40,10 +39,10 @@ async fn main() -> anyhow::Result<()> {
}
println!("{:?}", target_markets);
let pool = connect_to_database(&config).await?;
let pool = connect_to_database().await?;
let (fill_sender, mut fill_receiver) = mpsc::channel::<OpenBookFillEventLog>(1000);
tokio::spawn(async move {
tokio::spawn(async move {
loop {
persist_fill_events(&pool, &mut fill_receiver)
.await
@ -60,7 +59,6 @@ pub async fn backfill(
fill_sender: &Sender<OpenBookFillEventLog>,
target_markets: &HashMap<Pubkey, u8>,
) -> anyhow::Result<()> {
println!("backfill started");
let mut before_sig: Option<Signature> = None;
let mut now_time = Utc::now().timestamp();
@ -69,7 +67,8 @@ pub async fn backfill(
let mut handles = vec![];
while now_time > end_time {
let rpc_client = RpcClient::new_with_commitment(rpc_url.clone(), CommitmentConfig::confirmed());
let rpc_client =
RpcClient::new_with_commitment(rpc_url.clone(), CommitmentConfig::confirmed());
let maybe_r = get_signatures(&rpc_client, before_sig).await;
match maybe_r {
@ -90,10 +89,10 @@ pub async fn backfill(
get_transactions(&rpc_client, sigs, &cloned_sender, &cloned_markets).await;
});
handles.push(handle);
},
None => {},
}
None => {}
}
};
}
futures::future::join_all(handles).await;
@ -101,39 +100,46 @@ pub async fn backfill(
Ok(())
}
pub async fn get_signatures(
rpc_client: &RpcClient,
before_sig: Option<Signature>,
) -> Option<(
Signature,
i64,
Vec<RpcConfirmedTransactionStatusWithSignature>,
)> {
let rpc_config = GetConfirmedSignaturesForAddress2Config {
before: before_sig,
until: None,
limit: None,
commitment: Some(CommitmentConfig::confirmed()),
};
pub async fn get_signatures(rpc_client: &RpcClient,
before_sig: Option<Signature>) -> Option<(Signature, i64, Vec<RpcConfirmedTransactionStatusWithSignature>)> {
let rpc_config = GetConfirmedSignaturesForAddress2Config {
before: before_sig,
until: None,
limit: None,
commitment: Some(CommitmentConfig::confirmed()),
};
let sigs = match rpc_client
.get_signatures_for_address_with_config(
&Pubkey::from_str("srmqPvymJeFKQ4zGQed1GFppgkRHL9kaELCbyksJtPX").unwrap(),
rpc_config,
)
.await
{
Ok(s) => s,
Err(e) => {
println!("Error in get_signatures_for_address_with_config: {}", e);
return None;
}
};
if sigs.len() == 0 {
println!("No signatures found");
let sigs = match rpc_client
.get_signatures_for_address_with_config(
&Pubkey::from_str("srmqPvymJeFKQ4zGQed1GFppgkRHL9kaELCbyksJtPX").unwrap(),
rpc_config,
)
.await
{
Ok(s) => s,
Err(e) => {
println!("Error in get_signatures_for_address_with_config: {}", e);
return None;
}
let last = sigs.last().unwrap();
return Some((Signature::from_str(&last.signature).unwrap(), last.block_time.unwrap(), sigs));
};
if sigs.len() == 0 {
println!("No signatures found");
return None;
}
let last = sigs.last().unwrap();
return Some((
Signature::from_str(&last.signature).unwrap(),
last.block_time.unwrap(),
sigs,
));
}
pub async fn get_transactions(
rpc_client: &RpcClient,
@ -180,4 +186,4 @@ fn backfill_time_left(current_time: i64, backfill_end: i64) -> Duration {
let cur_date = DateTime::<Utc>::from_utc(naive_cur, Utc);
let bf_date = DateTime::<Utc>::from_utc(naive_bf, Utc);
cur_date - bf_date
}
}

View File

@ -1,28 +1,21 @@
use std::{fs, time::Duration};
use deadpool_postgres::{
Config as PgConfig, ManagerConfig, Pool, PoolConfig, RecyclingMethod, Runtime, SslMode,
Timeouts,
ManagerConfig, Pool, PoolConfig, RecyclingMethod, Runtime, SslMode, Timeouts,
};
use native_tls::{Certificate, Identity, TlsConnector};
use postgres_native_tls::MakeTlsConnector;
use crate::utils::Config;
use crate::utils::PgConfig;
pub async fn connect_to_database(config: &Config) -> anyhow::Result<Pool> {
let mut x = PgConfig::new();
pub async fn connect_to_database() -> anyhow::Result<Pool> {
let mut pg_config = PgConfig::from_env()?;
// TODO: fix
x.host = Some("".to_owned());
x.user = Some("".to_owned());
x.password = Some("".to_owned());
x.dbname = Some("postgres".to_owned());
x.manager = Some(ManagerConfig {
pg_config.pg.manager = Some(ManagerConfig {
recycling_method: RecyclingMethod::Fast,
});
x.pool = Some(PoolConfig {
max_size: config.max_pg_pool_connections,
pg_config.pg.pool = Some(PoolConfig {
max_size: pg_config.pg_max_pool_connections,
timeouts: Timeouts::default(),
});
@ -30,10 +23,16 @@ pub async fn connect_to_database(config: &Config) -> anyhow::Result<Pool> {
// base64 -i ca.cer -o ca.cer.b64 && base64 -i client.pks -o client.pks.b64
// fly secrets set PG_CA_CERT=- < ./ca.cer.b64 -a mango-fills
// fly secrets set PG_CLIENT_KEY=- < ./client.pks.b64 -a mango-fills
let tls = if config.use_ssl {
x.ssl_mode = Some(SslMode::Require);
let ca_cert = fs::read(&config.ca_cert_path).expect("reading client cert from file");
let client_key = fs::read(&config.client_key_path).expect("reading client key from file");
let tls = if pg_config.pg_use_ssl {
pg_config.pg.ssl_mode = Some(SslMode::Require);
let ca_cert = fs::read(&pg_config.pg_ca_cert_path.expect("reading ca cert from env"))
.expect("reading ca cert from file");
let client_key = fs::read(
&pg_config
.pg_client_key_path
.expect("reading client key from env"),
)
.expect("reading client key from file");
MakeTlsConnector::new(
TlsConnector::builder()
.add_root_certificate(Certificate::from_pem(&ca_cert)?)
@ -50,7 +49,10 @@ pub async fn connect_to_database(config: &Config) -> anyhow::Result<Pool> {
)
};
let pool = x.create_pool(Some(Runtime::Tokio1), tls).unwrap();
let pool = pg_config
.pg
.create_pool(Some(Runtime::Tokio1), tls)
.unwrap();
match pool.get().await {
Ok(_) => println!("Database connected"),
Err(e) => {

View File

@ -57,7 +57,6 @@ pub async fn persist_fill_events(
// }
}
}
Ok(())
}
pub async fn persist_candles(
@ -93,7 +92,6 @@ pub async fn persist_candles(
// }
// };
}
Ok(())
}
fn build_fills_upsert_statement(events: HashMap<OpenBookFillEventLog, u8>) -> String {

View File

@ -10,7 +10,7 @@ use openbook_candles::{
CoinGecko24HourVolume, CoinGeckoOrderBook, CoinGeckoPair, CoinGeckoTicker,
PgCoinGecko24HighLow,
},
slab::{get_best_bids_and_asks, get_orderbooks_with_depth},
slab::get_orderbooks_with_depth,
},
utils::WebContext,
};

View File

@ -29,27 +29,15 @@ async fn main() -> std::io::Result<()> {
assert!(args.len() == 2);
let path_to_markets_json = &args[1];
let rpc_url: String = dotenv::var("RPC_URL").unwrap();
let database_url: String = dotenv::var("DATABASE_URL").unwrap();
let use_ssl: bool = dotenv::var("USE_SSL").unwrap().parse::<bool>().unwrap();
let ca_cert_path: String = dotenv::var("CA_CERT_PATH").unwrap();
let client_key_path: String = dotenv::var("CLIENT_KEY_PATH").unwrap();
let max_pg_pool_connections: usize = dotenv::var("MAX_PG_POOL_CONNS_WORKER")
.unwrap()
.parse::<usize>()
.unwrap();
let bind_addr: String = dotenv::var("SERVER_BIND_ADDR").expect("reading bind addr from env");
let config = Config {
rpc_url: rpc_url.clone(),
database_url,
max_pg_pool_connections,
use_ssl,
ca_cert_path,
client_key_path,
};
let markets = load_markets(path_to_markets_json);
let market_infos = fetch_market_infos(&config, markets).await.unwrap();
let pool = connect_to_database(&config).await.unwrap();
let pool = connect_to_database().await.unwrap();
let context = Data::new(WebContext {
rpc_url,
@ -71,7 +59,7 @@ async fn main() -> std::io::Result<()> {
.service(coingecko::service()),
)
})
.bind(("127.0.0.1", 8080))?
.bind(&bind_addr)?
.run()
.await
}

View File

@ -16,14 +16,27 @@ impl<T, E: std::fmt::Debug> AnyhowWrap for Result<T, E> {
}
}
#[derive(Debug, serde::Deserialize)]
pub struct PgConfig {
pub pg: deadpool_postgres::Config,
pub pg_max_pool_connections: usize,
pub pg_use_ssl: bool,
pub pg_ca_cert_path: Option<String>,
pub pg_client_key_path: Option<String>,
}
impl PgConfig {
pub fn from_env() -> Result<Self, config::ConfigError> {
config::Config::builder()
.add_source(config::Environment::default().separator("_"))
.build()?
.try_deserialize()
}
}
#[derive(Clone, Debug, Deserialize)]
pub struct Config {
pub rpc_url: String,
pub database_url: String,
pub max_pg_pool_connections: usize,
pub use_ssl: bool,
pub ca_cert_path: String,
pub client_key_path: String,
}
pub struct WebContext {

View File

@ -16,7 +16,7 @@ use std::env;
use std::{collections::HashMap, str::FromStr};
use tokio::sync::mpsc;
#[tokio::main]
#[tokio::main(flavor = "multi_thread", worker_threads = 10)]
async fn main() -> anyhow::Result<()> {
dotenv::dotenv().ok();
@ -24,22 +24,9 @@ async fn main() -> anyhow::Result<()> {
assert!(args.len() == 2);
let path_to_markets_json = &args[1];
let rpc_url: String = dotenv::var("RPC_URL").unwrap();
let database_url: String = dotenv::var("DATABASE_URL").unwrap();
let use_ssl: bool = dotenv::var("USE_SSL").unwrap().parse::<bool>().unwrap();
let ca_cert_path: String = dotenv::var("CA_CERT_PATH").unwrap();
let client_key_path: String = dotenv::var("CLIENT_KEY_PATH").unwrap();
let max_pg_pool_connections: usize = dotenv::var("MAX_PG_POOL_CONNS_WORKER")
.unwrap()
.parse::<usize>()
.unwrap();
let config = Config {
rpc_url: rpc_url.clone(),
database_url,
max_pg_pool_connections,
use_ssl,
ca_cert_path,
client_key_path,
};
let markets = load_markets(&path_to_markets_json);
@ -50,7 +37,7 @@ async fn main() -> anyhow::Result<()> {
}
println!("{:?}", target_markets);
let pool = connect_to_database(&config).await?;
let pool = connect_to_database().await?;
setup_database(&pool).await?;
let mut handles = vec![];