Add docker build and fly deployment actions (#2)
* Test dockerfile with caching * Fix action * Remove tags * Use sccache release * Use max mode * idk * Try no runtime build * Use cargo-chef * Test rebuild * Revert "Test rebuild" This reverts commit d7e1654c63a0cc99a3898c1b17df0cddd13ac0bd. * Publish image to GCR * Trigger build * Read config from env * Fix typo * Bind to correct address * Lint * Add fly config * Switch trigger branch to main * Use both common branch names
This commit is contained in:
parent
43b5cf5780
commit
2f09d0a8f2
|
@ -0,0 +1 @@
|
||||||
|
target
|
|
@ -0,0 +1,56 @@
|
||||||
|
name: Publish Docker Image to GCR
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [main, master]
|
||||||
|
workflow_call:
|
||||||
|
secrets:
|
||||||
|
GCR_PROJECT:
|
||||||
|
required: false
|
||||||
|
GCR_SA_KEY:
|
||||||
|
required: false
|
||||||
|
|
||||||
|
env:
|
||||||
|
PROJECT_ID: ${{ secrets.GCR_PROJECT }}
|
||||||
|
IMAGE: mango-geyser-services
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v2
|
||||||
|
with:
|
||||||
|
submodules: recursive
|
||||||
|
|
||||||
|
- name: Set up Docker Buildx
|
||||||
|
id: buildx
|
||||||
|
uses: docker/setup-buildx-action@master
|
||||||
|
|
||||||
|
# Login to Google Cloud
|
||||||
|
- name: 'Login to Google Cloud'
|
||||||
|
uses: 'google-github-actions/auth@v0'
|
||||||
|
id: auth
|
||||||
|
with:
|
||||||
|
token_format: 'access_token'
|
||||||
|
credentials_json: '${{ secrets.GCR_SA_KEY }}'
|
||||||
|
|
||||||
|
# Login to GCR
|
||||||
|
- name: Login to GCR
|
||||||
|
uses: docker/login-action@v2
|
||||||
|
with:
|
||||||
|
registry: us-docker.pkg.dev
|
||||||
|
username: oauth2accesstoken
|
||||||
|
password: ${{ steps.auth.outputs.access_token }}
|
||||||
|
|
||||||
|
# Build and push the image
|
||||||
|
- name: Build and Push Image
|
||||||
|
uses: docker/build-push-action@v2
|
||||||
|
with:
|
||||||
|
context: .
|
||||||
|
push: true
|
||||||
|
tags: |
|
||||||
|
us-docker.pkg.dev/${{ env.PROJECT_ID }}/gcr.io/${{ env.IMAGE }}:${{ github.sha }}
|
||||||
|
us-docker.pkg.dev/${{ env.PROJECT_ID }}/gcr.io/${{ env.IMAGE }}:latest
|
||||||
|
cache-from: type=gha
|
||||||
|
cache-to: type=gha,mode=max
|
|
@ -0,0 +1,19 @@
|
||||||
|
name: Deploy to Fly
|
||||||
|
|
||||||
|
on: workflow_dispatch
|
||||||
|
|
||||||
|
env:
|
||||||
|
FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
deploy:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v2
|
||||||
|
|
||||||
|
- name: Setup Fly
|
||||||
|
uses: superfly/flyctl-actions/setup-flyctl@master
|
||||||
|
|
||||||
|
- name: Deploy
|
||||||
|
run: flyctl deploy
|
|
@ -0,0 +1,23 @@
|
||||||
|
# syntax = docker/dockerfile:1.2
|
||||||
|
# Base image containing all binaries, deployed to gcr.io/mango-markets/mango-geyser-services:latest
|
||||||
|
FROM rust:1.59.0 as base
|
||||||
|
RUN cargo install cargo-chef
|
||||||
|
RUN rustup component add rustfmt
|
||||||
|
RUN apt-get update && apt-get install -y clang cmake
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
FROM base AS plan
|
||||||
|
COPY . .
|
||||||
|
RUN cargo chef prepare --recipe-path recipe.json
|
||||||
|
|
||||||
|
FROM base as build
|
||||||
|
COPY --from=plan /app/recipe.json recipe.json
|
||||||
|
RUN cargo chef cook --release --recipe-path recipe.json
|
||||||
|
COPY . .
|
||||||
|
RUN cargo build --release --bin service-mango-fills --bin service-mango-pnl
|
||||||
|
|
||||||
|
FROM debian:bullseye-slim as run
|
||||||
|
RUN apt-get update && apt-get -y install ca-certificates libc6
|
||||||
|
COPY --from=build /app/target/release/service-mango-* /usr/local/bin/
|
||||||
|
COPY --from=build /app/service-mango-pnl/template-config.toml ./pnl-config.toml
|
||||||
|
COPY --from=build /app/service-mango-fills/template-config.toml ./fills-config.toml
|
|
@ -0,0 +1,37 @@
|
||||||
|
app = "mango-geyser-services"
|
||||||
|
|
||||||
|
kill_signal = "SIGINT"
|
||||||
|
kill_timeout = 5
|
||||||
|
|
||||||
|
[build]
|
||||||
|
image = "us-docker.pkg.dev/mango-markets/gcr.io/mango-geyser-services:latest"
|
||||||
|
|
||||||
|
[processes]
|
||||||
|
fills = "service-mango-fills fills-config.toml"
|
||||||
|
pnl = "service-mango-pnl pnl-config.toml"
|
||||||
|
|
||||||
|
[[services]]
|
||||||
|
processes = ["fills"]
|
||||||
|
internal_port = 8080
|
||||||
|
protocol = "tcp"
|
||||||
|
|
||||||
|
[[services.ports]]
|
||||||
|
port = "8080"
|
||||||
|
|
||||||
|
[services.concurrency]
|
||||||
|
type = "connections"
|
||||||
|
hard_limit = 1024
|
||||||
|
soft_limit = 1024
|
||||||
|
|
||||||
|
[[services]]
|
||||||
|
processes = ["pnl"]
|
||||||
|
internal_port = 2052
|
||||||
|
protocol = "tcp"
|
||||||
|
|
||||||
|
[[services.ports]]
|
||||||
|
port = "2052"
|
||||||
|
|
||||||
|
[services.concurrency]
|
||||||
|
type = "connections"
|
||||||
|
hard_limit = 1024
|
||||||
|
soft_limit = 1024
|
|
@ -11,7 +11,7 @@ use futures::{future, future::FutureExt};
|
||||||
use tonic::transport::{Certificate, ClientTlsConfig, Endpoint, Identity};
|
use tonic::transport::{Certificate, ClientTlsConfig, Endpoint, Identity};
|
||||||
|
|
||||||
use log::*;
|
use log::*;
|
||||||
use std::{collections::HashMap, str::FromStr, time::Duration};
|
use std::{collections::HashMap, env, str::FromStr, time::Duration};
|
||||||
|
|
||||||
pub mod geyser_proto {
|
pub mod geyser_proto {
|
||||||
tonic::include_proto!("accountsdb");
|
tonic::include_proto!("accountsdb");
|
||||||
|
@ -71,8 +71,17 @@ async fn feed_data_geyser(
|
||||||
sender: async_channel::Sender<Message>,
|
sender: async_channel::Sender<Message>,
|
||||||
) -> anyhow::Result<()> {
|
) -> anyhow::Result<()> {
|
||||||
let program_id = Pubkey::from_str(&snapshot_config.program_id)?;
|
let program_id = Pubkey::from_str(&snapshot_config.program_id)?;
|
||||||
|
let connection_string = match &grpc_config.connection_string.chars().next().unwrap() {
|
||||||
let endpoint = Endpoint::from_str(&grpc_config.connection_string)?;
|
'$' => env::var(&grpc_config.connection_string[1..])
|
||||||
|
.expect("reading connection string from env"),
|
||||||
|
_ => grpc_config.connection_string.clone(),
|
||||||
|
};
|
||||||
|
let rpc_http_url = match &snapshot_config.rpc_http_url.chars().next().unwrap() {
|
||||||
|
'$' => env::var(&snapshot_config.rpc_http_url[1..])
|
||||||
|
.expect("reading connection string from env"),
|
||||||
|
_ => snapshot_config.rpc_http_url.clone(),
|
||||||
|
};
|
||||||
|
let endpoint = Endpoint::from_str(&connection_string)?;
|
||||||
let channel = if let Some(tls) = tls_config {
|
let channel = if let Some(tls) = tls_config {
|
||||||
endpoint.tls_config(tls)?
|
endpoint.tls_config(tls)?
|
||||||
} else {
|
} else {
|
||||||
|
@ -162,7 +171,7 @@ async fn feed_data_geyser(
|
||||||
}
|
}
|
||||||
if snapshot_needed && max_rooted_slot - rooted_to_finalized_slots > first_full_slot {
|
if snapshot_needed && max_rooted_slot - rooted_to_finalized_slots > first_full_slot {
|
||||||
snapshot_needed = false;
|
snapshot_needed = false;
|
||||||
snapshot_future = tokio::spawn(get_snapshot(snapshot_config.rpc_http_url.clone(), program_id)).fuse();
|
snapshot_future = tokio::spawn(get_snapshot(rpc_http_url.clone(), program_id)).fuse();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
@ -233,16 +242,34 @@ async fn feed_data_geyser(
|
||||||
}
|
}
|
||||||
|
|
||||||
fn make_tls_config(config: &TlsConfig) -> ClientTlsConfig {
|
fn make_tls_config(config: &TlsConfig) -> ClientTlsConfig {
|
||||||
let server_root_ca_cert =
|
let server_root_ca_cert = match &config.ca_cert_path.chars().next().unwrap() {
|
||||||
std::fs::read(&config.ca_cert_path).expect("reading server root ca cert");
|
'$' => env::var(&config.ca_cert_path[1..])
|
||||||
|
.expect("reading server root ca cert from env")
|
||||||
|
.into_bytes(),
|
||||||
|
_ => std::fs::read(&config.ca_cert_path).expect("reading server root ca cert from file"),
|
||||||
|
};
|
||||||
let server_root_ca_cert = Certificate::from_pem(server_root_ca_cert);
|
let server_root_ca_cert = Certificate::from_pem(server_root_ca_cert);
|
||||||
let client_cert = std::fs::read(&config.client_cert_path).expect("reading client cert");
|
let client_cert = match &config.client_cert_path.chars().next().unwrap() {
|
||||||
let client_key = std::fs::read(&config.client_key_path).expect("reading client key");
|
'$' => env::var(&config.client_cert_path[1..])
|
||||||
|
.expect("reading client cert from env")
|
||||||
|
.into_bytes(),
|
||||||
|
_ => std::fs::read(&config.client_cert_path).expect("reading client cert from file"),
|
||||||
|
};
|
||||||
|
let client_key = match &config.client_key_path.chars().next().unwrap() {
|
||||||
|
'$' => env::var(&config.client_key_path[1..])
|
||||||
|
.expect("reading client key from env")
|
||||||
|
.into_bytes(),
|
||||||
|
_ => std::fs::read(&config.client_key_path).expect("reading client key from file"),
|
||||||
|
};
|
||||||
let client_identity = Identity::from_pem(client_cert, client_key);
|
let client_identity = Identity::from_pem(client_cert, client_key);
|
||||||
|
let domain_name = match &config.domain_name.chars().next().unwrap() {
|
||||||
|
'$' => env::var(&config.domain_name[1..]).expect("reading domain name from env"),
|
||||||
|
_ => config.domain_name.clone(),
|
||||||
|
};
|
||||||
ClientTlsConfig::new()
|
ClientTlsConfig::new()
|
||||||
.ca_certificate(server_root_ca_cert)
|
.ca_certificate(server_root_ca_cert)
|
||||||
.identity(client_identity)
|
.identity(client_identity)
|
||||||
.domain_name(&config.domain_name)
|
.domain_name(domain_name)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn process_events(
|
pub async fn process_events(
|
||||||
|
|
|
@ -0,0 +1,64 @@
|
||||||
|
bind_ws_addr = "0.0.0.0:8080"
|
||||||
|
|
||||||
|
[source]
|
||||||
|
dedup_queue_size = 50000
|
||||||
|
rpc_ws_url = ""
|
||||||
|
|
||||||
|
[[source.grpc_sources]]
|
||||||
|
name = "accountsdb-client"
|
||||||
|
connection_string = "$GEYSER_CONNECTION_STRING"
|
||||||
|
retry_connection_sleep_secs = 30
|
||||||
|
|
||||||
|
[source.grpc_sources.tls]
|
||||||
|
ca_cert_path = "$GEYSER_CA_CERT"
|
||||||
|
client_cert_path = "$GEYSER_CLIENT_CERT"
|
||||||
|
client_key_path = "$GEYSER_CLIENT_CERT"
|
||||||
|
domain_name = "$GEYSER_CERT_DOMAIN"
|
||||||
|
|
||||||
|
[source.snapshot]
|
||||||
|
rpc_http_url = "$RPC_HTTP_URL"
|
||||||
|
program_id = "mv3ekLzLbnVPNxjSKvqBpU3ZeZXPQdEC3bp5MDEBG68"
|
||||||
|
|
||||||
|
[[markets]]
|
||||||
|
name = "BTC-PERP"
|
||||||
|
event_queue = "7t5Me8RieYKsFpfLEV8jnpqcqswNpyWD95ZqgUXuLV8Z"
|
||||||
|
|
||||||
|
[[markets]]
|
||||||
|
name = "ETH-PERP"
|
||||||
|
event_queue = "9vDfKNPJkCvQv9bzR4JNTGciQC2RVHPVNMMHiVDgT1mw"
|
||||||
|
|
||||||
|
[[markets]]
|
||||||
|
name = "SOL-PERP"
|
||||||
|
event_queue = "31cKs646dt1YkA3zPyxZ7rUAkxTBz279w4XEobFXcAKP"
|
||||||
|
|
||||||
|
[[markets]]
|
||||||
|
name = "MNGO-PERP"
|
||||||
|
event_queue = "7orixrhZpjvofZGWZyyLFxSEt2tfFiost5kHEzd7jdet"
|
||||||
|
|
||||||
|
[[markets]]
|
||||||
|
name = "SRM-PERP"
|
||||||
|
event_queue = "BXSPmdHWP6fMqsCsT6kG8UN9uugAJxdDkQWy87njUQnL"
|
||||||
|
|
||||||
|
[[markets]]
|
||||||
|
name = "RAY-PERP"
|
||||||
|
event_queue = "Css2MQhEvXMTKjp9REVZR9ZyUAYAZAPrnDvRoPxrQkeN"
|
||||||
|
|
||||||
|
[[markets]]
|
||||||
|
name = "FTT-PERP"
|
||||||
|
event_queue = "5pHAhyEphQRVvLqvYF7dziofR52yZWuq8DThQFJvJ7r5"
|
||||||
|
|
||||||
|
[[markets]]
|
||||||
|
name = "ADA-PERP"
|
||||||
|
event_queue = "G6Dsw9KnP4G38hePtedTH6gDfDQmPJGJw8zipBJvKc12"
|
||||||
|
|
||||||
|
[[markets]]
|
||||||
|
name = "BNB-PERP"
|
||||||
|
event_queue = "GmX4qXMpXvs1DuUXNB4eqL1rfF8LeYEjkKgpFeYsm55n"
|
||||||
|
|
||||||
|
[[markets]]
|
||||||
|
name = "AVAX-PERP"
|
||||||
|
event_queue = "5Grgo9kLu692SUcJ6S7jtbi1WkdwiyRWgThAfN1PcvbL"
|
||||||
|
|
||||||
|
[[markets]]
|
||||||
|
name = "GMT-PERP"
|
||||||
|
event_queue = "J2WYiw67VeGkPvmM3fi65H9KxDgCf79fNwspcD3ycubK"
|
|
@ -0,0 +1,27 @@
|
||||||
|
[source]
|
||||||
|
dedup_queue_size = 50000
|
||||||
|
rpc_ws_url = ""
|
||||||
|
|
||||||
|
[[source.grpc_sources]]
|
||||||
|
name = "accountsdb-client"
|
||||||
|
connection_string = "$GEYSER_CONNECTION_STRING"
|
||||||
|
retry_connection_sleep_secs = 30
|
||||||
|
|
||||||
|
[source.grpc_sources.tls]
|
||||||
|
ca_cert_path = "$GEYSER_CA_CERT"
|
||||||
|
client_cert_path = "$GEYSER_CLIENT_CERT"
|
||||||
|
client_key_path = "$GEYSER_CLIENT_CERT"
|
||||||
|
domain_name = "$GEYSER_CERT_DOMAIN"
|
||||||
|
|
||||||
|
[source.snapshot]
|
||||||
|
rpc_http_url = "$RPC_HTTP_URL"
|
||||||
|
program_id = "mv3ekLzLbnVPNxjSKvqBpU3ZeZXPQdEC3bp5MDEBG68"
|
||||||
|
|
||||||
|
[pnl]
|
||||||
|
update_interval_millis = 5000
|
||||||
|
mango_program = "mv3ekLzLbnVPNxjSKvqBpU3ZeZXPQdEC3bp5MDEBG68"
|
||||||
|
mango_group = "98pjRuQjK3qA6gXts96PqZT4Ze5QmnCmt3QYjhbUSPue"
|
||||||
|
mango_cache = "EBDRoayCDDUvDgCimta45ajQeXbexv7aKqJubruqpyvu"
|
||||||
|
|
||||||
|
[jsonrpc_server]
|
||||||
|
bind_address = "0.0.0.0:2052"
|
Loading…
Reference in New Issue