Merge pull request #3 from blockworks-foundation/max/yellowstone

pull geyser protos from yellowstone connector
This commit is contained in:
galactus 2023-04-04 13:59:12 +02:00 committed by GitHub
commit dce0c851d2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 28 additions and 476 deletions

161
Cargo.lock generated
View File

@ -1125,18 +1125,6 @@ dependencies = [
"thiserror",
]
[[package]]
name = "cargo-lock"
version = "8.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "031718ddb8f78aa5def78a09e90defe30151d1f6c672f937af4dd916429ed996"
dependencies = [
"semver 1.0.17",
"serde",
"toml 0.5.11",
"url 2.3.1",
]
[[package]]
name = "cc"
version = "1.0.79"
@ -1872,33 +1860,13 @@ dependencies = [
"cfg-if 1.0.0",
]
[[package]]
name = "enum-iterator"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4eeac5c5edb79e4e39fe8439ef35207780a11f69c52cbe424ce3dfad4cb78de6"
dependencies = [
"enum-iterator-derive 0.7.0",
]
[[package]]
name = "enum-iterator"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "706d9e7cf1c7664859d79cd524e4e53ea2b67ea03c98cc2870c5e539695d597e"
dependencies = [
"enum-iterator-derive 1.2.0",
]
[[package]]
name = "enum-iterator-derive"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c134c37760b27a871ba422106eedbb8247da973a09e82558bf26d619c882b159"
dependencies = [
"proc-macro2 1.0.52",
"quote 1.0.26",
"syn 1.0.109",
"enum-iterator-derive",
]
[[package]]
@ -2351,53 +2319,6 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "getset"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e45727250e75cc04ff2846a66397da8ef2b3db8e40e0cef4df67950a07621eb9"
dependencies = [
"proc-macro-error",
"proc-macro2 1.0.52",
"quote 1.0.26",
"syn 1.0.109",
]
[[package]]
name = "git-version"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6b0decc02f4636b9ccad390dcbe77b722a77efedfa393caf8379a51d5c61899"
dependencies = [
"git-version-macro",
"proc-macro-hack",
]
[[package]]
name = "git-version-macro"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fe69f1cbdb6e28af2bac214e943b99ce8a0a06b447d15d3e61161b0423139f3f"
dependencies = [
"proc-macro-hack",
"proc-macro2 1.0.52",
"quote 1.0.26",
"syn 1.0.109",
]
[[package]]
name = "git2"
version = "0.16.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ccf7f68c2995f392c49fffb4f95ae2c873297830eb25c6bc4c114ce8f4562acc"
dependencies = [
"bitflags",
"libc",
"libgit2-sys",
"log 0.4.17",
"url 2.3.1",
]
[[package]]
name = "glob"
version = "0.3.1"
@ -3150,18 +3071,6 @@ version = "0.2.140"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "99227334921fae1a979cf0bfdfcc6b3e5ce376ef57e16fb6fb3ea2ed6095f80c"
[[package]]
name = "libgit2-sys"
version = "0.14.2+1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f3d95f6b51075fe9810a7ae22c7095f12b98005ab364d8544797a825ce946a4"
dependencies = [
"cc",
"libc",
"libz-sys",
"pkg-config",
]
[[package]]
name = "libloading"
version = "0.7.4"
@ -3247,7 +3156,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9702761c3935f8cc2f101793272e202c72b99da8f4224a19ddcf1279a6450bbf"
dependencies = [
"cc",
"libc",
"pkg-config",
"vcpkg",
]
@ -3411,7 +3319,6 @@ dependencies = [
"async-trait",
"borsh 0.9.3",
"bytemuck",
"cargo-lock",
"chrono",
"clap 2.34.0",
"csv-async",
@ -3419,7 +3326,6 @@ dependencies = [
"fixed",
"fixed-macro",
"futures 0.3.27",
"git-version",
"iter_tools",
"jsonrpc-core 18.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
"jsonrpc-core-client",
@ -3428,7 +3334,6 @@ dependencies = [
"mango-common",
"multiqueue",
"prost 0.11.8",
"protobuf-src",
"rand 0.8.5",
"rayon",
"serde",
@ -3453,10 +3358,8 @@ dependencies = [
"thiserror",
"tokio",
"toml 0.7.3",
"tonic 0.8.3",
"tonic-build 0.8.4",
"vergen",
"warp",
"yellowstone-grpc-proto",
]
[[package]]
@ -4369,12 +4272,6 @@ dependencies = [
"version_check 0.9.4",
]
[[package]]
name = "proc-macro-hack"
version = "0.5.20+deprecated"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068"
[[package]]
name = "proc-macro2"
version = "0.4.30"
@ -5373,9 +5270,6 @@ name = "semver"
version = "1.0.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed"
dependencies = [
"serde",
]
[[package]]
name = "semver-parser"
@ -6506,7 +6400,7 @@ dependencies = [
"base64 0.13.1",
"bincode",
"eager",
"enum-iterator 1.4.0",
"enum-iterator",
"itertools 0.10.5",
"libc",
"libloading",
@ -6945,7 +6839,7 @@ dependencies = [
"bincode",
"bytes 1.4.0",
"bzip2",
"enum-iterator 1.4.0",
"enum-iterator",
"flate2 1.0.25",
"futures 0.3.27",
"goauth",
@ -7503,20 +7397,6 @@ dependencies = [
"walkdir",
]
[[package]]
name = "sysinfo"
version = "0.23.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3977ec2e0520829be45c8a2df70db2bf364714d8a748316a10c3c35d4d2b01c9"
dependencies = [
"cfg-if 1.0.0",
"core-foundation-sys",
"libc",
"ntapi",
"once_cell",
"winapi 0.3.9",
]
[[package]]
name = "tar"
version = "0.4.38"
@ -7969,7 +7849,6 @@ dependencies = [
"axum",
"base64 0.13.1",
"bytes 1.4.0",
"flate2 1.0.25",
"futures-core",
"futures-util",
"h2",
@ -7981,7 +7860,6 @@ dependencies = [
"pin-project",
"prost 0.11.8",
"prost-derive 0.11.8",
"rustls-native-certs",
"rustls-pemfile 1.0.2",
"tokio",
"tokio-rustls 0.23.4",
@ -8343,24 +8221,6 @@ version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
[[package]]
name = "vergen"
version = "7.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1f44ef1afcf5979e34748c12595f9589f3dc4e34abf156fb6d95f9b835568dc"
dependencies = [
"anyhow",
"cfg-if 1.0.0",
"enum-iterator 0.7.0",
"getset",
"git2",
"rustc_version 0.4.0",
"rustversion",
"sysinfo",
"thiserror",
"time 0.3.20",
]
[[package]]
name = "version_check"
version = "0.1.5"
@ -8818,6 +8678,19 @@ dependencies = [
"time 0.3.20",
]
[[package]]
name = "yellowstone-grpc-proto"
version = "1.1.0+solana.1.15.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c2688c6f1d930f21f580829f6f896d7d38f90d5b2272e53a8f69a82e72b656f"
dependencies = [
"anyhow",
"prost 0.11.8",
"protobuf-src",
"tonic 0.8.3",
"tonic-build 0.8.4",
]
[[package]]
name = "zeroize"
version = "1.3.0"

View File

@ -29,6 +29,7 @@ dashmap = "5.4.0"
mango = { git = "https://github.com/blockworks-foundation/mango-v3.git", tag = "v3.6.0", default-features = false }
mango-common = { git = "https://github.com/blockworks-foundation/mango-v3.git", tag = "v3.6.0" }
yellowstone-grpc-proto = "1.0.1"
solana-client = { git = "https://github.com/solana-labs/solana.git", tag="v1.15.2" }
solana-core = { git = "https://github.com/solana-labs/solana.git", tag="v1.15.2" }
@ -53,7 +54,6 @@ solana-program = "1.9.17"
thiserror = "1.0"
csv-async = "1.2"
async-std = "1.12.0"
tonic = { version = "0.8.2", features = ["gzip", "tls", "tls-roots"] }
tokio = { version = "1", features = ["full"] }
anyhow = "1.0"
async-channel = "1.6"
@ -67,15 +67,6 @@ arrayref = "*"
bytemuck = "1.7.2"
toml = "*"
[build-dependencies]
anyhow = "1.0.62"
cargo-lock = "8.0.2"
git-version = "0.3.5"
protobuf-src = "1.1.0"
tonic-build = "0.8.2"
vergen = "=7.2.1"
[package.metadata.docs.rs]
targets = ["x86_64-unknown-linux-gnu"]

View File

@ -1,44 +0,0 @@
use {
cargo_lock::Lockfile,
std::collections::HashSet,
vergen::{vergen, Config},
};
fn main() -> anyhow::Result<()> {
compile_protos()?;
generate_env()?;
Ok(())
}
fn compile_protos() -> anyhow::Result<()> {
std::env::set_var("PROTOC", protobuf_src::protoc());
tonic_build::compile_protos("./proto/geyser.proto")?;
Ok(())
}
fn generate_env() -> anyhow::Result<()> {
vergen(Config::default())?;
// vergen git version does not looks cool
println!(
"cargo:rustc-env=GIT_VERSION={}",
git_version::git_version!()
);
// Extract Solana version
let lockfile = Lockfile::load("./Cargo.lock")?;
println!(
"cargo:rustc-env=SOLANA_SDK_VERSION={}",
lockfile
.packages
.iter()
.filter(|pkg| pkg.name.as_str() == "solana-sdk")
.map(|pkg| pkg.version.to_string())
.collect::<HashSet<_>>()
.into_iter()
.collect::<Vec<_>>()
.join(",")
);
Ok(())
}

View File

@ -1,116 +0,0 @@
syntax = "proto3";
import public "solana-storage-v1.15.2.proto";
option go_package = "github.com/rpcpool/solana-geyser-grpc/golang/proto";
package geyser;
service Geyser {
rpc Subscribe(stream SubscribeRequest) returns (stream SubscribeUpdate) {}
}
message SubscribeRequest {
map<string, SubscribeRequestFilterAccounts> accounts = 1;
map<string, SubscribeRequestFilterSlots> slots = 2;
map<string, SubscribeRequestFilterTransactions> transactions = 3;
map<string, SubscribeRequestFilterBlocks> blocks = 4;
map<string, SubscribeRequestFilterBlocksMeta> blocks_meta = 5;
}
message SubscribeRequestFilterAccounts {
repeated string account = 2;
repeated string owner = 3;
}
message SubscribeRequestFilterSlots {}
message SubscribeRequestFilterTransactions {
optional bool vote = 1;
optional bool failed = 2;
optional string signature = 5;
repeated string account_include = 3;
repeated string account_exclude = 4;
}
message SubscribeRequestFilterBlocks {}
message SubscribeRequestFilterBlocksMeta {}
message SubscribeUpdate {
repeated string filters = 1;
oneof update_oneof {
SubscribeUpdateAccount account = 2;
SubscribeUpdateSlot slot = 3;
SubscribeUpdateTransaction transaction = 4;
SubscribeUpdateBlock block = 5;
SubscribeUpdatePing ping = 6;
SubscribeUpdateBlockMeta block_meta = 7;
}
}
message SubscribeUpdateAccount {
SubscribeUpdateAccountInfo account = 1;
uint64 slot = 2;
bool is_startup = 3;
}
message SubscribeUpdateAccountInfo {
bytes pubkey = 1;
uint64 lamports = 2;
bytes owner = 3;
bool executable = 4;
uint64 rent_epoch = 5;
bytes data = 6;
uint64 write_version = 7;
optional bytes txn_signature = 8;
}
message SubscribeUpdateSlot {
uint64 slot = 1;
optional uint64 parent = 2;
SubscribeUpdateSlotStatus status = 3;
}
enum SubscribeUpdateSlotStatus {
PROCESSED = 0;
CONFIRMED = 1;
FINALIZED = 2;
}
message SubscribeUpdateTransaction {
SubscribeUpdateTransactionInfo transaction = 1;
uint64 slot = 2;
}
message SubscribeUpdateTransactionInfo {
bytes signature = 1;
bool is_vote = 2;
solana.storage.ConfirmedBlock.Transaction transaction = 3;
solana.storage.ConfirmedBlock.TransactionStatusMeta meta = 4;
uint64 index = 5;
}
message SubscribeUpdateBlock {
uint64 slot = 1;
string blockhash = 2;
solana.storage.ConfirmedBlock.Rewards rewards = 3;
solana.storage.ConfirmedBlock.UnixTimestamp block_time = 4;
solana.storage.ConfirmedBlock.BlockHeight block_height = 5;
repeated SubscribeUpdateTransactionInfo transactions = 6;
uint64 parent_slot = 7;
string parent_blockhash = 8;
}
message SubscribeUpdateBlockMeta {
uint64 slot = 1;
string blockhash = 2;
solana.storage.ConfirmedBlock.Rewards rewards = 3;
solana.storage.ConfirmedBlock.UnixTimestamp block_time = 4;
solana.storage.ConfirmedBlock.BlockHeight block_height = 5;
uint64 parent_slot = 6;
string parent_blockhash = 7;
uint64 executed_transaction_count = 8;
}
message SubscribeUpdatePing {}

View File

@ -1,143 +0,0 @@
syntax = "proto3";
package solana.storage.ConfirmedBlock;
option go_package = "github.com/rpcpool/solana-geyser-grpc/golang/proto";
message ConfirmedBlock {
string previous_blockhash = 1;
string blockhash = 2;
uint64 parent_slot = 3;
repeated ConfirmedTransaction transactions = 4;
repeated Reward rewards = 5;
UnixTimestamp block_time = 6;
BlockHeight block_height = 7;
}
message ConfirmedTransaction {
Transaction transaction = 1;
TransactionStatusMeta meta = 2;
}
message Transaction {
repeated bytes signatures = 1;
Message message = 2;
}
message Message {
MessageHeader header = 1;
repeated bytes account_keys = 2;
bytes recent_blockhash = 3;
repeated CompiledInstruction instructions = 4;
bool versioned = 5;
repeated MessageAddressTableLookup address_table_lookups = 6;
}
message MessageHeader {
uint32 num_required_signatures = 1;
uint32 num_readonly_signed_accounts = 2;
uint32 num_readonly_unsigned_accounts = 3;
}
message MessageAddressTableLookup {
bytes account_key = 1;
bytes writable_indexes = 2;
bytes readonly_indexes = 3;
}
message TransactionStatusMeta {
TransactionError err = 1;
uint64 fee = 2;
repeated uint64 pre_balances = 3;
repeated uint64 post_balances = 4;
repeated InnerInstructions inner_instructions = 5;
bool inner_instructions_none = 10;
repeated string log_messages = 6;
bool log_messages_none = 11;
repeated TokenBalance pre_token_balances = 7;
repeated TokenBalance post_token_balances = 8;
repeated Reward rewards = 9;
repeated bytes loaded_writable_addresses = 12;
repeated bytes loaded_readonly_addresses = 13;
ReturnData return_data = 14;
bool return_data_none = 15;
// Sum of compute units consumed by all instructions.
// Available since Solana v1.10.35 / v1.11.6.
// Set to `None` for txs executed on earlier versions.
optional uint64 compute_units_consumed = 16;
}
message TransactionError {
bytes err = 1;
}
message InnerInstructions {
uint32 index = 1;
repeated InnerInstruction instructions = 2;
}
message InnerInstruction {
uint32 program_id_index = 1;
bytes accounts = 2;
bytes data = 3;
// Invocation stack height of an inner instruction.
// Available since Solana v1.14.6
// Set to `None` for txs executed on earlier versions.
optional uint32 stack_height = 4;
}
message CompiledInstruction {
uint32 program_id_index = 1;
bytes accounts = 2;
bytes data = 3;
}
message TokenBalance {
uint32 account_index = 1;
string mint = 2;
UiTokenAmount ui_token_amount = 3;
string owner = 4;
string program_id = 5;
}
message UiTokenAmount {
double ui_amount = 1;
uint32 decimals = 2;
string amount = 3;
string ui_amount_string = 4;
}
message ReturnData {
bytes program_id = 1;
bytes data = 2;
}
enum RewardType {
Unspecified = 0;
Fee = 1;
Rent = 2;
Staking = 3;
Voting = 4;
}
message Reward {
string pubkey = 1;
int64 lamports = 2;
uint64 post_balance = 3;
RewardType reward_type = 4;
string commission = 5;
}
message Rewards {
repeated Reward rewards = 1;
}
message UnixTimestamp {
int64 timestamp = 1;
}
message BlockHeight {
uint64 block_height = 1;
}

View File

@ -1,5 +1,4 @@
use futures::stream::once;
use geyser::geyser_client::GeyserClient;
use jsonrpc_core::futures::StreamExt;
use jsonrpc_core_client::transports::http;
@ -12,7 +11,7 @@ use solana_rpc::rpc::rpc_accounts_scan::AccountsScanClient;
use solana_sdk::{account::Account, commitment_config::CommitmentConfig, pubkey::Pubkey};
use futures::{future, future::FutureExt};
use tonic::{
use yellowstone_grpc_proto::tonic::{
metadata::MetadataValue,
transport::{Certificate, Channel, ClientTlsConfig, Identity},
Request,
@ -21,20 +20,11 @@ use tonic::{
use log::*;
use std::{collections::HashMap, env, str::FromStr, time::Duration};
pub mod geyser {
tonic::include_proto!("geyser");
}
pub mod solana {
pub mod storage {
pub mod confirmed_block {
tonic::include_proto!("solana.storage.confirmed_block");
}
}
}
pub use geyser::*;
pub use solana::storage::confirmed_block::*;
use yellowstone_grpc_proto::prelude::{
geyser_client::GeyserClient, subscribe_update, SubscribeRequest,
SubscribeRequestFilterAccounts, SubscribeRequestFilterSlots, SubscribeUpdate,
SubscribeUpdateSlotStatus,
};
use crate::websocket_source::KeeperConfig;
use crate::{
@ -84,7 +74,7 @@ struct SnapshotData {
accounts: Vec<(String, Option<UiAccount>)>,
}
enum Message {
GrpcUpdate(geyser::SubscribeUpdate),
GrpcUpdate(SubscribeUpdate),
Snapshot(SnapshotData),
}
@ -186,6 +176,7 @@ async fn feed_data_geyser(
SubscribeRequestFilterAccounts {
account: filter_config.account_ids.clone(),
owner: filter_config.program_ids.clone(),
filters: vec![],
},
);
let mut slots = HashMap::new();
@ -265,7 +256,7 @@ async fn feed_data_geyser(
loop {
tokio::select! {
update = update_stream.next() => {
use geyser::{subscribe_update::UpdateOneof};
use subscribe_update::UpdateOneof;
let mut update = update.ok_or(anyhow::anyhow!("geyser plugin has closed the stream"))??;
trace!("update={:?}", update);
match update.update_oneof.as_mut().expect("invalid grpc") {
@ -512,7 +503,7 @@ pub async fn process_events(
loop {
metric_dedup_queue.set(msg_receiver.len() as u64);
let msg = msg_receiver.recv().await.expect("sender must not close");
use geyser::subscribe_update::UpdateOneof;
use subscribe_update::UpdateOneof;
match msg {
Message::GrpcUpdate(update) => {
match update.update_oneof.expect("invalid grpc") {