Merge pull request #3 from rpcpool/fixes_1.9

Compatibility fixes for 1.9

Cargo.toml

@@ -1,6 +1,6 @@
 [workspace]
 members = [
-        "accountsdb-plugin-grpc",
+        "geyser-plugin-grpc",
         "lib",
         "connector-raw",
         "connector-mango",
@@ -11,12 +11,3 @@ members = [
 # for gzip encoded responses
 jsonrpc-core-client = { git = "https://github.com/ckamm/jsonrpc.git", branch = "ckamm/http-with-gzip" }
 
-#solana-accountsdb-plugin-interface = { path = "../solana-accountsdb/accountsdb-plugin-interface" }
-#solana-logger = { path = "../solana-accountsdb/logger" }
-#solana-metrics = { path = "../solana-accountsdb/metrics" }
-#solana-sdk = { path = "../solana-accountsdb/sdk" }
-#solana-program = { path = "../solana-accountsdb/sdk/program" }
-#anchor-lang = { path = "../anchor/lang" } # armani/solana branch for 1.8
-
-#mango = { path = "../mango-v3-dep/program" }
-

README.md

@@ -15,7 +15,7 @@ and `getProgramAccounts` queries generally. That would reduce load on Solana RPC
 nodes while decreasing response times.
 
 Supported Solana sources:
-- AccountsDB plugin (preferred) plus JSONRPC HTTP API (for initial snapshots)
+- Geyser plugin (preferred) plus JSONRPC HTTP API (for initial snapshots)
 
 Unfinished Solana sources:
 - JSONRPC websocket subscriptions plus JSONRPC HTTP API (for initial snapshots)
@@ -27,9 +27,9 @@ Supported targets:
 Components
 ==========
 
-- [`accountsdb-plugin-grpc/`](accountsdb-plugin-grpc/)
+- [`geyser-plugin-grpc/`](geyser-plugin-grpc/)
 
-  The Solana AccountsDB plugin. It opens a gRPC server (see [`proto/`](proto/)) and
+  The Solana Geyser plugin. It opens a gRPC server (see [`proto/`](proto/)) and
   broadcasts account and slot updates to all clients that connect.
 
 - [`lib/`](lib/)
@@ -56,17 +56,17 @@ Setup Tutorial
 1. Compile the project.
 
    Make sure that you are using _exactly_ the same Rust version for compiling the
-   AccountsDb plugin that was used for compiling your `solana-validator`! Otherwise
+   Geyser plugin that was used for compiling your `solana-validator`! Otherwise
    the plugin will crash the validator during startup!
 
 2. Prepare the plugin configuration file.
 
-   [Here is an example](accountsdb-plugin-grpc/example-config.json). This file
+   [Here is an example](geyser-plugin-grpc/example-config.json). This file
    points the validator to your plugin shared library, controls which accounts
    will be exported, which address the gRPC server will bind to and internal
    queue sizes.
 
-3. Run `solana-validator` with `--accountsdb-plugin-config myconfig.json`.
+3. Run `solana-validator` with `--geyser-plugin-config myconfig.json`.
 
    Check the logs to ensure the plugin was loaded.
 

accountsdb-plugin-grpc/src/lib.rs

@@ -1,2 +0,0 @@
-pub mod accounts_selector;
-pub mod accountsdb_plugin_grpc;

connector-mango/Cargo.toml

@@ -1,12 +1,12 @@
 [package]
-name = "solana-accountsdb-connector-mango"
+name = "solana-geyser-connector-mango"
 version = "0.1.0"
 authors = ["Christian Kamm <mail@ckamm.de>"]
 edition = "2018"
 
 [dependencies]
-solana-accountsdb-connector-lib = { path = "../lib" }
+solana-geyser-connector-lib = { path = "../lib" }
-solana-logger = "=1.8.14"
+solana-logger = "=1.9.13"
 log = "0.4"
 anyhow = "1.0"
 toml = "0.5"

connector-mango/src/main.rs

@@ -2,7 +2,7 @@ mod mango;
 
 use {
     log::*,
-    solana_accountsdb_connector_lib::*,
+    solana_geyser_connector_lib::*,
     std::{fs::File, io::Read, sync::Arc},
 };
 
@@ -37,8 +37,8 @@ async fn main() -> anyhow::Result<()> {
         postgres_target::init(&config.postgres_target, account_tables, metrics_tx.clone()).await?;
 
     info!("postgres done");
-    let use_accountsdb = true;
+    let use_geyser = true;
-    if use_accountsdb {
+    if use_geyser {
         grpc_plugin_source::process_events(
             config,
             account_write_queue_sender,

connector-raw/Cargo.toml

@@ -1,12 +1,12 @@
 [package]
-name = "solana-accountsdb-connector-raw"
+name = "solana-geyser-connector-raw"
 version = "0.1.0"
 authors = ["Christian Kamm <mail@ckamm.de>"]
 edition = "2018"
 
 [dependencies]
-solana-accountsdb-connector-lib = { path = "../lib" }
+solana-geyser-connector-lib = { path = "../lib" }
-solana-logger = "=1.8.14"
+solana-logger = "=1.9.13"
 log = "0.4"
 tokio = { version = "1", features = ["full"] }
 anyhow = "1.0"

connector-raw/src/main.rs

@@ -1,6 +1,6 @@
 use {
     log::*,
-    solana_accountsdb_connector_lib::*,
+    solana_geyser_connector_lib::*,
     std::{fs::File, io::Read, sync::Arc},
 };
 
@@ -30,8 +30,8 @@ async fn main() -> anyhow::Result<()> {
         postgres_target::init(&config.postgres_target, account_tables, metrics_tx.clone()).await?;
 
     info!("postgres done");
-    let use_accountsdb = true;
+    let use_geyser = true;
-    if use_accountsdb {
+    if use_geyser {
         grpc_plugin_source::process_events(
             config,
             account_write_queue_sender,

geyser-plugin-grpc/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "solana-accountsdb-connector-plugin-grpc"
+name = "solana-geyser-connector-plugin-grpc"
 version = "0.1.0"
 authors = ["Christian Kamm <mail@ckamm.de>"]
 edition = "2018"
@@ -18,10 +18,10 @@ serde = "1.0.130"
 serde_derive = "1.0.103"
 serde_json = "1.0.67"
 
-solana-accountsdb-plugin-interface = "=1.8.14"
+solana-geyser-plugin-interface = "=1.9.13"
-solana-logger = "=1.8.14"
+solana-logger = "=1.9.13"
-solana-metrics = "=1.8.14"
+solana-metrics = "=1.9.13"
-solana-sdk = "=1.8.14"
+solana-sdk = "=1.9.13"
 
 tonic = "0.6"
 prost = "0.9"

geyser-plugin-grpc/build.rs

@@ -1,4 +1,4 @@
 fn main() {
-    tonic_build::compile_protos("../proto/accountsdb.proto")
+    tonic_build::compile_protos("../proto/geyser.proto")
         .unwrap_or_else(|e| panic!("Failed to compile protos {:?}", e));
 }

geyser-plugin-grpc/src/geyser_plugin_grpc.rs

@@ -1,16 +1,16 @@
 use {
     crate::accounts_selector::AccountsSelector,
-    accountsdb_proto::{
+    bs58,
+    geyser_proto::{
         slot_update::Status as SlotUpdateStatus, update::UpdateOneof, AccountWrite, Ping,
         SlotUpdate, SubscribeRequest, SubscribeResponse, Update,
     },
-    bs58,
     log::*,
     serde_derive::Deserialize,
     serde_json,
-    solana_accountsdb_plugin_interface::accountsdb_plugin_interface::{
+    solana_geyser_plugin_interface::geyser_plugin_interface::{
-        AccountsDbPlugin, AccountsDbPluginError, ReplicaAccountInfoVersions,
+        GeyserPlugin, GeyserPluginError, ReplicaAccountInfoVersions, Result as PluginResult,
-        Result as PluginResult, SlotStatus,
+        SlotStatus,
     },
     std::collections::HashSet,
     std::convert::TryInto,
@@ -21,14 +21,14 @@ use {
     tonic::transport::Server,
 };
 
-pub mod accountsdb_proto {
+pub mod geyser_proto {
-    tonic::include_proto!("accountsdb");
+    tonic::include_proto!("geyser");
 }
 
-pub mod accountsdb_service {
+pub mod geyser_service {
     use super::*;
     use {
-        accountsdb_proto::accounts_db_server::AccountsDb,
+        geyser_proto::accounts_db_server::AccountsDb,
         tokio_stream::wrappers::ReceiverStream,
         tonic::{Code, Request, Response, Status},
     };
@@ -130,7 +130,7 @@ impl std::fmt::Debug for Plugin {
 #[derive(Clone, Debug, Deserialize)]
 pub struct PluginConfig {
     pub bind_address: String,
-    pub service_config: accountsdb_service::ServiceConfig,
+    pub service_config: geyser_service::ServiceConfig,
 }
 
 impl PluginData {
@@ -142,9 +142,9 @@ impl PluginData {
     }
 }
 
-impl AccountsDbPlugin for Plugin {
+impl GeyserPlugin for Plugin {
     fn name(&self) -> &'static str {
-        "AccountsDbPluginGrpc"
+        "GeyserPluginGrpc"
     }
 
     fn on_load(&mut self, config_file: &str) -> PluginResult<()> {
@@ -163,7 +163,7 @@ impl AccountsDbPlugin for Plugin {
         let accounts_selector = Self::create_accounts_selector_from_config(&result);
 
         let config: PluginConfig = serde_json::from_str(&contents).map_err(|err| {
-            AccountsDbPluginError::ConfigFileReadError {
+            GeyserPluginError::ConfigFileReadError {
                 msg: format!(
                     "The config file is not in the JSON format expected: {:?}",
                     err
@@ -171,19 +171,21 @@ impl AccountsDbPlugin for Plugin {
             }
         })?;
 
-        let addr = config.bind_address.parse().map_err(|err| {
+        let addr =
-            AccountsDbPluginError::ConfigFileReadError {
+            config
-                msg: format!("Error parsing the bind_address {:?}", err),
+                .bind_address
-            }
+                .parse()
-        })?;
+                .map_err(|err| GeyserPluginError::ConfigFileReadError {
+                    msg: format!("Error parsing the bind_address {:?}", err),
+                })?;
 
         let highest_write_slot = Arc::new(AtomicU64::new(0));
         let service =
-            accountsdb_service::Service::new(config.service_config, highest_write_slot.clone());
+            geyser_service::Service::new(config.service_config, highest_write_slot.clone());
         let (server_exit_sender, mut server_exit_receiver) = broadcast::channel::<()>(1);
         let server_broadcast = service.sender.clone();
 
-        let server = accountsdb_proto::accounts_db_server::AccountsDbServer::new(service);
+        let server = geyser_proto::accounts_db_server::AccountsDbServer::new(service);
         let runtime = tokio::runtime::Runtime::new().unwrap();
         runtime.spawn(Server::builder().add_service(server).serve_with_shutdown(
             addr,
@@ -363,10 +365,10 @@ impl Plugin {
 #[allow(improper_ctypes_definitions)]
 /// # Safety
 ///
-/// This function returns the Plugin pointer as trait AccountsDbPlugin.
+/// This function returns the Plugin pointer as trait GeyserPlugin.
-pub unsafe extern "C" fn _create_plugin() -> *mut dyn AccountsDbPlugin {
+pub unsafe extern "C" fn _create_plugin() -> *mut dyn GeyserPlugin {
     let plugin = Plugin::default();
-    let plugin: Box<dyn AccountsDbPlugin> = Box::new(plugin);
+    let plugin: Box<dyn GeyserPlugin> = Box::new(plugin);
     Box::into_raw(plugin)
 }
 

geyser-plugin-grpc/src/lib.rs

@@ -0,0 +1,2 @@
+pub mod accounts_selector;
+pub mod geyser_plugin_grpc;

geyser-plugin-grpc/src/test_server.rs

@@ -2,15 +2,15 @@ use rand::Rng;
 use tokio::sync::{broadcast, mpsc};
 use tonic::transport::Server;
 
-pub mod accountsdb_proto {
+pub mod geyser_proto {
-    tonic::include_proto!("accountsdb");
+    tonic::include_proto!("geyser");
 }
-use accountsdb_proto::{update::UpdateOneof, SlotUpdate, SubscribeRequest, Update};
+use geyser_proto::{update::UpdateOneof, SlotUpdate, SubscribeRequest, Update};
 
-pub mod accountsdb_service {
+pub mod geyser_service {
     use super::*;
     use {
-        accountsdb_proto::accounts_db_server::AccountsDb,
+        geyser_proto::accounts_db_server::AccountsDb,
         tokio_stream::wrappers::ReceiverStream,
         tonic::{Request, Response, Status},
     };
@@ -53,9 +53,9 @@ pub mod accountsdb_service {
 async fn main() -> Result<(), Box<dyn std::error::Error>> {
     let addr = "[::1]:10000".parse().unwrap();
 
-    let service = accountsdb_service::Service::new();
+    let service = geyser_service::Service::new();
     let sender = service.sender.clone();
-    let svc = accountsdb_proto::accounts_db_server::AccountsDbServer::new(service);
+    let svc = geyser_proto::accounts_db_server::AccountsDbServer::new(service);
 
     tokio::spawn(async move {
         let mut slot = 1;

lib/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "solana-accountsdb-connector-lib"
+name = "solana-geyser-connector-lib"
 version = "0.1.0"
 authors = ["Christian Kamm <mail@ckamm.de>"]
 edition = "2018"
@@ -11,10 +11,10 @@ edition = "2018"
 jsonrpc-core = "18.0.0"
 jsonrpc-core-client = { version = "18.0.0", features = ["ws", "http"] }
 
-solana-rpc = "=1.8.14"
+solana-rpc = "=1.9.13"
-solana-client = "=1.8.14"
+solana-client = "=1.9.13"
-solana-account-decoder = "=1.8.14"
+solana-account-decoder = "=1.9.13"
-solana-sdk = "=1.8.14"
+solana-sdk = "=1.9.13"
 
 tokio = { version = "1", features = ["full"] }
 tokio-stream = "0.1"

lib/build.rs

@@ -1,4 +1,4 @@
 fn main() {
-    tonic_build::compile_protos("../proto/accountsdb.proto")
+    tonic_build::compile_protos("../proto/geyser.proto")
         .unwrap_or_else(|e| panic!("Failed to compile protos {:?}", e));
 }

lib/src/grpc_plugin_source.rs

@@ -4,7 +4,7 @@ use jsonrpc_core_client::transports::http;
 use solana_account_decoder::UiAccountEncoding;
 use solana_client::rpc_config::{RpcAccountInfoConfig, RpcProgramAccountsConfig};
 use solana_client::rpc_response::{Response, RpcKeyedAccount};
-use solana_rpc::{rpc::rpc_full::FullClient, rpc::OptionalContext};
+use solana_rpc::{rpc::rpc_accounts::AccountsDataClient, rpc::OptionalContext};
 use solana_sdk::{account::Account, commitment_config::CommitmentConfig, pubkey::Pubkey};
 
 use futures::{future, future::FutureExt};
@@ -13,10 +13,10 @@ use tonic::transport::{Certificate, ClientTlsConfig, Endpoint, Identity};
 use log::*;
 use std::{collections::HashMap, str::FromStr, time::Duration};
 
-pub mod accountsdb_proto {
+pub mod geyser_proto {
-    tonic::include_proto!("accountsdb");
+    tonic::include_proto!("geyser");
 }
-use accountsdb_proto::accounts_db_client::AccountsDbClient;
+use geyser_proto::accounts_db_client::AccountsDbClient;
 
 use crate::{
     metrics, AccountWrite, AnyhowWrap, Config, GrpcSourceConfig, SlotStatus, SlotUpdate,
@@ -26,7 +26,7 @@ use crate::{
 type SnapshotData = Response<Vec<RpcKeyedAccount>>;
 
 enum Message {
-    GrpcUpdate(accountsdb_proto::Update),
+    GrpcUpdate(geyser_proto::Update),
     Snapshot(SnapshotData),
 }
 
@@ -34,7 +34,7 @@ async fn get_snapshot(
     rpc_http_url: String,
     program_id: Pubkey,
 ) -> anyhow::Result<OptionalContext<Vec<RpcKeyedAccount>>> {
-    let rpc_client = http::connect_with_options::<FullClient>(&rpc_http_url, true)
+    let rpc_client = http::connect_with_options::<AccountsDataClient>(&rpc_http_url, true)
         .await
         .map_err_anyhow()?;
 
@@ -61,7 +61,7 @@ async fn get_snapshot(
     Ok(account_snapshot)
 }
 
-async fn feed_data_accountsdb(
+async fn feed_data_geyser(
     grpc_config: &GrpcSourceConfig,
     tls_config: Option<ClientTlsConfig>,
     snapshot_config: &SnapshotSourceConfig,
@@ -80,7 +80,7 @@ async fn feed_data_accountsdb(
     let mut client = AccountsDbClient::new(channel);
 
     let mut update_stream = client
-        .subscribe(accountsdb_proto::SubscribeRequest {})
+        .subscribe(geyser_proto::SubscribeRequest {})
         .await?
         .into_inner();
 
@@ -139,8 +139,8 @@ async fn feed_data_accountsdb(
     loop {
         tokio::select! {
             update = update_stream.next() => {
-                use accountsdb_proto::{update::UpdateOneof, slot_update::Status};
+                use geyser_proto::{update::UpdateOneof, slot_update::Status};
-                let mut update = update.ok_or(anyhow::anyhow!("accountsdb plugin has closed the stream"))??;
+                let mut update = update.ok_or(anyhow::anyhow!("geyser plugin has closed the stream"))??;
                 match update.update_oneof.as_mut().expect("invalid grpc") {
                     UpdateOneof::SubscribeResponse(subscribe_response) => {
                         first_full_slot = subscribe_response.highest_write_slot + 1;
@@ -188,7 +188,7 @@ async fn feed_data_accountsdb(
                         write.write_version = *writes as u64;
                         *writes += 1;
                     },
-                    accountsdb_proto::update::UpdateOneof::Ping(_) => {},
+                    geyser_proto::update::UpdateOneof::Ping(_) => {},
                 }
                 sender.send(Message::GrpcUpdate(update)).await.expect("send success");
             },
@@ -216,7 +216,7 @@ async fn feed_data_accountsdb(
                 }
             },
             _ = tokio::time::sleep(fatal_idle_timeout) => {
-                anyhow::bail!("accountsdb plugin hasn't sent a message in too long");
+                anyhow::bail!("geyser plugin hasn't sent a message in too long");
             }
         }
     }
@@ -241,7 +241,7 @@ pub async fn process_events(
     slot_queue_sender: async_channel::Sender<SlotUpdate>,
     metrics_sender: metrics::Metrics,
 ) {
-    // Subscribe to accountsdb
+    // Subscribe to geyser
     let (msg_sender, msg_receiver) =
         async_channel::bounded::<Message>(config.postgres_target.account_write_max_queue_size);
     for grpc_source in config.grpc_sources {
@@ -263,7 +263,7 @@ pub async fn process_events(
             // Continuously reconnect on failure
             loop {
                 metric_status.set("connected".into());
-                let out = feed_data_accountsdb(
+                let out = feed_data_geyser(
                     &grpc_source,
                     tls_config.clone(),
                     &snapshot_source,
@@ -273,7 +273,7 @@ pub async fn process_events(
                 assert!(result.is_err());
                 if let Err(err) = result {
                     warn!(
-                        "error during communication with the accountsdb plugin. retrying. {:?}",
+                        "error during communication with the geyser plugin. retrying. {:?}",
                         err
                     );
                 }
@@ -313,7 +313,7 @@ pub async fn process_events(
         match msg {
             Message::GrpcUpdate(update) => {
                 match update.update_oneof.expect("invalid grpc") {
-                    accountsdb_proto::update::UpdateOneof::AccountWrite(update) => {
+                    geyser_proto::update::UpdateOneof::AccountWrite(update) => {
                         assert!(update.pubkey.len() == 32);
                         assert!(update.owner.len() == 32);
 
@@ -345,11 +345,11 @@ pub async fn process_events(
                             .await
                             .expect("send success");
                     }
-                    accountsdb_proto::update::UpdateOneof::SlotUpdate(update) => {
+                    geyser_proto::update::UpdateOneof::SlotUpdate(update) => {
                         metric_slot_updates.increment();
                         metric_slot_queue.set(slot_queue_sender.len() as u64);
 
-                        use accountsdb_proto::slot_update::Status;
+                        use geyser_proto::slot_update::Status;
                         let status = Status::from_i32(update.status).map(|v| match v {
                             Status::Processed => SlotStatus::Processed,
                             Status::Confirmed => SlotStatus::Confirmed,
@@ -370,8 +370,8 @@ pub async fn process_events(
                             .await
                             .expect("send success");
                     }
-                    accountsdb_proto::update::UpdateOneof::Ping(_) => {}
+                    geyser_proto::update::UpdateOneof::Ping(_) => {}
-                    accountsdb_proto::update::UpdateOneof::SubscribeResponse(_) => {}
+                    geyser_proto::update::UpdateOneof::SubscribeResponse(_) => {}
                 }
             }
             Message::Snapshot(update) => {

lib/src/websocket_source.rs

@@ -7,7 +7,9 @@ use solana_client::{
     //rpc_filter::RpcFilterType,
     rpc_response::{Response, RpcKeyedAccount},
 };
-use solana_rpc::{rpc::rpc_full::FullClient, rpc::OptionalContext, rpc_pubsub::RpcSolPubSubClient};
+use solana_rpc::{
+    rpc::rpc_accounts::AccountsDataClient, rpc::OptionalContext, rpc_pubsub::RpcSolPubSubClient,
+};
 use solana_sdk::{account::Account, commitment_config::CommitmentConfig, pubkey::Pubkey};
 
 use log::*;
@@ -36,10 +38,12 @@ async fn feed_data(
     let connect = ws::try_connect::<RpcSolPubSubClient>(&config.rpc_ws_url).map_err_anyhow()?;
     let client = connect.await.map_err_anyhow()?;
 
-    let rpc_client =
+    let rpc_client = http::connect_with_options::<AccountsDataClient>(
-        http::connect_with_options::<FullClient>(&config.snapshot_source.rpc_http_url, true)
+        &config.snapshot_source.rpc_http_url,
-            .await
+        true,
-            .map_err_anyhow()?;
+    )
+    .await
+    .map_err_anyhow()?;
 
     let account_info_config = RpcAccountInfoConfig {
         encoding: Some(UiAccountEncoding::Base64),

proto/geyser.proto

@@ -1,10 +1,10 @@
 syntax = "proto3";
 
 option java_multiple_files = true;
-option java_package = "mango.v3.accountsdb";
+option java_package = "mango.v3.geyser";
-option java_outer_classname = "AccountsDbProto";
+option java_outer_classname = "GeyserProto";
 
-package accountsdb;
+package geyser;
 
 service AccountsDb {
   rpc Subscribe(SubscribeRequest) returns (stream Update) {}
@@ -55,4 +55,4 @@ message Ping {
 
 message SubscribeResponse {
   uint64 highest_write_slot = 1;
-}
+}