From 5bf9a20d42f7665e989bb1a550854da24dd99768 Mon Sep 17 00:00:00 2001 From: Michael Vines Date: Fri, 14 Dec 2018 17:42:46 -0800 Subject: [PATCH] fullnode-config no longer depends on src/ --- Cargo.lock | 28 ++++++--------------- Cargo.toml | 1 - bench-streamer/Cargo.toml | 1 + bench-streamer/src/main.rs | 3 +-- fullnode-config/Cargo.toml | 9 +++++-- fullnode-config/src/lib.rs | 48 +++++++++++++++++++++++++++++++++++ fullnode-config/src/main.rs | 50 ++++++++++--------------------------- fullnode/Cargo.toml | 3 +++ fullnode/src/main.rs | 40 ++++++++++++++--------------- netutil/src/lib.rs | 10 ++++---- replicator/Cargo.toml | 1 + replicator/src/main.rs | 25 +++++++++---------- src/fullnode.rs | 32 ------------------------ 13 files changed, 118 insertions(+), 133 deletions(-) create mode 100644 fullnode-config/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 1d15750699..4377a009ef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -649,11 +649,6 @@ dependencies = [ "tokio-io 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "half" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - [[package]] name = "hashbrown" version = "0.1.7" @@ -1631,16 +1626,6 @@ dependencies = [ "serde_derive 1.0.82 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "serde_cbor" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "byteorder 1.2.7 (registry+https://github.com/rust-lang/crates.io-index)", - "half 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.82 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "serde_derive" version = "1.0.82" @@ -1756,7 +1741,6 @@ dependencies = [ "ring 0.13.5 (registry+https://github.com/rust-lang/crates.io-index)", "rocksdb 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.82 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_cbor 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.82 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)", "sha2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1882,11 +1866,14 @@ dependencies = [ "clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "rayon 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "reqwest 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)", "solana 0.11.0", "solana-drone 0.11.0", + "solana-fullnode-config 0.11.0", "solana-logger 0.11.0", "solana-metrics 0.11.0", + "solana-netutil 0.11.0", "solana-sdk 0.11.0", ] @@ -1897,10 +1884,12 @@ dependencies = [ "clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)", "dirs 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.82 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.82 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)", - "solana 0.11.0", - "solana-logger 0.11.0", + "solana-netutil 0.11.0", "solana-sdk 0.11.0", + "untrusted 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -2141,6 +2130,7 @@ dependencies = [ "dirs 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)", "solana 0.11.0", + "solana-fullnode-config 0.11.0", "solana-logger 0.11.0", "solana-sdk 0.11.0", ] @@ -2820,7 +2810,6 @@ dependencies = [ "checksum glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "8be18de09a56b60ed0edf84bc9df007e30040691af7acd1c41874faac5895bfb" "checksum globset 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4743617a7464bbda3c8aec8558ff2f9429047e025771037df561d383337ff865" "checksum h2 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "7dd33bafe2e6370e6c8eb0cf1b8c5f93390b90acde7e9b03723f166b28b648ed" -"checksum half 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9353c2a89d550b58fa0061d8ed8d002a7d8cdf2494eb0e432859bd3a9e543836" "checksum hashbrown 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "64b7d419d0622ae02fe5da6b9a5e1964b610a65bb37923b976aeebb6dbb8f86e" "checksum hex-literal 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4da5f0e01bd8a71a224a4eedecaacfcabda388dbb7a80faf04d3514287572d95" "checksum hex-literal-impl 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1d340b6514f232f6db1bd16db65302a5278a04fef9ce867cb932e7e5fa21130a" @@ -2932,7 +2921,6 @@ dependencies = [ "checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" "checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" "checksum serde 1.0.82 (registry+https://github.com/rust-lang/crates.io-index)" = "6fa52f19aee12441d5ad11c9a00459122bd8f98707cadf9778c540674f1935b6" -"checksum serde_cbor 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "45cd6d95391b16cd57e88b68be41d504183b7faae22030c0cc3b3f73dd57b2fd" "checksum serde_derive 1.0.82 (registry+https://github.com/rust-lang/crates.io-index)" = "96a7f9496ac65a2db5929afa087b54f8fc5008dcfbe48a8874ed20049b0d6154" "checksum serde_json 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)" = "c37ccd6be3ed1fdf419ee848f7c758eb31b054d7cd3ae3600e3bae0adf569811" "checksum serde_urlencoded 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d48f9f99cd749a2de71d29da5f948de7f2764cc5a9d7f3c97e3514d4ee6eabf2" diff --git a/Cargo.toml b/Cargo.toml index bfe7eb094c..ac1c795e60 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,7 +46,6 @@ reqwest = "0.9.0" ring = "0.13.2" rocksdb = "0.10.1" serde = "1.0.82" -serde_cbor = "0.9.0" serde_derive = "1.0.82" serde_json = "1.0.10" sha2 = "0.8.0" diff --git a/bench-streamer/Cargo.toml b/bench-streamer/Cargo.toml index ddb70097b8..a2e32f597f 100644 --- a/bench-streamer/Cargo.toml +++ b/bench-streamer/Cargo.toml @@ -11,6 +11,7 @@ homepage = "https://solana.com/" clap = "2.32.0" solana = { path = "..", version = "0.11.0" } solana-logger = { path = "../logger", version = "0.11.0" } +solana-netutil = { path = "../netutil", version = "0.11.0" } [features] cuda = [] diff --git a/bench-streamer/src/main.rs b/bench-streamer/src/main.rs index 4ec7921a8b..5651c8500f 100644 --- a/bench-streamer/src/main.rs +++ b/bench-streamer/src/main.rs @@ -1,5 +1,4 @@ use clap::{App, Arg}; -use solana::netutil::bind_to; use solana::packet::{Packet, SharedPackets, BLOB_SIZE, PACKET_DATA_SIZE}; use solana::result::Result; use solana::streamer::{receiver, PacketReceiver}; @@ -74,7 +73,7 @@ fn main() -> Result<()> { let mut read_channels = Vec::new(); let mut read_threads = Vec::new(); for _ in 0..num_sockets { - let read = bind_to(port, false).unwrap(); + let read = solana_netutil::bind_to(port, false).unwrap(); read.set_read_timeout(Some(Duration::new(1, 0))).unwrap(); addr = read.local_addr().unwrap(); diff --git a/fullnode-config/Cargo.toml b/fullnode-config/Cargo.toml index 3e93b20cda..a52a9bcdc5 100644 --- a/fullnode-config/Cargo.toml +++ b/fullnode-config/Cargo.toml @@ -11,10 +11,15 @@ homepage = "https://solana.com/" clap = "2.32.0" dirs = "1.0.2" log = "0.4.2" +serde = "1.0.82" +serde_derive = "1.0.82" serde_json = "1.0.10" -solana = { path = "..", version = "0.11.0" } -solana-logger = { path = "../logger", version = "0.11.0" } +solana-netutil = { path = "../netutil", version = "0.11.0" } solana-sdk = { path = "../sdk", version = "0.11.0" } +untrusted = "0.6.2" [features] cuda = [] + +[lib] +name = "solana_fullnode_config" diff --git a/fullnode-config/src/lib.rs b/fullnode-config/src/lib.rs new file mode 100644 index 0000000000..28e1f965df --- /dev/null +++ b/fullnode-config/src/lib.rs @@ -0,0 +1,48 @@ +use serde_derive::{Deserialize, Serialize}; +use solana_sdk::signature::Keypair; +use std::net::SocketAddr; +use untrusted::Input; + +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] +#[serde(rename_all = "PascalCase")] +pub struct Config { + /// Bind to port or address + pub bind_port_or_address: Option, + + /// Detect public network address using public servers + pub use_public_address: bool, + + /// Detect network address from local machine configuration + pub use_local_address: bool, + + /// Fullnode identity + pub identity_pkcs8: Vec, + + /// Fullnode vote account + pub vote_account_pkcs8: Vec, +} + +impl Config { + pub fn bind_addr(&self, default_port: u16) -> SocketAddr { + let mut bind_addr = + solana_netutil::parse_port_or_addr(&self.bind_port_or_address, default_port); + if self.use_local_address { + let ip = solana_netutil::get_ip_addr().unwrap(); + bind_addr.set_ip(ip); + } + if self.use_public_address { + let ip = solana_netutil::get_public_ip_addr().unwrap(); + bind_addr.set_ip(ip); + } + bind_addr + } + + pub fn keypair(&self) -> Keypair { + Keypair::from_pkcs8(Input::from(&self.identity_pkcs8)) + .expect("from_pkcs8 in fullnode::Config keypair") + } + pub fn vote_account_keypair(&self) -> Keypair { + Keypair::from_pkcs8(Input::from(&self.vote_account_pkcs8)) + .expect("from_pkcs8 in fullnode::Config vote_account_keypair") + } +} diff --git a/fullnode-config/src/main.rs b/fullnode-config/src/main.rs index a1f59490d5..ae8f26b63e 100644 --- a/fullnode-config/src/main.rs +++ b/fullnode-config/src/main.rs @@ -1,31 +1,18 @@ -#[macro_use] -extern crate clap; -use dirs; - -use serde_json; - -use clap::{App, Arg}; -use solana::cluster_info::FULLNODE_PORT_RANGE; -use solana::fullnode::Config; - -use solana::netutil::{get_ip_addr, get_public_ip_addr, parse_port_or_addr}; use solana_sdk::signature::{gen_pkcs8, read_pkcs8}; use std::io; -use std::net::SocketAddr; fn main() { - solana_logger::setup(); - let matches = App::new("fullnode-config") - .version(crate_version!()) + let matches = clap::App::new("fullnode-config") + .version(clap::crate_version!()) .arg( - Arg::with_name("local") + clap::Arg::with_name("local") .short("l") .long("local") .takes_value(false) .help("Detect network address from local machine configuration"), ) .arg( - Arg::with_name("keypair") + clap::Arg::with_name("keypair") .short("k") .long("keypair") .value_name("PATH") @@ -33,14 +20,14 @@ fn main() { .help("/path/to/id.json"), ) .arg( - Arg::with_name("public") + clap::Arg::with_name("public") .short("p") .long("public") .takes_value(false) .help("Detect public network address using public servers"), ) .arg( - Arg::with_name("bind") + clap::Arg::with_name("bind") .short("b") .long("bind") .value_name("PORT") @@ -49,19 +36,6 @@ fn main() { ) .get_matches(); - let bind_addr: SocketAddr = { - let mut bind_addr = parse_port_or_addr(matches.value_of("bind"), FULLNODE_PORT_RANGE.0); - if matches.is_present("local") { - let ip = get_ip_addr().unwrap(); - bind_addr.set_ip(ip); - } - if matches.is_present("public") { - let ip = get_public_ip_addr().unwrap(); - bind_addr.set_ip(ip); - } - bind_addr - }; - let mut path = dirs::home_dir().expect("home directory"); let id_path = if matches.is_present("keypair") { matches.value_of("keypair").unwrap() @@ -69,13 +43,15 @@ fn main() { path.extend(&[".config", "solana", "id.json"]); path.to_str().unwrap() }; - let pkcs8 = read_pkcs8(id_path).expect("client keypair"); - let vote_account_pkcs8 = gen_pkcs8().unwrap(); + let config = solana_fullnode_config::Config { + bind_port_or_address: matches.value_of("bind").map(|s| s.to_string()), + use_local_address: matches.is_present("local"), + use_public_address: matches.is_present("public"), + identity_pkcs8: read_pkcs8(id_path).expect("invalid keypair"), + vote_account_pkcs8: gen_pkcs8().unwrap(), + }; - // we need all the receiving sockets to be bound within the expected - // port range that we open on aws - let config = Config::new(&bind_addr, pkcs8, vote_account_pkcs8); let stdout = io::stdout(); serde_json::to_writer(stdout, &config).expect("serialize"); } diff --git a/fullnode/Cargo.toml b/fullnode/Cargo.toml index 103d84a3d4..14adcb19e4 100644 --- a/fullnode/Cargo.toml +++ b/fullnode/Cargo.toml @@ -11,10 +11,13 @@ homepage = "https://solana.com/" clap = "2.32.0" log = "0.4.2" rayon = "1.0.3" +reqwest = "0.9.0" serde_json = "1.0.10" solana = { path = "..", version = "0.11.0" } solana-drone = { path = "../drone", version = "0.11.0" } +solana-fullnode-config = { path = "../fullnode-config", version = "0.11.0" } solana-logger = { path = "../logger", version = "0.11.0" } +solana-netutil = { path = "../netutil", version = "0.11.0" } solana-metrics = { path = "../metrics", version = "0.11.0" } solana-sdk = { path = "../sdk", version = "0.11.0" } diff --git a/fullnode/src/main.rs b/fullnode/src/main.rs index f4e9b31f22..4bb3d3514b 100644 --- a/fullnode/src/main.rs +++ b/fullnode/src/main.rs @@ -1,20 +1,10 @@ -#[macro_use] -extern crate clap; - -#[macro_use] -extern crate log; -use serde_json; -#[macro_use] -extern crate solana; -use solana_metrics; - -use clap::{App, Arg}; +use clap::{crate_version, App, Arg}; +use log::*; use solana::client::mk_client; -use solana::cluster_info::{Node, FULLNODE_PORT_RANGE}; -use solana::fullnode::{Config, Fullnode, FullnodeReturnType}; +use solana::cluster_info::{Node, NodeInfo, FULLNODE_PORT_RANGE}; +use solana::fullnode::{Fullnode, FullnodeReturnType}; use solana::leader_scheduler::LeaderScheduler; - -use solana::netutil::find_available_port_in_range; +use solana::socketaddr; use solana::thin_client::poll_gossip_for_leader; use solana_sdk::signature::{Keypair, KeypairUtil}; use solana_sdk::vote_program::VoteProgram; @@ -82,12 +72,20 @@ fn main() { let (keypair, vote_account_keypair, gossip) = if let Some(i) = matches.value_of("identity") { let path = i.to_string(); if let Ok(file) = File::open(path.clone()) { - let parse: serde_json::Result = serde_json::from_reader(file); - if let Ok(data) = parse { + let parse: serde_json::Result = + serde_json::from_reader(file); + + if let Ok(config_data) = parse { + let keypair = config_data.keypair(); + let node_info = NodeInfo::new_with_pubkey_socketaddr( + keypair.pubkey(), + &config_data.bind_addr(FULLNODE_PORT_RANGE.0), + ); + ( - data.keypair(), - data.vote_account_keypair(), - data.node_info.gossip, + keypair, + config_data.vote_account_keypair(), + node_info.gossip, ) } else { eprintln!("failed to parse {}", path); @@ -131,7 +129,7 @@ fn main() { } Some(port_number) } else { - match find_available_port_in_range(FULLNODE_PORT_RANGE) { + match solana_netutil::find_available_port_in_range(FULLNODE_PORT_RANGE) { Ok(port) => Some(port), Err(_) => None, } diff --git a/netutil/src/lib.rs b/netutil/src/lib.rs index afa5223754..99f1ad372d 100644 --- a/netutil/src/lib.rs +++ b/netutil/src/lib.rs @@ -30,7 +30,7 @@ pub fn get_public_ip_addr() -> Result { } } -pub fn parse_port_or_addr(optstr: Option<&str>, default_port: u16) -> SocketAddr { +pub fn parse_port_or_addr(optstr: &Option, default_port: u16) -> SocketAddr { let daddr = SocketAddr::from(([0, 0, 0, 0], default_port)); if let Some(addrstr) = optstr { @@ -280,13 +280,13 @@ mod tests { #[test] fn test_parse_port_or_addr() { - let p1 = parse_port_or_addr(Some("9000"), 1); + let p1 = parse_port_or_addr(&Some("9000".to_string()), 1); assert_eq!(p1.port(), 9000); - let p2 = parse_port_or_addr(Some("127.0.0.1:7000"), 1); + let p2 = parse_port_or_addr(&Some("127.0.0.1:7000".to_string()), 1); assert_eq!(p2.port(), 7000); - let p2 = parse_port_or_addr(Some("hi there"), 1); + let p2 = parse_port_or_addr(&Some("hi there".to_string()), 1); assert_eq!(p2.port(), 1); - let p3 = parse_port_or_addr(None, 1); + let p3 = parse_port_or_addr(&None, 1); assert_eq!(p3.port(), 1); } diff --git a/replicator/Cargo.toml b/replicator/Cargo.toml index 47fb7bae8e..effb156acb 100644 --- a/replicator/Cargo.toml +++ b/replicator/Cargo.toml @@ -13,6 +13,7 @@ dirs = "1.0.2" serde_json = "1.0.10" solana = { path = "..", version = "0.11.0" } solana-logger = { path = "../logger", version = "0.11.0" } +solana-fullnode-config = { path = "../fullnode-config", version = "0.11.0" } solana-sdk = { path = "../sdk", version = "0.11.0" } [features] diff --git a/replicator/src/main.rs b/replicator/src/main.rs index bbbc831d49..778f06073e 100644 --- a/replicator/src/main.rs +++ b/replicator/src/main.rs @@ -1,15 +1,8 @@ -#[macro_use] -extern crate clap; - +use clap::{crate_version, App, Arg}; use serde_json; -#[macro_use] -extern crate solana; - -use clap::{App, Arg}; -use solana::cluster_info::{Node, NodeInfo}; -use solana::fullnode::Config; - +use solana::cluster_info::{Node, NodeInfo, FULLNODE_PORT_RANGE}; use solana::replicator::Replicator; +use solana::socketaddr; use solana_sdk::signature::{Keypair, KeypairUtil}; use std::fs::File; use std::net::{Ipv4Addr, SocketAddr}; @@ -52,9 +45,15 @@ fn main() { let (keypair, gossip) = if let Some(i) = matches.value_of("identity") { let path = i.to_string(); if let Ok(file) = File::open(path.clone()) { - let parse: serde_json::Result = serde_json::from_reader(file); - if let Ok(data) = parse { - (data.keypair(), data.node_info.gossip) + let parse: serde_json::Result = + serde_json::from_reader(file); + if let Ok(config_data) = parse { + let keypair = config_data.keypair(); + let node_info = NodeInfo::new_with_pubkey_socketaddr( + keypair.pubkey(), + &config_data.bind_addr(FULLNODE_PORT_RANGE.0), + ); + (keypair, node_info.gossip) } else { eprintln!("failed to parse {}", path); exit(1); diff --git a/src/fullnode.rs b/src/fullnode.rs index 88e0bcfdf1..91d0e92eba 100644 --- a/src/fullnode.rs +++ b/src/fullnode.rs @@ -24,7 +24,6 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::{Arc, RwLock}; use std::thread::Result; -use untrusted::Input; pub enum NodeRole { Leader(LeaderServices), @@ -112,37 +111,6 @@ pub struct Fullnode { db_ledger: Arc>, } -#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] -#[serde(rename_all = "PascalCase")] -/// Fullnode configuration to be stored in file -pub struct Config { - pub node_info: NodeInfo, - pkcs8: Vec, - vote_account_pkcs8: Vec, -} - -impl Config { - pub fn new(bind_addr: &SocketAddr, pkcs8: Vec, vote_account_pkcs8: Vec) -> Self { - let keypair = - Keypair::from_pkcs8(Input::from(&pkcs8)).expect("from_pkcs8 in fullnode::Config new"); - let pubkey = keypair.pubkey(); - let node_info = NodeInfo::new_with_pubkey_socketaddr(pubkey, bind_addr); - Config { - node_info, - pkcs8, - vote_account_pkcs8, - } - } - pub fn keypair(&self) -> Keypair { - Keypair::from_pkcs8(Input::from(&self.pkcs8)) - .expect("from_pkcs8 in fullnode::Config keypair") - } - pub fn vote_account_keypair(&self) -> Keypair { - Keypair::from_pkcs8(Input::from(&self.vote_account_pkcs8)) - .expect("from_pkcs8 in fullnode::Config vote_account_keypair") - } -} - impl Fullnode { pub fn new( node: Node,