From e7777281d602c3bed994dbae1a2f5880268e3a3d Mon Sep 17 00:00:00 2001 From: Jeff Biseda Date: Thu, 20 Jan 2022 12:38:42 -0800 Subject: [PATCH] regularly report network limits (#22563) --- Cargo.lock | 2 +- core/Cargo.toml | 3 + core/src/system_monitor_service.rs | 143 +++++++++++++++++++++++++---- validator/Cargo.toml | 1 - validator/src/main.rs | 87 ++---------------- 5 files changed, 135 insertions(+), 101 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 66cfc94add..cfd46b0388 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4751,6 +4751,7 @@ dependencies = [ "solana-vote-program", "static_assertions", "sys-info", + "sysctl", "systemstat", "tempfile", "thiserror", @@ -6022,7 +6023,6 @@ dependencies = [ "solana-version", "solana-vote-program", "symlink", - "sysctl", "tikv-jemallocator", ] diff --git a/core/Cargo.toml b/core/Cargo.toml index 1d9fb5bd08..5810faea29 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -80,6 +80,9 @@ solana-version = { path = "../version", version = "=1.10.0" } static_assertions = "1.1.0" systemstat = "0.1.10" +[target."cfg(unix)".dependencies] +sysctl = "0.4.3" + [build-dependencies] rustc_version = "0.4" diff --git a/core/src/system_monitor_service.rs b/core/src/system_monitor_service.rs index e610d49e82..75d3e41c6f 100644 --- a/core/src/system_monitor_service.rs +++ b/core/src/system_monitor_service.rs @@ -15,7 +15,10 @@ use { }; const MS_PER_S: u64 = 1_000; +const MS_PER_M: u64 = MS_PER_S * 60; +const MS_PER_H: u64 = MS_PER_M * 60; const SAMPLE_INTERVAL_UDP_MS: u64 = 2 * MS_PER_S; +const SAMPLE_INTERVAL_OS_NETWORK_LIMITS_MS: u64 = MS_PER_H; const SAMPLE_INTERVAL_MEM_MS: u64 = MS_PER_S; const SLEEP_INTERVAL: Duration = Duration::from_millis(500); @@ -38,6 +41,30 @@ struct UdpStats { ignored_multi: usize, } +impl UdpStats { + fn from_map(udp_stats: &HashMap) -> Self { + Self { + in_datagrams: *udp_stats.get("InDatagrams").unwrap_or(&0), + no_ports: *udp_stats.get("NoPorts").unwrap_or(&0), + in_errors: *udp_stats.get("InErrors").unwrap_or(&0), + out_datagrams: *udp_stats.get("OutDatagrams").unwrap_or(&0), + rcvbuf_errors: *udp_stats.get("RcvbufErrors").unwrap_or(&0), + sndbuf_errors: *udp_stats.get("SndbufErrors").unwrap_or(&0), + in_csum_errors: *udp_stats.get("InCsumErrors").unwrap_or(&0), + ignored_multi: *udp_stats.get("IgnoredMulti").unwrap_or(&0), + } + } +} + +fn platform_id() -> String { + format!( + "{}/{}/{}", + std::env::consts::FAMILY, + std::env::consts::OS, + std::env::consts::ARCH + ) +} + #[cfg(target_os = "linux")] fn read_udp_stats(file_path: impl AsRef) -> Result { let file = File::open(file_path).map_err(|e| e.to_string())?; @@ -73,16 +100,7 @@ fn parse_udp_stats(reader: &mut impl BufRead) -> Result { .map(|(label, val)| (label.to_string(), val.parse::().unwrap())) .collect(); - let stats = UdpStats { - in_datagrams: *udp_stats.get("InDatagrams").unwrap_or(&0), - no_ports: *udp_stats.get("NoPorts").unwrap_or(&0), - in_errors: *udp_stats.get("InErrors").unwrap_or(&0), - out_datagrams: *udp_stats.get("OutDatagrams").unwrap_or(&0), - rcvbuf_errors: *udp_stats.get("RcvbufErrors").unwrap_or(&0), - sndbuf_errors: *udp_stats.get("SndbufErrors").unwrap_or(&0), - in_csum_errors: *udp_stats.get("InCsumErrors").unwrap_or(&0), - ignored_multi: *udp_stats.get("IgnoredMulti").unwrap_or(&0), - }; + let stats = UdpStats::from_map(&udp_stats); Ok(stats) } @@ -111,12 +129,98 @@ impl SystemMonitorService { Self { thread_hdl } } + #[cfg_attr(not(target_os = "linux"), allow(dead_code))] + fn linux_get_recommended_network_limits() -> HashMap<&'static str, i64> { + // Reference: https://medium.com/@CameronSparr/increase-os-udp-buffers-to-improve-performance-51d167bb1360 + let mut recommended_limits: HashMap<&str, i64> = HashMap::default(); + recommended_limits.insert("net.core.rmem_max", 134217728); + recommended_limits.insert("net.core.rmem_default", 134217728); + recommended_limits.insert("net.core.wmem_max", 134217728); + recommended_limits.insert("net.core.wmem_default", 134217728); + recommended_limits.insert("vm.max_map_count", 1000000); + + // Additionally collect the following limits + recommended_limits.insert("net.core.optmem_max", 0); + recommended_limits.insert("net.core.netdev_max_backlog", 0); + + recommended_limits + } + + #[cfg(target_os = "linux")] + fn linux_get_current_network_limits( + recommended_limits: &HashMap<&'static str, i64>, + ) -> HashMap<&'static str, i64> { + use sysctl::Sysctl; + + fn sysctl_read(name: &str) -> Result { + let ctl = sysctl::Ctl::new(name)?; + let val = ctl.value_string()?; + Ok(val) + } + + let mut current_limits: HashMap<&str, i64> = HashMap::default(); + for (key, _) in recommended_limits.iter() { + let current_val = match sysctl_read(key) { + Ok(val) => val.parse::().unwrap(), + Err(e) => { + error!("Failed to query value for {}: {}", key, e); + -1 + } + }; + current_limits.insert(key, current_val); + } + current_limits + } + + #[cfg_attr(not(target_os = "linux"), allow(dead_code))] + fn linux_report_network_limits( + current_limits: &HashMap<&str, i64>, + recommended_limits: &HashMap<&'static str, i64>, + ) -> bool { + let mut check_failed = false; + for (key, recommended_val) in recommended_limits.iter() { + let current_val = *current_limits.get(key).unwrap_or(&-1); + if current_val < *recommended_val { + datapoint_warn!("os-config", (key, current_val, i64)); + warn!( + " {}: recommended={} current={}, too small", + key, recommended_val, current_val + ); + check_failed = true; + } else { + datapoint_info!("os-config", (key, current_val, i64)); + info!( + " {}: recommended={} current={}", + key, recommended_val, current_val + ); + } + } + if check_failed { + datapoint_warn!("os-config", ("network_limit_test_failed", 1, i64)); + } + !check_failed + } + + #[cfg(not(target_os = "linux"))] + pub fn check_os_network_limits() -> bool { + datapoint_info!("os-config", ("platform", platform_id(), String)); + true + } + + #[cfg(target_os = "linux")] + pub fn check_os_network_limits() -> bool { + datapoint_info!("os-config", ("platform", platform_id(), String)); + let recommended_limits = Self::linux_get_recommended_network_limits(); + let current_limits = Self::linux_get_current_network_limits(&recommended_limits); + Self::linux_report_network_limits(¤t_limits, &recommended_limits) + } + #[cfg(target_os = "linux")] fn process_udp_stats(udp_stats: &mut Option) { match read_udp_stats(PROC_NET_SNMP_PATH) { Ok(new_stats) => { if let Some(old_stats) = udp_stats { - SystemMonitorService::report_udp_stats(old_stats, &new_stats); + Self::report_udp_stats(old_stats, &new_stats); } *udp_stats = Some(new_stats); } @@ -229,22 +333,25 @@ impl SystemMonitorService { pub fn run(exit: Arc, report_os_network_stats: bool) { let mut udp_stats = None; - + let network_limits_timer = AtomicInterval::default(); let udp_timer = AtomicInterval::default(); let mem_timer = AtomicInterval::default(); + loop { if exit.load(Ordering::Relaxed) { break; } - - if report_os_network_stats && udp_timer.should_update(SAMPLE_INTERVAL_UDP_MS) { - SystemMonitorService::process_udp_stats(&mut udp_stats); + if report_os_network_stats { + if network_limits_timer.should_update(SAMPLE_INTERVAL_OS_NETWORK_LIMITS_MS) { + Self::check_os_network_limits(); + } + if udp_timer.should_update(SAMPLE_INTERVAL_UDP_MS) { + Self::process_udp_stats(&mut udp_stats); + } } - if mem_timer.should_update(SAMPLE_INTERVAL_MEM_MS) { - SystemMonitorService::report_mem_stats(); + Self::report_mem_stats(); } - sleep(SLEEP_INTERVAL); } } diff --git a/validator/Cargo.toml b/validator/Cargo.toml index 97194e572b..941e4dac49 100644 --- a/validator/Cargo.toml +++ b/validator/Cargo.toml @@ -60,7 +60,6 @@ jemallocator = {package = "tikv-jemallocator", version = "0.4.1", features = ["u [target."cfg(unix)".dependencies] libc = "0.2.112" signal-hook = "0.3.13" -sysctl = "0.4.3" [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] diff --git a/validator/src/main.rs b/validator/src/main.rs index 3e2220da6e..0f5b9ebe66 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -23,6 +23,7 @@ use { }, solana_core::{ ledger_cleanup_service::{DEFAULT_MAX_LEDGER_SHREDS, DEFAULT_MIN_MAX_LEDGER_SHREDS}, + system_monitor_service::SystemMonitorService, tower_storage, tpu::DEFAULT_TPU_COALESCE_MS, validator::{is_snapshot_config_valid, Validator, ValidatorConfig, ValidatorStartProgress}, @@ -32,7 +33,6 @@ use { contact_info::ContactInfo, }, solana_ledger::blockstore_db::BlockstoreRecoveryMode, - solana_metrics::datapoint_info, solana_perf::recycler::enable_recycler_warming, solana_poh::poh_service, solana_replica_lib::accountsdb_repl_server::AccountsDbReplServiceConfig, @@ -411,85 +411,6 @@ fn get_cluster_shred_version(entrypoints: &[SocketAddr]) -> Option { None } -fn platform_id() -> String { - format!( - "{}/{}/{}", - std::env::consts::FAMILY, - std::env::consts::OS, - std::env::consts::ARCH - ) -} - -#[cfg(target_os = "linux")] -fn check_os_network_limits() { - use {solana_metrics::datapoint_warn, std::collections::HashMap, sysctl::Sysctl}; - - fn sysctl_read(name: &str) -> Result { - let ctl = sysctl::Ctl::new(name)?; - let val = ctl.value_string()?; - Ok(val) - } - let mut check_failed = false; - - info!("Testing OS network limits:"); - - // Reference: https://medium.com/@CameronSparr/increase-os-udp-buffers-to-improve-performance-51d167bb1360 - let mut recommended_limits: HashMap<&str, i64> = HashMap::default(); - recommended_limits.insert("net.core.rmem_max", 134217728); - recommended_limits.insert("net.core.rmem_default", 134217728); - recommended_limits.insert("net.core.wmem_max", 134217728); - recommended_limits.insert("net.core.wmem_default", 134217728); - recommended_limits.insert("vm.max_map_count", 1000000); - - // Additionally collect the following limits - recommended_limits.insert("net.core.optmem_max", 0); - recommended_limits.insert("net.core.netdev_max_backlog", 0); - - let mut current_limits: HashMap<&str, i64> = HashMap::default(); - for (key, _) in recommended_limits.iter() { - let current_val = match sysctl_read(key) { - Ok(val) => val.parse::().unwrap(), - Err(e) => { - error!("Failed to query value for {}: {}", key, e); - check_failed = true; - -1 - } - }; - current_limits.insert(key, current_val); - } - - for (key, recommended_val) in recommended_limits.iter() { - let current_val = *current_limits.get(key).unwrap(); - if current_val < *recommended_val { - datapoint_warn!("os-config", (key, current_val, i64)); - warn!( - " {}: recommended={} current={}, too small", - key, recommended_val, current_val - ); - check_failed = true; - } else { - datapoint_info!("os-config", (key, current_val, i64)); - info!( - " {}: recommended={} current={}", - key, recommended_val, current_val - ); - } - } - datapoint_info!("os-config", ("platform", platform_id(), String)); - - if check_failed { - datapoint_warn!("os-config", ("network_limit_test_failed", 1, i64)); - warn!("OS network limit test failed. solana-sys-tuner may be used to configure OS network limits. Bypass check with --no-os-network-limits-test."); - } else { - info!("OS network limits test passed."); - } -} - -#[cfg(not(target_os = "linux"))] -fn check_os_network_limits() { - datapoint_info!("os-config", ("platform", platform_id(), String)); -} - pub fn main() { let default_dynamic_port_range = &format!("{}-{}", VALIDATOR_PORT_RANGE.0, VALIDATOR_PORT_RANGE.1); @@ -2516,7 +2437,11 @@ pub fn main() { }); if !matches.is_present("no_os_network_limits_test") { - check_os_network_limits(); + if SystemMonitorService::check_os_network_limits() { + info!("OS network limits test passed."); + } else { + eprintln!("OS network limit test failed. solana-sys-tuner may be used to configure OS network limits. Bypass check with --no-os-network-limits-test."); + } } let mut ledger_lock = ledger_lockfile(&ledger_path);