diff --git a/bucket_map/src/bucket_map.rs b/bucket_map/src/bucket_map.rs index 1b2fe2009..698ec18f3 100644 --- a/bucket_map/src/bucket_map.rs +++ b/bucket_map/src/bucket_map.rs @@ -210,7 +210,7 @@ pub fn get_mmap_count() -> Option { None } -/// Utility function to get open_fd limits +/// Utility function to get open files limits #[cfg(target_os = "linux")] pub fn get_open_fd_limits() -> Option<(usize, usize)> { let run = |cmd, args| -> Option { @@ -237,7 +237,38 @@ pub fn get_open_fd_limits() -> Option<(usize, usize)> { } #[cfg(not(target_os = "linux"))] -pub fn get_open_fd_limits() -> Option<(usize, usize, usize)> { +pub fn get_open_fd_limits() -> Option<(usize, usize)> { + None +} + +/// Utility function to get num of open file descriptors +#[cfg(target_os = "linux")] +pub fn get_num_open_fd() -> Option { + let pid = std::process::id(); + let fd_path = format!("/proc/{}/fd", pid); + let cmd = format!("ls -l {} | wc -l", fd_path); + + let output = std::process::Command::new("sh") + .args(["-c", &cmd]) + .output() + .unwrap(); + if output.status.success() { + let n: usize = std::str::from_utf8(&output.stdout) + .unwrap() + .split_whitespace() + .next() + .unwrap() + .parse() + .unwrap(); + + Some(n) + } else { + None + } +} + +#[cfg(not(target_os = "linux"))] +pub fn get_num_open_fd() -> Option { None } diff --git a/bucket_map/src/bucket_storage.rs b/bucket_map/src/bucket_storage.rs index bc6218ff3..a9e09e329 100644 --- a/bucket_map/src/bucket_storage.rs +++ b/bucket_map/src/bucket_storage.rs @@ -1,6 +1,6 @@ use { crate::{ - bucket_map::{get_mmap_count, get_open_fd_limits}, + bucket_map::{get_mmap_count, get_num_open_fd, get_open_fd_limits}, bucket_stats::BucketStats, MaxSearch, }, @@ -288,6 +288,10 @@ impl BucketStorage { .map(|mmap_count| format!("current mmap_count: {}", mmap_count)) .unwrap_or_default(); + let open_fd_msg = get_num_open_fd() + .map(|open_fd| format!("current open_fd: {}", open_fd)) + .unwrap_or_default(); + let limit_msg = get_open_fd_limits() .map(|(soft_limit, hard_limit)| { format!("soft_limit: {}, hard_limit: {}", soft_limit, hard_limit,) @@ -295,10 +299,11 @@ impl BucketStorage { .unwrap_or_default(); panic!( - "Unable to create data file {} in current dir({:?}): {:?}. {}, {}", + "Unable to create data file {} in current dir({:?}): {:?}. {}, {}, {}", file.display(), std::env::current_dir(), e, + open_fd_msg, mmap_msg, limit_msg, ); @@ -432,9 +437,11 @@ mod test { // test get_mmap_fd_stats let mmap_count = get_mmap_count().unwrap(); + let open_fd = get_num_open_fd().unwrap(); let (soft_limit, hard_limit) = get_open_fd_limits().unwrap(); assert!(mmap_count > 0); + assert!(open_fd > 0); assert!(soft_limit > 0); assert!(hard_limit > 0); } diff --git a/core/src/system_monitor_service.rs b/core/src/system_monitor_service.rs index 2748dc45c..42f0d5940 100644 --- a/core/src/system_monitor_service.rs +++ b/core/src/system_monitor_service.rs @@ -7,7 +7,7 @@ use num_enum::{IntoPrimitive, TryFromPrimitive}; #[cfg(target_os = "linux")] use std::{fs::File, io::BufReader}; use { - solana_bucket_map::bucket_map::get_mmap_count, + solana_bucket_map::bucket_map::{get_mmap_count, get_num_open_fd}, solana_sdk::timing::AtomicInterval, std::{ collections::HashMap, @@ -835,10 +835,13 @@ impl SystemMonitorService { #[cfg(target_os = "linux")] fn report_open_fd_stats() { if let Some(curr_mmap_count) = get_mmap_count() { - datapoint_info!( - "open-mmap-stats", - ("number_mmap_files", curr_mmap_count, i64), - ); + if let Some(curr_open_fd) = get_num_open_fd() { + datapoint_info!( + "open-mmap-stats", + ("number_mmap_files", curr_mmap_count, i64), + ("number_open_fd", curr_open_fd, i64), + ); + } } }