report number of open files

bucket_map/src/bucket_map.rs

@@ -210,7 +210,7 @@ pub fn get_mmap_count() -> Option<usize> {
     None
 }
 
-/// Utility function to get open_fd limits
+/// Utility function to get open files limits
 #[cfg(target_os = "linux")]
 pub fn get_open_fd_limits() -> Option<(usize, usize)> {
     let run = |cmd, args| -> Option<usize> {
@@ -237,7 +237,38 @@ pub fn get_open_fd_limits() -> Option<(usize, usize)> {
 }
 
 #[cfg(not(target_os = "linux"))]
-pub fn get_open_fd_limits() -> Option<(usize, usize, usize)> {
+pub fn get_open_fd_limits() -> Option<(usize, usize)> {
+    None
+}
+
+/// Utility function to get num of open file descriptors
+#[cfg(target_os = "linux")]
+pub fn get_num_open_fd() -> Option<usize> {
+    let pid = std::process::id();
+    let fd_path = format!("/proc/{}/fd", pid);
+    let cmd = format!("ls -l {} | wc -l", fd_path);
+
+    let output = std::process::Command::new("sh")
+        .args(["-c", &cmd])
+        .output()
+        .unwrap();
+    if output.status.success() {
+        let n: usize = std::str::from_utf8(&output.stdout)
+            .unwrap()
+            .split_whitespace()
+            .next()
+            .unwrap()
+            .parse()
+            .unwrap();
+
+        Some(n)
+    } else {
+        None
+    }
+}
+
+#[cfg(not(target_os = "linux"))]
+pub fn get_num_open_fd() -> Option<usize> {
     None
 }
 

bucket_map/src/bucket_storage.rs

@@ -1,6 +1,6 @@
 use {
     crate::{
-        bucket_map::{get_mmap_count, get_open_fd_limits},
+        bucket_map::{get_mmap_count, get_num_open_fd, get_open_fd_limits},
         bucket_stats::BucketStats,
         MaxSearch,
     },
@@ -288,6 +288,10 @@ impl BucketStorage {
                     .map(|mmap_count| format!("current mmap_count: {}", mmap_count))
                     .unwrap_or_default();
 
+                let open_fd_msg = get_num_open_fd()
+                    .map(|open_fd| format!("current open_fd: {}", open_fd))
+                    .unwrap_or_default();
+
                 let limit_msg = get_open_fd_limits()
                     .map(|(soft_limit, hard_limit)| {
                         format!("soft_limit: {}, hard_limit: {}", soft_limit, hard_limit,)
@@ -295,10 +299,11 @@ impl BucketStorage {
                     .unwrap_or_default();
 
                 panic!(
-                    "Unable to create data file {} in current dir({:?}): {:?}. {}, {}",
+                    "Unable to create data file {} in current dir({:?}): {:?}. {}, {}, {}",
                     file.display(),
                     std::env::current_dir(),
                     e,
+                    open_fd_msg,
                     mmap_msg,
                     limit_msg,
                 );
@@ -432,9 +437,11 @@ mod test {
 
         // test get_mmap_fd_stats
         let mmap_count = get_mmap_count().unwrap();
+        let open_fd = get_num_open_fd().unwrap();
         let (soft_limit, hard_limit) = get_open_fd_limits().unwrap();
 
         assert!(mmap_count > 0);
+        assert!(open_fd > 0);
         assert!(soft_limit > 0);
         assert!(hard_limit > 0);
     }

core/src/system_monitor_service.rs

@@ -7,7 +7,7 @@ use num_enum::{IntoPrimitive, TryFromPrimitive};
 #[cfg(target_os = "linux")]
 use std::{fs::File, io::BufReader};
 use {
-    solana_bucket_map::bucket_map::get_mmap_count,
+    solana_bucket_map::bucket_map::{get_mmap_count, get_num_open_fd},
     solana_sdk::timing::AtomicInterval,
     std::{
         collections::HashMap,
@@ -835,10 +835,13 @@ impl SystemMonitorService {
     #[cfg(target_os = "linux")]
     fn report_open_fd_stats() {
         if let Some(curr_mmap_count) = get_mmap_count() {
-            datapoint_info!(
+            if let Some(curr_open_fd) = get_num_open_fd() {
-                "open-mmap-stats",
+                datapoint_info!(
-                ("number_mmap_files", curr_mmap_count, i64),
+                    "open-mmap-stats",
-            );
+                    ("number_mmap_files", curr_mmap_count, i64),
+                    ("number_open_fd", curr_open_fd, i64),
+                );
+            }
         }
     }