share code

Cargo.lock

@@ -5202,7 +5202,6 @@ dependencies = [
  "matches",
  "min-max-heap",
  "num_enum",
- "procfs",
  "rand 0.7.3",
  "rand_chacha 0.2.2",
  "raptorq",
@@ -5214,6 +5213,7 @@ dependencies = [
  "serial_test",
  "solana-address-lookup-table-program",
  "solana-bloom",
+ "solana-bucket-map",
  "solana-client",
  "solana-entry",
  "solana-frozen-abi 1.15.0",

bucket_map/src/bucket_map.rs

@@ -2,6 +2,7 @@
 
 use {
     crate::{bucket_api::BucketApi, bucket_stats::BucketMapStats, MaxSearch, RefCount},
+    procfs::process::{LimitValue, Process},
     solana_sdk::pubkey::Pubkey,
     std::{convert::TryInto, fmt::Debug, fs, path::PathBuf, sync::Arc},
     tempfile::TempDir,
@@ -181,6 +182,53 @@ fn read_be_u64(input: &[u8]) -> u64 {
     u64::from_be_bytes(input[0..std::mem::size_of::<u64>()].try_into().unwrap())
 }
 
+/// Utility function to get number of Mmap files for current process
+pub fn get_mmap_count() -> Option<usize> {
+    let pid = std::process::id();
+    let map_path = format!("/proc/{}/maps", pid);
+    let output = std::process::Command::new("wc")
+        .args(["-l", &map_path])
+        .output()
+        .unwrap();
+    if output.status.success() {
+        let n: usize = std::str::from_utf8(&output.stdout)
+            .unwrap()
+            .split_whitespace()
+            .next()
+            .unwrap()
+            .parse()
+            .unwrap();
+
+        Some(n)
+    } else {
+        None
+    }
+}
+
+/// Utility function to get open_fd stats
+pub fn get_open_fd_stats() -> Option<(usize, usize, usize, usize)> {
+    let proc = Process::myself().ok()?;
+    let curr_num_open_fd = proc.fd_count().unwrap();
+    let curr_mmap_count = get_mmap_count().unwrap();
+    let max_open_fd_limit = proc.limits().unwrap().max_open_files;
+
+    let max_open_fd_soft_limit = match max_open_fd_limit.soft_limit {
+        LimitValue::Unlimited => usize::MAX,
+        LimitValue::Value(x) => x as usize,
+    };
+    let max_open_fd_hard_limit = match max_open_fd_limit.hard_limit {
+        LimitValue::Unlimited => usize::MAX,
+        LimitValue::Value(x) => x as usize,
+    };
+
+    Some((
+        curr_num_open_fd,
+        curr_mmap_count,
+        max_open_fd_soft_limit,
+        max_open_fd_hard_limit,
+    ))
+}
+
 #[cfg(test)]
 mod tests {
     use {

bucket_map/src/bucket_storage.rs

@@ -1,7 +1,6 @@
 use {
-    crate::{bucket_stats::BucketStats, MaxSearch},
+    crate::{bucket_map::get_open_fd_stats, bucket_stats::BucketStats, MaxSearch},
     memmap2::MmapMut,
-    procfs::process::{Limit, Process},
     rand::{thread_rng, Rng},
     solana_measure::measure::Measure,
     std::{
@@ -263,37 +262,6 @@ impl BucketStorage {
         }
     }
 
-    fn get_mmap_count() -> Option<usize> {
-        let pid = std::process::id();
-        let map_path = format!("/proc/{}/maps", pid);
-        let output = std::process::Command::new("wc")
-            .args(["-l", &map_path])
-            .output()
-            .unwrap();
-        if output.status.success() {
-            let n: usize = std::str::from_utf8(&output.stdout)
-                .unwrap()
-                .split_whitespace()
-                .next()
-                .unwrap()
-                .parse()
-                .unwrap();
-
-            Some(n)
-        } else {
-            None
-        }
-    }
-
-    fn get_mmap_fd_stats() -> Option<(usize, usize, Limit)> {
-        let proc = Process::myself().ok()?;
-        let mmap_count = Self::get_mmap_count().unwrap();
-        let max_open_files_limit = proc.limits().unwrap().max_open_files;
-        let num_open_files = proc.fd_count().unwrap();
-
-        Some((mmap_count, num_open_files, max_open_files_limit))
-    }
-
     fn new_map(
         drives: &[PathBuf],
         cell_size: usize,
@@ -312,12 +280,13 @@ impl BucketStorage {
             .create(true)
             .open(file.clone())
             .map_err(|e| {
-                let mmap_msg = Self::get_mmap_fd_stats()
-                    .map(|(mmap_count, num_open_files, max_open_files_limit)| {
-                        format!("current mmap_count: {}, current number of open files: {}, max limit of open files: {:?}",
+                let mmap_msg = get_open_fd_stats()
+                    .map(|(mmap_count, num_open_files, soft_limit, hard_limit)| {
+                        format!("current mmap_count: {}, current number of open files: {}, soft_limit: {}, hard_limit: {}",
                                         mmap_count,
                                         num_open_files,
-                                        max_open_files_limit,
+                                        soft_limit, 
+                                        hard_limit, 
                         )
                     }).unwrap_or_default();
                 panic!(
@@ -491,8 +460,7 @@ mod test {
 
         // test get_mmap_fd_stats
         let start = Instant::now();
-        let (mmap_count, num_open_files, max_open_files_limit) =
-            BucketStorage::get_mmap_fd_stats().unwrap();
+        let (mmap_count, num_open_files, soft_limit, hard_limit) = get_open_fd_stats().unwrap();
         let duration = start.elapsed();
 
         println!(

core/Cargo.toml

@@ -31,13 +31,13 @@ log = "0.4.17"
 lru = "0.7.7"
 min-max-heap = "1.3.0"
 num_enum = "0.5.7"
-procfs = "0.14.1"
 rand = "0.7.0"
 rand_chacha = "0.2.2"
 rayon = "1.5.3"
 serde = "1.0.144"
 serde_derive = "1.0.103"
 solana-address-lookup-table-program = { path = "../programs/address-lookup-table", version = "=1.15.0" }
+solana-bucket-map = { path = "../bucket_map", version = "=1.15.0" }
 solana-bloom = { path = "../bloom", version = "=1.15.0" }
 solana-client = { path = "../client", version = "=1.15.0" }
 solana-entry = { path = "../entry", version = "=1.15.0" }

core/src/system_monitor_service.rs

@@ -7,7 +7,7 @@ use num_enum::{IntoPrimitive, TryFromPrimitive};
 #[cfg(target_os = "linux")]
 use std::{fs::File, io::BufReader};
 use {
-    procfs::process::{LimitValue, Process},
+    solana_bucket_map::bucket_map::get_open_fd_stats,
     solana_sdk::timing::AtomicInterval,
     std::{
         collections::HashMap,
@@ -832,58 +832,13 @@ impl SystemMonitorService {
         Self::report_cpuid_values();
     }
 
-    fn get_mmap_count() -> Option<usize> {
-        let pid = std::process::id();
-        let map_path = format!("/proc/{}/maps", pid);
-        let output = std::process::Command::new("wc")
-            .args(["-l", &map_path])
-            .output()
-            .unwrap();
-        if output.status.success() {
-            let n: usize = std::str::from_utf8(&output.stdout)
-                .unwrap()
-                .split_whitespace()
-                .next()
-                .unwrap()
-                .parse()
-                .unwrap();
-
-            Some(n)
-        } else {
-            None
-        }
-    }
-
-    fn get_open_fd_stats() -> Option<(usize, usize, usize, usize)> {
-        let proc = Process::myself().ok()?;
-        let curr_num_open_fd = proc.fd_count().unwrap();
-        let curr_mmap_count = Self::get_mmap_count().unwrap();
-        let max_open_fd_limit = proc.limits().unwrap().max_open_files;
-
-        let max_open_fd_soft_limit = match max_open_fd_limit.soft_limit {
-            LimitValue::Unlimited => usize::MAX,
-            LimitValue::Value(x) => x as usize,
-        };
-        let max_open_fd_hard_limit = match max_open_fd_limit.hard_limit {
-            LimitValue::Unlimited => usize::MAX,
-            LimitValue::Value(x) => x as usize,
-        };
-
-        Some((
-            curr_num_open_fd,
-            curr_mmap_count,
-            max_open_fd_soft_limit,
-            max_open_fd_hard_limit,
-        ))
-    }
-
     fn report_open_fd_stats() {
         if let Some((
             curr_num_open_fd,
             curr_mmap_count,
             max_open_fd_soft_limit,
             max_open_fd_hard_limit,
-        )) = Self::get_open_fd_stats()
+        )) = get_open_fd_stats()
         {
             datapoint_info!(
                 "open-fd-stats",