Tool to tune system parameters like PoH service priority (#7155)

* New daemon to tune system parameters like PoH service priority

* fixes for Linux

* integrate with poh_service

* fixes

* address review comments

* remove `dead_code` directive

Cargo.lock

@@ -1865,6 +1865,18 @@ dependencies = [
  "void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "nix"
+version = "0.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "cc 1.0.47 (registry+https://github.com/rust-lang/crates.io-index)",
+ "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "nodrop"
 version = "0.1.13"
@@ -3351,6 +3363,7 @@ dependencies = [
  "solana-sdk 0.22.0",
  "solana-stake-program 0.22.0",
  "solana-storage-program 0.22.0",
+ "solana-sys-tuner 0.22.0",
  "solana-vote-program 0.22.0",
  "solana-vote-signer 0.22.0",
  "symlink 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -3960,6 +3973,19 @@ dependencies = [
  "solana-storage-program 0.22.0",
 ]
 
+[[package]]
+name = "solana-sys-tuner"
+version = "0.22.0"
+dependencies = [
+ "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "nix 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "solana-logger 0.22.0",
+ "unix_socket2 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "users 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "solana-upload-perf"
 version = "0.22.0"
@@ -5073,6 +5099,14 @@ name = "unicode-xid"
 version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "unix_socket2"
+version = "0.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "unsigned-varint"
 version = "0.2.2"
@@ -5103,6 +5137,14 @@ dependencies = [
  "percent-encoding 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "users"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "utf8-ranges"
 version = "0.1.3"
@@ -5600,6 +5642,7 @@ dependencies = [
 "checksum nibble_vec 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "c8d77f3db4bce033f4d04db08079b2ef1c3d02b44e86f25d08886fafa7756ffa"
 "checksum nix 0.14.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6c722bee1037d430d0f8e687bbdbf222f27cc6e4e68d5caf630857bb2b6dbdce"
 "checksum nix 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3b2e0b4f3320ed72aaedb9a5ac838690a8047c7b275da22711fddff4f8a14229"
+"checksum nix 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)" = "19a8300bf427d432716764070ff70d5b2b7801c958b9049686e6cbd8b06fad92"
 "checksum nodrop 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "2f9667ddcc6cc8a43afc9b7917599d7216aa09c463919ea32c59ed6cac8bc945"
 "checksum nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2ad2a91a8e869eeb30b9cb3119ae87773a8f4ae617f41b1eb9c154b2905f7bd6"
 "checksum num-derive 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "eafd0b45c5537c3ba526f79d3e75120036502bebacbb3f3220914067ce39dbf2"
@@ -5824,10 +5867,12 @@ dependencies = [
 "checksum unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f860d7d29cf02cb2f3f359fd35991af3d30bac52c57d265a3c461074cb4dc"
 "checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc"
 "checksum unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c"
+"checksum unix_socket2 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)" = "b57c6eace16c00eccb98a28e85db3370eab0685bdd5e13831d59e2bcb49a1d8a"
 "checksum unsigned-varint 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "2c64cdf40b4a9645534a943668681bcb219faf51874d4b65d2e0abda1b10a2ab"
 "checksum untrusted 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "60369ef7a31de49bcb3f6ca728d4ba7300d9a1658f94c727d4cab8c8d9f4aece"
 "checksum url 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dd4e7c0d531266369519a4aa4f399d748bd37043b00bde1e4ff1f60a120b355a"
 "checksum url 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "75b414f6c464c879d7f9babf951f23bc3743fb7313c081b2e6ca719067ea9d61"
+"checksum users 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c72f4267aea0c3ec6d07eaabea6ead7c5ddacfafc5e22bcf8d186706851fb4cf"
 "checksum utf8-ranges 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a1ca13c08c41c9c3e04224ed9ff80461d97e121589ff27c753a16cb10830ae0f"
 "checksum uuid 0.7.4 (registry+https://github.com/rust-lang/crates.io-index)" = "90dbc611eb48397705a6b0f6e917da23ae517e4d127123d2cf7674206627d32a"
 "checksum vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a"

Cargo.toml

@@ -44,6 +44,7 @@ members = [
     "runtime",
     "sdk",
     "sdk-c",
+    "sys-tuner",
     "upload-perf",
     "net-utils",
     "fixed-buf",

core/Cargo.toml

@@ -60,6 +60,7 @@ solana-stake-program = { path = "../programs/stake", version = "0.22.0" }
 solana-storage-program = { path = "../programs/storage", version = "0.22.0" }
 solana-vote-program = { path = "../programs/vote", version = "0.22.0" }
 solana-vote-signer = { path = "../vote-signer", version = "0.22.0" }
+solana-sys-tuner = { path = "../sys-tuner", version = "0.22.0" }
 symlink = "0.1.0"
 sys-info = "0.5.8"
 tempfile = "3.1.0"

core/src/poh_service.rs

@@ -3,6 +3,7 @@
 use crate::poh_recorder::PohRecorder;
 use core_affinity;
 use solana_sdk::poh_config::PohConfig;
+use solana_sys_tuner;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, Mutex};
 use std::thread::{self, sleep, Builder, JoinHandle};
@@ -30,6 +31,7 @@ impl PohService {
         let tick_producer = Builder::new()
             .name("solana-poh-service-tick_producer".to_string())
             .spawn(move || {
+                solana_sys_tuner::request_realtime_poh();
                 if poh_config.hashes_per_tick.is_none() {
                     if poh_config.target_tick_count.is_none() {
                         Self::sleepy_tick_producer(poh_recorder, &poh_config, &poh_exit_);

net/remote/remote-node.sh

@@ -129,6 +129,9 @@ cat >> ~/solana/on-reboot <<EOF
   # shellcheck source=/dev/null
   SUDO_OK=1 source scripts/tune-system.sh
 
+  sudo RUST_LOG=info ~solana/.cargo/bin/solana-sys-tuner > sys-tuner.log 2>&1 &
+  echo \$! > sys-tuner.pid
+
   (
     sudo SOLANA_METRICS_CONFIG="$SOLANA_METRICS_CONFIG" scripts/oom-monitor.sh
   ) > oom-monitor.log 2>&1 &

scripts/cargo-install-all.sh

@@ -82,6 +82,7 @@ BINS=(
   solana-log-analyzer
   solana-net-shaper
   solana-archiver
+  solana-sys-tuner
   solana-validator
 )
 

sys-tuner/Cargo.toml

@@ -0,0 +1,28 @@
+[package]
+authors = ["Solana Maintainers <maintainers@solana.com>"]
+edition = "2018"
+name = "solana-sys-tuner"
+description = "The solana cluster system tuner daemon"
+version = "0.22.0"
+repository = "https://github.com/solana-labs/solana"
+license = "Apache-2.0"
+homepage = "https://solana.com/"
+publish = true
+
+[dependencies]
+clap = "2.33.0"
+log = "0.4.8"
+semver = "0.9.0"
+solana-logger = { path = "../logger", version = "0.22.0" }
+
+[target."cfg(unix)".dependencies]
+unix_socket2 = "0.5.4"
+users = "0.9.1"
+nix = "0.16.0"
+
+[lib]
+name = "solana_sys_tuner"
+
+[[bin]]
+name = "solana-sys-tuner"
+path = "src/main.rs"

sys-tuner/src/lib.rs

@@ -0,0 +1,10 @@
+use log::*;
+use unix_socket::UnixStream;
+
+pub const SOLANA_SYS_TUNER_PATH: &str = "/tmp/solana-sys-tuner";
+
+pub fn request_realtime_poh() {
+    info!("Sending tuning request");
+    let status = UnixStream::connect(SOLANA_SYS_TUNER_PATH);
+    info!("Tuning request status {:?}", status);
+}

sys-tuner/src/main.rs

@@ -0,0 +1,113 @@
+use log::*;
+use std::{fs, io};
+
+use solana_sys_tuner::SOLANA_SYS_TUNER_PATH;
+
+#[cfg(unix)]
+use unix_socket::UnixListener;
+
+#[cfg(target_os = "linux")]
+use std::fs::DirEntry;
+#[cfg(target_os = "linux")]
+use std::path::Path;
+
+#[cfg(target_os = "linux")]
+fn find_pid<P: AsRef<Path>, F>(name: &str, path: P, processor: F) -> Option<u64>
+where
+    F: Fn(&DirEntry) -> Option<u64>,
+{
+    for entry in fs::read_dir(path).expect("Failed to read /proc folder") {
+        if let Ok(dir) = entry {
+            let mut path = dir.path();
+            path.push("comm");
+            if let Ok(comm) = fs::read_to_string(path.as_path()) {
+                if comm.starts_with(name) {
+                    if let Some(pid) = processor(&dir) {
+                        return Some(pid);
+                    }
+                }
+            }
+        }
+    }
+
+    None
+}
+
+#[cfg(target_os = "linux")]
+fn tune_system() {
+    use std::process::Command;
+    use std::str::from_utf8;
+
+    if let Some(pid) = find_pid("solana-validato", "/proc", |dir| {
+        let mut path = dir.path();
+        path.push("task");
+        find_pid("solana-poh-serv", path, |dir1| {
+            if let Ok(pid) = dir1.file_name().into_string() {
+                pid.parse::<u64>().ok()
+            } else {
+                None
+            }
+        })
+    }) {
+        info!("POH thread PID is {}", pid);
+        let pid = format!("{}", pid);
+        let output = Command::new("chrt")
+            .args(&["-r", "-p", "99", pid.as_str()])
+            .output()
+            .expect("Expected to set priority of thread");
+        if output.status.success() {
+            info!("Done setting thread priority");
+        } else {
+            error!("chrt stderr: {}", from_utf8(&output.stderr).unwrap_or("?"));
+        }
+    } else {
+        error!("Could not find pid for POH thread");
+    }
+}
+
+#[cfg(any(not(unix), target_os = "macos"))]
+fn tune_system() {}
+
+#[allow(dead_code)]
+#[cfg(target_os = "linux")]
+fn set_socket_permissions() {
+    if let Some(user) = users::get_user_by_name("solana") {
+        let uid = format!("{}", user.uid());
+        info!("UID for solana is {}", uid);
+        nix::unistd::chown(
+            SOLANA_SYS_TUNER_PATH,
+            Some(nix::unistd::Uid::from_raw(user.uid())),
+            None,
+        )
+        .expect("Expected to change UID of the socket file");
+    } else {
+        error!("Could not find UID for solana user");
+    }
+}
+
+#[cfg(any(not(unix), target_os = "macos"))]
+fn set_socket_permissions() {}
+
+fn main() {
+    solana_logger::setup();
+    if let Err(e) = fs::remove_file(SOLANA_SYS_TUNER_PATH) {
+        if e.kind() != io::ErrorKind::NotFound {
+            panic!("Failed to remove stale socket file: {:?}", e)
+        }
+    }
+
+    let listener =
+        UnixListener::bind(SOLANA_SYS_TUNER_PATH).expect("Failed to bind to the socket file");
+
+    set_socket_permissions();
+
+    info!("Waiting for tuning requests");
+    for stream in listener.incoming() {
+        if stream.is_ok() {
+            info!("Tuning the system now");
+            tune_system();
+        }
+    }
+
+    info!("exiting");
+}