Tune UDP rmem/wmem using sys-tuner daemon (#7273)

Cargo.lock

@@ -3969,6 +3969,7 @@ dependencies = [
  "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "solana-clap-utils 0.22.0",
  "solana-logger 0.22.0",
+ "sysctl 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "unix_socket2 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "users 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
@@ -4628,6 +4629,18 @@ dependencies = [
  "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "sysctl"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "walkdir 2.2.9 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "take_mut"
 version = "0.2.2"
@@ -5805,6 +5818,7 @@ dependencies = [
 "checksum synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a393066ed9010ebaed60b9eafa373d4b1baac186dd7e008555b0f702b51945b6"
 "checksum synstructure 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)" = "02353edf96d6e4dc81aea2d8490a7e9db177bf8acb0e951c24940bf866cb313f"
 "checksum sys-info 0.5.8 (registry+https://github.com/rust-lang/crates.io-index)" = "0079fe39cec2c8215e21b0bc4ccec9031004c160b88358f531b601e96b77f0df"
+"checksum sysctl 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0501f0d0c2aa64b419abff97c209f4b82c4e67caa63e8dc5b222ecc1b574cb5c"
 "checksum take_mut 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60"
 "checksum tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)" = "b3196bfbffbba3e57481b6ea32249fbaf590396a52505a2615adbb79d9d826d3"
 "checksum tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8"

net/remote/remote-node.sh

@@ -126,9 +126,6 @@ cat >> ~/solana/on-reboot <<EOF
   PATH="$HOME"/.cargo/bin:"$PATH"
   export USE_INSTALL=1
 
-  # shellcheck source=/dev/null
-  SUDO_OK=1 source scripts/tune-system.sh
-
   sudo RUST_LOG=info ~solana/.cargo/bin/solana-sys-tuner --user $(whoami) > sys-tuner.log 2>&1 &
   echo \$! > sys-tuner.pid
 

sys-tuner/Cargo.toml

@@ -21,6 +21,7 @@ solana-logger = { path = "../logger", version = "0.22.0" }
 unix_socket2 = "0.5.4"
 users = "0.9.1"
 nix = "0.16.0"
+sysctl = "0.4.0"
 
 [lib]
 name = "solana_sys_tuner"

sys-tuner/src/main.rs

@@ -2,7 +2,7 @@ use clap::{crate_description, crate_name, value_t_or_exit, App, Arg};
 use log::*;
 
 #[cfg(target_os = "linux")]
-fn tune_system(uid: u32) {
+fn tune_poh_service_priority(uid: u32) {
     fn find_pid<P: AsRef<std::path::Path>, F>(
         name: &str,
         path: P,
@@ -64,6 +64,34 @@ fn tune_system(uid: u32) {
     }
 }
 
+#[cfg(target_os = "linux")]
+fn tune_kernel_udp_buffers() {
+    use sysctl::CtlValue::String;
+    use sysctl::Sysctl;
+    fn sysctl_write(name: &str, value: &str) {
+        if let Ok(ctl) = sysctl::Ctl::new(name) {
+            info!("Old {} value {:?}", name, ctl.value());
+            let ctl_value = String(value.to_string());
+            match ctl.set_value(String(value.to_string())) {
+                Ok(v) if v == ctl_value => info!("Updated {} to {:?}", name, ctl_value),
+                Ok(v) => info!(
+                    "Update returned success but {} was set to {:?}, instead of {:?}",
+                    name, v, ctl_value
+                ),
+                Err(e) => error!("Failed to set {} to {:?}. Err {:?}", name, ctl_value, e),
+            }
+        } else {
+            error!("Failed to find sysctl {}", name);
+        }
+    }
+
+    // Reference: https://medium.com/@CameronSparr/increase-os-udp-buffers-to-improve-performance-51d167bb1360
+    sysctl_write("net.core.rmem_max", "134217728");
+    sysctl_write("net.core.rmem_default", "134217728");
+    sysctl_write("net.core.wmem_max", "134217728");
+    sysctl_write("net.core.wmem_default", "134217728");
+}
+
 #[cfg(unix)]
 fn main() {
     solana_logger::setup();
@@ -115,7 +143,10 @@ fn main() {
         if stream.is_ok() {
             info!("Tuning the system now");
             #[cfg(target_os = "linux")]
-            tune_system(peer_uid);
+            {
+                tune_kernel_udp_buffers();
+                tune_poh_service_priority(peer_uid);
+            }
         }
     }