Validators now log to a file by default (use `-o -`/`--log -` for stderr) (#6768)

automerge

Cargo.lock

@@ -1110,6 +1110,15 @@ dependencies = [
  "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "gag"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "gcc"
 version = "0.3.55"
@@ -3960,8 +3969,10 @@ name = "solana-validator"
 version = "0.21.0"
 dependencies = [
  "bzip2 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "chrono 0.4.9 (registry+https://github.com/rust-lang/crates.io-index)",
  "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "console 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "gag 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
  "indicatif 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "reqwest 0.9.22 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -5499,6 +5510,7 @@ dependencies = [
 "checksum futures 0.1.28 (registry+https://github.com/rust-lang/crates.io-index)" = "45dc39533a6cae6da2b56da48edae506bb767ec07370f86f70fc062e9d435869"
 "checksum futures-cpupool 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "ab90cde24b3319636588d0c35fe03b1333857621051837ed769faefb4c2162e4"
 "checksum fxhash 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
+"checksum gag 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "8cc0b9f53275dc5fada808f1d2f82e3688a6c14d735633d1590b7be8eb2307b5"
 "checksum gcc 0.3.55 (registry+https://github.com/rust-lang/crates.io-index)" = "8f5f3913fa0bfe7ee1fd8248b6b9f42a5af4b9d65ec2dd2c3c26132b950ecfc2"
 "checksum generic-array 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c68f0274ae0e023facc3c97b2e00f076be70e254bc851d972503b328db79b2ec"
 "checksum generic-array 0.13.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0ed1e761351b56f54eb9dcd0cfaca9fd0daecf93918e1cfc01c8a3d26ee7adcd"

multinode-demo/bootstrap-leader.sh

@@ -37,6 +37,9 @@ while [[ -n $1 ]]; do
     elif [[ $1 = --skip-poh-verify ]]; then
       args+=("$1")
       shift
+    elif [[ $1 = --log ]]; then
+      args+=("$1" "$2")
+      shift 2
     else
       echo "Unknown argument: $1"
       $program --help
@@ -79,6 +82,7 @@ args+=(
   --rpc-drone-address 127.0.0.1:9900
 )
 default_arg --gossip-port 8001
+default_arg --log -
 
 set -x
 # shellcheck disable=SC2086 # Don't want to double quote $program

multinode-demo/validator.sh

@@ -143,6 +143,9 @@ while [[ -n $1 ]]; do
     elif [[ $1 = --skip-poh-verify ]]; then
       args+=("$1")
       shift
+    elif [[ $1 = --log ]]; then
+      args+=("$1" "$2")
+      shift 2
     elif [[ $1 = -h ]]; then
       usage "$@"
     else
@@ -216,6 +219,7 @@ default_arg --identity "$identity_keypair_path"
 default_arg --voting-keypair "$voting_keypair_path"
 default_arg --storage-keypair "$storage_keypair_path"
 default_arg --ledger "$ledger_dir"
+default_arg --log -
 
 if [[ -n $SOLANA_CUDA ]]; then
   program=$solana_validator_cuda

validator/Cargo.toml

@@ -11,6 +11,7 @@ homepage = "https://solana.com/"
 [dependencies]
 bzip2 = "0.3.3"
 clap = "2.33.0"
+chrono = { version = "0.4.9", features = ["serde"] }
 console = "0.9.1"
 log = "0.4.8"
 indicatif = "0.13.0"
@@ -30,3 +31,6 @@ solana-vote-api = { path = "../programs/vote_api", version = "0.21.0" }
 solana-vote-signer = { path = "../vote-signer", version = "0.21.0" }
 tempfile = "3.1.0"
 tar = "0.4.26"
+
+[target."cfg(unix)".dependencies]
+gag = "0.1.10"

validator/src/main.rs

@@ -416,9 +416,17 @@ pub fn main() {
                 .validator(hash_validator)
                 .help("Require the genesis block have this blockhash"),
         )
+        .arg(
+            Arg::with_name("logfile")
+                .short("o")
+                .long("log")
+                .value_name("FILE")
+                .takes_value(true)
+                .help("Redirect logging to the specified file, '-' for standard error"),
+        )
         .get_matches();
 
-    let keypair = if let Some(identity) = matches.value_of("identity") {
+    let identity_keypair = if let Some(identity) = matches.value_of("identity") {
         read_keypair_file(identity).unwrap_or_else(|err| {
             error!("{}: Unable to open keypair file: {}", err, identity);
             exit(1);
@@ -524,6 +532,34 @@ pub fn main() {
         option_env!("CI_BRANCH").unwrap_or("unknown"),
         option_env!("CI_COMMIT").unwrap_or("unknown")
     );
+
+    let default_logfile = format!(
+        "solana-validator-{}-{}.log",
+        identity_keypair.pubkey(),
+        chrono::Local::now().to_rfc3339()
+    );
+    let logfile = matches.value_of("logfile").unwrap_or(&default_logfile);
+
+    let _log_redirect = if logfile == "-" {
+        None
+    } else {
+        #[cfg(unix)]
+        {
+            println!("log file: {}", logfile);
+            Some(gag::Redirect::stderr(File::create(logfile).unwrap_or_else(
+                |err| {
+                    eprintln!("Unable to create {}: {:?}", logfile, err);
+                    exit(1);
+                },
+            )))
+        }
+        #[cfg(not(unix))]
+        {
+            println!("logging to a file is not supported on this platform");
+            None
+        }
+    };
+
     solana_logger::setup_with_filter(
         &[
             "solana=info", /* info logging for all solana modules */
@@ -531,7 +567,7 @@ pub fn main() {
         ]
         .join(","),
     );
-    solana_metrics::set_host_id(keypair.pubkey().to_string());
+    solana_metrics::set_host_id(identity_keypair.pubkey().to_string());
     solana_metrics::set_panic_hook("validator");
 
     if cuda {
@@ -563,16 +599,13 @@ pub fn main() {
     });
 
     let mut tcp_ports = vec![];
-    let mut node = Node::new_with_external_ip(&keypair.pubkey(), &gossip_addr, dynamic_port_range);
+    let mut node =
+        Node::new_with_external_ip(&identity_keypair.pubkey(), &gossip_addr, dynamic_port_range);
     if let Ok(rpc_port) = rpc_port {
-        let port_number = rpc_port.to_string().parse().expect("integer");
-        if port_number == 0 {
-            error!("Invalid RPC port requested: {:?}", rpc_port);
-            exit(1);
-        }
-        node.info.rpc = SocketAddr::new(node.info.gossip.ip(), port_number);
-        node.info.rpc_pubsub = SocketAddr::new(node.info.gossip.ip(), port_number + 1);
-        tcp_ports = vec![port_number, port_number + 1];
+        let rpc_pubsub_port = rpc_port + 1;
+        node.info.rpc = SocketAddr::new(node.info.gossip.ip(), rpc_port);
+        node.info.rpc_pubsub = SocketAddr::new(node.info.gossip.ip(), rpc_pubsub_port);
+        tcp_ports = vec![rpc_port, rpc_pubsub_port];
     };
 
     if let Some(ref cluster_entrypoint) = cluster_entrypoint {
@@ -637,7 +670,7 @@ pub fn main() {
 
     let validator = Validator::new(
         node,
-        &Arc::new(keypair),
+        &Arc::new(identity_keypair),
         &ledger_path,
         &vote_account,
         &Arc::new(voting_keypair),