diff --git a/net-shaper/src/main.rs b/net-shaper/src/main.rs index c7482af90..53b1c00a1 100644 --- a/net-shaper/src/main.rs +++ b/net-shaper/src/main.rs @@ -1,4 +1,6 @@ -use clap::{crate_description, crate_name, value_t_or_exit, App, Arg, ArgMatches, SubCommand}; +use clap::{ + crate_description, crate_name, crate_version, value_t_or_exit, App, Arg, ArgMatches, SubCommand, +}; use serde::{Deserialize, Serialize}; use std::fs; @@ -108,12 +110,13 @@ fn flush_iptables_rule() { ); } -fn insert_tc_root(interface: &str) -> bool { +fn insert_tc_root(interface: &str, num_bands: &str) -> bool { // tc qdisc add dev root handle 1: prio + // tc qdisc add dev root handle 1: prio bands run( "tc", &[ - "qdisc", "add", "dev", interface, "root", "handle", "1:", "prio", + "qdisc", "add", "dev", interface, "root", "handle", "1:", "prio", "bands", num_bands, ], "Failed to add root qdisc", "tc add root qdisc", @@ -161,11 +164,11 @@ fn delete_tc_netem(interface: &str, class: &str, handle: &str, filter: &str) { } fn insert_tos_filter(interface: &str, class: &str, tos: &str) -> bool { - // tc filter add dev parent 1:0 protocol ip prio 10 u32 match ip tos 0xff flowid 1: + // tc filter add dev protocol ip parent 1: prio 1 u32 match ip tos 0xff flowid 1: run( "tc", &[ - "filter", "add", "dev", interface, "parent", "1:0", "protocol", "ip", "prio", "10", + "filter", "add", "dev", interface, "protocol", "ip", "parent", "1:", "prio", "1", "u32", "match", "ip", "tos", tos, "0xff", "flowid", class, ], "Failed to add tos filter", @@ -175,11 +178,11 @@ fn insert_tos_filter(interface: &str, class: &str, tos: &str) -> bool { } fn delete_tos_filter(interface: &str, class: &str, tos: &str) { - // tc filter delete dev parent 1:0 protocol ip prio 10 u32 match ip tos 0xff flowid 1: + // tc filter delete dev protocol ip parent 1: prio 10 u32 match ip tos 0xff flowid 1: run( "tc", &[ - "filter", "delete", "dev", interface, "parent", "1:0", "protocol", "ip", "prio", "10", + "filter", "delete", "dev", interface, "protocol", "ip", "parent", "1:", "prio", "1", "u32", "match", "ip", "tos", tos, "0xff", "flowid", class, ], "Failed to delete tos filter", @@ -188,6 +191,45 @@ fn delete_tos_filter(interface: &str, class: &str, tos: &str) { ); } +fn insert_default_filter(interface: &str, class: &str) -> bool { + // tc filter add dev protocol ip parent 1: prio 2 u32 match ip src 0/0 flowid 1: + run( + "tc", + &[ + "filter", "add", "dev", interface, "protocol", "ip", "parent", "1:", "prio", "2", + "u32", "match", "ip", "tos", "0", "0xff", "flowid", class, + ], + "Failed to add default filter", + "tc add default filter", + false, + ) +} + +fn delete_default_filter(interface: &str, class: &str) { + // tc filter delete dev protocol ip parent 1: prio 2 flowid 1: + run( + "tc", + &[ + "filter", "delete", "dev", interface, "protocol", "ip", "parent", "1:", "prio", "2", + "flowid", class, + ], + "Failed to delete default filter", + "tc delete default filter", + true, + ); +} + +fn delete_all_filters(interface: &str) { + // tc filter delete dev + run( + "tc", + &["filter", "delete", "dev", interface], + "Failed to delete all filters", + "tc delete all filters", + true, + ); +} + fn identify_my_partition(partitions: &[u8], index: u64, size: u64) -> usize { let mut my_partition = 0; let mut watermark = 0; @@ -236,9 +278,17 @@ fn shape_network(matches: &ArgMatches) { } delete_tc_root(interface.as_str()); - if !topology.interconnects.is_empty() && !insert_tc_root(interface.as_str()) { - flush_iptables_rule(); - return; + let num_bands = topology.partitions.len() + 1; + let default_filter_class = format!("1:{}", num_bands); + if !topology.interconnects.is_empty() { + let num_bands_str = num_bands.to_string(); + if !insert_tc_root(interface.as_str(), num_bands_str.as_str()) + || !insert_default_filter(interface.as_str(), default_filter_class.as_str()) + { + delete_tc_root(interface.as_str()); + flush_iptables_rule(); + return; + } } topology.interconnects.iter().for_each(|i| { @@ -246,9 +296,12 @@ fn shape_network(matches: &ArgMatches) { let tos = partition_id_to_tos(i.a as usize); if tos == 0 { println!("Incorrect value of TOS/Partition in config {}", i.a); + delete_default_filter(interface.as_str(), default_filter_class.as_str()); + delete_tc_root(interface.as_str()); return; } let tos_string = tos.to_string(); + // First valid class is 1:1 let class = format!("1:{}", i.a + 1); if !insert_tc_netem( interface.as_str(), @@ -256,6 +309,7 @@ fn shape_network(matches: &ArgMatches) { tos_string.as_str(), i.config.as_str(), ) { + delete_default_filter(interface.as_str(), default_filter_class.as_str()); delete_tc_root(interface.as_str()); return; } @@ -267,6 +321,7 @@ fn shape_network(matches: &ArgMatches) { tos_string.as_str(), i.config.as_str(), ); + delete_default_filter(interface.as_str(), default_filter_class.as_str()); delete_tc_root(interface.as_str()); return; } @@ -296,8 +351,9 @@ fn cleanup_network(matches: &ArgMatches) { topology.interconnects.iter().for_each(|i| { if i.b as usize == my_partition { let handle = (i.a + 1).to_string(); + // First valid class is 1:1 + let class = format!("1:{}", i.a + 1); let tos_string = i.a.to_string(); - let class = format!("1:{}", i.a); delete_tos_filter(interface.as_str(), class.as_str(), tos_string.as_str()); delete_tc_netem( interface.as_str(), @@ -307,6 +363,16 @@ fn cleanup_network(matches: &ArgMatches) { ); } }); + let num_bands = topology.partitions.len() + 1; + let default_filter_class = format!("1:{}", num_bands); + delete_default_filter(interface.as_str(), default_filter_class.as_str()); + delete_tc_root(interface.as_str()); + flush_iptables_rule(); +} + +fn force_cleanup_network(matches: &ArgMatches) { + let interface = value_t_or_exit!(matches, "iface", String); + delete_all_filters(interface.as_str()); delete_tc_root(interface.as_str()); flush_iptables_rule(); } @@ -323,7 +389,7 @@ fn main() { let matches = App::new(crate_name!()) .about(crate_description!()) - .version(solana_clap_utils::version!()) + .version(crate_version!()) .subcommand( SubCommand::with_name("shape") .about("Shape the network using config file") @@ -404,12 +470,26 @@ fn main() { .help("Position of current node in the network"), ), ) + .subcommand( + SubCommand::with_name("force_cleanup") + .about("Remove the network filters") + .arg( + Arg::with_name("iface") + .short("i") + .long("iface") + .value_name("network interface name") + .takes_value(true) + .required(true) + .help("Name of network interface"), + ), + ) .subcommand(SubCommand::with_name("configure").about("Generate a config file")) .get_matches(); match matches.subcommand() { ("shape", Some(args_matches)) => shape_network(args_matches), ("cleanup", Some(args_matches)) => cleanup_network(args_matches), + ("force_cleanup", Some(args_matches)) => force_cleanup_network(args_matches), ("configure", Some(args_matches)) => configure(args_matches), _ => {} }; diff --git a/net/net.sh b/net/net.sh index 549d3c656..d6b0418b6 100755 --- a/net/net.sh +++ b/net/net.sh @@ -61,7 +61,7 @@ Operate a configured testnet - Override the default --hashes-per-tick for the cluster --no-airdrop - If set, disables airdrops. Nodes must be funded in genesis config when airdrops are disabled. - --faucet-lamports NUM_LAMPORTS_TO_MINT + --lamports NUM_LAMPORTS_TO_MINT - Override the default 500000000000000000 lamports minted in genesis --internal-nodes-stake-lamports NUM_LAMPORTS_PER_NODE - Amount to stake internal nodes. @@ -88,10 +88,6 @@ Operate a configured testnet --use-move - Build the move-loader-program and add it to the cluster - --operating-mode development|softlaunch - - Specify whether or not to launch the cluster in "development" mode with all features enabled at epoch 0, - or "softlaunch" mode with some features disabled at epoch 0 (default: development) - sanity/start-specific options: -F - Discard validator nodes that didn't bootup successfully -o noInstallCheck - Skip solana-install sanity @@ -172,18 +168,7 @@ while [[ -n $1 ]]; do elif [[ $1 = --target-lamports-per-signature ]]; then genesisOptions="$genesisOptions $1 $2" shift 2 - elif [[ $1 = --faucet-lamports ]]; then - genesisOptions="$genesisOptions $1 $2" - shift 2 - elif [[ $1 = --operating-mode ]]; then - case "$2" in - development|softlaunch) - ;; - *) - echo "Unexpected operating mode: \"$2\"" - exit 1 - ;; - esac + elif [[ $1 = --lamports ]]; then genesisOptions="$genesisOptions $1 $2" shift 2 elif [[ $1 = --no-snapshot-fetch ]]; then @@ -860,6 +845,7 @@ deploy() { echo "Network start logs in $netLogDir" } + stopNode() { local ipAddress=$1 local block=$2 @@ -891,6 +877,7 @@ stopNode() { solana/scripts/netem.sh delete < solana/netem.cfg rm -f solana/netem.cfg fi + solana/scripts/net-shaper.sh force_cleanup for pattern in node solana- remote-; do pkill -9 \$pattern done @@ -931,6 +918,7 @@ stop() { echo "Stopping nodes took $SECONDS seconds" } + checkPremptibleInstances() { # The validatorIpList nodes may be preemptible instances that can disappear at # any time. Try to detect when a validator has been preempted to help the user @@ -1020,12 +1008,14 @@ logs) ;; netem) if [[ -n $netemConfigFile ]]; then - for ipAddress in "${validatorIpList[@]}"; do - "$here"/scp.sh "$netemConfigFile" solana@"$ipAddress":~/solana - done + if [[ $netemCommand = "add" ]]; then + for ipAddress in "${validatorIpList[@]}"; do + "$here"/scp.sh "$netemConfigFile" solana@"$ipAddress":~/solana + done + fi for i in "${!validatorIpList[@]}"; do "$here"/ssh.sh solana@"${validatorIpList[$i]}" 'solana/scripts/net-shaper.sh' \ - "$netemCommand" solana/"$netemConfigFile" "${#validatorIpList[@]}" "$i" + "$netemCommand" ~solana/solana/"$netemConfigFile" "${#validatorIpList[@]}" "$i" done else num_nodes=$((${#validatorIpList[@]}*netemPartition/100)) diff --git a/scripts/net-shaper.sh b/scripts/net-shaper.sh index 28b277a0a..8bd39cafa 100755 --- a/scripts/net-shaper.sh +++ b/scripts/net-shaper.sh @@ -19,6 +19,8 @@ iface="$(ifconfig | grep mtu | grep -iv loopback | grep -i running | awk 'BEGIN if [[ "$1" = cleanup ]]; then $sudo ~solana/.cargo/bin/solana-net-shaper cleanup -f "$2" -s "$3" -p "$4" -i "$iface" +elif [[ "$1" = force_cleanup ]]; then + $sudo ~solana/.cargo/bin/solana-net-shaper force_cleanup -i "$iface" else $sudo ~solana/.cargo/bin/solana-net-shaper shape -f "$2" -s "$3" -p "$4" -i "$iface" fi