Fixes to net-shaper and net.sh (#7002)
* Fixes to net-shaper and net.sh * fixes to default filters and cleanup
This commit is contained in:
parent
e0a2bb9d86
commit
955aaef2e6
|
@ -1,4 +1,6 @@
|
||||||
use clap::{crate_description, crate_name, value_t_or_exit, App, Arg, ArgMatches, SubCommand};
|
use clap::{
|
||||||
|
crate_description, crate_name, crate_version, value_t_or_exit, App, Arg, ArgMatches, SubCommand,
|
||||||
|
};
|
||||||
|
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use std::fs;
|
use std::fs;
|
||||||
|
@ -108,12 +110,13 @@ fn flush_iptables_rule() {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
fn insert_tc_root(interface: &str) -> bool {
|
fn insert_tc_root(interface: &str, num_bands: &str) -> bool {
|
||||||
// tc qdisc add dev <if> root handle 1: prio
|
// tc qdisc add dev <if> root handle 1: prio
|
||||||
|
// tc qdisc add dev <if> root handle 1: prio bands <num_bands>
|
||||||
run(
|
run(
|
||||||
"tc",
|
"tc",
|
||||||
&[
|
&[
|
||||||
"qdisc", "add", "dev", interface, "root", "handle", "1:", "prio",
|
"qdisc", "add", "dev", interface, "root", "handle", "1:", "prio", "bands", num_bands,
|
||||||
],
|
],
|
||||||
"Failed to add root qdisc",
|
"Failed to add root qdisc",
|
||||||
"tc add root qdisc",
|
"tc add root qdisc",
|
||||||
|
@ -161,11 +164,11 @@ fn delete_tc_netem(interface: &str, class: &str, handle: &str, filter: &str) {
|
||||||
}
|
}
|
||||||
|
|
||||||
fn insert_tos_filter(interface: &str, class: &str, tos: &str) -> bool {
|
fn insert_tos_filter(interface: &str, class: &str, tos: &str) -> bool {
|
||||||
// tc filter add dev <if> parent 1:0 protocol ip prio 10 u32 match ip tos <i.a> 0xff flowid 1:<i.a>
|
// tc filter add dev <if> protocol ip parent 1: prio 1 u32 match ip tos <i.a> 0xff flowid 1:<i.a>
|
||||||
run(
|
run(
|
||||||
"tc",
|
"tc",
|
||||||
&[
|
&[
|
||||||
"filter", "add", "dev", interface, "parent", "1:0", "protocol", "ip", "prio", "10",
|
"filter", "add", "dev", interface, "protocol", "ip", "parent", "1:", "prio", "1",
|
||||||
"u32", "match", "ip", "tos", tos, "0xff", "flowid", class,
|
"u32", "match", "ip", "tos", tos, "0xff", "flowid", class,
|
||||||
],
|
],
|
||||||
"Failed to add tos filter",
|
"Failed to add tos filter",
|
||||||
|
@ -175,11 +178,11 @@ fn insert_tos_filter(interface: &str, class: &str, tos: &str) -> bool {
|
||||||
}
|
}
|
||||||
|
|
||||||
fn delete_tos_filter(interface: &str, class: &str, tos: &str) {
|
fn delete_tos_filter(interface: &str, class: &str, tos: &str) {
|
||||||
// tc filter delete dev <if> parent 1:0 protocol ip prio 10 u32 match ip tos <i.a> 0xff flowid 1:<i.a>
|
// tc filter delete dev <if> protocol ip parent 1: prio 10 u32 match ip tos <i.a> 0xff flowid 1:<i.a>
|
||||||
run(
|
run(
|
||||||
"tc",
|
"tc",
|
||||||
&[
|
&[
|
||||||
"filter", "delete", "dev", interface, "parent", "1:0", "protocol", "ip", "prio", "10",
|
"filter", "delete", "dev", interface, "protocol", "ip", "parent", "1:", "prio", "1",
|
||||||
"u32", "match", "ip", "tos", tos, "0xff", "flowid", class,
|
"u32", "match", "ip", "tos", tos, "0xff", "flowid", class,
|
||||||
],
|
],
|
||||||
"Failed to delete tos filter",
|
"Failed to delete tos filter",
|
||||||
|
@ -188,6 +191,45 @@ fn delete_tos_filter(interface: &str, class: &str, tos: &str) {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn insert_default_filter(interface: &str, class: &str) -> bool {
|
||||||
|
// tc filter add dev <if> protocol ip parent 1: prio 2 u32 match ip src 0/0 flowid 1:<class>
|
||||||
|
run(
|
||||||
|
"tc",
|
||||||
|
&[
|
||||||
|
"filter", "add", "dev", interface, "protocol", "ip", "parent", "1:", "prio", "2",
|
||||||
|
"u32", "match", "ip", "tos", "0", "0xff", "flowid", class,
|
||||||
|
],
|
||||||
|
"Failed to add default filter",
|
||||||
|
"tc add default filter",
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn delete_default_filter(interface: &str, class: &str) {
|
||||||
|
// tc filter delete dev <if> protocol ip parent 1: prio 2 flowid 1:<class>
|
||||||
|
run(
|
||||||
|
"tc",
|
||||||
|
&[
|
||||||
|
"filter", "delete", "dev", interface, "protocol", "ip", "parent", "1:", "prio", "2",
|
||||||
|
"flowid", class,
|
||||||
|
],
|
||||||
|
"Failed to delete default filter",
|
||||||
|
"tc delete default filter",
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn delete_all_filters(interface: &str) {
|
||||||
|
// tc filter delete dev <if>
|
||||||
|
run(
|
||||||
|
"tc",
|
||||||
|
&["filter", "delete", "dev", interface],
|
||||||
|
"Failed to delete all filters",
|
||||||
|
"tc delete all filters",
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
fn identify_my_partition(partitions: &[u8], index: u64, size: u64) -> usize {
|
fn identify_my_partition(partitions: &[u8], index: u64, size: u64) -> usize {
|
||||||
let mut my_partition = 0;
|
let mut my_partition = 0;
|
||||||
let mut watermark = 0;
|
let mut watermark = 0;
|
||||||
|
@ -236,19 +278,30 @@ fn shape_network(matches: &ArgMatches) {
|
||||||
}
|
}
|
||||||
|
|
||||||
delete_tc_root(interface.as_str());
|
delete_tc_root(interface.as_str());
|
||||||
if !topology.interconnects.is_empty() && !insert_tc_root(interface.as_str()) {
|
let num_bands = topology.partitions.len() + 1;
|
||||||
|
let default_filter_class = format!("1:{}", num_bands);
|
||||||
|
if !topology.interconnects.is_empty() {
|
||||||
|
let num_bands_str = num_bands.to_string();
|
||||||
|
if !insert_tc_root(interface.as_str(), num_bands_str.as_str())
|
||||||
|
|| !insert_default_filter(interface.as_str(), default_filter_class.as_str())
|
||||||
|
{
|
||||||
|
delete_tc_root(interface.as_str());
|
||||||
flush_iptables_rule();
|
flush_iptables_rule();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
topology.interconnects.iter().for_each(|i| {
|
topology.interconnects.iter().for_each(|i| {
|
||||||
if i.b as usize == my_partition {
|
if i.b as usize == my_partition {
|
||||||
let tos = partition_id_to_tos(i.a as usize);
|
let tos = partition_id_to_tos(i.a as usize);
|
||||||
if tos == 0 {
|
if tos == 0 {
|
||||||
println!("Incorrect value of TOS/Partition in config {}", i.a);
|
println!("Incorrect value of TOS/Partition in config {}", i.a);
|
||||||
|
delete_default_filter(interface.as_str(), default_filter_class.as_str());
|
||||||
|
delete_tc_root(interface.as_str());
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
let tos_string = tos.to_string();
|
let tos_string = tos.to_string();
|
||||||
|
// First valid class is 1:1
|
||||||
let class = format!("1:{}", i.a + 1);
|
let class = format!("1:{}", i.a + 1);
|
||||||
if !insert_tc_netem(
|
if !insert_tc_netem(
|
||||||
interface.as_str(),
|
interface.as_str(),
|
||||||
|
@ -256,6 +309,7 @@ fn shape_network(matches: &ArgMatches) {
|
||||||
tos_string.as_str(),
|
tos_string.as_str(),
|
||||||
i.config.as_str(),
|
i.config.as_str(),
|
||||||
) {
|
) {
|
||||||
|
delete_default_filter(interface.as_str(), default_filter_class.as_str());
|
||||||
delete_tc_root(interface.as_str());
|
delete_tc_root(interface.as_str());
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -267,6 +321,7 @@ fn shape_network(matches: &ArgMatches) {
|
||||||
tos_string.as_str(),
|
tos_string.as_str(),
|
||||||
i.config.as_str(),
|
i.config.as_str(),
|
||||||
);
|
);
|
||||||
|
delete_default_filter(interface.as_str(), default_filter_class.as_str());
|
||||||
delete_tc_root(interface.as_str());
|
delete_tc_root(interface.as_str());
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -296,8 +351,9 @@ fn cleanup_network(matches: &ArgMatches) {
|
||||||
topology.interconnects.iter().for_each(|i| {
|
topology.interconnects.iter().for_each(|i| {
|
||||||
if i.b as usize == my_partition {
|
if i.b as usize == my_partition {
|
||||||
let handle = (i.a + 1).to_string();
|
let handle = (i.a + 1).to_string();
|
||||||
|
// First valid class is 1:1
|
||||||
|
let class = format!("1:{}", i.a + 1);
|
||||||
let tos_string = i.a.to_string();
|
let tos_string = i.a.to_string();
|
||||||
let class = format!("1:{}", i.a);
|
|
||||||
delete_tos_filter(interface.as_str(), class.as_str(), tos_string.as_str());
|
delete_tos_filter(interface.as_str(), class.as_str(), tos_string.as_str());
|
||||||
delete_tc_netem(
|
delete_tc_netem(
|
||||||
interface.as_str(),
|
interface.as_str(),
|
||||||
|
@ -307,6 +363,16 @@ fn cleanup_network(matches: &ArgMatches) {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
let num_bands = topology.partitions.len() + 1;
|
||||||
|
let default_filter_class = format!("1:{}", num_bands);
|
||||||
|
delete_default_filter(interface.as_str(), default_filter_class.as_str());
|
||||||
|
delete_tc_root(interface.as_str());
|
||||||
|
flush_iptables_rule();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn force_cleanup_network(matches: &ArgMatches) {
|
||||||
|
let interface = value_t_or_exit!(matches, "iface", String);
|
||||||
|
delete_all_filters(interface.as_str());
|
||||||
delete_tc_root(interface.as_str());
|
delete_tc_root(interface.as_str());
|
||||||
flush_iptables_rule();
|
flush_iptables_rule();
|
||||||
}
|
}
|
||||||
|
@ -323,7 +389,7 @@ fn main() {
|
||||||
|
|
||||||
let matches = App::new(crate_name!())
|
let matches = App::new(crate_name!())
|
||||||
.about(crate_description!())
|
.about(crate_description!())
|
||||||
.version(solana_clap_utils::version!())
|
.version(crate_version!())
|
||||||
.subcommand(
|
.subcommand(
|
||||||
SubCommand::with_name("shape")
|
SubCommand::with_name("shape")
|
||||||
.about("Shape the network using config file")
|
.about("Shape the network using config file")
|
||||||
|
@ -404,12 +470,26 @@ fn main() {
|
||||||
.help("Position of current node in the network"),
|
.help("Position of current node in the network"),
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
.subcommand(
|
||||||
|
SubCommand::with_name("force_cleanup")
|
||||||
|
.about("Remove the network filters")
|
||||||
|
.arg(
|
||||||
|
Arg::with_name("iface")
|
||||||
|
.short("i")
|
||||||
|
.long("iface")
|
||||||
|
.value_name("network interface name")
|
||||||
|
.takes_value(true)
|
||||||
|
.required(true)
|
||||||
|
.help("Name of network interface"),
|
||||||
|
),
|
||||||
|
)
|
||||||
.subcommand(SubCommand::with_name("configure").about("Generate a config file"))
|
.subcommand(SubCommand::with_name("configure").about("Generate a config file"))
|
||||||
.get_matches();
|
.get_matches();
|
||||||
|
|
||||||
match matches.subcommand() {
|
match matches.subcommand() {
|
||||||
("shape", Some(args_matches)) => shape_network(args_matches),
|
("shape", Some(args_matches)) => shape_network(args_matches),
|
||||||
("cleanup", Some(args_matches)) => cleanup_network(args_matches),
|
("cleanup", Some(args_matches)) => cleanup_network(args_matches),
|
||||||
|
("force_cleanup", Some(args_matches)) => force_cleanup_network(args_matches),
|
||||||
("configure", Some(args_matches)) => configure(args_matches),
|
("configure", Some(args_matches)) => configure(args_matches),
|
||||||
_ => {}
|
_ => {}
|
||||||
};
|
};
|
||||||
|
|
26
net/net.sh
26
net/net.sh
|
@ -61,7 +61,7 @@ Operate a configured testnet
|
||||||
- Override the default --hashes-per-tick for the cluster
|
- Override the default --hashes-per-tick for the cluster
|
||||||
--no-airdrop
|
--no-airdrop
|
||||||
- If set, disables airdrops. Nodes must be funded in genesis config when airdrops are disabled.
|
- If set, disables airdrops. Nodes must be funded in genesis config when airdrops are disabled.
|
||||||
--faucet-lamports NUM_LAMPORTS_TO_MINT
|
--lamports NUM_LAMPORTS_TO_MINT
|
||||||
- Override the default 500000000000000000 lamports minted in genesis
|
- Override the default 500000000000000000 lamports minted in genesis
|
||||||
--internal-nodes-stake-lamports NUM_LAMPORTS_PER_NODE
|
--internal-nodes-stake-lamports NUM_LAMPORTS_PER_NODE
|
||||||
- Amount to stake internal nodes.
|
- Amount to stake internal nodes.
|
||||||
|
@ -88,10 +88,6 @@ Operate a configured testnet
|
||||||
|
|
||||||
--use-move - Build the move-loader-program and add it to the cluster
|
--use-move - Build the move-loader-program and add it to the cluster
|
||||||
|
|
||||||
--operating-mode development|softlaunch
|
|
||||||
- Specify whether or not to launch the cluster in "development" mode with all features enabled at epoch 0,
|
|
||||||
or "softlaunch" mode with some features disabled at epoch 0 (default: development)
|
|
||||||
|
|
||||||
sanity/start-specific options:
|
sanity/start-specific options:
|
||||||
-F - Discard validator nodes that didn't bootup successfully
|
-F - Discard validator nodes that didn't bootup successfully
|
||||||
-o noInstallCheck - Skip solana-install sanity
|
-o noInstallCheck - Skip solana-install sanity
|
||||||
|
@ -172,18 +168,7 @@ while [[ -n $1 ]]; do
|
||||||
elif [[ $1 = --target-lamports-per-signature ]]; then
|
elif [[ $1 = --target-lamports-per-signature ]]; then
|
||||||
genesisOptions="$genesisOptions $1 $2"
|
genesisOptions="$genesisOptions $1 $2"
|
||||||
shift 2
|
shift 2
|
||||||
elif [[ $1 = --faucet-lamports ]]; then
|
elif [[ $1 = --lamports ]]; then
|
||||||
genesisOptions="$genesisOptions $1 $2"
|
|
||||||
shift 2
|
|
||||||
elif [[ $1 = --operating-mode ]]; then
|
|
||||||
case "$2" in
|
|
||||||
development|softlaunch)
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
echo "Unexpected operating mode: \"$2\""
|
|
||||||
exit 1
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
genesisOptions="$genesisOptions $1 $2"
|
genesisOptions="$genesisOptions $1 $2"
|
||||||
shift 2
|
shift 2
|
||||||
elif [[ $1 = --no-snapshot-fetch ]]; then
|
elif [[ $1 = --no-snapshot-fetch ]]; then
|
||||||
|
@ -860,6 +845,7 @@ deploy() {
|
||||||
echo "Network start logs in $netLogDir"
|
echo "Network start logs in $netLogDir"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
stopNode() {
|
stopNode() {
|
||||||
local ipAddress=$1
|
local ipAddress=$1
|
||||||
local block=$2
|
local block=$2
|
||||||
|
@ -891,6 +877,7 @@ stopNode() {
|
||||||
solana/scripts/netem.sh delete < solana/netem.cfg
|
solana/scripts/netem.sh delete < solana/netem.cfg
|
||||||
rm -f solana/netem.cfg
|
rm -f solana/netem.cfg
|
||||||
fi
|
fi
|
||||||
|
solana/scripts/net-shaper.sh force_cleanup
|
||||||
for pattern in node solana- remote-; do
|
for pattern in node solana- remote-; do
|
||||||
pkill -9 \$pattern
|
pkill -9 \$pattern
|
||||||
done
|
done
|
||||||
|
@ -931,6 +918,7 @@ stop() {
|
||||||
echo "Stopping nodes took $SECONDS seconds"
|
echo "Stopping nodes took $SECONDS seconds"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
checkPremptibleInstances() {
|
checkPremptibleInstances() {
|
||||||
# The validatorIpList nodes may be preemptible instances that can disappear at
|
# The validatorIpList nodes may be preemptible instances that can disappear at
|
||||||
# any time. Try to detect when a validator has been preempted to help the user
|
# any time. Try to detect when a validator has been preempted to help the user
|
||||||
|
@ -1020,12 +1008,14 @@ logs)
|
||||||
;;
|
;;
|
||||||
netem)
|
netem)
|
||||||
if [[ -n $netemConfigFile ]]; then
|
if [[ -n $netemConfigFile ]]; then
|
||||||
|
if [[ $netemCommand = "add" ]]; then
|
||||||
for ipAddress in "${validatorIpList[@]}"; do
|
for ipAddress in "${validatorIpList[@]}"; do
|
||||||
"$here"/scp.sh "$netemConfigFile" solana@"$ipAddress":~/solana
|
"$here"/scp.sh "$netemConfigFile" solana@"$ipAddress":~/solana
|
||||||
done
|
done
|
||||||
|
fi
|
||||||
for i in "${!validatorIpList[@]}"; do
|
for i in "${!validatorIpList[@]}"; do
|
||||||
"$here"/ssh.sh solana@"${validatorIpList[$i]}" 'solana/scripts/net-shaper.sh' \
|
"$here"/ssh.sh solana@"${validatorIpList[$i]}" 'solana/scripts/net-shaper.sh' \
|
||||||
"$netemCommand" solana/"$netemConfigFile" "${#validatorIpList[@]}" "$i"
|
"$netemCommand" ~solana/solana/"$netemConfigFile" "${#validatorIpList[@]}" "$i"
|
||||||
done
|
done
|
||||||
else
|
else
|
||||||
num_nodes=$((${#validatorIpList[@]}*netemPartition/100))
|
num_nodes=$((${#validatorIpList[@]}*netemPartition/100))
|
||||||
|
|
|
@ -19,6 +19,8 @@ iface="$(ifconfig | grep mtu | grep -iv loopback | grep -i running | awk 'BEGIN
|
||||||
|
|
||||||
if [[ "$1" = cleanup ]]; then
|
if [[ "$1" = cleanup ]]; then
|
||||||
$sudo ~solana/.cargo/bin/solana-net-shaper cleanup -f "$2" -s "$3" -p "$4" -i "$iface"
|
$sudo ~solana/.cargo/bin/solana-net-shaper cleanup -f "$2" -s "$3" -p "$4" -i "$iface"
|
||||||
|
elif [[ "$1" = force_cleanup ]]; then
|
||||||
|
$sudo ~solana/.cargo/bin/solana-net-shaper force_cleanup -i "$iface"
|
||||||
else
|
else
|
||||||
$sudo ~solana/.cargo/bin/solana-net-shaper shape -f "$2" -s "$3" -p "$4" -i "$iface"
|
$sudo ~solana/.cargo/bin/solana-net-shaper shape -f "$2" -s "$3" -p "$4" -i "$iface"
|
||||||
fi
|
fi
|
||||||
|
|
Loading…
Reference in New Issue