Tool to partition network and induce packet drops/delays (#6933)
* Tool to partition network and induce packet drops/delays * clippy fixes * review comments
This commit is contained in:
parent
e2fb9ac829
commit
5f38fa379c
|
@ -3644,6 +3644,20 @@ dependencies = [
|
||||||
"tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
"tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "solana-log-analyzer"
|
||||||
|
version = "0.1.0"
|
||||||
|
dependencies = [
|
||||||
|
"byte-unit 3.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
|
"clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
|
"log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
|
"semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
|
"serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
|
"serde_derive 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
|
"serde_json 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
|
"solana-logger 0.21.0",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "solana-logger"
|
name = "solana-logger"
|
||||||
version = "0.21.0"
|
version = "0.21.0"
|
||||||
|
@ -3722,6 +3736,19 @@ dependencies = [
|
||||||
"solana-sdk 0.21.0",
|
"solana-sdk 0.21.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "solana-net-shaper"
|
||||||
|
version = "0.1.0"
|
||||||
|
dependencies = [
|
||||||
|
"clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
|
"log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
|
"semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
|
"serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
|
"serde_derive 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
|
"serde_json 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
|
"solana-logger 0.21.0",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "solana-net-utils"
|
name = "solana-net-utils"
|
||||||
version = "0.21.0"
|
version = "0.21.0"
|
||||||
|
@ -3740,20 +3767,6 @@ dependencies = [
|
||||||
"tokio-codec 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
"tokio-codec 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "solana-network-tool"
|
|
||||||
version = "0.1.0"
|
|
||||||
dependencies = [
|
|
||||||
"byte-unit 3.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
|
||||||
"clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
|
||||||
"log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
|
|
||||||
"semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
|
||||||
"serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)",
|
|
||||||
"serde_derive 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)",
|
|
||||||
"serde_json 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)",
|
|
||||||
"solana-logger 0.21.0",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "solana-noop-program"
|
name = "solana-noop-program"
|
||||||
version = "0.21.0"
|
version = "0.21.0"
|
||||||
|
|
|
@ -23,6 +23,7 @@ members = [
|
||||||
"merkle-tree",
|
"merkle-tree",
|
||||||
"measure",
|
"measure",
|
||||||
"metrics",
|
"metrics",
|
||||||
|
"net-shaper",
|
||||||
"programs/bpf_loader_api",
|
"programs/bpf_loader_api",
|
||||||
"programs/bpf_loader_program",
|
"programs/bpf_loader_program",
|
||||||
"programs/budget_api",
|
"programs/budget_api",
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
[package]
|
[package]
|
||||||
authors = ["Solana Maintainers <maintainers@solana.com>"]
|
authors = ["Solana Maintainers <maintainers@solana.com>"]
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
name = "solana-network-tool"
|
name = "solana-log-analyzer"
|
||||||
description = "The solana cluster network analysis tool"
|
description = "The solana cluster network analysis tool"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
repository = "https://github.com/solana-labs/solana"
|
repository = "https://github.com/solana-labs/solana"
|
||||||
|
|
|
@ -0,0 +1,23 @@
|
||||||
|
[package]
|
||||||
|
authors = ["Solana Maintainers <maintainers@solana.com>"]
|
||||||
|
edition = "2018"
|
||||||
|
name = "solana-net-shaper"
|
||||||
|
description = "The solana cluster network shaping tool"
|
||||||
|
version = "0.1.0"
|
||||||
|
repository = "https://github.com/solana-labs/solana"
|
||||||
|
license = "Apache-2.0"
|
||||||
|
homepage = "https://solana.com/"
|
||||||
|
publish = false
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
clap = "2.33.0"
|
||||||
|
log = "0.4.8"
|
||||||
|
semver = "0.9.0"
|
||||||
|
serde = "1.0.102"
|
||||||
|
serde_derive = "1.0.102"
|
||||||
|
serde_json = "1.0.41"
|
||||||
|
solana-logger = { path = "../logger", version = "0.21.0" }
|
||||||
|
|
||||||
|
[[bin]]
|
||||||
|
name = "solana-net-shaper"
|
||||||
|
path = "src/main.rs"
|
|
@ -0,0 +1,387 @@
|
||||||
|
use clap::{
|
||||||
|
crate_description, crate_name, crate_version, value_t_or_exit, App, Arg, ArgMatches, SubCommand,
|
||||||
|
};
|
||||||
|
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use std::fs;
|
||||||
|
use std::path::PathBuf;
|
||||||
|
|
||||||
|
#[derive(Deserialize, Serialize, Debug)]
|
||||||
|
struct NetworkInterconnect {
|
||||||
|
pub a: u8,
|
||||||
|
pub b: u8,
|
||||||
|
pub config: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize, Serialize, Debug)]
|
||||||
|
struct NetworkTopology {
|
||||||
|
pub partitions: Vec<u8>,
|
||||||
|
pub interconnects: Vec<NetworkInterconnect>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for NetworkTopology {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
partitions: vec![100],
|
||||||
|
interconnects: vec![],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl NetworkTopology {
|
||||||
|
pub fn verify(&self) -> bool {
|
||||||
|
let sum: u8 = self.partitions.iter().sum();
|
||||||
|
if sum != 100 {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
for x in self.interconnects.iter() {
|
||||||
|
if x.a as usize >= self.partitions.len() || x.b as usize >= self.partitions.len() {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn run(cmd: &str, args: &[&str], launch_err_msg: &str, status_err_msg: &str) -> bool {
|
||||||
|
let output = std::process::Command::new(cmd)
|
||||||
|
.args(args)
|
||||||
|
.output()
|
||||||
|
.expect(launch_err_msg);
|
||||||
|
|
||||||
|
if !output.status.success() {
|
||||||
|
eprintln!(
|
||||||
|
"{} command failed with exit code: {}",
|
||||||
|
status_err_msg, output.status
|
||||||
|
);
|
||||||
|
use std::str::from_utf8;
|
||||||
|
println!("stdout: {}", from_utf8(&output.stdout).unwrap_or("?"));
|
||||||
|
println!("stderr: {}", from_utf8(&output.stderr).unwrap_or("?"));
|
||||||
|
false
|
||||||
|
} else {
|
||||||
|
true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn insert_iptables_rule(my_partition: usize) -> bool {
|
||||||
|
let my_tos = my_partition.to_string();
|
||||||
|
|
||||||
|
// iptables -t mangle -A PREROUTING -p udp -j TOS --set-tos <my_parition_index>
|
||||||
|
run(
|
||||||
|
"iptables",
|
||||||
|
&[
|
||||||
|
"-t",
|
||||||
|
"mangle",
|
||||||
|
"-A",
|
||||||
|
"PREROUTING",
|
||||||
|
"-p",
|
||||||
|
"udp",
|
||||||
|
"-j",
|
||||||
|
"TOS",
|
||||||
|
"--set-tos",
|
||||||
|
my_tos.as_str(),
|
||||||
|
],
|
||||||
|
"Failed to add iptables rule",
|
||||||
|
"iptables",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn flush_iptables_rule() {
|
||||||
|
run(
|
||||||
|
"iptables",
|
||||||
|
&["-F", "-t", "mangle"],
|
||||||
|
"Failed to flush iptables",
|
||||||
|
"iptables flush",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn insert_tc_root(interface: &str) -> bool {
|
||||||
|
// tc qdisc add dev <if> root handle 1: prio
|
||||||
|
run(
|
||||||
|
"tc",
|
||||||
|
&[
|
||||||
|
"qdisc", "add", "dev", interface, "root", "handle", "1:", "prio",
|
||||||
|
],
|
||||||
|
"Failed to add root qdisc",
|
||||||
|
"tc add root qdisc",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn delete_tc_root(interface: &str) {
|
||||||
|
// tc qdisc delete dev <if> root handle 1: prio
|
||||||
|
run(
|
||||||
|
"tc",
|
||||||
|
&[
|
||||||
|
"qdisc", "delete", "dev", interface, "root", "handle", "1:", "prio",
|
||||||
|
],
|
||||||
|
"Failed to delete root qdisc",
|
||||||
|
"tc qdisc delete root",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn insert_tc_netem(interface: &str, class: &str, tos: &str, filter: &str) -> bool {
|
||||||
|
// tc qdisc add dev <if> parent 1:<i.a> handle <i.a>: netem <filters>
|
||||||
|
run(
|
||||||
|
"tc",
|
||||||
|
&[
|
||||||
|
"qdisc", "add", "dev", interface, "parent", class, "handle", tos, "netem", filter,
|
||||||
|
],
|
||||||
|
"Failed to add tc child",
|
||||||
|
"tc add child",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn delete_tc_netem(interface: &str, class: &str, tos: &str, filter: &str) {
|
||||||
|
// tc qdisc delete dev <if> parent 1:<i.a> handle <i.a>: netem <filters>
|
||||||
|
run(
|
||||||
|
"tc",
|
||||||
|
&[
|
||||||
|
"qdisc", "delete", "dev", interface, "parent", class, "handle", tos, "netem", filter,
|
||||||
|
],
|
||||||
|
"Failed to delete child qdisc",
|
||||||
|
"tc delete child qdisc",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn insert_tos_filter(interface: &str, class: &str, tos: &str) -> bool {
|
||||||
|
// tc filter add dev <if> parent 1:0 protocol ip prio 10 u32 match ip tos <i.a> 0xff flowid 1:<i.a>
|
||||||
|
run(
|
||||||
|
"tc",
|
||||||
|
&[
|
||||||
|
"filter", "add", "dev", interface, "parent", "1:0", "protocol", "ip", "prio", "10",
|
||||||
|
"u32", "match", "ip", "tos", tos, "0xff", "flowid", class,
|
||||||
|
],
|
||||||
|
"Failed to add tos filter",
|
||||||
|
"tc add filter",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn delete_tos_filter(interface: &str, class: &str, tos: &str) {
|
||||||
|
// tc filter delete dev <if> parent 1:0 protocol ip prio 10 u32 match ip tos <i.a> 0xff flowid 1:<i.a>
|
||||||
|
run(
|
||||||
|
"tc",
|
||||||
|
&[
|
||||||
|
"filter", "delete", "dev", interface, "parent", "1:0", "protocol", "ip", "prio", "10",
|
||||||
|
"u32", "match", "ip", "tos", tos, "0xff", "flowid", class,
|
||||||
|
],
|
||||||
|
"Failed to delete tos filter",
|
||||||
|
"tc delete filter",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn identify_my_partition(partitions: &[u8], index: u64, size: u64) -> usize {
|
||||||
|
let mut my_partition = 0;
|
||||||
|
let mut watermark = 0;
|
||||||
|
for (i, p) in partitions.iter().enumerate() {
|
||||||
|
watermark += *p;
|
||||||
|
if u64::from(watermark) >= index * 100 / size {
|
||||||
|
my_partition = i;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
my_partition
|
||||||
|
}
|
||||||
|
|
||||||
|
fn shape_network(matches: &ArgMatches) {
|
||||||
|
let config_path = PathBuf::from(value_t_or_exit!(matches, "file", String));
|
||||||
|
let config = fs::read_to_string(&config_path).expect("Unable to read config file");
|
||||||
|
let topology: NetworkTopology =
|
||||||
|
serde_json::from_str(&config).expect("Failed to parse log as JSON");
|
||||||
|
|
||||||
|
if !topology.verify() {
|
||||||
|
panic!("Failed to verify the configuration file");
|
||||||
|
}
|
||||||
|
|
||||||
|
let network_size = value_t_or_exit!(matches, "size", u64);
|
||||||
|
let my_index = value_t_or_exit!(matches, "position", u64);
|
||||||
|
let interface = value_t_or_exit!(matches, "iface", String);
|
||||||
|
|
||||||
|
assert!(my_index < network_size);
|
||||||
|
|
||||||
|
let my_partition = identify_my_partition(&topology.partitions, my_index, network_size);
|
||||||
|
println!("My partition is {}", my_partition);
|
||||||
|
|
||||||
|
flush_iptables_rule();
|
||||||
|
|
||||||
|
if !insert_iptables_rule(my_partition) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
delete_tc_root(interface.as_str());
|
||||||
|
if !topology.interconnects.is_empty() && !insert_tc_root(interface.as_str()) {
|
||||||
|
flush_iptables_rule();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
topology.interconnects.iter().for_each(|i| {
|
||||||
|
if i.b as usize == my_partition {
|
||||||
|
let tos_string = i.a.to_string();
|
||||||
|
let class = format!("1:{}", i.a);
|
||||||
|
if !insert_tc_netem(
|
||||||
|
interface.as_str(),
|
||||||
|
class.as_str(),
|
||||||
|
tos_string.as_str(),
|
||||||
|
i.config.as_str(),
|
||||||
|
) {
|
||||||
|
flush_iptables_rule();
|
||||||
|
delete_tc_root(interface.as_str());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if !insert_tos_filter(interface.as_str(), tos_string.as_str(), class.as_str()) {
|
||||||
|
flush_iptables_rule();
|
||||||
|
delete_tc_netem(
|
||||||
|
interface.as_str(),
|
||||||
|
class.as_str(),
|
||||||
|
tos_string.as_str(),
|
||||||
|
i.config.as_str(),
|
||||||
|
);
|
||||||
|
delete_tc_root(interface.as_str());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn cleanup_network(matches: &ArgMatches) {
|
||||||
|
let config_path = PathBuf::from(value_t_or_exit!(matches, "file", String));
|
||||||
|
let config = fs::read_to_string(&config_path).expect("Unable to read config file");
|
||||||
|
let topology: NetworkTopology =
|
||||||
|
serde_json::from_str(&config).expect("Failed to parse log as JSON");
|
||||||
|
|
||||||
|
if !topology.verify() {
|
||||||
|
panic!("Failed to verify the configuration file");
|
||||||
|
}
|
||||||
|
|
||||||
|
let network_size = value_t_or_exit!(matches, "size", u64);
|
||||||
|
let my_index = value_t_or_exit!(matches, "position", u64);
|
||||||
|
let interface = value_t_or_exit!(matches, "iface", String);
|
||||||
|
|
||||||
|
assert!(my_index < network_size);
|
||||||
|
|
||||||
|
let my_partition = identify_my_partition(&topology.partitions, my_index, network_size);
|
||||||
|
println!("My partition is {}", my_partition);
|
||||||
|
|
||||||
|
topology.interconnects.iter().for_each(|i| {
|
||||||
|
if i.b as usize == my_partition {
|
||||||
|
let tos_string = i.a.to_string();
|
||||||
|
let class = format!("1:{}", i.a);
|
||||||
|
delete_tos_filter(interface.as_str(), class.as_str(), tos_string.as_str());
|
||||||
|
delete_tc_netem(
|
||||||
|
interface.as_str(),
|
||||||
|
class.as_str(),
|
||||||
|
tos_string.as_str(),
|
||||||
|
i.config.as_str(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
delete_tc_root(interface.as_str());
|
||||||
|
flush_iptables_rule();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn configure(_matches: &ArgMatches) {
|
||||||
|
let config = NetworkTopology::default();
|
||||||
|
let topology = serde_json::to_string(&config).expect("Failed to write as JSON");
|
||||||
|
|
||||||
|
println!("{}", topology);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn main() {
|
||||||
|
solana_logger::setup();
|
||||||
|
|
||||||
|
let matches = App::new(crate_name!())
|
||||||
|
.about(crate_description!())
|
||||||
|
.version(crate_version!())
|
||||||
|
.subcommand(
|
||||||
|
SubCommand::with_name("shape")
|
||||||
|
.about("Shape the network using config file")
|
||||||
|
.arg(
|
||||||
|
Arg::with_name("file")
|
||||||
|
.short("f")
|
||||||
|
.long("file")
|
||||||
|
.value_name("config file")
|
||||||
|
.takes_value(true)
|
||||||
|
.required(true)
|
||||||
|
.help("Location of the network config file"),
|
||||||
|
)
|
||||||
|
.arg(
|
||||||
|
Arg::with_name("size")
|
||||||
|
.short("s")
|
||||||
|
.long("size")
|
||||||
|
.value_name("network size")
|
||||||
|
.takes_value(true)
|
||||||
|
.required(true)
|
||||||
|
.help("Number of nodes in the network"),
|
||||||
|
)
|
||||||
|
.arg(
|
||||||
|
Arg::with_name("iface")
|
||||||
|
.short("i")
|
||||||
|
.long("iface")
|
||||||
|
.value_name("network interface name")
|
||||||
|
.takes_value(true)
|
||||||
|
.required(true)
|
||||||
|
.help("Name of network interface"),
|
||||||
|
)
|
||||||
|
.arg(
|
||||||
|
Arg::with_name("position")
|
||||||
|
.short("p")
|
||||||
|
.long("position")
|
||||||
|
.value_name("position of node")
|
||||||
|
.takes_value(true)
|
||||||
|
.required(true)
|
||||||
|
.help("Position of current node in the network"),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.subcommand(
|
||||||
|
SubCommand::with_name("cleanup")
|
||||||
|
.about("Remove the network filters using config file")
|
||||||
|
.arg(
|
||||||
|
Arg::with_name("file")
|
||||||
|
.short("f")
|
||||||
|
.long("file")
|
||||||
|
.value_name("config file")
|
||||||
|
.takes_value(true)
|
||||||
|
.required(true)
|
||||||
|
.help("Location of the network config file"),
|
||||||
|
)
|
||||||
|
.arg(
|
||||||
|
Arg::with_name("size")
|
||||||
|
.short("s")
|
||||||
|
.long("size")
|
||||||
|
.value_name("network size")
|
||||||
|
.takes_value(true)
|
||||||
|
.required(true)
|
||||||
|
.help("Number of nodes in the network"),
|
||||||
|
)
|
||||||
|
.arg(
|
||||||
|
Arg::with_name("iface")
|
||||||
|
.short("i")
|
||||||
|
.long("iface")
|
||||||
|
.value_name("network interface name")
|
||||||
|
.takes_value(true)
|
||||||
|
.required(true)
|
||||||
|
.help("Name of network interface"),
|
||||||
|
)
|
||||||
|
.arg(
|
||||||
|
Arg::with_name("position")
|
||||||
|
.short("p")
|
||||||
|
.long("position")
|
||||||
|
.value_name("position of node")
|
||||||
|
.takes_value(true)
|
||||||
|
.required(true)
|
||||||
|
.help("Position of current node in the network"),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.subcommand(SubCommand::with_name("configure").about("Generate a config file"))
|
||||||
|
.get_matches();
|
||||||
|
|
||||||
|
match matches.subcommand() {
|
||||||
|
("shape", Some(args_matches)) => shape_network(args_matches),
|
||||||
|
("cleanup", Some(args_matches)) => cleanup_network(args_matches),
|
||||||
|
("configure", Some(args_matches)) => configure(args_matches),
|
||||||
|
_ => {}
|
||||||
|
};
|
||||||
|
}
|
Loading…
Reference in New Issue