Add fullnode --dynamic-port-range option

This commit is contained in:
Michael Vines 2019-04-12 18:17:34 -07:00
parent c7a7d6db84
commit f6aa90e193
11 changed files with 203 additions and 101 deletions

View File

@ -1,6 +1,6 @@
## Testnet Participation
This document describes how to participate in a public testnet as a
validator node using the *Grandview v0.13* release.
This document describes how to participate in the beta testnet as a
validator node.
Please note some of the information and instructions described here may change
in future releases.
@ -125,6 +125,12 @@ $ RUST_LOG=info solana-gossip --network ${ip:?}:8001
Congratulations, you're now participating in the testnet cluster!
#### Controlling local network port allocation
By default the validator will dynamically select available network ports in the
8000-10000 range, and may be overridden with `--dynamic-port-range`. For
example, `fullnode-x.sh --dynamic-port-range 11000-11010 ...` will restrict the
validator to ports 11000-11011.
### Sharing Metrics From Your Validator
If you'd like to share metrics perform the following steps before starting the
validator node:

View File

@ -66,6 +66,19 @@ echo --- Creating tarball
cp solana-release-cuda/bin/solana-fullnode solana-release/bin/solana-fullnode-cuda
cp -a scripts multinode-demo solana-release/
# Add a wrapper script for fullnode.sh
# TODO: Remove multinode/... from tarball
cat > solana-release/bin/fullnode.sh <<'EOF'
#!/usr/bin/env bash
set -e
cd "$(dirname "$0")"/..
export USE_INSTALL=1
exec multinode-demo/fullnode.sh "$@"
EOF
chmod +x solana-release/bin/fullnode.sh
# Add a wrapper script for fullnode-x.sh
# TODO: Remove multinode/... from tarball
cat > solana-release/bin/fullnode-x.sh <<'EOF'
#!/usr/bin/env bash
set -e

View File

@ -31,11 +31,12 @@ use rand::{thread_rng, Rng};
use rayon::prelude::*;
use solana_metrics::counter::Counter;
use solana_metrics::{influxdb, submit};
use solana_netutil::{bind_in_range, bind_to, find_available_port_in_range, multi_bind_in_range};
use solana_netutil::{
bind_in_range, bind_to, find_available_port_in_range, multi_bind_in_range, PortRange,
};
use solana_runtime::bloom::Bloom;
use solana_sdk::hash::Hash;
use solana_sdk::pubkey::Pubkey;
use solana_sdk::rpc_port;
use solana_sdk::signature::{Keypair, KeypairUtil, Signable, Signature};
use solana_sdk::timing::{duration_as_ms, timestamp};
use solana_sdk::transaction::Transaction;
@ -48,7 +49,7 @@ use std::sync::{Arc, RwLock};
use std::thread::{sleep, Builder, JoinHandle};
use std::time::{Duration, Instant};
pub const FULLNODE_PORT_RANGE: (u16, u16) = (8000, 10_000);
pub const FULLNODE_PORT_RANGE: PortRange = (8000, 10_000);
/// The fanout for Ledger Replication
pub const DATA_PLANE_FANOUT: usize = 200;
@ -1510,7 +1511,7 @@ impl Node {
},
}
}
fn get_gossip_port(gossip_addr: &SocketAddr) -> (u16, UdpSocket) {
fn get_gossip_port(gossip_addr: &SocketAddr, port_range: PortRange) -> (u16, UdpSocket) {
if gossip_addr.port() != 0 {
(
gossip_addr.port(),
@ -1519,27 +1520,29 @@ impl Node {
}),
)
} else {
Self::bind()
Self::bind(port_range)
}
}
fn bind() -> (u16, UdpSocket) {
bind_in_range(FULLNODE_PORT_RANGE).expect("Failed to bind")
fn bind(port_range: PortRange) -> (u16, UdpSocket) {
bind_in_range(port_range).expect("Failed to bind")
}
pub fn new_with_external_ip(pubkey: &Pubkey, gossip_addr: &SocketAddr) -> Node {
let (gossip_port, gossip) = Self::get_gossip_port(gossip_addr);
pub fn new_with_external_ip(
pubkey: &Pubkey,
gossip_addr: &SocketAddr,
port_range: PortRange,
) -> Node {
let (gossip_port, gossip) = Self::get_gossip_port(gossip_addr, port_range);
let (tvu_port, tvu_sockets) =
multi_bind_in_range(FULLNODE_PORT_RANGE, 8).expect("tvu multi_bind");
let (tvu_port, tvu_sockets) = multi_bind_in_range(port_range, 8).expect("tvu multi_bind");
let (tpu_port, tpu_sockets) =
multi_bind_in_range(FULLNODE_PORT_RANGE, 32).expect("tpu multi_bind");
let (tpu_port, tpu_sockets) = multi_bind_in_range(port_range, 32).expect("tpu multi_bind");
let (tpu_via_blobs_port, tpu_via_blobs_sockets) =
multi_bind_in_range(FULLNODE_PORT_RANGE, 8).expect("tpu multi_bind");
multi_bind_in_range(port_range, 8).expect("tpu multi_bind");
let (_, repair) = Self::bind();
let (_, broadcast) = Self::bind();
let (_, retransmit) = Self::bind();
let (_, repair) = Self::bind(port_range);
let (_, broadcast) = Self::bind(port_range);
let (_, retransmit) = Self::bind(port_range);
let info = ContactInfo::new(
pubkey,
@ -1547,9 +1550,9 @@ impl Node {
SocketAddr::new(gossip_addr.ip(), tvu_port),
SocketAddr::new(gossip_addr.ip(), tpu_port),
SocketAddr::new(gossip_addr.ip(), tpu_via_blobs_port),
"0.0.0.0:0".parse().unwrap(),
SocketAddr::new(gossip_addr.ip(), rpc_port::DEFAULT_RPC_PORT),
SocketAddr::new(gossip_addr.ip(), rpc_port::DEFAULT_RPC_PUBSUB_PORT),
socketaddr_any!(),
socketaddr_any!(),
socketaddr_any!(),
0,
);
trace!("new ContactInfo: {:?}", info);
@ -1568,14 +1571,18 @@ impl Node {
},
}
}
pub fn new_replicator_with_external_ip(pubkey: &Pubkey, gossip_addr: &SocketAddr) -> Node {
let mut new = Self::new_with_external_ip(pubkey, gossip_addr);
let (storage_port, storage_socket) = Self::bind();
pub fn new_replicator_with_external_ip(
pubkey: &Pubkey,
gossip_addr: &SocketAddr,
port_range: PortRange,
) -> Node {
let mut new = Self::new_with_external_ip(pubkey, gossip_addr, port_range);
let (storage_port, storage_socket) = Self::bind(port_range);
new.info.storage_addr = SocketAddr::new(gossip_addr.ip(), storage_port);
new.sockets.storage = Some(storage_socket);
let empty = "0.0.0.0:0".parse().unwrap();
let empty = socketaddr_any!();
new.info.tpu = empty;
new.info.tpu_via_blobs = empty;
new.sockets.tpu = vec![];
@ -1904,7 +1911,11 @@ mod tests {
#[test]
fn new_with_external_ip_test_random() {
let ip = Ipv4Addr::from(0);
let node = Node::new_with_external_ip(&Pubkey::new_rand(), &socketaddr!(ip, 0));
let node = Node::new_with_external_ip(
&Pubkey::new_rand(),
&socketaddr!(ip, 0),
FULLNODE_PORT_RANGE,
);
check_node_sockets(&node, IpAddr::V4(ip), FULLNODE_PORT_RANGE);
}
@ -1917,7 +1928,11 @@ mod tests {
.expect("Failed to bind")
.0
};
let node = Node::new_with_external_ip(&Pubkey::new_rand(), &socketaddr!(0, port));
let node = Node::new_with_external_ip(
&Pubkey::new_rand(),
&socketaddr!(0, port),
FULLNODE_PORT_RANGE,
);
check_node_sockets(&node, ip, FULLNODE_PORT_RANGE);
@ -1927,7 +1942,11 @@ mod tests {
#[test]
fn new_replicator_external_ip_test() {
let ip = Ipv4Addr::from(0);
let node = Node::new_replicator_with_external_ip(&Pubkey::new_rand(), &socketaddr!(ip, 0));
let node = Node::new_replicator_with_external_ip(
&Pubkey::new_rand(),
&socketaddr!(ip, 0),
FULLNODE_PORT_RANGE,
);
let ip = IpAddr::V4(ip);
check_socket(&node.sockets.storage.unwrap(), ip, FULLNODE_PORT_RANGE);

View File

@ -1,7 +1,10 @@
use bincode::serialize;
use solana_sdk::pubkey::Pubkey;
#[cfg(test)]
use solana_sdk::rpc_port;
use solana_sdk::signature::{Keypair, KeypairUtil, Signable, Signature};
#[cfg(test)]
use solana_sdk::signature::{Keypair, KeypairUtil};
use solana_sdk::signature::{Signable, Signature};
use solana_sdk::timing::timestamp;
use std::cmp::{Ord, Ordering, PartialEq, PartialOrd};
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
@ -141,16 +144,19 @@ impl ContactInfo {
0,
)
}
fn next_port(addr: &SocketAddr, nxt: u16) -> SocketAddr {
let mut nxt_addr = *addr;
nxt_addr.set_port(addr.port() + nxt);
nxt_addr
}
#[cfg(test)]
fn new_with_pubkey_socketaddr(pubkey: &Pubkey, bind_addr: &SocketAddr) -> Self {
fn next_port(addr: &SocketAddr, nxt: u16) -> SocketAddr {
let mut nxt_addr = *addr;
nxt_addr.set_port(addr.port() + nxt);
nxt_addr
}
let tpu_addr = *bind_addr;
let gossip_addr = Self::next_port(&bind_addr, 1);
let tvu_addr = Self::next_port(&bind_addr, 2);
let tpu_via_blobs_addr = Self::next_port(&bind_addr, 3);
let gossip_addr = next_port(&bind_addr, 1);
let tvu_addr = next_port(&bind_addr, 2);
let tpu_via_blobs_addr = next_port(&bind_addr, 3);
let rpc_addr = SocketAddr::new(bind_addr.ip(), rpc_port::DEFAULT_RPC_PORT);
let rpc_pubsub_addr = SocketAddr::new(bind_addr.ip(), rpc_port::DEFAULT_RPC_PUBSUB_PORT);
Self::new(
@ -165,7 +171,9 @@ impl ContactInfo {
timestamp(),
)
}
pub fn new_with_socketaddr(bind_addr: &SocketAddr) -> Self {
#[cfg(test)]
pub(crate) fn new_with_socketaddr(bind_addr: &SocketAddr) -> Self {
let keypair = Keypair::new();
Self::new_with_pubkey_socketaddr(&keypair.pubkey(), bind_addr)
}
@ -185,11 +193,13 @@ impl ContactInfo {
timestamp(),
)
}
fn is_valid_ip(addr: IpAddr) -> bool {
!(addr.is_unspecified() || addr.is_multicast())
// || (addr.is_loopback() && !cfg_test))
// TODO: boot loopback in production networks
}
/// port must not be 0
/// ip must be specified and not mulitcast
/// loopback ip is only allowed in tests

View File

@ -142,24 +142,32 @@ impl Fullnode {
let storage_state = StorageState::new();
let rpc_service = JsonRpcService::new(
&cluster_info,
SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), node.info.rpc.port()),
storage_state.clone(),
config.rpc_config.clone(),
bank_forks.clone(),
&exit,
);
let rpc_service = if node.info.rpc.port() == 0 {
None
} else {
Some(JsonRpcService::new(
&cluster_info,
SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), node.info.rpc.port()),
storage_state.clone(),
config.rpc_config.clone(),
bank_forks.clone(),
&exit,
))
};
let subscriptions = Arc::new(RpcSubscriptions::default());
let rpc_pubsub_service = PubSubService::new(
&subscriptions,
SocketAddr::new(
IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)),
node.info.rpc_pubsub.port(),
),
&exit,
);
let rpc_pubsub_service = if node.info.rpc_pubsub.port() == 0 {
None
} else {
Some(PubSubService::new(
&subscriptions,
SocketAddr::new(
IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)),
node.info.rpc_pubsub.port(),
),
&exit,
))
};
let gossip_service = GossipService::new(
&cluster_info,
@ -243,8 +251,8 @@ impl Fullnode {
Self {
id,
gossip_service,
rpc_service: Some(rpc_service),
rpc_pubsub_service: Some(rpc_pubsub_service),
rpc_service,
rpc_pubsub_service,
tpu,
tvu,
exit,

View File

@ -1,7 +1,7 @@
//! The `local_vote_signer_service` can be started locally to sign fullnode votes
use crate::cluster_info::FULLNODE_PORT_RANGE;
use crate::service::Service;
use solana_netutil::PortRange;
use solana_vote_signer::rpc::VoteSignerRpcService;
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
use std::sync::atomic::{AtomicBool, Ordering};
@ -24,8 +24,8 @@ impl Service for LocalVoteSignerService {
impl LocalVoteSignerService {
#[allow(clippy::new_ret_no_self)]
pub fn new() -> (Self, SocketAddr) {
let addr = match solana_netutil::find_available_port_in_range(FULLNODE_PORT_RANGE) {
pub fn new(port_range: PortRange) -> (Self, SocketAddr) {
let addr = match solana_netutil::find_available_port_in_range(port_range) {
Ok(port) => SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), port),
Err(_e) => panic!("Failed to find an available port for local vote signer service"),
};

View File

@ -5,15 +5,27 @@ use solana::contact_info::ContactInfo;
use solana::fullnode::{Fullnode, FullnodeConfig};
use solana::local_vote_signer_service::LocalVoteSignerService;
use solana::service::Service;
use solana_netutil::parse_port_range;
use solana_sdk::signature::{read_keypair, Keypair, KeypairUtil};
use std::fs::File;
use std::process::exit;
use std::sync::Arc;
fn port_range_validator(port_range: String) -> Result<(), String> {
if parse_port_range(&port_range).is_some() {
Ok(())
} else {
Err("Invalid port range".to_string())
}
}
fn main() {
solana_logger::setup();
solana_metrics::set_panic_hook("fullnode");
let default_dynamic_port_range =
&format!("{}-{}", FULLNODE_PORT_RANGE.0, FULLNODE_PORT_RANGE.1);
let matches = App::new(crate_name!()).about(crate_description!())
.version(crate_version!())
.arg(
@ -131,6 +143,15 @@ fn main() {
.takes_value(true)
.help("Gossip port number for the node"),
)
.arg(
clap::Arg::with_name("dynamic_port_range")
.long("dynamic-port-range")
.value_name("MIN_PORT-MAX_PORT")
.takes_value(true)
.default_value(default_dynamic_port_range)
.validator(port_range_validator)
.help("Range to use for dynamically assigned ports"),
)
.get_matches();
let mut fullnode_config = FullnodeConfig::default();
@ -170,10 +191,14 @@ fn main() {
.value_of("rpc_drone_address")
.map(|address| address.parse().expect("failed to parse drone address"));
let dynamic_port_range = parse_port_range(matches.value_of("dynamic_port_range").unwrap())
.expect("invalid dynamic_port_range");
let gossip_addr = {
let mut addr = solana_netutil::parse_port_or_addr(
matches.value_of("gossip_port"),
FULLNODE_PORT_RANGE.0 + 1,
solana_netutil::find_available_port_in_range(dynamic_port_range)
.expect("unable to allocate gossip_port"),
);
if matches.is_present("public_address") {
addr.set_ip(solana_netutil::get_public_ip_addr().unwrap());
@ -199,31 +224,23 @@ fn main() {
)
} else {
// Run a local vote signer if a vote signer service address was not provided
let (signer_service, signer_addr) = LocalVoteSignerService::new();
let (signer_service, signer_addr) = LocalVoteSignerService::new(dynamic_port_range);
(Some(signer_service), signer_addr)
};
let (rpc_port, rpc_pubsub_port) = if let Some(port) = matches.value_of("rpc_port") {
let port_number = port.to_string().parse().expect("integer");
if port_number == 0 {
eprintln!("Invalid RPC port requested: {:?}", port);
exit(1);
}
(port_number, port_number + 1)
} else {
(
solana_netutil::find_available_port_in_range(FULLNODE_PORT_RANGE)
.expect("unable to allocate rpc_port"),
solana_netutil::find_available_port_in_range(FULLNODE_PORT_RANGE)
.expect("unable to allocate rpc_pubsub_port"),
)
};
let init_complete_file = matches.value_of("init_complete_file");
fullnode_config.blockstream = matches.value_of("blockstream").map(|s| s.to_string());
let keypair = Arc::new(keypair);
let mut node = Node::new_with_external_ip(&keypair.pubkey(), &gossip_addr);
node.info.rpc.set_port(rpc_port);
node.info.rpc_pubsub.set_port(rpc_pubsub_port);
let mut node = Node::new_with_external_ip(&keypair.pubkey(), &gossip_addr, dynamic_port_range);
if let Some(port) = matches.value_of("rpc_port") {
let port_number = port.to_string().parse().expect("integer");
if port_number == 0 {
eprintln!("Invalid RPC port requested: {:?}", port);
exit(1);
}
node.info.rpc.set_port(port_number);
node.info.rpc_pubsub.set_port(port_number + 1);
};
let fullnode = Fullnode::new(
node,

View File

@ -35,6 +35,9 @@ while [[ ${1:0:1} = - ]]; do
elif [[ $1 = --rpc-port ]]; then
extra_fullnode_args+=("$1" "$2")
shift 2
elif [[ $1 = --dynamic-port-range ]]; then
extra_fullnode_args+=("$1" "$2")
shift 2
else
echo "Unknown argument: $1"
exit 1
@ -74,6 +77,7 @@ $program \
--vote-account "$bootstrap_leader_vote_id" \
--ledger "$SOLANA_CONFIG_DIR"/bootstrap-leader-ledger \
--accounts "$SOLANA_CONFIG_DIR"/bootstrap-leader-accounts \
--gossip-port 8001 \
--rpc-port 8899 \
--rpc-drone-address 127.0.0.1:9900 \
"${extra_fullnode_args[@]}" \

View File

@ -13,7 +13,7 @@ if [[ $1 = -h ]]; then
fullnode_usage "$@"
fi
gossip_port=9000
gossip_port=
extra_fullnode_args=()
self_setup=0
setup_stakes=1
@ -51,10 +51,14 @@ while [[ ${1:0:1} = - ]]; do
shift
elif [[ $1 = --gossip-port ]]; then
gossip_port=$2
extra_fullnode_args+=("$1" "$2")
shift 2
elif [[ $1 = --rpc-port ]]; then
extra_fullnode_args+=("$1" "$2")
shift 2
elif [[ $1 = --dynamic-port-range ]]; then
extra_fullnode_args+=("$1" "$2")
shift 2
else
echo "Unknown argument: $1"
exit 1
@ -117,26 +121,16 @@ if ((!self_setup)); then
fullnode_vote_id_path=$SOLANA_CONFIG_DIR/fullnode-vote-id.json
ledger_config_dir=$SOLANA_CONFIG_DIR/fullnode-ledger
accounts_config_dir=$SOLANA_CONFIG_DIR/fullnode-accounts
if [[ -z $gossip_port ]]; then
extra_fullnode_args+=("--gossip-port" 9000)
fi
else
mkdir -p "$SOLANA_CONFIG_DIR"
fullnode_id_path=$SOLANA_CONFIG_DIR/fullnode-id-x$self_setup_label.json
fullnode_vote_id_path=$SOLANA_CONFIG_DIR/fullnode-vote-id-x$self_setup_label.json
[[ -f "$fullnode_id_path" ]] || $solana_keygen -o "$fullnode_id_path"
[[ -f "$fullnode_vote_id_path" ]] || $solana_keygen -o "$fullnode_vote_id_path"
echo "Finding a port.."
# Find an available port in the range 9100-9899
(( gossip_port = 9100 + ($$ % 800) ))
while true; do
(( gossip_port = gossip_port >= 9900 ? 9100 : ++gossip_port ))
echo "Testing $gossip_port"
if ! nc -w 10 -z 127.0.0.1 $gossip_port; then
echo "Selected gossip_port $gossip_port"
break;
fi
echo "Port $gossip_port is in use"
done
ledger_config_dir=$SOLANA_CONFIG_DIR/fullnode-ledger-x$self_setup_label
accounts_config_dir=$SOLANA_CONFIG_DIR/fullnode-accounts-x$self_setup_label
fi
@ -188,7 +182,6 @@ while true; do
trap 'kill "$pid" && wait "$pid"' INT TERM ERR
$program \
--gossip-port "$gossip_port" \
--identity "$fullnode_id_path" \
--voting-keypair "$fullnode_vote_id_path" \
--vote-account "$fullnode_vote_id" \

View File

@ -17,6 +17,8 @@ pub struct UdpSocketPair {
pub sender: UdpSocket, // Locally bound socket to send via public address
}
pub type PortRange = (u16, u16);
/// Tries to determine the public IP address of this machine
pub fn get_public_ip_addr() -> Result<IpAddr, String> {
let body = reqwest::get("http://ifconfig.co/ip")
@ -48,6 +50,26 @@ pub fn parse_port_or_addr(optstr: Option<&str>, default_port: u16) -> SocketAddr
}
}
pub fn parse_port_range(port_range: &str) -> Option<PortRange> {
let ports: Vec<&str> = port_range.split('-').collect();
if ports.len() != 2 {
return None;
}
let start_port = ports[0].parse();
let end_port = ports[1].parse();
if start_port.is_err() || end_port.is_err() {
return None;
}
let start_port = start_port.unwrap();
let end_port = end_port.unwrap();
if end_port < start_port {
return None;
}
Some((start_port, end_port))
}
fn find_eth0ish_ip_addr(
ifaces: &mut Vec<datalink::NetworkInterface>,
enable_ipv6: bool,
@ -122,7 +144,7 @@ fn udp_socket(reuseaddr: bool) -> io::Result<Socket> {
Ok(sock)
}
pub fn bind_in_range(range: (u16, u16)) -> io::Result<(u16, UdpSocket)> {
pub fn bind_in_range(range: PortRange) -> io::Result<(u16, UdpSocket)> {
let sock = udp_socket(false)?;
let (start, end) = range;
@ -151,7 +173,7 @@ pub fn bind_in_range(range: (u16, u16)) -> io::Result<(u16, UdpSocket)> {
}
// binds many sockets to the same port in a range
pub fn multi_bind_in_range(range: (u16, u16), num: usize) -> io::Result<(u16, Vec<UdpSocket>)> {
pub fn multi_bind_in_range(range: PortRange, num: usize) -> io::Result<(u16, Vec<UdpSocket>)> {
let mut sockets = Vec::with_capacity(num);
let port = {
@ -176,7 +198,7 @@ pub fn bind_to(port: u16, reuseaddr: bool) -> io::Result<UdpSocket> {
}
}
pub fn find_available_port_in_range(range: (u16, u16)) -> io::Result<u16> {
pub fn find_available_port_in_range(range: PortRange) -> io::Result<u16> {
let (start, end) = range;
let mut tries_left = end - start;
let mut rand_port = thread_rng().gen_range(start, end);
@ -303,6 +325,15 @@ mod tests {
assert_eq!(p3.port(), 1);
}
#[test]
fn test_parse_port_range() {
assert_eq!(parse_port_range("garbage"), None);
assert_eq!(parse_port_range("1-"), None);
assert_eq!(parse_port_range("1-2"), Some((1, 2)));
assert_eq!(parse_port_range("1-2-3"), None);
assert_eq!(parse_port_range("2-1"), None);
}
#[test]
fn test_bind() {
assert_eq!(bind_in_range((2000, 2001)).unwrap().0, 2000);

View File

@ -1,5 +1,5 @@
use clap::{crate_description, crate_name, crate_version, App, Arg};
use solana::cluster_info::Node;
use solana::cluster_info::{Node, FULLNODE_PORT_RANGE};
use solana::contact_info::ContactInfo;
use solana::replicator::Replicator;
use solana::socketaddr;
@ -67,7 +67,8 @@ fn main() {
}
addr
};
let node = Node::new_replicator_with_external_ip(&keypair.pubkey(), &gossip_addr);
let node =
Node::new_replicator_with_external_ip(&keypair.pubkey(), &gossip_addr, FULLNODE_PORT_RANGE);
println!(
"replicating the data with keypair={:?} gossip_addr={:?}",