zecfoundation
/
zebra
mirror of https://github.com/ZcashFoundation/zebra.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add metrics for initial peer network protocol versions (#2804) 

* Add tracing and metrics for seed peer DNS resolution

* Add a grafana dashboard for seed peers

Currently this just shows the initial peer count from each seed.

* Add tracing and metrics for peer network protocol versions

* Update peers dashboard with network protocol versions

* Show peer network protocol versions for each seeder in dashboard

* Add per-seed filter to dashboard

Co-authored-by: Deirdre Connolly <deirdre@zfnd.org>
This commit is contained in:
teor 2021-09-30 04:08:20 +10:00 committed by GitHub
parent 679b4768df
commit 20b2e0549e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 1031 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

963
grafana/peers.json Normal file
View File

@ -0,0 +1,963 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"gnetId": null,
"graphTooltip": 0,
"id": 7,
"iteration": 1632879075996,
"links": [],
"panels": [
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 0
},
"hiddenSeries": false,
"id": 2,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.7",
"pointradius": 2,
"points": false,
"renderer": "flot",
"repeat": "job",
"scopedVars": {
"job": {
"selected": false,
"text": "zebrad-mainnet",
"value": "zebrad-mainnet"
}
},
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"exemplar": true,
"expr": "sum by (remote_version, seed) (zcash_net_peers_initial{job=\"$job\",seed=~\"$seed\"} * on(remote_ip) group_left(remote_version) zcash_net_peers_connected{job=\"$job\"})",
"instant": false,
"interval": "",
"legendFormat": "{{remote_version}} - {{seed}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Compatible Seed Peers - $job",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:65",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:66",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 0
},
"hiddenSeries": false,
"id": 12,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.7",
"pointradius": 2,
"points": false,
"renderer": "flot",
"repeatIteration": 1632879075996,
"repeatPanelId": 2,
"scopedVars": {
"job": {
"selected": false,
"text": "zebrad-testnet",
"value": "zebrad-testnet"
}
},
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"exemplar": true,
"expr": "sum by (remote_version, seed) (zcash_net_peers_initial{job=\"$job\",seed=~\"$seed\"} * on(remote_ip) group_left(remote_version) zcash_net_peers_connected{job=\"$job\"})",
"instant": false,
"interval": "",
"legendFormat": "{{remote_version}} - {{seed}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Compatible Seed Peers - $job",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:65",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:66",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 9
},
"hiddenSeries": false,
"id": 11,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.7",
"pointradius": 2,
"points": false,
"renderer": "flot",
"repeat": "job",
"scopedVars": {
"job": {
"selected": false,
"text": "zebrad-mainnet",
"value": "zebrad-mainnet"
}
},
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"exemplar": true,
"expr": "sum by (remote_version, seed) (zcash_net_peers_initial{job=\"$job\",seed=~\"$seed\"} * on(remote_ip) group_left(remote_version) zcash_net_peers_obsolete{job=\"$job\"})",
"instant": false,
"interval": "",
"legendFormat": "{{remote_version}} - {{seed}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Obsolete Seed Peers - $job",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:65",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:66",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 9
},
"hiddenSeries": false,
"id": 13,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.7",
"pointradius": 2,
"points": false,
"renderer": "flot",
"repeatIteration": 1632879075996,
"repeatPanelId": 11,
"scopedVars": {
"job": {
"selected": false,
"text": "zebrad-testnet",
"value": "zebrad-testnet"
}
},
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"exemplar": true,
"expr": "sum by (remote_version, seed) (zcash_net_peers_initial{job=\"$job\",seed=~\"$seed\"} * on(remote_ip) group_left(remote_version) zcash_net_peers_obsolete{job=\"$job\"})",
"instant": false,
"interval": "",
"legendFormat": "{{remote_version}} - {{seed}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Obsolete Seed Peers - $job",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:65",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:66",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 17
},
"hiddenSeries": false,
"id": 4,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.7",
"pointradius": 2,
"points": false,
"renderer": "flot",
"repeat": "job",
"scopedVars": {
"job": {
"selected": false,
"text": "zebrad-mainnet",
"value": "zebrad-mainnet"
}
},
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"exemplar": true,
"expr": "sum by (remote_version) (zcash_net_peers_connected{job=\"$job\"})",
"interval": "",
"legendFormat": "{{remote_version}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Compatible Peers - $job",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:65",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:66",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 17
},
"hiddenSeries": false,
"id": 14,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.7",
"pointradius": 2,
"points": false,
"renderer": "flot",
"repeatIteration": 1632879075996,
"repeatPanelId": 4,
"scopedVars": {
"job": {
"selected": false,
"text": "zebrad-testnet",
"value": "zebrad-testnet"
}
},
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"exemplar": true,
"expr": "sum by (remote_version) (zcash_net_peers_connected{job=\"$job\"})",
"interval": "",
"legendFormat": "{{remote_version}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Compatible Peers - $job",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:65",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:66",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 25
},
"hiddenSeries": false,
"id": 7,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.7",
"pointradius": 2,
"points": false,
"renderer": "flot",
"repeat": "job",
"scopedVars": {
"job": {
"selected": false,
"text": "zebrad-mainnet",
"value": "zebrad-mainnet"
}
},
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"exemplar": true,
"expr": "sum by (remote_version) (zcash_net_peers_obsolete{job=\"$job\"})",
"interval": "",
"legendFormat": "{{remote_version}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Obsolete Peers - $job",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:65",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:66",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 25
},
"hiddenSeries": false,
"id": 15,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.7",
"pointradius": 2,
"points": false,
"renderer": "flot",
"repeatIteration": 1632879075996,
"repeatPanelId": 7,
"scopedVars": {
"job": {
"selected": false,
"text": "zebrad-testnet",
"value": "zebrad-testnet"
}
},
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"exemplar": true,
"expr": "sum by (remote_version) (zcash_net_peers_obsolete{job=\"$job\"})",
"interval": "",
"legendFormat": "{{remote_version}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Obsolete Peers - $job",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:65",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:66",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
}
],
"refresh": "5s",
"schemaVersion": 27,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"allValue": null,
"current": {
"selected": false,
"text": [
"All"
],
"value": [
"$__all"
]
},
"datasource": null,
"definition": "label_values(zcash_net_in_bytes_total, job)",
"description": null,
"error": null,
"hide": 0,
"includeAll": true,
"label": null,
"multi": true,
"name": "job",
"options": [],
"query": {
"query": "label_values(zcash_net_in_bytes_total, job)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 1,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": ".+",
"current": {
"selected": true,
"tags": [],
"text": [
"dnsseed.str4d.xyz:8233",
"dnsseed.testnet.z.cash:18233",
"dnsseed.z.cash:8233",
"mainnet.is.yolo.money:8233",
"mainnet.seeder.zfnd.org:8233",
"testnet.is.yolo.money:18233",
"testnet.seeder.zfnd.org:18233"
],
"value": [
"dnsseed.str4d.xyz:8233",
"dnsseed.testnet.z.cash:18233",
"dnsseed.z.cash:8233",
"mainnet.is.yolo.money:8233",
"mainnet.seeder.zfnd.org:8233",
"testnet.is.yolo.money:18233",
"testnet.seeder.zfnd.org:18233"
]
},
"datasource": null,
"definition": "label_values(zcash_net_peers_initial, seed)",
"description": null,
"error": null,
"hide": 0,
"includeAll": true,
"label": null,
"multi": true,
"name": "seed",
"options": [],
"query": {
"query": "label_values(zcash_net_peers_initial, seed)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 1,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
}
]
},
"time": {
"from": "now-15m",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "peers",
"uid": "S29TgUH7k",
"version": 22
}

30
zebra-network/src/config.rs
View File

@ -144,13 +144,37 @@ impl Config {
let fut = tokio::time::timeout(crate::constants::DNS_LOOKUP_TIMEOUT, fut);
match fut.await {
Ok(Ok(ips)) => Ok(ips.map(canonical_socket_addr).collect()),
Ok(Ok(ip_addrs)) => {
let ip_addrs: Vec<SocketAddr> = ip_addrs.map(canonical_socket_addr).collect();
// if we're logging at debug level,
// the full list of IP addresses will be shown in the log message
let debug_span = debug_span!("", remote_ip_addrs = ?ip_addrs);
let _span_guard = debug_span.enter();
info!(seed = ?host, remote_ip_count = ?ip_addrs.len(), "resolved seed peer IP addresses");
for ip in &ip_addrs {
// Count each initial peer, recording the seed config and resolved IP address.
//
// If an IP is returned by multiple seeds,
// each duplicate adds 1 to the initial peer count.
// (But we only make one initial connection attempt to each IP.)
metrics::counter!(
"zcash.net.peers.initial",
1,
"seed" => host.to_string(),
"remote_ip" => ip.to_string()
);
}
Ok(ip_addrs.into_iter().collect())
}
Ok(Err(e)) => {
tracing::info!(?host, ?e, "DNS error resolving peer IP address");
tracing::info!(?host, ?e, "DNS error resolving peer IP addresses");
Err(e.into())
}
Err(e) => {
tracing::info!(?host, ?e, "DNS timeout resolving peer IP address");
tracing::info!(?host, ?e, "DNS timeout resolving peer IP addresses");
Err(e.into())
}
}

35
zebra-network/src/peer/handshake.rs
View File

@ -1,4 +1,5 @@
use std::{
cmp::min,
collections::HashSet,
fmt,
future::Future,
@ -588,8 +589,42 @@ pub async fn negotiate_version(
let height = latest_chain_tip.best_tip_height();
let min_version = Version::min_remote_for_height(config.network, height);
if remote_version < min_version {
debug!(
remote_ip = ?their_addr,
?remote_version,
?min_version,
"disconnecting from peer with obsolete network protocol version"
);
metrics::counter!(
"zcash.net.peers.obsolete",
1,
"remote_ip" => their_addr.to_string(),
"remote_version" => remote_version.to_string(),
"min_version" => min_version.to_string(),
);
// Disconnect if peer is using an obsolete version.
Err(HandshakeError::ObsoleteVersion(remote_version))?;
} else {
let negotiated_version = min(constants::CURRENT_NETWORK_PROTOCOL_VERSION, remote_version);
debug!(
remote_ip = ?their_addr,
?remote_version,
?negotiated_version,
?min_version,
"negotiated network protocol version with peer"
);
metrics::counter!(
"zcash.net.peers.connected",
1,
"remote_ip" => their_addr.to_string(),
"remote_version" => remote_version.to_string(),
"negotiated_version" => negotiated_version.to_string(),
"min_version" => min_version.to_string(),
);
}
peer_conn.send(Message::Verack).await?;

6
zebra-network/src/protocol/external/types.rs vendored
View File

@ -41,6 +41,12 @@ impl From<Network> for Magic {
#[cfg_attr(any(test, feature = "proptest-impl"), derive(Arbitrary))]
pub struct Version(pub u32);
impl fmt::Display for Version {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(&self.0.to_string())
}
}
impl Version {
/// Returns the minimum remote node network protocol version for `network` and
/// `height`. Zebra disconnects from peers with lower versions.