From 20b2e0549eb6d4abaca83c81cb884d964dbd0d84 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 30 Sep 2021 04:08:20 +1000 Subject: [PATCH] Add metrics for initial peer network protocol versions (#2804) * Add tracing and metrics for seed peer DNS resolution * Add a grafana dashboard for seed peers Currently this just shows the initial peer count from each seed. * Add tracing and metrics for peer network protocol versions * Update peers dashboard with network protocol versions * Show peer network protocol versions for each seeder in dashboard * Add per-seed filter to dashboard Co-authored-by: Deirdre Connolly --- grafana/peers.json | 963 +++++++++++++++++++ zebra-network/src/config.rs | 30 +- zebra-network/src/peer/handshake.rs | 35 + zebra-network/src/protocol/external/types.rs | 6 + 4 files changed, 1031 insertions(+), 3 deletions(-) create mode 100644 grafana/peers.json diff --git a/grafana/peers.json b/grafana/peers.json new file mode 100644 index 000000000..afb7589ba --- /dev/null +++ b/grafana/peers.json @@ -0,0 +1,963 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 7, + "iteration": 1632879075996, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "job", + "scopedVars": { + "job": { + "selected": false, + "text": "zebrad-mainnet", + "value": "zebrad-mainnet" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (remote_version, seed) (zcash_net_peers_initial{job=\"$job\",seed=~\"$seed\"} * on(remote_ip) group_left(remote_version) zcash_net_peers_connected{job=\"$job\"})", + "instant": false, + "interval": "", + "legendFormat": "{{remote_version}} - {{seed}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Compatible Seed Peers - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:65", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:66", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeatIteration": 1632879075996, + "repeatPanelId": 2, + "scopedVars": { + "job": { + "selected": false, + "text": "zebrad-testnet", + "value": "zebrad-testnet" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (remote_version, seed) (zcash_net_peers_initial{job=\"$job\",seed=~\"$seed\"} * on(remote_ip) group_left(remote_version) zcash_net_peers_connected{job=\"$job\"})", + "instant": false, + "interval": "", + "legendFormat": "{{remote_version}} - {{seed}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Compatible Seed Peers - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:65", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:66", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "hiddenSeries": false, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "job", + "scopedVars": { + "job": { + "selected": false, + "text": "zebrad-mainnet", + "value": "zebrad-mainnet" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (remote_version, seed) (zcash_net_peers_initial{job=\"$job\",seed=~\"$seed\"} * on(remote_ip) group_left(remote_version) zcash_net_peers_obsolete{job=\"$job\"})", + "instant": false, + "interval": "", + "legendFormat": "{{remote_version}} - {{seed}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Obsolete Seed Peers - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:65", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:66", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "hiddenSeries": false, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeatIteration": 1632879075996, + "repeatPanelId": 11, + "scopedVars": { + "job": { + "selected": false, + "text": "zebrad-testnet", + "value": "zebrad-testnet" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (remote_version, seed) (zcash_net_peers_initial{job=\"$job\",seed=~\"$seed\"} * on(remote_ip) group_left(remote_version) zcash_net_peers_obsolete{job=\"$job\"})", + "instant": false, + "interval": "", + "legendFormat": "{{remote_version}} - {{seed}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Obsolete Seed Peers - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:65", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:66", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "job", + "scopedVars": { + "job": { + "selected": false, + "text": "zebrad-mainnet", + "value": "zebrad-mainnet" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (remote_version) (zcash_net_peers_connected{job=\"$job\"})", + "interval": "", + "legendFormat": "{{remote_version}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Compatible Peers - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:65", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:66", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 17 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeatIteration": 1632879075996, + "repeatPanelId": 4, + "scopedVars": { + "job": { + "selected": false, + "text": "zebrad-testnet", + "value": "zebrad-testnet" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (remote_version) (zcash_net_peers_connected{job=\"$job\"})", + "interval": "", + "legendFormat": "{{remote_version}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Compatible Peers - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:65", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:66", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 25 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "job", + "scopedVars": { + "job": { + "selected": false, + "text": "zebrad-mainnet", + "value": "zebrad-mainnet" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (remote_version) (zcash_net_peers_obsolete{job=\"$job\"})", + "interval": "", + "legendFormat": "{{remote_version}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Obsolete Peers - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:65", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:66", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 25 + }, + "hiddenSeries": false, + "id": 15, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeatIteration": 1632879075996, + "repeatPanelId": 7, + "scopedVars": { + "job": { + "selected": false, + "text": "zebrad-testnet", + "value": "zebrad-testnet" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (remote_version) (zcash_net_peers_obsolete{job=\"$job\"})", + "interval": "", + "legendFormat": "{{remote_version}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Obsolete Peers - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:65", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:66", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "5s", + "schemaVersion": 27, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": false, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": null, + "definition": "label_values(zcash_net_in_bytes_total, job)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "job", + "options": [], + "query": { + "query": "label_values(zcash_net_in_bytes_total, job)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "current": { + "selected": true, + "tags": [], + "text": [ + "dnsseed.str4d.xyz:8233", + "dnsseed.testnet.z.cash:18233", + "dnsseed.z.cash:8233", + "mainnet.is.yolo.money:8233", + "mainnet.seeder.zfnd.org:8233", + "testnet.is.yolo.money:18233", + "testnet.seeder.zfnd.org:18233" + ], + "value": [ + "dnsseed.str4d.xyz:8233", + "dnsseed.testnet.z.cash:18233", + "dnsseed.z.cash:8233", + "mainnet.is.yolo.money:8233", + "mainnet.seeder.zfnd.org:8233", + "testnet.is.yolo.money:18233", + "testnet.seeder.zfnd.org:18233" + ] + }, + "datasource": null, + "definition": "label_values(zcash_net_peers_initial, seed)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "seed", + "options": [], + "query": { + "query": "label_values(zcash_net_peers_initial, seed)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "peers", + "uid": "S29TgUH7k", + "version": 22 +} \ No newline at end of file diff --git a/zebra-network/src/config.rs b/zebra-network/src/config.rs index 72142fe41..bc1a26fc9 100644 --- a/zebra-network/src/config.rs +++ b/zebra-network/src/config.rs @@ -144,13 +144,37 @@ impl Config { let fut = tokio::time::timeout(crate::constants::DNS_LOOKUP_TIMEOUT, fut); match fut.await { - Ok(Ok(ips)) => Ok(ips.map(canonical_socket_addr).collect()), + Ok(Ok(ip_addrs)) => { + let ip_addrs: Vec = ip_addrs.map(canonical_socket_addr).collect(); + + // if we're logging at debug level, + // the full list of IP addresses will be shown in the log message + let debug_span = debug_span!("", remote_ip_addrs = ?ip_addrs); + let _span_guard = debug_span.enter(); + info!(seed = ?host, remote_ip_count = ?ip_addrs.len(), "resolved seed peer IP addresses"); + + for ip in &ip_addrs { + // Count each initial peer, recording the seed config and resolved IP address. + // + // If an IP is returned by multiple seeds, + // each duplicate adds 1 to the initial peer count. + // (But we only make one initial connection attempt to each IP.) + metrics::counter!( + "zcash.net.peers.initial", + 1, + "seed" => host.to_string(), + "remote_ip" => ip.to_string() + ); + } + + Ok(ip_addrs.into_iter().collect()) + } Ok(Err(e)) => { - tracing::info!(?host, ?e, "DNS error resolving peer IP address"); + tracing::info!(?host, ?e, "DNS error resolving peer IP addresses"); Err(e.into()) } Err(e) => { - tracing::info!(?host, ?e, "DNS timeout resolving peer IP address"); + tracing::info!(?host, ?e, "DNS timeout resolving peer IP addresses"); Err(e.into()) } } diff --git a/zebra-network/src/peer/handshake.rs b/zebra-network/src/peer/handshake.rs index ce8482936..24f4b9b4a 100644 --- a/zebra-network/src/peer/handshake.rs +++ b/zebra-network/src/peer/handshake.rs @@ -1,4 +1,5 @@ use std::{ + cmp::min, collections::HashSet, fmt, future::Future, @@ -588,8 +589,42 @@ pub async fn negotiate_version( let height = latest_chain_tip.best_tip_height(); let min_version = Version::min_remote_for_height(config.network, height); if remote_version < min_version { + debug!( + remote_ip = ?their_addr, + ?remote_version, + ?min_version, + "disconnecting from peer with obsolete network protocol version" + ); + + metrics::counter!( + "zcash.net.peers.obsolete", + 1, + "remote_ip" => their_addr.to_string(), + "remote_version" => remote_version.to_string(), + "min_version" => min_version.to_string(), + ); + // Disconnect if peer is using an obsolete version. Err(HandshakeError::ObsoleteVersion(remote_version))?; + } else { + let negotiated_version = min(constants::CURRENT_NETWORK_PROTOCOL_VERSION, remote_version); + + debug!( + remote_ip = ?their_addr, + ?remote_version, + ?negotiated_version, + ?min_version, + "negotiated network protocol version with peer" + ); + + metrics::counter!( + "zcash.net.peers.connected", + 1, + "remote_ip" => their_addr.to_string(), + "remote_version" => remote_version.to_string(), + "negotiated_version" => negotiated_version.to_string(), + "min_version" => min_version.to_string(), + ); } peer_conn.send(Message::Verack).await?; diff --git a/zebra-network/src/protocol/external/types.rs b/zebra-network/src/protocol/external/types.rs index a69bce09b..fce1fc145 100644 --- a/zebra-network/src/protocol/external/types.rs +++ b/zebra-network/src/protocol/external/types.rs @@ -41,6 +41,12 @@ impl From for Magic { #[cfg_attr(any(test, feature = "proptest-impl"), derive(Arbitrary))] pub struct Version(pub u32); +impl fmt::Display for Version { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0.to_string()) + } +} + impl Version { /// Returns the minimum remote node network protocol version for `network` and /// `height`. Zebra disconnects from peers with lower versions.