Add erasure recovery stats to dashboard (#6079)

automerge

core/src/blocktree.rs

@@ -323,7 +323,7 @@ impl Blocktree {
         // 3. Before trying recovery, check if enough number of shreds have been received
         // 3a. Enough number of shreds = (#data + #coding shreds) > erasure.num_data
         for (&(slot, set_index), erasure_meta) in erasure_metas.iter() {
-            let submit_metrics = |attempted: bool, status: String| {
+            let submit_metrics = |attempted: bool, status: String, recovered: usize| {
                 datapoint_info!(
                     "blocktree-erasure",
                     ("slot", slot as i64, i64),
@@ -331,6 +331,7 @@ impl Blocktree {
                     ("end_index", erasure_meta.end_indexes().0 as i64, i64),
                     ("recovery_attempted", attempted, bool),
                     ("recovery_status", status, String),
+                    ("recovered", recovered as i64, i64),
                 );
             };
 
@@ -387,17 +388,17 @@ impl Blocktree {
                         set_index as usize,
                         slot,
                     ) {
-                        submit_metrics(true, format!("complete. recovered: {}", result.len()));
+                        submit_metrics(true, "complete".into(), result.len());
                         recovered_data_shreds.append(&mut result);
                     } else {
-                        submit_metrics(true, "incomplete".into());
+                        submit_metrics(true, "incomplete".into(), 0);
                     }
                 }
                 ErasureMetaStatus::DataFull => {
-                    submit_metrics(false, "complete".into());
+                    submit_metrics(false, "complete".into(), 0);
                 }
                 ErasureMetaStatus::StillNeed(needed) => {
-                    submit_metrics(false, format!("still need: {}", needed));
+                    submit_metrics(false, format!("still need: {}", needed), 0);
                 }
             };
         }

metrics/scripts/grafana-provisioning/dashboards/testnet-monitor.json

@@ -15,8 +15,8 @@
   "editable": true,
   "gnetId": null,
   "graphTooltip": 0,
-  "id": 947,
-  "iteration": 1568522114558,
+  "id": 948,
+  "iteration": 1568522114560,
   "links": [
     {
       "asDropdown": true,
@@ -6924,7 +6924,7 @@
       "fill": 1,
       "gridPos": {
         "h": 5,
-        "w": 12,
+        "w": 8,
         "x": 0,
         "y": 74
       },
@@ -7077,8 +7077,8 @@
       "fill": 1,
       "gridPos": {
         "h": 5,
-        "w": 12,
-        "x": 12,
+        "w": 8,
+        "x": 8,
         "y": 74
       },
       "id": 52,
@@ -7223,6 +7223,123 @@
         "alignLevel": null
       }
     },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "$datasource",
+      "fill": 1,
+      "gridPos": {
+        "h": 5,
+        "w": 8,
+        "x": 16,
+        "y": 74
+      },
+      "id": 67,
+      "legend": {
+        "alignAsTable": false,
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "rightSide": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "percentage": false,
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "groupBy": [
+            {
+              "params": [
+                "$__interval"
+              ],
+              "type": "time"
+            },
+            {
+              "params": [
+                "null"
+              ],
+              "type": "fill"
+            }
+          ],
+          "measurement": "cluster_info-vote-count",
+          "orderByTime": "ASC",
+          "policy": "autogen",
+          "query": "SELECT mean(\"recovered\") AS \"recovered\" FROM \"$testnet\".\"autogen\".\"blocktree-erasure\" WHERE host_id::tag =~ /$hostid/ AND $timeFilter GROUP BY time($__interval) FILL(0)",
+          "rawQuery": true,
+          "refId": "B",
+          "resultFormat": "time_series",
+          "select": [
+            [
+              {
+                "params": [
+                  "count"
+                ],
+                "type": "field"
+              },
+              {
+                "params": [],
+                "type": "sum"
+              }
+            ]
+          ],
+          "tags": []
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Erasure Recovery ($hostid)",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
     {
       "collapsed": false,
       "gridPos": {