Add a docker-compose.yml for prometheus/grafana metrics collection.

@str4d provided the zcashd-metrics dashboard.

Co-authored-by: Jack Grigg <jack@z.cash>
This commit is contained in:
Kris Nuttycombe 2022-09-21 18:27:45 -06:00
parent bb675d9799
commit aef66f4845
7 changed files with 1048 additions and 6 deletions

View File

@ -0,0 +1,52 @@
version: "3.7"
services:
grafana:
image: grafana/grafana
container_name: zcashd-grafana
depends_on:
- prometheus
environment:
GF_SERVER_HTTP_PORT: "3030"
ports:
- "3030:3030"
volumes:
- type: volume
source: grafana-storage
target: /var/lib/grafana
- type: bind
source: ./grafana/grafana.ini
target: /etc/grafana/grafana.ini
- type: bind
source: ./grafana/provisioning
target: /etc/grafana/provisioning
- type: bind
source: ./grafana/dashboards
target: /etc/grafana/dashboards
networks:
- zcashd-metrics
prometheus:
image: prom/prometheus
container_name: zcashd-prometheus
ports:
- "9090:9090"
volumes:
- type: volume
source: prometheus-storage
target: /prometheus
- type: bind
source: ./prometheus.yaml
target: /etc/prometheus/prometheus.yml
read_only: true
networks:
- zcashd-metrics
extra_hosts:
- "host.docker.internal:host-gateway"
volumes:
grafana-storage:
prometheus-storage:
networks:
zcashd-metrics:

View File

@ -0,0 +1,911 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "datasource",
"uid": "grafana"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 2,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "T9XAoUn4k"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 0
},
"id": 2,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "9.1.6",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "T9XAoUn4k"
},
"editorMode": "code",
"expr": "zcash_chain_verified_block_height",
"interval": "",
"legendFormat": "Chain tip",
"range": true,
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "T9XAoUn4k"
},
"editorMode": "builder",
"expr": "zcashd_wallet_synced_block_height",
"hide": false,
"legendFormat": "Wallet",
"range": true,
"refId": "B"
}
],
"title": "Block height",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "T9XAoUn4k"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"log": 2,
"type": "log"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 0
},
"id": 12,
"interval": "",
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.1.6",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "T9XAoUn4k"
},
"expr": "zcash_chain_verified_block_seconds",
"format": "time_series",
"interval": "",
"legendFormat": "{{quantile}}",
"refId": "A"
}
],
"title": "Block verification",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "T9XAoUn4k"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byFrameRefID",
"options": "Heap Usage"
},
"properties": [
{
"id": "custom.axisPlacement",
"value": "right"
},
{
"id": "unit",
"value": "bytes"
}
]
}
]
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 8
},
"id": 16,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "T9XAoUn4k"
},
"editorMode": "builder",
"expr": "zcashd_wallet_batchscanner_size_transactions",
"hide": false,
"legendFormat": "Transactions",
"range": true,
"refId": "Transactions"
},
{
"datasource": {
"type": "prometheus",
"uid": "T9XAoUn4k"
},
"editorMode": "builder",
"expr": "avg_over_time(zcashd_wallet_batchscanner_usage_byes[$__interval])",
"hide": false,
"legendFormat": "Heap usage",
"range": true,
"refId": "Heap Usage"
},
{
"datasource": {
"type": "prometheus",
"uid": "T9XAoUn4k"
},
"editorMode": "builder",
"expr": "rate(zcashd_wallet_batchscanner_outputs_scanned[$__rate_interval])",
"legendFormat": "{{kind}}",
"range": true,
"refId": "Output Rate"
}
],
"title": "Wallet batch scanner",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "T9XAoUn4k"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"decimals": 0,
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "txs"
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "Size"
},
"properties": [
{
"id": "unit",
"value": "bytes"
}
]
},
{
"matcher": {
"id": "byName",
"options": "Usage"
},
"properties": [
{
"id": "unit",
"value": "bytes"
}
]
}
]
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 8
},
"id": 10,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.1.6",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "T9XAoUn4k"
},
"expr": "zcash_mempool_size_transactions",
"interval": "",
"legendFormat": "Count",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "T9XAoUn4k"
},
"expr": "zcash_mempool_size_bytes",
"interval": "",
"legendFormat": "Size",
"refId": "B"
},
{
"datasource": {
"type": "prometheus",
"uid": "T9XAoUn4k"
},
"expr": "zcash_mempool_usage_bytes",
"interval": "",
"legendFormat": "Usage",
"refId": "C"
}
],
"title": "Mempool",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "T9XAoUn4k"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "binBps"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 16
},
"id": 4,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.1.6",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "T9XAoUn4k"
},
"expr": "rate(zcash_net_in_bytes[5m])",
"interval": "",
"legendFormat": "{{command}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "T9XAoUn4k"
},
"expr": "rate(zcash_net_in_bytes_total[5m])",
"interval": "",
"legendFormat": "Total",
"refId": "B"
}
],
"title": "Inbound messages",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "T9XAoUn4k"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "ZEC"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 16
},
"id": 8,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.1.6",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "T9XAoUn4k"
},
"expr": "zcash_pool_value_zatoshis/100000000",
"interval": "",
"legendFormat": "{{name}}",
"refId": "A"
}
],
"title": "Shielded pools",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "T9XAoUn4k"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "binBps"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 24
},
"id": 6,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.1.6",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "T9XAoUn4k"
},
"expr": "rate(zcash_net_out_bytes[5m])",
"interval": "",
"legendFormat": "{{command}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "T9XAoUn4k"
},
"expr": "rate(zcash_net_out_bytes_total[5m])",
"interval": "",
"legendFormat": "Total",
"refId": "B"
}
],
"title": "Outbound messages",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "T9XAoUn4k"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 24
},
"id": 14,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.1.6",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "T9XAoUn4k"
},
"expr": "zcash_pool_notes_created",
"interval": "",
"legendFormat": "{{name}}",
"refId": "A"
}
],
"title": "Commitment tree sizes",
"type": "timeseries"
}
],
"refresh": "5s",
"schemaVersion": 37,
"style": "dark",
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-5m",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "zcashd metrics",
"uid": "U4U58t-Gk",
"version": 1,
"weekStart": ""
}

View File

@ -0,0 +1,2 @@
[paths]
provisioning = /etc/grafana/provisioning

View File

@ -0,0 +1,18 @@
---
apiVersion: 1
providers:
- name: 'Zcash metrics'
# <string> provider type. Default to 'file'
type: file
# <bool> disable dashboard deletion
disableDeletion: false
# <int> how often Grafana will scan for changed dashboards
updateIntervalSeconds: 10
# <bool> allow updating provisioned dashboards from the UI
allowUiUpdates: false
options:
# <string, required> path to dashboard files on disk. Required when using the 'file' type
path: /etc/grafana/dashboards
# <bool> use folder names from filesystem to create folders in Grafana
foldersFromFilesStructure: true

View File

@ -0,0 +1,12 @@
---
apiVersion: 1
datasources:
-
access: proxy
editable: false
name: Prometheus
orgId: 1
type: prometheus
uid: T9XAoUn4k
url: "http://prometheus:9090"
version: 1

View File

@ -3,4 +3,4 @@ scrape_configs:
scrape_interval: 500ms
metrics_path: '/'
static_configs:
- targets: ['127.0.0.1:9969']
- targets: ['host.docker.internal:9969']

View File

@ -54,14 +54,60 @@ zcash_chain_verified_block_total 162
```
By default, access is restricted to localhost. This can be expanded with
`-metricsallowip=<ip>`, which can specify IPs or subnets. Note that HTTPS is not
supported, and therefore connections to the endpoint are not encrypted or
`-metricsallowip=<ip>`, which can specify IPs or subnets. Note that HTTPS is
not supported, and therefore connections to the endpoint are not encrypted or
authenticated. Access to the endpoint should be assumed to compromise the
privacy of node operations, by the provided metrics and/or by timing side
channels. Non-localhost access is **strongly discouraged** if the node has a
wallet holding live funds.
### Example metrics collection with Docker
### Metrics collection with Docker
A docker-compose.yml has been provided in `./contrib/metrics` that sets up
local instances of `prometheus` and `grafana` and provides a dashboard that
charts several of the various metrics exposed by `zcashd`'s `prometheus`
endpoint.
`sudo docker-compose up` will start local instances of `prometheus` and `grafana`,
accessible on ports `9090` and `3030` respectively.
```
cd $(zcash_root)/contrib/metrics
sudo docker-compose up -d
```
`~/.zcash/zcash.conf` must be updated to enable `prometheus` and to allow the
`prometheus` server launched via `docker-compose` to connect to the `zcashd`
prometheus endpoint. The following script can be used to detect the local IP
address for the `prometheus` server and add it to the `~/.zcash/zcash.conf`
file.
```
export PROMETHEUS_DOCKER_IP=$(sudo docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' zcashd-prometheus)
cat << PROM_CONF >> ~/.zcash/zcash.conf
prometheusport=9969
metricsallowip=$PROMETHEUS_DOCKER_IP/32
PROM_CONF
```
If you prefer to update the `~/.zcash/zcash.conf` file manually, make sure
to add both the `prometheusport` and `metricsallowip` properties, using the
`docker inspect ...` command above to determine the relevant IP address to
allow.
You may then (re)start `zcashd` and navigate to
[http://localhost:9090/targets?search=](http://localhost:9090/targets?search=)
to verify that the `prometheus` server is able to connect to the `zcashd`
prometheus endpoint; you should see the host
`http://host.docker.internal:9969/` having `UP` status. Once this is working,
navigate to
[http://localhost:3030/d/U4U58t-Gk/zcashd-metrics](http://localhost:3030/d/U4U58t-Gk/zcashd-metrics)
to view the dashboard. The username and password for this local grafana
instance are set by default to `admin`/`admin`; the UI will ask you to change
this on first use. Data gathered by the running prometheus instance and changes
that you make to the grafana interface will be persisted across restarts.
#### Manual Docker Setup Without docker-compose
The example instructions below were tested on Windows 10 using Docker Desktop
with the WSL 2 backend, connected to a `zcashd` running inside WSL2 (but not
@ -75,11 +121,12 @@ docker volume create grafana-storage
docker volume create prometheus-storage
# Run Prometheus
# You will need to modify ~/contrib/metrics/prometheus.yaml to match the
# You will need to modify $(zcash_root)/contrib/metrics/prometheus.yaml to match the
# port configured with -prometheusport and -metricsbind / -metricsallowip
# (and possibly also for your Docker network setup).
docker run --detach -p 9090:9090 --volume prometheus-storage:/prometheus --volume ~/contrib/metrics/prometheus.yaml:/etc/prometheus/prometheus.yml prom/prometheus
docker run --detach -p 9090:9090 --volume prometheus-storage:/prometheus --volume $(zcash_root)/contrib/metrics/prometheus.yaml:/etc/prometheus/prometheus.yml prom/prometheus
# Run Grafana
docker run --detach -p 3030:3030 --env GF_SERVER_HTTP_PORT=3030 --volume grafana-storage:/var/lib/grafana grafana/grafana
```