add Prometheus + Grafana for Guardian metrics

This commit is contained in:
justinschuldt 2022-07-29 17:01:39 -05:00 committed by Evan Gray
parent 126c9dcf57
commit 2c52f86546
3 changed files with 683 additions and 33 deletions

View File

@ -50,6 +50,7 @@ config.define_bool("e2e", False, "Enable E2E testing stack")
config.define_bool("ci_tests", False, "Enable tests runner component")
config.define_bool("bridge_ui_hot", False, "Enable hot loading bridge_ui")
config.define_bool("guardiand_debug", False, "Enable dlv endpoint for guardiand")
config.define_bool("node_metrics", False, "Enable Prometheus & Grafana for Guardian metrics")
cfg = config.parse()
num_guardians = int(cfg.get("num", "1"))
@ -69,6 +70,7 @@ spy_relayer = cfg.get("spy_relayer", ci)
e2e = cfg.get("e2e", ci)
ci_tests = cfg.get("ci_tests", ci)
guardiand_debug = cfg.get("guardiand_debug", False)
node_metrics = cfg.get("node_metrics", False)
bridge_ui_hot = not ci
@ -260,6 +262,47 @@ if num_guardians >= 2 and ci == False:
trigger_mode = trigger_mode,
)
# grafana + prometheus for node metrics
if node_metrics:
dashboard = read_json("dashboards/Wormhole.json")
dashboard_yaml = {
"apiVersion": "v1",
"kind": "ConfigMap",
"metadata": {
"name": "grafana-dashboards-json"
},
"data": {
"wormhole.json": encode_json(dashboard)
}
}
k8s_yaml_with_ns(encode_yaml(dashboard_yaml))
k8s_yaml_with_ns("devnet/node-metrics.yaml")
k8s_resource(
"prometheus-server",
resource_deps = ["guardian"],
port_forwards = [
port_forward(9099, name = "Prometheus [:9099]", host = webHost),
],
labels = ["guardian"],
trigger_mode = trigger_mode,
)
k8s_resource(
"grafana",
resource_deps = ["prometheus-server"],
port_forwards = [
port_forward(3033, name = "Grafana UI [:3033]", host = webHost),
],
labels = ["guardian"],
trigger_mode = trigger_mode,
)
# spy
k8s_yaml_with_ns("devnet/spy.yaml")

View File

@ -1,14 +1,5 @@
{
"__inputs": [
{
"name": "DS_MAIN_PROMETHEUS",
"label": "Main Prometheus",
"description": "",
"type": "datasource",
"pluginId": "prometheus",
"pluginName": "Prometheus"
}
],
"__inputs": [],
"__requires": [
{
"type": "grafana",
@ -39,7 +30,10 @@
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"datasource": {
"type": "datasource",
"uid": "grafana"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
@ -60,7 +54,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_MAIN_PROMETHEUS}",
"datasource": "prometheus",
"fieldConfig": {
"defaults": {
"custom": {}
@ -159,7 +153,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_MAIN_PROMETHEUS}",
"datasource": "prometheus",
"fieldConfig": {
"defaults": {
"custom": {}
@ -257,7 +251,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_MAIN_PROMETHEUS}",
"datasource": "prometheus",
"fieldConfig": {
"defaults": {
"custom": {}
@ -355,7 +349,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_MAIN_PROMETHEUS}",
"datasource": "prometheus",
"fieldConfig": {
"defaults": {
"custom": {}
@ -454,7 +448,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_MAIN_PROMETHEUS}",
"datasource": "prometheus",
"fieldConfig": {
"defaults": {
"custom": {}
@ -553,7 +547,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_MAIN_PROMETHEUS}",
"datasource": "prometheus",
"fieldConfig": {
"defaults": {
"custom": {}
@ -652,7 +646,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_MAIN_PROMETHEUS}",
"datasource": "prometheus",
"fieldConfig": {
"defaults": {
"custom": {}
@ -751,7 +745,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_MAIN_PROMETHEUS}",
"datasource": "prometheus",
"fieldConfig": {
"defaults": {
"custom": {}
@ -850,7 +844,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_MAIN_PROMETHEUS}",
"datasource": "prometheus",
"fieldConfig": {
"defaults": {
"custom": {}
@ -978,7 +972,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_MAIN_PROMETHEUS}",
"datasource": "prometheus",
"fieldConfig": {
"defaults": {
"custom": {}
@ -1076,7 +1070,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_MAIN_PROMETHEUS}",
"datasource": "prometheus",
"fieldConfig": {
"defaults": {
"custom": {}
@ -1195,7 +1189,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_MAIN_PROMETHEUS}",
"datasource": "prometheus",
"fieldConfig": {
"defaults": {
"custom": {}
@ -1293,7 +1287,7 @@
"bars": true,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_MAIN_PROMETHEUS}",
"datasource": "prometheus",
"fieldConfig": {
"defaults": {
"custom": {}
@ -1405,7 +1399,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_MAIN_PROMETHEUS}",
"datasource": "prometheus",
"fieldConfig": {
"defaults": {
"custom": {}
@ -1507,7 +1501,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_MAIN_PROMETHEUS}",
"datasource": "prometheus",
"fieldConfig": {
"defaults": {
"custom": {}
@ -1609,7 +1603,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_MAIN_PROMETHEUS}",
"datasource": "prometheus",
"fieldConfig": {
"defaults": {
"custom": {}
@ -1719,7 +1713,7 @@
"mode": "spectrum"
},
"dataFormat": "timeseries",
"datasource": "${DS_MAIN_PROMETHEUS}",
"datasource": "prometheus",
"fieldConfig": {
"defaults": {
"custom": {}
@ -1789,7 +1783,7 @@
"mode": "spectrum"
},
"dataFormat": "timeseries",
"datasource": "${DS_MAIN_PROMETHEUS}",
"datasource": "prometheus",
"fieldConfig": {
"defaults": {
"custom": {}
@ -1859,7 +1853,7 @@
"mode": "spectrum"
},
"dataFormat": "timeseries",
"datasource": "${DS_MAIN_PROMETHEUS}",
"datasource": "prometheus",
"fieldConfig": {
"defaults": {
"custom": {}
@ -1929,7 +1923,7 @@
"mode": "spectrum"
},
"dataFormat": "timeseries",
"datasource": "${DS_MAIN_PROMETHEUS}",
"datasource": "prometheus",
"fieldConfig": {
"defaults": {
"custom": {}
@ -1996,7 +1990,7 @@
{
"allValue": null,
"current": {},
"datasource": "${DS_MAIN_PROMETHEUS}",
"datasource": "prometheus",
"definition": "label_values(wormhole_network_node_height, instance)",
"error": null,
"hide": 0,
@ -2027,4 +2021,4 @@
"title": "Wormhole v2",
"uid": "-GcztLMnz",
"version": 16
}
}

613
devnet/node-metrics.yaml Normal file
View File

@ -0,0 +1,613 @@
---
apiVersion: v1
kind: Service
metadata:
name: prometheus-server
labels:
app: prometheus-server
spec:
selector:
app: prometheus-server
clusterIP: None
ports:
- port: 9099
targetPort: http
protocol: TCP
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: prometheus-server
labels:
app: prometheus-server
spec:
replicas: 1
selector:
matchLabels:
app: prometheus-server
template:
metadata:
labels:
app: prometheus-server
spec:
containers:
- name: prometheus
image: prom/prometheus
args:
- "--config.file=/etc/prometheus/prometheus.yml"
- "--storage.tsdb.path=/prometheus/"
ports:
- name: http
containerPort: 9090
volumeMounts:
- name: prometheus-config-volume
mountPath: /etc/prometheus/
- name: prometheus-storage-volume
mountPath: /prometheus/
volumes:
- name: prometheus-config-volume
configMap:
defaultMode: 420
name: prometheus-server-conf
- name: prometheus-storage-volume
emptyDir: {}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: prometheus-server-conf
data:
prometheus.yml: |-
global:
scrape_interval: 15s
scrape_timeout: 10s
evaluation_interval: 15s
alerting:
alertmanagers:
- follow_redirects: true
scheme: http
timeout: 10s
api_version: v2
static_configs:
- targets: []
scrape_configs:
- job_name: guardian
honor_timestamps: true
scrape_interval: 3s
scrape_timeout: 2s
metrics_path: /metrics
scheme: http
follow_redirects: true
static_configs:
- targets:
- guardian:6060
---
apiVersion: v1
kind: Service
metadata:
name: grafana
labels:
app: grafana
spec:
selector:
app: grafana
clusterIP: None
ports:
- port: 3033
targetPort: grafana
protocol: TCP
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: grafana
spec:
replicas: 1
selector:
matchLabels:
app: grafana
template:
metadata:
name: grafana
labels:
app: grafana
spec:
containers:
- name: grafana
image: grafana/grafana:9.0.5
ports:
- name: grafana
containerPort: 3000
volumeMounts:
- mountPath: /var/lib/grafana
name: grafana-storage
- mountPath: /etc/grafana/provisioning/datasources
name: grafana-datasources
readOnly: false
- mountPath: /etc/grafana
name: grafana-config
- mountPath: /var/lib/grafana/dashboards
name: grafana-dashboards-json
readOnly: false
- mountPath: /etc/grafana/provisioning/dashboards
name: grafana-dashboards-yaml
volumes:
- name: grafana-storage
emptyDir: {}
- name: grafana-datasources
configMap:
defaultMode: 420
name: grafana-datasources
- name: grafana-config
configMap:
name: grafana-config
- name: grafana-dashboards-json
configMap:
defaultMode: 420
name: grafana-dashboards-json
- name: grafana-dashboards-yaml
configMap:
name: grafana-dashboards-yaml
---
apiVersion: v1
kind: ConfigMap
metadata:
name: grafana-datasources
data:
prometheus.yaml: |-
{
"apiVersion": 1,
"datasources": [
{
"access":"proxy",
"editable": true,
"name": "prometheus",
"orgId": 1,
"type": "prometheus",
"url": "http://prometheus-server:9090",
"version": 1,
"default": true
}
]
}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: grafana-config
data:
grafana.ini: |
##################### Grafana Configuration Example #####################
#
# Everything has defaults so you only need to uncomment things you want to
# change
# possible values : production, development
; app_mode = production
# instance name, defaults to HOSTNAME environment variable value or hostname if HOSTNAME var is empty
; instance_name = ${HOSTNAME}
#################################### Paths ####################################
[paths]
# Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used)
#
;data = /var/lib/grafana
#
# Directory where grafana can store logs
#
;logs = /var/log/grafana
#
# Directory where grafana will automatically scan and look for plugins
#
;plugins = /var/lib/grafana/plugins
#
#################################### Server ####################################
[server]
# Protocol (http, https, socket)
;protocol = http
# The ip address to bind to, empty will bind to all interfaces
;http_addr =
# The http port to use
;http_port = 3000
# The public facing domain name used to access grafana from a browser
;domain = localhost
# Redirect to correct domain if host header does not match domain
# Prevents DNS rebinding attacks
;enforce_domain = false
# The full public facing url you use in browser, used for redirects and emails
# If you use reverse proxy and sub path specify full url (with sub path)
;root_url = http://localhost:3000
# Log web requests
;router_logging = false
# the path relative working path
;static_root_path = public
# enable gzip
;enable_gzip = false
# https certs & key file
;cert_file =
;cert_key =
# Unix socket path
;socket =
#################################### Database ####################################
[database]
# You can configure the database connection by specifying type, host, name, user and password
# as seperate properties or as on string using the url propertie.
# Either "mysql", "postgres" or "sqlite3", it's your choice
;type = sqlite3
;host = 127.0.0.1:3306
;name = grafana
;user = root
# If the password contains # or ; you have to wrap it with trippel quotes. Ex """#password;"""
;password =
# Use either URL or the previous fields to configure the database
# Example: mysql://user:secret@host:port/database
;url =
# For "postgres" only, either "disable", "require" or "verify-full"
;ssl_mode = disable
# For "sqlite3" only, path relative to data_path setting
;path = grafana.db
# Max conn setting default is 0 (mean not set)
;max_idle_conn =
;max_open_conn =
#################################### Session ####################################
[session]
# Either "memory", "file", "redis", "mysql", "postgres", default is "file"
;provider = file
# Provider config options
# memory: not have any config yet
# file: session dir path, is relative to grafana data_path
# redis: config like redis server e.g. `addr=127.0.0.1:6379,pool_size=100,db=grafana`
# mysql: go-sql-driver/mysql dsn config string, e.g. `user:password@tcp(127.0.0.1:3306)/database_name`
# postgres: user=a password=b host=localhost port=5432 dbname=c sslmode=disable
;provider_config = sessions
# Session cookie name
;cookie_name = grafana_sess
# If you use session in https only, default is false
;cookie_secure = false
# Session life time, default is 86400
;session_life_time = 86400
#################################### Data proxy ###########################
[dataproxy]
# This enables data proxy logging, default is false
;logging = false
#################################### Analytics ####################################
[analytics]
# Server reporting, sends usage counters to stats.grafana.org every 24 hours.
# No ip addresses are being tracked, only simple counters to track
# running instances, dashboard and error counts. It is very helpful to us.
# Change this option to false to disable reporting.
;reporting_enabled = true
# Set to false to disable all checks to https://grafana.net
# for new vesions (grafana itself and plugins), check is used
# in some UI views to notify that grafana or plugin update exists
# This option does not cause any auto updates, nor send any information
# only a GET request to http://grafana.com to get latest versions
;check_for_updates = true
# Google Analytics universal tracking code, only enabled if you specify an id here
;google_analytics_ua_id =
#################################### Security ####################################
[security]
# default admin user, created on startup
;admin_user = admin
# default admin password, can be changed before first start of grafana, or in profile settings
;admin_password = admin
# used for signing
;secret_key = SW2YcwTIb9zpOOhoPsMm
# Auto-login remember days
;login_remember_days = 7
;cookie_username = grafana_user
;cookie_remember_name = grafana_remember
# disable gravatar profile images
;disable_gravatar = false
# data source proxy whitelist (ip_or_domain:port separated by spaces)
;data_source_proxy_whitelist =
[snapshots]
# snapshot sharing options
;external_enabled = true
;external_snapshot_url = https://snapshots-origin.raintank.io
;external_snapshot_name = Publish to snapshot.raintank.io
# remove expired snapshot
;snapshot_remove_expired = true
# remove snapshots after 90 days
;snapshot_TTL_days = 90
#################################### Users ####################################
[users]
# disable user signup / registration
;allow_sign_up = true
# Allow non admin users to create organizations
;allow_org_create = true
# Set to true to automatically assign new users to the default organization (id 1)
;auto_assign_org = true
# Default role new users will be automatically assigned (if disabled above is set to true)
;auto_assign_org_role = Viewer
# Background text for the user field on the login page
;login_hint = email or username
# Default UI theme ("dark" or "light")
;default_theme = dark
[auth]
# Set to true to disable (hide) the login form, useful if you use OAuth, defaults to false
;disable_login_form = false
# Set to true to disable the signout link in the side menu. useful if you use auth.proxy, defaults to false
;disable_signout_menu = false
#################################### Anonymous Auth ##########################
[auth.anonymous]
# enable anonymous access
enabled = true
# specify organization name that should be used for unauthenticated users
;org_name = devnet.
# specify role for unauthenticated users
;org_role = Viewer
#################################### Github Auth ##########################
[auth.github]
;enabled = false
;allow_sign_up = true
;client_id = some_id
;client_secret = some_secret
;scopes = user:email,read:org
;auth_url = https://github.com/login/oauth/authorize
;token_url = https://github.com/login/oauth/access_token
;api_url = https://api.github.com/user
;team_ids =
;allowed_organizations =
#################################### Google Auth ##########################
[auth.google]
;enabled = false
;allow_sign_up = true
;client_id = some_client_id
;client_secret = some_client_secret
;scopes = https://www.googleapis.com/auth/userinfo.profile https://www.googleapis.com/auth/userinfo.email
;auth_url = https://accounts.google.com/o/oauth2/auth
;token_url = https://accounts.google.com/o/oauth2/token
;api_url = https://www.googleapis.com/oauth2/v1/userinfo
;allowed_domains =
#################################### Generic OAuth ##########################
[auth.generic_oauth]
;enabled = false
;name = OAuth
;allow_sign_up = true
;client_id = some_id
;client_secret = some_secret
;scopes = user:email,read:org
;auth_url = https://foo.bar/login/oauth/authorize
;token_url = https://foo.bar/login/oauth/access_token
;api_url = https://foo.bar/user
;team_ids =
;allowed_organizations =
#################################### Grafana.com Auth ####################
[auth.grafana_com]
;enabled = false
;allow_sign_up = true
;client_id = some_id
;client_secret = some_secret
;scopes = user:email
;allowed_organizations =
#################################### Auth Proxy ##########################
[auth.proxy]
;enabled = false
;header_name = X-WEBAUTH-USER
;header_property = username
;auto_sign_up = true
;ldap_sync_ttl = 60
;whitelist = 192.168.1.1, 192.168.2.1
#################################### Basic Auth ##########################
[auth.basic]
;enabled = true
#################################### Auth LDAP ##########################
[auth.ldap]
;enabled = false
;config_file = /etc/grafana/ldap.toml
;allow_sign_up = true
#################################### SMTP / Emailing ##########################
[smtp]
;enabled = false
;host = localhost:25
;user =
# If the password contains # or ; you have to wrap it with trippel quotes. Ex """#password;"""
;password =
;cert_file =
;key_file =
;skip_verify = false
;from_address = admin@grafana.localhost
;from_name = Grafana
[emails]
;welcome_email_on_sign_up = false
#################################### Logging ##########################
[log]
# Either "console", "file", "syslog". Default is console and file
# Use space to separate multiple modes, e.g. "console file"
;mode = console file
# Either "debug", "info", "warn", "error", "critical", default is "info"
;level = info
# optional settings to set different levels for specific loggers. Ex filters = sqlstore:debug
;filters =
# For "console" mode only
[log.console]
;level =
# log line format, valid options are text, console and json
;format = console
# For "file" mode only
[log.file]
;level =
# log line format, valid options are text, console and json
;format = text
# This enables automated log rotate(switch of following options), default is true
;log_rotate = true
# Max line number of single file, default is 1000000
;max_lines = 1000000
# Max size shift of single file, default is 28 means 1 << 28, 256MB
;max_size_shift = 28
# Segment log daily, default is true
;daily_rotate = true
# Expired days of log file(delete after max days), default is 7
;max_days = 7
[log.syslog]
;level =
# log line format, valid options are text, console and json
;format = text
# Syslog network type and address. This can be udp, tcp, or unix. If left blank, the default unix endpoints will be used.
;network =
;address =
# Syslog facility. user, daemon and local0 through local7 are valid.
;facility =
# Syslog tag. By default, the process' argv[0] is used.
;tag =
#################################### AMQP Event Publisher ##########################
[event_publisher]
;enabled = false
;rabbitmq_url = amqp://localhost/
;exchange = grafana_events
;#################################### Dashboard JSON files ##########################
[dashboards.json]
;enabled = false
enabled = true
;path = /var/lib/grafana/dashboards
#################################### Alerting ############################
[alerting]
# Disable alerting engine & UI features
;enabled = true
# Makes it possible to turn off alert rule execution but alerting UI is visible
;execute_alerts = true
#################################### Internal Grafana Metrics ##########################
# Metrics available at HTTP API Url /api/metrics
[metrics]
# Disable / Enable internal metrics
;enabled = true
# Publish interval
;interval_seconds = 10
# Send internal metrics to Graphite
[metrics.graphite]
# Enable by setting the address setting (ex localhost:2003)
;address =
;prefix = prod.grafana.%(instance_name)s.
#################################### Grafana.com integration ##########################
# Url used to to import dashboards directly from Grafana.com
[grafana_com]
;url = https://grafana.com
#################################### External image storage ##########################
[external_image_storage]
# Used for uploading images to public servers so they can be included in slack/email messages.
# you can choose between (s3, webdav)
;provider =
[external_image_storage.s3]
;bucket_url =
;access_key =
;secret_key =
[external_image_storage.webdav]
;url =
;public_url =
;username =
;password =
---
apiVersion: v1
kind: ConfigMap
metadata:
name: grafana-dashboards-yaml
data:
default.yaml: |-
{
"apiVersion": 1,
"providers": [
{
"name": "Default",
"folder": "",
"type": "file",
allowUiUpdates: true,
"options": {
"path": "/var/lib/grafana/dashboards",
"foldersFromFilesStructure": true
}
}
]
}
---