diff --git a/Tiltfile b/Tiltfile index 0d1acf827..d711d2bdf 100644 --- a/Tiltfile +++ b/Tiltfile @@ -50,6 +50,7 @@ config.define_bool("e2e", False, "Enable E2E testing stack") config.define_bool("ci_tests", False, "Enable tests runner component") config.define_bool("bridge_ui_hot", False, "Enable hot loading bridge_ui") config.define_bool("guardiand_debug", False, "Enable dlv endpoint for guardiand") +config.define_bool("node_metrics", False, "Enable Prometheus & Grafana for Guardian metrics") cfg = config.parse() num_guardians = int(cfg.get("num", "1")) @@ -69,6 +70,7 @@ spy_relayer = cfg.get("spy_relayer", ci) e2e = cfg.get("e2e", ci) ci_tests = cfg.get("ci_tests", ci) guardiand_debug = cfg.get("guardiand_debug", False) +node_metrics = cfg.get("node_metrics", False) bridge_ui_hot = not ci @@ -260,6 +262,47 @@ if num_guardians >= 2 and ci == False: trigger_mode = trigger_mode, ) + +# grafana + prometheus for node metrics +if node_metrics: + + dashboard = read_json("dashboards/Wormhole.json") + + dashboard_yaml = { + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": { + "name": "grafana-dashboards-json" + }, + "data": { + "wormhole.json": encode_json(dashboard) + } + } + k8s_yaml_with_ns(encode_yaml(dashboard_yaml)) + + k8s_yaml_with_ns("devnet/node-metrics.yaml") + + k8s_resource( + "prometheus-server", + resource_deps = ["guardian"], + port_forwards = [ + port_forward(9099, name = "Prometheus [:9099]", host = webHost), + ], + labels = ["guardian"], + trigger_mode = trigger_mode, + ) + + k8s_resource( + "grafana", + resource_deps = ["prometheus-server"], + port_forwards = [ + port_forward(3033, name = "Grafana UI [:3033]", host = webHost), + ], + labels = ["guardian"], + trigger_mode = trigger_mode, + ) + + # spy k8s_yaml_with_ns("devnet/spy.yaml") diff --git a/dashboards/Wormhole.json b/dashboards/Wormhole.json index 6ae93207e..fbf28d863 100644 --- a/dashboards/Wormhole.json +++ b/dashboards/Wormhole.json @@ -1,14 +1,5 @@ { - "__inputs": [ - { - "name": "DS_MAIN_PROMETHEUS", - "label": "Main Prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], + "__inputs": [], "__requires": [ { "type": "grafana", @@ -39,7 +30,10 @@ "list": [ { "builtIn": 1, - "datasource": "-- Grafana --", + "datasource": { + "type": "datasource", + "uid": "grafana" + }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -60,7 +54,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_MAIN_PROMETHEUS}", + "datasource": "prometheus", "fieldConfig": { "defaults": { "custom": {} @@ -159,7 +153,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_MAIN_PROMETHEUS}", + "datasource": "prometheus", "fieldConfig": { "defaults": { "custom": {} @@ -257,7 +251,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_MAIN_PROMETHEUS}", + "datasource": "prometheus", "fieldConfig": { "defaults": { "custom": {} @@ -355,7 +349,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_MAIN_PROMETHEUS}", + "datasource": "prometheus", "fieldConfig": { "defaults": { "custom": {} @@ -454,7 +448,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_MAIN_PROMETHEUS}", + "datasource": "prometheus", "fieldConfig": { "defaults": { "custom": {} @@ -553,7 +547,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_MAIN_PROMETHEUS}", + "datasource": "prometheus", "fieldConfig": { "defaults": { "custom": {} @@ -652,7 +646,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_MAIN_PROMETHEUS}", + "datasource": "prometheus", "fieldConfig": { "defaults": { "custom": {} @@ -751,7 +745,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_MAIN_PROMETHEUS}", + "datasource": "prometheus", "fieldConfig": { "defaults": { "custom": {} @@ -850,7 +844,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_MAIN_PROMETHEUS}", + "datasource": "prometheus", "fieldConfig": { "defaults": { "custom": {} @@ -978,7 +972,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_MAIN_PROMETHEUS}", + "datasource": "prometheus", "fieldConfig": { "defaults": { "custom": {} @@ -1076,7 +1070,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_MAIN_PROMETHEUS}", + "datasource": "prometheus", "fieldConfig": { "defaults": { "custom": {} @@ -1195,7 +1189,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_MAIN_PROMETHEUS}", + "datasource": "prometheus", "fieldConfig": { "defaults": { "custom": {} @@ -1293,7 +1287,7 @@ "bars": true, "dashLength": 10, "dashes": false, - "datasource": "${DS_MAIN_PROMETHEUS}", + "datasource": "prometheus", "fieldConfig": { "defaults": { "custom": {} @@ -1405,7 +1399,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_MAIN_PROMETHEUS}", + "datasource": "prometheus", "fieldConfig": { "defaults": { "custom": {} @@ -1507,7 +1501,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_MAIN_PROMETHEUS}", + "datasource": "prometheus", "fieldConfig": { "defaults": { "custom": {} @@ -1609,7 +1603,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_MAIN_PROMETHEUS}", + "datasource": "prometheus", "fieldConfig": { "defaults": { "custom": {} @@ -1719,7 +1713,7 @@ "mode": "spectrum" }, "dataFormat": "timeseries", - "datasource": "${DS_MAIN_PROMETHEUS}", + "datasource": "prometheus", "fieldConfig": { "defaults": { "custom": {} @@ -1789,7 +1783,7 @@ "mode": "spectrum" }, "dataFormat": "timeseries", - "datasource": "${DS_MAIN_PROMETHEUS}", + "datasource": "prometheus", "fieldConfig": { "defaults": { "custom": {} @@ -1859,7 +1853,7 @@ "mode": "spectrum" }, "dataFormat": "timeseries", - "datasource": "${DS_MAIN_PROMETHEUS}", + "datasource": "prometheus", "fieldConfig": { "defaults": { "custom": {} @@ -1929,7 +1923,7 @@ "mode": "spectrum" }, "dataFormat": "timeseries", - "datasource": "${DS_MAIN_PROMETHEUS}", + "datasource": "prometheus", "fieldConfig": { "defaults": { "custom": {} @@ -1996,7 +1990,7 @@ { "allValue": null, "current": {}, - "datasource": "${DS_MAIN_PROMETHEUS}", + "datasource": "prometheus", "definition": "label_values(wormhole_network_node_height, instance)", "error": null, "hide": 0, @@ -2027,4 +2021,4 @@ "title": "Wormhole v2", "uid": "-GcztLMnz", "version": 16 -} \ No newline at end of file +} diff --git a/devnet/node-metrics.yaml b/devnet/node-metrics.yaml new file mode 100644 index 000000000..a35600260 --- /dev/null +++ b/devnet/node-metrics.yaml @@ -0,0 +1,613 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: prometheus-server + labels: + app: prometheus-server +spec: + selector: + app: prometheus-server + clusterIP: None + ports: + - port: 9099 + targetPort: http + protocol: TCP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus-server + labels: + app: prometheus-server +spec: + replicas: 1 + selector: + matchLabels: + app: prometheus-server + template: + metadata: + labels: + app: prometheus-server + spec: + containers: + - name: prometheus + image: prom/prometheus + args: + - "--config.file=/etc/prometheus/prometheus.yml" + - "--storage.tsdb.path=/prometheus/" + ports: + - name: http + containerPort: 9090 + volumeMounts: + - name: prometheus-config-volume + mountPath: /etc/prometheus/ + - name: prometheus-storage-volume + mountPath: /prometheus/ + volumes: + - name: prometheus-config-volume + configMap: + defaultMode: 420 + name: prometheus-server-conf + + - name: prometheus-storage-volume + emptyDir: {} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-server-conf +data: + prometheus.yml: |- + global: + scrape_interval: 15s + scrape_timeout: 10s + evaluation_interval: 15s + alerting: + alertmanagers: + - follow_redirects: true + scheme: http + timeout: 10s + api_version: v2 + static_configs: + - targets: [] + scrape_configs: + - job_name: guardian + honor_timestamps: true + scrape_interval: 3s + scrape_timeout: 2s + metrics_path: /metrics + scheme: http + follow_redirects: true + static_configs: + - targets: + - guardian:6060 +--- +apiVersion: v1 +kind: Service +metadata: + name: grafana + labels: + app: grafana +spec: + selector: + app: grafana + clusterIP: None + ports: + - port: 3033 + targetPort: grafana + protocol: TCP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: grafana +spec: + replicas: 1 + selector: + matchLabels: + app: grafana + template: + metadata: + name: grafana + labels: + app: grafana + spec: + containers: + - name: grafana + image: grafana/grafana:9.0.5 + ports: + - name: grafana + containerPort: 3000 + volumeMounts: + - mountPath: /var/lib/grafana + name: grafana-storage + - mountPath: /etc/grafana/provisioning/datasources + name: grafana-datasources + readOnly: false + - mountPath: /etc/grafana + name: grafana-config + - mountPath: /var/lib/grafana/dashboards + name: grafana-dashboards-json + readOnly: false + - mountPath: /etc/grafana/provisioning/dashboards + name: grafana-dashboards-yaml + + volumes: + - name: grafana-storage + emptyDir: {} + - name: grafana-datasources + configMap: + defaultMode: 420 + name: grafana-datasources + - name: grafana-config + configMap: + name: grafana-config + - name: grafana-dashboards-json + configMap: + defaultMode: 420 + name: grafana-dashboards-json + - name: grafana-dashboards-yaml + configMap: + name: grafana-dashboards-yaml +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-datasources +data: + prometheus.yaml: |- + { + "apiVersion": 1, + "datasources": [ + { + "access":"proxy", + "editable": true, + "name": "prometheus", + "orgId": 1, + "type": "prometheus", + "url": "http://prometheus-server:9090", + "version": 1, + "default": true + } + ] + } +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-config +data: + grafana.ini: | + ##################### Grafana Configuration Example ##################### + # + # Everything has defaults so you only need to uncomment things you want to + # change + + # possible values : production, development + ; app_mode = production + + # instance name, defaults to HOSTNAME environment variable value or hostname if HOSTNAME var is empty + ; instance_name = ${HOSTNAME} + + #################################### Paths #################################### + [paths] + # Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used) + # + ;data = /var/lib/grafana + # + # Directory where grafana can store logs + # + ;logs = /var/log/grafana + # + # Directory where grafana will automatically scan and look for plugins + # + ;plugins = /var/lib/grafana/plugins + + # + #################################### Server #################################### + [server] + # Protocol (http, https, socket) + ;protocol = http + + # The ip address to bind to, empty will bind to all interfaces + ;http_addr = + + # The http port to use + ;http_port = 3000 + + # The public facing domain name used to access grafana from a browser + ;domain = localhost + + # Redirect to correct domain if host header does not match domain + # Prevents DNS rebinding attacks + ;enforce_domain = false + + # The full public facing url you use in browser, used for redirects and emails + # If you use reverse proxy and sub path specify full url (with sub path) + ;root_url = http://localhost:3000 + + # Log web requests + ;router_logging = false + + # the path relative working path + ;static_root_path = public + + # enable gzip + ;enable_gzip = false + + # https certs & key file + ;cert_file = + ;cert_key = + + # Unix socket path + ;socket = + + #################################### Database #################################### + [database] + # You can configure the database connection by specifying type, host, name, user and password + # as seperate properties or as on string using the url propertie. + + # Either "mysql", "postgres" or "sqlite3", it's your choice + ;type = sqlite3 + ;host = 127.0.0.1:3306 + ;name = grafana + ;user = root + # If the password contains # or ; you have to wrap it with trippel quotes. Ex """#password;""" + ;password = + + # Use either URL or the previous fields to configure the database + # Example: mysql://user:secret@host:port/database + ;url = + + # For "postgres" only, either "disable", "require" or "verify-full" + ;ssl_mode = disable + + # For "sqlite3" only, path relative to data_path setting + ;path = grafana.db + + # Max conn setting default is 0 (mean not set) + ;max_idle_conn = + ;max_open_conn = + + + #################################### Session #################################### + [session] + # Either "memory", "file", "redis", "mysql", "postgres", default is "file" + ;provider = file + + # Provider config options + # memory: not have any config yet + # file: session dir path, is relative to grafana data_path + # redis: config like redis server e.g. `addr=127.0.0.1:6379,pool_size=100,db=grafana` + # mysql: go-sql-driver/mysql dsn config string, e.g. `user:password@tcp(127.0.0.1:3306)/database_name` + # postgres: user=a password=b host=localhost port=5432 dbname=c sslmode=disable + ;provider_config = sessions + + # Session cookie name + ;cookie_name = grafana_sess + + # If you use session in https only, default is false + ;cookie_secure = false + + # Session life time, default is 86400 + ;session_life_time = 86400 + + #################################### Data proxy ########################### + [dataproxy] + + # This enables data proxy logging, default is false + ;logging = false + + + #################################### Analytics #################################### + [analytics] + # Server reporting, sends usage counters to stats.grafana.org every 24 hours. + # No ip addresses are being tracked, only simple counters to track + # running instances, dashboard and error counts. It is very helpful to us. + # Change this option to false to disable reporting. + ;reporting_enabled = true + + # Set to false to disable all checks to https://grafana.net + # for new vesions (grafana itself and plugins), check is used + # in some UI views to notify that grafana or plugin update exists + # This option does not cause any auto updates, nor send any information + # only a GET request to http://grafana.com to get latest versions + ;check_for_updates = true + + # Google Analytics universal tracking code, only enabled if you specify an id here + ;google_analytics_ua_id = + + #################################### Security #################################### + [security] + # default admin user, created on startup + ;admin_user = admin + + # default admin password, can be changed before first start of grafana, or in profile settings + ;admin_password = admin + + # used for signing + ;secret_key = SW2YcwTIb9zpOOhoPsMm + + # Auto-login remember days + ;login_remember_days = 7 + ;cookie_username = grafana_user + ;cookie_remember_name = grafana_remember + + # disable gravatar profile images + ;disable_gravatar = false + + # data source proxy whitelist (ip_or_domain:port separated by spaces) + ;data_source_proxy_whitelist = + + [snapshots] + # snapshot sharing options + ;external_enabled = true + ;external_snapshot_url = https://snapshots-origin.raintank.io + ;external_snapshot_name = Publish to snapshot.raintank.io + + # remove expired snapshot + ;snapshot_remove_expired = true + + # remove snapshots after 90 days + ;snapshot_TTL_days = 90 + + #################################### Users #################################### + [users] + # disable user signup / registration + ;allow_sign_up = true + + # Allow non admin users to create organizations + ;allow_org_create = true + + # Set to true to automatically assign new users to the default organization (id 1) + ;auto_assign_org = true + + # Default role new users will be automatically assigned (if disabled above is set to true) + ;auto_assign_org_role = Viewer + + # Background text for the user field on the login page + ;login_hint = email or username + + # Default UI theme ("dark" or "light") + ;default_theme = dark + + [auth] + # Set to true to disable (hide) the login form, useful if you use OAuth, defaults to false + ;disable_login_form = false + + # Set to true to disable the signout link in the side menu. useful if you use auth.proxy, defaults to false + ;disable_signout_menu = false + + #################################### Anonymous Auth ########################## + [auth.anonymous] + # enable anonymous access + enabled = true + + # specify organization name that should be used for unauthenticated users + ;org_name = devnet. + + # specify role for unauthenticated users + ;org_role = Viewer + + #################################### Github Auth ########################## + [auth.github] + ;enabled = false + ;allow_sign_up = true + ;client_id = some_id + ;client_secret = some_secret + ;scopes = user:email,read:org + ;auth_url = https://github.com/login/oauth/authorize + ;token_url = https://github.com/login/oauth/access_token + ;api_url = https://api.github.com/user + ;team_ids = + ;allowed_organizations = + + #################################### Google Auth ########################## + [auth.google] + ;enabled = false + ;allow_sign_up = true + ;client_id = some_client_id + ;client_secret = some_client_secret + ;scopes = https://www.googleapis.com/auth/userinfo.profile https://www.googleapis.com/auth/userinfo.email + ;auth_url = https://accounts.google.com/o/oauth2/auth + ;token_url = https://accounts.google.com/o/oauth2/token + ;api_url = https://www.googleapis.com/oauth2/v1/userinfo + ;allowed_domains = + + #################################### Generic OAuth ########################## + [auth.generic_oauth] + ;enabled = false + ;name = OAuth + ;allow_sign_up = true + ;client_id = some_id + ;client_secret = some_secret + ;scopes = user:email,read:org + ;auth_url = https://foo.bar/login/oauth/authorize + ;token_url = https://foo.bar/login/oauth/access_token + ;api_url = https://foo.bar/user + ;team_ids = + ;allowed_organizations = + + #################################### Grafana.com Auth #################### + [auth.grafana_com] + ;enabled = false + ;allow_sign_up = true + ;client_id = some_id + ;client_secret = some_secret + ;scopes = user:email + ;allowed_organizations = + + #################################### Auth Proxy ########################## + [auth.proxy] + ;enabled = false + ;header_name = X-WEBAUTH-USER + ;header_property = username + ;auto_sign_up = true + ;ldap_sync_ttl = 60 + ;whitelist = 192.168.1.1, 192.168.2.1 + + #################################### Basic Auth ########################## + [auth.basic] + ;enabled = true + + #################################### Auth LDAP ########################## + [auth.ldap] + ;enabled = false + ;config_file = /etc/grafana/ldap.toml + ;allow_sign_up = true + + #################################### SMTP / Emailing ########################## + [smtp] + ;enabled = false + ;host = localhost:25 + ;user = + # If the password contains # or ; you have to wrap it with trippel quotes. Ex """#password;""" + ;password = + ;cert_file = + ;key_file = + ;skip_verify = false + ;from_address = admin@grafana.localhost + ;from_name = Grafana + + [emails] + ;welcome_email_on_sign_up = false + + #################################### Logging ########################## + [log] + # Either "console", "file", "syslog". Default is console and file + # Use space to separate multiple modes, e.g. "console file" + ;mode = console file + + # Either "debug", "info", "warn", "error", "critical", default is "info" + ;level = info + + # optional settings to set different levels for specific loggers. Ex filters = sqlstore:debug + ;filters = + + + # For "console" mode only + [log.console] + ;level = + + # log line format, valid options are text, console and json + ;format = console + + # For "file" mode only + [log.file] + ;level = + + # log line format, valid options are text, console and json + ;format = text + + # This enables automated log rotate(switch of following options), default is true + ;log_rotate = true + + # Max line number of single file, default is 1000000 + ;max_lines = 1000000 + + # Max size shift of single file, default is 28 means 1 << 28, 256MB + ;max_size_shift = 28 + + # Segment log daily, default is true + ;daily_rotate = true + + # Expired days of log file(delete after max days), default is 7 + ;max_days = 7 + + [log.syslog] + ;level = + + # log line format, valid options are text, console and json + ;format = text + + # Syslog network type and address. This can be udp, tcp, or unix. If left blank, the default unix endpoints will be used. + ;network = + ;address = + + # Syslog facility. user, daemon and local0 through local7 are valid. + ;facility = + + # Syslog tag. By default, the process' argv[0] is used. + ;tag = + + + #################################### AMQP Event Publisher ########################## + [event_publisher] + ;enabled = false + ;rabbitmq_url = amqp://localhost/ + ;exchange = grafana_events + + ;#################################### Dashboard JSON files ########################## + [dashboards.json] + ;enabled = false + enabled = true + ;path = /var/lib/grafana/dashboards + + #################################### Alerting ############################ + [alerting] + # Disable alerting engine & UI features + ;enabled = true + # Makes it possible to turn off alert rule execution but alerting UI is visible + ;execute_alerts = true + + #################################### Internal Grafana Metrics ########################## + # Metrics available at HTTP API Url /api/metrics + [metrics] + # Disable / Enable internal metrics + ;enabled = true + + # Publish interval + ;interval_seconds = 10 + + # Send internal metrics to Graphite + [metrics.graphite] + # Enable by setting the address setting (ex localhost:2003) + ;address = + ;prefix = prod.grafana.%(instance_name)s. + + #################################### Grafana.com integration ########################## + # Url used to to import dashboards directly from Grafana.com + [grafana_com] + ;url = https://grafana.com + + #################################### External image storage ########################## + [external_image_storage] + # Used for uploading images to public servers so they can be included in slack/email messages. + # you can choose between (s3, webdav) + ;provider = + + [external_image_storage.s3] + ;bucket_url = + ;access_key = + ;secret_key = + + [external_image_storage.webdav] + ;url = + ;public_url = + ;username = + ;password = +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboards-yaml +data: + default.yaml: |- + { + "apiVersion": 1, + "providers": [ + { + "name": "Default", + "folder": "", + "type": "file", + allowUiUpdates: true, + "options": { + "path": "/var/lib/grafana/dashboards", + "foldersFromFilesStructure": true + } + } + ] + } +--- +