Add support for local metric collection/viewing (#4170)

metrics/scripts/.gitignore

@@ -0,0 +1 @@
+/lib

metrics/scripts/README.md

@@ -0,0 +1,49 @@
+
+This directory contains scripts to manage a local instance of [InfluxDB OSS](https://docs.influxdata.com/influxdb/v1.5/) and [Grafana](https://grafana.com/docs/v5.2/)
+
+### Setup
+
+Start the local metric services:
+
+`$ ./start.sh`
+
+Metrics are enabled on a per-shell basis which means you must `source` the following scripts in each shell in which you start an application you wish to collect metrics from.  For example, if running a Solana fullnode you must call `source ./enable.sh` before starting the node.
+
+`$ source ./enable.sh`
+
+Once metrics have been started and you have an application running you can view the metrics at:
+
+http://localhost:3000/d/local/local-monitor
+
+Stop metric services:
+
+`$ ./stop.sh`
+
+### InfluxDB CLI
+
+You may find it useful to install the InfluxDB client for
+adhoc metrics collection/viewing
+* Linux - `sudo apt-get install influxdb-client`
+* macOS - `brew install influxdb`
+
+
+Simple example of pulling all `thinclient` measurements out
+of the `local` database:
+```sh
+$ influx -database local -username admin -password admin -execute 'select * from thinclient'
+```
+
+Reference: https://docs.influxdata.com/influxdb/v1.5/query_language/
+
+### Monitoring
+
+To monitor activity, run one of:
+
+```
+$ docker logs -f influxdb
+$ docker logs -f grafana
+```
+
+### Reference
+* https://hub.docker.com/_/influxdata-influxdb
+* https://hub.docker.com/r/grafana/grafana

metrics/scripts/enable.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -e
+
+SOLANA_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/../.. || exit 1; pwd)"
+
+export SOLANA_METRICS_CONFIG="host=http://localhost:8086,db=local,u=admin,p=admin"
+
+# shellcheck source=scripts/configure-metrics.sh
+source "$SOLANA_ROOT"/scripts/configure-metrics.sh
+
+echo Local metrics enabled

metrics/scripts/grafana-provisioning/dashboards/dashboard.yml

@@ -0,0 +1,11 @@
+apiVersion: 1
+
+providers:
+- name: 'InfluxDB'
+  orgId: 1
+  folder: ''
+  type: file
+  disableDeletion: false
+  editable: true
+  options:
+    path: /etc/grafana/provisioning/dashboards

metrics/scripts/grafana-provisioning/datasources/datasource.yml

@@ -0,0 +1,16 @@
+# config file version
+apiVersion: 1
+
+datasources:
+- name: gdev-influxdb
+  type: influxdb
+  isDefault: true
+  access: proxy
+  database: local
+  user: admin
+  password: admin
+  basicAuth: true
+  basicAuthUser: admin
+  basicAuthPassword: admin
+  url: http://influxdb:8086
+  editable: true

metrics/scripts/grafana.ini

@@ -0,0 +1,455 @@
+##################### Grafana Configuration Example #####################
+#
+# Everything has defaults so you only need to uncomment things you want to
+# change
+
+# possible values : production, development
+;app_mode = production
+
+# instance name, defaults to HOSTNAME environment variable value or hostname if HOSTNAME var is empty
+;instance_name = ${HOSTNAME}
+
+#################################### Paths ####################################
+[paths]
+# Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used)
+;data = /var/lib/grafana
+
+# Directory where grafana can store logs
+;logs = /var/log/grafana
+
+# Directory where grafana will automatically scan and look for plugins
+;plugins = /var/lib/grafana/plugins
+
+# folder that contains provisioning config files that grafana will apply on startup and while running.
+;provisioning = conf/provisioning
+
+#################################### Server ####################################
+[server]
+# Protocol (http, https, socket)
+;protocol = http
+
+# The ip address to bind to, empty will bind to all interfaces
+;http_addr =
+
+# The http port  to use
+;http_port = 3000
+
+# The public facing domain name used to access grafana from a browser
+;domain = localhost
+
+# Redirect to correct domain if host header does not match domain
+# Prevents DNS rebinding attacks
+;enforce_domain = false
+
+# The full public facing url you use in browser, used for redirects and emails
+# If you use reverse proxy and sub path specify full url (with sub path)
+;root_url = http://localhost:3000
+
+# Log web requests
+;router_logging = false
+
+# the path relative working path
+;static_root_path = public
+
+# enable gzip
+;enable_gzip = false
+
+# https certs & key file
+;cert_file =
+;cert_key =
+
+# Unix socket path
+;socket =
+
+#################################### Database ####################################
+[database]
+# You can configure the database connection by specifying type, host, name, user and password
+# as separate properties or as on string using the url properties.
+
+# Either "mysql", "postgres" or "sqlite3", it's your choice
+;type = sqlite3
+;host = 127.0.0.1:3306
+;name = grafana
+;user = root
+# If the password contains # or ; you have to wrap it with triple quotes. Ex """#password;"""
+;password =
+
+# Use either URL or the previous fields to configure the database
+# Example: mysql://user:secret@host:port/database
+;url =
+
+# For "postgres" only, either "disable", "require" or "verify-full"
+;ssl_mode = disable
+
+# For "sqlite3" only, path relative to data_path setting
+;path = grafana.db
+
+# Max idle conn setting default is 2
+;max_idle_conn = 2
+
+# Max conn setting default is 0 (mean not set)
+;max_open_conn =
+
+# Connection Max Lifetime default is 14400 (means 14400 seconds or 4 hours)
+;conn_max_lifetime = 14400
+
+# Set to true to log the sql calls and execution times.
+log_queries =
+
+#################################### Session ####################################
+[session]
+# Either "memory", "file", "redis", "mysql", "postgres", default is "file"
+;provider = file
+
+# Provider config options
+# memory: not have any config yet
+# file: session dir path, is relative to grafana data_path
+# redis: config like redis server e.g. `addr=127.0.0.1:6379,pool_size=100,db=grafana`
+# mysql: go-sql-driver/mysql dsn config string, e.g. `user:password@tcp(127.0.0.1:3306)/database_name`
+# postgres: user=a password=b host=localhost port=5432 dbname=c sslmode=disable
+;provider_config = sessions
+
+# Session cookie name
+;cookie_name = grafana_sess
+
+# If you use session in https only, default is false
+;cookie_secure = false
+
+# Session life time, default is 86400
+;session_life_time = 86400
+
+#################################### Data proxy ###########################
+[dataproxy]
+
+# This enables data proxy logging, default is false
+;logging = false
+
+#################################### Analytics ####################################
+[analytics]
+# Server reporting, sends usage counters to stats.grafana.org every 24 hours.
+# No ip addresses are being tracked, only simple counters to track
+# running instances, dashboard and error counts. It is very helpful to us.
+# Change this option to false to disable reporting.
+;reporting_enabled = true
+
+# Set to false to disable all checks to https://grafana.net
+# for new vesions (grafana itself and plugins), check is used
+# in some UI views to notify that grafana or plugin update exists
+# This option does not cause any auto updates, nor send any information
+# only a GET request to http://grafana.com to get latest versions
+;check_for_updates = true
+
+# Google Analytics universal tracking code, only enabled if you specify an id here
+;google_analytics_ua_id =
+
+#################################### Security ####################################
+[security]
+# default admin user, created on startup
+;admin_user = admin
+
+# default admin password, can be changed before first start of grafana,  or in profile settings
+;admin_password = admin
+
+# used for signing
+;secret_key = SW2YcwTIb9zpOOhoPsMm
+
+# Auto-login remember days
+;login_remember_days = 7
+;cookie_username = grafana_user
+;cookie_remember_name = grafana_remember
+
+# disable gravatar profile images
+;disable_gravatar = false
+
+# data source proxy whitelist (ip_or_domain:port separated by spaces)
+;data_source_proxy_whitelist =
+
+# disable protection against brute force login attempts
+;disable_brute_force_login_protection = false
+
+#################################### Snapshots ###########################
+[snapshots]
+# snapshot sharing options
+;external_enabled = true
+;external_snapshot_url = https://snapshots-origin.raintank.io
+;external_snapshot_name = Publish to snapshot.raintank.io
+
+# remove expired snapshot
+;snapshot_remove_expired = true
+
+#################################### Dashboards History ##################
+[dashboards]
+# Number dashboard versions to keep (per dashboard). Default: 20, Minimum: 1
+;versions_to_keep = 20
+
+#################################### Users ###############################
+[users]
+# disable user signup / registration
+;allow_sign_up = true
+
+# Allow non admin users to create organizations
+;allow_org_create = true
+
+# Set to true to automatically assign new users to the default organization (id 1)
+auto_assign_org = true
+
+# Default role new users will be automatically assigned (if disabled above is set to true)
+auto_assign_org_role = Editor
+
+# Background text for the user field on the login page
+;login_hint = email or username
+
+# Default UI theme ("dark" or "light")
+;default_theme = dark
+
+# External user management, these options affect the organization users view
+;external_manage_link_url =
+;external_manage_link_name =
+;external_manage_info =
+
+# Viewers can edit/inspect dashboard settings in the browser. But not save the dashboard.
+viewers_can_edit = false
+
+[auth]
+# Set to true to disable (hide) the login form, useful if you use OAuth, defaults to false
+;disable_login_form = false
+
+# Set to true to disable the signout link in the side menu. useful if you use auth.proxy, defaults to false
+;disable_signout_menu = false
+
+# URL to redirect the user to after sign out
+;signout_redirect_url =
+
+#################################### Anonymous Auth ##########################
+[auth.anonymous]
+# enable anonymous access
+enabled = true
+
+# specify organization name that should be used for unauthenticated users
+;org_name = Main Org.
+
+# specify role for unauthenticated users
+;org_role = Viewer
+
+#################################### Github Auth ##########################
+[auth.github]
+;enabled = false
+allow_sign_up = true
+client_id = 5c585250dc68a3641f2a
+client_secret = 830996c62f790af4c4acc25174a79a2f88bdba09
+scopes = user:email,read:org
+auth_url = https://github.com/login/oauth/authorize
+token_url = https://github.com/login/oauth/access_token
+api_url = https://api.github.com/user
+;team_ids =
+allowed_organizations = solana-labs
+
+#################################### Google Auth ##########################
+[auth.google]
+;enabled = false
+;allow_sign_up = true
+;client_id = some_client_id
+;client_secret = some_client_secret
+;scopes = https://www.googleapis.com/auth/userinfo.profile https://www.googleapis.com/auth/userinfo.email
+;auth_url = https://accounts.google.com/o/oauth2/auth
+;token_url = https://accounts.google.com/o/oauth2/token
+;api_url = https://www.googleapis.com/oauth2/v1/userinfo
+;allowed_domains =
+
+#################################### Generic OAuth ##########################
+[auth.generic_oauth]
+;enabled = false
+;name = OAuth
+;allow_sign_up = true
+;client_id = some_id
+;client_secret = some_secret
+;scopes = user:email,read:org
+;auth_url = https://foo.bar/login/oauth/authorize
+;token_url = https://foo.bar/login/oauth/access_token
+;api_url = https://foo.bar/user
+;team_ids =
+;allowed_organizations =
+
+#################################### Grafana.com Auth ####################
+[auth.grafana_com]
+;enabled = false
+;allow_sign_up = true
+;client_id = some_id
+;client_secret = some_secret
+;scopes = user:email
+;allowed_organizations =
+
+#################################### Auth Proxy ##########################
+[auth.proxy]
+;enabled = false
+;header_name = X-WEBAUTH-USER
+;header_property = username
+;auto_sign_up = true
+;ldap_sync_ttl = 60
+;whitelist = 192.168.1.1, 192.168.2.1
+
+#################################### Basic Auth ##########################
+[auth.basic]
+;enabled = true
+
+#################################### Auth LDAP ##########################
+[auth.ldap]
+;enabled = false
+;config_file = /etc/grafana/ldap.toml
+;allow_sign_up = true
+
+#################################### SMTP / Emailing ##########################
+[smtp]
+;enabled = false
+;host = localhost:25
+;user =
+# If the password contains # or ; you have to wrap it with trippel quotes. Ex """#password;"""
+;password =
+;cert_file =
+;key_file =
+;skip_verify = false
+;from_address = admin@grafana.localhost
+;from_name = Grafana
+# EHLO identity in SMTP dialog (defaults to instance_name)
+;ehlo_identity = dashboard.example.com
+
+[emails]
+;welcome_email_on_sign_up = false
+
+#################################### Logging ##########################
+[log]
+# Either "console", "file", "syslog". Default is console and  file
+# Use space to separate multiple modes, e.g. "console file"
+;mode = console file
+
+# Either "debug", "info", "warn", "error", "critical", default is "info"
+;level = info
+
+# optional settings to set different levels for specific loggers. Ex filters = sqlstore:debug
+;filters =
+
+# For "console" mode only
+[log.console]
+;level =
+
+# log line format, valid options are text, console and json
+;format = console
+
+# For "file" mode only
+[log.file]
+;level =
+
+# log line format, valid options are text, console and json
+;format = text
+
+# This enables automated log rotate(switch of following options), default is true
+;log_rotate = true
+
+# Max line number of single file, default is 1000000
+;max_lines = 1000000
+
+# Max size shift of single file, default is 28 means 1 << 28, 256MB
+;max_size_shift = 28
+
+# Segment log daily, default is true
+;daily_rotate = true
+
+# Expired days of log file(delete after max days), default is 7
+;max_days = 7
+
+[log.syslog]
+;level =
+
+# log line format, valid options are text, console and json
+;format = text
+
+# Syslog network type and address. This can be udp, tcp, or unix. If left blank, the default unix endpoints will be used.
+;network =
+;address =
+
+# Syslog facility. user, daemon and local0 through local7 are valid.
+;facility =
+
+# Syslog tag. By default, the process' argv[0] is used.
+;tag =
+
+#################################### Alerting ############################
+[alerting]
+# Disable alerting engine & UI features
+;enabled = true
+# Makes it possible to turn off alert rule execution but alerting UI is visible
+;execute_alerts = true
+
+#################################### Explore #############################
+[explore]
+# Enable the Explore section
+enabled = false
+
+#################################### Internal Grafana Metrics ##########################
+# Metrics available at HTTP API Url /metrics
+[metrics]
+# Disable / Enable internal metrics
+;enabled           = true
+
+# Publish interval
+;interval_seconds  = 10
+
+# Send internal metrics to Graphite
+[metrics.graphite]
+# Enable by setting the address setting (ex localhost:2003)
+;address =
+;prefix = prod.grafana.%(instance_name)s.
+
+#################################### Distributed tracing ############
+[tracing.jaeger]
+# Enable by setting the address sending traces to jaeger (ex localhost:6831)
+;address = localhost:6831
+# Tag that will always be included in when creating new spans. ex (tag1:value1,tag2:value2)
+;always_included_tag = tag1:value1
+# Type specifies the type of the sampler: const, probabilistic, rateLimiting, or remote
+;sampler_type = const
+# jaeger samplerconfig param
+# for "const" sampler, 0 or 1 for always false/true respectively
+# for "probabilistic" sampler, a probability between 0 and 1
+# for "rateLimiting" sampler, the number of spans per second
+# for "remote" sampler, param is the same as for "probabilistic"
+# and indicates the initial sampling rate before the actual one
+# is received from the mothership
+;sampler_param = 1
+
+#################################### Grafana.com integration  ##########################
+# Url used to to import dashboards directly from Grafana.com
+[grafana_com]
+;url = https://grafana.com
+
+#################################### External image storage ##########################
+[external_image_storage]
+# Used for uploading images to public servers so they can be included in slack/email messages.
+# you can choose between (s3, webdav, gcs, azure_blob, local)
+;provider =
+
+[external_image_storage.s3]
+;bucket =
+;region =
+;path =
+;access_key =
+;secret_key =
+
+[external_image_storage.webdav]
+;url =
+;public_url =
+;username =
+;password =
+
+[external_image_storage.gcs]
+;key_file =
+;bucket =
+;path =
+
+[external_image_storage.azure_blob]
+;account_name =
+;account_key =
+;container_name =
+
+[external_image_storage.local]
+# does not require any configuration

metrics/scripts/influxdb.conf

@@ -0,0 +1,142 @@
+reporting-disabled = false
+bind-address = "127.0.0.1:8088"
+
+[meta]
+  dir = "/var/lib/influxdb/meta"
+  retention-autocreate = true
+  logging-enabled = true
+
+[data]
+  dir = "/var/lib/influxdb/data"
+  index-version = "inmem"
+  wal-dir = "/var/lib/influxdb/wal"
+  wal-fsync-delay = "0s"
+  query-log-enabled = true
+  cache-max-memory-size = 1073741824
+  cache-snapshot-memory-size = 26214400
+  cache-snapshot-write-cold-duration = "10m0s"
+  compact-full-write-cold-duration = "4h0m0s"
+  max-series-per-database = 1000000
+  max-values-per-tag = 100000
+  max-concurrent-compactions = 0
+  max-index-log-file-size = 1048576
+  trace-logging-enabled = false
+
+[coordinator]
+  write-timeout = "10s"
+  max-concurrent-queries = 0
+  query-timeout = "0s"
+  log-queries-after = "0s"
+  max-select-point = 0
+  max-select-series = 0
+  max-select-buckets = 0
+
+[retention]
+  enabled = true
+  check-interval = "30m0s"
+
+[shard-precreation]
+  enabled = true
+  check-interval = "10m0s"
+  advance-period = "30m0s"
+
+[monitor]
+  store-enabled = true
+  store-database = "_internal"
+  store-interval = "10s"
+
+[subscriber]
+  enabled = true
+  http-timeout = "30s"
+  insecure-skip-verify = false
+  ca-certs = ""
+  write-concurrency = 40
+  write-buffer-size = 1000
+
+[http]
+  enabled = true
+  bind-address = ":8086"
+  auth-enabled = false
+  log-enabled = true
+  write-tracing = false
+  pprof-enabled = true
+  debug-pprof-enabled = false
+  https-enabled = false
+  https-certificate = ""
+  https-private-key = ""
+  max-row-limit = 0
+  max-connection-limit = 0
+  shared-secret = ""
+realm = "InfluxDB"
+  unix-socket-enabled = false
+  bind-socket = "/var/run/influxdb.sock"
+  max-body-size = 25000000
+  access-log-path = ""
+
+[logging]
+  format = "auto"
+  level = "info"
+  suppress-logo = false
+
+[ifql]
+  enabled = false
+  log-enabled = true
+  bind-address = ":8082"
+
+[[graphite]]
+  enabled = false
+  bind-address = ":2003"
+  database = "graphite"
+  retention-policy = ""
+  protocol = "tcp"
+  batch-size = 5000
+  batch-pending = 10
+  batch-timeout = "1s"
+  consistency-level = "one"
+  separator = "."
+  udp-read-buffer = 0
+
+[[collectd]]
+  enabled = false
+  bind-address = ":25826"
+  database = "collectd"
+  retention-policy = ""
+  batch-size = 5000
+  batch-pending = 10
+  batch-timeout = "10s"
+  read-buffer = 0
+  typesdb = "/usr/share/collectd/types.db"
+  security-level = "none"
+  auth-file = "/etc/collectd/auth_file"
+  parse-multivalue-plugin = "split"
+
+[[opentsdb]]
+  enabled = false
+  bind-address = ":4242"
+  database = "opentsdb"
+  retention-policy = ""
+  consistency-level = "one"
+  tls-enabled = false
+  certificate = ""
+  batch-size = 1000
+  batch-pending = 5
+  batch-timeout = "1s"
+  log-point-errors = true
+
+[[udp]]
+  enabled = false
+  bind-address = ":8089"
+  database = "udp"
+  retention-policy = ""
+  batch-size = 5000
+  batch-pending = 10
+  read-buffer = 0
+  batch-timeout = "1s"
+  precision = ""
+
+[continuous_queries]
+  log-enabled = true
+  enabled = true
+  query-stats-enabled = false
+  run-interval = "1s"
+

metrics/scripts/start.sh

@@ -0,0 +1,49 @@
+#!/bin/bash -ex
+#
+# (Re)starts the local metrics
+#
+
+cd "$(dirname "$0")"
+
+# Stop if already running
+./stop.sh
+
+: "${INFLUXDB_IMAGE:=influxdb:1.6}"
+: "${GRAFANA_IMAGE:=grafana/grafana:5.2.3}"
+
+docker pull $INFLUXDB_IMAGE
+docker pull $GRAFANA_IMAGE
+
+docker network remove influxdb || true
+docker network create influxdb
+
+docker run \
+  --detach \
+  --name=influxdb \
+  --net=influxdb \
+  --publish 8086:8086 \
+  --user "$(id -u):$(id -g)" \
+  --volume "$PWD"/influxdb.conf:/etc/influxdb/influxdb.conf:ro \
+  --volume "$PWD"/lib/influxdb:/var/lib/influxdb \
+  --env INFLUXDB_DB=local \
+  --env INFLUXDB_ADMIN_USER=admin \
+  --env INFLUXDB_ADMIN_PASSWORD=admin \
+  $INFLUXDB_IMAGE -config /etc/influxdb/influxdb.conf /init-influxdb.sh
+
+docker run \
+  --detach \
+  --name=grafana \
+  --net=influxdb \
+  --publish 3000:3000 \
+  --user "$(id -u):$(id -g)" \
+  --env GF_PATHS_CONFIG=/grafana.ini \
+  --volume "$PWD"/grafana.ini:/grafana.ini:ro \
+  --volume "$PWD"/lib/grafana:/var/lib/grafana \
+  --volume "$PWD"/grafana-provisioning/:/etc/grafana/provisioning \
+  $GRAFANA_IMAGE
+
+sleep 5
+
+./status.sh
+
+exit 0

metrics/scripts/status.sh

@@ -0,0 +1,27 @@
+#!/bin/bash -e
+#
+# Checks the status of lcoal metrics
+#
+
+cd "$(dirname "$0")"
+
+(
+  set -x
+  docker ps --no-trunc --size
+)
+
+if ! timeout 10s curl -v --head http://localhost:8086/ping; then
+  echo Error: InfluxDB not running
+  exit 1
+fi
+
+if ! timeout 10s curl -v --head http://localhost:3000; then
+  echo Error: Grafana not running
+  exit 1
+fi
+
+echo Local metrics up and running
+echo - Enable local metric collection per shell by running \'source ./enable.sh\'
+echo - View dashboard at http://localhost:3000/d/local/local-monitor
+exit 0
+

metrics/scripts/stop.sh

@@ -0,0 +1,20 @@
+#!/bin/bash -e
+#
+# Stops local metrics
+#
+
+cd "$(dirname "$0")"
+
+for container in influxdb grafana; do
+  if [ "$(docker ps -q -f name=$container)" ]; then
+  (
+    set +e
+    docker kill $container
+    docker rm $container
+    exit 0
+  )
+  fi
+ done
+
+ echo Local metrics stopped
+ exit 0

scripts/configure-metrics.sh

@@ -10,7 +10,6 @@
 # file:
 # shellcheck disable=2034
 #
-metricsWriteDatapoint="$(dirname "${BASH_SOURCE[0]}")"/metrics-write-datapoint.sh
 
 configureMetrics() {
   [[ -n $SOLANA_METRICS_CONFIG ]] || return 0
@@ -49,3 +48,5 @@ configureMetrics() {
   done
 }
 configureMetrics
+
+metricsWriteDatapoint="$(dirname "${BASH_SOURCE[0]}")"/metrics-write-datapoint.sh