ci: update kapacitor config (#32069)

* add two new kapacitor post endpoints

* use kapacitor.sh to start

* add pager duty channels

* fix shell check

metrics/metrics-main/kapacitor.conf

@@ -193,15 +193,34 @@ default-retention-policy = ""
   url = "https://api.pushover.net/1/messages.json"
 
 [[httppost]]
-  endpoint = "example"
-  url = "http://example.com"
-  alert-template = ""
-  alert-template-file = ""
-  row-template = ""
-  row-template-file = ""
-  [httppost.basic-auth]
-    username = ""
-    password = ""
+  endpoint = "discord-canaries-monitoring"
+  url = "$DISCORD_WEBHOOK_CANARIES_MONITORING"
+  headers = { "Content-Type" = "application/json" }
+  alert-template = "{ \"username\": \"kapacitor\", \"content\": \"{{.Message}}\"}"
+
+[[httppost]]
+  endpoint = "discord-canaries-monitoring-panic-check"
+  url = "$DISCORD_WEBHOOK_CANARIES_MONITORING"
+  headers = { "Content-Type" = "application/json" }
+  alert-template = "{ \"username\": \"kapacitor\", \"content\": \"{{range .Data.Series}}{{$columns := .Columns}}{{range .Values}}⚠️\\n`{{index . 1}}`\\n{{index . 2}}){{end}}{{end}}\" }"
+
+[[httppost]]
+  endpoint = "discord-mb-pager-duty"
+  url = "$DISCORD_WEBHOOK_MB_PAGER_DUTY"
+  headers = { "Content-Type" = "application/json" }
+  alert-template = "{ \"username\": \"kapacitor\", \"content\": \"{{.Message}}\"}"
+
+[[httppost]]
+  endpoint = "discord-testnet-pager-duty"
+  url = "$DISCORD_WEBHOOK_TESTNET_PAGER_DUTY"
+  headers = { "Content-Type" = "application/json" }
+  alert-template = "{ \"username\": \"kapacitor\", \"content\": \"{{.Message}}\"}"
+
+[[httppost]]
+  endpoint = "discord-devnet-pager-duty"
+  url = "$DISCORD_WEBHOOK_DEVNET_PAGER_DUTY"
+  headers = { "Content-Type" = "application/json" }
+  alert-template = "{ \"username\": \"kapacitor\", \"content\": \"{{.Message}}\"}"
 
 [smtp]
   enabled = false

metrics/metrics-main/kapacitor.sh

@@ -24,9 +24,18 @@ container=kapacitor
   exit 0
 )
 
+# shellcheck disable=SC2016
+sed -i 's|$DISCORD_WEBHOOK_CANARIES_MONITORING|'"$DISCORD_WEBHOOK_CANARIES_MONITORING"'|g' "$PWD"/kapacitor.conf
+# shellcheck disable=SC2016
+sed -i 's|$DISCORD_WEBHOOK_MB_PAGER_DUTY|'"$DISCORD_WEBHOOK_MB_PAGER_DUTY"'|g' "$PWD"/kapacitor.conf
+# shellcheck disable=SC2016
+sed -i 's|$DISCORD_WEBHOOK_TESTNET_PAGER_DUTY|'"$DISCORD_WEBHOOK_TESTNET_PAGER_DUTY"'|g' "$PWD"/kapacitor.conf
+# shellcheck disable=SC2016
+sed -i 's|$DISCORD_WEBHOOK_DEVNET_PAGER_DUTY|'"$DISCORD_WEBHOOK_DEVNET_PAGER_DUTY"'|g' "$PWD"/kapacitor.conf
+
 #running influx kapacitor service
 sudo docker run \
-  --memory=10g \
+  --memory=30g \
   --detach \
   --name=kapacitor \
   --env KAPACITOR_USERNAME="$KAPACITOR_USERNAME" \

metrics/metrics-main/start.sh

@@ -12,7 +12,6 @@ echo "HOST: $HOST"
 
 : "${INFLUXDB_IMAGE:=influxdb:1.7}"
 : "${CHRONOGRAF_IMAGE:=chronograf:1.9.4}"
-: "${KAPACITOR_IMAGE:=kapacitor:1.6.5}"
 : "${GRAFANA_IMAGE:=grafana/grafana:9.4.7}"
 : "${PROMETHEUS_IMAGE:=prom/prometheus:v2.28.0}"
 : "${ALERTMANAGER_IMAGE:=prom/alertmanager:v0.23.0}"
@@ -20,7 +19,6 @@ echo "HOST: $HOST"
 
 docker pull $INFLUXDB_IMAGE
 docker pull $CHRONOGRAF_IMAGE
-docker pull $KAPACITOR_IMAGE
 docker pull $GRAFANA_IMAGE
 docker pull $PROMETHEUS_IMAGE
 docker pull $ALERTMANAGER_IMAGE
@@ -140,19 +138,8 @@ sudo docker run \
   --log-opt max-file=5 \
   $CHRONOGRAF_IMAGE --influxdb-url=https://"$HOST":8086 --auth-duration="720h" --inactivity-duration="48h"
 
-sudo docker run \
-  --memory=10g \
-  --detach \
-  --name=kapacitor \
-  --env KAPACITOR_USERNAME="$KAPACITOR_USERNAME" \
-  --env KAPACITOR_USERNAME="$KAPACITOR_PASSWORD" \
-  --publish 9092:9092 \
-  --volume "$PWD"/kapacitor.conf:/etc/kapacitor/kapacitor.conf \
-  --volume /var/lib/kapacitor:/var/lib/kapacitor \
-  --user "$(id -u):$(id -g)" \
-  --log-opt max-size=1g \
-  --log-opt max-file=5  \
-  $KAPACITOR_IMAGE
+#shellcheck source=metrics/metrics-main/kapacitor.sh
+source kapacitor.sh
 
 curl -h | sed -ne '/--tlsv/p'
 curl --retry 10 --retry-delay 5 -v --head https://"$HOST":8086/ping