2022-10-03 17:15:56 -07:00
|
|
|
/// To activate Slack, Discord, PagerDuty and/or Telegram notifications, define these environment variables
|
2020-05-14 17:32:08 -07:00
|
|
|
/// before using the `Notifier`
|
|
|
|
/// ```bash
|
|
|
|
/// export SLACK_WEBHOOK=...
|
|
|
|
/// export DISCORD_WEBHOOK=...
|
|
|
|
/// ```
|
|
|
|
///
|
|
|
|
/// Telegram requires the following two variables:
|
|
|
|
/// ```bash
|
|
|
|
/// export TELEGRAM_BOT_TOKEN=...
|
|
|
|
/// export TELEGRAM_CHAT_ID=...
|
|
|
|
/// ```
|
|
|
|
///
|
2022-10-03 17:15:56 -07:00
|
|
|
/// PagerDuty requires an Integration Key from the Events API v2 (Add this integration to your PagerDuty service to get this)
|
|
|
|
///
|
|
|
|
/// ```bash
|
|
|
|
/// export PAGERDUTY_INTEGRATION_KEY=...
|
|
|
|
/// ```
|
|
|
|
///
|
2020-05-14 17:32:08 -07:00
|
|
|
/// To receive a Twilio SMS notification on failure, having a Twilio account,
|
|
|
|
/// and a sending number owned by that account,
|
|
|
|
/// define environment variable before running `solana-watchtower`:
|
|
|
|
/// ```bash
|
|
|
|
/// export TWILIO_CONFIG='ACCOUNT=<account>,TOKEN=<securityToken>,TO=<receivingNumber>,FROM=<sendingNumber>'
|
|
|
|
/// ```
|
2019-12-12 23:49:16 -08:00
|
|
|
use log::*;
|
2021-12-03 09:00:31 -08:00
|
|
|
use {
|
|
|
|
reqwest::{blocking::Client, StatusCode},
|
|
|
|
serde_json::json,
|
2022-10-05 11:55:45 -07:00
|
|
|
solana_sdk::hash::Hash,
|
2021-12-03 09:00:31 -08:00
|
|
|
std::{env, str::FromStr, thread::sleep, time::Duration},
|
|
|
|
};
|
2019-12-12 23:49:16 -08:00
|
|
|
|
2019-12-16 10:48:56 -08:00
|
|
|
struct TelegramWebHook {
|
|
|
|
bot_token: String,
|
|
|
|
chat_id: String,
|
|
|
|
}
|
|
|
|
|
2020-03-02 22:37:57 -08:00
|
|
|
#[derive(Debug, Default)]
|
|
|
|
struct TwilioWebHook {
|
|
|
|
account: String,
|
|
|
|
token: String,
|
|
|
|
to: String,
|
|
|
|
from: String,
|
|
|
|
}
|
|
|
|
|
|
|
|
impl TwilioWebHook {
|
|
|
|
fn complete(&self) -> bool {
|
|
|
|
!(self.account.is_empty()
|
|
|
|
|| self.token.is_empty()
|
|
|
|
|| self.to.is_empty()
|
|
|
|
|| self.from.is_empty())
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
fn get_twilio_config() -> Result<Option<TwilioWebHook>, String> {
|
|
|
|
let config_var = env::var("TWILIO_CONFIG");
|
|
|
|
|
|
|
|
if config_var.is_err() {
|
|
|
|
info!("Twilio notifications disabled");
|
|
|
|
return Ok(None);
|
|
|
|
}
|
|
|
|
|
|
|
|
let mut config = TwilioWebHook::default();
|
|
|
|
|
|
|
|
for pair in config_var.unwrap().split(',') {
|
|
|
|
let nv: Vec<_> = pair.split('=').collect();
|
|
|
|
if nv.len() != 2 {
|
|
|
|
return Err(format!("TWILIO_CONFIG is invalid: '{}'", pair));
|
|
|
|
}
|
|
|
|
let v = nv[1].to_string();
|
|
|
|
match nv[0] {
|
|
|
|
"ACCOUNT" => config.account = v,
|
|
|
|
"TOKEN" => config.token = v,
|
|
|
|
"TO" => config.to = v,
|
|
|
|
"FROM" => config.from = v,
|
|
|
|
_ => return Err(format!("TWILIO_CONFIG is invalid: '{}'", pair)),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if !config.complete() {
|
|
|
|
return Err("TWILIO_CONFIG is incomplete".to_string());
|
|
|
|
}
|
|
|
|
Ok(Some(config))
|
|
|
|
}
|
|
|
|
|
2022-10-05 11:55:45 -07:00
|
|
|
enum NotificationChannel {
|
2021-01-19 09:20:25 -08:00
|
|
|
Discord(String),
|
|
|
|
Slack(String),
|
2022-10-03 17:15:56 -07:00
|
|
|
PagerDuty(String),
|
2021-01-19 09:20:25 -08:00
|
|
|
Telegram(TelegramWebHook),
|
|
|
|
Twilio(TwilioWebHook),
|
2021-02-24 21:32:36 -08:00
|
|
|
Log(Level),
|
2021-01-19 09:20:25 -08:00
|
|
|
}
|
|
|
|
|
2022-10-05 11:55:45 -07:00
|
|
|
#[derive(Clone)]
|
|
|
|
pub enum NotificationType {
|
|
|
|
Trigger { incident: Hash },
|
|
|
|
Resolve { incident: Hash },
|
|
|
|
}
|
|
|
|
|
2019-12-12 23:49:16 -08:00
|
|
|
pub struct Notifier {
|
|
|
|
client: Client,
|
2022-10-05 11:55:45 -07:00
|
|
|
notifiers: Vec<NotificationChannel>,
|
2019-12-12 23:49:16 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
impl Notifier {
|
2020-05-14 17:32:08 -07:00
|
|
|
pub fn default() -> Self {
|
|
|
|
Self::new("")
|
2020-05-01 17:48:22 -07:00
|
|
|
}
|
|
|
|
|
2020-05-14 17:32:08 -07:00
|
|
|
pub fn new(env_prefix: &str) -> Self {
|
2020-05-01 17:48:22 -07:00
|
|
|
info!("Initializing {}Notifier", env_prefix);
|
|
|
|
|
2021-01-19 09:20:25 -08:00
|
|
|
let mut notifiers = vec![];
|
|
|
|
|
|
|
|
if let Ok(webhook) = env::var(format!("{}DISCORD_WEBHOOK", env_prefix)) {
|
2022-10-05 11:55:45 -07:00
|
|
|
notifiers.push(NotificationChannel::Discord(webhook));
|
2021-01-19 09:20:25 -08:00
|
|
|
}
|
|
|
|
if let Ok(webhook) = env::var(format!("{}SLACK_WEBHOOK", env_prefix)) {
|
2022-10-05 11:55:45 -07:00
|
|
|
notifiers.push(NotificationChannel::Slack(webhook));
|
2021-01-19 09:20:25 -08:00
|
|
|
}
|
2022-10-03 17:15:56 -07:00
|
|
|
if let Ok(routing_key) = env::var(format!("{}PAGERDUTY_INTEGRATION_KEY", env_prefix)) {
|
2022-10-05 11:55:45 -07:00
|
|
|
notifiers.push(NotificationChannel::PagerDuty(routing_key));
|
2022-10-03 17:15:56 -07:00
|
|
|
}
|
2021-01-19 09:20:25 -08:00
|
|
|
|
|
|
|
if let (Ok(bot_token), Ok(chat_id)) = (
|
2020-05-01 17:48:22 -07:00
|
|
|
env::var(format!("{}TELEGRAM_BOT_TOKEN", env_prefix)),
|
|
|
|
env::var(format!("{}TELEGRAM_CHAT_ID", env_prefix)),
|
|
|
|
) {
|
2022-10-05 11:55:45 -07:00
|
|
|
notifiers.push(NotificationChannel::Telegram(TelegramWebHook {
|
2021-01-19 09:20:25 -08:00
|
|
|
bot_token,
|
|
|
|
chat_id,
|
|
|
|
}));
|
|
|
|
}
|
|
|
|
|
|
|
|
if let Ok(Some(webhook)) = get_twilio_config() {
|
2022-10-05 11:55:45 -07:00
|
|
|
notifiers.push(NotificationChannel::Twilio(webhook));
|
2021-01-19 09:20:25 -08:00
|
|
|
}
|
|
|
|
|
2021-02-24 21:32:36 -08:00
|
|
|
if let Ok(log_level) = env::var(format!("{}LOG_NOTIFIER_LEVEL", env_prefix)) {
|
|
|
|
match Level::from_str(&log_level) {
|
2022-10-05 11:55:45 -07:00
|
|
|
Ok(level) => notifiers.push(NotificationChannel::Log(level)),
|
2021-02-24 21:32:36 -08:00
|
|
|
Err(e) => warn!(
|
|
|
|
"could not parse specified log notifier level string ({}): {}",
|
|
|
|
log_level, e
|
|
|
|
),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-01-19 09:20:25 -08:00
|
|
|
info!("{} notifiers", notifiers.len());
|
2019-12-16 10:48:56 -08:00
|
|
|
|
2019-12-12 23:49:16 -08:00
|
|
|
Notifier {
|
|
|
|
client: Client::new(),
|
2021-01-19 09:20:25 -08:00
|
|
|
notifiers,
|
2019-12-12 23:49:16 -08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-01-19 09:20:25 -08:00
|
|
|
pub fn is_empty(&self) -> bool {
|
|
|
|
self.notifiers.is_empty()
|
|
|
|
}
|
|
|
|
|
2022-10-05 11:55:45 -07:00
|
|
|
pub fn send(&self, msg: &str, notification_type: &NotificationType) {
|
2021-01-19 09:20:25 -08:00
|
|
|
for notifier in &self.notifiers {
|
|
|
|
match notifier {
|
2022-10-05 11:55:45 -07:00
|
|
|
NotificationChannel::Discord(webhook) => {
|
2021-01-19 09:20:25 -08:00
|
|
|
for line in msg.split('\n') {
|
|
|
|
// Discord rate limiting is aggressive, limit to 1 message a second
|
|
|
|
sleep(Duration::from_millis(1000));
|
|
|
|
|
|
|
|
info!("Sending {}", line);
|
|
|
|
let data = json!({ "content": line });
|
|
|
|
|
|
|
|
loop {
|
|
|
|
let response = self.client.post(webhook).json(&data).send();
|
|
|
|
|
|
|
|
if let Err(err) = response {
|
|
|
|
warn!("Failed to send Discord message: \"{}\": {:?}", line, err);
|
|
|
|
break;
|
|
|
|
} else if let Ok(response) = response {
|
|
|
|
info!("response status: {}", response.status());
|
|
|
|
if response.status() == StatusCode::TOO_MANY_REQUESTS {
|
|
|
|
warn!("rate limited!...");
|
|
|
|
warn!("response text: {:?}", response.text());
|
|
|
|
sleep(Duration::from_secs(2));
|
|
|
|
} else {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
2020-05-14 17:32:08 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2022-10-05 11:55:45 -07:00
|
|
|
NotificationChannel::Slack(webhook) => {
|
2021-01-19 09:20:25 -08:00
|
|
|
let data = json!({ "text": msg });
|
|
|
|
if let Err(err) = self.client.post(webhook).json(&data).send() {
|
|
|
|
warn!("Failed to send Slack message: {:?}", err);
|
|
|
|
}
|
|
|
|
}
|
2022-10-05 11:55:45 -07:00
|
|
|
NotificationChannel::PagerDuty(routing_key) => {
|
|
|
|
let event_action = match notification_type {
|
|
|
|
NotificationType::Trigger { incident: _ } => String::from("trigger"),
|
|
|
|
NotificationType::Resolve { incident: _ } => String::from("resolve"),
|
|
|
|
};
|
|
|
|
let dedup_key = match notification_type {
|
|
|
|
NotificationType::Trigger { ref incident } => incident.clone().to_string(),
|
|
|
|
NotificationType::Resolve { ref incident } => incident.clone().to_string(),
|
|
|
|
};
|
|
|
|
|
|
|
|
let data = json!({"payload":{"summary":msg,"source":"solana-watchtower","severity":"critical"},"routing_key":routing_key,"event_action":event_action,"dedup_key":dedup_key});
|
2022-10-03 17:15:56 -07:00
|
|
|
let url = "https://events.pagerduty.com/v2/enqueue";
|
|
|
|
|
|
|
|
if let Err(err) = self.client.post(url).json(&data).send() {
|
|
|
|
warn!("Failed to send PagerDuty alert: {:?}", err);
|
|
|
|
}
|
|
|
|
}
|
2019-12-16 10:48:56 -08:00
|
|
|
|
2022-10-05 11:55:45 -07:00
|
|
|
NotificationChannel::Telegram(TelegramWebHook { chat_id, bot_token }) => {
|
2021-01-19 09:20:25 -08:00
|
|
|
let data = json!({ "chat_id": chat_id, "text": msg });
|
|
|
|
let url = format!("https://api.telegram.org/bot{}/sendMessage", bot_token);
|
2019-12-16 10:48:56 -08:00
|
|
|
|
2021-01-19 09:20:25 -08:00
|
|
|
if let Err(err) = self.client.post(&url).json(&data).send() {
|
|
|
|
warn!("Failed to send Telegram message: {:?}", err);
|
|
|
|
}
|
|
|
|
}
|
2020-03-02 22:37:57 -08:00
|
|
|
|
2022-10-05 11:55:45 -07:00
|
|
|
NotificationChannel::Twilio(TwilioWebHook {
|
2021-01-19 09:20:25 -08:00
|
|
|
account,
|
|
|
|
token,
|
|
|
|
to,
|
|
|
|
from,
|
|
|
|
}) => {
|
|
|
|
let url = format!(
|
|
|
|
"https://{}:{}@api.twilio.com/2010-04-01/Accounts/{}/Messages.json",
|
|
|
|
account, token, account
|
|
|
|
);
|
|
|
|
let params = [("To", to), ("From", from), ("Body", &msg.to_string())];
|
|
|
|
if let Err(err) = self.client.post(&url).form(¶ms).send() {
|
|
|
|
warn!("Failed to send Twilio message: {:?}", err);
|
|
|
|
}
|
|
|
|
}
|
2022-10-05 11:55:45 -07:00
|
|
|
NotificationChannel::Log(level) => {
|
2021-02-24 21:32:36 -08:00
|
|
|
log!(*level, "{}", msg)
|
|
|
|
}
|
2020-03-02 22:37:57 -08:00
|
|
|
}
|
|
|
|
}
|
2019-12-12 23:49:16 -08:00
|
|
|
}
|
|
|
|
}
|