sanity check metrics configuration (#32799)
This commit is contained in:
parent
b83cef43fc
commit
58cca78067
|
@ -6411,6 +6411,7 @@ dependencies = [
|
||||||
"reqwest",
|
"reqwest",
|
||||||
"serial_test",
|
"serial_test",
|
||||||
"solana-sdk",
|
"solana-sdk",
|
||||||
|
"thiserror",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|
|
@ -65,7 +65,9 @@ use {
|
||||||
use_snapshot_archives_at_startup::UseSnapshotArchivesAtStartup,
|
use_snapshot_archives_at_startup::UseSnapshotArchivesAtStartup,
|
||||||
},
|
},
|
||||||
solana_measure::measure::Measure,
|
solana_measure::measure::Measure,
|
||||||
solana_metrics::{datapoint_info, poh_timing_point::PohTimingSender},
|
solana_metrics::{
|
||||||
|
datapoint_info, metrics::metrics_config_sanity_check, poh_timing_point::PohTimingSender,
|
||||||
|
},
|
||||||
solana_poh::{
|
solana_poh::{
|
||||||
poh_recorder::PohRecorder,
|
poh_recorder::PohRecorder,
|
||||||
poh_service::{self, PohService},
|
poh_service::{self, PohService},
|
||||||
|
@ -1250,10 +1252,14 @@ impl Validator {
|
||||||
config.generator_config.clone(),
|
config.generator_config.clone(),
|
||||||
);
|
);
|
||||||
|
|
||||||
|
let cluster_type = bank_forks.read().unwrap().root_bank().cluster_type();
|
||||||
|
metrics_config_sanity_check(cluster_type)?;
|
||||||
|
|
||||||
datapoint_info!(
|
datapoint_info!(
|
||||||
"validator-new",
|
"validator-new",
|
||||||
("id", id.to_string(), String),
|
("id", id.to_string(), String),
|
||||||
("version", solana_version::version!(), String)
|
("version", solana_version::version!(), String),
|
||||||
|
("cluster_type", cluster_type as u32, i64),
|
||||||
);
|
);
|
||||||
|
|
||||||
*start_progress.write().unwrap() = ValidatorStartProgress::Running;
|
*start_progress.write().unwrap() = ValidatorStartProgress::Running;
|
||||||
|
|
|
@ -16,6 +16,7 @@ lazy_static = { workspace = true }
|
||||||
log = { workspace = true }
|
log = { workspace = true }
|
||||||
reqwest = { workspace = true, features = ["blocking", "brotli", "deflate", "gzip", "rustls-tls", "json"] }
|
reqwest = { workspace = true, features = ["blocking", "brotli", "deflate", "gzip", "rustls-tls", "json"] }
|
||||||
solana-sdk = { workspace = true }
|
solana-sdk = { workspace = true }
|
||||||
|
thiserror = { workspace = true }
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
env_logger = { workspace = true }
|
env_logger = { workspace = true }
|
||||||
|
|
|
@ -6,7 +6,7 @@ use {
|
||||||
gethostname::gethostname,
|
gethostname::gethostname,
|
||||||
lazy_static::lazy_static,
|
lazy_static::lazy_static,
|
||||||
log::*,
|
log::*,
|
||||||
solana_sdk::hash::hash,
|
solana_sdk::{genesis_config::ClusterType, hash::hash},
|
||||||
std::{
|
std::{
|
||||||
cmp,
|
cmp,
|
||||||
collections::HashMap,
|
collections::HashMap,
|
||||||
|
@ -17,10 +17,31 @@ use {
|
||||||
thread,
|
thread,
|
||||||
time::{Duration, Instant, UNIX_EPOCH},
|
time::{Duration, Instant, UNIX_EPOCH},
|
||||||
},
|
},
|
||||||
|
thiserror::Error,
|
||||||
};
|
};
|
||||||
|
|
||||||
type CounterMap = HashMap<(&'static str, u64), CounterPoint>;
|
type CounterMap = HashMap<(&'static str, u64), CounterPoint>;
|
||||||
|
|
||||||
|
#[derive(Debug, Error)]
|
||||||
|
pub enum MetricsError {
|
||||||
|
#[error(transparent)]
|
||||||
|
VarError(#[from] std::env::VarError),
|
||||||
|
#[error(transparent)]
|
||||||
|
ReqwestError(#[from] reqwest::Error),
|
||||||
|
#[error("SOLANA_METRICS_CONFIG is invalid: '{0}'")]
|
||||||
|
ConfigInvalid(String),
|
||||||
|
#[error("SOLANA_METRICS_CONFIG is incomplete")]
|
||||||
|
ConfigIncomplete,
|
||||||
|
#[error("SOLANA_METRICS_CONFIG database mismatch: {0}")]
|
||||||
|
DbMismatch(String),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<MetricsError> for String {
|
||||||
|
fn from(error: MetricsError) -> Self {
|
||||||
|
error.to_string()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
impl From<&CounterPoint> for DataPoint {
|
impl From<&CounterPoint> for DataPoint {
|
||||||
fn from(counter_point: &CounterPoint) -> Self {
|
fn from(counter_point: &CounterPoint) -> Self {
|
||||||
let mut point = Self::new(counter_point.name);
|
let mut point = Self::new(counter_point.name);
|
||||||
|
@ -58,7 +79,7 @@ impl InfluxDbMetricsWriter {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn build_write_url() -> Result<String, String> {
|
fn build_write_url() -> Result<String, MetricsError> {
|
||||||
let config = get_metrics_config().map_err(|err| {
|
let config = get_metrics_config().map_err(|err| {
|
||||||
info!("metrics disabled: {}", err);
|
info!("metrics disabled: {}", err);
|
||||||
err
|
err
|
||||||
|
@ -381,16 +402,14 @@ impl MetricsConfig {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn get_metrics_config() -> Result<MetricsConfig, String> {
|
fn get_metrics_config() -> Result<MetricsConfig, MetricsError> {
|
||||||
let mut config = MetricsConfig::default();
|
let mut config = MetricsConfig::default();
|
||||||
|
let config_var = env::var("SOLANA_METRICS_CONFIG")?;
|
||||||
let config_var =
|
|
||||||
env::var("SOLANA_METRICS_CONFIG").map_err(|err| format!("SOLANA_METRICS_CONFIG: {err}"))?;
|
|
||||||
|
|
||||||
for pair in config_var.split(',') {
|
for pair in config_var.split(',') {
|
||||||
let nv: Vec<_> = pair.split('=').collect();
|
let nv: Vec<_> = pair.split('=').collect();
|
||||||
if nv.len() != 2 {
|
if nv.len() != 2 {
|
||||||
return Err(format!("SOLANA_METRICS_CONFIG is invalid: '{pair}'"));
|
return Err(MetricsError::ConfigInvalid(pair.to_string()));
|
||||||
}
|
}
|
||||||
let v = nv[1].to_string();
|
let v = nv[1].to_string();
|
||||||
match nv[0] {
|
match nv[0] {
|
||||||
|
@ -398,27 +417,42 @@ fn get_metrics_config() -> Result<MetricsConfig, String> {
|
||||||
"db" => config.db = v,
|
"db" => config.db = v,
|
||||||
"u" => config.username = v,
|
"u" => config.username = v,
|
||||||
"p" => config.password = v,
|
"p" => config.password = v,
|
||||||
_ => return Err(format!("SOLANA_METRICS_CONFIG is invalid: '{pair}'")),
|
_ => return Err(MetricsError::ConfigInvalid(pair.to_string())),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if !config.complete() {
|
if !config.complete() {
|
||||||
return Err("SOLANA_METRICS_CONFIG is incomplete".to_string());
|
return Err(MetricsError::ConfigIncomplete);
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(config)
|
Ok(config)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn query(q: &str) -> Result<String, String> {
|
pub fn metrics_config_sanity_check(cluster_type: ClusterType) -> Result<(), MetricsError> {
|
||||||
|
let config = match get_metrics_config() {
|
||||||
|
Ok(config) => config,
|
||||||
|
Err(MetricsError::VarError(std::env::VarError::NotPresent)) => return Ok(()),
|
||||||
|
Err(e) => return Err(e),
|
||||||
|
};
|
||||||
|
match &config.db[..] {
|
||||||
|
"mainnet-beta" if cluster_type != ClusterType::MainnetBeta => (),
|
||||||
|
"tds" if cluster_type != ClusterType::Testnet => (),
|
||||||
|
"devnet" if cluster_type != ClusterType::Devnet => (),
|
||||||
|
_ => return Ok(()),
|
||||||
|
};
|
||||||
|
let (host, db) = (&config.host, &config.db);
|
||||||
|
let msg = format!("cluster_type={cluster_type:?} host={host} database={db}");
|
||||||
|
Err(MetricsError::DbMismatch(msg))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn query(q: &str) -> Result<String, MetricsError> {
|
||||||
let config = get_metrics_config()?;
|
let config = get_metrics_config()?;
|
||||||
let query_url = format!(
|
let query_url = format!(
|
||||||
"{}/query?u={}&p={}&q={}",
|
"{}/query?u={}&p={}&q={}",
|
||||||
&config.host, &config.username, &config.password, &q
|
&config.host, &config.username, &config.password, &q
|
||||||
);
|
);
|
||||||
|
|
||||||
let response = reqwest::blocking::get(query_url.as_str())
|
let response = reqwest::blocking::get(query_url.as_str())?.text()?;
|
||||||
.map_err(|err| err.to_string())?
|
|
||||||
.text()
|
|
||||||
.map_err(|err| err.to_string())?;
|
|
||||||
|
|
||||||
Ok(response)
|
Ok(response)
|
||||||
}
|
}
|
||||||
|
|
|
@ -5230,6 +5230,7 @@ dependencies = [
|
||||||
"log",
|
"log",
|
||||||
"reqwest",
|
"reqwest",
|
||||||
"solana-sdk",
|
"solana-sdk",
|
||||||
|
"thiserror",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|
Loading…
Reference in New Issue