Rate limit counter metrics points to one per second (#5496)
* Rate limit counter metrics points to one per second * Remove old env var * Test that metrics counter is incrementing * Fix typo
This commit is contained in:
parent
771d1a78fd
commit
0fde19239b
|
@ -3534,6 +3534,7 @@ dependencies = [
|
|||
"log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"reqwest 0.9.19 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde_json 1.0.40 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serial_test 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serial_test_derive 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"solana-sdk 0.18.0-pre1",
|
||||
|
|
|
@ -228,13 +228,10 @@ impl BankingStage {
|
|||
(new_tx_count as f32) / (proc_start.as_s())
|
||||
);
|
||||
|
||||
inc_new_high_rate_counter_info!("banking_stage-rebuffered_packets", rebuffered_packets);
|
||||
inc_new_high_rate_counter_info!("banking_stage-consumed_buffered_packets", new_tx_count);
|
||||
inc_new_high_rate_counter_debug!("banking_stage-process_transactions", new_tx_count);
|
||||
inc_new_high_rate_counter_debug!(
|
||||
"banking_stage-dropped_batches_count",
|
||||
dropped_batches_count
|
||||
);
|
||||
inc_new_counter_info!("banking_stage-rebuffered_packets", rebuffered_packets);
|
||||
inc_new_counter_info!("banking_stage-consumed_buffered_packets", new_tx_count);
|
||||
inc_new_counter_debug!("banking_stage-process_transactions", new_tx_count);
|
||||
inc_new_counter_debug!("banking_stage-dropped_batches_count", dropped_batches_count);
|
||||
|
||||
Ok(unprocessed_packets)
|
||||
}
|
||||
|
@ -813,7 +810,7 @@ impl BankingStage {
|
|||
count,
|
||||
id,
|
||||
);
|
||||
inc_new_high_rate_counter_debug!("banking_stage-transactions_received", count);
|
||||
inc_new_counter_debug!("banking_stage-transactions_received", count);
|
||||
let mut proc_start = Measure::start("process_received_packets_process");
|
||||
let mut new_tx_count = 0;
|
||||
|
||||
|
@ -885,12 +882,9 @@ impl BankingStage {
|
|||
count,
|
||||
id,
|
||||
);
|
||||
inc_new_high_rate_counter_debug!("banking_stage-process_packets", count);
|
||||
inc_new_high_rate_counter_debug!("banking_stage-process_transactions", new_tx_count);
|
||||
inc_new_high_rate_counter_debug!(
|
||||
"banking_stage-dropped_batches_count",
|
||||
dropped_batches_count
|
||||
);
|
||||
inc_new_counter_debug!("banking_stage-process_packets", count);
|
||||
inc_new_counter_debug!("banking_stage-process_transactions", new_tx_count);
|
||||
inc_new_counter_debug!("banking_stage-dropped_batches_count", dropped_batches_count);
|
||||
|
||||
*recv_start = Instant::now();
|
||||
|
||||
|
|
|
@ -1170,7 +1170,7 @@ impl ClusterInfo {
|
|||
stakes: &HashMap<Pubkey, u64>,
|
||||
) -> Vec<SharedBlob> {
|
||||
let self_id = me.read().unwrap().gossip.id;
|
||||
inc_new_high_rate_counter_debug!("cluster_info-push_message", 1);
|
||||
inc_new_counter_debug!("cluster_info-push_message", 1);
|
||||
|
||||
let updated: Vec<_> =
|
||||
me.write()
|
||||
|
|
|
@ -296,7 +296,7 @@ impl PohRecorder {
|
|||
pub fn tick(&mut self) {
|
||||
let now = Instant::now();
|
||||
let poh_entry = self.poh.lock().unwrap().tick();
|
||||
inc_new_high_rate_counter_warn!(
|
||||
inc_new_counter_warn!(
|
||||
"poh_recorder-tick_lock_contention",
|
||||
timing::duration_as_ms(&now.elapsed()) as usize
|
||||
);
|
||||
|
@ -306,7 +306,7 @@ impl PohRecorder {
|
|||
trace!("tick {}", self.tick_height);
|
||||
|
||||
if self.start_leader_at_tick.is_none() {
|
||||
inc_new_high_rate_counter_warn!(
|
||||
inc_new_counter_warn!(
|
||||
"poh_recorder-tick_overhead",
|
||||
timing::duration_as_ms(&now.elapsed()) as usize
|
||||
);
|
||||
|
@ -322,7 +322,7 @@ impl PohRecorder {
|
|||
self.tick_cache.push((entry, self.tick_height));
|
||||
let _ = self.flush_cache(true);
|
||||
}
|
||||
inc_new_high_rate_counter_warn!(
|
||||
inc_new_counter_warn!(
|
||||
"poh_recorder-tick_overhead",
|
||||
timing::duration_as_ms(&now.elapsed()) as usize
|
||||
);
|
||||
|
@ -350,7 +350,7 @@ impl PohRecorder {
|
|||
|
||||
let now = Instant::now();
|
||||
if let Some(poh_entry) = self.poh.lock().unwrap().record(mixin) {
|
||||
inc_new_high_rate_counter_warn!(
|
||||
inc_new_counter_warn!(
|
||||
"poh_recorder-record_lock_contention",
|
||||
timing::duration_as_ms(&now.elapsed()) as usize
|
||||
);
|
||||
|
|
|
@ -556,7 +556,7 @@ impl ReplayStage {
|
|||
);
|
||||
}
|
||||
}
|
||||
inc_new_high_rate_counter_info!("replay_stage-replay_transactions", tx_count);
|
||||
inc_new_counter_info!("replay_stage-replay_transactions", tx_count);
|
||||
did_complete_bank
|
||||
}
|
||||
|
||||
|
|
|
@ -39,7 +39,7 @@ fn retransmit_blobs(blobs: &[SharedBlob], retransmit: &BlobSender, id: &Pubkey)
|
|||
}
|
||||
|
||||
if !retransmit_queue.is_empty() {
|
||||
inc_new_high_rate_counter_debug!("streamer-recv_window-retransmit", retransmit_queue.len());
|
||||
inc_new_counter_debug!("streamer-recv_window-retransmit", retransmit_queue.len());
|
||||
retransmit.send(retransmit_queue)?;
|
||||
}
|
||||
Ok(())
|
||||
|
@ -113,7 +113,7 @@ where
|
|||
blobs.append(&mut blob)
|
||||
}
|
||||
let now = Instant::now();
|
||||
inc_new_high_rate_counter_debug!("streamer-recv_window-recv", blobs.len());
|
||||
inc_new_counter_debug!("streamer-recv_window-recv", blobs.len());
|
||||
|
||||
let blobs: Vec<_> = thread_pool.install(|| {
|
||||
blobs
|
||||
|
|
|
@ -19,6 +19,7 @@ sys-info = "0.5.7"
|
|||
|
||||
[dev-dependencies]
|
||||
rand = "0.6.5"
|
||||
serde_json = "1.0"
|
||||
serial_test = "0.2.0"
|
||||
serial_test_derive = "0.2.0"
|
||||
|
||||
|
|
|
@ -1,15 +1,12 @@
|
|||
use crate::{influxdb, submit};
|
||||
use crate::metrics::{submit_counter, CounterPoint};
|
||||
use log::*;
|
||||
use solana_sdk::timing;
|
||||
use std::env;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
|
||||
|
||||
const DEFAULT_LOG_RATE: usize = 1000;
|
||||
const DEFAULT_METRICS_RATE: usize = 1;
|
||||
const DEFAULT_METRICS_HIGH_RATE: usize = 10;
|
||||
|
||||
/// Use default metrics high rate
|
||||
pub const HIGH_RATE: usize = 999_999;
|
||||
// Submit a datapoint every second by default
|
||||
const DEFAULT_METRICS_RATE: u64 = 1000;
|
||||
|
||||
pub struct Counter {
|
||||
pub name: &'static str,
|
||||
|
@ -19,8 +16,7 @@ pub struct Counter {
|
|||
/// last accumulated value logged
|
||||
pub lastlog: AtomicUsize,
|
||||
pub lograte: AtomicUsize,
|
||||
pub metricsrate: AtomicUsize,
|
||||
pub point: Option<influxdb::Point>,
|
||||
pub metricsrate: AtomicU64,
|
||||
}
|
||||
|
||||
#[macro_export]
|
||||
|
@ -32,8 +28,7 @@ macro_rules! create_counter {
|
|||
times: std::sync::atomic::AtomicUsize::new(0),
|
||||
lastlog: std::sync::atomic::AtomicUsize::new(0),
|
||||
lograte: std::sync::atomic::AtomicUsize::new($lograte),
|
||||
metricsrate: std::sync::atomic::AtomicUsize::new($metricsrate),
|
||||
point: None,
|
||||
metricsrate: std::sync::atomic::AtomicU64::new($metricsrate),
|
||||
}
|
||||
};
|
||||
}
|
||||
|
@ -129,65 +124,13 @@ macro_rules! inc_new_counter_debug {
|
|||
}};
|
||||
}
|
||||
|
||||
#[macro_export]
|
||||
macro_rules! inc_new_high_rate_counter_error {
|
||||
($name:expr, $count:expr) => {{
|
||||
inc_new_counter!(
|
||||
$name,
|
||||
$count,
|
||||
log::Level::Error,
|
||||
0,
|
||||
$crate::counter::HIGH_RATE
|
||||
);
|
||||
}};
|
||||
}
|
||||
|
||||
#[macro_export]
|
||||
macro_rules! inc_new_high_rate_counter_warn {
|
||||
($name:expr, $count:expr) => {{
|
||||
inc_new_counter!(
|
||||
$name,
|
||||
$count,
|
||||
log::Level::Warn,
|
||||
0,
|
||||
$crate::counter::HIGH_RATE
|
||||
);
|
||||
}};
|
||||
}
|
||||
|
||||
#[macro_export]
|
||||
macro_rules! inc_new_high_rate_counter_info {
|
||||
($name:expr, $count:expr) => {{
|
||||
inc_new_counter!(
|
||||
$name,
|
||||
$count,
|
||||
log::Level::Info,
|
||||
0,
|
||||
$crate::counter::HIGH_RATE
|
||||
);
|
||||
}};
|
||||
}
|
||||
|
||||
#[macro_export]
|
||||
macro_rules! inc_new_high_rate_counter_debug {
|
||||
($name:expr, $count:expr) => {{
|
||||
inc_new_counter!(
|
||||
$name,
|
||||
$count,
|
||||
log::Level::Debug,
|
||||
0,
|
||||
$crate::counter::HIGH_RATE
|
||||
);
|
||||
}};
|
||||
}
|
||||
|
||||
impl Counter {
|
||||
fn default_metrics_high_rate() -> usize {
|
||||
let v = env::var("SOLANA_METRICS_HIGH_RATE")
|
||||
fn default_metrics_rate() -> u64 {
|
||||
let v = env::var("SOLANA_DEFAULT_METRICS_RATE")
|
||||
.map(|x| x.parse().unwrap_or(0))
|
||||
.unwrap_or(0);
|
||||
if v == 0 {
|
||||
DEFAULT_METRICS_HIGH_RATE
|
||||
DEFAULT_METRICS_RATE
|
||||
} else {
|
||||
v
|
||||
}
|
||||
|
@ -203,22 +146,13 @@ impl Counter {
|
|||
}
|
||||
}
|
||||
pub fn init(&mut self) {
|
||||
self.point = Some(
|
||||
influxdb::Point::new(&self.name)
|
||||
.add_field("count", influxdb::Value::Integer(0))
|
||||
.to_owned(),
|
||||
);
|
||||
self.lograte
|
||||
.compare_and_swap(0, Self::default_log_rate(), Ordering::Relaxed);
|
||||
self.metricsrate.compare_and_swap(
|
||||
HIGH_RATE,
|
||||
Self::default_metrics_high_rate(),
|
||||
Ordering::Relaxed,
|
||||
);
|
||||
self.metricsrate
|
||||
.compare_and_swap(0, DEFAULT_METRICS_RATE, Ordering::Relaxed);
|
||||
.compare_and_swap(0, Self::default_metrics_rate(), Ordering::Relaxed);
|
||||
}
|
||||
pub fn inc(&mut self, level: log::Level, events: usize) {
|
||||
let now = timing::timestamp();
|
||||
let counts = self.counts.fetch_add(events, Ordering::Relaxed);
|
||||
let times = self.times.fetch_add(1, Ordering::Relaxed);
|
||||
let lograte = self.lograte.load(Ordering::Relaxed);
|
||||
|
@ -230,36 +164,29 @@ impl Counter {
|
|||
self.name,
|
||||
counts + events,
|
||||
times,
|
||||
timing::timestamp(),
|
||||
now,
|
||||
events,
|
||||
);
|
||||
}
|
||||
|
||||
if times % metricsrate == 0 && times > 0 {
|
||||
let lastlog = self.lastlog.load(Ordering::Relaxed);
|
||||
let prev = self
|
||||
.lastlog
|
||||
.compare_and_swap(lastlog, counts, Ordering::Relaxed);
|
||||
if prev == lastlog {
|
||||
if let Some(ref mut point) = self.point {
|
||||
point
|
||||
.fields
|
||||
.entry("count".to_string())
|
||||
.and_modify(|v| {
|
||||
*v = influxdb::Value::Integer(counts as i64 - lastlog as i64)
|
||||
})
|
||||
.or_insert(influxdb::Value::Integer(0));
|
||||
}
|
||||
if let Some(ref mut point) = self.point {
|
||||
submit(point.to_owned(), level);
|
||||
}
|
||||
}
|
||||
let lastlog = self.lastlog.load(Ordering::Relaxed);
|
||||
let prev = self
|
||||
.lastlog
|
||||
.compare_and_swap(lastlog, counts, Ordering::Relaxed);
|
||||
if prev == lastlog {
|
||||
let bucket = now / metricsrate;
|
||||
let counter = CounterPoint {
|
||||
name: self.name,
|
||||
count: counts as i64 - lastlog as i64,
|
||||
timestamp: now,
|
||||
};
|
||||
submit_counter(counter, level, bucket);
|
||||
}
|
||||
}
|
||||
}
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::counter::{Counter, DEFAULT_LOG_RATE, DEFAULT_METRICS_HIGH_RATE, HIGH_RATE};
|
||||
use crate::counter::{Counter, DEFAULT_LOG_RATE, DEFAULT_METRICS_RATE};
|
||||
use log::Level;
|
||||
use log::*;
|
||||
use serial_test_derive::serial;
|
||||
|
@ -313,31 +240,31 @@ mod tests {
|
|||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_high_rate_counter() {
|
||||
fn test_metricsrate() {
|
||||
env_logger::Builder::from_env(env_logger::Env::new().default_filter_or("solana=info"))
|
||||
.try_init()
|
||||
.ok();
|
||||
let _readlock = get_env_lock().read();
|
||||
static mut COUNTER: Counter = create_counter!("test", 1000, HIGH_RATE);
|
||||
env::remove_var("SOLANA_METRICS_HIGH_RATE");
|
||||
env::remove_var("SOLANA_DEFAULT_METRICS_RATE");
|
||||
static mut COUNTER: Counter = create_counter!("test", 1000, 0);
|
||||
unsafe {
|
||||
COUNTER.init();
|
||||
assert_eq!(
|
||||
COUNTER.metricsrate.load(Ordering::Relaxed),
|
||||
DEFAULT_METRICS_HIGH_RATE
|
||||
DEFAULT_METRICS_RATE
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_high_rate_counter_env() {
|
||||
fn test_metricsrate_env() {
|
||||
env_logger::Builder::from_env(env_logger::Env::new().default_filter_or("solana=info"))
|
||||
.try_init()
|
||||
.ok();
|
||||
let _writelock = get_env_lock().write();
|
||||
static mut COUNTER: Counter = create_counter!("test", 1000, HIGH_RATE);
|
||||
env::set_var("SOLANA_METRICS_HIGH_RATE", "50");
|
||||
env::set_var("SOLANA_DEFAULT_METRICS_RATE", "50");
|
||||
static mut COUNTER: Counter = create_counter!("test", 1000, 0);
|
||||
unsafe {
|
||||
COUNTER.init();
|
||||
assert_eq!(COUNTER.metricsrate.load(Ordering::Relaxed), 50);
|
||||
|
|
|
@ -7,6 +7,7 @@ use log::*;
|
|||
use solana_sdk::hash::hash;
|
||||
use solana_sdk::timing;
|
||||
use std::collections::HashMap;
|
||||
use std::convert::Into;
|
||||
use std::sync::mpsc::{channel, Receiver, RecvTimeoutError, Sender};
|
||||
use std::sync::{Arc, Barrier, Mutex, Once};
|
||||
use std::thread;
|
||||
|
@ -128,10 +129,41 @@ lazy_static! {
|
|||
};
|
||||
}
|
||||
|
||||
type CounterMap = HashMap<(&'static str, u64), CounterPoint>;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct CounterPoint {
|
||||
pub name: &'static str,
|
||||
pub count: i64,
|
||||
pub timestamp: u64,
|
||||
}
|
||||
|
||||
impl CounterPoint {
|
||||
#[cfg(test)]
|
||||
fn new(name: &'static str) -> Self {
|
||||
CounterPoint {
|
||||
name,
|
||||
count: 0,
|
||||
timestamp: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Into<influxdb::Point> for CounterPoint {
|
||||
fn into(self) -> influxdb::Point {
|
||||
let mut point = influxdb::Point::new(self.name);
|
||||
point.add_tag("host_id", influxdb::Value::String(HOST_ID.to_string()));
|
||||
point.add_field("count", influxdb::Value::Integer(self.count));
|
||||
point.add_timestamp(self.timestamp as i64);
|
||||
point
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
enum MetricsCommand {
|
||||
Submit(influxdb::Point, log::Level),
|
||||
Flush(Arc<Barrier>),
|
||||
Submit(influxdb::Point, log::Level),
|
||||
SubmitCounter(CounterPoint, log::Level, u64),
|
||||
}
|
||||
|
||||
struct MetricsAgent {
|
||||
|
@ -270,7 +302,7 @@ impl MetricsAgent {
|
|||
) {
|
||||
trace!("run: enter");
|
||||
let mut last_write_time = Instant::now();
|
||||
let mut points_map = HashMap::<log::Level, (Instant, Vec<Point>)>::new();
|
||||
let mut points_map = HashMap::<log::Level, (Instant, CounterMap, Vec<Point>)>::new();
|
||||
let max_points = write_frequency_secs.as_secs() as usize * max_points_per_sec;
|
||||
|
||||
loop {
|
||||
|
@ -278,20 +310,38 @@ impl MetricsAgent {
|
|||
Ok(cmd) => match cmd {
|
||||
MetricsCommand::Flush(barrier) => {
|
||||
debug!("metrics_thread: flush");
|
||||
points_map.retain(|_, (_, points)| {
|
||||
writer.write(points.to_vec());
|
||||
points_map.drain().for_each(|(_, (_, counters, points))| {
|
||||
let counter_points = counters.into_iter().map(|(_, v)| v.into());
|
||||
let points: Vec<_> = points.into_iter().chain(counter_points).collect();
|
||||
writer.write(points);
|
||||
last_write_time = Instant::now();
|
||||
false
|
||||
});
|
||||
barrier.wait();
|
||||
}
|
||||
MetricsCommand::Submit(point, level) => {
|
||||
debug!("run: submit {:?}", point);
|
||||
let (_, points) = points_map
|
||||
.entry(level)
|
||||
.or_insert((last_write_time, Vec::new()));
|
||||
let (_, _, points) = points_map.entry(level).or_insert((
|
||||
last_write_time,
|
||||
HashMap::new(),
|
||||
Vec::new(),
|
||||
));
|
||||
points.push(point);
|
||||
}
|
||||
MetricsCommand::SubmitCounter(counter, level, bucket) => {
|
||||
debug!("run: submit counter {:?}", counter);
|
||||
let (_, counters, _) = points_map.entry(level).or_insert((
|
||||
last_write_time,
|
||||
HashMap::new(),
|
||||
Vec::new(),
|
||||
));
|
||||
|
||||
let key = (counter.name, bucket);
|
||||
if let Some(value) = counters.get_mut(&key) {
|
||||
value.count += counter.count;
|
||||
} else {
|
||||
counters.insert(key, counter);
|
||||
}
|
||||
}
|
||||
},
|
||||
Err(RecvTimeoutError::Timeout) => {
|
||||
trace!("run: receive timeout");
|
||||
|
@ -315,7 +365,9 @@ impl MetricsAgent {
|
|||
]
|
||||
.iter()
|
||||
.for_each(|x| {
|
||||
if let Some((last_time, points)) = points_map.remove(x) {
|
||||
if let Some((last_time, counters, points)) = points_map.remove(x) {
|
||||
let counter_points = counters.into_iter().map(|(_, v)| v.into());
|
||||
let points: Vec<_> = points.into_iter().chain(counter_points).collect();
|
||||
let num_written = Self::write(
|
||||
&points,
|
||||
last_time,
|
||||
|
@ -338,7 +390,7 @@ impl MetricsAgent {
|
|||
|
||||
pub fn submit(&self, mut point: influxdb::Point, level: log::Level) {
|
||||
if point.timestamp.is_none() {
|
||||
point.timestamp = Some(timing::timestamp() as i64);
|
||||
point.add_timestamp(timing::timestamp() as i64);
|
||||
}
|
||||
debug!("Submitting point: {:?}", point);
|
||||
self.sender
|
||||
|
@ -346,6 +398,13 @@ impl MetricsAgent {
|
|||
.unwrap();
|
||||
}
|
||||
|
||||
pub fn submit_counter(&self, point: CounterPoint, level: log::Level, bucket: u64) {
|
||||
debug!("Submitting counter point: {:?}", point);
|
||||
self.sender
|
||||
.send(MetricsCommand::SubmitCounter(point, level, bucket))
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
pub fn flush(&self) {
|
||||
debug!("Flush");
|
||||
let barrier = Arc::new(Barrier::new(2));
|
||||
|
@ -384,6 +443,14 @@ pub fn submit(mut point: influxdb::Point, level: log::Level) {
|
|||
agent.submit(point, level);
|
||||
}
|
||||
|
||||
/// Submits a new counter or updates an existing counter from any thread. Note that points are
|
||||
/// internally queued and transmitted periodically in batches.
|
||||
pub fn submit_counter(point: CounterPoint, level: log::Level, bucket: u64) {
|
||||
let agent_mutex = get_singleton_agent();
|
||||
let agent = agent_mutex.lock().unwrap();
|
||||
agent.submit_counter(point, level, bucket);
|
||||
}
|
||||
|
||||
fn get_env_settings() -> Result<(String, String, String, String), env::VarError> {
|
||||
let host =
|
||||
env::var("INFLUX_HOST").unwrap_or_else(|_| "https://metrics.solana.com:8086".to_string());
|
||||
|
@ -461,20 +528,20 @@ pub fn set_panic_hook(program: &'static str) {
|
|||
#[cfg(test)]
|
||||
mod test {
|
||||
use super::*;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use serde_json;
|
||||
|
||||
struct MockMetricsWriter {
|
||||
points_written: AtomicUsize,
|
||||
points_written: Arc<Mutex<Vec<influxdb::Point>>>,
|
||||
}
|
||||
impl MockMetricsWriter {
|
||||
fn new() -> Self {
|
||||
MockMetricsWriter {
|
||||
points_written: AtomicUsize::new(0),
|
||||
points_written: Arc::new(Mutex::new(Vec::new())),
|
||||
}
|
||||
}
|
||||
|
||||
fn points_written(&self) -> usize {
|
||||
return self.points_written.load(Ordering::Relaxed);
|
||||
self.points_written.lock().unwrap().len()
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -482,13 +549,16 @@ mod test {
|
|||
fn write(&self, points: Vec<influxdb::Point>) {
|
||||
assert!(!points.is_empty());
|
||||
|
||||
let new_points = points.len();
|
||||
self.points_written
|
||||
.fetch_add(points.len(), Ordering::Relaxed);
|
||||
.lock()
|
||||
.unwrap()
|
||||
.extend(points.into_iter());
|
||||
|
||||
info!(
|
||||
"Writing {} points ({} total)",
|
||||
points.len(),
|
||||
self.points_written.load(Ordering::Relaxed)
|
||||
new_points,
|
||||
self.points_written()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
@ -509,6 +579,63 @@ mod test {
|
|||
assert_eq!(writer.points_written(), 42);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_submit_counter() {
|
||||
let writer = Arc::new(MockMetricsWriter::new());
|
||||
let agent = MetricsAgent::new(writer.clone(), Duration::from_secs(10), 1000);
|
||||
|
||||
for i in 0..10 {
|
||||
agent.submit_counter(CounterPoint::new("counter - 1"), Level::Info, i);
|
||||
agent.submit_counter(CounterPoint::new("counter - 2"), Level::Info, i);
|
||||
}
|
||||
|
||||
agent.flush();
|
||||
assert_eq!(writer.points_written(), 20);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_submit_counter_increment() {
|
||||
let writer = Arc::new(MockMetricsWriter::new());
|
||||
let agent = MetricsAgent::new(writer.clone(), Duration::from_secs(10), 1000);
|
||||
|
||||
for _ in 0..10 {
|
||||
agent.submit_counter(
|
||||
CounterPoint {
|
||||
name: "counter",
|
||||
count: 10,
|
||||
timestamp: 0,
|
||||
},
|
||||
Level::Info,
|
||||
0, // use the same bucket
|
||||
);
|
||||
}
|
||||
|
||||
agent.flush();
|
||||
assert_eq!(writer.points_written(), 1);
|
||||
|
||||
let submitted_point = writer.points_written.lock().unwrap()[0].clone();
|
||||
let submitted_count = submitted_point.fields.get("count").unwrap();
|
||||
let expected_count = &influxdb::Value::Integer(100);
|
||||
assert_eq!(
|
||||
serde_json::to_string(submitted_count).unwrap(),
|
||||
serde_json::to_string(expected_count).unwrap()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_submit_bucketed_counter() {
|
||||
let writer = Arc::new(MockMetricsWriter::new());
|
||||
let agent = MetricsAgent::new(writer.clone(), Duration::from_secs(10), 1000);
|
||||
|
||||
for i in 0..50 {
|
||||
agent.submit_counter(CounterPoint::new("counter - 1"), Level::Info, i / 10);
|
||||
agent.submit_counter(CounterPoint::new("counter - 2"), Level::Info, i / 10);
|
||||
}
|
||||
|
||||
agent.flush();
|
||||
assert_eq!(writer.points_written(), 10);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_submit_with_delay() {
|
||||
let writer = Arc::new(MockMetricsWriter::new());
|
||||
|
|
|
@ -60,10 +60,6 @@ solana_replicator=$(solana_program replicator)
|
|||
|
||||
export RUST_BACKTRACE=1
|
||||
|
||||
if [[ -z $SOLANA_METRICS_HIGH_RATE ]]; then
|
||||
export SOLANA_METRICS_HIGH_RATE=1000
|
||||
fi
|
||||
|
||||
# shellcheck source=scripts/configure-metrics.sh
|
||||
source "$SOLANA_ROOT"/scripts/configure-metrics.sh
|
||||
|
||||
|
|
|
@ -4,7 +4,7 @@ use crate::exchange_instruction::*;
|
|||
use crate::exchange_state::*;
|
||||
use crate::faucet;
|
||||
use log::*;
|
||||
use solana_metrics::inc_new_high_rate_counter_info;
|
||||
use solana_metrics::inc_new_counter_info;
|
||||
use solana_sdk::account::KeyedAccount;
|
||||
use solana_sdk::instruction::InstructionError;
|
||||
use solana_sdk::pubkey::Pubkey;
|
||||
|
@ -296,7 +296,7 @@ impl ExchangeProcessor {
|
|||
// Trade holds the tokens in escrow
|
||||
account.tokens[from_token] -= info.tokens;
|
||||
|
||||
inc_new_high_rate_counter_info!("exchange_processor-trades", 1);
|
||||
inc_new_counter_info!("exchange_processor-trades", 1);
|
||||
|
||||
Self::serialize(
|
||||
&ExchangeState::Trade(OrderInfo {
|
||||
|
@ -390,7 +390,7 @@ impl ExchangeProcessor {
|
|||
Err(e)?
|
||||
}
|
||||
|
||||
inc_new_high_rate_counter_info!("exchange_processor-swaps", 1);
|
||||
inc_new_counter_info!("exchange_processor-swaps", 1);
|
||||
|
||||
if to_order.tokens == 0 {
|
||||
// Turn into token account
|
||||
|
|
|
@ -25,7 +25,7 @@ use log::*;
|
|||
use serde::{Deserialize, Serialize};
|
||||
use solana_measure::measure::Measure;
|
||||
use solana_metrics::{
|
||||
datapoint_info, inc_new_counter_debug, inc_new_counter_error, inc_new_high_rate_counter_info,
|
||||
datapoint_info, inc_new_counter_debug, inc_new_counter_error, inc_new_counter_info,
|
||||
};
|
||||
use solana_sdk::account::Account;
|
||||
use solana_sdk::fee_calculator::FeeCalculator;
|
||||
|
@ -859,43 +859,43 @@ impl Bank {
|
|||
|
||||
fn update_error_counters(error_counters: &ErrorCounters) {
|
||||
if 0 != error_counters.blockhash_not_found {
|
||||
inc_new_high_rate_counter_error!(
|
||||
inc_new_counter_error!(
|
||||
"bank-process_transactions-error-blockhash_not_found",
|
||||
error_counters.blockhash_not_found
|
||||
);
|
||||
}
|
||||
if 0 != error_counters.invalid_account_index {
|
||||
inc_new_high_rate_counter_error!(
|
||||
inc_new_counter_error!(
|
||||
"bank-process_transactions-error-invalid_account_index",
|
||||
error_counters.invalid_account_index
|
||||
);
|
||||
}
|
||||
if 0 != error_counters.reserve_blockhash {
|
||||
inc_new_high_rate_counter_error!(
|
||||
inc_new_counter_error!(
|
||||
"bank-process_transactions-error-reserve_blockhash",
|
||||
error_counters.reserve_blockhash
|
||||
);
|
||||
}
|
||||
if 0 != error_counters.duplicate_signature {
|
||||
inc_new_high_rate_counter_error!(
|
||||
inc_new_counter_error!(
|
||||
"bank-process_transactions-error-duplicate_signature",
|
||||
error_counters.duplicate_signature
|
||||
);
|
||||
}
|
||||
if 0 != error_counters.invalid_account_for_fee {
|
||||
inc_new_high_rate_counter_error!(
|
||||
inc_new_counter_error!(
|
||||
"bank-process_transactions-error-invalid_account_for_fee",
|
||||
error_counters.invalid_account_for_fee
|
||||
);
|
||||
}
|
||||
if 0 != error_counters.insufficient_funds {
|
||||
inc_new_high_rate_counter_error!(
|
||||
inc_new_counter_error!(
|
||||
"bank-process_transactions-error-insufficient_funds",
|
||||
error_counters.insufficient_funds
|
||||
);
|
||||
}
|
||||
if 0 != error_counters.account_loaded_twice {
|
||||
inc_new_high_rate_counter_error!(
|
||||
inc_new_counter_error!(
|
||||
"bank-process_transactions-account_loaded_twice",
|
||||
error_counters.account_loaded_twice
|
||||
);
|
||||
|
@ -916,7 +916,7 @@ impl Bank {
|
|||
usize,
|
||||
) {
|
||||
debug!("processing transactions: {}", txs.len());
|
||||
inc_new_high_rate_counter_info!("bank-process_transactions", txs.len());
|
||||
inc_new_counter_info!("bank-process_transactions", txs.len());
|
||||
let mut error_counters = ErrorCounters::default();
|
||||
let mut load_time = Measure::start("accounts_load");
|
||||
|
||||
|
@ -977,7 +977,7 @@ impl Bank {
|
|||
}
|
||||
if err_count > 0 {
|
||||
debug!("{} errors of {} txs", err_count, err_count + tx_count);
|
||||
inc_new_high_rate_counter_error!(
|
||||
inc_new_counter_error!(
|
||||
"bank-process_transactions-account_not_found",
|
||||
error_counters.account_not_found
|
||||
);
|
||||
|
@ -1053,8 +1053,8 @@ impl Bank {
|
|||
self.increment_transaction_count(tx_count);
|
||||
self.increment_signature_count(signature_count);
|
||||
|
||||
inc_new_high_rate_counter_info!("bank-process_transactions-txs", tx_count);
|
||||
inc_new_high_rate_counter_info!("bank-process_transactions-sigs", signature_count);
|
||||
inc_new_counter_info!("bank-process_transactions-txs", tx_count);
|
||||
inc_new_counter_info!("bank-process_transactions-sigs", signature_count);
|
||||
|
||||
if executed.iter().any(|res| Self::can_commit(res)) {
|
||||
self.is_delta.store(true, Ordering::Relaxed);
|
||||
|
|
Loading…
Reference in New Issue