diff --git a/lib/include/srsran/system/sys_metrics.h b/lib/include/srsran/system/sys_metrics.h index 150e9f36f..890e9c9d7 100644 --- a/lib/include/srsran/system/sys_metrics.h +++ b/lib/include/srsran/system/sys_metrics.h @@ -21,11 +21,12 @@ namespace srsran { struct sys_metrics_t { uint32_t process_realmem_kB = 0; uint32_t process_virtualmem_kB = 0; - float process_realmem = -1.f; - float process_virtualmem = -1.f; + float process_realmem = 0.f; uint32_t thread_count = 0; - float process_cpu_usage = -1.f; - float system_mem = -1.f; + float process_cpu_usage = 0.f; + float system_mem = 0.f; + uint32_t cpu_count = 0; + float cpu_load[128]; }; } // namespace srsran diff --git a/lib/include/srsran/system/sys_metrics_processor.h b/lib/include/srsran/system/sys_metrics_processor.h index 4c1d6309b..7da5c9af5 100644 --- a/lib/include/srsran/system/sys_metrics_processor.h +++ b/lib/include/srsran/system/sys_metrics_processor.h @@ -42,6 +42,18 @@ class sys_metrics_processor std::string comm; }; + /// Helper class to read the cpu metrics. + struct cpu_metrics_t { + std::string name = ""; + int32_t user = 0; + int32_t nice = 0; + int32_t system = 0; + int32_t idle = 0; + int32_t iowait = 0; + int32_t irq = 0; + int32_t softirq = 0; + }; + public: /// Measures and returns the system metrics. sys_metrics_t get_metrics(); @@ -56,9 +68,17 @@ private: /// NOTE: on error, metrics memory parameters are set to 0. void calculate_mem_usage(sys_metrics_t& metrics) const; + /// Calculate the cpu metrics and stores them in the given metrics. delta_time_in_seconds is the number of seconds + /// elapsed since the last cpu metrics measurement. + void calculate_cpu_metrics(sys_metrics_t& metrics, float delta_time_in_seconds); + + /// Returns the cpu metrics from the given line. + cpu_metrics_t read_cpu_idle_from_line(const std::string& line) const; + private: - proc_stats_info last_query = {}; - std::chrono::time_point last_query_time = std::chrono::steady_clock::now(); + proc_stats_info last_query = {}; + cpu_metrics_t last_cpu_thread[128] = {}; + std::chrono::time_point last_query_time = std::chrono::steady_clock::now(); }; } // namespace srsran diff --git a/lib/src/system/sys_metrics_processor.cc b/lib/src/system/sys_metrics_processor.cc index 89e52835d..4dbf3f39f 100644 --- a/lib/src/system/sys_metrics_processor.cc +++ b/lib/src/system/sys_metrics_processor.cc @@ -18,6 +18,9 @@ using namespace srsran; +static const uint32_t cpu_count = ::sysconf(_SC_NPROCESSORS_CONF); +static const float ticks_per_second = ::sysconf(_SC_CLK_TCK); + sys_metrics_processor::proc_stats_info::proc_stats_info() { std::string line; @@ -55,6 +58,9 @@ sys_metrics_t sys_metrics_processor::get_metrics() // Get the memory metrics. calculate_mem_usage(metrics); + // Calculate cpu metrics. + calculate_cpu_metrics(metrics, measure_interval_ms / 1000.f); + // Get the stats from the proc. proc_stats_info current_query; metrics.thread_count = current_query.num_threads; @@ -80,23 +86,54 @@ float sys_metrics_processor::calculate_cpu_usage(const proc_stats_info& current_ return 0.f; } - static const uint32_t cpu_count = ::sysconf(_SC_NPROCESSORS_CONF); - static const float ticks_per_second = ::sysconf(_SC_CLK_TCK); - return ((current_query.stime + current_query.utime) - (last_query.stime + last_query.utime)) * 100.f / (cpu_count * ticks_per_second * delta_time_in_seconds); } -/// Extracts and returns the memory size from the given line. -static uint32_t read_memory_value_from_line(const std::string& line) +sys_metrics_processor::cpu_metrics_t sys_metrics_processor::read_cpu_idle_from_line(const std::string& line) const { std::istringstream reader(line); - std::string label, unit; - uint32_t value; + cpu_metrics_t m; - reader >> label >> value >> unit; + reader >> m.name >> m.user >> m.nice >> m.system >> m.idle >> m.iowait >> m.irq >> m.softirq; - return value; + return m; +} + +void sys_metrics_processor::calculate_cpu_metrics(sys_metrics_t& metrics, float delta_time_in_seconds) +{ + metrics.cpu_count = cpu_count; + + std::ifstream file("/proc/stat"); + std::string line; + + if (!file) { + return; + } + + int count = -1; + while (std::getline(file, line)) { + // First line is the CPU field that contains all the cores and thread. For now, we skip this one. + if (count < 0) { + ++count; + continue; + } + + // Parse all the cpus. + if (line.find("cpu") != std::string::npos) { + auto tmp = read_cpu_idle_from_line(line); + auto index = count++; + if (tmp.idle < last_cpu_thread[index].idle) { + metrics.cpu_load[index] = 0.f; + continue; + } + + metrics.cpu_load[index] = std::max( + (1.f - (tmp.idle - last_cpu_thread[index].idle) / (ticks_per_second * delta_time_in_seconds)) * 100.f, 0.f); + + last_cpu_thread[index] = std::move(tmp); + } + } } /// Sets the memory parameters of the given metrics to zero. @@ -104,11 +141,22 @@ static void set_mem_to_zero(sys_metrics_t& metrics) { metrics.process_realmem_kB = 0; metrics.process_virtualmem_kB = 0; - metrics.process_virtualmem = 0; metrics.process_realmem = 0; metrics.system_mem = 0; } +/// Extracts and returns the memory size from the given line. +static int32_t read_memory_value_from_line(const std::string& line) +{ + std::istringstream reader(line); + std::string label, unit; + int32_t value; + + reader >> label >> value >> unit; + + return value; +} + static void calculate_percentage_memory(sys_metrics_t& metrics) { std::ifstream file("/proc/meminfo"); @@ -119,19 +167,42 @@ static void calculate_percentage_memory(sys_metrics_t& metrics) return; } - // Total system's memory is in the first line. - std::getline(file, line); - uint32_t total_mem_kB = read_memory_value_from_line(line); + struct meminfo_t { + uint32_t total_kB = 0; + uint32_t free_kB = 0; + uint32_t buffers_kB = 0; + uint32_t cached_kB = 0; + uint32_t slab_kB = 0; + }; - // System's available memory is in the third line. - std::getline(file, line); - std::getline(file, line); - uint32_t available_mem_kB = read_memory_value_from_line(line); + // Retrieve the data + meminfo_t m_info; + while (std::getline(file, line)) { + // Looks for Virtual memory. + if (line.find("MemTotal:") != std::string::npos) { + m_info.total_kB = std::max(read_memory_value_from_line(line), 0); + } + if (line.find("MemFree:") != std::string::npos) { + m_info.free_kB = std::max(read_memory_value_from_line(line), 0); + } + if (line.find("Buffers:") != std::string::npos) { + m_info.buffers_kB = std::max(read_memory_value_from_line(line), 0); + } + if (line.find("Cached:") != std::string::npos) { + m_info.cached_kB = std::max(read_memory_value_from_line(line), 0); + } + if (line.find("Slab:") != std::string::npos) { + m_info.slab_kB = std::max(read_memory_value_from_line(line), 0); + } + } // Calculate the metrics. - metrics.process_realmem = 100.f * (float(metrics.process_realmem_kB) / total_mem_kB); - metrics.process_virtualmem = 100.f * (float(metrics.process_virtualmem_kB) / total_mem_kB); - metrics.system_mem = (1.f - float(available_mem_kB) / float(total_mem_kB)) * 100.f; + metrics.process_realmem = (metrics.process_realmem_kB <= m_info.total_kB) + ? 100.f * (float(metrics.process_realmem_kB) / m_info.total_kB) + : 0; + metrics.system_mem = + (1.f - float(m_info.buffers_kB + m_info.cached_kB + m_info.free_kB + m_info.slab_kB) / float(m_info.total_kB)) * + 100.f; } void sys_metrics_processor::calculate_mem_usage(sys_metrics_t& metrics) const @@ -147,12 +218,14 @@ void sys_metrics_processor::calculate_mem_usage(sys_metrics_t& metrics) const while (std::getline(file, line)) { // Looks for Virtual memory. if (line.find("VmSize:") != std::string::npos) { - metrics.process_virtualmem_kB = read_memory_value_from_line(line); + // NOTE: std::max will clamp negative values to 0. + metrics.process_virtualmem_kB = std::max(read_memory_value_from_line(line), 0); continue; } // Looks for physical memory. if (line.find("VmRSS:") != std::string::npos) { - metrics.process_realmem_kB = read_memory_value_from_line(line); + // NOTE: std::max will clamp negative values to 0. + metrics.process_realmem_kB = std::max(read_memory_value_from_line(line), 0); continue; } } diff --git a/srsenb/src/metrics_csv.cc b/srsenb/src/metrics_csv.cc index f2a0b7ca1..34ffd0ac7 100644 --- a/srsenb/src/metrics_csv.cc +++ b/srsenb/src/metrics_csv.cc @@ -56,7 +56,15 @@ void metrics_csv::set_metrics(const enb_metrics_t& metrics, const uint32_t perio if (file.is_open() && enb != NULL) { if (n_reports == 0) { file << "time;nof_ue;dl_brate;ul_brate;" - "proc_rmem;proc_rmem_kB;proc_vmem;proc_vmem_kB;sys_mem;proc_cpu;thread_count\n"; + "proc_rmem;proc_rmem_kB;proc_vmem_kB;sys_mem;system_load;thread_count"; + + // Add the cores. + for (uint32_t i = 0, e = ::sysconf(_SC_NPROCESSORS_CONF); i != e; ++i) { + file << ";cpu_" << std::to_string(i); + } + + // Add the new line. + file << "\n"; } // Time @@ -90,11 +98,15 @@ void metrics_csv::set_metrics(const enb_metrics_t& metrics, const uint32_t perio const srsran::sys_metrics_t& m = metrics.sys; file << float_to_string(m.process_realmem, 2); file << std::to_string(m.process_realmem_kB) << ";"; - file << float_to_string(m.process_virtualmem, 2); file << std::to_string(m.process_virtualmem_kB) << ";"; file << float_to_string(m.system_mem, 2); file << float_to_string(m.process_cpu_usage, 2); - file << std::to_string(m.thread_count); + file << std::to_string(m.thread_count) << ";"; + + // Write the cpu metrics. + for (uint32_t i = 0, e = m.cpu_count, last_cpu_index = e - 1; i != e; ++i) { + file << float_to_string(m.cpu_load[i], 2, (i != last_cpu_index)); + } file << "\n"; diff --git a/srsue/src/metrics_csv.cc b/srsue/src/metrics_csv.cc index e0b19caf4..666fd4981 100644 --- a/srsue/src/metrics_csv.cc +++ b/srsue/src/metrics_csv.cc @@ -78,7 +78,15 @@ void metrics_csv::set_metrics(const ue_metrics_t& metrics, const uint32_t period "bler;" "rf_o;rf_" "u;rf_l;is_attached;" - "proc_rmem;proc_rmem_kB;proc_vmem;proc_vmem_kB;sys_mem;proc_cpu;thread_count\n"; + "proc_rmem;proc_rmem_kB;proc_vmem_kB;sys_mem;sys_load;thread_count"; + + // Add the cores. + for (uint32_t i = 0, e = ::sysconf(_SC_NPROCESSORS_CONF); i != e; ++i) { + file << ";cpu_" << std::to_string(i); + } + + // Add the new line. + file << "\n"; } for (uint32_t r = 0; r < metrics.phy.nof_active_cc; r++) { @@ -159,11 +167,15 @@ void metrics_csv::set_metrics(const ue_metrics_t& metrics, const uint32_t period const srsran::sys_metrics_t& m = metrics.sys; file << float_to_string(m.process_realmem, 2); file << std::to_string(m.process_realmem_kB) << ";"; - file << float_to_string(m.process_virtualmem, 2); file << std::to_string(m.process_virtualmem_kB) << ";"; file << float_to_string(m.system_mem, 2); file << float_to_string(m.process_cpu_usage, 2); - file << std::to_string(m.thread_count); + file << std::to_string(m.thread_count) << ";"; + + // Write the cpu metrics. + for (uint32_t i = 0, e = m.cpu_count, last_cpu_index = e - 1; i != e; ++i) { + file << float_to_string(m.cpu_load[i], 2, (i != last_cpu_index)); + } file << "\n"; }