From a2c8e3952f7cf9c168ff784552ff29dff657e201 Mon Sep 17 00:00:00 2001 From: Pankaj Garg Date: Wed, 17 Apr 2019 15:37:01 -0700 Subject: [PATCH] Fixes to TPS calculation and reporting (#3836) Fixes to TPS calculations and reporting --- ci/testnet-automation.sh | 4 ++-- core/src/banking_stage.rs | 23 +++++++++++++++++++---- metrics/testnet-monitor.json | 16 ++++++++-------- 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/ci/testnet-automation.sh b/ci/testnet-automation.sh index e913aef9e..df432fd41 100755 --- a/ci/testnet-automation.sh +++ b/ci/testnet-automation.sh @@ -52,14 +52,14 @@ launchTestnet() { declare q_mean_tps=' SELECT round(mean("sum_count")) AS "mean_tps" FROM ( SELECT sum("count") AS "sum_count" - FROM "testnet-automation"."autogen"."counter-bank-process_transactions-txs" + FROM "testnet-automation"."autogen"."counter-banking_stage-process_transactions" WHERE time > now() - 300s GROUP BY time(1s) )' declare q_max_tps=' SELECT round(max("sum_count")) AS "max_tps" FROM ( SELECT sum("count") AS "sum_count" - FROM "testnet-automation"."autogen"."counter-bank-process_transactions-txs" + FROM "testnet-automation"."autogen"."counter-banking_stage-process_transactions" WHERE time > now() - 300s GROUP BY time(1s) )' diff --git a/core/src/banking_stage.rs b/core/src/banking_stage.rs index 44ecea790..51cb1c750 100644 --- a/core/src/banking_stage.rs +++ b/core/src/banking_stage.rs @@ -115,6 +115,7 @@ impl BankingStage { .iter() .flat_map(|(p, start_index)| &p.packets[**start_index..]) .collect(); + inc_new_counter_info!("banking_stage-forwarded_packets", packets.len()); let blobs = packet::packets_to_blobs(&packets); for blob in blobs { @@ -132,12 +133,14 @@ impl BankingStage { let mut bank_shutdown = false; for (msgs, offset, vers) in buffered_packets { if bank_shutdown { + inc_new_counter_info!("banking_stage-rebuffered_packets", vers.len() - *offset); unprocessed_packets.push((msgs.to_owned(), *offset, vers.to_owned())); continue; } let bank = poh_recorder.lock().unwrap().bank(); if bank.is_none() { + inc_new_counter_info!("banking_stage-rebuffered_packets", vers.len() - *offset); unprocessed_packets.push((msgs.to_owned(), *offset, vers.to_owned())); continue; } @@ -145,7 +148,12 @@ impl BankingStage { let (processed, verified_txs, verified_indexes) = Self::process_received_packets(&bank, &poh_recorder, &msgs, &vers, *offset)?; - + inc_new_counter_info!("banking_stage-consumed_buffered_packets", processed); + inc_new_counter_info!("banking_stage-process_transactions", processed); + inc_new_counter_info!( + "banking_stage-rebuffered_packets", + verified_txs.len() - processed + ); if processed < verified_txs.len() { bank_shutdown = true; // Collect any unprocessed transactions in this batch for forwarding @@ -221,14 +229,14 @@ impl BankingStage { // Buffer the packets if I am the next leader // or, if it was getting sent to me + // or, the next leader is unknown let leader_id = match poh_recorder.lock().unwrap().bank() { Some(bank) => { leader_schedule_utils::slot_leader_at(bank.slot() + 1, &bank).unwrap_or_default() } None => rcluster_info .leader_data() - .map(|x| x.id) - .unwrap_or_default(), + .map_or(rcluster_info.id(), |x| x.id), }; leader_id == rcluster_info.id() @@ -269,6 +277,13 @@ impl BankingStage { Err(Error::RecvTimeoutError(RecvTimeoutError::Timeout)) => (), Ok(unprocessed_packets) => { if Self::should_buffer_packets(poh_recorder, cluster_info) { + let num = unprocessed_packets + .iter() + .map(|(x, start, _)| { + x.read().unwrap().packets.len().saturating_sub(*start) + }) + .sum(); + inc_new_counter_info!("banking_stage-buffered_packets", num); buffered_packets.extend_from_slice(&unprocessed_packets); continue; } @@ -426,6 +441,7 @@ impl BankingStage { poh, ); trace!("process_transcations: {:?}", result); + chunk_start = chunk_end; if let Err(Error::PohRecorderError(PohRecorderError::MaxHeightReached)) = result { info!( "process transactions: max height reached slot: {} height: {}", @@ -435,7 +451,6 @@ impl BankingStage { break; } result?; - chunk_start = chunk_end; } Ok(chunk_start) } diff --git a/metrics/testnet-monitor.json b/metrics/testnet-monitor.json index 330e71d1d..e7dd282a5 100644 --- a/metrics/testnet-monitor.json +++ b/metrics/testnet-monitor.json @@ -16,7 +16,7 @@ "gnetId": null, "graphTooltip": 0, "id": 399, - "iteration": 1554844431880, + "iteration": 1554844431890, "links": [ { "asDropdown": true, @@ -124,7 +124,7 @@ "hide": false, "orderByTime": "ASC", "policy": "default", - "query": "SELECT ROUND(MEAN(\"sum\")) FROM ( SELECT sum(\"count\") FROM \"$testnet\".\"autogen\".\"counter-bank-process_transactions-txs\" WHERE $timeFilter GROUP BY time(1s) )\n\n", + "query": "SELECT ROUND(MEAN(\"sum\")) FROM ( SELECT sum(\"count\") FROM \"$testnet\".\"autogen\".\"counter-banking_stage-process_transactions\" WHERE $timeFilter GROUP BY time(1s) )\n\n", "rawQuery": true, "refId": "A", "resultFormat": "time_series", @@ -236,7 +236,7 @@ "hide": false, "orderByTime": "ASC", "policy": "default", - "query": "SELECT MAX(\"sum\") FROM ( SELECT sum(\"count\") FROM \"$testnet\".\"autogen\".\"counter-bank-process_transactions-txs\" WHERE $timeFilter GROUP BY time(1s) )\n\n", + "query": "SELECT MAX(\"sum\") FROM ( SELECT sum(\"count\") FROM \"$testnet\".\"autogen\".\"counter-banking_stage-process_transactions\" WHERE $timeFilter GROUP BY time(1s) )\n\n", "rawQuery": true, "refId": "A", "resultFormat": "time_series", @@ -348,7 +348,7 @@ "hide": false, "orderByTime": "ASC", "policy": "default", - "query": "SELECT sum(\"count\") AS \"transactions\" FROM \"$testnet\".\"autogen\".\"counter-bank-process_transactions-txs\" WHERE $timeFilter \n\n", + "query": "SELECT sum(\"count\") AS \"transactions\" FROM \"$testnet\".\"autogen\".\"counter-banking_stage-process_transactions\" WHERE $timeFilter \n\n", "rawQuery": true, "refId": "A", "resultFormat": "time_series", @@ -1479,7 +1479,7 @@ ], "orderByTime": "ASC", "policy": "default", - "query": "SELECT sum(\"count\") AS \"transactions\" FROM \"$testnet\".\"autogen\".\"counter-bank-process_transactions-txs\" WHERE $timeFilter GROUP BY time(1s) FILL(0)\n", + "query": "SELECT sum(\"count\") AS \"transactions\" FROM \"$testnet\".\"autogen\".\"counter-banking_stage-process_transactions\" WHERE $timeFilter GROUP BY time(1s) FILL(0)\n", "rawQuery": true, "refId": "B", "resultFormat": "time_series", @@ -4238,7 +4238,7 @@ ], "orderByTime": "ASC", "policy": "default", - "query": "SELECT sum(\"count\") AS \"count\" FROM \"$testnet\".\"autogen\".\"fetch-stage\" WHERE $timeFilter GROUP BY time($__interval) FILL(0)\n", + "query": "SELECT sum(\"count\") AS \"count\" FROM \"$testnet\".\"autogen\".\"counter-packets-recv_count\" WHERE $timeFilter GROUP BY time($__interval) FILL(0)\n", "rawQuery": true, "refId": "G", "resultFormat": "time_series", @@ -6428,5 +6428,5 @@ "timezone": "", "title": "Testnet Monitor (edge)", "uid": "testnet-edge", - "version": 118 -} + "version": 119 +} \ No newline at end of file