read hash mismatch errors from influx and error if > 0 (#14240)

system-test/automation_utils.sh

@@ -140,6 +140,18 @@ function collect_performance_statistics {
     --data-urlencode "db=${TESTNET_TAG}" \
     --data-urlencode "q=$q_mean_tps;$q_max_tps;$q_mean_confirmation;$q_max_confirmation;$q_99th_confirmation;$q_max_tower_distance_observed;$q_last_tower_distance_observed" |
     python "${REPO_ROOT}"/system-test/testnet-automation-json-parser.py >>"$RESULT_FILE"
+
+  declare q_dropped_vote_hash_count='
+    SELECT sum("count") as "sum_dropped_vote_hash"
+      FROM "'$TESTNET_TAG'"."autogen"."dropped-vote-hash"
+      WHERE time > now() - '"$TEST_DURATION_SECONDS"'s'
+
+  # store in variable to be returned
+  dropped_vote_hash_count=$( \
+  curl -G "${INFLUX_HOST}/query?u=ro&p=topsecret" \
+    --data-urlencode "db=${TESTNET_TAG}" \
+    --data-urlencode "q=$q_dropped_vote_hash_count" |
+    python "${REPO_ROOT}"/system-test/testnet-automation-json-parser-missing.py)
 }
 
 function upload_results_to_slack() {

system-test/testnet-automation-json-parser-missing.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+import sys, json
+
+data=json.load(sys.stdin)
+
+# this code is designed for influx queries where 'no data' means 0
+if 'results' in data:
+   for result in data['results']:
+      val = "0"
+      if 'series' in result:
+         val = str(result['series'][0]['values'][0][1])
+      print(val)
+else:
+   print("No results returned from CURL request")

system-test/testnet-automation.sh

@@ -197,8 +197,15 @@ function launch_testnet() {
   execution_step "Average slot rate: $SLOTS_PER_SECOND slots/second over $((SLOT_COUNT_END_SECONDS - SLOT_COUNT_START_SECONDS)) seconds"
 
   if [[ "$SKIP_PERF_RESULTS" = "false" ]]; then
+    declare -g dropped_vote_hash_count
+
     collect_performance_statistics
     echo "slots_per_second: $SLOTS_PER_SECOND" >>"$RESULT_FILE"
+
+    if [[ $dropped_vote_hash_count -gt 0 ]]; then
+      execution_step "Checking for dropped vote hash count"
+      exit 1
+    fi
   fi
 
   RESULT_DETAILS=$(<"$RESULT_FILE")