From 0641244378c6dd60e61034aec5fc49c5a656c0fa Mon Sep 17 00:00:00 2001
From: Dan Albert <dan@solana.com>
Date: Mon, 16 Mar 2020 12:09:18 -0700
Subject: [PATCH] Add genesis token counter test to system test (#8824)

automerge
---
 .../genesis-test/cluster_token_count.sh       | 144 ++++++++++++++++++
 .../genesis-test/get_owned_accounts_info.sh   | 137 +++++++++++++++++
 .../genesis-test/get_program_accounts.sh      |  58 +++++++
 3 files changed, 339 insertions(+)
 create mode 100755 system-test/genesis-test/cluster_token_count.sh
 create mode 100755 system-test/genesis-test/get_owned_accounts_info.sh
 create mode 100755 system-test/genesis-test/get_program_accounts.sh

diff --git a/system-test/genesis-test/cluster_token_count.sh b/system-test/genesis-test/cluster_token_count.sh
new file mode 100755
index 000000000..b33d7e411
--- /dev/null
+++ b/system-test/genesis-test/cluster_token_count.sh
@@ -0,0 +1,144 @@
+#!/usr/bin/env bash
+
+# shellcheck disable=SC1090
+# shellcheck disable=SC1091
+source "$(dirname "$0")"/get_program_accounts.sh
+
+usage() {
+  exitcode=0
+  if [[ -n "$1" ]]; then
+    exitcode=1
+    echo "Error: $*"
+  fi
+  cat <<EOF
+usage: $0 [cluster_rpc_url]
+
+ Report total token distribution of a running cluster owned by the following programs:
+   STAKE
+   SYSTEM
+   VOTE
+   STORAGE
+   CONFIG
+
+ Required arguments:
+   cluster_rpc_url  - RPC URL and port for a running Solana cluster (ex: http://34.83.146.144:8899)
+EOF
+  exit $exitcode
+}
+
+function get_cluster_version {
+  clusterVersion="$(curl -s -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"getVersion"}' "$url" | jq '.result | ."solana-core" ')"
+  echo Cluster software version: "$clusterVersion"
+}
+
+function get_token_capitalization {
+  totalSupplyLamports="$(curl -s -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"getTotalSupply"}' "$url" | cut -d , -f 2 | cut -d : -f 2)"
+  totalSupplySol=$((totalSupplyLamports / LAMPORTS_PER_SOL))
+
+  printf "\n--- Token Capitalization ---\n"
+  printf "Total token capitalization %'d SOL\n" "$totalSupplySol"
+  printf "Total token capitalization %'d Lamports\n" "$totalSupplyLamports"
+
+}
+
+function get_program_account_balance_totals {
+  PROGRAM_NAME="$1"
+
+  # shellcheck disable=SC2002
+  accountBalancesLamports="$(cat "${PROGRAM_NAME}_account_data.json" | \
+    jq '.result | .[] | .account | .lamports')"
+
+  totalAccountBalancesLamports=0
+  numberOfAccounts=0
+
+  # shellcheck disable=SC2068
+  for account in ${accountBalancesLamports[@]}; do
+    totalAccountBalancesLamports=$((totalAccountBalancesLamports + account))
+    numberOfAccounts=$((numberOfAccounts + 1))
+  done
+  totalAccountBalancesSol=$((totalAccountBalancesLamports / LAMPORTS_PER_SOL))
+
+  printf "\n--- %s Account Balance Totals ---\n" "$PROGRAM_NAME"
+  printf "Number of %s Program accounts: %'.f\n" "$PROGRAM_NAME" "$numberOfAccounts"
+  printf "Total token balance in all %s accounts: %'d SOL\n" "$PROGRAM_NAME" "$totalAccountBalancesSol"
+  printf "Total token balance in all %s accounts: %'d Lamports\n" "$PROGRAM_NAME" "$totalAccountBalancesLamports"
+
+  case $PROGRAM_NAME in
+    SYSTEM)
+      systemAccountBalanceTotalSol=$totalAccountBalancesSol
+      systemAccountBalanceTotalLamports=$totalAccountBalancesLamports
+      ;;
+    STAKE)
+      stakeAccountBalanceTotalSol=$totalAccountBalancesSol
+      stakeAccountBalanceTotalLamports=$totalAccountBalancesLamports
+      ;;
+    VOTE)
+      voteAccountBalanceTotalSol=$totalAccountBalancesSol
+      voteAccountBalanceTotalLamports=$totalAccountBalancesLamports
+      ;;
+    STORAGE)
+      storageAccountBalanceTotalSol=$totalAccountBalancesSol
+      storageAccountBalanceTotalLamports=$totalAccountBalancesLamports
+      ;;
+    CONFIG)
+      configAccountBalanceTotalSol=$totalAccountBalancesSol
+      configAccountBalanceTotalLamports=$totalAccountBalancesLamports
+      ;;
+    *)
+      echo "Unknown program: $PROGRAM_NAME"
+      exit 1
+      ;;
+  esac
+}
+
+function sum_account_balances_totals {
+  grandTotalAccountBalancesSol=$((systemAccountBalanceTotalSol + stakeAccountBalanceTotalSol + voteAccountBalanceTotalSol + storageAccountBalanceTotalSol + configAccountBalanceTotalSol))
+  grandTotalAccountBalancesLamports=$((systemAccountBalanceTotalLamports + stakeAccountBalanceTotalLamports + voteAccountBalanceTotalLamports + storageAccountBalanceTotalLamports + configAccountBalanceTotalLamports))
+
+  printf "\n--- Total Token Distribution in all Account Balances ---\n"
+  printf "Total SOL in all Account Balances: %'d\n" "$grandTotalAccountBalancesSol"
+  printf "Total Lamports in all Account Balances: %'d\n" "$grandTotalAccountBalancesLamports"
+}
+
+url=$1
+[[ -n $url ]] || usage "Missing required RPC URL"
+shift
+
+LAMPORTS_PER_SOL=1000000000 # 1 billion
+
+stakeAccountBalanceTotalSol=
+systemAccountBalanceTotalSol=
+voteAccountBalanceTotalSol=
+storageAccountBalanceTotalSol=
+configAccountBalanceTotalSol=
+
+stakeAccountBalanceTotalLamports=
+systemAccountBalanceTotalLamports=
+voteAccountBalanceTotalLamports=
+storageAccountBalanceTotalLamports=
+configAccountBalanceTotalLamports=
+
+echo "--- Querying RPC URL: $url ---"
+get_cluster_version
+
+get_program_accounts STAKE "$STAKE_PROGRAM_PUBKEY" "$url"
+get_program_accounts SYSTEM "$SYSTEM_PROGRAM_PUBKEY" "$url"
+get_program_accounts VOTE "$VOTE_PROGRAM_PUBKEY" "$url"
+get_program_accounts STORAGE "$STORAGE_PROGRAM_PUBKEY" "$url"
+get_program_accounts CONFIG "$CONFIG_PROGRAM_PUBKEY" "$url"
+
+write_program_account_data_csv STAKE
+write_program_account_data_csv SYSTEM
+write_program_account_data_csv VOTE
+write_program_account_data_csv STORAGE
+write_program_account_data_csv CONFIG
+
+get_token_capitalization
+
+get_program_account_balance_totals STAKE
+get_program_account_balance_totals SYSTEM
+get_program_account_balance_totals VOTE
+get_program_account_balance_totals STORAGE
+get_program_account_balance_totals CONFIG
+
+sum_account_balances_totals
diff --git a/system-test/genesis-test/get_owned_accounts_info.sh b/system-test/genesis-test/get_owned_accounts_info.sh
new file mode 100755
index 000000000..f62ce3602
--- /dev/null
+++ b/system-test/genesis-test/get_owned_accounts_info.sh
@@ -0,0 +1,137 @@
+#!/usr/bin/env bash
+
+# shellcheck disable=SC1090
+# shellcheck disable=SC1091
+source "$(dirname "$0")"/get_program_accounts.sh
+
+usage() {
+  exitcode=0
+  if [[ -n "$1" ]]; then
+    exitcode=1
+    echo "Error: $*"
+  fi
+  cat <<EOF
+usage: $0 [cluster_rpc_url] [identity_pubkey]
+
+ Report the account addresses, balances and lockups of all stake accounts
+ for which a given key is the authorized staker.
+ Also report the system account balance for that same key.
+
+ Required arguments:
+   cluster_rpc_url  - RPC URL and port for a running Solana cluster (ex: http://34.83.146.144:8899)
+   identity_pubkey  - Base58 pubkey that is an authorized staker for at least one stake account on the cluster.
+EOF
+  exit $exitcode
+}
+
+function parse_stake_account_data_to_file {
+  account_key=$(echo "$1" | tr -d '"')
+  filter_key="$2"
+  csvfile="$3"
+
+  account_data="$(solana --url "$url" show-stake-account "$account_key")"
+  staker="$(echo "$account_data" | grep -i 'authorized staker' | cut -f3 -d " ")"
+  lockup_epoch="$(echo "$account_data" | grep -i 'lockup epoch' | cut -f3 -d " ")"
+  if [[ "$staker" == "$filter_key" ]] ; then
+    echo STAKE,"$account_key","$lamports","$lockup_epoch" >> "$csvfile"
+  fi
+}
+
+function display_results_summary {
+  stake_account_balance_total=0
+  num_stake_accounts=0
+  {
+  read -r
+  while IFS=, read -r program account_pubkey lamports lockup_epoch; do
+    case $program in
+      SYSTEM)
+        system_account_balance=$lamports
+        ;;
+      STAKE)
+        stake_account_balance_total=$((stake_account_balance_total + lamports))
+        num_stake_accounts=$((num_stake_accounts + 1))
+        ;;
+      *)
+        echo "Unknown program: $program"
+        exit 1
+        ;;
+    esac
+  done
+  } < "$results_file"
+
+  stake_account_balance_total_sol="$(bc <<< "scale=3; $stake_account_balance_total/$LAMPORTS_PER_SOL")"
+  system_account_balance_sol="$(bc <<< "scale=3; $system_account_balance/$LAMPORTS_PER_SOL")"
+
+  all_account_total_balance="$(bc <<< "scale=3; $system_account_balance+$stake_account_balance_total")"
+  all_account_total_balance_sol="$(bc <<< "scale=3; ($system_account_balance+$stake_account_balance_total)/$LAMPORTS_PER_SOL")"
+
+  echo "--------------------------------------------------------------------------------------"
+  echo "Results written to: $results_file"
+  echo "--------------------------------------------------------------------------------------"
+  echo "Summary of accounts owned by $filter_pubkey"
+  echo ""
+  printf "Number of STAKE accounts: %'d\n" "$num_stake_accounts"
+  printf "Balance of all STAKE accounts: %'d lamports\n" "$stake_account_balance_total"
+  printf "Balance of all STAKE accounts: %'.3f SOL\n" "$stake_account_balance_total_sol"
+  printf "\n"
+  printf "Balance of SYSTEM account: %'d lamports\n" "$system_account_balance"
+  printf "Balance of SYSTEM account: %'.3f SOL\n" "$system_account_balance_sol"
+  printf "\n"
+  printf "Total Balance of ALL accounts: %'d lamports\n" "$all_account_total_balance"
+  printf "Total Balance of ALL accounts: %'.3f SOL\n" "$all_account_total_balance_sol"
+  echo "--------------------------------------------------------------------------------------"
+}
+
+function display_results_details {
+  # shellcheck disable=SC2002
+  cat "$results_file" | column -t -s,
+}
+
+LAMPORTS_PER_SOL=1000000000 # 1 billion
+all_stake_accounts_json_file=all_stake_accounts_data.json
+all_stake_accounts_csv_file=all_stake_accounts_data.csv
+
+url=$1
+[[ -n $url ]] || usage "Missing required RPC URL"
+shift
+filter_pubkey=$1
+[[ -n $filter_pubkey ]] || usage "Missing required pubkey"
+shift
+
+results_file=accounts_owned_by_${filter_pubkey}.csv
+system_account_json_file=system_account_${filter_pubkey}.json
+
+echo "Program,Account_Pubkey,Lamports,Lockup_Epoch" > "$results_file"
+
+echo "Getting system account data"
+get_account_info "$filter_pubkey" "$url" "$system_account_json_file"
+# shellcheck disable=SC2002
+system_account_balance="$(cat "$system_account_json_file" | jq -r '(.result | .value | .lamports)')"
+if [[ "$system_account_balance" == "null" ]]; then
+  echo "The provided pubkey is not found in the system program: $filter_pubkey"
+  exit 1
+fi
+echo SYSTEM,"$filter_pubkey","$system_account_balance",N/A >> "$results_file"
+
+echo "Getting all stake program accounts"
+get_program_accounts STAKE "$STAKE_PROGRAM_PUBKEY" "$url" "$all_stake_accounts_json_file"
+write_program_account_data_csv STAKE "$all_stake_accounts_json_file" "$all_stake_accounts_csv_file"
+
+echo "Querying cluster at $url for stake accounts with authorized staker: $filter_pubkey"
+last_tick=$SECONDS
+{
+read -r
+while IFS=, read -r account_pubkey lamports; do
+  parse_stake_account_data_to_file "$account_pubkey" "$filter_pubkey" "$results_file" &
+  sleep 0.01
+  if [[ $((SECONDS - last_tick)) == 1 ]]; then
+    last_tick=$SECONDS
+    printf "."
+  fi
+done
+} < "$all_stake_accounts_csv_file"
+wait
+printf "\n"
+
+display_results_details
+display_results_summary
diff --git a/system-test/genesis-test/get_program_accounts.sh b/system-test/genesis-test/get_program_accounts.sh
new file mode 100755
index 000000000..d32e1c7bc
--- /dev/null
+++ b/system-test/genesis-test/get_program_accounts.sh
@@ -0,0 +1,58 @@
+# | source | this file
+
+# shellcheck disable=SC2034
+# shellcheck disable=SC2086
+
+STAKE_PROGRAM_PUBKEY=Stake11111111111111111111111111111111111111
+SYSTEM_PROGRAM_PUBKEY=11111111111111111111111111111111
+VOTE_PROGRAM_PUBKEY=Vote111111111111111111111111111111111111111
+STORAGE_PROGRAM_PUBKEY=Storage111111111111111111111111111111111111
+CONFIG_PROGRAM_PUBKEY=Config1111111111111111111111111111111111111
+
+function get_program_accounts {
+  PROGRAM_NAME="$1"
+  PROGRAM_PUBKEY="$2"
+  URL="$3"
+
+  if [[ -n "$4" ]] ; then
+    JSON_OUTFILE="$4"
+  else
+    JSON_OUTFILE="${PROGRAM_NAME}_account_data.json"
+  fi
+  curl -s -X POST -H "Content-Type: application/json" -d \
+    '{"jsonrpc":"2.0","id":1, "method":"getProgramAccounts", "params":["'$PROGRAM_PUBKEY'"]}' $URL | jq '.' \
+    > $JSON_OUTFILE
+}
+
+function write_program_account_data_csv {
+  PROGRAM_NAME="$1"
+  if [[ -n "$2" ]] ; then
+    JSON_INFILE="$2"
+  else
+    JSON_INFILE="${PROGRAM_NAME}_account_data.json"
+  fi
+  if [[ -n "$3" ]] ; then
+    CSV_OUTFILE="$3"
+  else
+    CSV_OUTFILE="${PROGRAM_NAME}_account_data.csv"
+  fi
+
+  echo "Account_Pubkey,Lamports" > $CSV_OUTFILE
+  # shellcheck disable=SC2002
+  cat "$JSON_INFILE" | jq -r '(.result | .[]) | [.pubkey, (.account | .lamports)] | @csv' \
+    >> $CSV_OUTFILE
+}
+
+function get_account_info {
+  ACCOUNT_PUBKEY="$1"
+  URL="$2"
+
+  if [[ -n "$3" ]] ; then
+    JSON_OUTFILE="$3"
+  else
+    JSON_OUTFILE="${ACCOUNT_PUBKEY}_account_info.json"
+  fi
+  curl -s -X POST -H "Content-Type: application/json" -d \
+    '{"jsonrpc":"2.0","id":1, "method":"getAccountInfo", "params":["'$ACCOUNT_PUBKEY'"]}' $URL | jq '.' \
+    > $JSON_OUTFILE
+}