From 2b899a7200fca3093431b0938c73980d60de5de5 Mon Sep 17 00:00:00 2001
From: Honza Rychnovsky <rychnovsky.honza@gmail.com>
Date: Thu, 22 Dec 2022 15:42:52 +0100
Subject: [PATCH] [#682] Testing Documentation Update

* [#682] Testing documentation update

* Bump benchmark dep to 1.2.0-alpha8

* Simplify ui-benchmark-test module build types

* Add referential benchmark test results

* Code clean
---
 ...rk_test_connectedBenchmarkAndroidTest.xml} |  2 +-
 app/src/main/AndroidManifest.xml              | 14 +++-
 docs/Architecture.md                          |  4 +-
 docs/CI.md                                    |  2 +-
 docs/Setup.md                                 | 13 +---
 docs/testing/Testing.md                       | 70 +++++++++++++++++++
 docs/testing/local_coverage.md                | 10 ---
 docs/testing/manual_testing/README.md         |  5 --
 gradle.properties                             |  2 +-
 ui-benchmark-test/build.gradle.kts            | 12 +---
 10 files changed, 93 insertions(+), 41 deletions(-)
 rename .idea/runConfigurations/{ui_benchmark_test_connectedZcashmainnetBenchmarkAndroidTest.xml => ui_benchmark_test_connectedBenchmarkAndroidTest.xml} (95%)
 create mode 100644 docs/testing/Testing.md
 delete mode 100644 docs/testing/local_coverage.md
 delete mode 100644 docs/testing/manual_testing/README.md

diff --git a/.idea/runConfigurations/ui_benchmark_test_connectedZcashmainnetBenchmarkAndroidTest.xml b/.idea/runConfigurations/ui_benchmark_test_connectedBenchmarkAndroidTest.xml
similarity index 95%
rename from .idea/runConfigurations/ui_benchmark_test_connectedZcashmainnetBenchmarkAndroidTest.xml
rename to .idea/runConfigurations/ui_benchmark_test_connectedBenchmarkAndroidTest.xml
index 34ab0698..d6eab902 100644
--- a/.idea/runConfigurations/ui_benchmark_test_connectedZcashmainnetBenchmarkAndroidTest.xml
+++ b/.idea/runConfigurations/ui_benchmark_test_connectedBenchmarkAndroidTest.xml
@@ -1,5 +1,5 @@
 <component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="ui-benchmark-test:connectedZcashmainnetBenchmarkAndroidTest" type="AndroidTestRunConfigurationType" factoryName="Android Instrumented Tests">
+  <configuration default="false" name="ui-benchmark-test:connectedBenchmarkAndroidTest" type="AndroidTestRunConfigurationType" factoryName="Android Instrumented Tests">
     <module name="zcash-android-app.ui-benchmark-test" />
     <option name="TESTING_TYPE" value="0" />
     <option name="METHOD_NAME" value="" />
diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml
index 2d6a7ec1..fe5c523f 100644
--- a/app/src/main/AndroidManifest.xml
+++ b/app/src/main/AndroidManifest.xml
@@ -21,11 +21,23 @@
             </intent-filter>
         </activity-alias>
 
-        <!-- enable profiling by benchmark -->
+        <!-- Enable profiling by benchmark -->
         <profileable
             android:shell="true"
             tools:targetApi="29" />
 
+        <!-- To bypass "The DROP_SHADER_CACHE broadcast was not received." error while benchmarking the app. See
+        https://issuetracker.google.com/issues/258619948 -->
+        <receiver
+            android:name="androidx.profileinstaller.ProfileInstallReceiver"
+            android:permission="android.permission.DUMP"
+            android:exported="true">
+            <intent-filter>
+                <action
+                    android:name="androidx.profileinstaller.action.BENCHMARK_OPERATION" />
+            </intent-filter>
+        </receiver>
+
     </application>
 
 </manifest>
diff --git a/docs/Architecture.md b/docs/Architecture.md
index e64f4d8e..eb0e4893 100644
--- a/docs/Architecture.md
+++ b/docs/Architecture.md
@@ -24,13 +24,13 @@ The main entrypoints of the application are:
 # Modules
 The logical components of the app are implemented as a number of Gradle modules.
 
- * `app` — Compiles all of the modules together into the final application.  This module contains minimal actual code.  Note that the Java package structure for this module is under `co.electriccoin.zcash.app` while the Android package name is `co.electriccoin.zcash`.
+ * `app` — Compiles all the modules together into the final application.  This module contains minimal actual code.  Note that the Java package structure for this module is under `co.electriccoin.zcash.app` while the Android package name is `co.electriccoin.zcash`.
  * `build-info-lib` — Collects information from the build environment (e.g. Git SHA, Git commit count) and compiles them into the application.  Can also be used for injection of API keys or other secrets.
  * crash — For collecting and reporting exceptions and crashes
      * `crash-lib` — Common crash collection logic for Kotlin and JVM.  This is not fully-featured by itself, but the long-term plan is multiplatform support.
      * `crash-android-lib` — Android-specific crash collection logic, built on top of the common and JVM implementation in `crash-lib`
  * ui
-     * `ui-design` — Contains UI theme elements only. Besides offering modularization, this allows for hiding of some Material Design components behind our own custom components.
+     * `ui-design` — Contains UI theme elements only. Besides offering modularization, this allows for hide of some Material Design components behind our own custom components.
      * `ui-lib` — User interface that the user interacts with.  This contains 99% of the UI code, along with localizations, icons, and other assets.
 * ui-test
     * `ui-integration-test` — Is a pure test module dedicated for integration tests only. It has Android Test Orchestrator turned on — it allows us to run each of our tests within its own invocation of Instrumentation, and thus brings us benefits for the testing environment (minimal shared state, crashes are isolated, permissions are reset).
diff --git a/docs/CI.md b/docs/CI.md
index ee9915a0..f1142ae5 100644
--- a/docs/CI.md
+++ b/docs/CI.md
@@ -18,7 +18,7 @@ To enhance security, [OpenID Connect](https://docs.github.com/en/actions/deploym
 * `FIREBASE_TEST_LAB_SERVICE_ACCOUNT` - Email address of Firebase Test Lab service account.
 * `FIREBASE_TEST_LAB_WORKLOAD_IDENTITY_PROVIDER` - Workload identity provider to generate temporary service account key.
 
-The Pull Request workflow supports testing of the app and libraries with both Emulator.wtf and Firebase Test Lab.  By default Emulator.wtf is used for library instrumentation tests, while Firebase Test Lab is used for a robo test.
+The Pull Request workflow supports testing of the app and libraries with both Emulator.wtf and Firebase Test Lab.  By default, Emulator.wtf is used for library instrumentation tests, while Firebase Test Lab is used for a robo test.
 
 To configure Firebase Test Lab, you'll need to enable the necessary Google Cloud APIs to enable automated access to Firebase Test Lab.
 * Configure Firebase Test Lab.  Google has [documentation for Jenkins](https://firebase.google.com/docs/test-lab/android/continuous).  Although we're using GitHub Actions, the initial requirements are the same.
diff --git a/docs/Setup.md b/docs/Setup.md
index b2d5cb41..ede042c7 100644
--- a/docs/Setup.md
+++ b/docs/Setup.md
@@ -112,12 +112,12 @@ Debug builds are up to 10x slower due to JIT being disabled by Android's runtime
 
 "mainnet" (main network) and "testnet" (test network) are terms used in the blockchain ecosystem to describe different blockchain networks.  Mainnet is responsible for executing actual transactions within the network and storing them on the blockchain. In contrast, the testnet provides an alternative environment that mimics the mainnet's functionality to allow developers to build and test projects without needing to facilitate live transactions or the use of cryptocurrencies, for example.
 
-Currently, we support 4 build variants for the `app` module: `zcashmainnetDebug`, `zcashtestnetDebug`, `zcashmainnetRelease`, `zcashtestnetRelease`. Library modules like `ui-lib`, `test-lib`, etc. support only `debug` and `release` variants. UI test modules like `ui-integration-test`, `ui-screenshot-test` and `ui-benchmark-test` provide variants extended by the network dimension similarly as app module does. Moreover, the `ui-benchmark-test` introduces a `benchmark` build type which results in 2 extra variants (`zcashmainnetBenchmark`, `zcashtestnetBenchmark`), which are supposed to be used only for benchmarking. 
+Currently, we support 4 build variants for the `app` module: `zcashmainnetDebug`, `zcashtestnetDebug`, `zcashmainnetRelease`, `zcashtestnetRelease`. Library modules like `ui-lib`, `test-lib`, etc. support only `debug` and `release` variants. UI test modules like `ui-integration-test`, `ui-screenshot-test` provide variants extended by the network dimension similarly as app module does. Moreover, the `ui-benchmark-test` introduces a `benchmark` build type, which is supposed to be used only for benchmarking. 
 
 App module build variants:
 - `zcashtestnetDebug` - build variant is built upon testnet network and with debug build type. You usually use this variant for development
 - `zcashmainnetDebug` - same as previous, but is built upon mainnet network
-- `zcashmainnetRelease` and `zcashtestnetRelease` - are usually used by our CI jobs to prepare binaries for testing and releasing to the Google Play store
+- `zcashmainnetRelease` and `zcashtestnetRelease` - are usually used by our CI jobs to prepare binaries for testing and releasing to the Google Play Store
 
 ### Included builds
 This section is optional.
@@ -133,13 +133,6 @@ There are some limitations of included builds:
 1. If `secant-android-wallet` is using a newer version of the Android Gradle plugin compared to `zcash-android-wallet-sdk`, the build will fail.  If this happens, you may need to modify the `zcash-android-wallet-sdk` gradle.properties so that the Android Gradle Plugin version matches that of `secant-android-wallet`.  After making this change, it will be necessary to run a build from the command line with the flag `--write-locks` e.g. `./gradlew assemble --write-locks` in order to update the dependency locks.  Similar problems can occur if projects are using different versions of Kotlin or different versions of Gradle
 1. Modules in each project cannot share the same name.  For this reason, build-conventions have different names in each repo (`zcash-android-wallet-sdk/build-conventions` vs `secant-android-wallet/build-conventions-secant`)
 
-### Benchmarking
-This section provides information about available benchmark tests integrated in this project as well as how to use them. Currently, we support macrobenchmark tests run locally as described in the Android [documentation](https://developer.android.com/topic/performance/benchmarking/benchmarking-overview).
-
-We provide dedicated benchmark test module `ui-benchamark-test` for this. If you want to run these benchmark tests against our application, make sure you have a physical device connected with Android SDK level 29, at least. Select `zcashmainnetBenchmark` or `zcashtestnetBenchmark` build variant for this module. Make sure that other modules are set to release variants of their available build variants too, as benchmarking is only allowed against minified build variants. The benchmark tests can be run with Android Studio run configuration `ui-benchmark-test:connectedZcashmainnetBenchmarkAndroidTest` with having the Gradle property `IS_SIGN_RELEASE_BUILD_WITH_DEBUG_KEY` set to true. Running the benchmark test this way automatically provides benchmarking results in Run panel. Or you can run the tests manually from the terminal with `./gradlew connectedZcashmainnetBenchmarkAndroidTest -PIS_SIGN_RELEASE_BUILD_WITH_DEBUG_KEY=true` or `./gradlew connectedZcashtestnetBenchmarkAndroidTest -PIS_SIGN_RELEASE_BUILD_WITH_DEBUG_KEY=true` and analyze results with Android Studio's Profiler or [Perfetto](https://ui.perfetto.dev/) tool, as described in this Android [documentation](https://developer.android.com/topic/performance/benchmarking/macrobenchmark-overview#access-trace).
-
-**Note**: We've enabled benchmarking also for emulators, although it's always better to run the tests on a real physical device. Emulator benchmark improvements might not carry over to a real user's experience (or even regress real device performance).
-
 ### Firebase Test Lab
 This section is optional.
 
@@ -166,7 +159,7 @@ For Continuous Integration, see [CI.md](CI.md).  The rest of this section is reg
     1. If you are an Electric Coin Co team member: We are still setting up a process for this, because emulator.wtf does not yet support individual API tokens
     1. If you are an open source contributor: Visit http://emulator.wtf and request an API key
 1. Set the emulator.wtf API key as a global Gradle property `ZCASH_EMULATOR_WTF_API_KEY` under `~/.gradle/gradle.properties`
-1. Run the Gradle task `./gradlew testDebugWithEmulatorWtf :app:testZcashmainnetDebugWithEmulatorWtf :ui-integration-test:testZcashmainnetDebugWithEmulatorWtf :ui-screenshot-test:testZcashmainnetDebugWithEmulatorWtf` (emulator.wtf tasks do build the app, so you don't need to build them beforehand)
+1. Run the Gradle task `./gradlew testDebugWithEmulatorWtf :app:testZcashmainnetDebugWithEmulatorWtf :ui-integration-test:testDebugWithEmulatorWtf :ui-screenshot-test:testDebugWithEmulatorWtf` (emulator.wtf tasks do build the app, so you don't need to build them beforehand)
 
 ## Testnet funds
 
diff --git a/docs/testing/Testing.md b/docs/testing/Testing.md
new file mode 100644
index 00000000..571b673c
--- /dev/null
+++ b/docs/testing/Testing.md
@@ -0,0 +1,70 @@
+# Testing
+This documentation outlines our approach to testing. By running tests against our app consistently, we verify the app's correctness, functional behavior, and usability before releasing it publicly.
+
+## Automated testing
+
+- TBD
+<!-- TODO [#682]: Testing documentation update --> 
+<!-- TODO [#682]: https://github.com/zcash/secant-android-wallet/issues/682 --> 
+
+## Manual testing
+
+We aim to automate as much as we possibly can. Still manual testing is really important for Quality Assurance.
+
+Here you'll find our manual testing scripts. When developing a new feature you can add your own that provide the proper steps to properly test it.
+
+## Gathering Code Coverage
+The app consists of different Gradle module types (e.g. Kotlin Multiplatform, Android).  Generating coverage for these different module types requires different command line invocations.
+
+### Kotlin Multiplatform
+Kotlin Multiplatform does not support coverage for all platforms.  Most of our code lives under commonMain, with a JVM target.  This effectively allows generation of coverage reports with Jacoco.  Coverage is enabled by default when running `./gradlew check`.
+
+### Android
+The Android Gradle plugin supports code coverage with Jacoco.  This integration can sometimes be buggy.  For that reason, coverage is disabled by default and can be enabled on a case-by-case basis, by passing `-PIS_ANDROID_INSTRUMENTATION_TEST_COVERAGE_ENABLED=true` as a command line argument for Gradle builds.  For example: `./gradlew connectedCheck -PIS_ANDROID_INSTRUMENTATION_TEST_COVERAGE_ENABLED=true`.
+
+When coverage is enabled, running instrumentation tests will automatically generate coverage reports stored under `$module/build/reports/coverage`.
+
+## Benchmarking
+This section provides information about available benchmark tests integrated in this project as well as how to use them. Currently, we support macrobenchmark tests run locally as described in the Android [documentation](https://developer.android.com/topic/performance/benchmarking/benchmarking-overview).
+
+We provide dedicated benchmark test module `ui-benchamark-test` for this. If you want to run these benchmark tests against our application, make sure you have a physical device connected with Android SDK level 29, at least. Select `benchmark` build variant for this module. Make sure that other modules are set to release variants of their available build variants too, as benchmarking is only allowed against minified build variants. The benchmark tests can be run with Android Studio run configuration `ui-benchmark-test:connectedBenchmarkAndroidTest` with having the Gradle property `IS_SIGN_RELEASE_BUILD_WITH_DEBUG_KEY` set to true. Running the benchmark test this way automatically provides benchmarking results in Run panel. Or you can run the tests manually from the terminal with `./gradlew connectedBenchmarkAndroidTest -PIS_SIGN_RELEASE_BUILD_WITH_DEBUG_KEY=true` and analyze results with Android Studio's Profiler or [Perfetto](https://ui.perfetto.dev/) tool, as described in this Android [documentation](https://developer.android.com/topic/performance/benchmarking/macrobenchmark-overview#access-trace).
+
+**Note**: We've enabled benchmarking also for emulators, although it's always better to run the tests on a real physical device. Emulator benchmark improvements might not carry over to a real user's experience (or even regress real device performance).
+
+### Referential benchmark tests results
+Every few months, or before a major app release, we run and compare benchmark test results to avoid making the app's mechanisms significantly slower.
+
+**Note**: If possible, run the benchmark tests on a physical device with sufficient empty disk space, connected to the
+internet and charged or plugged-in to a charger. It's always better to restart the device before approaching to
+running the benchmark tests. Also, please, ensure you're running it on the latest main branch
+commits of that date. Generate tests results with the Android Studio run configuration
+`ui-benchmark-test:connectedBenchmarkAndroidTest` and gather results from the Run panel.
+
+#### Dec 7, 2022:
+
+- APP version: `0.1 (1)`
+- SDK version: `1.10.0-beta01-SNAPSHOT`
+- Git branch: `682-testing-doc-update`
+- Device:
+    - Pixel 6 - Android 13:
+      ```
+      Starting 1 tests on Pixel 6 - 13
+
+      BasicStartupBenchmark_startup
+      timeToFullDisplayMs   min 288,1,   median 305,6,   max 339,1
+      timeToInitialDisplayMs   min 288,1,   median 305,6,   max 339,1
+      Traces: Iteration 0 1 2 3 4
+      
+      BUILD SUCCESSFUL in 32s
+      ```
+    - Pixel 3a - Android 12:
+      ```
+      Starting 1 tests on Pixel 3a - 12
+
+      BasicStartupBenchmark_startup
+      timeToFullDisplayMs   min 708.3,   median 763.1,   max 770.9
+      timeToInitialDisplayMs   min 708.3,   median 763.1,   max 770.9
+      Traces: Iteration 0 1 2 3 4
+      
+      BUILD SUCCESSFUL in 41s
+      ```
\ No newline at end of file
diff --git a/docs/testing/local_coverage.md b/docs/testing/local_coverage.md
deleted file mode 100644
index dd59163b..00000000
--- a/docs/testing/local_coverage.md
+++ /dev/null
@@ -1,10 +0,0 @@
-# Gathering Code Coverage
-The app consists of different Gradle module types (e.g. Kotlin Multiplatform, Android).  Generating coverage for these different module types requires different command line invocations.
-
-## Kotlin Multiplatform
-Kotlin Multiplatform does not support coverage for all platforms.  Most of our code lives under commonMain, with a JVM target.  This effectively allows generation of coverage reports with Jacoco.  Coverage is enabled by default when running `./gradlew check`.
-
-## Android
-The Android Gradle plugin supports code coverage with Jacoco.  This integration can sometimes be buggy.  For that reason, coverage is disabled by default and can be enabled on a case-by-case basis, by passing `-PIS_ANDROID_INSTRUMENTATION_TEST_COVERAGE_ENABLED=true` as a command line argument for Gradle builds.  For example: `./gradlew connectedCheck -PIS_ANDROID_INSTRUMENTATION_TEST_COVERAGE_ENABLED=true`.
-
-When coverage is enabled, running instrumentation tests will automatically generate coverage reports stored under `$module/build/reports/coverage`.
diff --git a/docs/testing/manual_testing/README.md b/docs/testing/manual_testing/README.md
deleted file mode 100644
index 9028053b..00000000
--- a/docs/testing/manual_testing/README.md
+++ /dev/null
@@ -1,5 +0,0 @@
-# Manual testing
-
-We aim to automate as much as we possibly can. Still manual testing is really important for Quality Assurance. 
-
-Here you'll find our manual testing scripts. When developing a new feature you can add your own that provide the proper steps to properly test it. 
\ No newline at end of file
diff --git a/gradle.properties b/gradle.properties
index 16504abc..c583667a 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -128,7 +128,7 @@ ANDROIDX_SPLASH_SCREEN_VERSION=1.0.0
 ANDROIDX_TEST_JUNIT_VERSION=1.1.4
 ANDROIDX_TEST_ORCHESTRATOR_VERSION=1.4.2
 ANDROIDX_TEST_CORE_VERSION=1.5.0
-ANDROIDX_TEST_MACROBENCHMARK_VERSION=1.2.0-alpha07
+ANDROIDX_TEST_MACROBENCHMARK_VERSION=1.2.0-alpha08
 ANDROIDX_TEST_RUNNER_VERSION=1.5.1
 ANDROIDX_STARTUP_VERSION=1.1.1
 ANDROIDX_UI_AUTOMATOR_VERSION=2.2.0-alpha1
diff --git a/ui-benchmark-test/build.gradle.kts b/ui-benchmark-test/build.gradle.kts
index f34ff58b..857ef643 100644
--- a/ui-benchmark-test/build.gradle.kts
+++ b/ui-benchmark-test/build.gradle.kts
@@ -13,16 +13,8 @@ android {
         testInstrumentationRunner = "co.electriccoin.zcash.test.ZcashUiTestRunner"
         // to enable benchmarking for emulators, although only a physical device gives real results
         testInstrumentationRunnerArguments["androidx.benchmark.suppressErrors"] = "EMULATOR"
-    }
-
-    flavorDimensions.add("network")
-    productFlavors {
-        create("zcashtestnet") {
-            dimension = "network"
-        }
-        create("zcashmainnet") {
-            dimension = "network"
-        }
+        // To simplify module variants, we assume to run benchmarking against mainnet only
+        missingDimensionStrategy("network", "zcashmainnet")
     }
 
     buildTypes {