[#599] Macrobenchmark test module

* Temporary enable FTL test

* Enable emulator.wft tests for ui-integration-test module on CI

* Rebase onto main

* Test Fladle configuration for ui-integration-test module

* Fix Fladle for ui-integration-test module

* Rename ui-integration-test module flade configuration

* Disable again FTL action from PRs

* Clear support for FTL from ui-integration-test module

* Refactor the emulator.wtf support across our modules.

* [#599] Macrobenchmark test module

- Created new dedicated macrobenchmark test module
- Updated related Architecture and Setup documentation
- Connected to app module
- New benchmark build type
- Related run configuration above custom Gradle task
- Basic startup benchmark test included

* Benchmark build variant simplification

* Run benchmarking simplification

* Documentation update.

* New property IS_SIGN_RELEASE_BUILD_WITH_DEBUG_KEY

- It enables signing the release build with debug key configuration. Default value is false.
- First, we check if we can sign it with release configs, otherwise with debug.
- Documentation updated.

* Benchmarking documentation update

* Adds support for Android SDK 32 and 33

- Bumped benchmark library to the latest alpha version
- Which results in a need of adding profile-installer library in target module (app)
- Now we're able to run benchmarking on Android SDK level 29 and higher (i.e. latest levels included)
- Updated related documentation

* Enables benchmarking for emulators

- I've decided to support emulator devices with our benchmark test module at the end. Documentation and code comments explain to a user that the results may not be the same as on a real physical device. But until we have it set up in CI, we can benefit from having it set up like this.
This commit is contained in:
Honza Rychnovsky 2022-10-27 12:51:24 +02:00 committed by GitHub
parent 570c98b994
commit bcacf93a04
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 207 additions and 5 deletions

View File

@ -0,0 +1,55 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="ui-benchmark-test:connectedZcashmainnetBenchmarkAndroidTest" type="AndroidTestRunConfigurationType" factoryName="Android Instrumented Tests">
<module name="zcash-android-app.ui-benchmark-test" />
<option name="TESTING_TYPE" value="0" />
<option name="METHOD_NAME" value="" />
<option name="CLASS_NAME" value="" />
<option name="PACKAGE_NAME" value="" />
<option name="TEST_NAME_REGEX" value="" />
<option name="INSTRUMENTATION_RUNNER_CLASS" value="" />
<option name="EXTRA_OPTIONS" value="" />
<option name="RETENTION_ENABLED" value="No" />
<option name="RETENTION_MAX_SNAPSHOTS" value="2" />
<option name="RETENTION_COMPRESS_SNAPSHOTS" value="false" />
<option name="CLEAR_LOGCAT" value="false" />
<option name="SHOW_LOGCAT_AUTOMATICALLY" value="false" />
<option name="INSPECTION_WITHOUT_ACTIVITY_RESTART" value="false" />
<option name="TARGET_SELECTION_MODE" value="DEVICE_AND_SNAPSHOT_COMBO_BOX" />
<option name="SELECTED_CLOUD_MATRIX_CONFIGURATION_ID" value="-1" />
<option name="SELECTED_CLOUD_MATRIX_PROJECT_ID" value="" />
<option name="DEBUGGER_TYPE" value="Auto" />
<Auto>
<option name="USE_JAVA_AWARE_DEBUGGER" value="false" />
<option name="SHOW_STATIC_VARS" value="true" />
<option name="WORKING_DIR" value="" />
<option name="TARGET_LOGGING_CHANNELS" value="lldb process:gdb-remote packets" />
<option name="SHOW_OPTIMIZED_WARNING" value="true" />
</Auto>
<Hybrid>
<option name="USE_JAVA_AWARE_DEBUGGER" value="false" />
<option name="SHOW_STATIC_VARS" value="true" />
<option name="WORKING_DIR" value="" />
<option name="TARGET_LOGGING_CHANNELS" value="lldb process:gdb-remote packets" />
<option name="SHOW_OPTIMIZED_WARNING" value="true" />
</Hybrid>
<Java />
<Native>
<option name="USE_JAVA_AWARE_DEBUGGER" value="false" />
<option name="SHOW_STATIC_VARS" value="true" />
<option name="WORKING_DIR" value="" />
<option name="TARGET_LOGGING_CHANNELS" value="lldb process:gdb-remote packets" />
<option name="SHOW_OPTIMIZED_WARNING" value="true" />
</Native>
<Profilers>
<option name="ADVANCED_PROFILING_ENABLED" value="false" />
<option name="STARTUP_PROFILING_ENABLED" value="false" />
<option name="STARTUP_CPU_PROFILING_ENABLED" value="false" />
<option name="STARTUP_CPU_PROFILING_CONFIGURATION_NAME" value="Java/Kotlin Method Sample (legacy)" />
<option name="STARTUP_NATIVE_MEMORY_PROFILING_ENABLED" value="false" />
<option name="NATIVE_MEMORY_SAMPLE_RATE_BYTES" value="2048" />
</Profilers>
<method v="2">
<option name="Android.Gradle.BeforeRunTask" enabled="true" />
</method>
</configuration>
</component>

View File

@ -89,8 +89,15 @@ android {
getDefaultProguardFile("proguard-android-optimize.txt"),
"proguard-project.txt"
)
val isSignReleaseBuildWithDebugKey = project.property("IS_SIGN_RELEASE_BUILD_WITH_DEBUG_KEY")
.toString().toBoolean()
if (isReleaseSigningConfigured) {
signingConfig = signingConfigs.getByName("release")
} else if (isSignReleaseBuildWithDebugKey) {
// Warning: in this case is the release build signed with the debug key
signingConfig = signingConfigs.getByName("debug")
}
}
all {
@ -161,6 +168,8 @@ dependencies {
implementation(libs.androidx.activity)
implementation(libs.androidx.annotation)
implementation(libs.androidx.core)
// just to support baseline profile installation needed by benchmark tests
implementation(libs.androidx.profileinstaller)
implementation(libs.kotlin.stdlib)
implementation(libs.kotlinx.coroutines.android)
implementation(libs.kotlinx.coroutines.core)

View File

@ -1,5 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android">
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
xmlns:tools="http://schemas.android.com/tools">
<application
android:name="co.electriccoin.zcash.app.ZcashApplication"
@ -19,6 +20,12 @@
<category android:name="android.intent.category.LAUNCHER" />
</intent-filter>
</activity-alias>
<!-- enable profiling by benchmark -->
<profileable
android:shell="true"
tools:targetApi="29" />
</application>
</manifest>

View File

@ -18,8 +18,8 @@ Note: Test coverage for multiplatform modules behaves differently than coverage
# App
The main entrypoints of the application are:
* [AppImpl.kt](../app/src/main/java/cash/z/ecc/app/AppImpl.kt) - The root Application object defined in the app module
* [MainActivity.kt](../ui-lib/src/main/java/cash/z/ecc/ui/MainActivity.kt) - The main Activity, defined in ui-lib. Note that the Activity is NOT exported. Instead, the app module defines an activity-alias in the AndroidManifest which is what presents the actual icon on the Android home screen.
* [ZcashApplication.kt](../app/src/main/java/co/electriccoin/zcash/app/ZcashApplication.kt) - The root Application object defined in the app module
* [MainActivity.kt](../ui-lib/src/main/java/co/electriccoin/zcash/ui/MainActivity.kt) - The main Activity, defined in ui-lib. Note that the Activity is NOT exported. Instead, the app module defines an activity-alias in the AndroidManifest which is what presents the actual icon on the Android home screen.
# Modules
The logical components of the app are implemented as a number of Gradle modules.
@ -35,6 +35,7 @@ The logical components of the app are implemented as a number of Gradle modules.
* ui-test
* `ui-integration-test` — Is a pure test module dedicated for integration tests only. It has Android Test Orchestrator turned on — it allows us to run each of our tests within its own invocation of Instrumentation, and thus brings us benefits for the testing environment (minimal shared state, crashes are isolated, permissions are reset).
* `ui-screenshot-test` — Is also a pure test module, whose purpose is to provide a wrapper for the ui screenshot tests. It has the Android Test Orchestrator turned on too.
* `ui-benchmark-test` — Test module, which we use to run macrobenchmark tests against the `app` module. Benchmarking is a way to inspect and monitor the performance of our application. We regularly run benchmarks to help analyze and debug performance problems and ensure that we don't introduce regressions in recent changes.
* preference
* `preference-api-lib` — Multiplatform interfaces for key-value storage of preferences.
* `preference-impl-android-lib` — Android-specific implementation for preference storage.
@ -75,6 +76,7 @@ The following diagram shows a rough depiction of dependencies between the module
subgraph ui-test
uiIntegrationTest[[ui-integration-test]];
uiScreenshotTest[[ui-screenshot-test]];
uiBenchmarkTest[[ui-benchmark-test]];
end
subgraph spackle
spackleLib[[spackle-lib]];

View File

@ -112,8 +112,9 @@ Debug builds are up to 10x slower due to JIT being disabled by Android's runtime
"mainnet" (main network) and "testnet" (test network) are terms used in the blockchain ecosystem to describe different blockchain networks. Mainnet is responsible for executing actual transactions within the network and storing them on the blockchain. In contrast, the testnet provides an alternative environment that mimics the mainnet's functionality to allow developers to build and test projects without needing to facilitate live transactions or the use of cryptocurrencies, for example.
Currently, we support 4 build variants for the `app` module: `zcashmainnetDebug`, `zcashtestnetDebug`, `zcashmainnetRelease`, `zcashtestnetRelease`. Library modules like `ui-lib`, `test-lib`, etc. support only `debug` and `release` variants.
Currently, we support 4 build variants for the `app` module: `zcashmainnetDebug`, `zcashtestnetDebug`, `zcashmainnetRelease`, `zcashtestnetRelease`. Library modules like `ui-lib`, `test-lib`, etc. support only `debug` and `release` variants. UI test modules like `ui-integration-test`, `ui-screenshot-test` and `ui-benchmark-test` provide variants extended by the network dimension similarly as app module does. Moreover, the `ui-benchmark-test` introduces a `benchmark` build type which results in 2 extra variants (`zcashmainnetBenchmark`, `zcashtestnetBenchmark`), which are supposed to be used only for benchmarking.
App module build variants:
- `zcashtestnetDebug` - build variant is built upon testnet network and with debug build type. You usually use this variant for development
- `zcashmainnetDebug` - same as previous, but is built upon mainnet network
- `zcashmainnetRelease` and `zcashtestnetRelease` - are usually used by our CI jobs to prepare binaries for testing and releasing to the Google Play store
@ -132,6 +133,12 @@ There are some limitations of included builds:
1. If `secant-android-wallet` is using a newer version of the Android Gradle plugin compared to `zcash-android-wallet-sdk`, the build will fail. If this happens, you may need to modify the `zcash-android-wallet-sdk` gradle.properties so that the Android Gradle Plugin version matches that of `secant-android-wallet`. After making this change, it will be necessary to run a build from the command line with the flag `--write-locks` e.g. `./gradlew assemble --write-locks` in order to update the dependency locks. Similar problems can occur if projects are using different versions of Kotlin or different versions of Gradle
1. Modules in each project cannot share the same name. For this reason, build-conventions have different names in each repo (`zcash-android-wallet-sdk/build-conventions` vs `secant-android-wallet/build-conventions-secant`)
### Benchmarking
This section provides information about available benchmark tests integrated in this project as well as how to use them. Currently, we support macrobenchmark tests run locally as described in the Android [documentation](https://developer.android.com/topic/performance/benchmarking/benchmarking-overview).
We provide dedicated benchmark test module `ui-benchamark-test` for this. If you want to run these benchmark tests against our application, make sure you have a physical device connected with Android SDK level 29, at least. Select `zcashmainnetBenchmark` or `zcashtestnetBenchmark` build variant for this module. Make sure that other modules are set to release variants of their available build variants too, as benchmarking is only allowed against minified build variants. The benchmark tests can be run with Android Studio run configuration `ui-benchmark-test:connectedZcashmainnetBenchmarkAndroidTest` with having the Gradle property `IS_SIGN_RELEASE_BUILD_WITH_DEBUG_KEY` set to true. Running the benchmark test this way automatically provides benchmarking results in Run panel. Or you can run the tests manually from the terminal with `./gradlew connectedZcashmainnetBenchmarkAndroidTest -PIS_SIGN_RELEASE_BUILD_WITH_DEBUG_KEY=true` or `./gradlew connectedZcashtestnetBenchmarkAndroidTest -PIS_SIGN_RELEASE_BUILD_WITH_DEBUG_KEY=true` and analyze results with Android Studio's Profiler or [Perfetto](https://ui.perfetto.dev/) tool, as described in this Android [documentation](https://developer.android.com/topic/performance/benchmarking/macrobenchmark-overview#access-trace).
**Note**: We've enabled benchmarking also for emulators, although it's always better to run the tests on a real physical device. Emulator benchmark improvements might not carry over to a real user's experience (or even regress real device performance).
### Firebase Test Lab
This section is optional.
@ -158,7 +165,7 @@ For Continuous Integration, see [CI.md](CI.md). The rest of this section is reg
1. Configure or request access to emulator.wtf
1. If you are an Electric Coin Co team member: We are still setting up a process for this, because emulator.wtf does not yet support individual API tokens
1. If you are an open source contributor: Visit http://emulator.wtf and request an API key
1. Set the emulator.wtf API key as a global Gradle property `ZCASH_EMULATOR_WTF_API_KEY` under `~/.gradle/gradle.properties`
1. Set the emulator.wtf API key as a global Gradle property `ZCASH_EMULATOR_WTF_API_KEY` under `~/.gradle/gradle.properties`
1. Run the Gradle task `./gradlew testDebugWithEmulatorWtf :app:testZcashmainnetDebugWithEmulatorWtf :ui-integration-test:testZcashmainnetDebugWithEmulatorWtf :ui-screenshot-test:testZcashmainnetDebugWithEmulatorWtf` (emulator.wtf tasks do build the app, so you don't need to build them beforehand)
## Testnet funds

View File

@ -65,6 +65,11 @@ ZCASH_RELEASE_KEYSTORE_PASSWORD=
ZCASH_RELEASE_KEY_ALIAS=
ZCASH_RELEASE_KEY_ALIAS_PASSWORD=
# Switch this property to true only if you need to sign the release build with a debug key. It can
# be useful, for example, for running benchmark tests against a release build of the app signed with
# the default debug key configuration.
IS_SIGN_RELEASE_BUILD_WITH_DEBUG_KEY=false
# Optionally set the Google Play Service Key path to enable deployment
ZCASH_GOOGLE_PLAY_SERVICE_KEY_FILE_PATH=
# Can be one of {build, deploy}.
@ -117,11 +122,13 @@ ANDROIDX_CORE_VERSION=1.9.0
ANDROIDX_ESPRESSO_VERSION=3.5.0-alpha07
ANDROIDX_LIFECYCLE_VERSION=2.5.1
ANDROIDX_NAVIGATION_COMPOSE_VERSION=2.5.2
ANDROIDX_PROFILE_INSTALLER_VERSION=1.3.0-alpha01
ANDROIDX_SECURITY_CRYPTO_VERSION=1.1.0-alpha03
ANDROIDX_SPLASH_SCREEN_VERSION=1.0.0
ANDROIDX_TEST_JUNIT_VERSION=1.1.4-alpha07
ANDROIDX_TEST_ORCHESTRATOR_VERSION=1.4.2-alpha04
ANDROIDX_TEST_CORE_VERSION=1.5.0-alpha02
ANDROIDX_TEST_MACROBENCHMARK_VERSION=1.2.0-alpha06
ANDROIDX_TEST_RUNNER_VERSION=1.5.0-alpha04
ANDROIDX_UI_AUTOMATOR_VERSION=2.2.0-alpha1
ANDROIDX_WORK_MANAGER_VERSION=2.7.1
@ -136,6 +143,7 @@ PLAY_CORE_KTX_VERSION=1.8.1
ZCASH_ANDROID_WALLET_PLUGINS_VERSION=1.0.0
ZCASH_BIP39_VERSION=1.0.4
# TODO [#279]: Revert to stable SDK before app release
# TODO [#279]: https://github.com/zcash/secant-android-wallet/issues/279
ZCASH_SDK_VERSION=1.9.0-beta02
ZXING_VERSION=3.5.0

View File

@ -124,10 +124,12 @@ dependencyResolutionManagement {
val androidxEspressoVersion = extra["ANDROIDX_ESPRESSO_VERSION"].toString()
val androidxLifecycleVersion = extra["ANDROIDX_LIFECYCLE_VERSION"].toString()
val androidxNavigationComposeVersion = extra["ANDROIDX_NAVIGATION_COMPOSE_VERSION"].toString()
val androidxProfileInstallerVersion = extra["ANDROIDX_PROFILE_INSTALLER_VERSION"].toString()
val androidxSecurityCryptoVersion = extra["ANDROIDX_SECURITY_CRYPTO_VERSION"].toString()
val androidxSplashScreenVersion = extra["ANDROIDX_SPLASH_SCREEN_VERSION"].toString()
val androidxTestCoreVersion = extra["ANDROIDX_TEST_CORE_VERSION"].toString()
val androidxTestJunitVersion = extra["ANDROIDX_TEST_JUNIT_VERSION"].toString()
val androidxTestMacrobenchmarkVersion = extra["ANDROIDX_TEST_MACROBENCHMARK_VERSION"].toString()
val androidxTestOrchestratorVersion = extra["ANDROIDX_TEST_ORCHESTRATOR_VERSION"].toString()
val androidxTestRunnerVersion = extra["ANDROIDX_TEST_RUNNER_VERSION"].toString()
val androidxUiAutomatorVersion = extra["ANDROIDX_UI_AUTOMATOR_VERSION"].toString()
@ -171,6 +173,7 @@ dependencyResolutionManagement {
library("androidx-constraintlayout", "androidx.constraintlayout:constraintlayout-compose:$androidxConstraintLayoutVersion")
library("androidx-lifecycle-livedata", "androidx.lifecycle:lifecycle-livedata-ktx:$androidxLifecycleVersion")
library("androidx-navigation-compose", "androidx.navigation:navigation-compose:$androidxNavigationComposeVersion")
library("androidx-profileinstaller", "androidx.profileinstaller:profileinstaller:$androidxProfileInstallerVersion")
library("androidx-security-crypto", "androidx.security:security-crypto-ktx:$androidxSecurityCryptoVersion")
library("androidx-splash", "androidx.core:core-splashscreen:$androidxSplashScreenVersion")
library("androidx-viewmodel-compose", "androidx.lifecycle:lifecycle-viewmodel-compose:$androidxLifecycleVersion")
@ -199,6 +202,7 @@ dependencyResolutionManagement {
library("androidx-espresso-intents", "androidx.test.espresso:espresso-intents:$androidxEspressoVersion")
library("androidx-test-core", "androidx.test:core-ktx:$androidxTestCoreVersion")
library("androidx-test-junit", "androidx.test.ext:junit-ktx:$androidxTestJunitVersion")
library("androidx-test-macrobenchmark", "androidx.benchmark:benchmark-macro-junit4:$androidxTestMacrobenchmarkVersion")
library("androidx-test-orchestrator", "androidx.test:orchestrator:$androidxTestOrchestratorVersion")
library("androidx-test-runner", "androidx.test:runner:$androidxTestRunnerVersion")
library("androidx-uiAutomator", "androidx.test.uiautomator:uiautomator-v18:$androidxUiAutomatorVersion")
@ -270,6 +274,7 @@ include("sdk-ext-ui-lib")
include("spackle-lib")
include("spackle-android-lib")
include("test-lib")
include("ui-benchmark-test")
include("ui-design-lib")
include("ui-integration-test")
include("ui-lib")

View File

@ -0,0 +1,56 @@
plugins {
id("com.android.test")
kotlin("android")
id("secant.android-build-conventions")
}
android {
namespace = "co.electriccoin.zcash.ui.benchmark"
targetProjectPath = ":${projects.app.name}"
experimentalProperties["android.experimental.self-instrumenting"] = true
defaultConfig {
testInstrumentationRunner = "co.electriccoin.zcash.test.ZcashUiTestRunner"
// to enable benchmarking for emulators, although only a physical device gives real results
testInstrumentationRunnerArguments["androidx.benchmark.suppressErrors"] = "EMULATOR"
}
flavorDimensions.add("network")
productFlavors {
create("zcashtestnet") {
dimension = "network"
}
create("zcashmainnet") {
dimension = "network"
}
}
buildTypes {
create("release") {
// To provide compatibility with other modules
}
create("benchmark") {
// We provide the extra benchmark build variants for benchmarking. We still need to support debug
// variants to be compatible with debug variants in other modules, although benchmarking does not allow
// not minified build variants - benchmarking with the debug build variants will fail.
isDebuggable = true
signingConfig = signingConfigs.getByName("debug")
matchingFallbacks += listOf("release")
}
}
}
dependencies {
implementation(projects.testLib)
implementation(libs.bundles.androidx.test)
implementation(libs.androidx.test.macrobenchmark)
if (project.property("IS_USE_TEST_ORCHESTRATOR").toString().toBoolean()) {
androidTestUtil(libs.androidx.test.orchestrator) {
artifact {
type = "apk"
}
}
}
}

View File

@ -0,0 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<manifest>
</manifest>

View File

@ -0,0 +1,39 @@
package co.electriccoin.zcash.ui.benchmark
import androidx.benchmark.macro.StartupMode
import androidx.benchmark.macro.StartupTimingMetric
import androidx.benchmark.macro.junit4.MacrobenchmarkRule
import org.junit.Rule
import org.junit.Test
/**
* This is an example startup benchmark. Its purpose is to provide basic startup measurements, and captured
* system traces for investigating the app's performance.
*
* It navigates to the device's home screen, and launches the default activity. Run this benchmark from Studio only
* against the release variant type and use one of the 'Benchmark' trailing build variants for this module.
* The 'Debug' trailing modules are also available, but they just provide compatibility with other debug modules.
*
* We ideally run this against a physical device with Android SDK level 29, at least, as profiling is provided by this
* version and later on.
*/
class BasicStartupBenchmark {
companion object {
private const val APP_TARGET_PACKAGE_NAME = "co.electriccoin.zcash"
}
@get:Rule
val benchmarkRule = MacrobenchmarkRule()
@Test
fun startup() = benchmarkRule.measureRepeated(
packageName = APP_TARGET_PACKAGE_NAME,
metrics = listOf(StartupTimingMetric()),
iterations = 5,
startupMode = StartupMode.COLD
) {
pressHome()
startActivityAndWait()
}
}

View File

@ -34,6 +34,11 @@ android {
dimension = "network"
}
}
buildTypes {
create("release") {
// to align with the benchmark module requirement - run against minified application
}
}
if (isOrchestratorEnabled) {
testOptions {

View File

@ -32,6 +32,11 @@ android {
dimension = "network"
}
}
buildTypes {
create("release") {
// to align with the benchmark module requirement - run against minified application
}
}
if (isOrchestratorEnabled) {
testOptions {