[#592] Instrumentation coverage

Co-authored-by: Carter Jernigan <git@carterjernigan.com>
This commit is contained in:
Alex 2023-01-17 20:54:40 +01:00 committed by GitHub
parent 4f144405c1
commit b84df3d9dc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
28 changed files with 248 additions and 34 deletions

View File

@ -275,7 +275,7 @@ jobs:
name: Test Android modules with FTL results
path: ~/artifacts
test_android_modules_wtf:
test_android_modules_wtf_coverage:
if: needs.check_emulator_wtf_secrets.outputs.has-secrets == 'true'
needs: [ prime_cache, check_emulator_wtf_secrets ]
runs-on: ubuntu-latest
@ -290,13 +290,14 @@ jobs:
timeout-minutes: 5
uses: ./.github/actions/setup
- name: Build and test
timeout-minutes: 25
timeout-minutes: 30
env:
# Force blank suffix for screenshot tests
ORG_GRADLE_PROJECT_ZCASH_DEBUG_APP_NAME_SUFFIX: ""
ORG_GRADLE_PROJECT_ZCASH_EMULATOR_WTF_API_KEY: ${{ secrets.EMULATOR_WTF_API_KEY }}
ORG_GRADLE_PROJECT_IS_ANDROID_INSTRUMENTATION_TEST_COVERAGE_ENABLED: true
run: |
./gradlew testDebugWithEmulatorWtf :app:testZcashmainnetDebugWithEmulatorWtf :ui-integration-test:testZcashmainnetDebugWithEmulatorWtf :ui-screenshot-test:testZcashmainnetDebugWithEmulatorWtf
./gradlew testDebugWithEmulatorWtf :ui-integration-test:testZcashmainnetDebugWithEmulatorWtf
- name: Collect Artifacts
if: ${{ always() }}
timeout-minutes: 1
@ -312,9 +313,48 @@ jobs:
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce
timeout-minutes: 1
with:
name: Test Android modules with WTF results
name: Test Android libs with WTF results
path: ~/artifacts
test_android_modules_wtf_no_coverage:
if: needs.check_emulator_wtf_secrets.outputs.has-secrets == 'true'
needs: [ prime_cache, check_emulator_wtf_secrets ]
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Checkout
timeout-minutes: 1
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
- name: Setup
id: setup
timeout-minutes: 5
uses: ./.github/actions/setup
- name: Build and test
timeout-minutes: 30
env:
# Force blank suffix for screenshot tests
ORG_GRADLE_PROJECT_ZCASH_DEBUG_APP_NAME_SUFFIX: ""
ORG_GRADLE_PROJECT_ZCASH_EMULATOR_WTF_API_KEY: ${{ secrets.EMULATOR_WTF_API_KEY }}
run: |
./gradlew :app:testZcashmainnetDebugWithEmulatorWtf :ui-screenshot-test:testZcashmainnetDebugWithEmulatorWtf
- name: Collect Artifacts
if: ${{ always() }}
timeout-minutes: 1
env:
ARTIFACTS_DIR_PATH: ${{ format('{0}/artifacts', env.home) }}
TEST_RESULTS_ZIP_PATH: ${{ format('{0}/artifacts/test_results.zip', env.home) }}
run: |
mkdir ${ARTIFACTS_DIR_PATH}
zip -r ${TEST_RESULTS_ZIP_PATH} . -i \*/build/test-results/\*
- name: Upload Artifacts
if: ${{ always() }}
uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb
timeout-minutes: 1
with:
name: Test Android app with WTF results
path: ~/artifacts
# Performs a button mash test on the debug build of the app with strict mode enabled
test_robo_debug:

View File

@ -23,9 +23,19 @@ android {
versionCode = project.property("ZCASH_VERSION_CODE").toString().toInt()
versionName = project.property("ZCASH_VERSION_NAME").toString()
if (project.property("IS_USE_TEST_ORCHESTRATOR").toString().toBoolean()) {
testInstrumentationRunnerArguments["clearPackageData"] = "true"
}
testInstrumentationRunner = "co.electriccoin.zcash.test.ZcashUiTestRunner"
}
if (project.property("IS_USE_TEST_ORCHESTRATOR").toString().toBoolean()) {
testOptions {
execution = "ANDROIDX_TEST_ORCHESTRATOR"
}
}
compileOptions {
isCoreLibraryDesugaringEnabled = true
}
@ -160,6 +170,10 @@ android {
enabled.set(false)
}
}
testCoverage {
jacocoVersion = project.property("JACOCO_VERSION").toString()
}
}
dependencies {
@ -180,6 +194,12 @@ dependencies {
androidTestImplementation(projects.testLib)
androidTestUtil(libs.androidx.test.services) {
artifact {
type = "apk"
}
}
if (project.property("IS_USE_TEST_ORCHESTRATOR").toString().toBoolean()) {
androidTestUtil(libs.androidx.test.orchestrator) {
artifact {
@ -234,7 +254,6 @@ if (googlePlayServiceKeyFilePath.isNotEmpty()) {
}
}
fladle {
// Firebase Test Lab has min and max values that might differ from our project's
// These are determined by `gcloud firebase test android models list`

View File

@ -1,4 +1,13 @@
<?xml version="1.0" encoding="utf-8"?>
<manifest>
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
>
<!-- For test coverage -->
<uses-permission
android:name="android.permission.WRITE_EXTERNAL_STORAGE"
android:maxSdkVersion="29" />
<!-- For test coverage on API 29 only -->
<application
android:requestLegacyExternalStorage="true" />
</manifest>

View File

@ -18,6 +18,7 @@ pluginManager.withPlugin("com.android.application") {
testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"
testInstrumentationRunnerArguments["useTestStorageService"] = "true"
if (project.property("IS_USE_TEST_ORCHESTRATOR").toString().toBoolean()) {
testInstrumentationRunnerArguments["clearPackageData"] = "true"
}
@ -42,6 +43,7 @@ pluginManager.withPlugin("com.android.library") {
testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"
consumerProguardFiles("proguard-consumer.txt")
testInstrumentationRunnerArguments["useTestStorageService"] = "true"
if (project.property("IS_USE_TEST_ORCHESTRATOR").toString().toBoolean()) {
testInstrumentationRunnerArguments["clearPackageData"] = "true"
}
@ -67,6 +69,11 @@ pluginManager.withPlugin("com.android.test") {
resourceConfigurations.addAll(listOf("en", "en-rUS", "en-rGB", "en-rAU", "en_XA", "ar_XB"))
testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"
testInstrumentationRunnerArguments["useTestStorageService"] = "true"
if (project.property("IS_USE_TEST_ORCHESTRATOR").toString().toBoolean()) {
testInstrumentationRunnerArguments["clearPackageData"] = "true"
}
}
testCoverage {
jacocoVersion = project.property("JACOCO_VERSION").toString()
@ -87,8 +94,11 @@ fun com.android.build.gradle.BaseExtension.configureBaseExtension(isLibrary: Boo
buildTypes {
getByName("debug").apply {
isTestCoverageEnabled =
val coverageEnabled =
project.property("IS_ANDROID_INSTRUMENTATION_TEST_COVERAGE_ENABLED").toString().toBoolean()
isTestCoverageEnabled = coverageEnabled
enableAndroidTestCoverage = coverageEnabled
enableUnitTestCoverage = coverageEnabled
}
}

View File

@ -1,3 +1,6 @@
import com.android.build.gradle.internal.cxx.configure.buildTypeOf
import com.android.build.gradle.internal.dsl.decorator.androidPluginDslDecorator
// Emulator WTF has min and max values that might differ from our project's
// These are determined by `ew-cli --models`
@ -16,6 +19,11 @@ pluginManager.withPlugin("wtf.emulator.gradle") {
token.set(tokenString)
}
if (project.properties["IS_ANDROID_INSTRUMENTATION_TEST_COVERAGE_ENABLED"].toString().toBoolean()) {
withCoverage.set(true)
environmentVariables.set(mapOf("useTestStorageService" to "false"))
}
val buildMinSdk = if (pluginManager.hasPlugin("com.android.application")) {
project.properties["ANDROID_APP_MIN_SDK_VERSION"].toString().toInt()
} else if (pluginManager.hasPlugin("com.android.test")) {

View File

@ -1,3 +1,5 @@
import kotlinx.kover.api.KoverMergedConfig
buildscript {
dependencyLocking {
// This property is treated specially, as it is not defined by default in the root gradle.properties
@ -124,21 +126,30 @@ fladle {
}
kover {
isDisabled = !project.property("IS_KOTLIN_TEST_COVERAGE_ENABLED").toString().toBoolean()
isDisabled.set(!project.property("IS_KOTLIN_TEST_COVERAGE_ENABLED").toString().toBoolean())
engine.set(kotlinx.kover.api.JacocoEngine(project.property("JACOCO_VERSION").toString()))
// Don't run on the Android projects, as they have coverage generated in a different way
// through Android's instrumented tests
disabledProjects = setOf(
"app",
"crash-android-lib",
"preference-impl-android-lib",
"sdk-ext-lib",
"sdk-ext-ui-lib",
"spackle-android-lib",
"test-lib",
"ui-design-lib",
"ui-integration-test",
"ui-lib",
"ui-screenshot-test",
)
extensions.configure<KoverMergedConfig> {
enable()
filters {
projects {
excludes.addAll(setOf(
"app",
"crash-android-lib",
"preference-impl-android-lib",
"sdk-ext-lib",
"sdk-ext-ui-lib",
"spackle-android-lib",
"test-lib",
"ui-benchmark-test",
"ui-design-lib",
"ui-integration-test",
"ui-lib",
"ui-screenshot-test"
))
}
}
}
}

View File

@ -147,13 +147,13 @@ org.jetbrains.kotlin:kotlin-stdlib:1.7.10=classpath
org.jetbrains.kotlin:kotlin-tooling-core:1.7.21=classpath
org.jetbrains.kotlin:kotlin-util-io:1.7.21=classpath
org.jetbrains.kotlin:kotlin-util-klib:1.7.21=classpath
org.jetbrains.kotlinx.kover:org.jetbrains.kotlinx.kover.gradle.plugin:0.5.1=classpath
org.jetbrains.kotlinx.kover:org.jetbrains.kotlinx.kover.gradle.plugin:0.6.1=classpath
org.jetbrains.kotlinx:kotlinx-coroutines-core-jvm:1.5.0=classpath
org.jetbrains.kotlinx:kotlinx-serialization-core-jvm:1.1.0=classpath
org.jetbrains.kotlinx:kotlinx-serialization-core:1.1.0=classpath
org.jetbrains.kotlinx:kotlinx-serialization-json-jvm:1.1.0=classpath
org.jetbrains.kotlinx:kotlinx-serialization-json:1.1.0=classpath
org.jetbrains.kotlinx:kover:0.5.1=classpath
org.jetbrains.kotlinx:kover:0.6.1=classpath
org.jetbrains:annotations:13.0=classpath
org.json:json:20180813=classpath
org.jvnet.staxex:stax-ex:1.8.1=classpath

View File

@ -38,6 +38,12 @@ dependencies {
androidTestImplementation(libs.bundles.androidx.test)
androidTestImplementation(libs.kotlinx.coroutines.test)
androidTestUtil(libs.androidx.test.services) {
artifact {
type = "apk"
}
}
androidTestUtil(libs.androidx.test.orchestrator) {
artifact {
type = "apk"

View File

@ -1,6 +1,12 @@
<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android">
<application/>
<!-- For test coverage -->
<uses-permission
android:name="android.permission.WRITE_EXTERNAL_STORAGE"
android:maxSdkVersion="29" />
<!-- For test coverage on API 29 only -->
<application android:requestLegacyExternalStorage="true" />
</manifest>

View File

@ -7,6 +7,9 @@ This documentation outlines our approach to testing. By running tests against ou
<!-- TODO [#682]: Testing documentation update -->
<!-- TODO [#682]: https://github.com/zcash/secant-android-wallet/issues/682 -->
<!-- TODO [#705] Instrumentation coverage generation fails when run locally -->
<!-- TODO [#705] https://github.com/zcash/secant-android-wallet/issues/705-->
## Manual testing
We aim to automate as much as we possibly can. Still manual testing is really important for Quality Assurance.

View File

@ -20,6 +20,9 @@ IS_KOTLIN_TEST_COVERAGE_ENABLED=true
# Optionally configure Android instumentation test coverage.
# The app module will crash at launch when coverage is enabled, so coverage is only enabled explicitly for tests.
# generation of instrumentation coverage is flaky, particularly when running ui-lib:connectedCheck
# TODO: [#705] Instrumentation coverage generation fails when run locally
# TODO: [#705] https://github.com/zcash/secant-android-wallet/issues/705
IS_ANDROID_INSTRUMENTATION_TEST_COVERAGE_ENABLED=false
# Optionally configure test orchestrator.
@ -131,6 +134,7 @@ ANDROIDX_TEST_CORE_VERSION=1.5.0
ANDROIDX_TEST_MACROBENCHMARK_VERSION=1.2.0-alpha08
ANDROIDX_TEST_RUNNER_VERSION=1.5.1
ANDROIDX_STARTUP_VERSION=1.1.1
ANDROIDX_TEST_SERVICE_VERSION=1.4.2
ANDROIDX_UI_AUTOMATOR_VERSION=2.2.0-alpha1
ANDROIDX_WORK_MANAGER_VERSION=2.7.1
CORE_LIBRARY_DESUGARING_VERSION=1.1.6
@ -138,7 +142,7 @@ JACOCO_VERSION=0.8.8
KOTLIN_VERSION=1.7.21
KOTLINX_COROUTINES_VERSION=1.6.4
KOTLINX_DATETIME_VERSION=0.4.0
KOVER_VERSION=0.5.1
KOVER_VERSION=0.6.1
PLAY_APP_UPDATE_VERSION=2.0.1
PLAY_APP_UPDATE_KTX_VERSION=2.0.1
ZCASH_ANDROID_WALLET_PLUGINS_VERSION=1.0.0

View File

@ -34,6 +34,12 @@ dependencies {
androidTestImplementation(libs.bundles.androidx.test)
androidTestImplementation(libs.kotlinx.coroutines.test)
androidTestUtil(libs.androidx.test.services) {
artifact {
type = "apk"
}
}
if (isOrchestratorEnabled) {
androidTestUtil(libs.androidx.test.orchestrator) {
artifact {

View File

@ -1,6 +1,14 @@
<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android">
<application android:label="zcash-preference-test" />
<!-- For test coverage -->
<uses-permission
android:name="android.permission.WRITE_EXTERNAL_STORAGE"
android:maxSdkVersion="29" />
<!-- For test coverage on API 29 only -->
<application
android:label="zcash-preference-test"
android:requestLegacyExternalStorage="true" />
</manifest>

View File

@ -22,6 +22,12 @@ dependencies {
androidTestImplementation(libs.kotlinx.coroutines.test)
androidTestImplementation(libs.kotlin.test)
androidTestUtil(libs.androidx.test.services) {
artifact {
type = "apk"
}
}
if (project.property("IS_USE_TEST_ORCHESTRATOR").toString().toBoolean()) {
androidTestUtil(libs.androidx.test.orchestrator) {
artifact {

View File

@ -1,7 +1,13 @@
<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android">
<!-- For test coverage -->
<uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE"
android:maxSdkVersion="29"/>
<!-- For test coverage on API 29 only -->
<application
android:label="sdk-ext-test"/>
android:label="sdk-ext-test"
android:requestLegacyExternalStorage="true" />
</manifest>

View File

@ -26,6 +26,12 @@ dependencies {
androidTestImplementation(libs.bundles.androidx.test)
androidTestImplementation(libs.kotlin.test)
androidTestUtil(libs.androidx.test.services) {
artifact {
type = "apk"
}
}
if (project.property("IS_USE_TEST_ORCHESTRATOR").toString().toBoolean()) {
androidTestUtil(libs.androidx.test.orchestrator) {
artifact {

View File

@ -1,7 +1,15 @@
<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android">
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
>
<!-- For test coverage -->
<uses-permission
android:name="android.permission.WRITE_EXTERNAL_STORAGE"
android:maxSdkVersion="29" />
<!-- For test coverage on API 29 only -->
<application
android:label="sdk-ext-ui-test"/>
android:label="sdk-ext-ui-test"
android:requestLegacyExternalStorage="true" />
</manifest>

View File

@ -163,6 +163,7 @@ dependencyResolutionManagement {
val androidxTestJunitVersion = extra["ANDROIDX_TEST_JUNIT_VERSION"].toString()
val androidxTestMacrobenchmarkVersion = extra["ANDROIDX_TEST_MACROBENCHMARK_VERSION"].toString()
val androidxTestOrchestratorVersion = extra["ANDROIDX_TEST_ORCHESTRATOR_VERSION"].toString()
val androidxTestServices = extra["ANDROIDX_TEST_SERVICE_VERSION"].toString()
val androidxTestRunnerVersion = extra["ANDROIDX_TEST_RUNNER_VERSION"].toString()
val androidxUiAutomatorVersion = extra["ANDROIDX_UI_AUTOMATOR_VERSION"].toString()
val androidxWorkManagerVersion = extra["ANDROIDX_WORK_MANAGER_VERSION"].toString()
@ -239,9 +240,9 @@ dependencyResolutionManagement {
library("androidx-test-macrobenchmark", "androidx.benchmark:benchmark-macro-junit4:$androidxTestMacrobenchmarkVersion")
library("androidx-test-orchestrator", "androidx.test:orchestrator:$androidxTestOrchestratorVersion")
library("androidx-test-runner", "androidx.test:runner:$androidxTestRunnerVersion")
library("androidx-test-services","androidx.test.services:test-services:$androidxTestServices")
library("androidx-uiAutomator", "androidx.test.uiautomator:uiautomator-v18:$androidxUiAutomatorVersion")
library("kotlinx-coroutines-test", "org.jetbrains.kotlinx:kotlinx-coroutines-test:$kotlinxCoroutinesVersion")
// Bundles
bundle(
"androidx-camera",

View File

@ -29,6 +29,12 @@ dependencies {
androidTestImplementation(libs.bundles.androidx.test)
androidTestImplementation(libs.kotlinx.coroutines.test)
androidTestUtil(libs.androidx.test.services) {
artifact {
type = "apk"
}
}
androidTestUtil(libs.androidx.test.orchestrator) {
artifact {
type = "apk"

View File

@ -1,6 +1,14 @@
<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android">
<application android:label="zcash-preference-test" />
<!-- For test coverage -->
<uses-permission
android:name="android.permission.WRITE_EXTERNAL_STORAGE"
android:maxSdkVersion="29" />
<!-- For test coverage on API 29 only -->
<application
android:label="zcash-preference-test"
android:requestLegacyExternalStorage="true" />
</manifest>

View File

@ -38,6 +38,12 @@ dependencies {
implementation(libs.bundles.androidx.test)
implementation(libs.androidx.test.macrobenchmark)
androidTestUtil(libs.androidx.test.services) {
artifact {
type = "apk"
}
}
if (project.property("IS_USE_TEST_ORCHESTRATOR").toString().toBoolean()) {
androidTestUtil(libs.androidx.test.orchestrator) {
artifact {

View File

@ -42,6 +42,12 @@ dependencies {
androidTestImplementation(libs.kotlin.reflect)
androidTestImplementation(libs.kotlin.test)
androidTestUtil(libs.androidx.test.services) {
artifact {
type = "apk"
}
}
if (project.property("IS_USE_TEST_ORCHESTRATOR").toString().toBoolean()) {
androidTestUtil(libs.androidx.test.orchestrator) {
artifact {

View File

@ -69,6 +69,12 @@ dependencies {
implementation(libs.androidx.navigation.compose)
implementation(libs.androidx.uiAutomator)
androidTestUtil(libs.androidx.test.services) {
artifact {
type = "apk"
}
}
if (isOrchestratorEnabled) {
androidTestUtil(libs.androidx.test.orchestrator) {
artifact {

View File

@ -1,10 +1,16 @@
<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android">
<!-- For test coverage -->
<uses-permission
android:name="android.permission.WRITE_EXTERNAL_STORAGE"
android:maxSdkVersion="29" />
<application
android:label="zcash-ui-integration-test"
android:icon="@mipmap/ic_launcher_square"
android:roundIcon="@mipmap/ic_launcher_round">
android:roundIcon="@mipmap/ic_launcher_round"
android:requestLegacyExternalStorage="true">
<activity
android:name="co.electriccoin.zcash.ui.integration.test.common.IntegrationTestingActivity"
android:exported="false" />

View File

@ -92,6 +92,12 @@ dependencies {
androidTestImplementation(libs.kotlin.reflect)
androidTestImplementation(libs.kotlin.test)
androidTestUtil(libs.androidx.test.services) {
artifact {
type = "apk"
}
}
if (project.property("IS_USE_TEST_ORCHESTRATOR").toString().toBoolean()) {
androidTestUtil(libs.androidx.test.orchestrator) {
artifact {

View File

@ -2,8 +2,14 @@
<manifest
xmlns:android="http://schemas.android.com/apk/res/android">
<!-- For test coverage -->
<uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE"
android:maxSdkVersion="29"/>
<!-- Legacy external storage is for coverage on API 29 -->
<application
android:label="zcash-ui-test" >
android:label="zcash-ui-test"
android:requestLegacyExternalStorage="true" >
<activity
android:name="co.electriccoin.zcash.ui.common.UiTestingActivity"
android:exported="false" />

View File

@ -70,6 +70,12 @@ dependencies {
implementation(libs.androidx.startup)
implementation(libs.androidx.uiAutomator)
androidTestUtil(libs.androidx.test.services) {
artifact {
type = "apk"
}
}
if (isOrchestratorEnabled) {
androidTestUtil(libs.androidx.test.orchestrator) {
artifact {

View File

@ -1,5 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
<manifest
xmlns:android="http://schemas.android.com/apk/res/android"
xmlns:tools="http://schemas.android.com/tools">
<application