From 9226adea9798cf85eccf6d66f5de200eff69d870 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Fri, 6 May 2022 01:30:38 -0400 Subject: [PATCH] feat(ci): add `sending_transactions_using_lightwalletd` test to CI (#4267) * feat(ci): add lightwalletd_*_sync tests to CI * feat(ci): add lightwalletd RPC call test * feat(ci): add send transactions test with lwd to CI * fix(ci): create a variable to run transactions test * refactor(ci): use docker in docker This is a workaround for an issue related to disk partitioning, caused by a GCP service called Konlet, while mounting the cached disks to the VM and then to the container * fix(build): persist docker login credentials * fix(ci): get sync height from docker logs instead of gcp * try: use gha cache for faster building * fix(ci): mount disk in container to make it available in vm * fix(build): do not invalidate cache between images * try(docker): invalidate cache as less as possible * fix(ci): GHA terminal is not a TTY * fix(build): do not ignore entrypoint.sh * fix * fix(ci): mount using root priveleges * fix(ci): use existing disk as cached state * fix(ci): wait for disks to get mounted * force rebuild * fix failed force * fix failed commit * WIP * fix(ci): some tests does not use a cached state * wip * refactor(ci): disk names and job segregation * fix(ci): do not name boot and attached disk the same * fix(ci): attach a disk to full sync, to snapshot the state * fix(ci): use correct disk implementations * fix(ci): use different disk name to allow test concurrency * feat(ci): add lightwalledt send transaction test * cleanup(ci): remove extra tests * fix(ci): allow disk concurrency with tests * fix(ci): add considerations for different tests * fix(reusable): last fixes * feat(ci): use reusable workflow for tests * fix(rw): remove nested worflow * fix(rw): minor fixes * force rebuild * fix(rw): do not use an input as job name * fix(rw): remove variable id * fix(ci): remove explicit conditions and id * fix(ci): docker does not need the variable sign ($) to work * fix(ci): mount typo * fix(ci): if a sync fails, always delete the instance This also reduces the amount of jobs needed. * refactor(ci): make all test depend on the same build * fix(ci): some tests require multiple variables * fix(docker): variable substitution * fix(ci): allow to run multiple commits from a PR at once * fix(docker): lower the NETWORK env var for test names * reduce uneeded diff * imp(keys): use better naming for builds_disks * imp(ci): use input defaults * imp(ci): remove test_name in favor of test_id * fix(ci): better key naming * fix(ci): long disk names breaks GCP naming convention * feat(ci): validate local state version with cached state * fix(ci): add condition to run tests * fix: typo * fix: app_name should not be required * fix: zebra_state_path shouldn't be required * fix: reduce diff * fix(ci): checkout to grep local state version * Update .github/workflows/test.yml Co-authored-by: teor * revert: merge all tests into a single workflow * Remove unused STATE_VERSION env var * fix: minor fixes * fix(ci): make test.patch the same as test * fix(ci): negate the input value * imp(ci): better cached state conditional handling * imp(ci): exit code is captured by `docker run` * fix(deploy): mount disks with better write performance * fix(ci): change sync id to a broader id name * fix(ci): use correct input validation * fix(ci): do not make test with cached state dependant on other * imp(ci): organiza keys better * fix(ci): use appropiate naming * fix(ci): create docker volume before mounting * fix(lint): do not fail on all new changes * imp(ci): do not report in pr review * fix(ci): partition clean disks * fix: typo * fix: test called the wrong way * fix(build): stop using gha cache * ref(ci): validate run condition before calling reusable workflow * fix(ci): use a better filesystem dir and fix other values * fix: linting errors * fix(ci): typo * Revert "fix(build): stop using gha cache" This reverts commit a8fbc5f416df561e58b388e065d1dc9696983508. Cache expiration is a lesser evil than not using caching at all and then failing with a 401 * imp(ci): do not set a default for needs_zebra_state * Update .github/workflows/test.yml Co-authored-by: teor * fix(deps): remove dependencies * force build * Update .github/workflows/test.yml Co-authored-by: teor * fix(docker): add RUST_LOG as an ARG and ENV * fix(test): add `#[ignore]` to send transactions test This test needs state then it should be marked as #[ignore] * fix(ci): differentiate between root cache path and its dir * Remove extra `state` directory That was a workaround for an issue that has been fixed. * imp(docs): use better test descriptions Co-authored-by: teor * fix: reduce unwanted diff with main * fix(ci): make lwd conditions consistent * Remove another extra `state` directory Was also part of a workaround for an issue that has been fixed. * fix(ci): use better conditionals to run test jobs Co-authored-by: teor * Tweak to support different lightwalletd versions Some versions print `Waiting for block`, and some versions print `Ingestor waiting for block`. Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: teor Co-authored-by: Janito Vaqueiro Ferreira Filho --- .github/workflows/gcp-test-deploy.yml | 10 +++++--- .github/workflows/test.yml | 25 ++++++++++++++++--- docker/entrypoint.sh | 2 ++ zebrad/tests/acceptance.rs | 1 + .../lightwalletd/send_transaction_test.rs | 6 ++--- .../tests/common/lightwalletd/wallet_grpc.rs | 2 +- 6 files changed, 36 insertions(+), 10 deletions(-) diff --git a/.github/workflows/gcp-test-deploy.yml b/.github/workflows/gcp-test-deploy.yml index 2904d301d..27d488f08 100644 --- a/.github/workflows/gcp-test-deploy.yml +++ b/.github/workflows/gcp-test-deploy.yml @@ -19,10 +19,14 @@ on: test_variables: required: true type: string - zebra_state_path: + root_state_path: required: false type: string default: '/zebrad-cache' + zebra_state_dir: + required: false + type: string + default: '' disk_prefix: required: false type: string @@ -110,7 +114,7 @@ jobs: ${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }} \ && \ docker run ${{ inputs.test_variables }} -t --name ${{ inputs.test_id }} \ - --mount type=volume,src=${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }},dst=${{ inputs.zebra_state_path }} \ + --mount type=volume,src=${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }},dst=${{ inputs.root_state_path }}/${{ inputs.zebra_state_dir }} \ ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}" - name: Get state version from constants.rs @@ -243,7 +247,7 @@ jobs: ${{ inputs.disk_prefix }}-${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }} \ && \ docker run ${{ inputs.test_variables }} -t --name ${{ inputs.test_id }} \ - --mount type=volume,src=${{ inputs.disk_prefix }}-${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }},dst=${{ inputs.zebra_state_path }} \ + --mount type=volume,src=${{ inputs.disk_prefix }}-${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }},dst=${{ inputs.root_state_path }}/${{ inputs.zebra_state_dir }} \ ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}" - name: Delete test instance diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cc58f244e..186f40088 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -198,7 +198,7 @@ jobs: test-stateful-sync: needs: build uses: ./.github/workflows/gcp-test-deploy.yml - if: ${{ github.event.inputs.regenerate-disks != 'true' }} + if: ${{ github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' }} with: test_id: sync-past-checkpoint test_description: Test full validation sync from a cached state @@ -228,10 +228,11 @@ jobs: disk_suffix: tip height_grep_text: 'finished initial sync to chain tip, using gossiped blocks sync_percent=100.* current_height=Height' + # Test that Zebra can answer a synthetic RPC call, using a cached Zebra tip state lightwalletd-rpc-test: - if: ${{ github.event.inputs.regenerate-disks != 'true' }} needs: build uses: ./.github/workflows/gcp-test-deploy.yml + if: ${{ github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' }} with: app_name: lightwalletd test_id: fully-synced-rpc @@ -240,4 +241,22 @@ jobs: needs_zebra_state: true saves_to_disk: false disk_suffix: tip - zebra_state_path: '/var/cache' + root_state_path: '/var/cache' + zebra_state_dir: 'zebrad-cache' + + # Test that Zebra can handle a lightwalletd send transaction RPC call, using a cached Zebra tip state + # TODO: use a cached lightwalletd tip state to speed up the test (#4303) + lightwalletd-transactions-test: + needs: build + uses: ./.github/workflows/gcp-test-deploy.yml + if: ${{ github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' }} + with: + app_name: lightwalletd + test_id: lwd-send-transactions + test_description: Test sending transactions via lightwalletd + test_variables: '-e TEST_LWD_TRANSACTIONS=1 -e ZEBRA_TEST_LIGHTWALLETD=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache -e LIGHTWALLETD_DATA_DIR=/var/cache/lightwalletd-cache' + needs_zebra_state: true + saves_to_disk: false + disk_suffix: tip + root_state_path: '/var/cache' + zebra_state_dir: 'zebrad-cache' \ No newline at end of file diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index e2df8cf52..30e2b0bf9 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -14,6 +14,8 @@ case "$1" in exec cargo "test" "--locked" "--release" "--features" "enable-sentry,test_sync_past_mandatory_checkpoint_${NETWORK,,}" "--manifest-path" "zebrad/Cargo.toml" "sync_past_mandatory_checkpoint_${NETWORK,,}" elif [[ "$TEST_LWD_RPC_CALL" -eq "1" ]]; then exec cargo "test" "--locked" "--release" "--features" "enable-sentry" "--test" "acceptance" "--" "--nocapture" "--ignored" "fully_synced_rpc_test" + elif [[ "$TEST_LWD_TRANSACTIONS" -eq "1" ]]; then + exec cargo "test" "--locked" "--release" "--features" "enable-sentry" "--test" "acceptance" "--" "--nocapture" "--ignored" "sending_transactions_using_lightwalletd" else exec "$@" fi diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index 75ca9f62d..6776beff7 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -1526,6 +1526,7 @@ async fn fully_synced_rpc_test() -> Result<()> { /// /// See [`common::lightwalletd::send_transaction_test`] for more information. #[tokio::test] +#[ignore] async fn sending_transactions_using_lightwalletd() -> Result<()> { common::lightwalletd::send_transaction_test::run().await } diff --git a/zebrad/tests/common/lightwalletd/send_transaction_test.rs b/zebrad/tests/common/lightwalletd/send_transaction_test.rs index 7cb2731f3..cfc6b2c75 100644 --- a/zebrad/tests/common/lightwalletd/send_transaction_test.rs +++ b/zebrad/tests/common/lightwalletd/send_transaction_test.rs @@ -141,8 +141,8 @@ async fn prepare_partial_sync( cached_zebra_state: PathBuf, ) -> Result<(TempDir, block::Height)> { let partial_sync_path = copy_state_directory(cached_zebra_state).await?; - let partial_sync_state_dir = partial_sync_path.as_ref().join("state"); - let tip_height = load_tip_height_from_state_directory(network, &partial_sync_state_dir).await?; + let tip_height = + load_tip_height_from_state_directory(network, partial_sync_path.as_ref()).await?; Ok((partial_sync_path, tip_height)) } @@ -162,7 +162,7 @@ async fn load_transactions_from_block_after( state_path: &Path, ) -> Result>> { let (_read_write_state_service, mut state, latest_chain_tip, _chain_tip_change) = - start_state_service_with_cache_dir(network, state_path.join("state")).await?; + start_state_service_with_cache_dir(network, state_path).await?; let tip_height = latest_chain_tip .best_tip_height() diff --git a/zebrad/tests/common/lightwalletd/wallet_grpc.rs b/zebrad/tests/common/lightwalletd/wallet_grpc.rs index 5ed23c8a7..2bb48ef02 100644 --- a/zebrad/tests/common/lightwalletd/wallet_grpc.rs +++ b/zebrad/tests/common/lightwalletd/wallet_grpc.rs @@ -42,7 +42,7 @@ pub fn spawn_lightwalletd_with_rpc_server( lightwalletd.expect_stdout_line_matches("Starting gRPC server")?; if wait_for_blocks { - lightwalletd.expect_stdout_line_matches("Waiting for block")?; + lightwalletd.expect_stdout_line_matches("[Ww]aiting for block")?; } Ok((lightwalletd, lightwalletd_rpc_port)) }