2022-05-09 12:54:16 -07:00
|
|
|
|
name: CI Docker
|
2020-11-19 12:21:03 -08:00
|
|
|
|
|
2022-08-29 17:11:05 -07:00
|
|
|
|
# Ensures that only one workflow task will run at a time. Previous builds, if
|
|
|
|
|
# already in process, will get cancelled. Only the latest commit will be allowed
|
|
|
|
|
# to run, cancelling any workflows in between
|
|
|
|
|
concurrency:
|
|
|
|
|
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
|
|
|
|
|
cancel-in-progress: true
|
|
|
|
|
|
2020-11-19 12:21:03 -08:00
|
|
|
|
on:
|
2023-06-12 18:27:22 -07:00
|
|
|
|
schedule:
|
|
|
|
|
# Run this job every Friday at mid-day UTC
|
|
|
|
|
# This is limited to the Zebra and lightwalletd Full Sync jobs
|
|
|
|
|
# TODO: we should move this behavior to a separate workflow
|
|
|
|
|
- cron: '0 12 * * 5'
|
|
|
|
|
|
2020-11-19 12:21:03 -08:00
|
|
|
|
workflow_dispatch:
|
refactor(test): dockerize tests and run sync in detached mode (#3459)
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): use newer google auth action
* fix (cd): use newer secret as gcp credential
* fix (docker): do not create extra directories
* fix (docker): ignore .github for caching purposes
* fix (docker): use latest rust
* fix (cd): bump build timeout
* fix: use a better name for manual deployment
* refactor (docker): use standard directories for executable
* fix (cd): most systems expect a "latest" tag
Caching from the latest image is one of the main reasons to add this extra tag. Before this commit, the inline cache was not being used.
* fix (cd): push the build image and the cache separately
The inline cache exporter only supports `min` cache mode. To enable `max` cache mode, push the image and the cache separately by using the registry cache exporter.
This also allows for smaller release images.
* fix (cd): remove unused GHA cache
We're leveraging the registry to cache the actions, instead of using the 10GB limits from Github Actions cache storage
* refactor (cd): use cargo-chef for caching rust deps
* fix: move build system deps before cargo cheg cook
* fix (release): use newer debian to reduce vulnerabilities
* fix (cd): use same zone, region and service accounts
* fix (cd): use same disk size and type for all deployments
* refactor (cd): activate interactive shells
Use interactive shells for manual and test deployments. This allow greater flexibility if troubleshooting is needed inside the machines
* refactor (test): use docker artifact from registry
Instead of using a VM to SSH into in to build and test. Build in GHA (to have the logs available), run the workspace tests in GHA, and just run the sync tests in GCP
Use a cintainer VM with zebra's image directly on it, and pass the needed parameters to run the Sync past mandatory checkpoint.
* tmp (cd): bump timeout for building from scratch
* tmp (test): bump build time
* fix (cd, test): bump build time-out to 210 minutes
* fix (docker): do not build with different settings
Compiling might be slow because different steps are compiling the same code 2-4 times because of the variations
* revert (docker): do not fix the rust version
* fix (docker): build on the root directory
* refactor(docker): Use base image commands and tools
* fix (cd): use correct variables & values, add build concurrency
* fix(cd): use Mainnet instead of mainnet
* imp: remove checkout as Buildkit uses the git context
* fix (docker): just Buildkit uses a .dockerignore in a path
* imp (cd): just use needed variables in the right place
* imp (cd): do not checkout if not needed
* test: run on push
* refactor(docker): reduce build changes
* fix(cd): not checking out was limiting some variables
* refactor(test): add an multistage exclusive for testing
* fix(cd): remove tests as a runtime dependency
* fix(cd): use default service account with cloud-platform scope
* fix(cd): revert checkout actions
* fix: use GA c2 instead of Preview c2d machine types
* fix(actions): remove workflow_dispatch from patched actions
This causes GitHub confusion as it can't determined which of the actions using workflow_dispatch is the right one
* fix(actions): remove patches from push actions
* test: validate changes on each push
* fix(test): wrong file syntax on test job
* fix(test): add missing env parameters
* fix(docker): Do not rebuild to download params and run tests
* fix(test): setup gcloud and loginto artifact just when needed
Try not to rebuild the tests
* fix(test): use GCP container to sync past mandatory checkpoint
* fix(test): missing separators
* test
* fix(test): mount the available disk
* push
* refactor(test): merge disk regeneration into test.yml
* fix(cd): minor typo fixes
* fix(docker): rebuild on .github changes
* fix(cd): keep compatibility with gcr.io
To prevent conflicts between registries, and migrate when the time is right, we'll keep pushing to both registries and use github actions cache to prevent conflicts between artifacts.
* fix(cd): typo and scope
* fix(cd): typos everywhere
* refactor(test): use smarter docker wait and keep old registry
* fix(cd): do not constraint the CPUs for bigger machines
* revert(cd): reduce PR diff as there's a separate one for tests
* fix(docker): add .github as it has no impact on caching
* fix(test): run command correctly
* fix(test): wiat and create image if previous step succeded
* force rebuild
* fix(test): do not restrict interdependant steps based on event
* force push
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* fix(test): remove all hardoced values and increase disks
* fix(test): use correct commands on deploy
* fix(test): use args as required by google
* fix(docker): try not to invalidate zebrad download cache
* fix(test): minor typo
* refactor(test): decouple jobs for better modularity
This also allows faster tests as testing Zunstable won't be a dependency and it can't stop already started jobs if it fails.
* fix(test): Do not try to execute ss and commands in one line
* fix(test): do not show undeeded information in the terminal
* fix(test): sleep befor/after machine creation/deletion
* fix(docker): do not download zcash params twice
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* merge: docker-actions-refactor into docker-test-refactor
* test docker wait scenarios
* fix(docker): $HOME variables is not being expanded
* fix(test): allow docker wait to work correctly
* fix(docker): do not use variables while using COPY
* fix(docker): allow to use zebrad as a command
* fix(cd): use test .yml from main
* fix(cd): Do not duplicate network values
The Dockerfile has an ARG with a default value of 'Mainnet', if this value is changed it will be done manually on a workflow_dispatch, making the ENV option a uneeded duplicate in this workflow
* fix(test): use bigger machine type for compute intensive tasks
* refactor(test): add tests in CI file
* fix(test): remove duplicated tests
* fix(test): typo
* test: build on .github changes temporarily
* fix(test): bigger machines have no effect on sync times
* feat: add an image to inherit from with zcash params
* fix(cd): use the right image name and allow push to test
* fix(cd): use the right docker target and remove extra builds
* refactor(docker): use cached zcash params from previous build
* fix(cd): finalize for merging
* imp(cd): add double safety measure for production
* fix(cd): use specific SHA for containers
* fix(cd): use latest gcloud action version
* fix(test): use the network as Mainnet and remove the uppercase from tests
* fix(test): run disk regeneration on specific file change
Just run this regeneration when changing the following files:
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state/disk_format.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/constants.rs
* refactor(test): seggregate disks regeneration from tests
Allow to regenerate disks without running tests, and to run tests from previous disk regeneration.
Disk will be regenerated just if specific files were changed, or triggered manually.
Tests will run just if a disk regeneration was not manually triggered.
* fix(test): gcp disks require lower case conventions
* fix(test): validate logs being emmited by docker
GHA is transforming is somehow transforwing the variable to lowercase also, so we're changint it to adapt to it
* test
* fix(test): force tty terminal
* fix(test): use a one line command to test terminal output
* fix(test): always delete test instance
* fix(test): use short SHA from the PR head
Using the SHA from the base, creates confusion and it's not accurate with the SHA being shown and used on GitHub.
We have to keep both as manual runs with `workflow_dispatch` does not have a PR SHA
* fix(ci): do not trigger CI on docker changes
There's no impact in this workflow when a change is done in the dockerfile
* Instead of runing cargo test when the instance gets created, run this commands afterwards in a different step.
As GHA TTY is not working as expected, and workarounds does not play nicely with `gcloud compute ssh` actions/runner#241 (comment) we decided to get the container name from the logs, log directly to the container and run the cargo command from there.
* doc(test): document reasoning for new steps
* fix(test): increase machine type and ssh timeout
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
* finalize(test): do not rebuild image when changing actions
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
2022-02-15 16:54:16 -08:00
|
|
|
|
inputs:
|
|
|
|
|
network:
|
|
|
|
|
default: 'Mainnet'
|
2022-04-11 22:06:37 -07:00
|
|
|
|
description: 'Network to deploy: Mainnet or Testnet'
|
|
|
|
|
required: true
|
refactor(test): dockerize tests and run sync in detached mode (#3459)
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): use newer google auth action
* fix (cd): use newer secret as gcp credential
* fix (docker): do not create extra directories
* fix (docker): ignore .github for caching purposes
* fix (docker): use latest rust
* fix (cd): bump build timeout
* fix: use a better name for manual deployment
* refactor (docker): use standard directories for executable
* fix (cd): most systems expect a "latest" tag
Caching from the latest image is one of the main reasons to add this extra tag. Before this commit, the inline cache was not being used.
* fix (cd): push the build image and the cache separately
The inline cache exporter only supports `min` cache mode. To enable `max` cache mode, push the image and the cache separately by using the registry cache exporter.
This also allows for smaller release images.
* fix (cd): remove unused GHA cache
We're leveraging the registry to cache the actions, instead of using the 10GB limits from Github Actions cache storage
* refactor (cd): use cargo-chef for caching rust deps
* fix: move build system deps before cargo cheg cook
* fix (release): use newer debian to reduce vulnerabilities
* fix (cd): use same zone, region and service accounts
* fix (cd): use same disk size and type for all deployments
* refactor (cd): activate interactive shells
Use interactive shells for manual and test deployments. This allow greater flexibility if troubleshooting is needed inside the machines
* refactor (test): use docker artifact from registry
Instead of using a VM to SSH into in to build and test. Build in GHA (to have the logs available), run the workspace tests in GHA, and just run the sync tests in GCP
Use a cintainer VM with zebra's image directly on it, and pass the needed parameters to run the Sync past mandatory checkpoint.
* tmp (cd): bump timeout for building from scratch
* tmp (test): bump build time
* fix (cd, test): bump build time-out to 210 minutes
* fix (docker): do not build with different settings
Compiling might be slow because different steps are compiling the same code 2-4 times because of the variations
* revert (docker): do not fix the rust version
* fix (docker): build on the root directory
* refactor(docker): Use base image commands and tools
* fix (cd): use correct variables & values, add build concurrency
* fix(cd): use Mainnet instead of mainnet
* imp: remove checkout as Buildkit uses the git context
* fix (docker): just Buildkit uses a .dockerignore in a path
* imp (cd): just use needed variables in the right place
* imp (cd): do not checkout if not needed
* test: run on push
* refactor(docker): reduce build changes
* fix(cd): not checking out was limiting some variables
* refactor(test): add an multistage exclusive for testing
* fix(cd): remove tests as a runtime dependency
* fix(cd): use default service account with cloud-platform scope
* fix(cd): revert checkout actions
* fix: use GA c2 instead of Preview c2d machine types
* fix(actions): remove workflow_dispatch from patched actions
This causes GitHub confusion as it can't determined which of the actions using workflow_dispatch is the right one
* fix(actions): remove patches from push actions
* test: validate changes on each push
* fix(test): wrong file syntax on test job
* fix(test): add missing env parameters
* fix(docker): Do not rebuild to download params and run tests
* fix(test): setup gcloud and loginto artifact just when needed
Try not to rebuild the tests
* fix(test): use GCP container to sync past mandatory checkpoint
* fix(test): missing separators
* test
* fix(test): mount the available disk
* push
* refactor(test): merge disk regeneration into test.yml
* fix(cd): minor typo fixes
* fix(docker): rebuild on .github changes
* fix(cd): keep compatibility with gcr.io
To prevent conflicts between registries, and migrate when the time is right, we'll keep pushing to both registries and use github actions cache to prevent conflicts between artifacts.
* fix(cd): typo and scope
* fix(cd): typos everywhere
* refactor(test): use smarter docker wait and keep old registry
* fix(cd): do not constraint the CPUs for bigger machines
* revert(cd): reduce PR diff as there's a separate one for tests
* fix(docker): add .github as it has no impact on caching
* fix(test): run command correctly
* fix(test): wiat and create image if previous step succeded
* force rebuild
* fix(test): do not restrict interdependant steps based on event
* force push
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* fix(test): remove all hardoced values and increase disks
* fix(test): use correct commands on deploy
* fix(test): use args as required by google
* fix(docker): try not to invalidate zebrad download cache
* fix(test): minor typo
* refactor(test): decouple jobs for better modularity
This also allows faster tests as testing Zunstable won't be a dependency and it can't stop already started jobs if it fails.
* fix(test): Do not try to execute ss and commands in one line
* fix(test): do not show undeeded information in the terminal
* fix(test): sleep befor/after machine creation/deletion
* fix(docker): do not download zcash params twice
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* merge: docker-actions-refactor into docker-test-refactor
* test docker wait scenarios
* fix(docker): $HOME variables is not being expanded
* fix(test): allow docker wait to work correctly
* fix(docker): do not use variables while using COPY
* fix(docker): allow to use zebrad as a command
* fix(cd): use test .yml from main
* fix(cd): Do not duplicate network values
The Dockerfile has an ARG with a default value of 'Mainnet', if this value is changed it will be done manually on a workflow_dispatch, making the ENV option a uneeded duplicate in this workflow
* fix(test): use bigger machine type for compute intensive tasks
* refactor(test): add tests in CI file
* fix(test): remove duplicated tests
* fix(test): typo
* test: build on .github changes temporarily
* fix(test): bigger machines have no effect on sync times
* feat: add an image to inherit from with zcash params
* fix(cd): use the right image name and allow push to test
* fix(cd): use the right docker target and remove extra builds
* refactor(docker): use cached zcash params from previous build
* fix(cd): finalize for merging
* imp(cd): add double safety measure for production
* fix(cd): use specific SHA for containers
* fix(cd): use latest gcloud action version
* fix(test): use the network as Mainnet and remove the uppercase from tests
* fix(test): run disk regeneration on specific file change
Just run this regeneration when changing the following files:
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state/disk_format.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/constants.rs
* refactor(test): seggregate disks regeneration from tests
Allow to regenerate disks without running tests, and to run tests from previous disk regeneration.
Disk will be regenerated just if specific files were changed, or triggered manually.
Tests will run just if a disk regeneration was not manually triggered.
* fix(test): gcp disks require lower case conventions
* fix(test): validate logs being emmited by docker
GHA is transforming is somehow transforwing the variable to lowercase also, so we're changint it to adapt to it
* test
* fix(test): force tty terminal
* fix(test): use a one line command to test terminal output
* fix(test): always delete test instance
* fix(test): use short SHA from the PR head
Using the SHA from the base, creates confusion and it's not accurate with the SHA being shown and used on GitHub.
We have to keep both as manual runs with `workflow_dispatch` does not have a PR SHA
* fix(ci): do not trigger CI on docker changes
There's no impact in this workflow when a change is done in the dockerfile
* Instead of runing cargo test when the instance gets created, run this commands afterwards in a different step.
As GHA TTY is not working as expected, and workarounds does not play nicely with `gcloud compute ssh` actions/runner#241 (comment) we decided to get the container name from the logs, log directly to the container and run the cargo command from there.
* doc(test): document reasoning for new steps
* fix(test): increase machine type and ssh timeout
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
* finalize(test): do not rebuild image when changing actions
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
2022-02-15 16:54:16 -08:00
|
|
|
|
regenerate-disks:
|
|
|
|
|
type: boolean
|
|
|
|
|
default: false
|
2022-10-19 05:01:26 -07:00
|
|
|
|
description: 'Just run a Zebra checkpoint sync and update checkpoint disks'
|
2022-04-11 22:06:37 -07:00
|
|
|
|
required: true
|
2022-05-04 15:55:02 -07:00
|
|
|
|
run-full-sync:
|
|
|
|
|
type: boolean
|
|
|
|
|
default: false
|
2023-05-03 16:14:22 -07:00
|
|
|
|
description: 'Just run a Zebra full sync on `network`, and update tip disks'
|
2022-10-19 05:01:26 -07:00
|
|
|
|
required: true
|
|
|
|
|
run-lwd-sync:
|
|
|
|
|
type: boolean
|
|
|
|
|
default: false
|
|
|
|
|
description: 'Just run a lightwalletd full sync and update tip disks'
|
2022-05-04 15:55:02 -07:00
|
|
|
|
required: true
|
2023-07-13 14:36:15 -07:00
|
|
|
|
force_save_to_disk:
|
|
|
|
|
required: false
|
|
|
|
|
type: boolean
|
|
|
|
|
default: false
|
|
|
|
|
description: 'Force tests to always create a cached state disk, if they already create disks'
|
2023-06-07 22:44:30 -07:00
|
|
|
|
no_cache:
|
|
|
|
|
description: 'Disable the Docker cache for this build'
|
|
|
|
|
required: false
|
|
|
|
|
type: boolean
|
|
|
|
|
default: false
|
2022-10-19 05:01:26 -07:00
|
|
|
|
|
refactor(test): dockerize tests and run sync in detached mode (#3459)
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): use newer google auth action
* fix (cd): use newer secret as gcp credential
* fix (docker): do not create extra directories
* fix (docker): ignore .github for caching purposes
* fix (docker): use latest rust
* fix (cd): bump build timeout
* fix: use a better name for manual deployment
* refactor (docker): use standard directories for executable
* fix (cd): most systems expect a "latest" tag
Caching from the latest image is one of the main reasons to add this extra tag. Before this commit, the inline cache was not being used.
* fix (cd): push the build image and the cache separately
The inline cache exporter only supports `min` cache mode. To enable `max` cache mode, push the image and the cache separately by using the registry cache exporter.
This also allows for smaller release images.
* fix (cd): remove unused GHA cache
We're leveraging the registry to cache the actions, instead of using the 10GB limits from Github Actions cache storage
* refactor (cd): use cargo-chef for caching rust deps
* fix: move build system deps before cargo cheg cook
* fix (release): use newer debian to reduce vulnerabilities
* fix (cd): use same zone, region and service accounts
* fix (cd): use same disk size and type for all deployments
* refactor (cd): activate interactive shells
Use interactive shells for manual and test deployments. This allow greater flexibility if troubleshooting is needed inside the machines
* refactor (test): use docker artifact from registry
Instead of using a VM to SSH into in to build and test. Build in GHA (to have the logs available), run the workspace tests in GHA, and just run the sync tests in GCP
Use a cintainer VM with zebra's image directly on it, and pass the needed parameters to run the Sync past mandatory checkpoint.
* tmp (cd): bump timeout for building from scratch
* tmp (test): bump build time
* fix (cd, test): bump build time-out to 210 minutes
* fix (docker): do not build with different settings
Compiling might be slow because different steps are compiling the same code 2-4 times because of the variations
* revert (docker): do not fix the rust version
* fix (docker): build on the root directory
* refactor(docker): Use base image commands and tools
* fix (cd): use correct variables & values, add build concurrency
* fix(cd): use Mainnet instead of mainnet
* imp: remove checkout as Buildkit uses the git context
* fix (docker): just Buildkit uses a .dockerignore in a path
* imp (cd): just use needed variables in the right place
* imp (cd): do not checkout if not needed
* test: run on push
* refactor(docker): reduce build changes
* fix(cd): not checking out was limiting some variables
* refactor(test): add an multistage exclusive for testing
* fix(cd): remove tests as a runtime dependency
* fix(cd): use default service account with cloud-platform scope
* fix(cd): revert checkout actions
* fix: use GA c2 instead of Preview c2d machine types
* fix(actions): remove workflow_dispatch from patched actions
This causes GitHub confusion as it can't determined which of the actions using workflow_dispatch is the right one
* fix(actions): remove patches from push actions
* test: validate changes on each push
* fix(test): wrong file syntax on test job
* fix(test): add missing env parameters
* fix(docker): Do not rebuild to download params and run tests
* fix(test): setup gcloud and loginto artifact just when needed
Try not to rebuild the tests
* fix(test): use GCP container to sync past mandatory checkpoint
* fix(test): missing separators
* test
* fix(test): mount the available disk
* push
* refactor(test): merge disk regeneration into test.yml
* fix(cd): minor typo fixes
* fix(docker): rebuild on .github changes
* fix(cd): keep compatibility with gcr.io
To prevent conflicts between registries, and migrate when the time is right, we'll keep pushing to both registries and use github actions cache to prevent conflicts between artifacts.
* fix(cd): typo and scope
* fix(cd): typos everywhere
* refactor(test): use smarter docker wait and keep old registry
* fix(cd): do not constraint the CPUs for bigger machines
* revert(cd): reduce PR diff as there's a separate one for tests
* fix(docker): add .github as it has no impact on caching
* fix(test): run command correctly
* fix(test): wiat and create image if previous step succeded
* force rebuild
* fix(test): do not restrict interdependant steps based on event
* force push
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* fix(test): remove all hardoced values and increase disks
* fix(test): use correct commands on deploy
* fix(test): use args as required by google
* fix(docker): try not to invalidate zebrad download cache
* fix(test): minor typo
* refactor(test): decouple jobs for better modularity
This also allows faster tests as testing Zunstable won't be a dependency and it can't stop already started jobs if it fails.
* fix(test): Do not try to execute ss and commands in one line
* fix(test): do not show undeeded information in the terminal
* fix(test): sleep befor/after machine creation/deletion
* fix(docker): do not download zcash params twice
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* merge: docker-actions-refactor into docker-test-refactor
* test docker wait scenarios
* fix(docker): $HOME variables is not being expanded
* fix(test): allow docker wait to work correctly
* fix(docker): do not use variables while using COPY
* fix(docker): allow to use zebrad as a command
* fix(cd): use test .yml from main
* fix(cd): Do not duplicate network values
The Dockerfile has an ARG with a default value of 'Mainnet', if this value is changed it will be done manually on a workflow_dispatch, making the ENV option a uneeded duplicate in this workflow
* fix(test): use bigger machine type for compute intensive tasks
* refactor(test): add tests in CI file
* fix(test): remove duplicated tests
* fix(test): typo
* test: build on .github changes temporarily
* fix(test): bigger machines have no effect on sync times
* feat: add an image to inherit from with zcash params
* fix(cd): use the right image name and allow push to test
* fix(cd): use the right docker target and remove extra builds
* refactor(docker): use cached zcash params from previous build
* fix(cd): finalize for merging
* imp(cd): add double safety measure for production
* fix(cd): use specific SHA for containers
* fix(cd): use latest gcloud action version
* fix(test): use the network as Mainnet and remove the uppercase from tests
* fix(test): run disk regeneration on specific file change
Just run this regeneration when changing the following files:
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state/disk_format.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/constants.rs
* refactor(test): seggregate disks regeneration from tests
Allow to regenerate disks without running tests, and to run tests from previous disk regeneration.
Disk will be regenerated just if specific files were changed, or triggered manually.
Tests will run just if a disk regeneration was not manually triggered.
* fix(test): gcp disks require lower case conventions
* fix(test): validate logs being emmited by docker
GHA is transforming is somehow transforwing the variable to lowercase also, so we're changint it to adapt to it
* test
* fix(test): force tty terminal
* fix(test): use a one line command to test terminal output
* fix(test): always delete test instance
* fix(test): use short SHA from the PR head
Using the SHA from the base, creates confusion and it's not accurate with the SHA being shown and used on GitHub.
We have to keep both as manual runs with `workflow_dispatch` does not have a PR SHA
* fix(ci): do not trigger CI on docker changes
There's no impact in this workflow when a change is done in the dockerfile
* Instead of runing cargo test when the instance gets created, run this commands afterwards in a different step.
As GHA TTY is not working as expected, and workarounds does not play nicely with `gcloud compute ssh` actions/runner#241 (comment) we decided to get the container name from the logs, log directly to the container and run the cargo command from there.
* doc(test): document reasoning for new steps
* fix(test): increase machine type and ssh timeout
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
* finalize(test): do not rebuild image when changing actions
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
2022-02-15 16:54:16 -08:00
|
|
|
|
pull_request:
|
2022-02-11 06:05:19 -08:00
|
|
|
|
paths:
|
2022-03-02 06:23:05 -08:00
|
|
|
|
# code and tests
|
2022-01-25 03:58:11 -08:00
|
|
|
|
- '**/*.rs'
|
2022-03-04 11:55:49 -08:00
|
|
|
|
# hard-coded checkpoints and proptest regressions
|
2022-01-25 03:58:11 -08:00
|
|
|
|
- '**/*.txt'
|
2022-03-02 06:23:05 -08:00
|
|
|
|
# test data snapshots
|
|
|
|
|
- '**/*.snap'
|
|
|
|
|
# dependencies
|
2022-01-25 03:58:11 -08:00
|
|
|
|
- '**/Cargo.toml'
|
|
|
|
|
- '**/Cargo.lock'
|
2022-08-26 11:06:32 -07:00
|
|
|
|
# configuration files
|
|
|
|
|
- '.cargo/config.toml'
|
|
|
|
|
- '**/clippy.toml'
|
2022-03-02 06:23:05 -08:00
|
|
|
|
# workflow definitions
|
2022-01-25 03:58:11 -08:00
|
|
|
|
- 'docker/**'
|
2022-05-09 12:54:16 -07:00
|
|
|
|
- '.github/workflows/continous-integration-docker.yml'
|
|
|
|
|
- '.github/workflows/deploy-gcp-tests.yml'
|
|
|
|
|
- '.github/workflows/build-docker-image.yml'
|
2023-05-03 05:04:16 -07:00
|
|
|
|
- '.github/workflows/find-cached-disks.yml'
|
2020-11-19 12:21:03 -08:00
|
|
|
|
|
2022-04-13 18:44:39 -07:00
|
|
|
|
push:
|
|
|
|
|
branches:
|
|
|
|
|
- main
|
|
|
|
|
paths:
|
|
|
|
|
# code and tests
|
|
|
|
|
- '**/*.rs'
|
|
|
|
|
# hard-coded checkpoints and proptest regressions
|
|
|
|
|
- '**/*.txt'
|
|
|
|
|
# test data snapshots
|
|
|
|
|
- '**/*.snap'
|
|
|
|
|
# dependencies
|
|
|
|
|
- '**/Cargo.toml'
|
|
|
|
|
- '**/Cargo.lock'
|
2022-11-23 06:00:11 -08:00
|
|
|
|
# configuration files
|
|
|
|
|
- '.cargo/config.toml'
|
|
|
|
|
- '**/clippy.toml'
|
2022-04-13 18:44:39 -07:00
|
|
|
|
# workflow definitions
|
|
|
|
|
- 'docker/**'
|
2023-06-25 16:24:10 -07:00
|
|
|
|
- '.dockerignore'
|
2022-05-09 12:54:16 -07:00
|
|
|
|
- '.github/workflows/continous-integration-docker.yml'
|
|
|
|
|
- '.github/workflows/deploy-gcp-tests.yml'
|
2023-05-03 05:04:16 -07:00
|
|
|
|
- '.github/workflows/find-cached-disks.yml'
|
2023-06-25 16:24:10 -07:00
|
|
|
|
- '.github/workflows/build-docker-image.yml'
|
2022-04-13 18:44:39 -07:00
|
|
|
|
|
2020-11-19 12:21:03 -08:00
|
|
|
|
jobs:
|
2023-06-12 18:27:22 -07:00
|
|
|
|
# to also run a job on Mergify head branches,
|
|
|
|
|
# add `|| (github.event_name == 'push' && startsWith(github.head_ref, 'mergify/merge-queue/'))`:
|
|
|
|
|
# https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#running-your-workflow-based-on-the-head-or-base-branch-of-a-pull-request-1
|
2022-05-24 11:59:31 -07:00
|
|
|
|
|
2023-05-03 05:04:16 -07:00
|
|
|
|
# Check if the cached state disks used by the tests are available for the default network.
|
|
|
|
|
#
|
2023-05-03 16:14:22 -07:00
|
|
|
|
# The default network is mainnet unless a manually triggered workflow or repository variable
|
2023-05-03 05:04:16 -07:00
|
|
|
|
# is configured differently.
|
|
|
|
|
#
|
|
|
|
|
# The outputs for this job have the same names as the workflow outputs in find-cached-disks.yml
|
|
|
|
|
get-available-disks:
|
|
|
|
|
name: Check if cached state disks exist for ${{ inputs.network || vars.ZCASH_NETWORK }}
|
|
|
|
|
uses: ./.github/workflows/find-cached-disks.yml
|
|
|
|
|
with:
|
|
|
|
|
network: ${{ inputs.network || vars.ZCASH_NETWORK }}
|
2022-05-19 16:07:28 -07:00
|
|
|
|
|
2023-05-03 16:14:22 -07:00
|
|
|
|
# Check if the cached state disks used by the tests are available for testnet.
|
2023-05-03 05:04:16 -07:00
|
|
|
|
#
|
|
|
|
|
# The outputs for this job have the same names as the workflow outputs in find-cached-disks.yml
|
2023-05-03 16:14:22 -07:00
|
|
|
|
# Some outputs are ignored, because we don't run those jobs on testnet.
|
2023-05-03 05:04:16 -07:00
|
|
|
|
get-available-disks-testnet:
|
2023-05-03 16:14:22 -07:00
|
|
|
|
name: Check if cached state disks exist for testnet
|
2023-05-03 05:04:16 -07:00
|
|
|
|
uses: ./.github/workflows/find-cached-disks.yml
|
|
|
|
|
with:
|
|
|
|
|
network: 'Testnet'
|
2022-05-04 15:55:02 -07:00
|
|
|
|
|
2023-05-03 05:04:16 -07:00
|
|
|
|
# Build the docker image used by the tests.
|
|
|
|
|
#
|
2023-05-03 16:14:22 -07:00
|
|
|
|
# The default network in the Zebra config in the image is mainnet, unless a manually triggered
|
2023-05-03 05:04:16 -07:00
|
|
|
|
# workflow or repository variable is configured differently. Testnet jobs change that config to
|
|
|
|
|
# testnet when running the image.
|
refactor(test): dockerize tests and run sync in detached mode (#3459)
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): use newer google auth action
* fix (cd): use newer secret as gcp credential
* fix (docker): do not create extra directories
* fix (docker): ignore .github for caching purposes
* fix (docker): use latest rust
* fix (cd): bump build timeout
* fix: use a better name for manual deployment
* refactor (docker): use standard directories for executable
* fix (cd): most systems expect a "latest" tag
Caching from the latest image is one of the main reasons to add this extra tag. Before this commit, the inline cache was not being used.
* fix (cd): push the build image and the cache separately
The inline cache exporter only supports `min` cache mode. To enable `max` cache mode, push the image and the cache separately by using the registry cache exporter.
This also allows for smaller release images.
* fix (cd): remove unused GHA cache
We're leveraging the registry to cache the actions, instead of using the 10GB limits from Github Actions cache storage
* refactor (cd): use cargo-chef for caching rust deps
* fix: move build system deps before cargo cheg cook
* fix (release): use newer debian to reduce vulnerabilities
* fix (cd): use same zone, region and service accounts
* fix (cd): use same disk size and type for all deployments
* refactor (cd): activate interactive shells
Use interactive shells for manual and test deployments. This allow greater flexibility if troubleshooting is needed inside the machines
* refactor (test): use docker artifact from registry
Instead of using a VM to SSH into in to build and test. Build in GHA (to have the logs available), run the workspace tests in GHA, and just run the sync tests in GCP
Use a cintainer VM with zebra's image directly on it, and pass the needed parameters to run the Sync past mandatory checkpoint.
* tmp (cd): bump timeout for building from scratch
* tmp (test): bump build time
* fix (cd, test): bump build time-out to 210 minutes
* fix (docker): do not build with different settings
Compiling might be slow because different steps are compiling the same code 2-4 times because of the variations
* revert (docker): do not fix the rust version
* fix (docker): build on the root directory
* refactor(docker): Use base image commands and tools
* fix (cd): use correct variables & values, add build concurrency
* fix(cd): use Mainnet instead of mainnet
* imp: remove checkout as Buildkit uses the git context
* fix (docker): just Buildkit uses a .dockerignore in a path
* imp (cd): just use needed variables in the right place
* imp (cd): do not checkout if not needed
* test: run on push
* refactor(docker): reduce build changes
* fix(cd): not checking out was limiting some variables
* refactor(test): add an multistage exclusive for testing
* fix(cd): remove tests as a runtime dependency
* fix(cd): use default service account with cloud-platform scope
* fix(cd): revert checkout actions
* fix: use GA c2 instead of Preview c2d machine types
* fix(actions): remove workflow_dispatch from patched actions
This causes GitHub confusion as it can't determined which of the actions using workflow_dispatch is the right one
* fix(actions): remove patches from push actions
* test: validate changes on each push
* fix(test): wrong file syntax on test job
* fix(test): add missing env parameters
* fix(docker): Do not rebuild to download params and run tests
* fix(test): setup gcloud and loginto artifact just when needed
Try not to rebuild the tests
* fix(test): use GCP container to sync past mandatory checkpoint
* fix(test): missing separators
* test
* fix(test): mount the available disk
* push
* refactor(test): merge disk regeneration into test.yml
* fix(cd): minor typo fixes
* fix(docker): rebuild on .github changes
* fix(cd): keep compatibility with gcr.io
To prevent conflicts between registries, and migrate when the time is right, we'll keep pushing to both registries and use github actions cache to prevent conflicts between artifacts.
* fix(cd): typo and scope
* fix(cd): typos everywhere
* refactor(test): use smarter docker wait and keep old registry
* fix(cd): do not constraint the CPUs for bigger machines
* revert(cd): reduce PR diff as there's a separate one for tests
* fix(docker): add .github as it has no impact on caching
* fix(test): run command correctly
* fix(test): wiat and create image if previous step succeded
* force rebuild
* fix(test): do not restrict interdependant steps based on event
* force push
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* fix(test): remove all hardoced values and increase disks
* fix(test): use correct commands on deploy
* fix(test): use args as required by google
* fix(docker): try not to invalidate zebrad download cache
* fix(test): minor typo
* refactor(test): decouple jobs for better modularity
This also allows faster tests as testing Zunstable won't be a dependency and it can't stop already started jobs if it fails.
* fix(test): Do not try to execute ss and commands in one line
* fix(test): do not show undeeded information in the terminal
* fix(test): sleep befor/after machine creation/deletion
* fix(docker): do not download zcash params twice
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* merge: docker-actions-refactor into docker-test-refactor
* test docker wait scenarios
* fix(docker): $HOME variables is not being expanded
* fix(test): allow docker wait to work correctly
* fix(docker): do not use variables while using COPY
* fix(docker): allow to use zebrad as a command
* fix(cd): use test .yml from main
* fix(cd): Do not duplicate network values
The Dockerfile has an ARG with a default value of 'Mainnet', if this value is changed it will be done manually on a workflow_dispatch, making the ENV option a uneeded duplicate in this workflow
* fix(test): use bigger machine type for compute intensive tasks
* refactor(test): add tests in CI file
* fix(test): remove duplicated tests
* fix(test): typo
* test: build on .github changes temporarily
* fix(test): bigger machines have no effect on sync times
* feat: add an image to inherit from with zcash params
* fix(cd): use the right image name and allow push to test
* fix(cd): use the right docker target and remove extra builds
* refactor(docker): use cached zcash params from previous build
* fix(cd): finalize for merging
* imp(cd): add double safety measure for production
* fix(cd): use specific SHA for containers
* fix(cd): use latest gcloud action version
* fix(test): use the network as Mainnet and remove the uppercase from tests
* fix(test): run disk regeneration on specific file change
Just run this regeneration when changing the following files:
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state/disk_format.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/constants.rs
* refactor(test): seggregate disks regeneration from tests
Allow to regenerate disks without running tests, and to run tests from previous disk regeneration.
Disk will be regenerated just if specific files were changed, or triggered manually.
Tests will run just if a disk regeneration was not manually triggered.
* fix(test): gcp disks require lower case conventions
* fix(test): validate logs being emmited by docker
GHA is transforming is somehow transforwing the variable to lowercase also, so we're changint it to adapt to it
* test
* fix(test): force tty terminal
* fix(test): use a one line command to test terminal output
* fix(test): always delete test instance
* fix(test): use short SHA from the PR head
Using the SHA from the base, creates confusion and it's not accurate with the SHA being shown and used on GitHub.
We have to keep both as manual runs with `workflow_dispatch` does not have a PR SHA
* fix(ci): do not trigger CI on docker changes
There's no impact in this workflow when a change is done in the dockerfile
* Instead of runing cargo test when the instance gets created, run this commands afterwards in a different step.
As GHA TTY is not working as expected, and workarounds does not play nicely with `gcloud compute ssh` actions/runner#241 (comment) we decided to get the container name from the logs, log directly to the container and run the cargo command from there.
* doc(test): document reasoning for new steps
* fix(test): increase machine type and ssh timeout
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
* finalize(test): do not rebuild image when changing actions
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
2022-02-15 16:54:16 -08:00
|
|
|
|
build:
|
2022-11-06 17:59:32 -08:00
|
|
|
|
name: Build CI Docker
|
2022-05-09 12:54:16 -07:00
|
|
|
|
uses: ./.github/workflows/build-docker-image.yml
|
2022-04-28 01:13:35 -07:00
|
|
|
|
with:
|
|
|
|
|
dockerfile_path: ./docker/Dockerfile
|
2022-05-10 14:00:09 -07:00
|
|
|
|
dockerfile_target: tests
|
2023-04-12 23:56:21 -07:00
|
|
|
|
image_name: ${{ vars.CI_IMAGE_NAME }}
|
2023-06-07 22:44:30 -07:00
|
|
|
|
no_cache: ${{ inputs.no_cache || false }}
|
2022-04-28 01:13:35 -07:00
|
|
|
|
rust_backtrace: full
|
|
|
|
|
rust_lib_backtrace: full
|
|
|
|
|
rust_log: info
|
refactor(test): dockerize tests and run sync in detached mode (#3459)
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): use newer google auth action
* fix (cd): use newer secret as gcp credential
* fix (docker): do not create extra directories
* fix (docker): ignore .github for caching purposes
* fix (docker): use latest rust
* fix (cd): bump build timeout
* fix: use a better name for manual deployment
* refactor (docker): use standard directories for executable
* fix (cd): most systems expect a "latest" tag
Caching from the latest image is one of the main reasons to add this extra tag. Before this commit, the inline cache was not being used.
* fix (cd): push the build image and the cache separately
The inline cache exporter only supports `min` cache mode. To enable `max` cache mode, push the image and the cache separately by using the registry cache exporter.
This also allows for smaller release images.
* fix (cd): remove unused GHA cache
We're leveraging the registry to cache the actions, instead of using the 10GB limits from Github Actions cache storage
* refactor (cd): use cargo-chef for caching rust deps
* fix: move build system deps before cargo cheg cook
* fix (release): use newer debian to reduce vulnerabilities
* fix (cd): use same zone, region and service accounts
* fix (cd): use same disk size and type for all deployments
* refactor (cd): activate interactive shells
Use interactive shells for manual and test deployments. This allow greater flexibility if troubleshooting is needed inside the machines
* refactor (test): use docker artifact from registry
Instead of using a VM to SSH into in to build and test. Build in GHA (to have the logs available), run the workspace tests in GHA, and just run the sync tests in GCP
Use a cintainer VM with zebra's image directly on it, and pass the needed parameters to run the Sync past mandatory checkpoint.
* tmp (cd): bump timeout for building from scratch
* tmp (test): bump build time
* fix (cd, test): bump build time-out to 210 minutes
* fix (docker): do not build with different settings
Compiling might be slow because different steps are compiling the same code 2-4 times because of the variations
* revert (docker): do not fix the rust version
* fix (docker): build on the root directory
* refactor(docker): Use base image commands and tools
* fix (cd): use correct variables & values, add build concurrency
* fix(cd): use Mainnet instead of mainnet
* imp: remove checkout as Buildkit uses the git context
* fix (docker): just Buildkit uses a .dockerignore in a path
* imp (cd): just use needed variables in the right place
* imp (cd): do not checkout if not needed
* test: run on push
* refactor(docker): reduce build changes
* fix(cd): not checking out was limiting some variables
* refactor(test): add an multistage exclusive for testing
* fix(cd): remove tests as a runtime dependency
* fix(cd): use default service account with cloud-platform scope
* fix(cd): revert checkout actions
* fix: use GA c2 instead of Preview c2d machine types
* fix(actions): remove workflow_dispatch from patched actions
This causes GitHub confusion as it can't determined which of the actions using workflow_dispatch is the right one
* fix(actions): remove patches from push actions
* test: validate changes on each push
* fix(test): wrong file syntax on test job
* fix(test): add missing env parameters
* fix(docker): Do not rebuild to download params and run tests
* fix(test): setup gcloud and loginto artifact just when needed
Try not to rebuild the tests
* fix(test): use GCP container to sync past mandatory checkpoint
* fix(test): missing separators
* test
* fix(test): mount the available disk
* push
* refactor(test): merge disk regeneration into test.yml
* fix(cd): minor typo fixes
* fix(docker): rebuild on .github changes
* fix(cd): keep compatibility with gcr.io
To prevent conflicts between registries, and migrate when the time is right, we'll keep pushing to both registries and use github actions cache to prevent conflicts between artifacts.
* fix(cd): typo and scope
* fix(cd): typos everywhere
* refactor(test): use smarter docker wait and keep old registry
* fix(cd): do not constraint the CPUs for bigger machines
* revert(cd): reduce PR diff as there's a separate one for tests
* fix(docker): add .github as it has no impact on caching
* fix(test): run command correctly
* fix(test): wiat and create image if previous step succeded
* force rebuild
* fix(test): do not restrict interdependant steps based on event
* force push
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* fix(test): remove all hardoced values and increase disks
* fix(test): use correct commands on deploy
* fix(test): use args as required by google
* fix(docker): try not to invalidate zebrad download cache
* fix(test): minor typo
* refactor(test): decouple jobs for better modularity
This also allows faster tests as testing Zunstable won't be a dependency and it can't stop already started jobs if it fails.
* fix(test): Do not try to execute ss and commands in one line
* fix(test): do not show undeeded information in the terminal
* fix(test): sleep befor/after machine creation/deletion
* fix(docker): do not download zcash params twice
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* merge: docker-actions-refactor into docker-test-refactor
* test docker wait scenarios
* fix(docker): $HOME variables is not being expanded
* fix(test): allow docker wait to work correctly
* fix(docker): do not use variables while using COPY
* fix(docker): allow to use zebrad as a command
* fix(cd): use test .yml from main
* fix(cd): Do not duplicate network values
The Dockerfile has an ARG with a default value of 'Mainnet', if this value is changed it will be done manually on a workflow_dispatch, making the ENV option a uneeded duplicate in this workflow
* fix(test): use bigger machine type for compute intensive tasks
* refactor(test): add tests in CI file
* fix(test): remove duplicated tests
* fix(test): typo
* test: build on .github changes temporarily
* fix(test): bigger machines have no effect on sync times
* feat: add an image to inherit from with zcash params
* fix(cd): use the right image name and allow push to test
* fix(cd): use the right docker target and remove extra builds
* refactor(docker): use cached zcash params from previous build
* fix(cd): finalize for merging
* imp(cd): add double safety measure for production
* fix(cd): use specific SHA for containers
* fix(cd): use latest gcloud action version
* fix(test): use the network as Mainnet and remove the uppercase from tests
* fix(test): run disk regeneration on specific file change
Just run this regeneration when changing the following files:
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state/disk_format.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/constants.rs
* refactor(test): seggregate disks regeneration from tests
Allow to regenerate disks without running tests, and to run tests from previous disk regeneration.
Disk will be regenerated just if specific files were changed, or triggered manually.
Tests will run just if a disk regeneration was not manually triggered.
* fix(test): gcp disks require lower case conventions
* fix(test): validate logs being emmited by docker
GHA is transforming is somehow transforwing the variable to lowercase also, so we're changint it to adapt to it
* test
* fix(test): force tty terminal
* fix(test): use a one line command to test terminal output
* fix(test): always delete test instance
* fix(test): use short SHA from the PR head
Using the SHA from the base, creates confusion and it's not accurate with the SHA being shown and used on GitHub.
We have to keep both as manual runs with `workflow_dispatch` does not have a PR SHA
* fix(ci): do not trigger CI on docker changes
There's no impact in this workflow when a change is done in the dockerfile
* Instead of runing cargo test when the instance gets created, run this commands afterwards in a different step.
As GHA TTY is not working as expected, and workarounds does not play nicely with `gcloud compute ssh` actions/runner#241 (comment) we decided to get the container name from the logs, log directly to the container and run the cargo command from there.
* doc(test): document reasoning for new steps
* fix(test): increase machine type and ssh timeout
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
* finalize(test): do not rebuild image when changing actions
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
2022-02-15 16:54:16 -08:00
|
|
|
|
|
2022-09-06 22:06:34 -07:00
|
|
|
|
# zebrad tests without cached state
|
|
|
|
|
|
2023-07-10 11:20:50 -07:00
|
|
|
|
# TODO: make the non-cached-state tests use:
|
|
|
|
|
# network: ${{ inputs.network || vars.ZCASH_NETWORK }}
|
|
|
|
|
|
2022-05-05 14:52:56 -07:00
|
|
|
|
# Run all the zebra tests, including tests that are ignored by default.
|
|
|
|
|
# Skips tests that need a cached state disk or a lightwalletd binary.
|
2022-05-11 07:06:58 -07:00
|
|
|
|
#
|
2022-10-24 16:38:48 -07:00
|
|
|
|
# - We run all the tests behind the `getblocktemplate-rpcs` feature as a separated step.
|
|
|
|
|
# - We activate the gRPC feature to avoid recompiling `zebrad`, but we don't actually run any gRPC tests.
|
2022-11-03 23:57:36 -07:00
|
|
|
|
#
|
|
|
|
|
# TODO: turn this test and the getblocktemplate test into a matrix, so the jobs use exactly the same diagnostics settings
|
refactor(test): dockerize tests and run sync in detached mode (#3459)
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): use newer google auth action
* fix (cd): use newer secret as gcp credential
* fix (docker): do not create extra directories
* fix (docker): ignore .github for caching purposes
* fix (docker): use latest rust
* fix (cd): bump build timeout
* fix: use a better name for manual deployment
* refactor (docker): use standard directories for executable
* fix (cd): most systems expect a "latest" tag
Caching from the latest image is one of the main reasons to add this extra tag. Before this commit, the inline cache was not being used.
* fix (cd): push the build image and the cache separately
The inline cache exporter only supports `min` cache mode. To enable `max` cache mode, push the image and the cache separately by using the registry cache exporter.
This also allows for smaller release images.
* fix (cd): remove unused GHA cache
We're leveraging the registry to cache the actions, instead of using the 10GB limits from Github Actions cache storage
* refactor (cd): use cargo-chef for caching rust deps
* fix: move build system deps before cargo cheg cook
* fix (release): use newer debian to reduce vulnerabilities
* fix (cd): use same zone, region and service accounts
* fix (cd): use same disk size and type for all deployments
* refactor (cd): activate interactive shells
Use interactive shells for manual and test deployments. This allow greater flexibility if troubleshooting is needed inside the machines
* refactor (test): use docker artifact from registry
Instead of using a VM to SSH into in to build and test. Build in GHA (to have the logs available), run the workspace tests in GHA, and just run the sync tests in GCP
Use a cintainer VM with zebra's image directly on it, and pass the needed parameters to run the Sync past mandatory checkpoint.
* tmp (cd): bump timeout for building from scratch
* tmp (test): bump build time
* fix (cd, test): bump build time-out to 210 minutes
* fix (docker): do not build with different settings
Compiling might be slow because different steps are compiling the same code 2-4 times because of the variations
* revert (docker): do not fix the rust version
* fix (docker): build on the root directory
* refactor(docker): Use base image commands and tools
* fix (cd): use correct variables & values, add build concurrency
* fix(cd): use Mainnet instead of mainnet
* imp: remove checkout as Buildkit uses the git context
* fix (docker): just Buildkit uses a .dockerignore in a path
* imp (cd): just use needed variables in the right place
* imp (cd): do not checkout if not needed
* test: run on push
* refactor(docker): reduce build changes
* fix(cd): not checking out was limiting some variables
* refactor(test): add an multistage exclusive for testing
* fix(cd): remove tests as a runtime dependency
* fix(cd): use default service account with cloud-platform scope
* fix(cd): revert checkout actions
* fix: use GA c2 instead of Preview c2d machine types
* fix(actions): remove workflow_dispatch from patched actions
This causes GitHub confusion as it can't determined which of the actions using workflow_dispatch is the right one
* fix(actions): remove patches from push actions
* test: validate changes on each push
* fix(test): wrong file syntax on test job
* fix(test): add missing env parameters
* fix(docker): Do not rebuild to download params and run tests
* fix(test): setup gcloud and loginto artifact just when needed
Try not to rebuild the tests
* fix(test): use GCP container to sync past mandatory checkpoint
* fix(test): missing separators
* test
* fix(test): mount the available disk
* push
* refactor(test): merge disk regeneration into test.yml
* fix(cd): minor typo fixes
* fix(docker): rebuild on .github changes
* fix(cd): keep compatibility with gcr.io
To prevent conflicts between registries, and migrate when the time is right, we'll keep pushing to both registries and use github actions cache to prevent conflicts between artifacts.
* fix(cd): typo and scope
* fix(cd): typos everywhere
* refactor(test): use smarter docker wait and keep old registry
* fix(cd): do not constraint the CPUs for bigger machines
* revert(cd): reduce PR diff as there's a separate one for tests
* fix(docker): add .github as it has no impact on caching
* fix(test): run command correctly
* fix(test): wiat and create image if previous step succeded
* force rebuild
* fix(test): do not restrict interdependant steps based on event
* force push
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* fix(test): remove all hardoced values and increase disks
* fix(test): use correct commands on deploy
* fix(test): use args as required by google
* fix(docker): try not to invalidate zebrad download cache
* fix(test): minor typo
* refactor(test): decouple jobs for better modularity
This also allows faster tests as testing Zunstable won't be a dependency and it can't stop already started jobs if it fails.
* fix(test): Do not try to execute ss and commands in one line
* fix(test): do not show undeeded information in the terminal
* fix(test): sleep befor/after machine creation/deletion
* fix(docker): do not download zcash params twice
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* merge: docker-actions-refactor into docker-test-refactor
* test docker wait scenarios
* fix(docker): $HOME variables is not being expanded
* fix(test): allow docker wait to work correctly
* fix(docker): do not use variables while using COPY
* fix(docker): allow to use zebrad as a command
* fix(cd): use test .yml from main
* fix(cd): Do not duplicate network values
The Dockerfile has an ARG with a default value of 'Mainnet', if this value is changed it will be done manually on a workflow_dispatch, making the ENV option a uneeded duplicate in this workflow
* fix(test): use bigger machine type for compute intensive tasks
* refactor(test): add tests in CI file
* fix(test): remove duplicated tests
* fix(test): typo
* test: build on .github changes temporarily
* fix(test): bigger machines have no effect on sync times
* feat: add an image to inherit from with zcash params
* fix(cd): use the right image name and allow push to test
* fix(cd): use the right docker target and remove extra builds
* refactor(docker): use cached zcash params from previous build
* fix(cd): finalize for merging
* imp(cd): add double safety measure for production
* fix(cd): use specific SHA for containers
* fix(cd): use latest gcloud action version
* fix(test): use the network as Mainnet and remove the uppercase from tests
* fix(test): run disk regeneration on specific file change
Just run this regeneration when changing the following files:
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state/disk_format.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/constants.rs
* refactor(test): seggregate disks regeneration from tests
Allow to regenerate disks without running tests, and to run tests from previous disk regeneration.
Disk will be regenerated just if specific files were changed, or triggered manually.
Tests will run just if a disk regeneration was not manually triggered.
* fix(test): gcp disks require lower case conventions
* fix(test): validate logs being emmited by docker
GHA is transforming is somehow transforwing the variable to lowercase also, so we're changint it to adapt to it
* test
* fix(test): force tty terminal
* fix(test): use a one line command to test terminal output
* fix(test): always delete test instance
* fix(test): use short SHA from the PR head
Using the SHA from the base, creates confusion and it's not accurate with the SHA being shown and used on GitHub.
We have to keep both as manual runs with `workflow_dispatch` does not have a PR SHA
* fix(ci): do not trigger CI on docker changes
There's no impact in this workflow when a change is done in the dockerfile
* Instead of runing cargo test when the instance gets created, run this commands afterwards in a different step.
As GHA TTY is not working as expected, and workarounds does not play nicely with `gcloud compute ssh` actions/runner#241 (comment) we decided to get the container name from the logs, log directly to the container and run the cargo command from there.
* doc(test): document reasoning for new steps
* fix(test): increase machine type and ssh timeout
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
* finalize(test): do not rebuild image when changing actions
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
2022-02-15 16:54:16 -08:00
|
|
|
|
test-all:
|
|
|
|
|
name: Test all
|
|
|
|
|
runs-on: ubuntu-latest
|
|
|
|
|
needs: build
|
2023-01-31 12:39:11 -08:00
|
|
|
|
if: ${{ github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
|
refactor(test): dockerize tests and run sync in detached mode (#3459)
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): use newer google auth action
* fix (cd): use newer secret as gcp credential
* fix (docker): do not create extra directories
* fix (docker): ignore .github for caching purposes
* fix (docker): use latest rust
* fix (cd): bump build timeout
* fix: use a better name for manual deployment
* refactor (docker): use standard directories for executable
* fix (cd): most systems expect a "latest" tag
Caching from the latest image is one of the main reasons to add this extra tag. Before this commit, the inline cache was not being used.
* fix (cd): push the build image and the cache separately
The inline cache exporter only supports `min` cache mode. To enable `max` cache mode, push the image and the cache separately by using the registry cache exporter.
This also allows for smaller release images.
* fix (cd): remove unused GHA cache
We're leveraging the registry to cache the actions, instead of using the 10GB limits from Github Actions cache storage
* refactor (cd): use cargo-chef for caching rust deps
* fix: move build system deps before cargo cheg cook
* fix (release): use newer debian to reduce vulnerabilities
* fix (cd): use same zone, region and service accounts
* fix (cd): use same disk size and type for all deployments
* refactor (cd): activate interactive shells
Use interactive shells for manual and test deployments. This allow greater flexibility if troubleshooting is needed inside the machines
* refactor (test): use docker artifact from registry
Instead of using a VM to SSH into in to build and test. Build in GHA (to have the logs available), run the workspace tests in GHA, and just run the sync tests in GCP
Use a cintainer VM with zebra's image directly on it, and pass the needed parameters to run the Sync past mandatory checkpoint.
* tmp (cd): bump timeout for building from scratch
* tmp (test): bump build time
* fix (cd, test): bump build time-out to 210 minutes
* fix (docker): do not build with different settings
Compiling might be slow because different steps are compiling the same code 2-4 times because of the variations
* revert (docker): do not fix the rust version
* fix (docker): build on the root directory
* refactor(docker): Use base image commands and tools
* fix (cd): use correct variables & values, add build concurrency
* fix(cd): use Mainnet instead of mainnet
* imp: remove checkout as Buildkit uses the git context
* fix (docker): just Buildkit uses a .dockerignore in a path
* imp (cd): just use needed variables in the right place
* imp (cd): do not checkout if not needed
* test: run on push
* refactor(docker): reduce build changes
* fix(cd): not checking out was limiting some variables
* refactor(test): add an multistage exclusive for testing
* fix(cd): remove tests as a runtime dependency
* fix(cd): use default service account with cloud-platform scope
* fix(cd): revert checkout actions
* fix: use GA c2 instead of Preview c2d machine types
* fix(actions): remove workflow_dispatch from patched actions
This causes GitHub confusion as it can't determined which of the actions using workflow_dispatch is the right one
* fix(actions): remove patches from push actions
* test: validate changes on each push
* fix(test): wrong file syntax on test job
* fix(test): add missing env parameters
* fix(docker): Do not rebuild to download params and run tests
* fix(test): setup gcloud and loginto artifact just when needed
Try not to rebuild the tests
* fix(test): use GCP container to sync past mandatory checkpoint
* fix(test): missing separators
* test
* fix(test): mount the available disk
* push
* refactor(test): merge disk regeneration into test.yml
* fix(cd): minor typo fixes
* fix(docker): rebuild on .github changes
* fix(cd): keep compatibility with gcr.io
To prevent conflicts between registries, and migrate when the time is right, we'll keep pushing to both registries and use github actions cache to prevent conflicts between artifacts.
* fix(cd): typo and scope
* fix(cd): typos everywhere
* refactor(test): use smarter docker wait and keep old registry
* fix(cd): do not constraint the CPUs for bigger machines
* revert(cd): reduce PR diff as there's a separate one for tests
* fix(docker): add .github as it has no impact on caching
* fix(test): run command correctly
* fix(test): wiat and create image if previous step succeded
* force rebuild
* fix(test): do not restrict interdependant steps based on event
* force push
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* fix(test): remove all hardoced values and increase disks
* fix(test): use correct commands on deploy
* fix(test): use args as required by google
* fix(docker): try not to invalidate zebrad download cache
* fix(test): minor typo
* refactor(test): decouple jobs for better modularity
This also allows faster tests as testing Zunstable won't be a dependency and it can't stop already started jobs if it fails.
* fix(test): Do not try to execute ss and commands in one line
* fix(test): do not show undeeded information in the terminal
* fix(test): sleep befor/after machine creation/deletion
* fix(docker): do not download zcash params twice
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* merge: docker-actions-refactor into docker-test-refactor
* test docker wait scenarios
* fix(docker): $HOME variables is not being expanded
* fix(test): allow docker wait to work correctly
* fix(docker): do not use variables while using COPY
* fix(docker): allow to use zebrad as a command
* fix(cd): use test .yml from main
* fix(cd): Do not duplicate network values
The Dockerfile has an ARG with a default value of 'Mainnet', if this value is changed it will be done manually on a workflow_dispatch, making the ENV option a uneeded duplicate in this workflow
* fix(test): use bigger machine type for compute intensive tasks
* refactor(test): add tests in CI file
* fix(test): remove duplicated tests
* fix(test): typo
* test: build on .github changes temporarily
* fix(test): bigger machines have no effect on sync times
* feat: add an image to inherit from with zcash params
* fix(cd): use the right image name and allow push to test
* fix(cd): use the right docker target and remove extra builds
* refactor(docker): use cached zcash params from previous build
* fix(cd): finalize for merging
* imp(cd): add double safety measure for production
* fix(cd): use specific SHA for containers
* fix(cd): use latest gcloud action version
* fix(test): use the network as Mainnet and remove the uppercase from tests
* fix(test): run disk regeneration on specific file change
Just run this regeneration when changing the following files:
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state/disk_format.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/constants.rs
* refactor(test): seggregate disks regeneration from tests
Allow to regenerate disks without running tests, and to run tests from previous disk regeneration.
Disk will be regenerated just if specific files were changed, or triggered manually.
Tests will run just if a disk regeneration was not manually triggered.
* fix(test): gcp disks require lower case conventions
* fix(test): validate logs being emmited by docker
GHA is transforming is somehow transforwing the variable to lowercase also, so we're changint it to adapt to it
* test
* fix(test): force tty terminal
* fix(test): use a one line command to test terminal output
* fix(test): always delete test instance
* fix(test): use short SHA from the PR head
Using the SHA from the base, creates confusion and it's not accurate with the SHA being shown and used on GitHub.
We have to keep both as manual runs with `workflow_dispatch` does not have a PR SHA
* fix(ci): do not trigger CI on docker changes
There's no impact in this workflow when a change is done in the dockerfile
* Instead of runing cargo test when the instance gets created, run this commands afterwards in a different step.
As GHA TTY is not working as expected, and workarounds does not play nicely with `gcloud compute ssh` actions/runner#241 (comment) we decided to get the container name from the logs, log directly to the container and run the cargo command from there.
* doc(test): document reasoning for new steps
* fix(test): increase machine type and ssh timeout
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
* finalize(test): do not rebuild image when changing actions
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
2022-02-15 16:54:16 -08:00
|
|
|
|
steps:
|
2023-05-18 09:44:39 -07:00
|
|
|
|
- uses: r7kamura/rust-problem-matchers@v1.3.0
|
|
|
|
|
|
refactor(test): dockerize tests and run sync in detached mode (#3459)
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): use newer google auth action
* fix (cd): use newer secret as gcp credential
* fix (docker): do not create extra directories
* fix (docker): ignore .github for caching purposes
* fix (docker): use latest rust
* fix (cd): bump build timeout
* fix: use a better name for manual deployment
* refactor (docker): use standard directories for executable
* fix (cd): most systems expect a "latest" tag
Caching from the latest image is one of the main reasons to add this extra tag. Before this commit, the inline cache was not being used.
* fix (cd): push the build image and the cache separately
The inline cache exporter only supports `min` cache mode. To enable `max` cache mode, push the image and the cache separately by using the registry cache exporter.
This also allows for smaller release images.
* fix (cd): remove unused GHA cache
We're leveraging the registry to cache the actions, instead of using the 10GB limits from Github Actions cache storage
* refactor (cd): use cargo-chef for caching rust deps
* fix: move build system deps before cargo cheg cook
* fix (release): use newer debian to reduce vulnerabilities
* fix (cd): use same zone, region and service accounts
* fix (cd): use same disk size and type for all deployments
* refactor (cd): activate interactive shells
Use interactive shells for manual and test deployments. This allow greater flexibility if troubleshooting is needed inside the machines
* refactor (test): use docker artifact from registry
Instead of using a VM to SSH into in to build and test. Build in GHA (to have the logs available), run the workspace tests in GHA, and just run the sync tests in GCP
Use a cintainer VM with zebra's image directly on it, and pass the needed parameters to run the Sync past mandatory checkpoint.
* tmp (cd): bump timeout for building from scratch
* tmp (test): bump build time
* fix (cd, test): bump build time-out to 210 minutes
* fix (docker): do not build with different settings
Compiling might be slow because different steps are compiling the same code 2-4 times because of the variations
* revert (docker): do not fix the rust version
* fix (docker): build on the root directory
* refactor(docker): Use base image commands and tools
* fix (cd): use correct variables & values, add build concurrency
* fix(cd): use Mainnet instead of mainnet
* imp: remove checkout as Buildkit uses the git context
* fix (docker): just Buildkit uses a .dockerignore in a path
* imp (cd): just use needed variables in the right place
* imp (cd): do not checkout if not needed
* test: run on push
* refactor(docker): reduce build changes
* fix(cd): not checking out was limiting some variables
* refactor(test): add an multistage exclusive for testing
* fix(cd): remove tests as a runtime dependency
* fix(cd): use default service account with cloud-platform scope
* fix(cd): revert checkout actions
* fix: use GA c2 instead of Preview c2d machine types
* fix(actions): remove workflow_dispatch from patched actions
This causes GitHub confusion as it can't determined which of the actions using workflow_dispatch is the right one
* fix(actions): remove patches from push actions
* test: validate changes on each push
* fix(test): wrong file syntax on test job
* fix(test): add missing env parameters
* fix(docker): Do not rebuild to download params and run tests
* fix(test): setup gcloud and loginto artifact just when needed
Try not to rebuild the tests
* fix(test): use GCP container to sync past mandatory checkpoint
* fix(test): missing separators
* test
* fix(test): mount the available disk
* push
* refactor(test): merge disk regeneration into test.yml
* fix(cd): minor typo fixes
* fix(docker): rebuild on .github changes
* fix(cd): keep compatibility with gcr.io
To prevent conflicts between registries, and migrate when the time is right, we'll keep pushing to both registries and use github actions cache to prevent conflicts between artifacts.
* fix(cd): typo and scope
* fix(cd): typos everywhere
* refactor(test): use smarter docker wait and keep old registry
* fix(cd): do not constraint the CPUs for bigger machines
* revert(cd): reduce PR diff as there's a separate one for tests
* fix(docker): add .github as it has no impact on caching
* fix(test): run command correctly
* fix(test): wiat and create image if previous step succeded
* force rebuild
* fix(test): do not restrict interdependant steps based on event
* force push
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* fix(test): remove all hardoced values and increase disks
* fix(test): use correct commands on deploy
* fix(test): use args as required by google
* fix(docker): try not to invalidate zebrad download cache
* fix(test): minor typo
* refactor(test): decouple jobs for better modularity
This also allows faster tests as testing Zunstable won't be a dependency and it can't stop already started jobs if it fails.
* fix(test): Do not try to execute ss and commands in one line
* fix(test): do not show undeeded information in the terminal
* fix(test): sleep befor/after machine creation/deletion
* fix(docker): do not download zcash params twice
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* merge: docker-actions-refactor into docker-test-refactor
* test docker wait scenarios
* fix(docker): $HOME variables is not being expanded
* fix(test): allow docker wait to work correctly
* fix(docker): do not use variables while using COPY
* fix(docker): allow to use zebrad as a command
* fix(cd): use test .yml from main
* fix(cd): Do not duplicate network values
The Dockerfile has an ARG with a default value of 'Mainnet', if this value is changed it will be done manually on a workflow_dispatch, making the ENV option a uneeded duplicate in this workflow
* fix(test): use bigger machine type for compute intensive tasks
* refactor(test): add tests in CI file
* fix(test): remove duplicated tests
* fix(test): typo
* test: build on .github changes temporarily
* fix(test): bigger machines have no effect on sync times
* feat: add an image to inherit from with zcash params
* fix(cd): use the right image name and allow push to test
* fix(cd): use the right docker target and remove extra builds
* refactor(docker): use cached zcash params from previous build
* fix(cd): finalize for merging
* imp(cd): add double safety measure for production
* fix(cd): use specific SHA for containers
* fix(cd): use latest gcloud action version
* fix(test): use the network as Mainnet and remove the uppercase from tests
* fix(test): run disk regeneration on specific file change
Just run this regeneration when changing the following files:
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state/disk_format.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/constants.rs
* refactor(test): seggregate disks regeneration from tests
Allow to regenerate disks without running tests, and to run tests from previous disk regeneration.
Disk will be regenerated just if specific files were changed, or triggered manually.
Tests will run just if a disk regeneration was not manually triggered.
* fix(test): gcp disks require lower case conventions
* fix(test): validate logs being emmited by docker
GHA is transforming is somehow transforwing the variable to lowercase also, so we're changint it to adapt to it
* test
* fix(test): force tty terminal
* fix(test): use a one line command to test terminal output
* fix(test): always delete test instance
* fix(test): use short SHA from the PR head
Using the SHA from the base, creates confusion and it's not accurate with the SHA being shown and used on GitHub.
We have to keep both as manual runs with `workflow_dispatch` does not have a PR SHA
* fix(ci): do not trigger CI on docker changes
There's no impact in this workflow when a change is done in the dockerfile
* Instead of runing cargo test when the instance gets created, run this commands afterwards in a different step.
As GHA TTY is not working as expected, and workarounds does not play nicely with `gcloud compute ssh` actions/runner#241 (comment) we decided to get the container name from the logs, log directly to the container and run the cargo command from there.
* doc(test): document reasoning for new steps
* fix(test): increase machine type and ssh timeout
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
* finalize(test): do not rebuild image when changing actions
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
2022-02-15 16:54:16 -08:00
|
|
|
|
- name: Inject slug/short variables
|
|
|
|
|
uses: rlespinasse/github-slug-action@v4
|
2022-03-21 16:07:01 -07:00
|
|
|
|
with:
|
|
|
|
|
short-length: 7
|
refactor(test): dockerize tests and run sync in detached mode (#3459)
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): use newer google auth action
* fix (cd): use newer secret as gcp credential
* fix (docker): do not create extra directories
* fix (docker): ignore .github for caching purposes
* fix (docker): use latest rust
* fix (cd): bump build timeout
* fix: use a better name for manual deployment
* refactor (docker): use standard directories for executable
* fix (cd): most systems expect a "latest" tag
Caching from the latest image is one of the main reasons to add this extra tag. Before this commit, the inline cache was not being used.
* fix (cd): push the build image and the cache separately
The inline cache exporter only supports `min` cache mode. To enable `max` cache mode, push the image and the cache separately by using the registry cache exporter.
This also allows for smaller release images.
* fix (cd): remove unused GHA cache
We're leveraging the registry to cache the actions, instead of using the 10GB limits from Github Actions cache storage
* refactor (cd): use cargo-chef for caching rust deps
* fix: move build system deps before cargo cheg cook
* fix (release): use newer debian to reduce vulnerabilities
* fix (cd): use same zone, region and service accounts
* fix (cd): use same disk size and type for all deployments
* refactor (cd): activate interactive shells
Use interactive shells for manual and test deployments. This allow greater flexibility if troubleshooting is needed inside the machines
* refactor (test): use docker artifact from registry
Instead of using a VM to SSH into in to build and test. Build in GHA (to have the logs available), run the workspace tests in GHA, and just run the sync tests in GCP
Use a cintainer VM with zebra's image directly on it, and pass the needed parameters to run the Sync past mandatory checkpoint.
* tmp (cd): bump timeout for building from scratch
* tmp (test): bump build time
* fix (cd, test): bump build time-out to 210 minutes
* fix (docker): do not build with different settings
Compiling might be slow because different steps are compiling the same code 2-4 times because of the variations
* revert (docker): do not fix the rust version
* fix (docker): build on the root directory
* refactor(docker): Use base image commands and tools
* fix (cd): use correct variables & values, add build concurrency
* fix(cd): use Mainnet instead of mainnet
* imp: remove checkout as Buildkit uses the git context
* fix (docker): just Buildkit uses a .dockerignore in a path
* imp (cd): just use needed variables in the right place
* imp (cd): do not checkout if not needed
* test: run on push
* refactor(docker): reduce build changes
* fix(cd): not checking out was limiting some variables
* refactor(test): add an multistage exclusive for testing
* fix(cd): remove tests as a runtime dependency
* fix(cd): use default service account with cloud-platform scope
* fix(cd): revert checkout actions
* fix: use GA c2 instead of Preview c2d machine types
* fix(actions): remove workflow_dispatch from patched actions
This causes GitHub confusion as it can't determined which of the actions using workflow_dispatch is the right one
* fix(actions): remove patches from push actions
* test: validate changes on each push
* fix(test): wrong file syntax on test job
* fix(test): add missing env parameters
* fix(docker): Do not rebuild to download params and run tests
* fix(test): setup gcloud and loginto artifact just when needed
Try not to rebuild the tests
* fix(test): use GCP container to sync past mandatory checkpoint
* fix(test): missing separators
* test
* fix(test): mount the available disk
* push
* refactor(test): merge disk regeneration into test.yml
* fix(cd): minor typo fixes
* fix(docker): rebuild on .github changes
* fix(cd): keep compatibility with gcr.io
To prevent conflicts between registries, and migrate when the time is right, we'll keep pushing to both registries and use github actions cache to prevent conflicts between artifacts.
* fix(cd): typo and scope
* fix(cd): typos everywhere
* refactor(test): use smarter docker wait and keep old registry
* fix(cd): do not constraint the CPUs for bigger machines
* revert(cd): reduce PR diff as there's a separate one for tests
* fix(docker): add .github as it has no impact on caching
* fix(test): run command correctly
* fix(test): wiat and create image if previous step succeded
* force rebuild
* fix(test): do not restrict interdependant steps based on event
* force push
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* fix(test): remove all hardoced values and increase disks
* fix(test): use correct commands on deploy
* fix(test): use args as required by google
* fix(docker): try not to invalidate zebrad download cache
* fix(test): minor typo
* refactor(test): decouple jobs for better modularity
This also allows faster tests as testing Zunstable won't be a dependency and it can't stop already started jobs if it fails.
* fix(test): Do not try to execute ss and commands in one line
* fix(test): do not show undeeded information in the terminal
* fix(test): sleep befor/after machine creation/deletion
* fix(docker): do not download zcash params twice
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* merge: docker-actions-refactor into docker-test-refactor
* test docker wait scenarios
* fix(docker): $HOME variables is not being expanded
* fix(test): allow docker wait to work correctly
* fix(docker): do not use variables while using COPY
* fix(docker): allow to use zebrad as a command
* fix(cd): use test .yml from main
* fix(cd): Do not duplicate network values
The Dockerfile has an ARG with a default value of 'Mainnet', if this value is changed it will be done manually on a workflow_dispatch, making the ENV option a uneeded duplicate in this workflow
* fix(test): use bigger machine type for compute intensive tasks
* refactor(test): add tests in CI file
* fix(test): remove duplicated tests
* fix(test): typo
* test: build on .github changes temporarily
* fix(test): bigger machines have no effect on sync times
* feat: add an image to inherit from with zcash params
* fix(cd): use the right image name and allow push to test
* fix(cd): use the right docker target and remove extra builds
* refactor(docker): use cached zcash params from previous build
* fix(cd): finalize for merging
* imp(cd): add double safety measure for production
* fix(cd): use specific SHA for containers
* fix(cd): use latest gcloud action version
* fix(test): use the network as Mainnet and remove the uppercase from tests
* fix(test): run disk regeneration on specific file change
Just run this regeneration when changing the following files:
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state/disk_format.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/constants.rs
* refactor(test): seggregate disks regeneration from tests
Allow to regenerate disks without running tests, and to run tests from previous disk regeneration.
Disk will be regenerated just if specific files were changed, or triggered manually.
Tests will run just if a disk regeneration was not manually triggered.
* fix(test): gcp disks require lower case conventions
* fix(test): validate logs being emmited by docker
GHA is transforming is somehow transforwing the variable to lowercase also, so we're changint it to adapt to it
* test
* fix(test): force tty terminal
* fix(test): use a one line command to test terminal output
* fix(test): always delete test instance
* fix(test): use short SHA from the PR head
Using the SHA from the base, creates confusion and it's not accurate with the SHA being shown and used on GitHub.
We have to keep both as manual runs with `workflow_dispatch` does not have a PR SHA
* fix(ci): do not trigger CI on docker changes
There's no impact in this workflow when a change is done in the dockerfile
* Instead of runing cargo test when the instance gets created, run this commands afterwards in a different step.
As GHA TTY is not working as expected, and workarounds does not play nicely with `gcloud compute ssh` actions/runner#241 (comment) we decided to get the container name from the logs, log directly to the container and run the cargo command from there.
* doc(test): document reasoning for new steps
* fix(test): increase machine type and ssh timeout
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
* finalize(test): do not rebuild image when changing actions
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
2022-02-15 16:54:16 -08:00
|
|
|
|
|
2022-12-07 22:11:46 -08:00
|
|
|
|
# Run unit, basic acceptance tests, and ignored tests, only showing command output if the test fails.
|
2022-11-03 23:57:36 -07:00
|
|
|
|
#
|
|
|
|
|
# If some tests hang, add "-- --nocapture" for just that test, or for all the tests.
|
2022-12-07 22:11:46 -08:00
|
|
|
|
- name: Run zebrad tests
|
refactor(test): dockerize tests and run sync in detached mode (#3459)
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): use newer google auth action
* fix (cd): use newer secret as gcp credential
* fix (docker): do not create extra directories
* fix (docker): ignore .github for caching purposes
* fix (docker): use latest rust
* fix (cd): bump build timeout
* fix: use a better name for manual deployment
* refactor (docker): use standard directories for executable
* fix (cd): most systems expect a "latest" tag
Caching from the latest image is one of the main reasons to add this extra tag. Before this commit, the inline cache was not being used.
* fix (cd): push the build image and the cache separately
The inline cache exporter only supports `min` cache mode. To enable `max` cache mode, push the image and the cache separately by using the registry cache exporter.
This also allows for smaller release images.
* fix (cd): remove unused GHA cache
We're leveraging the registry to cache the actions, instead of using the 10GB limits from Github Actions cache storage
* refactor (cd): use cargo-chef for caching rust deps
* fix: move build system deps before cargo cheg cook
* fix (release): use newer debian to reduce vulnerabilities
* fix (cd): use same zone, region and service accounts
* fix (cd): use same disk size and type for all deployments
* refactor (cd): activate interactive shells
Use interactive shells for manual and test deployments. This allow greater flexibility if troubleshooting is needed inside the machines
* refactor (test): use docker artifact from registry
Instead of using a VM to SSH into in to build and test. Build in GHA (to have the logs available), run the workspace tests in GHA, and just run the sync tests in GCP
Use a cintainer VM with zebra's image directly on it, and pass the needed parameters to run the Sync past mandatory checkpoint.
* tmp (cd): bump timeout for building from scratch
* tmp (test): bump build time
* fix (cd, test): bump build time-out to 210 minutes
* fix (docker): do not build with different settings
Compiling might be slow because different steps are compiling the same code 2-4 times because of the variations
* revert (docker): do not fix the rust version
* fix (docker): build on the root directory
* refactor(docker): Use base image commands and tools
* fix (cd): use correct variables & values, add build concurrency
* fix(cd): use Mainnet instead of mainnet
* imp: remove checkout as Buildkit uses the git context
* fix (docker): just Buildkit uses a .dockerignore in a path
* imp (cd): just use needed variables in the right place
* imp (cd): do not checkout if not needed
* test: run on push
* refactor(docker): reduce build changes
* fix(cd): not checking out was limiting some variables
* refactor(test): add an multistage exclusive for testing
* fix(cd): remove tests as a runtime dependency
* fix(cd): use default service account with cloud-platform scope
* fix(cd): revert checkout actions
* fix: use GA c2 instead of Preview c2d machine types
* fix(actions): remove workflow_dispatch from patched actions
This causes GitHub confusion as it can't determined which of the actions using workflow_dispatch is the right one
* fix(actions): remove patches from push actions
* test: validate changes on each push
* fix(test): wrong file syntax on test job
* fix(test): add missing env parameters
* fix(docker): Do not rebuild to download params and run tests
* fix(test): setup gcloud and loginto artifact just when needed
Try not to rebuild the tests
* fix(test): use GCP container to sync past mandatory checkpoint
* fix(test): missing separators
* test
* fix(test): mount the available disk
* push
* refactor(test): merge disk regeneration into test.yml
* fix(cd): minor typo fixes
* fix(docker): rebuild on .github changes
* fix(cd): keep compatibility with gcr.io
To prevent conflicts between registries, and migrate when the time is right, we'll keep pushing to both registries and use github actions cache to prevent conflicts between artifacts.
* fix(cd): typo and scope
* fix(cd): typos everywhere
* refactor(test): use smarter docker wait and keep old registry
* fix(cd): do not constraint the CPUs for bigger machines
* revert(cd): reduce PR diff as there's a separate one for tests
* fix(docker): add .github as it has no impact on caching
* fix(test): run command correctly
* fix(test): wiat and create image if previous step succeded
* force rebuild
* fix(test): do not restrict interdependant steps based on event
* force push
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* fix(test): remove all hardoced values and increase disks
* fix(test): use correct commands on deploy
* fix(test): use args as required by google
* fix(docker): try not to invalidate zebrad download cache
* fix(test): minor typo
* refactor(test): decouple jobs for better modularity
This also allows faster tests as testing Zunstable won't be a dependency and it can't stop already started jobs if it fails.
* fix(test): Do not try to execute ss and commands in one line
* fix(test): do not show undeeded information in the terminal
* fix(test): sleep befor/after machine creation/deletion
* fix(docker): do not download zcash params twice
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* merge: docker-actions-refactor into docker-test-refactor
* test docker wait scenarios
* fix(docker): $HOME variables is not being expanded
* fix(test): allow docker wait to work correctly
* fix(docker): do not use variables while using COPY
* fix(docker): allow to use zebrad as a command
* fix(cd): use test .yml from main
* fix(cd): Do not duplicate network values
The Dockerfile has an ARG with a default value of 'Mainnet', if this value is changed it will be done manually on a workflow_dispatch, making the ENV option a uneeded duplicate in this workflow
* fix(test): use bigger machine type for compute intensive tasks
* refactor(test): add tests in CI file
* fix(test): remove duplicated tests
* fix(test): typo
* test: build on .github changes temporarily
* fix(test): bigger machines have no effect on sync times
* feat: add an image to inherit from with zcash params
* fix(cd): use the right image name and allow push to test
* fix(cd): use the right docker target and remove extra builds
* refactor(docker): use cached zcash params from previous build
* fix(cd): finalize for merging
* imp(cd): add double safety measure for production
* fix(cd): use specific SHA for containers
* fix(cd): use latest gcloud action version
* fix(test): use the network as Mainnet and remove the uppercase from tests
* fix(test): run disk regeneration on specific file change
Just run this regeneration when changing the following files:
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state/disk_format.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/constants.rs
* refactor(test): seggregate disks regeneration from tests
Allow to regenerate disks without running tests, and to run tests from previous disk regeneration.
Disk will be regenerated just if specific files were changed, or triggered manually.
Tests will run just if a disk regeneration was not manually triggered.
* fix(test): gcp disks require lower case conventions
* fix(test): validate logs being emmited by docker
GHA is transforming is somehow transforwing the variable to lowercase also, so we're changint it to adapt to it
* test
* fix(test): force tty terminal
* fix(test): use a one line command to test terminal output
* fix(test): always delete test instance
* fix(test): use short SHA from the PR head
Using the SHA from the base, creates confusion and it's not accurate with the SHA being shown and used on GitHub.
We have to keep both as manual runs with `workflow_dispatch` does not have a PR SHA
* fix(ci): do not trigger CI on docker changes
There's no impact in this workflow when a change is done in the dockerfile
* Instead of runing cargo test when the instance gets created, run this commands afterwards in a different step.
As GHA TTY is not working as expected, and workarounds does not play nicely with `gcloud compute ssh` actions/runner#241 (comment) we decided to get the container name from the logs, log directly to the container and run the cargo command from there.
* doc(test): document reasoning for new steps
* fix(test): increase machine type and ssh timeout
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
* finalize(test): do not rebuild image when changing actions
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
2022-02-15 16:54:16 -08:00
|
|
|
|
run: |
|
2023-04-12 23:56:21 -07:00
|
|
|
|
docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
|
2023-07-25 15:50:45 -07:00
|
|
|
|
docker run -e NETWORK --name zebrad-tests --tty ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features "lightwalletd-grpc-tests" --workspace -- --include-ignored
|
|
|
|
|
env:
|
|
|
|
|
NETWORK: ${{ inputs.network || vars.ZCASH_NETWORK }}
|
2022-10-24 16:38:48 -07:00
|
|
|
|
|
2022-11-03 23:57:36 -07:00
|
|
|
|
# zebrad tests without cached state with `getblocktemplate-rpcs` feature
|
|
|
|
|
#
|
2022-10-24 16:38:48 -07:00
|
|
|
|
# Same as above but we run all the tests behind the `getblocktemplate-rpcs` feature.
|
|
|
|
|
test-all-getblocktemplate-rpcs:
|
|
|
|
|
name: Test all with getblocktemplate-rpcs feature
|
|
|
|
|
runs-on: ubuntu-latest
|
|
|
|
|
needs: build
|
2023-01-31 12:39:11 -08:00
|
|
|
|
if: ${{ github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
|
2022-10-24 16:38:48 -07:00
|
|
|
|
steps:
|
2023-05-18 09:44:39 -07:00
|
|
|
|
- uses: r7kamura/rust-problem-matchers@v1.3.0
|
|
|
|
|
|
2022-10-24 16:38:48 -07:00
|
|
|
|
- name: Inject slug/short variables
|
|
|
|
|
uses: rlespinasse/github-slug-action@v4
|
|
|
|
|
with:
|
|
|
|
|
short-length: 7
|
|
|
|
|
|
2022-12-07 22:11:46 -08:00
|
|
|
|
- name: Run zebrad tests
|
2022-10-24 16:38:48 -07:00
|
|
|
|
run: |
|
2023-04-12 23:56:21 -07:00
|
|
|
|
docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
|
2023-07-25 15:50:45 -07:00
|
|
|
|
docker run -e NETWORK --name zebrad-tests --tty -e ${{ inputs.network || vars.ZCASH_NETWORK }} ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features "lightwalletd-grpc-tests getblocktemplate-rpcs" --workspace -- --include-ignored
|
|
|
|
|
env:
|
|
|
|
|
NETWORK: ${{ inputs.network || vars.ZCASH_NETWORK }}
|
refactor(test): dockerize tests and run sync in detached mode (#3459)
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): use newer google auth action
* fix (cd): use newer secret as gcp credential
* fix (docker): do not create extra directories
* fix (docker): ignore .github for caching purposes
* fix (docker): use latest rust
* fix (cd): bump build timeout
* fix: use a better name for manual deployment
* refactor (docker): use standard directories for executable
* fix (cd): most systems expect a "latest" tag
Caching from the latest image is one of the main reasons to add this extra tag. Before this commit, the inline cache was not being used.
* fix (cd): push the build image and the cache separately
The inline cache exporter only supports `min` cache mode. To enable `max` cache mode, push the image and the cache separately by using the registry cache exporter.
This also allows for smaller release images.
* fix (cd): remove unused GHA cache
We're leveraging the registry to cache the actions, instead of using the 10GB limits from Github Actions cache storage
* refactor (cd): use cargo-chef for caching rust deps
* fix: move build system deps before cargo cheg cook
* fix (release): use newer debian to reduce vulnerabilities
* fix (cd): use same zone, region and service accounts
* fix (cd): use same disk size and type for all deployments
* refactor (cd): activate interactive shells
Use interactive shells for manual and test deployments. This allow greater flexibility if troubleshooting is needed inside the machines
* refactor (test): use docker artifact from registry
Instead of using a VM to SSH into in to build and test. Build in GHA (to have the logs available), run the workspace tests in GHA, and just run the sync tests in GCP
Use a cintainer VM with zebra's image directly on it, and pass the needed parameters to run the Sync past mandatory checkpoint.
* tmp (cd): bump timeout for building from scratch
* tmp (test): bump build time
* fix (cd, test): bump build time-out to 210 minutes
* fix (docker): do not build with different settings
Compiling might be slow because different steps are compiling the same code 2-4 times because of the variations
* revert (docker): do not fix the rust version
* fix (docker): build on the root directory
* refactor(docker): Use base image commands and tools
* fix (cd): use correct variables & values, add build concurrency
* fix(cd): use Mainnet instead of mainnet
* imp: remove checkout as Buildkit uses the git context
* fix (docker): just Buildkit uses a .dockerignore in a path
* imp (cd): just use needed variables in the right place
* imp (cd): do not checkout if not needed
* test: run on push
* refactor(docker): reduce build changes
* fix(cd): not checking out was limiting some variables
* refactor(test): add an multistage exclusive for testing
* fix(cd): remove tests as a runtime dependency
* fix(cd): use default service account with cloud-platform scope
* fix(cd): revert checkout actions
* fix: use GA c2 instead of Preview c2d machine types
* fix(actions): remove workflow_dispatch from patched actions
This causes GitHub confusion as it can't determined which of the actions using workflow_dispatch is the right one
* fix(actions): remove patches from push actions
* test: validate changes on each push
* fix(test): wrong file syntax on test job
* fix(test): add missing env parameters
* fix(docker): Do not rebuild to download params and run tests
* fix(test): setup gcloud and loginto artifact just when needed
Try not to rebuild the tests
* fix(test): use GCP container to sync past mandatory checkpoint
* fix(test): missing separators
* test
* fix(test): mount the available disk
* push
* refactor(test): merge disk regeneration into test.yml
* fix(cd): minor typo fixes
* fix(docker): rebuild on .github changes
* fix(cd): keep compatibility with gcr.io
To prevent conflicts between registries, and migrate when the time is right, we'll keep pushing to both registries and use github actions cache to prevent conflicts between artifacts.
* fix(cd): typo and scope
* fix(cd): typos everywhere
* refactor(test): use smarter docker wait and keep old registry
* fix(cd): do not constraint the CPUs for bigger machines
* revert(cd): reduce PR diff as there's a separate one for tests
* fix(docker): add .github as it has no impact on caching
* fix(test): run command correctly
* fix(test): wiat and create image if previous step succeded
* force rebuild
* fix(test): do not restrict interdependant steps based on event
* force push
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* fix(test): remove all hardoced values and increase disks
* fix(test): use correct commands on deploy
* fix(test): use args as required by google
* fix(docker): try not to invalidate zebrad download cache
* fix(test): minor typo
* refactor(test): decouple jobs for better modularity
This also allows faster tests as testing Zunstable won't be a dependency and it can't stop already started jobs if it fails.
* fix(test): Do not try to execute ss and commands in one line
* fix(test): do not show undeeded information in the terminal
* fix(test): sleep befor/after machine creation/deletion
* fix(docker): do not download zcash params twice
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* merge: docker-actions-refactor into docker-test-refactor
* test docker wait scenarios
* fix(docker): $HOME variables is not being expanded
* fix(test): allow docker wait to work correctly
* fix(docker): do not use variables while using COPY
* fix(docker): allow to use zebrad as a command
* fix(cd): use test .yml from main
* fix(cd): Do not duplicate network values
The Dockerfile has an ARG with a default value of 'Mainnet', if this value is changed it will be done manually on a workflow_dispatch, making the ENV option a uneeded duplicate in this workflow
* fix(test): use bigger machine type for compute intensive tasks
* refactor(test): add tests in CI file
* fix(test): remove duplicated tests
* fix(test): typo
* test: build on .github changes temporarily
* fix(test): bigger machines have no effect on sync times
* feat: add an image to inherit from with zcash params
* fix(cd): use the right image name and allow push to test
* fix(cd): use the right docker target and remove extra builds
* refactor(docker): use cached zcash params from previous build
* fix(cd): finalize for merging
* imp(cd): add double safety measure for production
* fix(cd): use specific SHA for containers
* fix(cd): use latest gcloud action version
* fix(test): use the network as Mainnet and remove the uppercase from tests
* fix(test): run disk regeneration on specific file change
Just run this regeneration when changing the following files:
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state/disk_format.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/constants.rs
* refactor(test): seggregate disks regeneration from tests
Allow to regenerate disks without running tests, and to run tests from previous disk regeneration.
Disk will be regenerated just if specific files were changed, or triggered manually.
Tests will run just if a disk regeneration was not manually triggered.
* fix(test): gcp disks require lower case conventions
* fix(test): validate logs being emmited by docker
GHA is transforming is somehow transforwing the variable to lowercase also, so we're changint it to adapt to it
* test
* fix(test): force tty terminal
* fix(test): use a one line command to test terminal output
* fix(test): always delete test instance
* fix(test): use short SHA from the PR head
Using the SHA from the base, creates confusion and it's not accurate with the SHA being shown and used on GitHub.
We have to keep both as manual runs with `workflow_dispatch` does not have a PR SHA
* fix(ci): do not trigger CI on docker changes
There's no impact in this workflow when a change is done in the dockerfile
* Instead of runing cargo test when the instance gets created, run this commands afterwards in a different step.
As GHA TTY is not working as expected, and workarounds does not play nicely with `gcloud compute ssh` actions/runner#241 (comment) we decided to get the container name from the logs, log directly to the container and run the cargo command from there.
* doc(test): document reasoning for new steps
* fix(test): increase machine type and ssh timeout
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
* finalize(test): do not rebuild image when changing actions
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
2022-02-15 16:54:16 -08:00
|
|
|
|
|
2022-05-11 07:06:58 -07:00
|
|
|
|
# Run state tests with fake activation heights.
|
|
|
|
|
#
|
2022-03-09 13:10:05 -08:00
|
|
|
|
# This test changes zebra-chain's activation heights,
|
|
|
|
|
# which can recompile all the Zebra crates,
|
|
|
|
|
# so we want its build products to be cached separately.
|
|
|
|
|
#
|
|
|
|
|
# Also, we don't want to accidentally use the fake heights in other tests.
|
2022-05-11 07:06:58 -07:00
|
|
|
|
#
|
|
|
|
|
# (The gRPC feature is a zebrad feature, so it isn't needed here.)
|
refactor(test): dockerize tests and run sync in detached mode (#3459)
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): use newer google auth action
* fix (cd): use newer secret as gcp credential
* fix (docker): do not create extra directories
* fix (docker): ignore .github for caching purposes
* fix (docker): use latest rust
* fix (cd): bump build timeout
* fix: use a better name for manual deployment
* refactor (docker): use standard directories for executable
* fix (cd): most systems expect a "latest" tag
Caching from the latest image is one of the main reasons to add this extra tag. Before this commit, the inline cache was not being used.
* fix (cd): push the build image and the cache separately
The inline cache exporter only supports `min` cache mode. To enable `max` cache mode, push the image and the cache separately by using the registry cache exporter.
This also allows for smaller release images.
* fix (cd): remove unused GHA cache
We're leveraging the registry to cache the actions, instead of using the 10GB limits from Github Actions cache storage
* refactor (cd): use cargo-chef for caching rust deps
* fix: move build system deps before cargo cheg cook
* fix (release): use newer debian to reduce vulnerabilities
* fix (cd): use same zone, region and service accounts
* fix (cd): use same disk size and type for all deployments
* refactor (cd): activate interactive shells
Use interactive shells for manual and test deployments. This allow greater flexibility if troubleshooting is needed inside the machines
* refactor (test): use docker artifact from registry
Instead of using a VM to SSH into in to build and test. Build in GHA (to have the logs available), run the workspace tests in GHA, and just run the sync tests in GCP
Use a cintainer VM with zebra's image directly on it, and pass the needed parameters to run the Sync past mandatory checkpoint.
* tmp (cd): bump timeout for building from scratch
* tmp (test): bump build time
* fix (cd, test): bump build time-out to 210 minutes
* fix (docker): do not build with different settings
Compiling might be slow because different steps are compiling the same code 2-4 times because of the variations
* revert (docker): do not fix the rust version
* fix (docker): build on the root directory
* refactor(docker): Use base image commands and tools
* fix (cd): use correct variables & values, add build concurrency
* fix(cd): use Mainnet instead of mainnet
* imp: remove checkout as Buildkit uses the git context
* fix (docker): just Buildkit uses a .dockerignore in a path
* imp (cd): just use needed variables in the right place
* imp (cd): do not checkout if not needed
* test: run on push
* refactor(docker): reduce build changes
* fix(cd): not checking out was limiting some variables
* refactor(test): add an multistage exclusive for testing
* fix(cd): remove tests as a runtime dependency
* fix(cd): use default service account with cloud-platform scope
* fix(cd): revert checkout actions
* fix: use GA c2 instead of Preview c2d machine types
* fix(actions): remove workflow_dispatch from patched actions
This causes GitHub confusion as it can't determined which of the actions using workflow_dispatch is the right one
* fix(actions): remove patches from push actions
* test: validate changes on each push
* fix(test): wrong file syntax on test job
* fix(test): add missing env parameters
* fix(docker): Do not rebuild to download params and run tests
* fix(test): setup gcloud and loginto artifact just when needed
Try not to rebuild the tests
* fix(test): use GCP container to sync past mandatory checkpoint
* fix(test): missing separators
* test
* fix(test): mount the available disk
* push
* refactor(test): merge disk regeneration into test.yml
* fix(cd): minor typo fixes
* fix(docker): rebuild on .github changes
* fix(cd): keep compatibility with gcr.io
To prevent conflicts between registries, and migrate when the time is right, we'll keep pushing to both registries and use github actions cache to prevent conflicts between artifacts.
* fix(cd): typo and scope
* fix(cd): typos everywhere
* refactor(test): use smarter docker wait and keep old registry
* fix(cd): do not constraint the CPUs for bigger machines
* revert(cd): reduce PR diff as there's a separate one for tests
* fix(docker): add .github as it has no impact on caching
* fix(test): run command correctly
* fix(test): wiat and create image if previous step succeded
* force rebuild
* fix(test): do not restrict interdependant steps based on event
* force push
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* fix(test): remove all hardoced values and increase disks
* fix(test): use correct commands on deploy
* fix(test): use args as required by google
* fix(docker): try not to invalidate zebrad download cache
* fix(test): minor typo
* refactor(test): decouple jobs for better modularity
This also allows faster tests as testing Zunstable won't be a dependency and it can't stop already started jobs if it fails.
* fix(test): Do not try to execute ss and commands in one line
* fix(test): do not show undeeded information in the terminal
* fix(test): sleep befor/after machine creation/deletion
* fix(docker): do not download zcash params twice
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* merge: docker-actions-refactor into docker-test-refactor
* test docker wait scenarios
* fix(docker): $HOME variables is not being expanded
* fix(test): allow docker wait to work correctly
* fix(docker): do not use variables while using COPY
* fix(docker): allow to use zebrad as a command
* fix(cd): use test .yml from main
* fix(cd): Do not duplicate network values
The Dockerfile has an ARG with a default value of 'Mainnet', if this value is changed it will be done manually on a workflow_dispatch, making the ENV option a uneeded duplicate in this workflow
* fix(test): use bigger machine type for compute intensive tasks
* refactor(test): add tests in CI file
* fix(test): remove duplicated tests
* fix(test): typo
* test: build on .github changes temporarily
* fix(test): bigger machines have no effect on sync times
* feat: add an image to inherit from with zcash params
* fix(cd): use the right image name and allow push to test
* fix(cd): use the right docker target and remove extra builds
* refactor(docker): use cached zcash params from previous build
* fix(cd): finalize for merging
* imp(cd): add double safety measure for production
* fix(cd): use specific SHA for containers
* fix(cd): use latest gcloud action version
* fix(test): use the network as Mainnet and remove the uppercase from tests
* fix(test): run disk regeneration on specific file change
Just run this regeneration when changing the following files:
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state/disk_format.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/constants.rs
* refactor(test): seggregate disks regeneration from tests
Allow to regenerate disks without running tests, and to run tests from previous disk regeneration.
Disk will be regenerated just if specific files were changed, or triggered manually.
Tests will run just if a disk regeneration was not manually triggered.
* fix(test): gcp disks require lower case conventions
* fix(test): validate logs being emmited by docker
GHA is transforming is somehow transforwing the variable to lowercase also, so we're changint it to adapt to it
* test
* fix(test): force tty terminal
* fix(test): use a one line command to test terminal output
* fix(test): always delete test instance
* fix(test): use short SHA from the PR head
Using the SHA from the base, creates confusion and it's not accurate with the SHA being shown and used on GitHub.
We have to keep both as manual runs with `workflow_dispatch` does not have a PR SHA
* fix(ci): do not trigger CI on docker changes
There's no impact in this workflow when a change is done in the dockerfile
* Instead of runing cargo test when the instance gets created, run this commands afterwards in a different step.
As GHA TTY is not working as expected, and workarounds does not play nicely with `gcloud compute ssh` actions/runner#241 (comment) we decided to get the container name from the logs, log directly to the container and run the cargo command from there.
* doc(test): document reasoning for new steps
* fix(test): increase machine type and ssh timeout
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
* finalize(test): do not rebuild image when changing actions
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
2022-02-15 16:54:16 -08:00
|
|
|
|
test-fake-activation-heights:
|
|
|
|
|
name: Test with fake activation heights
|
|
|
|
|
runs-on: ubuntu-latest
|
|
|
|
|
needs: build
|
2023-01-31 12:39:11 -08:00
|
|
|
|
if: ${{ github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
|
refactor(test): dockerize tests and run sync in detached mode (#3459)
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): use newer google auth action
* fix (cd): use newer secret as gcp credential
* fix (docker): do not create extra directories
* fix (docker): ignore .github for caching purposes
* fix (docker): use latest rust
* fix (cd): bump build timeout
* fix: use a better name for manual deployment
* refactor (docker): use standard directories for executable
* fix (cd): most systems expect a "latest" tag
Caching from the latest image is one of the main reasons to add this extra tag. Before this commit, the inline cache was not being used.
* fix (cd): push the build image and the cache separately
The inline cache exporter only supports `min` cache mode. To enable `max` cache mode, push the image and the cache separately by using the registry cache exporter.
This also allows for smaller release images.
* fix (cd): remove unused GHA cache
We're leveraging the registry to cache the actions, instead of using the 10GB limits from Github Actions cache storage
* refactor (cd): use cargo-chef for caching rust deps
* fix: move build system deps before cargo cheg cook
* fix (release): use newer debian to reduce vulnerabilities
* fix (cd): use same zone, region and service accounts
* fix (cd): use same disk size and type for all deployments
* refactor (cd): activate interactive shells
Use interactive shells for manual and test deployments. This allow greater flexibility if troubleshooting is needed inside the machines
* refactor (test): use docker artifact from registry
Instead of using a VM to SSH into in to build and test. Build in GHA (to have the logs available), run the workspace tests in GHA, and just run the sync tests in GCP
Use a cintainer VM with zebra's image directly on it, and pass the needed parameters to run the Sync past mandatory checkpoint.
* tmp (cd): bump timeout for building from scratch
* tmp (test): bump build time
* fix (cd, test): bump build time-out to 210 minutes
* fix (docker): do not build with different settings
Compiling might be slow because different steps are compiling the same code 2-4 times because of the variations
* revert (docker): do not fix the rust version
* fix (docker): build on the root directory
* refactor(docker): Use base image commands and tools
* fix (cd): use correct variables & values, add build concurrency
* fix(cd): use Mainnet instead of mainnet
* imp: remove checkout as Buildkit uses the git context
* fix (docker): just Buildkit uses a .dockerignore in a path
* imp (cd): just use needed variables in the right place
* imp (cd): do not checkout if not needed
* test: run on push
* refactor(docker): reduce build changes
* fix(cd): not checking out was limiting some variables
* refactor(test): add an multistage exclusive for testing
* fix(cd): remove tests as a runtime dependency
* fix(cd): use default service account with cloud-platform scope
* fix(cd): revert checkout actions
* fix: use GA c2 instead of Preview c2d machine types
* fix(actions): remove workflow_dispatch from patched actions
This causes GitHub confusion as it can't determined which of the actions using workflow_dispatch is the right one
* fix(actions): remove patches from push actions
* test: validate changes on each push
* fix(test): wrong file syntax on test job
* fix(test): add missing env parameters
* fix(docker): Do not rebuild to download params and run tests
* fix(test): setup gcloud and loginto artifact just when needed
Try not to rebuild the tests
* fix(test): use GCP container to sync past mandatory checkpoint
* fix(test): missing separators
* test
* fix(test): mount the available disk
* push
* refactor(test): merge disk regeneration into test.yml
* fix(cd): minor typo fixes
* fix(docker): rebuild on .github changes
* fix(cd): keep compatibility with gcr.io
To prevent conflicts between registries, and migrate when the time is right, we'll keep pushing to both registries and use github actions cache to prevent conflicts between artifacts.
* fix(cd): typo and scope
* fix(cd): typos everywhere
* refactor(test): use smarter docker wait and keep old registry
* fix(cd): do not constraint the CPUs for bigger machines
* revert(cd): reduce PR diff as there's a separate one for tests
* fix(docker): add .github as it has no impact on caching
* fix(test): run command correctly
* fix(test): wiat and create image if previous step succeded
* force rebuild
* fix(test): do not restrict interdependant steps based on event
* force push
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* fix(test): remove all hardoced values and increase disks
* fix(test): use correct commands on deploy
* fix(test): use args as required by google
* fix(docker): try not to invalidate zebrad download cache
* fix(test): minor typo
* refactor(test): decouple jobs for better modularity
This also allows faster tests as testing Zunstable won't be a dependency and it can't stop already started jobs if it fails.
* fix(test): Do not try to execute ss and commands in one line
* fix(test): do not show undeeded information in the terminal
* fix(test): sleep befor/after machine creation/deletion
* fix(docker): do not download zcash params twice
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* merge: docker-actions-refactor into docker-test-refactor
* test docker wait scenarios
* fix(docker): $HOME variables is not being expanded
* fix(test): allow docker wait to work correctly
* fix(docker): do not use variables while using COPY
* fix(docker): allow to use zebrad as a command
* fix(cd): use test .yml from main
* fix(cd): Do not duplicate network values
The Dockerfile has an ARG with a default value of 'Mainnet', if this value is changed it will be done manually on a workflow_dispatch, making the ENV option a uneeded duplicate in this workflow
* fix(test): use bigger machine type for compute intensive tasks
* refactor(test): add tests in CI file
* fix(test): remove duplicated tests
* fix(test): typo
* test: build on .github changes temporarily
* fix(test): bigger machines have no effect on sync times
* feat: add an image to inherit from with zcash params
* fix(cd): use the right image name and allow push to test
* fix(cd): use the right docker target and remove extra builds
* refactor(docker): use cached zcash params from previous build
* fix(cd): finalize for merging
* imp(cd): add double safety measure for production
* fix(cd): use specific SHA for containers
* fix(cd): use latest gcloud action version
* fix(test): use the network as Mainnet and remove the uppercase from tests
* fix(test): run disk regeneration on specific file change
Just run this regeneration when changing the following files:
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state/disk_format.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/constants.rs
* refactor(test): seggregate disks regeneration from tests
Allow to regenerate disks without running tests, and to run tests from previous disk regeneration.
Disk will be regenerated just if specific files were changed, or triggered manually.
Tests will run just if a disk regeneration was not manually triggered.
* fix(test): gcp disks require lower case conventions
* fix(test): validate logs being emmited by docker
GHA is transforming is somehow transforwing the variable to lowercase also, so we're changint it to adapt to it
* test
* fix(test): force tty terminal
* fix(test): use a one line command to test terminal output
* fix(test): always delete test instance
* fix(test): use short SHA from the PR head
Using the SHA from the base, creates confusion and it's not accurate with the SHA being shown and used on GitHub.
We have to keep both as manual runs with `workflow_dispatch` does not have a PR SHA
* fix(ci): do not trigger CI on docker changes
There's no impact in this workflow when a change is done in the dockerfile
* Instead of runing cargo test when the instance gets created, run this commands afterwards in a different step.
As GHA TTY is not working as expected, and workarounds does not play nicely with `gcloud compute ssh` actions/runner#241 (comment) we decided to get the container name from the logs, log directly to the container and run the cargo command from there.
* doc(test): document reasoning for new steps
* fix(test): increase machine type and ssh timeout
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
* finalize(test): do not rebuild image when changing actions
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
2022-02-15 16:54:16 -08:00
|
|
|
|
steps:
|
2023-05-18 09:44:39 -07:00
|
|
|
|
- uses: r7kamura/rust-problem-matchers@v1.3.0
|
|
|
|
|
|
refactor(test): dockerize tests and run sync in detached mode (#3459)
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): use newer google auth action
* fix (cd): use newer secret as gcp credential
* fix (docker): do not create extra directories
* fix (docker): ignore .github for caching purposes
* fix (docker): use latest rust
* fix (cd): bump build timeout
* fix: use a better name for manual deployment
* refactor (docker): use standard directories for executable
* fix (cd): most systems expect a "latest" tag
Caching from the latest image is one of the main reasons to add this extra tag. Before this commit, the inline cache was not being used.
* fix (cd): push the build image and the cache separately
The inline cache exporter only supports `min` cache mode. To enable `max` cache mode, push the image and the cache separately by using the registry cache exporter.
This also allows for smaller release images.
* fix (cd): remove unused GHA cache
We're leveraging the registry to cache the actions, instead of using the 10GB limits from Github Actions cache storage
* refactor (cd): use cargo-chef for caching rust deps
* fix: move build system deps before cargo cheg cook
* fix (release): use newer debian to reduce vulnerabilities
* fix (cd): use same zone, region and service accounts
* fix (cd): use same disk size and type for all deployments
* refactor (cd): activate interactive shells
Use interactive shells for manual and test deployments. This allow greater flexibility if troubleshooting is needed inside the machines
* refactor (test): use docker artifact from registry
Instead of using a VM to SSH into in to build and test. Build in GHA (to have the logs available), run the workspace tests in GHA, and just run the sync tests in GCP
Use a cintainer VM with zebra's image directly on it, and pass the needed parameters to run the Sync past mandatory checkpoint.
* tmp (cd): bump timeout for building from scratch
* tmp (test): bump build time
* fix (cd, test): bump build time-out to 210 minutes
* fix (docker): do not build with different settings
Compiling might be slow because different steps are compiling the same code 2-4 times because of the variations
* revert (docker): do not fix the rust version
* fix (docker): build on the root directory
* refactor(docker): Use base image commands and tools
* fix (cd): use correct variables & values, add build concurrency
* fix(cd): use Mainnet instead of mainnet
* imp: remove checkout as Buildkit uses the git context
* fix (docker): just Buildkit uses a .dockerignore in a path
* imp (cd): just use needed variables in the right place
* imp (cd): do not checkout if not needed
* test: run on push
* refactor(docker): reduce build changes
* fix(cd): not checking out was limiting some variables
* refactor(test): add an multistage exclusive for testing
* fix(cd): remove tests as a runtime dependency
* fix(cd): use default service account with cloud-platform scope
* fix(cd): revert checkout actions
* fix: use GA c2 instead of Preview c2d machine types
* fix(actions): remove workflow_dispatch from patched actions
This causes GitHub confusion as it can't determined which of the actions using workflow_dispatch is the right one
* fix(actions): remove patches from push actions
* test: validate changes on each push
* fix(test): wrong file syntax on test job
* fix(test): add missing env parameters
* fix(docker): Do not rebuild to download params and run tests
* fix(test): setup gcloud and loginto artifact just when needed
Try not to rebuild the tests
* fix(test): use GCP container to sync past mandatory checkpoint
* fix(test): missing separators
* test
* fix(test): mount the available disk
* push
* refactor(test): merge disk regeneration into test.yml
* fix(cd): minor typo fixes
* fix(docker): rebuild on .github changes
* fix(cd): keep compatibility with gcr.io
To prevent conflicts between registries, and migrate when the time is right, we'll keep pushing to both registries and use github actions cache to prevent conflicts between artifacts.
* fix(cd): typo and scope
* fix(cd): typos everywhere
* refactor(test): use smarter docker wait and keep old registry
* fix(cd): do not constraint the CPUs for bigger machines
* revert(cd): reduce PR diff as there's a separate one for tests
* fix(docker): add .github as it has no impact on caching
* fix(test): run command correctly
* fix(test): wiat and create image if previous step succeded
* force rebuild
* fix(test): do not restrict interdependant steps based on event
* force push
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* fix(test): remove all hardoced values and increase disks
* fix(test): use correct commands on deploy
* fix(test): use args as required by google
* fix(docker): try not to invalidate zebrad download cache
* fix(test): minor typo
* refactor(test): decouple jobs for better modularity
This also allows faster tests as testing Zunstable won't be a dependency and it can't stop already started jobs if it fails.
* fix(test): Do not try to execute ss and commands in one line
* fix(test): do not show undeeded information in the terminal
* fix(test): sleep befor/after machine creation/deletion
* fix(docker): do not download zcash params twice
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* merge: docker-actions-refactor into docker-test-refactor
* test docker wait scenarios
* fix(docker): $HOME variables is not being expanded
* fix(test): allow docker wait to work correctly
* fix(docker): do not use variables while using COPY
* fix(docker): allow to use zebrad as a command
* fix(cd): use test .yml from main
* fix(cd): Do not duplicate network values
The Dockerfile has an ARG with a default value of 'Mainnet', if this value is changed it will be done manually on a workflow_dispatch, making the ENV option a uneeded duplicate in this workflow
* fix(test): use bigger machine type for compute intensive tasks
* refactor(test): add tests in CI file
* fix(test): remove duplicated tests
* fix(test): typo
* test: build on .github changes temporarily
* fix(test): bigger machines have no effect on sync times
* feat: add an image to inherit from with zcash params
* fix(cd): use the right image name and allow push to test
* fix(cd): use the right docker target and remove extra builds
* refactor(docker): use cached zcash params from previous build
* fix(cd): finalize for merging
* imp(cd): add double safety measure for production
* fix(cd): use specific SHA for containers
* fix(cd): use latest gcloud action version
* fix(test): use the network as Mainnet and remove the uppercase from tests
* fix(test): run disk regeneration on specific file change
Just run this regeneration when changing the following files:
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state/disk_format.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/constants.rs
* refactor(test): seggregate disks regeneration from tests
Allow to regenerate disks without running tests, and to run tests from previous disk regeneration.
Disk will be regenerated just if specific files were changed, or triggered manually.
Tests will run just if a disk regeneration was not manually triggered.
* fix(test): gcp disks require lower case conventions
* fix(test): validate logs being emmited by docker
GHA is transforming is somehow transforwing the variable to lowercase also, so we're changint it to adapt to it
* test
* fix(test): force tty terminal
* fix(test): use a one line command to test terminal output
* fix(test): always delete test instance
* fix(test): use short SHA from the PR head
Using the SHA from the base, creates confusion and it's not accurate with the SHA being shown and used on GitHub.
We have to keep both as manual runs with `workflow_dispatch` does not have a PR SHA
* fix(ci): do not trigger CI on docker changes
There's no impact in this workflow when a change is done in the dockerfile
* Instead of runing cargo test when the instance gets created, run this commands afterwards in a different step.
As GHA TTY is not working as expected, and workarounds does not play nicely with `gcloud compute ssh` actions/runner#241 (comment) we decided to get the container name from the logs, log directly to the container and run the cargo command from there.
* doc(test): document reasoning for new steps
* fix(test): increase machine type and ssh timeout
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
* finalize(test): do not rebuild image when changing actions
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
2022-02-15 16:54:16 -08:00
|
|
|
|
- name: Inject slug/short variables
|
|
|
|
|
uses: rlespinasse/github-slug-action@v4
|
2022-03-21 16:07:01 -07:00
|
|
|
|
with:
|
|
|
|
|
short-length: 7
|
refactor(test): dockerize tests and run sync in detached mode (#3459)
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): use newer google auth action
* fix (cd): use newer secret as gcp credential
* fix (docker): do not create extra directories
* fix (docker): ignore .github for caching purposes
* fix (docker): use latest rust
* fix (cd): bump build timeout
* fix: use a better name for manual deployment
* refactor (docker): use standard directories for executable
* fix (cd): most systems expect a "latest" tag
Caching from the latest image is one of the main reasons to add this extra tag. Before this commit, the inline cache was not being used.
* fix (cd): push the build image and the cache separately
The inline cache exporter only supports `min` cache mode. To enable `max` cache mode, push the image and the cache separately by using the registry cache exporter.
This also allows for smaller release images.
* fix (cd): remove unused GHA cache
We're leveraging the registry to cache the actions, instead of using the 10GB limits from Github Actions cache storage
* refactor (cd): use cargo-chef for caching rust deps
* fix: move build system deps before cargo cheg cook
* fix (release): use newer debian to reduce vulnerabilities
* fix (cd): use same zone, region and service accounts
* fix (cd): use same disk size and type for all deployments
* refactor (cd): activate interactive shells
Use interactive shells for manual and test deployments. This allow greater flexibility if troubleshooting is needed inside the machines
* refactor (test): use docker artifact from registry
Instead of using a VM to SSH into in to build and test. Build in GHA (to have the logs available), run the workspace tests in GHA, and just run the sync tests in GCP
Use a cintainer VM with zebra's image directly on it, and pass the needed parameters to run the Sync past mandatory checkpoint.
* tmp (cd): bump timeout for building from scratch
* tmp (test): bump build time
* fix (cd, test): bump build time-out to 210 minutes
* fix (docker): do not build with different settings
Compiling might be slow because different steps are compiling the same code 2-4 times because of the variations
* revert (docker): do not fix the rust version
* fix (docker): build on the root directory
* refactor(docker): Use base image commands and tools
* fix (cd): use correct variables & values, add build concurrency
* fix(cd): use Mainnet instead of mainnet
* imp: remove checkout as Buildkit uses the git context
* fix (docker): just Buildkit uses a .dockerignore in a path
* imp (cd): just use needed variables in the right place
* imp (cd): do not checkout if not needed
* test: run on push
* refactor(docker): reduce build changes
* fix(cd): not checking out was limiting some variables
* refactor(test): add an multistage exclusive for testing
* fix(cd): remove tests as a runtime dependency
* fix(cd): use default service account with cloud-platform scope
* fix(cd): revert checkout actions
* fix: use GA c2 instead of Preview c2d machine types
* fix(actions): remove workflow_dispatch from patched actions
This causes GitHub confusion as it can't determined which of the actions using workflow_dispatch is the right one
* fix(actions): remove patches from push actions
* test: validate changes on each push
* fix(test): wrong file syntax on test job
* fix(test): add missing env parameters
* fix(docker): Do not rebuild to download params and run tests
* fix(test): setup gcloud and loginto artifact just when needed
Try not to rebuild the tests
* fix(test): use GCP container to sync past mandatory checkpoint
* fix(test): missing separators
* test
* fix(test): mount the available disk
* push
* refactor(test): merge disk regeneration into test.yml
* fix(cd): minor typo fixes
* fix(docker): rebuild on .github changes
* fix(cd): keep compatibility with gcr.io
To prevent conflicts between registries, and migrate when the time is right, we'll keep pushing to both registries and use github actions cache to prevent conflicts between artifacts.
* fix(cd): typo and scope
* fix(cd): typos everywhere
* refactor(test): use smarter docker wait and keep old registry
* fix(cd): do not constraint the CPUs for bigger machines
* revert(cd): reduce PR diff as there's a separate one for tests
* fix(docker): add .github as it has no impact on caching
* fix(test): run command correctly
* fix(test): wiat and create image if previous step succeded
* force rebuild
* fix(test): do not restrict interdependant steps based on event
* force push
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* fix(test): remove all hardoced values and increase disks
* fix(test): use correct commands on deploy
* fix(test): use args as required by google
* fix(docker): try not to invalidate zebrad download cache
* fix(test): minor typo
* refactor(test): decouple jobs for better modularity
This also allows faster tests as testing Zunstable won't be a dependency and it can't stop already started jobs if it fails.
* fix(test): Do not try to execute ss and commands in one line
* fix(test): do not show undeeded information in the terminal
* fix(test): sleep befor/after machine creation/deletion
* fix(docker): do not download zcash params twice
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* merge: docker-actions-refactor into docker-test-refactor
* test docker wait scenarios
* fix(docker): $HOME variables is not being expanded
* fix(test): allow docker wait to work correctly
* fix(docker): do not use variables while using COPY
* fix(docker): allow to use zebrad as a command
* fix(cd): use test .yml from main
* fix(cd): Do not duplicate network values
The Dockerfile has an ARG with a default value of 'Mainnet', if this value is changed it will be done manually on a workflow_dispatch, making the ENV option a uneeded duplicate in this workflow
* fix(test): use bigger machine type for compute intensive tasks
* refactor(test): add tests in CI file
* fix(test): remove duplicated tests
* fix(test): typo
* test: build on .github changes temporarily
* fix(test): bigger machines have no effect on sync times
* feat: add an image to inherit from with zcash params
* fix(cd): use the right image name and allow push to test
* fix(cd): use the right docker target and remove extra builds
* refactor(docker): use cached zcash params from previous build
* fix(cd): finalize for merging
* imp(cd): add double safety measure for production
* fix(cd): use specific SHA for containers
* fix(cd): use latest gcloud action version
* fix(test): use the network as Mainnet and remove the uppercase from tests
* fix(test): run disk regeneration on specific file change
Just run this regeneration when changing the following files:
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state/disk_format.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/constants.rs
* refactor(test): seggregate disks regeneration from tests
Allow to regenerate disks without running tests, and to run tests from previous disk regeneration.
Disk will be regenerated just if specific files were changed, or triggered manually.
Tests will run just if a disk regeneration was not manually triggered.
* fix(test): gcp disks require lower case conventions
* fix(test): validate logs being emmited by docker
GHA is transforming is somehow transforwing the variable to lowercase also, so we're changint it to adapt to it
* test
* fix(test): force tty terminal
* fix(test): use a one line command to test terminal output
* fix(test): always delete test instance
* fix(test): use short SHA from the PR head
Using the SHA from the base, creates confusion and it's not accurate with the SHA being shown and used on GitHub.
We have to keep both as manual runs with `workflow_dispatch` does not have a PR SHA
* fix(ci): do not trigger CI on docker changes
There's no impact in this workflow when a change is done in the dockerfile
* Instead of runing cargo test when the instance gets created, run this commands afterwards in a different step.
As GHA TTY is not working as expected, and workarounds does not play nicely with `gcloud compute ssh` actions/runner#241 (comment) we decided to get the container name from the logs, log directly to the container and run the cargo command from there.
* doc(test): document reasoning for new steps
* fix(test): increase machine type and ssh timeout
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
* finalize(test): do not rebuild image when changing actions
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
2022-02-15 16:54:16 -08:00
|
|
|
|
|
|
|
|
|
- name: Run tests with fake activation heights
|
|
|
|
|
run: |
|
2023-04-12 23:56:21 -07:00
|
|
|
|
docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
|
2023-07-25 15:50:45 -07:00
|
|
|
|
docker run -e NETWORK -e TEST_FAKE_ACTIVATION_HEIGHTS --name zebrad-tests -t ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --package zebra-state --lib -- --nocapture --include-ignored with_fake_activation_heights
|
2022-03-05 09:32:46 -08:00
|
|
|
|
env:
|
|
|
|
|
TEST_FAKE_ACTIVATION_HEIGHTS: '1'
|
2023-07-25 15:50:45 -07:00
|
|
|
|
NETWORK: ${{ inputs.network || vars.ZCASH_NETWORK }}
|
refactor(test): dockerize tests and run sync in detached mode (#3459)
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): use newer google auth action
* fix (cd): use newer secret as gcp credential
* fix (docker): do not create extra directories
* fix (docker): ignore .github for caching purposes
* fix (docker): use latest rust
* fix (cd): bump build timeout
* fix: use a better name for manual deployment
* refactor (docker): use standard directories for executable
* fix (cd): most systems expect a "latest" tag
Caching from the latest image is one of the main reasons to add this extra tag. Before this commit, the inline cache was not being used.
* fix (cd): push the build image and the cache separately
The inline cache exporter only supports `min` cache mode. To enable `max` cache mode, push the image and the cache separately by using the registry cache exporter.
This also allows for smaller release images.
* fix (cd): remove unused GHA cache
We're leveraging the registry to cache the actions, instead of using the 10GB limits from Github Actions cache storage
* refactor (cd): use cargo-chef for caching rust deps
* fix: move build system deps before cargo cheg cook
* fix (release): use newer debian to reduce vulnerabilities
* fix (cd): use same zone, region and service accounts
* fix (cd): use same disk size and type for all deployments
* refactor (cd): activate interactive shells
Use interactive shells for manual and test deployments. This allow greater flexibility if troubleshooting is needed inside the machines
* refactor (test): use docker artifact from registry
Instead of using a VM to SSH into in to build and test. Build in GHA (to have the logs available), run the workspace tests in GHA, and just run the sync tests in GCP
Use a cintainer VM with zebra's image directly on it, and pass the needed parameters to run the Sync past mandatory checkpoint.
* tmp (cd): bump timeout for building from scratch
* tmp (test): bump build time
* fix (cd, test): bump build time-out to 210 minutes
* fix (docker): do not build with different settings
Compiling might be slow because different steps are compiling the same code 2-4 times because of the variations
* revert (docker): do not fix the rust version
* fix (docker): build on the root directory
* refactor(docker): Use base image commands and tools
* fix (cd): use correct variables & values, add build concurrency
* fix(cd): use Mainnet instead of mainnet
* imp: remove checkout as Buildkit uses the git context
* fix (docker): just Buildkit uses a .dockerignore in a path
* imp (cd): just use needed variables in the right place
* imp (cd): do not checkout if not needed
* test: run on push
* refactor(docker): reduce build changes
* fix(cd): not checking out was limiting some variables
* refactor(test): add an multistage exclusive for testing
* fix(cd): remove tests as a runtime dependency
* fix(cd): use default service account with cloud-platform scope
* fix(cd): revert checkout actions
* fix: use GA c2 instead of Preview c2d machine types
* fix(actions): remove workflow_dispatch from patched actions
This causes GitHub confusion as it can't determined which of the actions using workflow_dispatch is the right one
* fix(actions): remove patches from push actions
* test: validate changes on each push
* fix(test): wrong file syntax on test job
* fix(test): add missing env parameters
* fix(docker): Do not rebuild to download params and run tests
* fix(test): setup gcloud and loginto artifact just when needed
Try not to rebuild the tests
* fix(test): use GCP container to sync past mandatory checkpoint
* fix(test): missing separators
* test
* fix(test): mount the available disk
* push
* refactor(test): merge disk regeneration into test.yml
* fix(cd): minor typo fixes
* fix(docker): rebuild on .github changes
* fix(cd): keep compatibility with gcr.io
To prevent conflicts between registries, and migrate when the time is right, we'll keep pushing to both registries and use github actions cache to prevent conflicts between artifacts.
* fix(cd): typo and scope
* fix(cd): typos everywhere
* refactor(test): use smarter docker wait and keep old registry
* fix(cd): do not constraint the CPUs for bigger machines
* revert(cd): reduce PR diff as there's a separate one for tests
* fix(docker): add .github as it has no impact on caching
* fix(test): run command correctly
* fix(test): wiat and create image if previous step succeded
* force rebuild
* fix(test): do not restrict interdependant steps based on event
* force push
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* fix(test): remove all hardoced values and increase disks
* fix(test): use correct commands on deploy
* fix(test): use args as required by google
* fix(docker): try not to invalidate zebrad download cache
* fix(test): minor typo
* refactor(test): decouple jobs for better modularity
This also allows faster tests as testing Zunstable won't be a dependency and it can't stop already started jobs if it fails.
* fix(test): Do not try to execute ss and commands in one line
* fix(test): do not show undeeded information in the terminal
* fix(test): sleep befor/after machine creation/deletion
* fix(docker): do not download zcash params twice
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* merge: docker-actions-refactor into docker-test-refactor
* test docker wait scenarios
* fix(docker): $HOME variables is not being expanded
* fix(test): allow docker wait to work correctly
* fix(docker): do not use variables while using COPY
* fix(docker): allow to use zebrad as a command
* fix(cd): use test .yml from main
* fix(cd): Do not duplicate network values
The Dockerfile has an ARG with a default value of 'Mainnet', if this value is changed it will be done manually on a workflow_dispatch, making the ENV option a uneeded duplicate in this workflow
* fix(test): use bigger machine type for compute intensive tasks
* refactor(test): add tests in CI file
* fix(test): remove duplicated tests
* fix(test): typo
* test: build on .github changes temporarily
* fix(test): bigger machines have no effect on sync times
* feat: add an image to inherit from with zcash params
* fix(cd): use the right image name and allow push to test
* fix(cd): use the right docker target and remove extra builds
* refactor(docker): use cached zcash params from previous build
* fix(cd): finalize for merging
* imp(cd): add double safety measure for production
* fix(cd): use specific SHA for containers
* fix(cd): use latest gcloud action version
* fix(test): use the network as Mainnet and remove the uppercase from tests
* fix(test): run disk regeneration on specific file change
Just run this regeneration when changing the following files:
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state/disk_format.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/constants.rs
* refactor(test): seggregate disks regeneration from tests
Allow to regenerate disks without running tests, and to run tests from previous disk regeneration.
Disk will be regenerated just if specific files were changed, or triggered manually.
Tests will run just if a disk regeneration was not manually triggered.
* fix(test): gcp disks require lower case conventions
* fix(test): validate logs being emmited by docker
GHA is transforming is somehow transforwing the variable to lowercase also, so we're changint it to adapt to it
* test
* fix(test): force tty terminal
* fix(test): use a one line command to test terminal output
* fix(test): always delete test instance
* fix(test): use short SHA from the PR head
Using the SHA from the base, creates confusion and it's not accurate with the SHA being shown and used on GitHub.
We have to keep both as manual runs with `workflow_dispatch` does not have a PR SHA
* fix(ci): do not trigger CI on docker changes
There's no impact in this workflow when a change is done in the dockerfile
* Instead of runing cargo test when the instance gets created, run this commands afterwards in a different step.
As GHA TTY is not working as expected, and workarounds does not play nicely with `gcloud compute ssh` actions/runner#241 (comment) we decided to get the container name from the logs, log directly to the container and run the cargo command from there.
* doc(test): document reasoning for new steps
* fix(test): increase machine type and ssh timeout
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
* finalize(test): do not rebuild image when changing actions
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
2022-02-15 16:54:16 -08:00
|
|
|
|
|
2022-05-11 07:06:58 -07:00
|
|
|
|
# Test that Zebra syncs and checkpoints a few thousand blocks from an empty state.
|
|
|
|
|
#
|
|
|
|
|
# (We activate the gRPC feature to avoid recompiling `zebrad`, but we don't actually run any gRPC tests.)
|
2022-02-17 13:42:26 -08:00
|
|
|
|
test-empty-sync:
|
|
|
|
|
name: Test checkpoint sync from empty state
|
refactor(test): dockerize tests and run sync in detached mode (#3459)
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): use newer google auth action
* fix (cd): use newer secret as gcp credential
* fix (docker): do not create extra directories
* fix (docker): ignore .github for caching purposes
* fix (docker): use latest rust
* fix (cd): bump build timeout
* fix: use a better name for manual deployment
* refactor (docker): use standard directories for executable
* fix (cd): most systems expect a "latest" tag
Caching from the latest image is one of the main reasons to add this extra tag. Before this commit, the inline cache was not being used.
* fix (cd): push the build image and the cache separately
The inline cache exporter only supports `min` cache mode. To enable `max` cache mode, push the image and the cache separately by using the registry cache exporter.
This also allows for smaller release images.
* fix (cd): remove unused GHA cache
We're leveraging the registry to cache the actions, instead of using the 10GB limits from Github Actions cache storage
* refactor (cd): use cargo-chef for caching rust deps
* fix: move build system deps before cargo cheg cook
* fix (release): use newer debian to reduce vulnerabilities
* fix (cd): use same zone, region and service accounts
* fix (cd): use same disk size and type for all deployments
* refactor (cd): activate interactive shells
Use interactive shells for manual and test deployments. This allow greater flexibility if troubleshooting is needed inside the machines
* refactor (test): use docker artifact from registry
Instead of using a VM to SSH into in to build and test. Build in GHA (to have the logs available), run the workspace tests in GHA, and just run the sync tests in GCP
Use a cintainer VM with zebra's image directly on it, and pass the needed parameters to run the Sync past mandatory checkpoint.
* tmp (cd): bump timeout for building from scratch
* tmp (test): bump build time
* fix (cd, test): bump build time-out to 210 minutes
* fix (docker): do not build with different settings
Compiling might be slow because different steps are compiling the same code 2-4 times because of the variations
* revert (docker): do not fix the rust version
* fix (docker): build on the root directory
* refactor(docker): Use base image commands and tools
* fix (cd): use correct variables & values, add build concurrency
* fix(cd): use Mainnet instead of mainnet
* imp: remove checkout as Buildkit uses the git context
* fix (docker): just Buildkit uses a .dockerignore in a path
* imp (cd): just use needed variables in the right place
* imp (cd): do not checkout if not needed
* test: run on push
* refactor(docker): reduce build changes
* fix(cd): not checking out was limiting some variables
* refactor(test): add an multistage exclusive for testing
* fix(cd): remove tests as a runtime dependency
* fix(cd): use default service account with cloud-platform scope
* fix(cd): revert checkout actions
* fix: use GA c2 instead of Preview c2d machine types
* fix(actions): remove workflow_dispatch from patched actions
This causes GitHub confusion as it can't determined which of the actions using workflow_dispatch is the right one
* fix(actions): remove patches from push actions
* test: validate changes on each push
* fix(test): wrong file syntax on test job
* fix(test): add missing env parameters
* fix(docker): Do not rebuild to download params and run tests
* fix(test): setup gcloud and loginto artifact just when needed
Try not to rebuild the tests
* fix(test): use GCP container to sync past mandatory checkpoint
* fix(test): missing separators
* test
* fix(test): mount the available disk
* push
* refactor(test): merge disk regeneration into test.yml
* fix(cd): minor typo fixes
* fix(docker): rebuild on .github changes
* fix(cd): keep compatibility with gcr.io
To prevent conflicts between registries, and migrate when the time is right, we'll keep pushing to both registries and use github actions cache to prevent conflicts between artifacts.
* fix(cd): typo and scope
* fix(cd): typos everywhere
* refactor(test): use smarter docker wait and keep old registry
* fix(cd): do not constraint the CPUs for bigger machines
* revert(cd): reduce PR diff as there's a separate one for tests
* fix(docker): add .github as it has no impact on caching
* fix(test): run command correctly
* fix(test): wiat and create image if previous step succeded
* force rebuild
* fix(test): do not restrict interdependant steps based on event
* force push
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* fix(test): remove all hardoced values and increase disks
* fix(test): use correct commands on deploy
* fix(test): use args as required by google
* fix(docker): try not to invalidate zebrad download cache
* fix(test): minor typo
* refactor(test): decouple jobs for better modularity
This also allows faster tests as testing Zunstable won't be a dependency and it can't stop already started jobs if it fails.
* fix(test): Do not try to execute ss and commands in one line
* fix(test): do not show undeeded information in the terminal
* fix(test): sleep befor/after machine creation/deletion
* fix(docker): do not download zcash params twice
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* merge: docker-actions-refactor into docker-test-refactor
* test docker wait scenarios
* fix(docker): $HOME variables is not being expanded
* fix(test): allow docker wait to work correctly
* fix(docker): do not use variables while using COPY
* fix(docker): allow to use zebrad as a command
* fix(cd): use test .yml from main
* fix(cd): Do not duplicate network values
The Dockerfile has an ARG with a default value of 'Mainnet', if this value is changed it will be done manually on a workflow_dispatch, making the ENV option a uneeded duplicate in this workflow
* fix(test): use bigger machine type for compute intensive tasks
* refactor(test): add tests in CI file
* fix(test): remove duplicated tests
* fix(test): typo
* test: build on .github changes temporarily
* fix(test): bigger machines have no effect on sync times
* feat: add an image to inherit from with zcash params
* fix(cd): use the right image name and allow push to test
* fix(cd): use the right docker target and remove extra builds
* refactor(docker): use cached zcash params from previous build
* fix(cd): finalize for merging
* imp(cd): add double safety measure for production
* fix(cd): use specific SHA for containers
* fix(cd): use latest gcloud action version
* fix(test): use the network as Mainnet and remove the uppercase from tests
* fix(test): run disk regeneration on specific file change
Just run this regeneration when changing the following files:
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state/disk_format.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/constants.rs
* refactor(test): seggregate disks regeneration from tests
Allow to regenerate disks without running tests, and to run tests from previous disk regeneration.
Disk will be regenerated just if specific files were changed, or triggered manually.
Tests will run just if a disk regeneration was not manually triggered.
* fix(test): gcp disks require lower case conventions
* fix(test): validate logs being emmited by docker
GHA is transforming is somehow transforwing the variable to lowercase also, so we're changint it to adapt to it
* test
* fix(test): force tty terminal
* fix(test): use a one line command to test terminal output
* fix(test): always delete test instance
* fix(test): use short SHA from the PR head
Using the SHA from the base, creates confusion and it's not accurate with the SHA being shown and used on GitHub.
We have to keep both as manual runs with `workflow_dispatch` does not have a PR SHA
* fix(ci): do not trigger CI on docker changes
There's no impact in this workflow when a change is done in the dockerfile
* Instead of runing cargo test when the instance gets created, run this commands afterwards in a different step.
As GHA TTY is not working as expected, and workarounds does not play nicely with `gcloud compute ssh` actions/runner#241 (comment) we decided to get the container name from the logs, log directly to the container and run the cargo command from there.
* doc(test): document reasoning for new steps
* fix(test): increase machine type and ssh timeout
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
* finalize(test): do not rebuild image when changing actions
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
2022-02-15 16:54:16 -08:00
|
|
|
|
runs-on: ubuntu-latest
|
|
|
|
|
needs: build
|
2023-01-31 12:39:11 -08:00
|
|
|
|
if: ${{ github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
|
refactor(test): dockerize tests and run sync in detached mode (#3459)
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): use newer google auth action
* fix (cd): use newer secret as gcp credential
* fix (docker): do not create extra directories
* fix (docker): ignore .github for caching purposes
* fix (docker): use latest rust
* fix (cd): bump build timeout
* fix: use a better name for manual deployment
* refactor (docker): use standard directories for executable
* fix (cd): most systems expect a "latest" tag
Caching from the latest image is one of the main reasons to add this extra tag. Before this commit, the inline cache was not being used.
* fix (cd): push the build image and the cache separately
The inline cache exporter only supports `min` cache mode. To enable `max` cache mode, push the image and the cache separately by using the registry cache exporter.
This also allows for smaller release images.
* fix (cd): remove unused GHA cache
We're leveraging the registry to cache the actions, instead of using the 10GB limits from Github Actions cache storage
* refactor (cd): use cargo-chef for caching rust deps
* fix: move build system deps before cargo cheg cook
* fix (release): use newer debian to reduce vulnerabilities
* fix (cd): use same zone, region and service accounts
* fix (cd): use same disk size and type for all deployments
* refactor (cd): activate interactive shells
Use interactive shells for manual and test deployments. This allow greater flexibility if troubleshooting is needed inside the machines
* refactor (test): use docker artifact from registry
Instead of using a VM to SSH into in to build and test. Build in GHA (to have the logs available), run the workspace tests in GHA, and just run the sync tests in GCP
Use a cintainer VM with zebra's image directly on it, and pass the needed parameters to run the Sync past mandatory checkpoint.
* tmp (cd): bump timeout for building from scratch
* tmp (test): bump build time
* fix (cd, test): bump build time-out to 210 minutes
* fix (docker): do not build with different settings
Compiling might be slow because different steps are compiling the same code 2-4 times because of the variations
* revert (docker): do not fix the rust version
* fix (docker): build on the root directory
* refactor(docker): Use base image commands and tools
* fix (cd): use correct variables & values, add build concurrency
* fix(cd): use Mainnet instead of mainnet
* imp: remove checkout as Buildkit uses the git context
* fix (docker): just Buildkit uses a .dockerignore in a path
* imp (cd): just use needed variables in the right place
* imp (cd): do not checkout if not needed
* test: run on push
* refactor(docker): reduce build changes
* fix(cd): not checking out was limiting some variables
* refactor(test): add an multistage exclusive for testing
* fix(cd): remove tests as a runtime dependency
* fix(cd): use default service account with cloud-platform scope
* fix(cd): revert checkout actions
* fix: use GA c2 instead of Preview c2d machine types
* fix(actions): remove workflow_dispatch from patched actions
This causes GitHub confusion as it can't determined which of the actions using workflow_dispatch is the right one
* fix(actions): remove patches from push actions
* test: validate changes on each push
* fix(test): wrong file syntax on test job
* fix(test): add missing env parameters
* fix(docker): Do not rebuild to download params and run tests
* fix(test): setup gcloud and loginto artifact just when needed
Try not to rebuild the tests
* fix(test): use GCP container to sync past mandatory checkpoint
* fix(test): missing separators
* test
* fix(test): mount the available disk
* push
* refactor(test): merge disk regeneration into test.yml
* fix(cd): minor typo fixes
* fix(docker): rebuild on .github changes
* fix(cd): keep compatibility with gcr.io
To prevent conflicts between registries, and migrate when the time is right, we'll keep pushing to both registries and use github actions cache to prevent conflicts between artifacts.
* fix(cd): typo and scope
* fix(cd): typos everywhere
* refactor(test): use smarter docker wait and keep old registry
* fix(cd): do not constraint the CPUs for bigger machines
* revert(cd): reduce PR diff as there's a separate one for tests
* fix(docker): add .github as it has no impact on caching
* fix(test): run command correctly
* fix(test): wiat and create image if previous step succeded
* force rebuild
* fix(test): do not restrict interdependant steps based on event
* force push
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* fix(test): remove all hardoced values and increase disks
* fix(test): use correct commands on deploy
* fix(test): use args as required by google
* fix(docker): try not to invalidate zebrad download cache
* fix(test): minor typo
* refactor(test): decouple jobs for better modularity
This also allows faster tests as testing Zunstable won't be a dependency and it can't stop already started jobs if it fails.
* fix(test): Do not try to execute ss and commands in one line
* fix(test): do not show undeeded information in the terminal
* fix(test): sleep befor/after machine creation/deletion
* fix(docker): do not download zcash params twice
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* merge: docker-actions-refactor into docker-test-refactor
* test docker wait scenarios
* fix(docker): $HOME variables is not being expanded
* fix(test): allow docker wait to work correctly
* fix(docker): do not use variables while using COPY
* fix(docker): allow to use zebrad as a command
* fix(cd): use test .yml from main
* fix(cd): Do not duplicate network values
The Dockerfile has an ARG with a default value of 'Mainnet', if this value is changed it will be done manually on a workflow_dispatch, making the ENV option a uneeded duplicate in this workflow
* fix(test): use bigger machine type for compute intensive tasks
* refactor(test): add tests in CI file
* fix(test): remove duplicated tests
* fix(test): typo
* test: build on .github changes temporarily
* fix(test): bigger machines have no effect on sync times
* feat: add an image to inherit from with zcash params
* fix(cd): use the right image name and allow push to test
* fix(cd): use the right docker target and remove extra builds
* refactor(docker): use cached zcash params from previous build
* fix(cd): finalize for merging
* imp(cd): add double safety measure for production
* fix(cd): use specific SHA for containers
* fix(cd): use latest gcloud action version
* fix(test): use the network as Mainnet and remove the uppercase from tests
* fix(test): run disk regeneration on specific file change
Just run this regeneration when changing the following files:
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state/disk_format.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/constants.rs
* refactor(test): seggregate disks regeneration from tests
Allow to regenerate disks without running tests, and to run tests from previous disk regeneration.
Disk will be regenerated just if specific files were changed, or triggered manually.
Tests will run just if a disk regeneration was not manually triggered.
* fix(test): gcp disks require lower case conventions
* fix(test): validate logs being emmited by docker
GHA is transforming is somehow transforwing the variable to lowercase also, so we're changint it to adapt to it
* test
* fix(test): force tty terminal
* fix(test): use a one line command to test terminal output
* fix(test): always delete test instance
* fix(test): use short SHA from the PR head
Using the SHA from the base, creates confusion and it's not accurate with the SHA being shown and used on GitHub.
We have to keep both as manual runs with `workflow_dispatch` does not have a PR SHA
* fix(ci): do not trigger CI on docker changes
There's no impact in this workflow when a change is done in the dockerfile
* Instead of runing cargo test when the instance gets created, run this commands afterwards in a different step.
As GHA TTY is not working as expected, and workarounds does not play nicely with `gcloud compute ssh` actions/runner#241 (comment) we decided to get the container name from the logs, log directly to the container and run the cargo command from there.
* doc(test): document reasoning for new steps
* fix(test): increase machine type and ssh timeout
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
* finalize(test): do not rebuild image when changing actions
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
2022-02-15 16:54:16 -08:00
|
|
|
|
steps:
|
2023-05-18 09:44:39 -07:00
|
|
|
|
- uses: r7kamura/rust-problem-matchers@v1.3.0
|
|
|
|
|
|
refactor(test): dockerize tests and run sync in detached mode (#3459)
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): use newer google auth action
* fix (cd): use newer secret as gcp credential
* fix (docker): do not create extra directories
* fix (docker): ignore .github for caching purposes
* fix (docker): use latest rust
* fix (cd): bump build timeout
* fix: use a better name for manual deployment
* refactor (docker): use standard directories for executable
* fix (cd): most systems expect a "latest" tag
Caching from the latest image is one of the main reasons to add this extra tag. Before this commit, the inline cache was not being used.
* fix (cd): push the build image and the cache separately
The inline cache exporter only supports `min` cache mode. To enable `max` cache mode, push the image and the cache separately by using the registry cache exporter.
This also allows for smaller release images.
* fix (cd): remove unused GHA cache
We're leveraging the registry to cache the actions, instead of using the 10GB limits from Github Actions cache storage
* refactor (cd): use cargo-chef for caching rust deps
* fix: move build system deps before cargo cheg cook
* fix (release): use newer debian to reduce vulnerabilities
* fix (cd): use same zone, region and service accounts
* fix (cd): use same disk size and type for all deployments
* refactor (cd): activate interactive shells
Use interactive shells for manual and test deployments. This allow greater flexibility if troubleshooting is needed inside the machines
* refactor (test): use docker artifact from registry
Instead of using a VM to SSH into in to build and test. Build in GHA (to have the logs available), run the workspace tests in GHA, and just run the sync tests in GCP
Use a cintainer VM with zebra's image directly on it, and pass the needed parameters to run the Sync past mandatory checkpoint.
* tmp (cd): bump timeout for building from scratch
* tmp (test): bump build time
* fix (cd, test): bump build time-out to 210 minutes
* fix (docker): do not build with different settings
Compiling might be slow because different steps are compiling the same code 2-4 times because of the variations
* revert (docker): do not fix the rust version
* fix (docker): build on the root directory
* refactor(docker): Use base image commands and tools
* fix (cd): use correct variables & values, add build concurrency
* fix(cd): use Mainnet instead of mainnet
* imp: remove checkout as Buildkit uses the git context
* fix (docker): just Buildkit uses a .dockerignore in a path
* imp (cd): just use needed variables in the right place
* imp (cd): do not checkout if not needed
* test: run on push
* refactor(docker): reduce build changes
* fix(cd): not checking out was limiting some variables
* refactor(test): add an multistage exclusive for testing
* fix(cd): remove tests as a runtime dependency
* fix(cd): use default service account with cloud-platform scope
* fix(cd): revert checkout actions
* fix: use GA c2 instead of Preview c2d machine types
* fix(actions): remove workflow_dispatch from patched actions
This causes GitHub confusion as it can't determined which of the actions using workflow_dispatch is the right one
* fix(actions): remove patches from push actions
* test: validate changes on each push
* fix(test): wrong file syntax on test job
* fix(test): add missing env parameters
* fix(docker): Do not rebuild to download params and run tests
* fix(test): setup gcloud and loginto artifact just when needed
Try not to rebuild the tests
* fix(test): use GCP container to sync past mandatory checkpoint
* fix(test): missing separators
* test
* fix(test): mount the available disk
* push
* refactor(test): merge disk regeneration into test.yml
* fix(cd): minor typo fixes
* fix(docker): rebuild on .github changes
* fix(cd): keep compatibility with gcr.io
To prevent conflicts between registries, and migrate when the time is right, we'll keep pushing to both registries and use github actions cache to prevent conflicts between artifacts.
* fix(cd): typo and scope
* fix(cd): typos everywhere
* refactor(test): use smarter docker wait and keep old registry
* fix(cd): do not constraint the CPUs for bigger machines
* revert(cd): reduce PR diff as there's a separate one for tests
* fix(docker): add .github as it has no impact on caching
* fix(test): run command correctly
* fix(test): wiat and create image if previous step succeded
* force rebuild
* fix(test): do not restrict interdependant steps based on event
* force push
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* fix(test): remove all hardoced values and increase disks
* fix(test): use correct commands on deploy
* fix(test): use args as required by google
* fix(docker): try not to invalidate zebrad download cache
* fix(test): minor typo
* refactor(test): decouple jobs for better modularity
This also allows faster tests as testing Zunstable won't be a dependency and it can't stop already started jobs if it fails.
* fix(test): Do not try to execute ss and commands in one line
* fix(test): do not show undeeded information in the terminal
* fix(test): sleep befor/after machine creation/deletion
* fix(docker): do not download zcash params twice
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* merge: docker-actions-refactor into docker-test-refactor
* test docker wait scenarios
* fix(docker): $HOME variables is not being expanded
* fix(test): allow docker wait to work correctly
* fix(docker): do not use variables while using COPY
* fix(docker): allow to use zebrad as a command
* fix(cd): use test .yml from main
* fix(cd): Do not duplicate network values
The Dockerfile has an ARG with a default value of 'Mainnet', if this value is changed it will be done manually on a workflow_dispatch, making the ENV option a uneeded duplicate in this workflow
* fix(test): use bigger machine type for compute intensive tasks
* refactor(test): add tests in CI file
* fix(test): remove duplicated tests
* fix(test): typo
* test: build on .github changes temporarily
* fix(test): bigger machines have no effect on sync times
* feat: add an image to inherit from with zcash params
* fix(cd): use the right image name and allow push to test
* fix(cd): use the right docker target and remove extra builds
* refactor(docker): use cached zcash params from previous build
* fix(cd): finalize for merging
* imp(cd): add double safety measure for production
* fix(cd): use specific SHA for containers
* fix(cd): use latest gcloud action version
* fix(test): use the network as Mainnet and remove the uppercase from tests
* fix(test): run disk regeneration on specific file change
Just run this regeneration when changing the following files:
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state/disk_format.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/constants.rs
* refactor(test): seggregate disks regeneration from tests
Allow to regenerate disks without running tests, and to run tests from previous disk regeneration.
Disk will be regenerated just if specific files were changed, or triggered manually.
Tests will run just if a disk regeneration was not manually triggered.
* fix(test): gcp disks require lower case conventions
* fix(test): validate logs being emmited by docker
GHA is transforming is somehow transforwing the variable to lowercase also, so we're changint it to adapt to it
* test
* fix(test): force tty terminal
* fix(test): use a one line command to test terminal output
* fix(test): always delete test instance
* fix(test): use short SHA from the PR head
Using the SHA from the base, creates confusion and it's not accurate with the SHA being shown and used on GitHub.
We have to keep both as manual runs with `workflow_dispatch` does not have a PR SHA
* fix(ci): do not trigger CI on docker changes
There's no impact in this workflow when a change is done in the dockerfile
* Instead of runing cargo test when the instance gets created, run this commands afterwards in a different step.
As GHA TTY is not working as expected, and workarounds does not play nicely with `gcloud compute ssh` actions/runner#241 (comment) we decided to get the container name from the logs, log directly to the container and run the cargo command from there.
* doc(test): document reasoning for new steps
* fix(test): increase machine type and ssh timeout
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
* finalize(test): do not rebuild image when changing actions
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
2022-02-15 16:54:16 -08:00
|
|
|
|
- name: Inject slug/short variables
|
|
|
|
|
uses: rlespinasse/github-slug-action@v4
|
2022-03-21 16:07:01 -07:00
|
|
|
|
with:
|
|
|
|
|
short-length: 7
|
refactor(test): dockerize tests and run sync in detached mode (#3459)
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): use newer google auth action
* fix (cd): use newer secret as gcp credential
* fix (docker): do not create extra directories
* fix (docker): ignore .github for caching purposes
* fix (docker): use latest rust
* fix (cd): bump build timeout
* fix: use a better name for manual deployment
* refactor (docker): use standard directories for executable
* fix (cd): most systems expect a "latest" tag
Caching from the latest image is one of the main reasons to add this extra tag. Before this commit, the inline cache was not being used.
* fix (cd): push the build image and the cache separately
The inline cache exporter only supports `min` cache mode. To enable `max` cache mode, push the image and the cache separately by using the registry cache exporter.
This also allows for smaller release images.
* fix (cd): remove unused GHA cache
We're leveraging the registry to cache the actions, instead of using the 10GB limits from Github Actions cache storage
* refactor (cd): use cargo-chef for caching rust deps
* fix: move build system deps before cargo cheg cook
* fix (release): use newer debian to reduce vulnerabilities
* fix (cd): use same zone, region and service accounts
* fix (cd): use same disk size and type for all deployments
* refactor (cd): activate interactive shells
Use interactive shells for manual and test deployments. This allow greater flexibility if troubleshooting is needed inside the machines
* refactor (test): use docker artifact from registry
Instead of using a VM to SSH into in to build and test. Build in GHA (to have the logs available), run the workspace tests in GHA, and just run the sync tests in GCP
Use a cintainer VM with zebra's image directly on it, and pass the needed parameters to run the Sync past mandatory checkpoint.
* tmp (cd): bump timeout for building from scratch
* tmp (test): bump build time
* fix (cd, test): bump build time-out to 210 minutes
* fix (docker): do not build with different settings
Compiling might be slow because different steps are compiling the same code 2-4 times because of the variations
* revert (docker): do not fix the rust version
* fix (docker): build on the root directory
* refactor(docker): Use base image commands and tools
* fix (cd): use correct variables & values, add build concurrency
* fix(cd): use Mainnet instead of mainnet
* imp: remove checkout as Buildkit uses the git context
* fix (docker): just Buildkit uses a .dockerignore in a path
* imp (cd): just use needed variables in the right place
* imp (cd): do not checkout if not needed
* test: run on push
* refactor(docker): reduce build changes
* fix(cd): not checking out was limiting some variables
* refactor(test): add an multistage exclusive for testing
* fix(cd): remove tests as a runtime dependency
* fix(cd): use default service account with cloud-platform scope
* fix(cd): revert checkout actions
* fix: use GA c2 instead of Preview c2d machine types
* fix(actions): remove workflow_dispatch from patched actions
This causes GitHub confusion as it can't determined which of the actions using workflow_dispatch is the right one
* fix(actions): remove patches from push actions
* test: validate changes on each push
* fix(test): wrong file syntax on test job
* fix(test): add missing env parameters
* fix(docker): Do not rebuild to download params and run tests
* fix(test): setup gcloud and loginto artifact just when needed
Try not to rebuild the tests
* fix(test): use GCP container to sync past mandatory checkpoint
* fix(test): missing separators
* test
* fix(test): mount the available disk
* push
* refactor(test): merge disk regeneration into test.yml
* fix(cd): minor typo fixes
* fix(docker): rebuild on .github changes
* fix(cd): keep compatibility with gcr.io
To prevent conflicts between registries, and migrate when the time is right, we'll keep pushing to both registries and use github actions cache to prevent conflicts between artifacts.
* fix(cd): typo and scope
* fix(cd): typos everywhere
* refactor(test): use smarter docker wait and keep old registry
* fix(cd): do not constraint the CPUs for bigger machines
* revert(cd): reduce PR diff as there's a separate one for tests
* fix(docker): add .github as it has no impact on caching
* fix(test): run command correctly
* fix(test): wiat and create image if previous step succeded
* force rebuild
* fix(test): do not restrict interdependant steps based on event
* force push
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* fix(test): remove all hardoced values and increase disks
* fix(test): use correct commands on deploy
* fix(test): use args as required by google
* fix(docker): try not to invalidate zebrad download cache
* fix(test): minor typo
* refactor(test): decouple jobs for better modularity
This also allows faster tests as testing Zunstable won't be a dependency and it can't stop already started jobs if it fails.
* fix(test): Do not try to execute ss and commands in one line
* fix(test): do not show undeeded information in the terminal
* fix(test): sleep befor/after machine creation/deletion
* fix(docker): do not download zcash params twice
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* merge: docker-actions-refactor into docker-test-refactor
* test docker wait scenarios
* fix(docker): $HOME variables is not being expanded
* fix(test): allow docker wait to work correctly
* fix(docker): do not use variables while using COPY
* fix(docker): allow to use zebrad as a command
* fix(cd): use test .yml from main
* fix(cd): Do not duplicate network values
The Dockerfile has an ARG with a default value of 'Mainnet', if this value is changed it will be done manually on a workflow_dispatch, making the ENV option a uneeded duplicate in this workflow
* fix(test): use bigger machine type for compute intensive tasks
* refactor(test): add tests in CI file
* fix(test): remove duplicated tests
* fix(test): typo
* test: build on .github changes temporarily
* fix(test): bigger machines have no effect on sync times
* feat: add an image to inherit from with zcash params
* fix(cd): use the right image name and allow push to test
* fix(cd): use the right docker target and remove extra builds
* refactor(docker): use cached zcash params from previous build
* fix(cd): finalize for merging
* imp(cd): add double safety measure for production
* fix(cd): use specific SHA for containers
* fix(cd): use latest gcloud action version
* fix(test): use the network as Mainnet and remove the uppercase from tests
* fix(test): run disk regeneration on specific file change
Just run this regeneration when changing the following files:
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state/disk_format.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/constants.rs
* refactor(test): seggregate disks regeneration from tests
Allow to regenerate disks without running tests, and to run tests from previous disk regeneration.
Disk will be regenerated just if specific files were changed, or triggered manually.
Tests will run just if a disk regeneration was not manually triggered.
* fix(test): gcp disks require lower case conventions
* fix(test): validate logs being emmited by docker
GHA is transforming is somehow transforwing the variable to lowercase also, so we're changint it to adapt to it
* test
* fix(test): force tty terminal
* fix(test): use a one line command to test terminal output
* fix(test): always delete test instance
* fix(test): use short SHA from the PR head
Using the SHA from the base, creates confusion and it's not accurate with the SHA being shown and used on GitHub.
We have to keep both as manual runs with `workflow_dispatch` does not have a PR SHA
* fix(ci): do not trigger CI on docker changes
There's no impact in this workflow when a change is done in the dockerfile
* Instead of runing cargo test when the instance gets created, run this commands afterwards in a different step.
As GHA TTY is not working as expected, and workarounds does not play nicely with `gcloud compute ssh` actions/runner#241 (comment) we decided to get the container name from the logs, log directly to the container and run the cargo command from there.
* doc(test): document reasoning for new steps
* fix(test): increase machine type and ssh timeout
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
* finalize(test): do not rebuild image when changing actions
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
2022-02-15 16:54:16 -08:00
|
|
|
|
|
|
|
|
|
- name: Run zebrad large sync tests
|
|
|
|
|
run: |
|
2023-04-12 23:56:21 -07:00
|
|
|
|
docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
|
2023-07-25 15:50:45 -07:00
|
|
|
|
docker run -e NETWORK --name zebrad-tests -t ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features lightwalletd-grpc-tests --package zebrad --test acceptance -- --nocapture --include-ignored sync_large_checkpoints_
|
|
|
|
|
env:
|
|
|
|
|
NETWORK: ${{ inputs.network || vars.ZCASH_NETWORK }}
|
refactor(test): dockerize tests and run sync in detached mode (#3459)
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): use newer google auth action
* fix (cd): use newer secret as gcp credential
* fix (docker): do not create extra directories
* fix (docker): ignore .github for caching purposes
* fix (docker): use latest rust
* fix (cd): bump build timeout
* fix: use a better name for manual deployment
* refactor (docker): use standard directories for executable
* fix (cd): most systems expect a "latest" tag
Caching from the latest image is one of the main reasons to add this extra tag. Before this commit, the inline cache was not being used.
* fix (cd): push the build image and the cache separately
The inline cache exporter only supports `min` cache mode. To enable `max` cache mode, push the image and the cache separately by using the registry cache exporter.
This also allows for smaller release images.
* fix (cd): remove unused GHA cache
We're leveraging the registry to cache the actions, instead of using the 10GB limits from Github Actions cache storage
* refactor (cd): use cargo-chef for caching rust deps
* fix: move build system deps before cargo cheg cook
* fix (release): use newer debian to reduce vulnerabilities
* fix (cd): use same zone, region and service accounts
* fix (cd): use same disk size and type for all deployments
* refactor (cd): activate interactive shells
Use interactive shells for manual and test deployments. This allow greater flexibility if troubleshooting is needed inside the machines
* refactor (test): use docker artifact from registry
Instead of using a VM to SSH into in to build and test. Build in GHA (to have the logs available), run the workspace tests in GHA, and just run the sync tests in GCP
Use a cintainer VM with zebra's image directly on it, and pass the needed parameters to run the Sync past mandatory checkpoint.
* tmp (cd): bump timeout for building from scratch
* tmp (test): bump build time
* fix (cd, test): bump build time-out to 210 minutes
* fix (docker): do not build with different settings
Compiling might be slow because different steps are compiling the same code 2-4 times because of the variations
* revert (docker): do not fix the rust version
* fix (docker): build on the root directory
* refactor(docker): Use base image commands and tools
* fix (cd): use correct variables & values, add build concurrency
* fix(cd): use Mainnet instead of mainnet
* imp: remove checkout as Buildkit uses the git context
* fix (docker): just Buildkit uses a .dockerignore in a path
* imp (cd): just use needed variables in the right place
* imp (cd): do not checkout if not needed
* test: run on push
* refactor(docker): reduce build changes
* fix(cd): not checking out was limiting some variables
* refactor(test): add an multistage exclusive for testing
* fix(cd): remove tests as a runtime dependency
* fix(cd): use default service account with cloud-platform scope
* fix(cd): revert checkout actions
* fix: use GA c2 instead of Preview c2d machine types
* fix(actions): remove workflow_dispatch from patched actions
This causes GitHub confusion as it can't determined which of the actions using workflow_dispatch is the right one
* fix(actions): remove patches from push actions
* test: validate changes on each push
* fix(test): wrong file syntax on test job
* fix(test): add missing env parameters
* fix(docker): Do not rebuild to download params and run tests
* fix(test): setup gcloud and loginto artifact just when needed
Try not to rebuild the tests
* fix(test): use GCP container to sync past mandatory checkpoint
* fix(test): missing separators
* test
* fix(test): mount the available disk
* push
* refactor(test): merge disk regeneration into test.yml
* fix(cd): minor typo fixes
* fix(docker): rebuild on .github changes
* fix(cd): keep compatibility with gcr.io
To prevent conflicts between registries, and migrate when the time is right, we'll keep pushing to both registries and use github actions cache to prevent conflicts between artifacts.
* fix(cd): typo and scope
* fix(cd): typos everywhere
* refactor(test): use smarter docker wait and keep old registry
* fix(cd): do not constraint the CPUs for bigger machines
* revert(cd): reduce PR diff as there's a separate one for tests
* fix(docker): add .github as it has no impact on caching
* fix(test): run command correctly
* fix(test): wiat and create image if previous step succeded
* force rebuild
* fix(test): do not restrict interdependant steps based on event
* force push
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* fix(test): remove all hardoced values and increase disks
* fix(test): use correct commands on deploy
* fix(test): use args as required by google
* fix(docker): try not to invalidate zebrad download cache
* fix(test): minor typo
* refactor(test): decouple jobs for better modularity
This also allows faster tests as testing Zunstable won't be a dependency and it can't stop already started jobs if it fails.
* fix(test): Do not try to execute ss and commands in one line
* fix(test): do not show undeeded information in the terminal
* fix(test): sleep befor/after machine creation/deletion
* fix(docker): do not download zcash params twice
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* merge: docker-actions-refactor into docker-test-refactor
* test docker wait scenarios
* fix(docker): $HOME variables is not being expanded
* fix(test): allow docker wait to work correctly
* fix(docker): do not use variables while using COPY
* fix(docker): allow to use zebrad as a command
* fix(cd): use test .yml from main
* fix(cd): Do not duplicate network values
The Dockerfile has an ARG with a default value of 'Mainnet', if this value is changed it will be done manually on a workflow_dispatch, making the ENV option a uneeded duplicate in this workflow
* fix(test): use bigger machine type for compute intensive tasks
* refactor(test): add tests in CI file
* fix(test): remove duplicated tests
* fix(test): typo
* test: build on .github changes temporarily
* fix(test): bigger machines have no effect on sync times
* feat: add an image to inherit from with zcash params
* fix(cd): use the right image name and allow push to test
* fix(cd): use the right docker target and remove extra builds
* refactor(docker): use cached zcash params from previous build
* fix(cd): finalize for merging
* imp(cd): add double safety measure for production
* fix(cd): use specific SHA for containers
* fix(cd): use latest gcloud action version
* fix(test): use the network as Mainnet and remove the uppercase from tests
* fix(test): run disk regeneration on specific file change
Just run this regeneration when changing the following files:
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state/disk_format.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/constants.rs
* refactor(test): seggregate disks regeneration from tests
Allow to regenerate disks without running tests, and to run tests from previous disk regeneration.
Disk will be regenerated just if specific files were changed, or triggered manually.
Tests will run just if a disk regeneration was not manually triggered.
* fix(test): gcp disks require lower case conventions
* fix(test): validate logs being emmited by docker
GHA is transforming is somehow transforwing the variable to lowercase also, so we're changint it to adapt to it
* test
* fix(test): force tty terminal
* fix(test): use a one line command to test terminal output
* fix(test): always delete test instance
* fix(test): use short SHA from the PR head
Using the SHA from the base, creates confusion and it's not accurate with the SHA being shown and used on GitHub.
We have to keep both as manual runs with `workflow_dispatch` does not have a PR SHA
* fix(ci): do not trigger CI on docker changes
There's no impact in this workflow when a change is done in the dockerfile
* Instead of runing cargo test when the instance gets created, run this commands afterwards in a different step.
As GHA TTY is not working as expected, and workarounds does not play nicely with `gcloud compute ssh` actions/runner#241 (comment) we decided to get the container name from the logs, log directly to the container and run the cargo command from there.
* doc(test): document reasoning for new steps
* fix(test): increase machine type and ssh timeout
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
* finalize(test): do not rebuild image when changing actions
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
2022-02-15 16:54:16 -08:00
|
|
|
|
|
2022-05-11 07:06:58 -07:00
|
|
|
|
# Test launching lightwalletd with an empty lightwalletd and Zebra state.
|
|
|
|
|
#
|
|
|
|
|
# (We activate the gRPC feature to avoid recompiling `zebrad`, but we don't actually run any gRPC tests.)
|
2022-03-02 01:00:55 -08:00
|
|
|
|
test-lightwalletd-integration:
|
|
|
|
|
name: Test integration with lightwalletd
|
|
|
|
|
runs-on: ubuntu-latest
|
|
|
|
|
needs: build
|
2023-01-31 12:39:11 -08:00
|
|
|
|
if: ${{ github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
|
2022-03-02 01:00:55 -08:00
|
|
|
|
steps:
|
2023-05-18 09:44:39 -07:00
|
|
|
|
- uses: r7kamura/rust-problem-matchers@v1.3.0
|
|
|
|
|
|
2022-03-02 01:00:55 -08:00
|
|
|
|
- name: Inject slug/short variables
|
|
|
|
|
uses: rlespinasse/github-slug-action@v4
|
2022-03-21 16:07:01 -07:00
|
|
|
|
with:
|
|
|
|
|
short-length: 7
|
2022-03-02 01:00:55 -08:00
|
|
|
|
|
2022-05-04 15:55:02 -07:00
|
|
|
|
- name: Run tests with empty lightwalletd launch
|
2022-03-02 01:00:55 -08:00
|
|
|
|
run: |
|
2023-04-12 23:56:21 -07:00
|
|
|
|
docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
|
2023-07-25 15:50:45 -07:00
|
|
|
|
docker run -e NETWORK -e ZEBRA_TEST_LIGHTWALLETD --name lightwalletd-tests -t ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features lightwalletd-grpc-tests --package zebrad --test acceptance -- --nocapture --include-ignored lightwalletd_integration
|
2022-03-02 01:00:55 -08:00
|
|
|
|
env:
|
|
|
|
|
ZEBRA_TEST_LIGHTWALLETD: '1'
|
2023-07-25 15:50:45 -07:00
|
|
|
|
NETWORK: ${{ inputs.network || vars.ZCASH_NETWORK }}
|
2022-03-02 01:00:55 -08:00
|
|
|
|
|
2023-01-22 22:41:59 -08:00
|
|
|
|
# Test that Zebra works using the default config with the latest Zebra version
|
|
|
|
|
test-configuration-file:
|
|
|
|
|
name: Test Zebra default Docker config file
|
2023-05-30 05:57:08 -07:00
|
|
|
|
timeout-minutes: 15
|
2023-01-22 22:41:59 -08:00
|
|
|
|
runs-on: ubuntu-latest
|
|
|
|
|
needs: build
|
2023-01-31 12:39:11 -08:00
|
|
|
|
if: ${{ github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
|
2023-01-22 22:41:59 -08:00
|
|
|
|
steps:
|
2023-05-18 09:44:39 -07:00
|
|
|
|
- uses: r7kamura/rust-problem-matchers@v1.3.0
|
|
|
|
|
|
2023-01-22 22:41:59 -08:00
|
|
|
|
- name: Inject slug/short variables
|
|
|
|
|
uses: rlespinasse/github-slug-action@v4
|
|
|
|
|
with:
|
|
|
|
|
short-length: 7
|
|
|
|
|
|
|
|
|
|
- name: Run tests using the default config
|
|
|
|
|
run: |
|
|
|
|
|
set -ex
|
2023-04-12 23:56:21 -07:00
|
|
|
|
docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
|
2023-07-25 15:50:45 -07:00
|
|
|
|
docker run -e NETWORK --detach --name default-conf-tests -t ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} zebrad start
|
2023-01-22 22:41:59 -08:00
|
|
|
|
EXIT_STATUS=$(docker logs --tail all --follow default-conf-tests 2>&1 | grep -q --extended-regexp --max-count=1 -e 'estimated progress to chain tip.*BeforeOverwinter'; echo $?; )
|
|
|
|
|
docker stop default-conf-tests
|
|
|
|
|
docker logs default-conf-tests
|
|
|
|
|
exit "$EXIT_STATUS"
|
2023-07-25 15:50:45 -07:00
|
|
|
|
env:
|
|
|
|
|
NETWORK: ${{ inputs.network || vars.ZCASH_NETWORK }}
|
2023-01-22 22:41:59 -08:00
|
|
|
|
|
|
|
|
|
# Test that Zebra works using the $ZEBRA_CONF_PATH config
|
|
|
|
|
test-zebra-conf-path:
|
|
|
|
|
name: Test Zebra custom Docker config file
|
2023-05-30 05:57:08 -07:00
|
|
|
|
timeout-minutes: 15
|
2023-01-22 22:41:59 -08:00
|
|
|
|
runs-on: ubuntu-latest
|
|
|
|
|
needs: build
|
2023-01-31 12:39:11 -08:00
|
|
|
|
if: ${{ github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
|
2023-01-22 22:41:59 -08:00
|
|
|
|
steps:
|
2023-05-18 09:44:39 -07:00
|
|
|
|
- uses: r7kamura/rust-problem-matchers@v1.3.0
|
|
|
|
|
|
2023-01-22 22:41:59 -08:00
|
|
|
|
- name: Inject slug/short variables
|
|
|
|
|
uses: rlespinasse/github-slug-action@v4
|
|
|
|
|
with:
|
|
|
|
|
short-length: 7
|
|
|
|
|
|
|
|
|
|
- name: Run tests using the $ZEBRA_CONF_PATH
|
|
|
|
|
run: |
|
|
|
|
|
set -ex
|
2023-04-12 23:56:21 -07:00
|
|
|
|
docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
|
2023-07-25 15:50:45 -07:00
|
|
|
|
docker run -e NETWORK --detach -e ZEBRA_CONF_PATH --name variable-conf-tests -t ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} -c $ZEBRA_CONF_PATH start
|
2023-01-22 22:41:59 -08:00
|
|
|
|
EXIT_STATUS=$(docker logs --tail all --follow variable-conf-tests 2>&1 | grep -q --extended-regexp --max-count=1 -e 'v1.0.0-rc.2.toml'; echo $?; )
|
|
|
|
|
docker stop variable-conf-tests
|
|
|
|
|
docker logs variable-conf-tests
|
|
|
|
|
exit "$EXIT_STATUS"
|
|
|
|
|
env:
|
|
|
|
|
ZEBRA_CONF_PATH: 'zebrad/tests/common/configs/v1.0.0-rc.2.toml'
|
2023-07-25 15:50:45 -07:00
|
|
|
|
NETWORK: ${{ inputs.network || vars.ZCASH_NETWORK }}
|
2023-01-22 22:41:59 -08:00
|
|
|
|
|
2023-07-10 11:20:50 -07:00
|
|
|
|
# END TODO: make the non-cached-state tests use:
|
|
|
|
|
# network: ${{ inputs.network || vars.ZCASH_NETWORK }}
|
|
|
|
|
|
2022-09-06 22:06:34 -07:00
|
|
|
|
# zebrad cached checkpoint state tests
|
|
|
|
|
|
2022-05-19 16:07:28 -07:00
|
|
|
|
# Regenerate mandatory checkpoint Zebra cached state disks.
|
|
|
|
|
#
|
2022-05-05 14:52:56 -07:00
|
|
|
|
# Runs:
|
2022-05-19 16:07:28 -07:00
|
|
|
|
# - on every PR update, but only if there's no available disk matching the actual state version from constants.rs
|
|
|
|
|
# - on request, using workflow_dispatch with regenerate-disks
|
|
|
|
|
#
|
|
|
|
|
# Note: the output from get-available-disks should match with the caller workflow inputs
|
refactor(test): dockerize tests and run sync in detached mode (#3459)
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): use newer google auth action
* fix (cd): use newer secret as gcp credential
* fix (docker): do not create extra directories
* fix (docker): ignore .github for caching purposes
* fix (docker): use latest rust
* fix (cd): bump build timeout
* fix: use a better name for manual deployment
* refactor (docker): use standard directories for executable
* fix (cd): most systems expect a "latest" tag
Caching from the latest image is one of the main reasons to add this extra tag. Before this commit, the inline cache was not being used.
* fix (cd): push the build image and the cache separately
The inline cache exporter only supports `min` cache mode. To enable `max` cache mode, push the image and the cache separately by using the registry cache exporter.
This also allows for smaller release images.
* fix (cd): remove unused GHA cache
We're leveraging the registry to cache the actions, instead of using the 10GB limits from Github Actions cache storage
* refactor (cd): use cargo-chef for caching rust deps
* fix: move build system deps before cargo cheg cook
* fix (release): use newer debian to reduce vulnerabilities
* fix (cd): use same zone, region and service accounts
* fix (cd): use same disk size and type for all deployments
* refactor (cd): activate interactive shells
Use interactive shells for manual and test deployments. This allow greater flexibility if troubleshooting is needed inside the machines
* refactor (test): use docker artifact from registry
Instead of using a VM to SSH into in to build and test. Build in GHA (to have the logs available), run the workspace tests in GHA, and just run the sync tests in GCP
Use a cintainer VM with zebra's image directly on it, and pass the needed parameters to run the Sync past mandatory checkpoint.
* tmp (cd): bump timeout for building from scratch
* tmp (test): bump build time
* fix (cd, test): bump build time-out to 210 minutes
* fix (docker): do not build with different settings
Compiling might be slow because different steps are compiling the same code 2-4 times because of the variations
* revert (docker): do not fix the rust version
* fix (docker): build on the root directory
* refactor(docker): Use base image commands and tools
* fix (cd): use correct variables & values, add build concurrency
* fix(cd): use Mainnet instead of mainnet
* imp: remove checkout as Buildkit uses the git context
* fix (docker): just Buildkit uses a .dockerignore in a path
* imp (cd): just use needed variables in the right place
* imp (cd): do not checkout if not needed
* test: run on push
* refactor(docker): reduce build changes
* fix(cd): not checking out was limiting some variables
* refactor(test): add an multistage exclusive for testing
* fix(cd): remove tests as a runtime dependency
* fix(cd): use default service account with cloud-platform scope
* fix(cd): revert checkout actions
* fix: use GA c2 instead of Preview c2d machine types
* fix(actions): remove workflow_dispatch from patched actions
This causes GitHub confusion as it can't determined which of the actions using workflow_dispatch is the right one
* fix(actions): remove patches from push actions
* test: validate changes on each push
* fix(test): wrong file syntax on test job
* fix(test): add missing env parameters
* fix(docker): Do not rebuild to download params and run tests
* fix(test): setup gcloud and loginto artifact just when needed
Try not to rebuild the tests
* fix(test): use GCP container to sync past mandatory checkpoint
* fix(test): missing separators
* test
* fix(test): mount the available disk
* push
* refactor(test): merge disk regeneration into test.yml
* fix(cd): minor typo fixes
* fix(docker): rebuild on .github changes
* fix(cd): keep compatibility with gcr.io
To prevent conflicts between registries, and migrate when the time is right, we'll keep pushing to both registries and use github actions cache to prevent conflicts between artifacts.
* fix(cd): typo and scope
* fix(cd): typos everywhere
* refactor(test): use smarter docker wait and keep old registry
* fix(cd): do not constraint the CPUs for bigger machines
* revert(cd): reduce PR diff as there's a separate one for tests
* fix(docker): add .github as it has no impact on caching
* fix(test): run command correctly
* fix(test): wiat and create image if previous step succeded
* force rebuild
* fix(test): do not restrict interdependant steps based on event
* force push
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* fix(test): remove all hardoced values and increase disks
* fix(test): use correct commands on deploy
* fix(test): use args as required by google
* fix(docker): try not to invalidate zebrad download cache
* fix(test): minor typo
* refactor(test): decouple jobs for better modularity
This also allows faster tests as testing Zunstable won't be a dependency and it can't stop already started jobs if it fails.
* fix(test): Do not try to execute ss and commands in one line
* fix(test): do not show undeeded information in the terminal
* fix(test): sleep befor/after machine creation/deletion
* fix(docker): do not download zcash params twice
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* merge: docker-actions-refactor into docker-test-refactor
* test docker wait scenarios
* fix(docker): $HOME variables is not being expanded
* fix(test): allow docker wait to work correctly
* fix(docker): do not use variables while using COPY
* fix(docker): allow to use zebrad as a command
* fix(cd): use test .yml from main
* fix(cd): Do not duplicate network values
The Dockerfile has an ARG with a default value of 'Mainnet', if this value is changed it will be done manually on a workflow_dispatch, making the ENV option a uneeded duplicate in this workflow
* fix(test): use bigger machine type for compute intensive tasks
* refactor(test): add tests in CI file
* fix(test): remove duplicated tests
* fix(test): typo
* test: build on .github changes temporarily
* fix(test): bigger machines have no effect on sync times
* feat: add an image to inherit from with zcash params
* fix(cd): use the right image name and allow push to test
* fix(cd): use the right docker target and remove extra builds
* refactor(docker): use cached zcash params from previous build
* fix(cd): finalize for merging
* imp(cd): add double safety measure for production
* fix(cd): use specific SHA for containers
* fix(cd): use latest gcloud action version
* fix(test): use the network as Mainnet and remove the uppercase from tests
* fix(test): run disk regeneration on specific file change
Just run this regeneration when changing the following files:
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state/disk_format.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/constants.rs
* refactor(test): seggregate disks regeneration from tests
Allow to regenerate disks without running tests, and to run tests from previous disk regeneration.
Disk will be regenerated just if specific files were changed, or triggered manually.
Tests will run just if a disk regeneration was not manually triggered.
* fix(test): gcp disks require lower case conventions
* fix(test): validate logs being emmited by docker
GHA is transforming is somehow transforwing the variable to lowercase also, so we're changint it to adapt to it
* test
* fix(test): force tty terminal
* fix(test): use a one line command to test terminal output
* fix(test): always delete test instance
* fix(test): use short SHA from the PR head
Using the SHA from the base, creates confusion and it's not accurate with the SHA being shown and used on GitHub.
We have to keep both as manual runs with `workflow_dispatch` does not have a PR SHA
* fix(ci): do not trigger CI on docker changes
There's no impact in this workflow when a change is done in the dockerfile
* Instead of runing cargo test when the instance gets created, run this commands afterwards in a different step.
As GHA TTY is not working as expected, and workarounds does not play nicely with `gcloud compute ssh` actions/runner#241 (comment) we decided to get the container name from the logs, log directly to the container and run the cargo command from there.
* doc(test): document reasoning for new steps
* fix(test): increase machine type and ssh timeout
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
* finalize(test): do not rebuild image when changing actions
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
2022-02-15 16:54:16 -08:00
|
|
|
|
regenerate-stateful-disks:
|
2022-05-23 20:10:53 -07:00
|
|
|
|
name: Zebra checkpoint
|
2022-05-19 16:07:28 -07:00
|
|
|
|
needs: [ build, get-available-disks ]
|
2022-05-09 12:54:16 -07:00
|
|
|
|
uses: ./.github/workflows/deploy-gcp-tests.yml
|
2022-05-19 16:07:28 -07:00
|
|
|
|
if: ${{ !fromJSON(needs.get-available-disks.outputs.zebra_checkpoint_disk) || github.event.inputs.regenerate-disks == 'true' }}
|
2022-05-04 15:55:02 -07:00
|
|
|
|
with:
|
2022-09-08 13:25:00 -07:00
|
|
|
|
app_name: zebrad
|
2022-05-04 15:55:02 -07:00
|
|
|
|
test_id: sync-to-checkpoint
|
|
|
|
|
test_description: Test sync up to mandatory checkpoint
|
2023-07-25 15:50:45 -07:00
|
|
|
|
test_variables: '-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_DISK_REBUILD=1 -e ZEBRA_FORCE_USE_COLOR=1'
|
2022-05-04 15:55:02 -07:00
|
|
|
|
needs_zebra_state: false
|
|
|
|
|
saves_to_disk: true
|
2023-07-13 14:36:15 -07:00
|
|
|
|
force_save_to_disk: ${{ inputs.force_save_to_disk || false }}
|
2022-05-04 15:55:02 -07:00
|
|
|
|
disk_suffix: checkpoint
|
2022-09-06 22:06:34 -07:00
|
|
|
|
height_grep_text: 'flushing database to disk .*height.*=.*Height.*\('
|
2022-11-21 10:18:26 -08:00
|
|
|
|
secrets: inherit
|
2022-10-19 05:01:26 -07:00
|
|
|
|
# We want to prevent multiple checkpoint syncs running at the same time,
|
|
|
|
|
# but we don't want to cancel running syncs on `main` if a new PR gets merged,
|
|
|
|
|
# because we might never get a finished sync.
|
|
|
|
|
#
|
|
|
|
|
# See the concurrency comment on the zebrad test-full-sync job for details.
|
2022-11-07 14:29:37 -08:00
|
|
|
|
concurrency:
|
2023-05-15 16:04:33 -07:00
|
|
|
|
group: ${{ github.workflow }}−manual-${{ format('{0}', github.event.inputs.regenerate-disks == 'true') }}-regenerate-stateful-disks
|
2022-10-19 05:01:26 -07:00
|
|
|
|
cancel-in-progress: false
|
2022-05-01 09:52:57 -07:00
|
|
|
|
|
2022-05-05 14:52:56 -07:00
|
|
|
|
# Test that Zebra syncs and fully validates a few thousand blocks from a cached mandatory checkpoint disk
|
2022-05-19 16:07:28 -07:00
|
|
|
|
#
|
|
|
|
|
# If the state version has changed, waits for the new cached state to be created.
|
|
|
|
|
# Otherwise, if the state rebuild was skipped, runs immediately after the build job.
|
refactor(test): dockerize tests and run sync in detached mode (#3459)
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): overall pipeline improvement
- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
* refactor (cd): use newer google auth action
* fix (cd): use newer secret as gcp credential
* fix (docker): do not create extra directories
* fix (docker): ignore .github for caching purposes
* fix (docker): use latest rust
* fix (cd): bump build timeout
* fix: use a better name for manual deployment
* refactor (docker): use standard directories for executable
* fix (cd): most systems expect a "latest" tag
Caching from the latest image is one of the main reasons to add this extra tag. Before this commit, the inline cache was not being used.
* fix (cd): push the build image and the cache separately
The inline cache exporter only supports `min` cache mode. To enable `max` cache mode, push the image and the cache separately by using the registry cache exporter.
This also allows for smaller release images.
* fix (cd): remove unused GHA cache
We're leveraging the registry to cache the actions, instead of using the 10GB limits from Github Actions cache storage
* refactor (cd): use cargo-chef for caching rust deps
* fix: move build system deps before cargo cheg cook
* fix (release): use newer debian to reduce vulnerabilities
* fix (cd): use same zone, region and service accounts
* fix (cd): use same disk size and type for all deployments
* refactor (cd): activate interactive shells
Use interactive shells for manual and test deployments. This allow greater flexibility if troubleshooting is needed inside the machines
* refactor (test): use docker artifact from registry
Instead of using a VM to SSH into in to build and test. Build in GHA (to have the logs available), run the workspace tests in GHA, and just run the sync tests in GCP
Use a cintainer VM with zebra's image directly on it, and pass the needed parameters to run the Sync past mandatory checkpoint.
* tmp (cd): bump timeout for building from scratch
* tmp (test): bump build time
* fix (cd, test): bump build time-out to 210 minutes
* fix (docker): do not build with different settings
Compiling might be slow because different steps are compiling the same code 2-4 times because of the variations
* revert (docker): do not fix the rust version
* fix (docker): build on the root directory
* refactor(docker): Use base image commands and tools
* fix (cd): use correct variables & values, add build concurrency
* fix(cd): use Mainnet instead of mainnet
* imp: remove checkout as Buildkit uses the git context
* fix (docker): just Buildkit uses a .dockerignore in a path
* imp (cd): just use needed variables in the right place
* imp (cd): do not checkout if not needed
* test: run on push
* refactor(docker): reduce build changes
* fix(cd): not checking out was limiting some variables
* refactor(test): add an multistage exclusive for testing
* fix(cd): remove tests as a runtime dependency
* fix(cd): use default service account with cloud-platform scope
* fix(cd): revert checkout actions
* fix: use GA c2 instead of Preview c2d machine types
* fix(actions): remove workflow_dispatch from patched actions
This causes GitHub confusion as it can't determined which of the actions using workflow_dispatch is the right one
* fix(actions): remove patches from push actions
* test: validate changes on each push
* fix(test): wrong file syntax on test job
* fix(test): add missing env parameters
* fix(docker): Do not rebuild to download params and run tests
* fix(test): setup gcloud and loginto artifact just when needed
Try not to rebuild the tests
* fix(test): use GCP container to sync past mandatory checkpoint
* fix(test): missing separators
* test
* fix(test): mount the available disk
* push
* refactor(test): merge disk regeneration into test.yml
* fix(cd): minor typo fixes
* fix(docker): rebuild on .github changes
* fix(cd): keep compatibility with gcr.io
To prevent conflicts between registries, and migrate when the time is right, we'll keep pushing to both registries and use github actions cache to prevent conflicts between artifacts.
* fix(cd): typo and scope
* fix(cd): typos everywhere
* refactor(test): use smarter docker wait and keep old registry
* fix(cd): do not constraint the CPUs for bigger machines
* revert(cd): reduce PR diff as there's a separate one for tests
* fix(docker): add .github as it has no impact on caching
* fix(test): run command correctly
* fix(test): wiat and create image if previous step succeded
* force rebuild
* fix(test): do not restrict interdependant steps based on event
* force push
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* fix(test): remove all hardoced values and increase disks
* fix(test): use correct commands on deploy
* fix(test): use args as required by google
* fix(docker): try not to invalidate zebrad download cache
* fix(test): minor typo
* refactor(test): decouple jobs for better modularity
This also allows faster tests as testing Zunstable won't be a dependency and it can't stop already started jobs if it fails.
* fix(test): Do not try to execute ss and commands in one line
* fix(test): do not show undeeded information in the terminal
* fix(test): sleep befor/after machine creation/deletion
* fix(docker): do not download zcash params twice
* feat(docker): add google OS Config agent
Use a separate step to have better flexibility in case a better approach is available
* merge: docker-actions-refactor into docker-test-refactor
* test docker wait scenarios
* fix(docker): $HOME variables is not being expanded
* fix(test): allow docker wait to work correctly
* fix(docker): do not use variables while using COPY
* fix(docker): allow to use zebrad as a command
* fix(cd): use test .yml from main
* fix(cd): Do not duplicate network values
The Dockerfile has an ARG with a default value of 'Mainnet', if this value is changed it will be done manually on a workflow_dispatch, making the ENV option a uneeded duplicate in this workflow
* fix(test): use bigger machine type for compute intensive tasks
* refactor(test): add tests in CI file
* fix(test): remove duplicated tests
* fix(test): typo
* test: build on .github changes temporarily
* fix(test): bigger machines have no effect on sync times
* feat: add an image to inherit from with zcash params
* fix(cd): use the right image name and allow push to test
* fix(cd): use the right docker target and remove extra builds
* refactor(docker): use cached zcash params from previous build
* fix(cd): finalize for merging
* imp(cd): add double safety measure for production
* fix(cd): use specific SHA for containers
* fix(cd): use latest gcloud action version
* fix(test): use the network as Mainnet and remove the uppercase from tests
* fix(test): run disk regeneration on specific file change
Just run this regeneration when changing the following files:
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state/disk_format.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/constants.rs
* refactor(test): seggregate disks regeneration from tests
Allow to regenerate disks without running tests, and to run tests from previous disk regeneration.
Disk will be regenerated just if specific files were changed, or triggered manually.
Tests will run just if a disk regeneration was not manually triggered.
* fix(test): gcp disks require lower case conventions
* fix(test): validate logs being emmited by docker
GHA is transforming is somehow transforwing the variable to lowercase also, so we're changint it to adapt to it
* test
* fix(test): force tty terminal
* fix(test): use a one line command to test terminal output
* fix(test): always delete test instance
* fix(test): use short SHA from the PR head
Using the SHA from the base, creates confusion and it's not accurate with the SHA being shown and used on GitHub.
We have to keep both as manual runs with `workflow_dispatch` does not have a PR SHA
* fix(ci): do not trigger CI on docker changes
There's no impact in this workflow when a change is done in the dockerfile
* Instead of runing cargo test when the instance gets created, run this commands afterwards in a different step.
As GHA TTY is not working as expected, and workarounds does not play nicely with `gcloud compute ssh` actions/runner#241 (comment) we decided to get the container name from the logs, log directly to the container and run the cargo command from there.
* doc(test): document reasoning for new steps
* fix(test): increase machine type and ssh timeout
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
* finalize(test): do not rebuild image when changing actions
* fix(test): run tests on creation and follow container logs
This allows to follow logs in Github Actions terminal, while the GCP container is still running.
Just delete the instance when following the logs ends successfully or fails
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
2022-02-15 16:54:16 -08:00
|
|
|
|
test-stateful-sync:
|
2022-05-23 20:10:53 -07:00
|
|
|
|
name: Zebra checkpoint update
|
2023-05-11 04:15:20 -07:00
|
|
|
|
needs: [ regenerate-stateful-disks, get-available-disks ]
|
2022-05-09 20:28:07 -07:00
|
|
|
|
uses: ./.github/workflows/deploy-gcp-tests.yml
|
2023-05-11 04:15:20 -07:00
|
|
|
|
if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_checkpoint_disk) || needs.regenerate-stateful-disks.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
|
2022-05-04 15:55:02 -07:00
|
|
|
|
with:
|
2022-09-08 13:25:00 -07:00
|
|
|
|
app_name: zebrad
|
2022-05-04 15:55:02 -07:00
|
|
|
|
test_id: sync-past-checkpoint
|
|
|
|
|
test_description: Test full validation sync from a cached state
|
2023-07-25 15:50:45 -07:00
|
|
|
|
test_variables: '-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_CHECKPOINT_SYNC=1 -e ZEBRA_FORCE_USE_COLOR=1'
|
2022-05-04 15:55:02 -07:00
|
|
|
|
needs_zebra_state: true
|
|
|
|
|
saves_to_disk: false
|
|
|
|
|
disk_suffix: checkpoint
|
2022-11-21 10:18:26 -08:00
|
|
|
|
secrets: inherit
|
2022-05-04 15:55:02 -07:00
|
|
|
|
|
2022-09-06 22:06:34 -07:00
|
|
|
|
# zebrad cached tip state tests
|
|
|
|
|
|
2023-05-03 16:14:22 -07:00
|
|
|
|
# Test that Zebra can run a full sync on mainnet,
|
2022-05-19 16:07:28 -07:00
|
|
|
|
# and regenerate chain tip Zebra cached state disks.
|
|
|
|
|
#
|
2023-05-03 16:14:22 -07:00
|
|
|
|
# This test always runs on mainnet.
|
|
|
|
|
#
|
2022-05-05 14:52:56 -07:00
|
|
|
|
# Runs:
|
2023-06-12 18:27:22 -07:00
|
|
|
|
# - on schedule, as defined at the top of the workflow
|
2023-05-03 16:14:22 -07:00
|
|
|
|
# - on every PR update, but only if the state version in constants.rs has no cached disk
|
|
|
|
|
# - in manual workflow runs, when run-full-sync is 'true' and network is 'Mainnet'
|
2022-05-19 16:07:28 -07:00
|
|
|
|
#
|
|
|
|
|
# Note: the output from get-available-disks should match with the caller workflow inputs
|
2022-05-04 15:55:02 -07:00
|
|
|
|
test-full-sync:
|
2022-05-23 20:10:53 -07:00
|
|
|
|
name: Zebra tip
|
2022-05-19 16:07:28 -07:00
|
|
|
|
needs: [ build, get-available-disks ]
|
2022-05-09 12:54:16 -07:00
|
|
|
|
uses: ./.github/workflows/deploy-gcp-tests.yml
|
2023-07-18 00:14:06 -07:00
|
|
|
|
if: ${{ github.event_name == 'schedule' || !fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || (github.event.inputs.run-full-sync == 'true' && (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet') }}
|
2022-05-04 15:55:02 -07:00
|
|
|
|
with:
|
2022-09-08 13:25:00 -07:00
|
|
|
|
app_name: zebrad
|
2022-05-04 15:55:02 -07:00
|
|
|
|
test_id: full-sync-to-tip
|
|
|
|
|
test_description: Test a full sync up to the tip
|
2022-11-08 20:42:04 -08:00
|
|
|
|
# The value of FULL_SYNC_MAINNET_TIMEOUT_MINUTES is currently ignored.
|
2023-05-03 16:14:22 -07:00
|
|
|
|
# TODO: update the test to use {{ input.network }} instead?
|
2023-07-25 15:50:45 -07:00
|
|
|
|
test_variables: '-e NETWORK=Mainnet -e FULL_SYNC_MAINNET_TIMEOUT_MINUTES=0 -e ZEBRA_FORCE_USE_COLOR=1'
|
2022-11-07 14:29:37 -08:00
|
|
|
|
# This test runs for longer than 6 hours, so it needs multiple jobs
|
|
|
|
|
is_long_test: true
|
2022-05-04 15:55:02 -07:00
|
|
|
|
needs_zebra_state: false
|
|
|
|
|
saves_to_disk: true
|
2023-07-13 14:36:15 -07:00
|
|
|
|
force_save_to_disk: ${{ inputs.force_save_to_disk || false }}
|
2022-05-04 15:55:02 -07:00
|
|
|
|
disk_suffix: tip
|
2022-09-06 22:06:34 -07:00
|
|
|
|
height_grep_text: 'current_height.*=.*Height.*\('
|
2022-11-21 10:18:26 -08:00
|
|
|
|
secrets: inherit
|
2022-10-19 05:01:26 -07:00
|
|
|
|
# We want to prevent multiple full zebrad syncs running at the same time,
|
|
|
|
|
# but we don't want to cancel running syncs on `main` if a new PR gets merged,
|
|
|
|
|
# because we might never get a finished sync.
|
2022-11-07 14:29:37 -08:00
|
|
|
|
#
|
2022-10-19 05:01:26 -07:00
|
|
|
|
# Instead, we let the first sync complete, then queue the latest pending sync, cancelling any syncs in between.
|
2022-08-29 17:11:05 -07:00
|
|
|
|
# (As the general workflow concurrency group just gets matched in Pull Requests,
|
|
|
|
|
# it has no impact on this job.)
|
2022-08-30 05:09:26 -07:00
|
|
|
|
#
|
|
|
|
|
# TODO:
|
2023-05-11 04:15:20 -07:00
|
|
|
|
# - allow multiple manual syncs on a branch by adding '-${{ github.run_id }}' when github.event.inputs.run-full-sync is true
|
2022-11-07 14:29:37 -08:00
|
|
|
|
concurrency:
|
2023-05-15 16:04:33 -07:00
|
|
|
|
group: ${{ github.workflow }}−manual-${{ format('{0}', github.event.inputs.run-full-sync == 'true') }}-test-full-sync
|
2022-08-29 17:11:05 -07:00
|
|
|
|
cancel-in-progress: false
|
2022-05-05 02:27:07 -07:00
|
|
|
|
|
2022-07-25 15:13:25 -07:00
|
|
|
|
# Test that Zebra can sync to the chain tip, using a cached Zebra tip state,
|
|
|
|
|
# without launching `lightwalletd`.
|
|
|
|
|
#
|
|
|
|
|
# Runs:
|
|
|
|
|
# - after every PR is merged to `main`
|
|
|
|
|
# - on every PR update
|
|
|
|
|
#
|
|
|
|
|
# If the state version has changed, waits for the new cached state to be created.
|
|
|
|
|
# Otherwise, if the state rebuild was skipped, runs immediately after the build job.
|
|
|
|
|
test-update-sync:
|
|
|
|
|
name: Zebra tip update
|
2023-05-11 04:15:20 -07:00
|
|
|
|
needs: [ test-full-sync, get-available-disks ]
|
2022-07-25 15:13:25 -07:00
|
|
|
|
uses: ./.github/workflows/deploy-gcp-tests.yml
|
2023-05-11 04:15:20 -07:00
|
|
|
|
if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
|
2022-07-25 15:13:25 -07:00
|
|
|
|
with:
|
|
|
|
|
app_name: zebrad
|
|
|
|
|
test_id: update-to-tip
|
|
|
|
|
test_description: Test syncing to tip with a Zebra tip state
|
2023-07-25 15:50:45 -07:00
|
|
|
|
test_variables: '-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_UPDATE_SYNC=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache'
|
2022-07-25 15:13:25 -07:00
|
|
|
|
needs_zebra_state: true
|
2022-08-25 06:09:20 -07:00
|
|
|
|
# update the disk on every PR, to increase CI speed
|
|
|
|
|
saves_to_disk: true
|
2023-07-13 14:36:15 -07:00
|
|
|
|
force_save_to_disk: ${{ inputs.force_save_to_disk || false }}
|
2022-07-25 15:13:25 -07:00
|
|
|
|
disk_suffix: tip
|
|
|
|
|
root_state_path: '/var/cache'
|
|
|
|
|
zebra_state_dir: 'zebrad-cache'
|
2022-09-06 22:06:34 -07:00
|
|
|
|
height_grep_text: 'current_height.*=.*Height.*\('
|
2022-11-21 10:18:26 -08:00
|
|
|
|
secrets: inherit
|
2022-07-25 15:13:25 -07:00
|
|
|
|
|
2023-05-03 16:14:22 -07:00
|
|
|
|
# zebra mainnet checkpoint generation tests
|
2023-04-26 21:39:43 -07:00
|
|
|
|
|
|
|
|
|
# Test that Zebra can generate mainnet checkpoints after syncing to the chain tip,
|
|
|
|
|
# using a cached Zebra tip state,
|
|
|
|
|
#
|
2023-05-03 16:14:22 -07:00
|
|
|
|
# This test always runs on mainnet.
|
|
|
|
|
#
|
2023-04-26 21:39:43 -07:00
|
|
|
|
# Runs:
|
|
|
|
|
# - after every PR is merged to `main`
|
|
|
|
|
# - on every PR update
|
|
|
|
|
#
|
|
|
|
|
# If the state version has changed, waits for the new cached state to be created.
|
|
|
|
|
# Otherwise, if the state rebuild was skipped, runs immediately after the build job.
|
|
|
|
|
generate-checkpoints-mainnet:
|
|
|
|
|
name: Generate checkpoints mainnet
|
2023-05-11 04:15:20 -07:00
|
|
|
|
needs: [ test-full-sync, get-available-disks ]
|
2023-04-26 21:39:43 -07:00
|
|
|
|
uses: ./.github/workflows/deploy-gcp-tests.yml
|
2023-05-11 04:15:20 -07:00
|
|
|
|
if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
|
2023-04-26 21:39:43 -07:00
|
|
|
|
with:
|
|
|
|
|
app_name: zebrad
|
|
|
|
|
test_id: generate-checkpoints-mainnet
|
|
|
|
|
test_description: Generate Zebra checkpoints on mainnet
|
2023-05-03 16:14:22 -07:00
|
|
|
|
# TODO: update the test to use {{ input.network }} instead?
|
2023-07-25 15:50:45 -07:00
|
|
|
|
test_variables: '-e NETWORK=Mainnet -e GENERATE_CHECKPOINTS_MAINNET=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache'
|
2023-05-03 16:14:22 -07:00
|
|
|
|
needs_zebra_state: true
|
|
|
|
|
# test-update-sync updates the disk on every PR, so we don't need to do it here
|
|
|
|
|
saves_to_disk: false
|
|
|
|
|
disk_suffix: tip
|
|
|
|
|
root_state_path: '/var/cache'
|
|
|
|
|
zebra_state_dir: 'zebrad-cache'
|
|
|
|
|
height_grep_text: 'current_height.*=.*Height.*\('
|
|
|
|
|
secrets: inherit
|
|
|
|
|
|
|
|
|
|
# zebra testnet checkpoint generation tests
|
|
|
|
|
#
|
|
|
|
|
# These tests will fail when testnet is unstable, they should not be required to merge.
|
|
|
|
|
#
|
|
|
|
|
# TODO: ignore failures on testnet, so any failures don't appear in the GitHub interface.
|
|
|
|
|
|
|
|
|
|
# Test that Zebra can run a full testnet sync, and regenerate chain tip Zebra cached state disks.
|
|
|
|
|
# This job always runs on testnet, regardless of any inputs or variable settings.
|
|
|
|
|
#
|
|
|
|
|
# Runs:
|
2023-06-12 18:27:22 -07:00
|
|
|
|
# - on schedule, as defined at the top of the workflow
|
2023-05-03 16:14:22 -07:00
|
|
|
|
# - on every PR update, but only if the state version in constants.rs has no cached disk
|
|
|
|
|
# - in manual workflow runs, when run-full-sync is 'true' and network is 'Testnet'
|
|
|
|
|
#
|
|
|
|
|
# Note: the output from get-available-disks-testnet should match with the caller workflow inputs
|
|
|
|
|
test-full-sync-testnet:
|
|
|
|
|
name: Zebra tip on testnet
|
|
|
|
|
needs: [ build, get-available-disks-testnet ]
|
|
|
|
|
uses: ./.github/workflows/deploy-gcp-tests.yml
|
2023-07-18 00:14:06 -07:00
|
|
|
|
if: ${{ (github.event_name == 'schedule' && vars.SCHEDULE_TESTNET_FULL_SYNC == 'true') || !fromJSON(needs.get-available-disks-testnet.outputs.zebra_tip_disk) || (github.event.inputs.run-full-sync == 'true' && (inputs.network || vars.ZCASH_NETWORK) == 'Testnet') }}
|
2023-05-03 16:14:22 -07:00
|
|
|
|
with:
|
|
|
|
|
app_name: zebrad
|
|
|
|
|
test_id: full-sync-to-tip-testnet
|
|
|
|
|
test_description: Test a full sync up to the tip on testnet
|
|
|
|
|
# The value of FULL_SYNC_TESTNET_TIMEOUT_MINUTES is currently ignored.
|
2023-07-25 15:50:45 -07:00
|
|
|
|
test_variables: '-e NETWORK=Testnet -e FULL_SYNC_TESTNET_TIMEOUT_MINUTES=0 -e ZEBRA_FORCE_USE_COLOR=1'
|
2023-05-10 15:09:38 -07:00
|
|
|
|
# A full testnet sync could take 2-10 hours in April 2023.
|
|
|
|
|
# The time varies a lot due to the small number of nodes.
|
|
|
|
|
is_long_test: true
|
2023-05-03 16:14:22 -07:00
|
|
|
|
needs_zebra_state: false
|
|
|
|
|
saves_to_disk: true
|
2023-07-13 14:36:15 -07:00
|
|
|
|
force_save_to_disk: ${{ inputs.force_save_to_disk || false }}
|
2023-05-03 16:14:22 -07:00
|
|
|
|
disk_suffix: tip
|
|
|
|
|
height_grep_text: 'current_height.*=.*Height.*\('
|
|
|
|
|
secrets: inherit
|
|
|
|
|
# We want to prevent multiple full zebrad syncs running at the same time,
|
|
|
|
|
# but we don't want to cancel running syncs on `main` if a new PR gets merged,
|
|
|
|
|
# because we might never get a finished sync.
|
|
|
|
|
#
|
|
|
|
|
# Instead, we let the first sync complete, then queue the latest pending sync, cancelling any syncs in between.
|
|
|
|
|
# (As the general workflow concurrency group just gets matched in Pull Requests,
|
|
|
|
|
# it has no impact on this job.)
|
|
|
|
|
#
|
|
|
|
|
# TODO:
|
2023-05-11 04:15:20 -07:00
|
|
|
|
# - allow multiple manual syncs on a branch by adding '-${{ github.run_id }}' when github.event.inputs.run-full-sync is true
|
2023-05-03 16:14:22 -07:00
|
|
|
|
concurrency:
|
2023-05-15 16:04:33 -07:00
|
|
|
|
group: ${{ github.workflow }}−manual-${{ format('{0}', github.event.inputs.run-full-sync == 'true') }}-test-full-sync-testnet
|
2023-05-03 16:14:22 -07:00
|
|
|
|
cancel-in-progress: false
|
|
|
|
|
|
|
|
|
|
# Test that Zebra can generate testnet checkpoints after syncing to the chain tip,
|
|
|
|
|
# using a cached Zebra tip state.
|
|
|
|
|
#
|
|
|
|
|
# This test always runs on testnet.
|
|
|
|
|
#
|
|
|
|
|
# Runs:
|
|
|
|
|
# - after every PR is merged to `main`
|
|
|
|
|
# - on every PR update
|
|
|
|
|
#
|
|
|
|
|
# If the state version has changed, waits for the new cached state to be created.
|
|
|
|
|
# Otherwise, if the state rebuild was skipped, runs immediately after the build job.
|
|
|
|
|
generate-checkpoints-testnet:
|
|
|
|
|
name: Generate checkpoints testnet
|
2023-05-11 04:15:20 -07:00
|
|
|
|
needs: [ test-full-sync-testnet, get-available-disks-testnet ]
|
2023-05-03 16:14:22 -07:00
|
|
|
|
uses: ./.github/workflows/deploy-gcp-tests.yml
|
2023-05-11 04:15:20 -07:00
|
|
|
|
if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks-testnet.outputs.zebra_tip_disk) || needs.test-full-sync-testnet.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
|
2023-05-03 16:14:22 -07:00
|
|
|
|
with:
|
|
|
|
|
app_name: zebrad
|
|
|
|
|
test_id: generate-checkpoints-testnet
|
|
|
|
|
test_description: Generate Zebra checkpoints on testnet
|
2023-07-25 15:50:45 -07:00
|
|
|
|
test_variables: '-e NETWORK=Testnet -e GENERATE_CHECKPOINTS_TESTNET=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache'
|
2023-04-26 21:39:43 -07:00
|
|
|
|
needs_zebra_state: true
|
|
|
|
|
# update the disk on every PR, to increase CI speed
|
2023-05-03 16:14:22 -07:00
|
|
|
|
# we don't have a test-update-sync-testnet job, so we need to update the disk here
|
2023-04-26 21:39:43 -07:00
|
|
|
|
saves_to_disk: true
|
2023-07-13 14:36:15 -07:00
|
|
|
|
force_save_to_disk: ${{ inputs.force_save_to_disk || false }}
|
2023-04-26 21:39:43 -07:00
|
|
|
|
disk_suffix: tip
|
|
|
|
|
root_state_path: '/var/cache'
|
|
|
|
|
zebra_state_dir: 'zebrad-cache'
|
|
|
|
|
height_grep_text: 'current_height.*=.*Height.*\('
|
|
|
|
|
secrets: inherit
|
|
|
|
|
|
2022-09-06 22:06:34 -07:00
|
|
|
|
# lightwalletd cached tip state tests
|
|
|
|
|
|
|
|
|
|
# Test full sync of lightwalletd with a Zebra tip state
|
2022-05-19 16:07:28 -07:00
|
|
|
|
#
|
|
|
|
|
# Runs:
|
2023-06-12 18:27:22 -07:00
|
|
|
|
# - on schedule, as defined at the top of the workflow
|
|
|
|
|
# - on every PR update, but only if the state version in constants.rs has no cached disk
|
|
|
|
|
# - in manual workflow runs, when run-lwd-sync is 'true' and network is 'Mainnet' (the network is required by the test-full-sync job)
|
2022-05-19 16:07:28 -07:00
|
|
|
|
#
|
|
|
|
|
# If the state version has changed, waits for the new cached state to be created.
|
|
|
|
|
# Otherwise, if the state rebuild was skipped, runs immediately after the build job.
|
2022-09-06 22:06:34 -07:00
|
|
|
|
lightwalletd-full-sync:
|
|
|
|
|
name: lightwalletd tip
|
|
|
|
|
needs: [ test-full-sync, get-available-disks ]
|
2022-05-09 20:28:07 -07:00
|
|
|
|
uses: ./.github/workflows/deploy-gcp-tests.yml
|
2023-07-18 00:14:06 -07:00
|
|
|
|
# Currently the lightwalletd tests only work on Mainnet
|
|
|
|
|
if: ${{ !cancelled() && !failure() && (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet' && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && (github.event_name == 'schedule' || !fromJSON(needs.get-available-disks.outputs.lwd_tip_disk) || github.event.inputs.run-lwd-sync == 'true' ) }}
|
2022-05-05 02:27:07 -07:00
|
|
|
|
with:
|
|
|
|
|
app_name: lightwalletd
|
2022-09-06 22:06:34 -07:00
|
|
|
|
test_id: lwd-full-sync
|
|
|
|
|
test_description: Test lightwalletd full sync
|
2023-07-25 15:50:45 -07:00
|
|
|
|
test_variables: '-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_LWD_FULL_SYNC=1 -e ZEBRA_TEST_LIGHTWALLETD=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache -e LIGHTWALLETD_DATA_DIR=/var/cache/lwd-cache'
|
2023-07-10 11:20:50 -07:00
|
|
|
|
# This test runs for longer than 6 hours, so it needs multiple jobs
|
2023-02-15 10:12:30 -08:00
|
|
|
|
is_long_test: true
|
2022-05-05 02:27:07 -07:00
|
|
|
|
needs_zebra_state: true
|
2022-09-06 22:06:34 -07:00
|
|
|
|
needs_lwd_state: false
|
|
|
|
|
saves_to_disk: true
|
2023-07-13 14:36:15 -07:00
|
|
|
|
force_save_to_disk: ${{ inputs.force_save_to_disk || false }}
|
2022-09-06 22:06:34 -07:00
|
|
|
|
disk_prefix: lwd-cache
|
2022-05-05 02:27:07 -07:00
|
|
|
|
disk_suffix: tip
|
2022-05-05 22:30:38 -07:00
|
|
|
|
root_state_path: '/var/cache'
|
|
|
|
|
zebra_state_dir: 'zebrad-cache'
|
2022-09-06 22:06:34 -07:00
|
|
|
|
lwd_state_dir: 'lwd-cache'
|
|
|
|
|
height_grep_text: '(current_height.*=.*Height.*\()|(Adding block to cache )'
|
2022-11-21 10:18:26 -08:00
|
|
|
|
secrets: inherit
|
2022-10-19 05:01:26 -07:00
|
|
|
|
# We want to prevent multiple lightwalletd full syncs running at the same time,
|
|
|
|
|
# but we don't want to cancel running syncs on `main` if a new PR gets merged,
|
|
|
|
|
# because we might never get a finished sync.
|
|
|
|
|
#
|
|
|
|
|
# See the concurrency comment on the zebrad test-full-sync job for details.
|
2022-11-07 14:29:37 -08:00
|
|
|
|
concurrency:
|
2023-05-15 16:04:33 -07:00
|
|
|
|
group: ${{ github.workflow }}−manual-${{ format('{0}', github.event.inputs.run-lwd-sync == 'true') }}-lightwalletd-full-sync
|
2022-10-19 05:01:26 -07:00
|
|
|
|
cancel-in-progress: false
|
2022-05-05 22:30:38 -07:00
|
|
|
|
|
2022-09-06 22:06:34 -07:00
|
|
|
|
# Test update sync of lightwalletd with a lightwalletd and Zebra tip state
|
2022-05-19 16:07:28 -07:00
|
|
|
|
# Runs:
|
|
|
|
|
# - after every PR is merged to `main`
|
|
|
|
|
# - on every PR update
|
|
|
|
|
#
|
|
|
|
|
# If the state version has changed, waits for the new cached states to be created.
|
|
|
|
|
# Otherwise, if the state rebuild was skipped, runs immediately after the build job.
|
2022-09-06 22:06:34 -07:00
|
|
|
|
lightwalletd-update-sync:
|
|
|
|
|
name: lightwalletd tip update
|
2023-05-11 04:15:20 -07:00
|
|
|
|
needs: [ lightwalletd-full-sync, get-available-disks ]
|
2022-05-09 20:28:07 -07:00
|
|
|
|
uses: ./.github/workflows/deploy-gcp-tests.yml
|
2023-07-18 00:14:06 -07:00
|
|
|
|
if: ${{ !cancelled() && !failure() && (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet' && (fromJSON(needs.get-available-disks.outputs.lwd_tip_disk) || needs.lightwalletd-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
|
2022-05-05 22:30:38 -07:00
|
|
|
|
with:
|
|
|
|
|
app_name: lightwalletd
|
2022-09-06 22:06:34 -07:00
|
|
|
|
test_id: lwd-update-sync
|
|
|
|
|
test_description: Test lightwalletd update sync with both states
|
2023-07-25 15:50:45 -07:00
|
|
|
|
test_variables: '-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_LWD_UPDATE_SYNC=1 -e ZEBRA_TEST_LIGHTWALLETD=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache -e LIGHTWALLETD_DATA_DIR=/var/cache/lwd-cache'
|
2022-05-05 22:30:38 -07:00
|
|
|
|
needs_zebra_state: true
|
2022-05-13 16:01:31 -07:00
|
|
|
|
needs_lwd_state: true
|
2022-09-08 13:25:00 -07:00
|
|
|
|
# since we do a full sync in every PR, the new cached state will only be a few minutes newer than the original one
|
|
|
|
|
saves_to_disk: false
|
2022-09-06 22:06:34 -07:00
|
|
|
|
disk_prefix: lwd-cache
|
2022-05-05 22:30:38 -07:00
|
|
|
|
disk_suffix: tip
|
|
|
|
|
root_state_path: '/var/cache'
|
2022-05-11 16:41:33 -07:00
|
|
|
|
zebra_state_dir: 'zebrad-cache'
|
2022-05-13 16:01:31 -07:00
|
|
|
|
lwd_state_dir: 'lwd-cache'
|
2022-09-06 22:06:34 -07:00
|
|
|
|
height_grep_text: '(current_height.*=.*Height.*\()|(Adding block to cache )'
|
2022-11-21 10:18:26 -08:00
|
|
|
|
secrets: inherit
|
2022-05-13 08:20:17 -07:00
|
|
|
|
|
2022-09-06 22:06:34 -07:00
|
|
|
|
# Test that Zebra can answer a synthetic RPC call, using a cached Zebra tip state
|
2022-05-19 16:07:28 -07:00
|
|
|
|
#
|
|
|
|
|
# Runs:
|
|
|
|
|
# - after every PR is merged to `main`
|
|
|
|
|
# - on every PR update
|
|
|
|
|
#
|
|
|
|
|
# If the state version has changed, waits for the new cached state to be created.
|
|
|
|
|
# Otherwise, if the state rebuild was skipped, runs immediately after the build job.
|
2023-05-11 04:15:20 -07:00
|
|
|
|
#
|
|
|
|
|
# TODO: move this job below the rest of the mainnet jobs that just use Zebra cached state
|
2022-09-06 22:06:34 -07:00
|
|
|
|
lightwalletd-rpc-test:
|
|
|
|
|
name: Zebra tip JSON-RPC
|
2023-05-11 04:15:20 -07:00
|
|
|
|
needs: [ test-full-sync, get-available-disks ]
|
2022-05-13 08:20:17 -07:00
|
|
|
|
uses: ./.github/workflows/deploy-gcp-tests.yml
|
2023-07-18 00:14:06 -07:00
|
|
|
|
if: ${{ !cancelled() && !failure() && (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet' && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
|
2022-05-13 08:20:17 -07:00
|
|
|
|
with:
|
|
|
|
|
app_name: lightwalletd
|
2022-09-06 22:06:34 -07:00
|
|
|
|
test_id: fully-synced-rpc
|
|
|
|
|
test_description: Test lightwalletd RPC with a Zebra tip state
|
2023-07-25 15:50:45 -07:00
|
|
|
|
test_variables: '-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_LWD_RPC_CALL=1 -e ZEBRA_TEST_LIGHTWALLETD=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache'
|
2022-05-13 08:20:17 -07:00
|
|
|
|
needs_zebra_state: true
|
2022-09-06 22:06:34 -07:00
|
|
|
|
saves_to_disk: false
|
2022-05-13 08:20:17 -07:00
|
|
|
|
disk_suffix: tip
|
|
|
|
|
root_state_path: '/var/cache'
|
|
|
|
|
zebra_state_dir: 'zebrad-cache'
|
2022-11-21 10:18:26 -08:00
|
|
|
|
secrets: inherit
|
2022-05-13 15:02:05 -07:00
|
|
|
|
|
2022-09-06 22:06:34 -07:00
|
|
|
|
# Test that Zebra can handle a lightwalletd send transaction RPC call, using a cached Zebra tip state
|
|
|
|
|
#
|
2022-05-19 16:07:28 -07:00
|
|
|
|
# Runs:
|
|
|
|
|
# - after every PR is merged to `main`
|
2023-05-11 04:15:20 -07:00
|
|
|
|
# - on every PR update
|
2022-05-19 16:07:28 -07:00
|
|
|
|
#
|
|
|
|
|
# If the state version has changed, waits for the new cached states to be created.
|
|
|
|
|
# Otherwise, if the state rebuild was skipped, runs immediately after the build job.
|
2022-09-06 22:06:34 -07:00
|
|
|
|
lightwalletd-transactions-test:
|
|
|
|
|
name: lightwalletd tip send
|
2023-05-11 04:15:20 -07:00
|
|
|
|
needs: [ lightwalletd-full-sync, get-available-disks ]
|
2022-05-13 15:02:05 -07:00
|
|
|
|
uses: ./.github/workflows/deploy-gcp-tests.yml
|
2023-07-18 00:14:06 -07:00
|
|
|
|
if: ${{ !cancelled() && !failure() && (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet' && (fromJSON(needs.get-available-disks.outputs.lwd_tip_disk) || needs.lightwalletd-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
|
2022-05-13 15:02:05 -07:00
|
|
|
|
with:
|
|
|
|
|
app_name: lightwalletd
|
2022-09-06 22:06:34 -07:00
|
|
|
|
test_id: lwd-send-transactions
|
|
|
|
|
test_description: Test sending transactions via lightwalletd
|
2023-07-25 15:50:45 -07:00
|
|
|
|
test_variables: '-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_LWD_TRANSACTIONS=1 -e ZEBRA_TEST_LIGHTWALLETD=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache -e LIGHTWALLETD_DATA_DIR=/var/cache/lwd-cache'
|
2022-05-13 15:02:05 -07:00
|
|
|
|
needs_zebra_state: true
|
|
|
|
|
needs_lwd_state: true
|
2022-09-06 22:06:34 -07:00
|
|
|
|
saves_to_disk: false
|
2022-05-13 15:02:05 -07:00
|
|
|
|
disk_suffix: tip
|
|
|
|
|
root_state_path: '/var/cache'
|
|
|
|
|
zebra_state_dir: 'zebrad-cache'
|
|
|
|
|
lwd_state_dir: 'lwd-cache'
|
2022-11-21 10:18:26 -08:00
|
|
|
|
secrets: inherit
|
2022-05-26 04:38:00 -07:00
|
|
|
|
|
|
|
|
|
# Test that Zebra can handle gRPC wallet calls, using a cached Zebra tip state
|
|
|
|
|
#
|
|
|
|
|
# Runs:
|
|
|
|
|
# - after every PR is merged to `main`
|
|
|
|
|
# - on every PR update
|
|
|
|
|
#
|
|
|
|
|
# If the state version has changed, waits for the new cached states to be created.
|
|
|
|
|
# Otherwise, if the state rebuild was skipped, runs immediately after the build job.
|
|
|
|
|
lightwalletd-grpc-test:
|
|
|
|
|
name: lightwalletd GRPC tests
|
2023-05-11 04:15:20 -07:00
|
|
|
|
needs: [ lightwalletd-full-sync, get-available-disks ]
|
2022-05-26 04:38:00 -07:00
|
|
|
|
uses: ./.github/workflows/deploy-gcp-tests.yml
|
2023-07-18 00:14:06 -07:00
|
|
|
|
if: ${{ !cancelled() && !failure() && (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet' && (fromJSON(needs.get-available-disks.outputs.lwd_tip_disk) || needs.lightwalletd-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
|
2022-05-26 04:38:00 -07:00
|
|
|
|
with:
|
|
|
|
|
app_name: lightwalletd
|
|
|
|
|
test_id: lwd-grpc-wallet
|
|
|
|
|
test_description: Test gRPC calls via lightwalletd
|
2023-07-25 15:50:45 -07:00
|
|
|
|
test_variables: '-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_LWD_GRPC=1 -e ZEBRA_TEST_LIGHTWALLETD=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache -e LIGHTWALLETD_DATA_DIR=/var/cache/lwd-cache'
|
2022-05-26 04:38:00 -07:00
|
|
|
|
needs_zebra_state: true
|
|
|
|
|
needs_lwd_state: true
|
|
|
|
|
saves_to_disk: false
|
|
|
|
|
disk_suffix: tip
|
|
|
|
|
root_state_path: '/var/cache'
|
|
|
|
|
zebra_state_dir: 'zebrad-cache'
|
|
|
|
|
lwd_state_dir: 'lwd-cache'
|
2022-11-21 10:18:26 -08:00
|
|
|
|
secrets: inherit
|
2023-05-18 09:44:39 -07:00
|
|
|
|
|
2023-05-11 04:15:20 -07:00
|
|
|
|
## getblocktemplate-rpcs using cached Zebra state on mainnet
|
|
|
|
|
#
|
|
|
|
|
# TODO: move these below the rest of the mainnet jobs that just use Zebra cached state
|
2022-11-09 19:40:21 -08:00
|
|
|
|
|
2022-12-01 11:39:01 -08:00
|
|
|
|
# Test that Zebra can handle a getblocktemplate RPC call, using a cached Zebra tip state
|
|
|
|
|
#
|
|
|
|
|
# Runs:
|
|
|
|
|
# - after every PR is merged to `main`
|
|
|
|
|
# - on every PR update
|
|
|
|
|
#
|
|
|
|
|
# If the state version has changed, waits for the new cached states to be created.
|
|
|
|
|
# Otherwise, if the state rebuild was skipped, runs immediately after the build job.
|
|
|
|
|
get-block-template-test:
|
|
|
|
|
name: get block template
|
2023-05-11 04:15:20 -07:00
|
|
|
|
needs: [ test-full-sync, get-available-disks ]
|
2022-12-01 11:39:01 -08:00
|
|
|
|
uses: ./.github/workflows/deploy-gcp-tests.yml
|
2023-05-11 04:15:20 -07:00
|
|
|
|
if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
|
2022-12-01 11:39:01 -08:00
|
|
|
|
with:
|
|
|
|
|
app_name: zebrad
|
|
|
|
|
test_id: get-block-template
|
|
|
|
|
test_description: Test getblocktemplate RPC method via Zebra's rpc server
|
2023-07-25 15:50:45 -07:00
|
|
|
|
test_variables: '-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_GET_BLOCK_TEMPLATE=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache'
|
2022-12-01 11:39:01 -08:00
|
|
|
|
needs_zebra_state: true
|
|
|
|
|
needs_lwd_state: false
|
|
|
|
|
saves_to_disk: false
|
|
|
|
|
disk_suffix: tip
|
|
|
|
|
root_state_path: '/var/cache'
|
|
|
|
|
zebra_state_dir: 'zebrad-cache'
|
|
|
|
|
secrets: inherit
|
|
|
|
|
|
2022-11-09 19:40:21 -08:00
|
|
|
|
# Test that Zebra can handle a submit block RPC call, using a cached Zebra tip state
|
|
|
|
|
#
|
|
|
|
|
# Runs:
|
|
|
|
|
# - after every PR is merged to `main`
|
|
|
|
|
# - on every PR update
|
|
|
|
|
#
|
|
|
|
|
# If the state version has changed, waits for the new cached states to be created.
|
|
|
|
|
# Otherwise, if the state rebuild was skipped, runs immediately after the build job.
|
|
|
|
|
submit-block-test:
|
|
|
|
|
name: submit block
|
2023-05-11 04:15:20 -07:00
|
|
|
|
needs: [ test-full-sync, get-available-disks ]
|
2022-11-09 19:40:21 -08:00
|
|
|
|
uses: ./.github/workflows/deploy-gcp-tests.yml
|
2023-05-11 04:15:20 -07:00
|
|
|
|
if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
|
2022-11-09 19:40:21 -08:00
|
|
|
|
with:
|
|
|
|
|
app_name: zebrad
|
|
|
|
|
test_id: submit-block
|
|
|
|
|
test_description: Test submitting blocks via Zebra's rpc server
|
2023-07-25 15:50:45 -07:00
|
|
|
|
test_variables: '-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_SUBMIT_BLOCK=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache'
|
2022-11-09 19:40:21 -08:00
|
|
|
|
needs_zebra_state: true
|
|
|
|
|
needs_lwd_state: false
|
|
|
|
|
saves_to_disk: false
|
|
|
|
|
disk_suffix: tip
|
|
|
|
|
root_state_path: '/var/cache'
|
|
|
|
|
zebra_state_dir: 'zebrad-cache'
|
2022-11-21 10:18:26 -08:00
|
|
|
|
secrets: inherit
|