feat(deploy): Add CI/CD workflows

This commit is contained in:
Gustavo Valverde 2024-04-15 12:01:53 +01:00
parent bcd77dc546
commit cdd2156b3d
14 changed files with 665 additions and 81 deletions

17
.dockerignore Normal file
View File

@ -0,0 +1,17 @@
# Before the docker CLI sends the context to the docker daemon, it looks for a file
# named .dockerignore in the root directory of the context. If this file exists, the
# CLI modifies the context to exclude files and directories that match patterns in it.
#
# You may want to specify which files to include in the context, rather than which
# to exclude. To achieve this, specify * as the first pattern, followed by one or
# more ! exception patterns.
#
# https://docs.docker.com/engine/reference/builder/#dockerignore-file
# Exclude everything:
#
*
# Now un-exclude required files and folders:
#
!coredns

48
.github/workflows/cd-deploy-to-dev.yml vendored Normal file
View File

@ -0,0 +1,48 @@
name: Deploy to dev
on:
pull_request:
types: [opened, synchronize, reopened, labeled]
paths:
- Corefile
- Dockerfile
- .github/workflows/cd-deploy-to-dev.yml
- .github/workflows/sub-cloudrun-deploy.yml
concurrency:
# Ensures that only one workflow task will run at a time. Previous builds, if
# already in process, will get cancelled. Only the latest commit will be allowed
# to run, cancelling any workflows in between
group: ${{ github.workflow }}-${{ github.job }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
jobs:
# TODO: Add a job to run unit tests
# test:
# uses: ./.github/workflows/sub-unit-tests.yml
build:
uses: ./.github/workflows/sub-build-docker-image.yml
with:
environment: dev
dockerfile_path: ./docker/Dockerfile
dockerfile_target: runner
app_name: ${{ vars.APP_NAME }}
registry: ${{ vars.GAR_BASE }}
secrets: inherit
deploy:
needs: [build]
uses: ./.github/workflows/sub-cloudrun-deploy.yml
with:
environment: dev
project_id: ${{ vars.GCP_PROJECT }}
region: ${{ vars.GCP_REGION }}
app_name: ${{ vars.APP_NAME }}
registry: ${{ vars.GAR_BASE }}
image_digest: ${{ needs.build.outputs.image_digest }}
min_instances: '0'
max_instances: '30'
cpu: '1'
memory: 1Gi
secrets: inherit

46
.github/workflows/cd-deploy-to-prod.yml vendored Normal file
View File

@ -0,0 +1,46 @@
name: Deploy to prod
on:
release:
types:
- published
concurrency:
# Ensures that only one workflow task will run at a time. Previous builds, if
# already in process, will get cancelled. Only the latest commit will be allowed
# to run, cancelling any workflows in between
group: ${{ github.workflow }}-${{ github.job }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
jobs:
# TODO: Add a job to run unit tests
# test:
# uses: ./.github/workflows/sub-unit-tests.yml
build:
# needs: [test]
uses: ./.github/workflows/sub-build-docker-image.yml
with:
environment: prod
dockerfile_path: ./docker/Dockerfile
dockerfile_target: runner
app_name: ${{ vars.APP_NAME }}
registry: ${{ vars.GAR_BASE }}
secrets: inherit
deploy:
needs: [build]
uses: ./.github/workflows/sub-cloudrun-deploy.yml
with:
environment: prod
project_id: ${{ vars.GCP_PROJECT }}
region: ${{ vars.GCP_REGION }}
app_name: ${{ vars.APP_NAME }}
registry: ${{ vars.GAR_BASE }}
image_digest: ${{ needs.build.outputs.image_digest }}
min_instances: '1'
max_instances: '100'
cpu: '1'
memory: 1Gi
secrets: inherit

50
.github/workflows/cd-deploy-to-test.yml vendored Normal file
View File

@ -0,0 +1,50 @@
name: Deploy to test
on:
push:
branches:
- main
paths:
- Corefile
- Dockerfile
- .github/workflows/cd-deploy-to-test.yml
- .github/workflows/sub-cloudrun-deploy.yml
concurrency:
# Ensures that only one workflow task will run at a time. Previous builds, if
# already in process, will get cancelled. Only the latest commit will be allowed
# to run, cancelling any workflows in between
group: ${{ github.workflow }}-${{ github.job }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
jobs:
# TODO: Add a job to run unit tests
# test:
# uses: ./.github/workflows/sub-unit-tests.yml
build:
# needs: [test]
uses: ./.github/workflows/sub-build-docker-image.yml
with:
environment: test
dockerfile_path: ./docker/Dockerfile
dockerfile_target: runner
app_name: ${{ vars.APP_NAME }}
registry: ${{ vars.GAR_BASE }}
secrets: inherit
deploy:
needs: [build]
uses: ./.github/workflows/sub-cloudrun-deploy.yml
with:
environment: test
project_id: ${{ vars.GCP_PROJECT }}
region: ${{ vars.GCP_REGION }}
app_name: ${{ vars.APP_NAME }}
registry: ${{ vars.GAR_BASE }}
image_digest: ${{ needs.build.outputs.image_digest }}
min_instances: '0'
max_instances: '30'
cpu: '1'
memory: 1Gi
secrets: inherit

30
.github/workflows/chore-clean-dev.yml vendored Normal file
View File

@ -0,0 +1,30 @@
name: Clean dev instances
on:
delete:
pull_request:
branches:
- main
types:
- closed
jobs:
delete:
runs-on: ubuntu-latest
steps:
- name: Inject slug/short variables
uses: rlespinasse/github-slug-action@v4.4.1
- name: Authenticate to Google Cloud
id: auth
uses: google-github-actions/auth@v2.1.2
with:
workload_identity_provider: '${{ vars.GCP_WIF }}'
service_account: '${{ vars.GCP_DEPLOYMENTS_SA }}'
- name: Set up Cloud SDK
uses: google-github-actions/setup-gcloud@v1.1.1
- name: Removing CR service
run: |
gcloud run services delete ${{ vars.APP_NAME }}-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }} --region=${{ vars.GOOGLE_CLOUD_REGION }} --quiet

View File

@ -0,0 +1,15 @@
name: Lint Code Base
on:
pull_request:
branches: [main]
paths-ignore:
- Corefile
- Dockerfile
- .github/workflows/ci-lint-codebase.yml
jobs:
linter:
runs-on: ubuntu-latest
steps:
- run: echo "Job not required"

54
.github/workflows/ci-lint-codebase.yaml vendored Normal file
View File

@ -0,0 +1,54 @@
name: Lint Code Base
on:
pull_request:
branches: [main]
paths:
- '**.js*'
- '**.ts*'
- Dockerfile
- package.json
- pnpm-lock.yaml
- .github/workflows/ci-lint-codebase.yml
push:
branches: [main]
paths:
- Corefile
- Dockerfile
- .github/workflows/ci-lint-codebase.yml
concurrency:
# Ensures that only one workflow task will run at a time. Previous builds, if
# already in process, will get cancelled. Only the latest commit will be allowed
# to run, cancelling any workflows in between
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
jobs:
linter:
runs-on: ubuntu-latest
steps:
- name: Checkout Code Repository
uses: actions/checkout@v4.1.1
with:
# Full git history is needed to get a proper
# list of changed files within `super-linter`
fetch-depth: 0
- name: Lint Code Base
uses: super-linter/super-linter/slim@v5.2.1
env:
LOG_LEVEL: ERROR
VALIDATE_ALL_CODEBASE: false
VALIDATE_SHELL_SHFMT: false
VALIDATE_JSCPD: false
VALIDATE_CSS: false
VALIDATE_EDITORCONFIG: false
VALIDATE_MARKDOWN: false
VALIDATE_DOCKERFILE_HADOLINT: false
LINTER_RULES_PATH: /
JAVASCRIPT_DEFAULT_STYLE: prettier
TYPESCRIPT_DEFAULT_STYLE: prettier
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

View File

@ -1,54 +0,0 @@
name: gcloud
on:
workflow_dispatch:
push:
branches:
- master
- update-gcloud
env:
PROJECT_ID: ecosystem-infrastructure
GCLOUD_ZONE: us-central1-a
jobs:
build-deploy:
name: Google Cloud Build
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set project and image names
run: |
BRANCH_NAME=$GITHUB_REPOSITORY/$(expr $GITHUB_REF : '.*/\(.*\)') && \
BRANCH_NAME=${BRANCH_NAME,,} && \
echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV && \
echo "SHORT_SHA=$(git rev-parse --short=7 $GITHUB_SHA)" >> $GITHUB_ENV
# Setup gcloud CLI
- uses: google-github-actions/setup-gcloud@v2
with:
version: '295.0.0'
project_id: ${{ env.PROJECT_ID }}
service_account_key: ${{ secrets.GCLOUD_SA_KEY }}
# Build and push image to Google Container Registry
- name: Build
# Tagging w/ the commit SHA blocks the :latest tag on GCR
run: |
gcloud builds submit \
--tag "gcr.io/$PROJECT_ID/$BRANCH_NAME:$SHORT_SHA"
# Deploy image to Compute Engine
- name: Deploy
run: |
gcloud compute instances create-with-container "zfnd-seeder-$SHORT_SHA" \
--zone "$GCLOUD_ZONE" \
--service-account instance-service-account@ecosystem-infrastructure.iam.gserviceaccount.com \
--scopes cloud-platform \
--machine-type n1-highcpu-4 \
--container-image "gcr.io/$PROJECT_ID/$BRANCH_NAME:$SHORT_SHA" \
--tags seeder \
--metadata-from-file startup-script=scripts/seeder-zfnd-org-startup.sh \
--container-mount-host-path mount-path=/etc/dnsseeder,host-path=/etc/dnsseeder,mode=ro

View File

@ -0,0 +1,117 @@
name: Build docker image
on:
workflow_call:
inputs:
app_name:
required: true
type: string
dockerfile_path:
required: true
type: string
dockerfile_target:
required: true
type: string
registry:
required: true
type: string
environment:
required: true
type: string
outputs:
image_digest:
description: The image digest to be used on a caller workflow
value: ${{ jobs.build.outputs.image_digest }}
jobs:
build:
name: Build images
timeout-minutes: 15
runs-on: ubuntu-latest
outputs:
image_digest: ${{ steps.docker_build.outputs.digest }}
permissions:
contents: read
id-token: write
steps:
- uses: actions/checkout@v4.1.1
with:
persist-credentials: false
- name: Inject slug/short variables
uses: rlespinasse/github-slug-action@v4.4.1
with:
short-length: 7
# Automatic tag management and OCI Image Format Specification for labels
- name: Docker meta
id: meta
uses: docker/metadata-action@v5.5.0
with:
# list of Docker images to use as base name for tags
images: |
${{ inputs.registry }}/${{ inputs.app_name }}
# generate Docker tags based on the following events/attributes
tags: |
type=schedule
# semver and ref,tag automatically add a "latest" tag, but only on stable releases
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=semver,pattern={{major}}
type=ref,event=tag
type=ref,event=branch
type=ref,event=pr
type=sha
# edge is the latest commit on the default branch.
type=edge,enable={{is_default_branch}}
# Setup Docker Buildx to allow use of docker cache layers from GH
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v3.0.0
- name: Authenticate to Google Cloud
id: auth
uses: google-github-actions/auth@v2.1.2
with:
workload_identity_provider: '${{ vars.GCP_WIF }}'
service_account: '${{ vars.GCP_ARTIFACTS_SA }}'
token_format: access_token
# Some builds might take over an hour, and Google's default lifetime duration for
# an access token is 1 hour (3600s). We increase this to 3 hours (10800s)
# as some builds take over an hour.
access_token_lifetime: 10800s
- name: Login to Google Artifact Registry
uses: docker/login-action@v3.0.0
with:
registry: us-docker.pkg.dev
username: oauth2accesstoken
password: ${{ steps.auth.outputs.access_token }}
# Build and push image to Google Artifact Registry, and possibly DockerHub
- name: Build & push
id: docker_build
uses: docker/build-push-action@v5.1.0
with:
target: ${{ inputs.dockerfile_target }}
context: .
file: ${{ inputs.dockerfile_path }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
push: true
# To improve build speeds, for each branch we push an additional image to the registry,
# to be used as the caching layer, using the `max` caching mode.
#
# We use multiple cache sources to confirm a cache hit, starting from a per-branch cache,
# and if there's no hit, then continue with the `main` branch. When changes are added to a PR,
# they are usually smaller than the diff between the PR and `main` branch. So this provides the
# best performance.
#
# The caches are tried in top-down order, the first available cache is used:
# https://github.com/moby/moby/pull/26839#issuecomment-277383550
cache-from: |
type=registry,ref=${{ inputs.registry }}/${{ inputs.app_name }}:${{ env.GITHUB_REF_SLUG_URL }}-cache
type=registry,ref=${{ inputs.registry }}/${{ inputs.app_name }}:${{ github.event.repository.default_branch }}-cache
cache-to: |
type=registry,ref=${{ inputs.registry }}/${{ inputs.app_name }}:${{ env.GITHUB_REF_SLUG_URL }}-cache,mode=min

View File

@ -0,0 +1,112 @@
name: Deploy to Cloud Run
on:
workflow_call:
inputs:
app_name:
required: true
type: string
registry:
required: true
type: string
image_digest:
required: true
type: string
description: The image digest to deploy
project_id:
required: true
type: string
description: The project to deploy to
region:
required: true
type: string
description: The region to deploy to
environment:
required: false
type: string
description: The environment to deploy to
min_instances:
required: false
type: string
description: The minimum number of instances to deploy
max_instances:
required: false
type: string
description: The maximum number of instances to deploy
cpu:
required: false
type: string
description: The number of CPUs to use for the service
memory:
required: false
type: string
description: The amount of memory to use for the service
jobs:
versioning:
runs-on: ubuntu-latest
outputs:
version: ${{ steps.set.outputs.version }}
steps:
- name: Getting API Version
id: get
uses: actions/github-script@v7
if: ${{ github.event_name == 'release' }}
with:
result-encoding: string
script: |
return context.payload.release.tag_name.substring(0,2)
- name: Setting API Version
id: set
run: echo "version=${{ steps.get.outputs.result }}" >> "$GITHUB_OUTPUT"
deploy:
name: Deploy to Cloud Run
needs: [versioning]
timeout-minutes: 10
runs-on: ubuntu-latest
environment:
name: ${{ inputs.environment }}
url: ${{ steps.deploy.outputs.url }}
permissions:
contents: read
id-token: write
steps:
- name: Inject slug/short variables
uses: rlespinasse/github-slug-action@v4.4.1
- name: Authenticate to Google Cloud
id: auth
uses: google-github-actions/auth@v2.1.2
with:
workload_identity_provider: '${{ vars.GCP_WIF }}'
service_account: '${{ vars.GCP_DEPLOYMENTS_SA }}'
- name: Set up Cloud SDK
uses: google-github-actions/setup-gcloud@v2.1.0
- name: Deploy to cloud run
id: deploy
uses: google-github-actions/deploy-cloudrun@v2.2.0
with:
service: ${{ inputs.app_name }}-${{ needs.versioning.outputs.version || env.GITHUB_HEAD_REF_SLUG || inputs.environment }}
image: ${{ inputs.registry }}/${{ inputs.app_name }}@${{ inputs.image_digest }}
region: ${{ inputs.region }}
gcloud_component: alpha
# env_vars: |
# secrets: |
flags: |
--min-instances=${{ inputs.min_instances }}
--max-instances=${{ inputs.max_instances }}
--cpu=${{ inputs.cpu }}
--memory=${{ inputs.memory }}
--network=projects/zfnd-dev-net-spoke-0/global/networks/dev-spoke-0
--subnet=projects/zfnd-dev-net-spoke-0/regions/us-east1/subnetworks/dev-default-ue1
- name: Allow unauthenticated calls to the service
run: |
gcloud run services add-iam-policy-binding ${{ inputs.app_name }}-${{ needs.versioning.outputs.version || env.GITHUB_HEAD_REF_SLUG || inputs.environment }} \
--region=${{ inputs.region }} --member=allUsers --role=roles/run.invoker --quiet
- name: Test service with cURL
run: curl "${{ steps.deploy.outputs.url }}"

37
.github/workflows/sub-unit-tests.yml vendored Normal file
View File

@ -0,0 +1,37 @@
name: Unit Tests
on:
workflow_call:
concurrency:
# Ensures that only one workflow task will run at a time. Previous builds, if
# already in process, will get cancelled. Only the latest commit will be allowed
# to run, cancelling any workflows in between
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
jobs:
test:
name: Test with Go
timeout-minutes: 10
runs-on: ubuntu-latest
steps:
- name: Checkout Code Repository
uses: actions/checkout@v4.1.1
- name: Setup Go
uses: actions/setup-go@v4
with:
go-version: 1.21.x
- name: Install dependencies
run: |
echo "dnsseed:github.com/zcashfoundation/dnsseeder/dnsseed" >> plugin.cfg
echo "replace github.com/btcsuite/btcd => github.com/gtank/btcd v0.0.0-20191012142736-b43c61a68604" >> go.mod
go get github.com/zcashfoundation/dnsseeder/dnsseed@v0.2.4-beta
- name: Build
run: go build -v ./...
- name: Test with the Go CLI
run: go test

139
.gitignore vendored
View File

@ -1 +1,140 @@
# Created by https://www.toptal.com/developers/gitignore/api/go,windows,macos,linux,vim,visualstudiocode
# Edit at https://www.toptal.com/developers/gitignore?templates=go,windows,macos,linux,vim,visualstudiocode
### Go ###
# If you prefer the allow list template instead of the deny list, see community template:
# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
#
# Binaries for programs and plugins
*.exe
*.exe~
*.dll
*.so
*.dylib
# Test binary, built with `go test -c`
*.test
# Output of the go coverage tool, specifically when used with LiteIDE
*.out
# Dependency directories (remove the comment below to include it)
# vendor/
# Go workspace file
go.work
### Linux ###
*~
# temporary files which can be created if a process still has a handle open of a deleted file
.fuse_hidden*
# KDE directory preferences
.directory
# Linux trash folder which might appear on any partition or disk
.Trash-*
# .nfs files are created when an open file is removed but is still being accessed
.nfs*
### macOS ###
# General
.DS_Store
.AppleDouble
.LSOverride
# Icon must end with two \r
Icon
# Thumbnails
._*
# Files that might appear in the root of a volume
.DocumentRevisions-V100
.fseventsd
.Spotlight-V100
.TemporaryItems
.Trashes
.VolumeIcon.icns
.com.apple.timemachine.donotpresent
# Directories potentially created on remote AFP share
.AppleDB
.AppleDesktop
Network Trash Folder
Temporary Items
.apdisk
### macOS Patch ###
# iCloud generated files
*.icloud
### Vim ###
# Swap
[._]*.s[a-v][a-z]
!*.svg # comment out if you don't need vector files
[._]*.sw[a-p]
[._]s[a-rt-v][a-z]
[._]ss[a-gi-z]
[._]sw[a-p]
# Session
Session.vim
Sessionx.vim
# Temporary
.netrwhist
# Auto-generated tag files
tags
# Persistent undo
[._]*.un~
### VisualStudioCode ###
.vscode/*
!.vscode/settings.json
!.vscode/tasks.json
!.vscode/launch.json
!.vscode/extensions.json
!.vscode/*.code-snippets
# Local History for Visual Studio Code
.history/
# Built Visual Studio Code Extensions
*.vsix
### VisualStudioCode Patch ###
# Ignore all local history of files
.history
.ionide
### Windows ###
# Windows thumbnail cache files
Thumbs.db
Thumbs.db:encryptable
ehthumbs.db
ehthumbs_vista.db
# Dump file
*.stackdump
# Folder config file
[Dd]esktop.ini
# Recycle Bin used on file shares
$RECYCLE.BIN/
# Windows Installer files
*.cab
*.msi
*.msix
*.msm
*.msp
# Windows shortcuts
*.lnk
# End of https://www.toptal.com/developers/gitignore/api/go,windows,macos,linux,vim,visualstudiocode

View File

@ -1,27 +0,0 @@
#!/bin/bash
systemctl stop systemd-resolved
if [ ! -d /etc/dnsseeder ]; then
mkdir -p /etc/dnsseeder
fi
cat <<EOF > /etc/dnsseeder/Corefile
mainnet.seeder.zfnd.org {
dnsseed {
network mainnet
bootstrap_peers mainnet.z.cash:8233 dnsseed.str4d.xyz:8233 mainnet.is.yolo.money:8233 mainnet.seeder.zfnd.org:8233
crawl_interval 30m
record_ttl 600
}
}
testnet.seeder.zfnd.org {
dnsseed {
network testnet
bootstrap_peers testnet.z.cash:18233 testnet.is.yolo.money:18233 testnet.seeder.zfnd.org:18233
crawl_interval 15m
record_ttl 300
}
}
EOF