add basic tekton zcash env

This commit is contained in:
mdr0id 2022-08-23 13:17:25 -07:00
parent f84a57e71b
commit e6de9bd437
6 changed files with 2946 additions and 0 deletions

View File

@ -0,0 +1,77 @@
# infra-tekton-labs
Examples of CI/CD used to support Zcash build and testing
# Environment Tooling Setup
### Install Docker
https://docs.docker.com/engine/install/
#### Setup Docker
If you are installing Docker for the first time, to run `docker` without `sudo` priviledge for every docker command:
- Check if Docker group already exists:
- `cat /etc/group | grep docker`
- How to create docker group:
- `sudo groupadd docker`
- Add current user to docker group
- `sudo usermod -a -G docker $CURR_USER`
- Validate current user was added:
- `cat /etc/group | grep docker`
- Restart system to persist changes and graceful setup (had issues with just logout/login)
### Install Kind
https://kind.sigs.k8s.io/
### Install Kubctl
https://kubernetes.io/docs/tasks/tools/install-kubectl-linux/
### Install Tkn
https://tekton.dev/docs/cli/
# Setup Local Zcash CI environment
### Create cluster (e.g. micro k8, kind, minicube)
1. `kind create cluster --name zcash-tekton-lab`
Other Useful commands
- Validate cluster creation & context: `kubectl cluster-info --context kind-zcash-tekton-lab`
- Delete cluster: `kind delete cluster --name zcash-tekton-lab || true`
- List Kube contexts: `kubectl config get-contexts`
- Delete Kube contexts: `kubectl config delete-context <context_name>`
- Setup Kube context: `kubectl config current-context`
- Create new Kube context: `kubectl config set-context zcash_local_ci --user=cluster-admin`
- Switch to new context: `kubectl config use-context zcash_local_ci`
### Create Tekton Pipeline in cluster
See: https://github.com/tektoncd/pipeline for recent version
2. `kubectl apply -f releases/tekton-pipeline-v0.37.0.yaml`
(Alternative)
`kubectl apply -f https://storage.googleapis.com/tekton-releases/pipeline/previous/v0.37.0/release.yaml`
### Create Tekton Dashboard in cluster
See: https://github.com/tektoncd/dashboard for recent version
3. `kubectl apply -f releases/tekton-dashboard-readonly-v0.27.0.yaml`
Validate deployment of Tekton Dashboard & Pipeline
`kubectl get pods --namespace tekton-pipelines`
Forward Tekton Dashboard:
`kubectl --namespace tekton-pipelines port-forward svc/tekton-dashboard 9097:9097 &`
View Dashboard in Browser:
`http://localhost:9097/`
## Create Tekton Task
kubectl apply -f ./tasks/zcash-build.yml
tkn task start --showlog zcash-build
tkn task delete zcash-build
## Create Tekton Pipeline

View File

@ -0,0 +1,334 @@
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
labels:
app.kubernetes.io/component: dashboard
app.kubernetes.io/instance: default
app.kubernetes.io/part-of: tekton-dashboard
name: extensions.dashboard.tekton.dev
spec:
group: dashboard.tekton.dev
names:
categories:
- tekton
- tekton-dashboard
kind: Extension
plural: extensions
shortNames:
- ext
- exts
preserveUnknownFields: false
scope: Namespaced
versions:
- additionalPrinterColumns:
- jsonPath: .spec.apiVersion
name: API version
type: string
- jsonPath: .spec.name
name: Kind
type: string
- jsonPath: .spec.displayname
name: Display name
type: string
- jsonPath: .metadata.creationTimestamp
name: Age
type: date
name: v1alpha1
schema:
openAPIV3Schema:
type: object
x-kubernetes-preserve-unknown-fields: true
served: true
storage: true
subresources:
status: {}
---
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
app.kubernetes.io/component: dashboard
app.kubernetes.io/instance: default
app.kubernetes.io/part-of: tekton-dashboard
name: tekton-dashboard
namespace: tekton-pipelines
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
labels:
app.kubernetes.io/instance: default
app.kubernetes.io/part-of: tekton-dashboard
name: tekton-dashboard-info
namespace: tekton-pipelines
rules:
- apiGroups:
- ""
resourceNames:
- dashboard-info
resources:
- configmaps
verbs:
- get
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
app.kubernetes.io/component: dashboard
app.kubernetes.io/instance: default
app.kubernetes.io/part-of: tekton-dashboard
name: tekton-dashboard-backend
rules:
- apiGroups:
- apiextensions.k8s.io
resources:
- customresourcedefinitions
verbs:
- get
- list
- apiGroups:
- security.openshift.io
resources:
- securitycontextconstraints
verbs:
- use
- apiGroups:
- tekton.dev
resources:
- clustertasks
- clustertasks/status
verbs:
- get
- list
- watch
- apiGroups:
- triggers.tekton.dev
resources:
- clusterinterceptors
- clustertriggerbindings
verbs:
- get
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
app.kubernetes.io/component: dashboard
app.kubernetes.io/instance: default
app.kubernetes.io/part-of: tekton-dashboard
name: tekton-dashboard-tenant
rules:
- apiGroups:
- dashboard.tekton.dev
resources:
- extensions
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- events
- namespaces
- pods
- pods/log
verbs:
- get
- list
- watch
- apiGroups:
- tekton.dev
resources:
- tasks
- taskruns
- pipelines
- pipelineruns
- pipelineresources
- conditions
- tasks/status
- taskruns/status
- pipelines/status
- pipelineruns/status
- taskruns/finalizers
- pipelineruns/finalizers
verbs:
- get
- list
- watch
- apiGroups:
- triggers.tekton.dev
resources:
- eventlisteners
- triggerbindings
- triggers
- triggertemplates
verbs:
- get
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
labels:
app.kubernetes.io/instance: default
app.kubernetes.io/part-of: tekton-dashboard
name: tekton-dashboard-info
namespace: tekton-pipelines
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: tekton-dashboard-info
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: Group
name: system:authenticated
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
labels:
app.kubernetes.io/component: dashboard
app.kubernetes.io/instance: default
app.kubernetes.io/part-of: tekton-dashboard
rbac.dashboard.tekton.dev/subject: tekton-dashboard
name: tekton-dashboard-backend
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: tekton-dashboard-backend
subjects:
- kind: ServiceAccount
name: tekton-dashboard
namespace: tekton-pipelines
---
apiVersion: v1
data:
version: v0.27.0
kind: ConfigMap
metadata:
labels:
app.kubernetes.io/instance: default
app.kubernetes.io/part-of: tekton-dashboard
name: dashboard-info
namespace: tekton-pipelines
---
apiVersion: v1
kind: Service
metadata:
labels:
app: tekton-dashboard
app.kubernetes.io/component: dashboard
app.kubernetes.io/instance: default
app.kubernetes.io/name: dashboard
app.kubernetes.io/part-of: tekton-dashboard
app.kubernetes.io/version: v0.27.0
dashboard.tekton.dev/release: v0.27.0
version: v0.27.0
name: tekton-dashboard
namespace: tekton-pipelines
spec:
ports:
- name: http
port: 9097
protocol: TCP
targetPort: 9097
selector:
app.kubernetes.io/component: dashboard
app.kubernetes.io/instance: default
app.kubernetes.io/name: dashboard
app.kubernetes.io/part-of: tekton-dashboard
---
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: tekton-dashboard
app.kubernetes.io/component: dashboard
app.kubernetes.io/instance: default
app.kubernetes.io/name: dashboard
app.kubernetes.io/part-of: tekton-dashboard
app.kubernetes.io/version: v0.27.0
dashboard.tekton.dev/release: v0.27.0
version: v0.27.0
name: tekton-dashboard
namespace: tekton-pipelines
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/component: dashboard
app.kubernetes.io/instance: default
app.kubernetes.io/name: dashboard
app.kubernetes.io/part-of: tekton-dashboard
template:
metadata:
labels:
app: tekton-dashboard
app.kubernetes.io/component: dashboard
app.kubernetes.io/instance: default
app.kubernetes.io/name: dashboard
app.kubernetes.io/part-of: tekton-dashboard
app.kubernetes.io/version: v0.27.0
name: tekton-dashboard
spec:
containers:
- args:
- --port=9097
- --logout-url=
- --pipelines-namespace=tekton-pipelines
- --triggers-namespace=tekton-pipelines
- --read-only=true
- --log-level=info
- --log-format=json
- --namespace=
- --stream-logs=true
- --external-logs=
env:
- name: INSTALLED_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
image: gcr.io/tekton-releases/github.com/tektoncd/dashboard/cmd/dashboard:v0.27.0@sha256:d57860eb2ff1f97ca1b657eda6ec640584ec8b756e80a5f888c98b7b056c14c1
livenessProbe:
httpGet:
path: /health
port: 9097
name: tekton-dashboard
ports:
- containerPort: 9097
readinessProbe:
httpGet:
path: /readiness
port: 9097
nodeSelector:
kubernetes.io/os: linux
securityContext:
runAsNonRoot: true
runAsUser: 65532
serviceAccountName: tekton-dashboard
volumes: []
---
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
labels:
app.kubernetes.io/component: dashboard
app.kubernetes.io/instance: default
app.kubernetes.io/part-of: tekton-dashboard
rbac.dashboard.tekton.dev/subject: tekton-dashboard
name: tekton-dashboard-tenant
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: tekton-dashboard-tenant
subjects:
- kind: ServiceAccount
name: tekton-dashboard
namespace: tekton-pipelines

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,205 @@
---
apiVersion: tekton.dev/v1beta1
kind: Task
metadata:
name: zcash-build-bench
spec:
params:
- name: JOBS
default: "4"
type: string
- name: BUILD_FOR_HOST
default: x86_64-pc-linux-gnu
type: string
- name: CONFIGURE_FLAGS
default: ""
type: string
- name: BUILD_CONTAINER
default: electriccoinco/zcashd-bench-debian10
type: string
- name: TEST_TARGET
default: ""
type: string
resources:
inputs:
- name: source
type: git
steps:
- name: build
image: $(params.BUILD_CONTAINER)
script: |
#!/bin/bash
set -o pipefail
cd workspace/source
echo $(git rev-parse HEAD)
echo $(git rev-parse --short HEAD)
if [ "$(params.BUILD_FOR_HOST)" == "x86_64-apple-darwin11" ]
then
mkdir -p depends/SDKs
curl -fs https://ecc.mypinata.cloud/ipfs/QmeSwckvSCGL9SXGdEHoAyqXdzD7T9HYTwsC34Bj5EVDF5 \
-o depends/SDKs/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz
tar zxvf depends/SDKs/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz \
-C depends/SDKs
fi
CONFIGURE_FLAGS=$(params.CONFIGURE_FLAGS) HOST=$(params.BUILD_FOR_HOST) ./zcutil/build.sh -j$(params.JOBS)
- name: bench-sleep
image: $(params.BUILD_CONTAINER)
script: |
#!/bin/bash
set -o pipefail
python -m pip install pyblake2 pyflakes python-qpid-proton pyutil pyzmq requests simplejson
cp -a /home/.zcash-params $HOME/
cp -r /home/benchmark-200k-UTXOs.tar.xz /workspace/source
cp -r /home/block-107134.tar.xz /workspace/source
cd /workspace/source
./qa/zcash/performance-measurements.sh time sleep | jq '[.[]|.runningtime | tonumber] | add / (. | length | tonumber)'
- name: bench-createsaplingspend
image: $(params.BUILD_CONTAINER)
script: |
#!/bin/bash
set -o pipefail
cp -a /home/.zcash-params $HOME/
cp -r /home/benchmark-200k-UTXOs.tar.xz /workspace/source
cp -r /home/block-107134.tar.xz /workspace/source
cd /workspace/source
./qa/zcash/performance-measurements.sh time createsaplingspend | jq '[.[]|.runningtime | tonumber] | add / (. | length | tonumber)'
- name: bench-verifysaplingspend
image: $(params.BUILD_CONTAINER)
script: |
#!/bin/bash
set -o pipefail
cp -a /home/.zcash-params $HOME/
cp -r /home/benchmark-200k-UTXOs.tar.xz /workspace/source
cp -r /home/block-107134.tar.xz /workspace/source
cd /workspace/source
./qa/zcash/performance-measurements.sh time verifysaplingspend | jq '[.[]|.runningtime | tonumber] | add / (. | length | tonumber)'
- name: bench-createsaplingoutput
image: $(params.BUILD_CONTAINER)
script: |
#!/bin/bash
set -o pipefail
cp -a /home/.zcash-params $HOME/
cp -r /home/benchmark-200k-UTXOs.tar.xz /workspace/source
cp -r /home/block-107134.tar.xz /workspace/source
cd /workspace/source
./qa/zcash/performance-measurements.sh time createsaplingoutput | jq '[.[]|.runningtime | tonumber] | add / (. | length | tonumber)'
- name: bench-verifysaplingoutput
image: $(params.BUILD_CONTAINER)
script: |
#!/bin/bash
set -o pipefail
cp -a /home/.zcash-params $HOME/
cp -r /home/benchmark-200k-UTXOs.tar.xz /workspace/source
cp -r /home/block-107134.tar.xz /workspace/source
cd /workspace/source
./qa/zcash/performance-measurements.sh time verifysaplingoutput | jq '[.[]|.runningtime | tonumber] | add / (. | length | tonumber)'
- name: bench-createjoinsplit
image: $(params.BUILD_CONTAINER)
script: |
#!/bin/bash
set -o pipefail
cp -a /home/.zcash-params $HOME/
cp -r /home/benchmark-200k-UTXOs.tar.xz /workspace/source
cp -r /home/block-107134.tar.xz /workspace/source
cd /workspace/source
./qa/zcash/performance-measurements.sh time createjoinsplit | jq '[.[]|.runningtime | tonumber] | add / (. | length | tonumber)'
- name: bench-verifyjoinsplit
image: $(params.BUILD_CONTAINER)
script: |
#!/bin/bash
set -o pipefail
cp -a /home/.zcash-params $HOME/
cp -r /home/benchmark-200k-UTXOs.tar.xz /workspace/source
cp -r /home/block-107134.tar.xz /workspace/source
cd /workspace/source
./qa/zcash/performance-measurements.sh time verifyjoinsplit | jq '[.[]|.runningtime | tonumber] | add / (. | length | tonumber)'
- name: bench-verifyequihash
image: $(params.BUILD_CONTAINER)
script: |
#!/bin/bash
set -o pipefail
cp -a /home/.zcash-params $HOME/
cp -r /home/benchmark-200k-UTXOs.tar.xz /workspace/source
cp -r /home/block-107134.tar.xz /workspace/source
cd /workspace/source
./qa/zcash/performance-measurements.sh time verifyequihash | jq '[.[]|.runningtime | tonumber] | add / (. | length | tonumber)'
- name: bench-connectblockslow
image: $(params.BUILD_CONTAINER)
script: |
#!/bin/bash
set -o pipefail
cp -a /home/.zcash-params $HOME/
cp -r /home/benchmark-200k-UTXOs.tar.xz /workspace/source
cp -r /home/block-107134.tar.xz /workspace/source
cd /workspace/source
./qa/zcash/performance-measurements.sh time connectblockslow | sed -n '2,$p' | jq '[.[]|.runningtime | tonumber] | add / (. | length | tonumber)'
- name: bench-loadwallet-200k-recv
image: $(params.BUILD_CONTAINER)
script: |
#!/bin/bash
set -o pipefail
cp -a /home/.zcash-params $HOME/
cp -r /home/benchmark-200k-UTXOs.tar.xz /workspace/source
cp -r /home/block-107134.tar.xz /workspace/source
cd /workspace/source
./qa/zcash/performance-measurements.sh time loadwallet 200k-recv | sed 1d | jq '[.[]|.runningtime | tonumber] | add / (. | length | tonumber)'
- name: bench-loadwallet-200k-send
image: $(params.BUILD_CONTAINER)
script: |
#!/bin/bash
set -o pipefail
cp -a /home/.zcash-params $HOME/
cp -r /home/benchmark-200k-UTXOs.tar.xz /workspace/source
cp -r /home/block-107134.tar.xz /workspace/source
cd /workspace/source
./qa/zcash/performance-measurements.sh time loadwallet 200k-send | sed 1d | jq '[.[]|.runningtime | tonumber] | add / (. | length | tonumber)'
- name: bench-listunspent-200k-recv
image: $(params.BUILD_CONTAINER)
script: |
#!/bin/bash
set -o pipefail
cp -a /home/.zcash-params $HOME/
cp -r /home/benchmark-200k-UTXOs.tar.xz /workspace/source
cp -r /home/block-107134.tar.xz /workspace/source
cd /workspace/source
./qa/zcash/performance-measurements.sh time listunspent 200k-recv | sed 1d | jq '[.[]|.runningtime | tonumber] | add / (. | length | tonumber)'
- name: bench-listunspent-200k-send
image: $(params.BUILD_CONTAINER)
script: |
#!/bin/bash
set -o pipefail
cp -a /home/.zcash-params $HOME/
cp -r /home/benchmark-200k-UTXOs.tar.xz /workspace/source
cp -r /home/block-107134.tar.xz /workspace/source
cd /workspace/source
./qa/zcash/performance-measurements.sh time listunspent 200k-send | sed 1d | jq '[.[]|.runningtime | tonumber] | add / (. | length | tonumber)'

View File

@ -0,0 +1,57 @@
---
apiVersion: tekton.dev/v1beta1
kind: Task
metadata:
name: zcash-build-test
spec:
params:
- name: JOBS
default: "4"
type: string
- name: BUILD_FOR_HOST
default: x86_64-pc-linux-gnu
type: string
- name: CONFIGURE_FLAGS
default: ""
type: string
- name: BUILD_CONTAINER
default: electriccoinco/zcashd-worker-ubuntu2004v2
type: string
- name: TEST_TARGET
default: ""
type: string
resources:
inputs:
- name: source
type: git
steps:
- name: build
image: $(params.BUILD_CONTAINER)
script: |
#!/bin/bash
set -o pipefail
cd workspace/source
echo $(git rev-parse HEAD)
echo $(git rev-parse --short HEAD)
if [ "$(params.BUILD_FOR_HOST)" == "x86_64-apple-darwin11" ]
then
mkdir -p depends/SDKs
curl -fs https://ecc.mypinata.cloud/ipfs/QmeSwckvSCGL9SXGdEHoAyqXdzD7T9HYTwsC34Bj5EVDF5 \
-o depends/SDKs/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz
tar zxvf depends/SDKs/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz \
-C depends/SDKs
fi
CONFIGURE_FLAGS=$(params.CONFIGURE_FLAGS) HOST=$(params.BUILD_FOR_HOST) ./zcutil/build.sh -j$(params.JOBS)
- name: test
image: $(params.BUILD_CONTAINER)
script: |
#!/bin/bash
set -o pipefail
python -m pip install pyblake2 pyflakes python-qpid-proton pyutil pyzmq requests simplejson
cp -a /home/.zcash-params $HOME/
echo $(git rev-parse HEAD)
echo $(git rev-parse --short HEAD)
HOST=x86_64-pc-linux-gnu /workspace/source/qa/zcash/full_test_suite.py $(params.TEST_TARGET)

View File

@ -0,0 +1,51 @@
---
apiVersion: tekton.dev/v1beta1
kind: Task
metadata:
name: zcash-build
spec:
params:
- name: JOBS
default: "4"
type: string
- name: BUILD_FOR_HOST
default: x86_64-pc-linux-gnu
type: string
- name: CONFIGURE_FLAGS
default: ""
type: string
- name: BUILD_CONTAINER
default: electriccoinco/zcashd-worker-ubuntu2004v2
type: string
resources:
inputs:
- name: source
type: git
steps:
- name: setup-venv
image: $(params.BUILD_CONTAINER)
script: |
#!/bin/bash
set -e -o pipefail
python3 -m venv /venv/zcash-build-venv
source /venv/zcash-build-venv/bin/activate
python -m pip install pyblake2 pyflakes python-qpid-proton pyutil pyzmq requests simplejson
- name: build
image: $(params.BUILD_CONTAINER)
script: |
#!/bin/bash
set -o pipefail
cd workspace/source
echo $(git rev-parse HEAD)
echo $(git rev-parse --short HEAD)
if [ "$(params.BUILD_FOR_HOST)" == "x86_64-apple-darwin11" ]
then
mkdir -p depends/SDKs
curl -fs https://ecc.mypinata.cloud/ipfs/QmeSwckvSCGL9SXGdEHoAyqXdzD7T9HYTwsC34Bj5EVDF5 \
-o depends/SDKs/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz
tar zxvf depends/SDKs/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz \
-C depends/SDKs
fi
CONFIGURE_FLAGS=$(params.CONFIGURE_FLAGS) HOST=$(params.BUILD_FOR_HOST) ./zcutil/build.sh -j$(params.JOBS)