Auto merge of #5154 - mdr0id:ci_builder_updates, r=str4d

CI Builder Updates & Refactors

To support existing legacy pinned Docker builder/workers in Buildbot, it is valuable to move these into their own folder.

The new directory structured is intended to simplify development/deployment of Docker Tekton builder/worker images and remove overhead related to potential Buildbot CI depreciation.

contrib/ci-builders/tekton/Dockerfile-build.apt

@@ -0,0 +1,9 @@
+ARG FROMBASEOS
+ARG FROMBASEOS_BUILD_TAG
+FROM $FROMBASEOS:$FROMBASEOS_BUILD_TAG
+ARG DEBIAN_FRONTEND=noninteractive
+
+ADD apt-package-list.txt /tmp/apt-package-list.txt
+RUN apt-get update \
+    && apt-get install -y $(tr "\n" " " < /tmp/apt-package-list.txt) \
+    && update-alternatives --install /usr/bin/python python /usr/bin/python3 1

contrib/ci-builders/tekton/Dockerfile-build.arch

@@ -0,0 +1,14 @@
+ARG FROMBASEOS
+ARG FROMBASEOS_BUILD_TAG
+FROM $FROMBASEOS:$FROMBASEOS_BUILD_TAG
+
+RUN pacman -Syyu --noconfirm \
+    && pacman -S --noconfirm \
+      base-devel \
+      git \
+      python3 \
+      python-pip \
+      ncurses \
+      wget
+
+RUN sudo link /lib/libtinfo.so.6 /lib/libtinfo.so.5

contrib/ci-builders/tekton/Dockerfile-build.centos8

@@ -0,0 +1,14 @@
+FROM centos:8
+
+RUN yum update -y \
+    && dnf group install -y "Development Tools" \
+    && yum install -y \
+      ncurses-compat-libs \
+      python3 \
+      python3-devel \
+      wget
+
+RUN wget -O /usr/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.2/dumb-init_1.2.2_amd64
+RUN chmod +x /usr/bin/dumb-init
+RUN alternatives --set  python /usr/bin/python3 \
+    && python3 -m pip install virtualenv

contrib/ci-builders/tekton/Dockerfile-tekton-worker

@@ -0,0 +1,11 @@
+ARG FROMBASEOS
+ARG FROMBASEOS_BUILD_TAG
+FROM electriccoinco/zcashd-build-$FROMBASEOS$FROMBASEOS_BUILD_TAG
+
+ADD requirements.txt requirements.txt
+RUN python -m venv venv \
+    && . venv/bin/activate \
+    && pip install --upgrade pip \
+    && python -m pip install -r requirements.txt
+
+ADD ./zcash-params /home/.zcash-params

contrib/ci-builders/tekton/README.md

@@ -0,0 +1,35 @@
+# zcashd ci Docker images
+
+These Dockerfiles can be used to build zcashd.
+
+The current objective is to build a base image for each distribution that includes the system packages to build zcashd. From `build` images, more targeted images are created.
+
+The process is meant to be automated, but an example `docker-build.sh` script is included.
+
+
+## build images
+`apt-package-tekton-list.txt` contains the required packages for debian based systems.
+
+`Dockerfile-build.apt` uses that file, and some build time arguments, to build apt based build images.
+
+Currently available images are hosted at
+https://hub.docker.com/r/electriccoinco/zcashd-build/tags
+
+
+
+## Tekton worker images
+
+`Dockerfile-tekton-worker` uses the above build images as a base and layers on toolchains needed for testing
+
+- requirements.txt is the python package requirements for the tekton worker
+
+Currently available images are hosted at https://hub.docker.com/r/electriccoinco
+
+
+### Stand alone, best effort images
+
+Additional Tekton base builders for Centos8 and Arch. Can be used with `Dockerfile-tekton-worker` to create Tekton workers.
+
+- Dockerfile-build.arch
+- Dockerfile-build.centos8
+

contrib/ci-builders/tekton/apt-package-tekton-list.txt

@@ -0,0 +1,24 @@
+autoconf
+bsdmainutils
+build-essential
+cmake
+curl
+g++-aarch64-linux-gnu
+git
+lcov
+libcap-dev
+libffi-dev
+libtinfo5
+libtool
+libssl-dev
+libz-dev
+libbz2-dev
+mingw-w64
+pkg-config
+python3
+python3-pip
+python3-dev
+python3-venv
+valgrind
+wget
+zstd

contrib/ci-builders/tekton/docker-build.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+
+export LC_ALL=C
+set -exo pipefail
+
+# Debian 9
+docker build . -f Dockerfile-build-python.apt --build-arg FROMBASEOS=debian --build-arg FROMBASEOS_BUILD_TAG=9 -t electriccoinco/zcashd-build-debian9
+docker push electriccoinco/zcashd-build-debian9
+
+docker build . -f Dockerfile-tekton-worker --build-arg BASEOS=debian --build-arg FROMBASEOS=debian --build-arg FROMBASEOS_BUILD_TAG=9 -t electriccoinco/zcashd-worker-debian9
+docker push electriccoinco/zcashd-worker-debian9
+
+# Debian 10
+docker build . -f Dockerfile-build.apt --build-arg FROMBASEOS=debian --build-arg FROMBASEOS_BUILD_TAG=10 -t electriccoinco/zcashd-build-debian10
+docker push electriccoinco/zcashd-build-debian10
+
+docker build . -f Dockerfile-tekton-worker --build-arg BASEOS=debian --build-arg FROMBASEOS=debian --build-arg FROMBASEOS_BUILD_TAG=10 -t electriccoinco/zcashd-worker-debian10
+docker push electriccoinco/zcashd-worker-debian10
+
+# Debian 11
+docker build . -f Dockerfile-build.apt --build-arg FROMBASEOS=debian --build-arg FROMBASEOS_BUILD_TAG=bullseye -t electriccoinco/zcashd-build-debian11
+docker push electriccoinco/zcashd-build-debian11
+
+docker build . -f Dockerfile-tekton-worker --build-arg FROMBASEOS=debian --build-arg FROMBASEOS_BUILD_TAG=11 -t electriccoinco/zcashd-worker-debian11
+docker push electriccoinco/zcashd-worker-debian11
+
+# Ubuntu 16.04
+docker build . -f Dockerfile-build.apt --build-arg FROMBASEOS=ubuntu --build-arg FROMBASEOS_BUILD_TAG=16.04 -t electriccoinco/zcashd-build-ubuntu1604
+docker push electriccoinco/zcashd-build-ubuntu1604
+
+docker build . -f Dockerfile-tekton-worker --build-arg BASEOS=ubuntu --build-arg FROMBASEOS=ubuntu --build-arg FROMBASEOS_BUILD_TAG=1604 -t electriccoinco/zcashd-worker-ubuntu1604
+docker push electriccoinco/zcashd-worker-ubuntu1604
+
+# Ubuntu 18.04
+docker build . -f Dockerfile-build.apt --build-arg FROMBASEOS=ubuntu --build-arg FROMBASEOS_BUILD_TAG=18.04 -t electriccoinco/zcashd-build-ubuntu1804
+docker push electriccoinco/zcashd-build-ubuntu1804
+
+docker build . -f Dockerfile-tekton-worker --build-arg BASEOS=ubuntu --build-arg FROMBASEOS=ubuntu --build-arg FROMBASEOS_BUILD_TAG=1804 -t electriccoinco/zcashd-worker-ubuntu1804
+docker push electriccoinco/zcashd-worker-ubuntu1804
+
+# Ubuntu 20.04
+docker build . -f Dockerfile-build.apt --build-arg FROMBASEOS=ubuntu --build-arg FROMBASEOS_BUILD_TAG=20.04 -t electriccoinco/zcashd-build-ubuntu2004
+docker push electriccoinco/zcashd-build-ubuntu2004
+
+docker build . -f Dockerfile-tekton-worker --build-arg BASEOS=ubuntu --build-arg FROMBASEOS=ubuntu --build-arg FROMBASEOS_BUILD_TAG=2004 -t electriccoinco/zcashd-worker-ubuntu2004
+docker push electriccoinco/zcashd-worker-ubuntu2004
+
+# Centos8
+docker build . -f Dockerfile-build.centos8 -t electriccoinco/zcashd-build-centos8
+docker push electriccoinco/zcashd-build-centos8
+
+docker build . -f Dockerfile-tekton-worker --build-arg FROMBASEOS=centos --build-arg FROMBASEOS_BUILD_TAG=8  -t electriccoinco/zcashd-worker-centos8
+docker push electriccoinco/zcashd-worker-centos8
+
+# Arch 20210418.0.20194
+docker build . -f Dockerfile-build.arch --build-arg FROMBASEOS=archlinux --build-arg FROMBASEOS_BUILD_TAG=base-20210418.0.20194 -t electriccoinco/zcashd-build-archlinux
+docker push electriccoinco/zcashd-build-archlinux
+
+docker build . -f Dockerfile-tekton-worker --build-arg FROMBASEOS=archlinux -t electriccoinco/zcashd-worker-archlinux
+docker push electriccoinco/zcashd-worker-archlinux

contrib/ci-builders/tekton/requirements.txt

@@ -0,0 +1,8 @@
+wheel
+pyblake2
+pyflakes
+pyutil
+pyzmq
+requests
+simplejson
+twisted[tls]