From 134d8e4cd00c58dc48e82ef7402ef242ddff710c Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Fri, 1 May 2020 12:56:17 -0400 Subject: [PATCH] Dockerfiles for zcashd CI builds --- contrib/ci-builders/Dockerfile-bbworker.apt | 42 +++++++++++++++ contrib/ci-builders/Dockerfile-bbworker.arch | 52 +++++++++++++++++++ .../ci-builders/Dockerfile-bbworker.centos8 | 51 ++++++++++++++++++ contrib/ci-builders/Dockerfile-build.apt | 9 ++++ contrib/ci-builders/Dockerfile-gitian.apt | 8 +++ contrib/ci-builders/README.md | 42 +++++++++++++++ contrib/ci-builders/apt-package-list.txt | 16 ++++++ contrib/ci-builders/bbworker-buildbot.tac | 40 ++++++++++++++ contrib/ci-builders/bbworker-requirements.txt | 10 ++++ contrib/ci-builders/docker-build.sh | 37 +++++++++++++ 10 files changed, 307 insertions(+) create mode 100644 contrib/ci-builders/Dockerfile-bbworker.apt create mode 100644 contrib/ci-builders/Dockerfile-bbworker.arch create mode 100644 contrib/ci-builders/Dockerfile-bbworker.centos8 create mode 100644 contrib/ci-builders/Dockerfile-build.apt create mode 100644 contrib/ci-builders/Dockerfile-gitian.apt create mode 100644 contrib/ci-builders/README.md create mode 100644 contrib/ci-builders/apt-package-list.txt create mode 100644 contrib/ci-builders/bbworker-buildbot.tac create mode 100644 contrib/ci-builders/bbworker-requirements.txt create mode 100644 contrib/ci-builders/docker-build.sh diff --git a/contrib/ci-builders/Dockerfile-bbworker.apt b/contrib/ci-builders/Dockerfile-bbworker.apt new file mode 100644 index 000000000..89e00ffb1 --- /dev/null +++ b/contrib/ci-builders/Dockerfile-bbworker.apt @@ -0,0 +1,42 @@ +ARG FROMBASEOS +ARG FROMBASEOS_BUILD_TAG=latest +FROM electriccoinco/zcashd-build:$FROMBASEOS$FROMBASEOS_BUILD_TAG + +ARG DUMBINIT_VERSION=1.2.2 +RUN wget -O /usr/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v${DUMBINIT_VERSION}/dumb-init_${DUMBINIT_VERSION}_amd64 +RUN chmod +x /usr/bin/dumb-init + +# Buildbot user +ARG BUILDBOT_USER=zcbbworker +ARG BUILDBOT_UID=2001 +RUN useradd --home-dir /home/$BUILDBOT_USER \ + --shell /bin/bash \ + --create-home \ + --uid $BUILDBOT_UID\ + $BUILDBOT_USER + +USER $BUILDBOT_USER +WORKDIR /home/$BUILDBOT_USER + +ADD bbworker-requirements.txt requirements.txt +RUN python -m venv venv \ + && . venv/bin/activate \ + && python -m pip install wheel \ + && python -m pip install -r requirements.txt + +# Buildbot worker +ARG BASEOS +ENV BUILDBOT_WORKER_NAME=$BASEOS-docker +ENV BUILDBOT_WORKER_PASS=thisgetssetwhenpodisstarted +ENV BUILDBOT_MASTER_HOST=dev-ci.z.cash +ENV BUILDBOT_MASTER_PORT=9899 + +WORKDIR /home/$BUILDBOT_USER +RUN venv/bin/buildbot-worker create-worker $BUILDBOT_WORKER_NAME \ + $BUILDBOT_MASTER_HOST:$BUILDBOT_MASTER_PORT \ + $BUILDBOT_WORKER_NAME $BUILDBOT_WORKER_PASS \ + && echo "OS: $BASEOS" > $BUILDBOT_WORKER_NAME/info/host +ADD bbworker-buildbot.tac $BUILDBOT_WORKER_NAME/buildbot.tac + +WORKDIR /home/$BUILDBOT_USER/$BUILDBOT_WORKER_NAME +CMD ["/usr/bin/dumb-init", "../venv/bin/twistd", "--pidfile=", "-ny", "buildbot.tac"] diff --git a/contrib/ci-builders/Dockerfile-bbworker.arch b/contrib/ci-builders/Dockerfile-bbworker.arch new file mode 100644 index 000000000..0f81aa386 --- /dev/null +++ b/contrib/ci-builders/Dockerfile-bbworker.arch @@ -0,0 +1,52 @@ +ARG ARCHLINUX_TAG=20200205 +FROM archlinux:$ARCHLINUX_TAG + +RUN pacman -Syyu --noconfirm \ + && pacman -S --noconfirm \ + base-devel \ + git \ + python3 \ + python-pip \ + wget + +RUN wget -O /usr/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.2/dumb-init_1.2.2_amd64 +RUN chmod +x /usr/bin/dumb-init +RUN python -m pip install virtualenv + +# Buildbot user +ARG BUILDBOT_USER=zcbbworker +ARG BUILDBOT_UID=2001 +RUN useradd --home-dir /home/$BUILDBOT_USER \ + --shell /bin/bash \ + --create-home \ + --uid $BUILDBOT_UID\ + $BUILDBOT_USER + +USER $BUILDBOT_USER +WORKDIR /home/$BUILDBOT_USER + +RUN python -m venv venv \ + && . venv/bin/activate \ + && python -m pip install \ + buildbot-worker \ + pyflakes \ + pyblake2 \ + pyzmq \ + requests \ + twisted[tls] + +# Buildbot worker +ARG BUILDBOT_WORKER_NAME=arch-docker +ARG BUILDBOT_WORKER_PASS=thisgetssetwhenpodisstarted +ARG BUILDBOT_MASTER_HOST=dev-ci.z.cash +ARG BUILDBOT_MASTER_PORT=9899 + +WORKDIR /home/$BUILDBOT_USER +RUN venv/bin/buildbot-worker create-worker $BUILDBOT_WORKER_NAME \ + $BUILDBOT_MASTER_HOST:$BUILDBOT_MASTER_PORT \ + $BUILDBOT_WORKER_NAME $BUILDBOT_WORKER_PASS \ + && echo "OS: Centos 8" > $BUILDBOT_WORKER_NAME/info/host +ADD buildbot.tac $BUILDBOT_WORKER_NAME/buildbot.tac + +WORKDIR /home/$BUILDBOT_USER/$BUILDBOT_WORKER_NAME +CMD ["/usr/bin/dumb-init", "../venv/bin/twistd", "--pidfile=", "-ny", "buildbot.tac"] diff --git a/contrib/ci-builders/Dockerfile-bbworker.centos8 b/contrib/ci-builders/Dockerfile-bbworker.centos8 new file mode 100644 index 000000000..79adaf860 --- /dev/null +++ b/contrib/ci-builders/Dockerfile-bbworker.centos8 @@ -0,0 +1,51 @@ +FROM centos:8 + +RUN yum update -y \ + && dnf group install -y "Development Tools" \ + && yum install -y \ + python3 \ + python3-devel \ + wget + +RUN wget -O /usr/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.2/dumb-init_1.2.2_amd64 +RUN chmod +x /usr/bin/dumb-init +RUN alternatives --set python /usr/bin/python3 \ + && python3 -m pip install virtualenv + +# Buildbot user +ARG BUILDBOT_USER=zcbbworker +ARG BUILDBOT_UID=2001 +RUN useradd --home-dir /home/$BUILDBOT_USER \ + --shell /bin/bash \ + --create-home \ + --uid $BUILDBOT_UID\ + $BUILDBOT_USER + +USER $BUILDBOT_USER +WORKDIR /home/$BUILDBOT_USER + +RUN python3 -m venv venv \ + && . venv/bin/activate \ + && python3 -m pip install \ + buildbot-worker \ + pyflakes \ + pyblake2 \ + pyzmq \ + requests \ + twisted[tls] + +# Buildbot worker +ARG BUILDBOT_WORKER_NAME=centos8-docker +ARG BUILDBOT_WORKER_PASS=thisgetssetwhenpodisstarted +ARG BUILDBOT_MASTER_HOST=dev-ci.z.cash +ARG BUILDBOT_MASTER_PORT=9899 + +WORKDIR /home/$BUILDBOT_USER +RUN venv/bin/buildbot-worker create-worker $BUILDBOT_WORKER_NAME \ + $BUILDBOT_MASTER_HOST:$BUILDBOT_MASTER_PORT \ + $BUILDBOT_WORKER_NAME $BUILDBOT_WORKER_PASS \ + && echo "OS: Centos 8" > $BUILDBOT_WORKER_NAME/info/host +ADD buildbot.tac $BUILDBOT_WORKER_NAME/buildbot.tac + +WORKDIR /home/$BUILDBOT_USER/$BUILDBOT_WORKER_NAME +CMD ["/usr/bin/dumb-init", "../venv/bin/twistd", "--pidfile=", "-ny", "buildbot.tac"] diff --git a/contrib/ci-builders/Dockerfile-build.apt b/contrib/ci-builders/Dockerfile-build.apt new file mode 100644 index 000000000..3b7fd9541 --- /dev/null +++ b/contrib/ci-builders/Dockerfile-build.apt @@ -0,0 +1,9 @@ +ARG FROMBASEOS +ARG FROMBASEOS_BUILD_TAG +FROM $FROMBASEOS:$FROMBASEOS_BUILD_TAG +ARG DEBIAN_FRONTEND=noninteractive + +ADD apt-package-list.txt /tmp/apt-package-list.txt +RUN apt-get update \ + && apt-get install -y $(tr "\n" " " < /tmp/apt-package-list.txt) \ + && update-alternatives --install /usr/bin/python python /usr/bin/python3 1 diff --git a/contrib/ci-builders/Dockerfile-gitian.apt b/contrib/ci-builders/Dockerfile-gitian.apt new file mode 100644 index 000000000..247262c17 --- /dev/null +++ b/contrib/ci-builders/Dockerfile-gitian.apt @@ -0,0 +1,8 @@ +ARG FROMBASEOS +ARG FROMBASEOS_BUILD_TAG=latest +FROM electriccoinco/zcashd-build:$FROMBASEOS$FROMBASEOS_BUILD_TAG + +RUN useradd -ms /bin/bash -U debian +USER debian:debian +WORKDIR /home/debian +CMD ["sleep", "infinity"] \ No newline at end of file diff --git a/contrib/ci-builders/README.md b/contrib/ci-builders/README.md new file mode 100644 index 000000000..457fe1f3a --- /dev/null +++ b/contrib/ci-builders/README.md @@ -0,0 +1,42 @@ +# zcashd ci Docker images + +These Dockerfiles can be used to build zcashd. + +The current objective is to build a base image for each distribution that includes the system packages to build zcashd. From `build` images, more targeted images are created. + +The process is meant to be automated, but an example `docker-build.sh` script is included. + + +## build images +`apt-package-list.txt` contains the required packages for debian based systems. + +`Dockerfile-build.apt` uses that file, and some build time arguements, to build apt based build images. + +Currently available images are hosted at +https://hub.docker.com/r/electriccoinco/zcashd-build/tags + + +## gitian images + +`Dockerfile-gitian.apt` uses the build images to generate images for use with [gitian-builder](https://github.com/devrandom/gitian-builder) + +Currently available images are hosted at https://hub.docker.com/r/electriccoinco/zcashd-gitian/tags + + +## bbworker images + +`Dockerfile-bbworker.apt` uses the build images for use with [buildbot](http://buildbot.net/) + +- bbworker-buildbot.tac is a required worker boot file (most is overridden at instancitation) +- bbworker-requirements.txt is the python package requirements for the buildbot worker + +Currently available images are hosted at https://hub.docker.com/r/electriccoinco/zcashd-bbworker/tags + + +### Stand alone, best effort images + +Additional buildbot workers for Centos8 and Arch. + +- Dockerfile-bbworker.arch +- Dockerfile-bbworker.centos8 + diff --git a/contrib/ci-builders/apt-package-list.txt b/contrib/ci-builders/apt-package-list.txt new file mode 100644 index 000000000..c1725b15a --- /dev/null +++ b/contrib/ci-builders/apt-package-list.txt @@ -0,0 +1,16 @@ +autoconf +bsdmainutils +build-essential +cmake +curl +git +lcov +libffi-dev +libtool +libssl-dev +pkg-config +python3 +python3-dev +python3-venv +valgrind +wget diff --git a/contrib/ci-builders/bbworker-buildbot.tac b/contrib/ci-builders/bbworker-buildbot.tac new file mode 100644 index 000000000..477ae27d3 --- /dev/null +++ b/contrib/ci-builders/bbworker-buildbot.tac @@ -0,0 +1,40 @@ +import fnmatch +import os +import sys + +from twisted.application import service +from twisted.python.log import FileLogObserver +from twisted.python.log import ILogObserver + +from buildbot_worker.bot import Worker + +# setup worker +basedir = os.environ.get("BUILDBOT_BASEDIR", + os.path.abspath(os.path.dirname(__file__))) +application = service.Application('buildbot-worker') + + +application.setComponent(ILogObserver, FileLogObserver(sys.stdout).emit) +# and worker on the same process! +buildmaster_host = os.environ.get("BUILDMASTER", 'localhost') +port = int(os.environ.get("BUILDMASTER_PORT", 9989)) +workername = os.environ.get("WORKERNAME", 'docker') +passwd = os.environ.get("WORKERPASS") + +# delete the password from the environ so that it is not leaked in the log +blacklist = os.environ.get("WORKER_ENVIRONMENT_BLACKLIST", "WORKERPASS").split() +for name in list(os.environ.keys()): + for toremove in blacklist: + if fnmatch.fnmatch(name, toremove): + del os.environ[name] + +keepalive = 600 +umask = None +maxdelay = 300 +allow_shutdown = None +maxretries = 10 + +s = Worker(buildmaster_host, port, workername, passwd, basedir, + keepalive, umask=umask, maxdelay=maxdelay, + allow_shutdown=allow_shutdown, maxRetries=maxretries) +s.setServiceParent(application) diff --git a/contrib/ci-builders/bbworker-requirements.txt b/contrib/ci-builders/bbworker-requirements.txt new file mode 100644 index 000000000..d5492aaa8 --- /dev/null +++ b/contrib/ci-builders/bbworker-requirements.txt @@ -0,0 +1,10 @@ +buildbot-worker +pyblake2 +pyflakes +python-qpid-proton +pyutil +pyzmq +requests +simplejson +twisted[tls] +wheel diff --git a/contrib/ci-builders/docker-build.sh b/contrib/ci-builders/docker-build.sh new file mode 100644 index 000000000..6a37027af --- /dev/null +++ b/contrib/ci-builders/docker-build.sh @@ -0,0 +1,37 @@ +# Debian +docker build . -f Dockerfile-build.apt --build-arg FROMBASEOS=debian --build-arg FROMBASEOS_BUILD_TAG=9 -t electriccoinco/zcashd-build:debian9 +docker build . -f Dockerfile-build.apt --build-arg FROMBASEOS=debian --build-arg FROMBASEOS_BUILD_TAG=10 -t electriccoinco/zcashd-build:debian10 + +docker push electriccoinco/zcashd-build:debian9 +docker push electriccoinco/zcashd-build:debian10 + +docker build . -f Dockerfile-gitian.apt --build-arg FROMBASEOS=debian --build-arg FROMBASEOS_BUILD_TAG=9 -t electriccoinco/zcashd-gitian:debian9 +docker build . -f Dockerfile-gitian.apt --build-arg FROMBASEOS=debian --build-arg FROMBASEOS_BUILD_TAG=10 -t electriccoinco/zcashd-gitian:debian10 + +docker push electriccoinco/zcashd-gitian:debian9 +docker push electriccoinco/zcashd-gitian:debian10 + +docker build . -f Dockerfile-bbworker.apt --build-arg BASEOS=debian --build-arg FROMBASEOS=debian --build-arg FROMBASEOS_BUILD_TAG=9 -t electriccoinco/zcashd-bbworker:debian9 +docker build . -f Dockerfile-bbworker.apt --build-arg BASEOS=debian --build-arg FROMBASEOS=debian --build-arg FROMBASEOS_BUILD_TAG=10 -t electriccoinco/zcashd-bbworker:debian10 + +docker push electriccoinco/zcashd-bbworker:debian9 +docker push electriccoinco/zcashd-bbworker:debian10 + +# Ubuntu +docker build . -f Dockerfile-build.apt --build-arg FROMBASEOS=ubuntu --build-arg FROMBASEOS_BUILD_TAG=18.04 -t electriccoinco/zcashd-build:ubuntu18.04 +docker build . -f Dockerfile-build.apt --build-arg FROMBASEOS=ubuntu --build-arg FROMBASEOS_BUILD_TAG=20.04 -t electriccoinco/zcashd-build:ubuntu20.04 + +docker push electriccoinco/zcashd-build:ubuntu18.04 +docker push electriccoinco/zcashd-build:ubuntu20.04 + +docker build . -f Dockerfile-gitian.apt --build-arg FROMBASEOS=ubuntu --build-arg FROMBASEOS_BUILD_TAG=18.04 -t electriccoinco/zcashd-gitian:ubuntu18.04 +docker build . -f Dockerfile-gitian.apt --build-arg FROMBASEOS=ubuntu --build-arg FROMBASEOS_BUILD_TAG=20.04 -t electriccoinco/zcashd-gitian:ubuntu20.04 + +docker push electriccoinco/zcashd-gitian:ubuntu18.04 +docker push electriccoinco/zcashd-gitian:ubuntu20.04 + +docker build . -f Dockerfile-bbworker.apt --build-arg BASEOS=ubuntu --build-arg FROMBASEOS=ubuntu --build-arg FROMBASEOS_BUILD_TAG=18.04 -t electriccoinco/zcashd-bbworker:ubuntu18.04 +docker build . -f Dockerfile-bbworker.apt --build-arg BASEOS=ubuntu --build-arg FROMBASEOS=ubuntu --build-arg FROMBASEOS_BUILD_TAG=20.04 -t electriccoinco/zcashd-bbworker:ubuntu20.04 + +docker push electriccoinco/zcashd-bbworker:ubuntu18.04 +docker push electriccoinco/zcashd-bbworker:ubuntu20.04 \ No newline at end of file