Auto merge of #4490 - benzcash:ben/contrib/ci-builders, r=mdr0id
Dockerfiles for zcashd CI builds Please ensure this checklist is followed for any pull requests for this repo. This checklist must be checked by both the PR creator and by anyone who reviews the PR. * [ ] Relevant documentation for this PR has to be completed and reviewed by @mdr0id before the PR can be merged * [ ] A test plan for the PR must be documented in the PR notes and included in the test plan for the next regular release As a note, all buildbot tests need to be passing and all appropriate code reviews need to be done before this PR can be merged This PR is intended to move the existing buildbot worker docker images into this repo, and provide for a way to build gitian-builder images. The included README.md describes the purpose and an example script. When this is merged the existing buildbot CI will be updated to start using these images.
This commit is contained in:
Homu
commit
4b71f1633a
|
|
@ -0,0 +1,42 @@
|
|||
ARG FROMBASEOS
|
||||
ARG FROMBASEOS_BUILD_TAG=latest
|
||||
FROM electriccoinco/zcashd-build:$FROMBASEOS$FROMBASEOS_BUILD_TAG
|
||||
|
||||
ARG DUMBINIT_VERSION=1.2.2
|
||||
RUN wget -O /usr/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v${DUMBINIT_VERSION}/dumb-init_${DUMBINIT_VERSION}_amd64
|
||||
RUN chmod +x /usr/bin/dumb-init
|
||||
|
||||
# Buildbot user
|
||||
ARG BUILDBOT_USER=zcbbworker
|
||||
ARG BUILDBOT_UID=2001
|
||||
RUN useradd --home-dir /home/$BUILDBOT_USER \
|
||||
--shell /bin/bash \
|
||||
--create-home \
|
||||
--uid $BUILDBOT_UID\
|
||||
$BUILDBOT_USER
|
||||
|
||||
USER $BUILDBOT_USER
|
||||
WORKDIR /home/$BUILDBOT_USER
|
||||
|
||||
ADD bbworker-requirements.txt requirements.txt
|
||||
RUN python -m venv venv \
|
||||
&& . venv/bin/activate \
|
||||
&& python -m pip install wheel \
|
||||
&& python -m pip install -r requirements.txt
|
||||
|
||||
# Buildbot worker
|
||||
ARG BASEOS
|
||||
ENV BUILDBOT_WORKER_NAME=$BASEOS-docker
|
||||
ENV BUILDBOT_WORKER_PASS=thisgetssetwhenpodisstarted
|
||||
ENV BUILDBOT_MASTER_HOST=dev-ci.z.cash
|
||||
ENV BUILDBOT_MASTER_PORT=9899
|
||||
|
||||
WORKDIR /home/$BUILDBOT_USER
|
||||
RUN venv/bin/buildbot-worker create-worker $BUILDBOT_WORKER_NAME \
|
||||
$BUILDBOT_MASTER_HOST:$BUILDBOT_MASTER_PORT \
|
||||
$BUILDBOT_WORKER_NAME $BUILDBOT_WORKER_PASS \
|
||||
&& echo "OS: $BASEOS" > $BUILDBOT_WORKER_NAME/info/host
|
||||
ADD bbworker-buildbot.tac $BUILDBOT_WORKER_NAME/buildbot.tac
|
||||
|
||||
WORKDIR /home/$BUILDBOT_USER/$BUILDBOT_WORKER_NAME
|
||||
CMD ["/usr/bin/dumb-init", "../venv/bin/twistd", "--pidfile=", "-ny", "buildbot.tac"]
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
ARG ARCHLINUX_TAG=20200205
|
||||
FROM archlinux:$ARCHLINUX_TAG
|
||||
|
||||
RUN pacman -Syyu --noconfirm \
|
||||
&& pacman -S --noconfirm \
|
||||
base-devel \
|
||||
git \
|
||||
python3 \
|
||||
python-pip \
|
||||
wget
|
||||
|
||||
RUN wget -O /usr/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.2/dumb-init_1.2.2_amd64
|
||||
RUN chmod +x /usr/bin/dumb-init
|
||||
RUN python -m pip install virtualenv
|
||||
|
||||
# Buildbot user
|
||||
ARG BUILDBOT_USER=zcbbworker
|
||||
ARG BUILDBOT_UID=2001
|
||||
RUN useradd --home-dir /home/$BUILDBOT_USER \
|
||||
--shell /bin/bash \
|
||||
--create-home \
|
||||
--uid $BUILDBOT_UID\
|
||||
$BUILDBOT_USER
|
||||
|
||||
USER $BUILDBOT_USER
|
||||
WORKDIR /home/$BUILDBOT_USER
|
||||
|
||||
RUN python -m venv venv \
|
||||
&& . venv/bin/activate \
|
||||
&& python -m pip install \
|
||||
buildbot-worker \
|
||||
pyflakes \
|
||||
pyblake2 \
|
||||
pyzmq \
|
||||
requests \
|
||||
twisted[tls]
|
||||
|
||||
# Buildbot worker
|
||||
ARG BUILDBOT_WORKER_NAME=arch-docker
|
||||
ARG BUILDBOT_WORKER_PASS=thisgetssetwhenpodisstarted
|
||||
ARG BUILDBOT_MASTER_HOST=dev-ci.z.cash
|
||||
ARG BUILDBOT_MASTER_PORT=9899
|
||||
|
||||
WORKDIR /home/$BUILDBOT_USER
|
||||
RUN venv/bin/buildbot-worker create-worker $BUILDBOT_WORKER_NAME \
|
||||
$BUILDBOT_MASTER_HOST:$BUILDBOT_MASTER_PORT \
|
||||
$BUILDBOT_WORKER_NAME $BUILDBOT_WORKER_PASS \
|
||||
&& echo "OS: Centos 8" > $BUILDBOT_WORKER_NAME/info/host
|
||||
ADD buildbot.tac $BUILDBOT_WORKER_NAME/buildbot.tac
|
||||
|
||||
WORKDIR /home/$BUILDBOT_USER/$BUILDBOT_WORKER_NAME
|
||||
CMD ["/usr/bin/dumb-init", "../venv/bin/twistd", "--pidfile=", "-ny", "buildbot.tac"]
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
FROM centos:8
|
||||
|
||||
RUN yum update -y \
|
||||
&& dnf group install -y "Development Tools" \
|
||||
&& yum install -y \
|
||||
python3 \
|
||||
python3-devel \
|
||||
wget
|
||||
|
||||
RUN wget -O /usr/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.2/dumb-init_1.2.2_amd64
|
||||
RUN chmod +x /usr/bin/dumb-init
|
||||
RUN alternatives --set python /usr/bin/python3 \
|
||||
&& python3 -m pip install virtualenv
|
||||
|
||||
# Buildbot user
|
||||
ARG BUILDBOT_USER=zcbbworker
|
||||
ARG BUILDBOT_UID=2001
|
||||
RUN useradd --home-dir /home/$BUILDBOT_USER \
|
||||
--shell /bin/bash \
|
||||
--create-home \
|
||||
--uid $BUILDBOT_UID\
|
||||
$BUILDBOT_USER
|
||||
|
||||
USER $BUILDBOT_USER
|
||||
WORKDIR /home/$BUILDBOT_USER
|
||||
|
||||
RUN python3 -m venv venv \
|
||||
&& . venv/bin/activate \
|
||||
&& python3 -m pip install \
|
||||
buildbot-worker \
|
||||
pyflakes \
|
||||
pyblake2 \
|
||||
pyzmq \
|
||||
requests \
|
||||
twisted[tls]
|
||||
|
||||
# Buildbot worker
|
||||
ARG BUILDBOT_WORKER_NAME=centos8-docker
|
||||
ARG BUILDBOT_WORKER_PASS=thisgetssetwhenpodisstarted
|
||||
ARG BUILDBOT_MASTER_HOST=dev-ci.z.cash
|
||||
ARG BUILDBOT_MASTER_PORT=9899
|
||||
|
||||
WORKDIR /home/$BUILDBOT_USER
|
||||
RUN venv/bin/buildbot-worker create-worker $BUILDBOT_WORKER_NAME \
|
||||
$BUILDBOT_MASTER_HOST:$BUILDBOT_MASTER_PORT \
|
||||
$BUILDBOT_WORKER_NAME $BUILDBOT_WORKER_PASS \
|
||||
&& echo "OS: Centos 8" > $BUILDBOT_WORKER_NAME/info/host
|
||||
ADD buildbot.tac $BUILDBOT_WORKER_NAME/buildbot.tac
|
||||
|
||||
WORKDIR /home/$BUILDBOT_USER/$BUILDBOT_WORKER_NAME
|
||||
CMD ["/usr/bin/dumb-init", "../venv/bin/twistd", "--pidfile=", "-ny", "buildbot.tac"]
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
ARG FROMBASEOS
|
||||
ARG FROMBASEOS_BUILD_TAG
|
||||
FROM $FROMBASEOS:$FROMBASEOS_BUILD_TAG
|
||||
ARG DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
ADD apt-package-list.txt /tmp/apt-package-list.txt
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y $(tr "\n" " " < /tmp/apt-package-list.txt) \
|
||||
&& update-alternatives --install /usr/bin/python python /usr/bin/python3 1
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
ARG FROMBASEOS
|
||||
ARG FROMBASEOS_BUILD_TAG=latest
|
||||
FROM electriccoinco/zcashd-build:$FROMBASEOS$FROMBASEOS_BUILD_TAG
|
||||
|
||||
RUN useradd -ms /bin/bash -U debian
|
||||
USER debian:debian
|
||||
WORKDIR /home/debian
|
||||
CMD ["sleep", "infinity"]
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
# zcashd ci Docker images
|
||||
|
||||
These Dockerfiles can be used to build zcashd.
|
||||
|
||||
The current objective is to build a base image for each distribution that includes the system packages to build zcashd. From `build` images, more targeted images are created.
|
||||
|
||||
The process is meant to be automated, but an example `docker-build.sh` script is included.
|
||||
|
||||
|
||||
## build images
|
||||
`apt-package-list.txt` contains the required packages for debian based systems.
|
||||
|
||||
`Dockerfile-build.apt` uses that file, and some build time arguements, to build apt based build images.
|
||||
|
||||
Currently available images are hosted at
|
||||
https://hub.docker.com/r/electriccoinco/zcashd-build/tags
|
||||
|
||||
|
||||
## gitian images
|
||||
|
||||
`Dockerfile-gitian.apt` uses the build images to generate images for use with [gitian-builder](https://github.com/devrandom/gitian-builder)
|
||||
|
||||
Currently available images are hosted at https://hub.docker.com/r/electriccoinco/zcashd-gitian/tags
|
||||
|
||||
|
||||
## bbworker images
|
||||
|
||||
`Dockerfile-bbworker.apt` uses the build images for use with [buildbot](http://buildbot.net/)
|
||||
|
||||
- bbworker-buildbot.tac is a required worker boot file (most is overridden at instancitation)
|
||||
- bbworker-requirements.txt is the python package requirements for the buildbot worker
|
||||
|
||||
Currently available images are hosted at https://hub.docker.com/r/electriccoinco/zcashd-bbworker/tags
|
||||
|
||||
|
||||
### Stand alone, best effort images
|
||||
|
||||
Additional buildbot workers for Centos8 and Arch.
|
||||
|
||||
- Dockerfile-bbworker.arch
|
||||
- Dockerfile-bbworker.centos8
|
||||
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
autoconf
|
||||
bsdmainutils
|
||||
build-essential
|
||||
cmake
|
||||
curl
|
||||
git
|
||||
lcov
|
||||
libffi-dev
|
||||
libtool
|
||||
libssl-dev
|
||||
pkg-config
|
||||
python3
|
||||
python3-dev
|
||||
python3-venv
|
||||
valgrind
|
||||
wget
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
import fnmatch
|
||||
import os
|
||||
import sys
|
||||
|
||||
from twisted.application import service
|
||||
from twisted.python.log import FileLogObserver
|
||||
from twisted.python.log import ILogObserver
|
||||
|
||||
from buildbot_worker.bot import Worker
|
||||
|
||||
# setup worker
|
||||
basedir = os.environ.get("BUILDBOT_BASEDIR",
|
||||
os.path.abspath(os.path.dirname(__file__)))
|
||||
application = service.Application('buildbot-worker')
|
||||
|
||||
|
||||
application.setComponent(ILogObserver, FileLogObserver(sys.stdout).emit)
|
||||
# and worker on the same process!
|
||||
buildmaster_host = os.environ.get("BUILDMASTER", 'localhost')
|
||||
port = int(os.environ.get("BUILDMASTER_PORT", 9989))
|
||||
workername = os.environ.get("WORKERNAME", 'docker')
|
||||
passwd = os.environ.get("WORKERPASS")
|
||||
|
||||
# delete the password from the environ so that it is not leaked in the log
|
||||
blacklist = os.environ.get("WORKER_ENVIRONMENT_BLACKLIST", "WORKERPASS").split()
|
||||
for name in list(os.environ.keys()):
|
||||
for toremove in blacklist:
|
||||
if fnmatch.fnmatch(name, toremove):
|
||||
del os.environ[name]
|
||||
|
||||
keepalive = 600
|
||||
umask = None
|
||||
maxdelay = 300
|
||||
allow_shutdown = None
|
||||
maxretries = 10
|
||||
|
||||
s = Worker(buildmaster_host, port, workername, passwd, basedir,
|
||||
keepalive, umask=umask, maxdelay=maxdelay,
|
||||
allow_shutdown=allow_shutdown, maxRetries=maxretries)
|
||||
s.setServiceParent(application)
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
buildbot-worker
|
||||
pyblake2
|
||||
pyflakes
|
||||
python-qpid-proton
|
||||
pyutil
|
||||
pyzmq
|
||||
requests
|
||||
simplejson
|
||||
twisted[tls]
|
||||
wheel
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
# Debian
|
||||
docker build . -f Dockerfile-build.apt --build-arg FROMBASEOS=debian --build-arg FROMBASEOS_BUILD_TAG=9 -t electriccoinco/zcashd-build:debian9
|
||||
docker build . -f Dockerfile-build.apt --build-arg FROMBASEOS=debian --build-arg FROMBASEOS_BUILD_TAG=10 -t electriccoinco/zcashd-build:debian10
|
||||
|
||||
docker push electriccoinco/zcashd-build:debian9
|
||||
docker push electriccoinco/zcashd-build:debian10
|
||||
|
||||
docker build . -f Dockerfile-gitian.apt --build-arg FROMBASEOS=debian --build-arg FROMBASEOS_BUILD_TAG=9 -t electriccoinco/zcashd-gitian:debian9
|
||||
docker build . -f Dockerfile-gitian.apt --build-arg FROMBASEOS=debian --build-arg FROMBASEOS_BUILD_TAG=10 -t electriccoinco/zcashd-gitian:debian10
|
||||
|
||||
docker push electriccoinco/zcashd-gitian:debian9
|
||||
docker push electriccoinco/zcashd-gitian:debian10
|
||||
|
||||
docker build . -f Dockerfile-bbworker.apt --build-arg BASEOS=debian --build-arg FROMBASEOS=debian --build-arg FROMBASEOS_BUILD_TAG=9 -t electriccoinco/zcashd-bbworker:debian9
|
||||
docker build . -f Dockerfile-bbworker.apt --build-arg BASEOS=debian --build-arg FROMBASEOS=debian --build-arg FROMBASEOS_BUILD_TAG=10 -t electriccoinco/zcashd-bbworker:debian10
|
||||
|
||||
docker push electriccoinco/zcashd-bbworker:debian9
|
||||
docker push electriccoinco/zcashd-bbworker:debian10
|
||||
|
||||
# Ubuntu
|
||||
docker build . -f Dockerfile-build.apt --build-arg FROMBASEOS=ubuntu --build-arg FROMBASEOS_BUILD_TAG=18.04 -t electriccoinco/zcashd-build:ubuntu18.04
|
||||
docker build . -f Dockerfile-build.apt --build-arg FROMBASEOS=ubuntu --build-arg FROMBASEOS_BUILD_TAG=20.04 -t electriccoinco/zcashd-build:ubuntu20.04
|
||||
|
||||
docker push electriccoinco/zcashd-build:ubuntu18.04
|
||||
docker push electriccoinco/zcashd-build:ubuntu20.04
|
||||
|
||||
docker build . -f Dockerfile-gitian.apt --build-arg FROMBASEOS=ubuntu --build-arg FROMBASEOS_BUILD_TAG=18.04 -t electriccoinco/zcashd-gitian:ubuntu18.04
|
||||
docker build . -f Dockerfile-gitian.apt --build-arg FROMBASEOS=ubuntu --build-arg FROMBASEOS_BUILD_TAG=20.04 -t electriccoinco/zcashd-gitian:ubuntu20.04
|
||||
|
||||
docker push electriccoinco/zcashd-gitian:ubuntu18.04
|
||||
docker push electriccoinco/zcashd-gitian:ubuntu20.04
|
||||
|
||||
docker build . -f Dockerfile-bbworker.apt --build-arg BASEOS=ubuntu --build-arg FROMBASEOS=ubuntu --build-arg FROMBASEOS_BUILD_TAG=18.04 -t electriccoinco/zcashd-bbworker:ubuntu18.04
|
||||
docker build . -f Dockerfile-bbworker.apt --build-arg BASEOS=ubuntu --build-arg FROMBASEOS=ubuntu --build-arg FROMBASEOS_BUILD_TAG=20.04 -t electriccoinco/zcashd-bbworker:ubuntu20.04
|
||||
|
||||
docker push electriccoinco/zcashd-bbworker:ubuntu18.04
|
||||
docker push electriccoinco/zcashd-bbworker:ubuntu20.04
|
||||
Loading…
Reference in New Issue