diff --git a/contrib/ci-builders/Dockerfile-bbworker.apt b/contrib/ci-builders/Dockerfile-bbworker.apt index 89e00ffb1..40ee4b8a4 100644 --- a/contrib/ci-builders/Dockerfile-bbworker.apt +++ b/contrib/ci-builders/Dockerfile-bbworker.apt @@ -1,6 +1,6 @@ ARG FROMBASEOS ARG FROMBASEOS_BUILD_TAG=latest -FROM electriccoinco/zcashd-build:$FROMBASEOS$FROMBASEOS_BUILD_TAG +FROM electriccoinco/zcashd-build-$FROMBASEOS$FROMBASEOS_BUILD_TAG ARG DUMBINIT_VERSION=1.2.2 RUN wget -O /usr/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v${DUMBINIT_VERSION}/dumb-init_${DUMBINIT_VERSION}_amd64 diff --git a/contrib/ci-builders/Dockerfile-bbworker.arch b/contrib/ci-builders/Dockerfile-bbworker.arch index 0f81aa386..c3e6ff9d7 100644 --- a/contrib/ci-builders/Dockerfile-bbworker.arch +++ b/contrib/ci-builders/Dockerfile-bbworker.arch @@ -1,17 +1,4 @@ -ARG ARCHLINUX_TAG=20200205 -FROM archlinux:$ARCHLINUX_TAG - -RUN pacman -Syyu --noconfirm \ - && pacman -S --noconfirm \ - base-devel \ - git \ - python3 \ - python-pip \ - wget - -RUN wget -O /usr/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.2/dumb-init_1.2.2_amd64 -RUN chmod +x /usr/bin/dumb-init -RUN python -m pip install virtualenv +FROM electriccoinco/zcashd-build-arch # Buildbot user ARG BUILDBOT_USER=zcbbworker @@ -25,15 +12,11 @@ RUN useradd --home-dir /home/$BUILDBOT_USER \ USER $BUILDBOT_USER WORKDIR /home/$BUILDBOT_USER +ADD bbworker-requirements.txt requirements.txt RUN python -m venv venv \ && . venv/bin/activate \ - && python -m pip install \ - buildbot-worker \ - pyflakes \ - pyblake2 \ - pyzmq \ - requests \ - twisted[tls] + && python -m pip install wheel \ + && python -m pip install -r requirements.txt # Buildbot worker ARG BUILDBOT_WORKER_NAME=arch-docker @@ -46,7 +29,7 @@ RUN venv/bin/buildbot-worker create-worker $BUILDBOT_WORKER_NAME \ $BUILDBOT_MASTER_HOST:$BUILDBOT_MASTER_PORT \ $BUILDBOT_WORKER_NAME $BUILDBOT_WORKER_PASS \ && echo "OS: Centos 8" > $BUILDBOT_WORKER_NAME/info/host -ADD buildbot.tac $BUILDBOT_WORKER_NAME/buildbot.tac +ADD bbworker-buildbot.tac $BUILDBOT_WORKER_NAME/buildbot.tac WORKDIR /home/$BUILDBOT_USER/$BUILDBOT_WORKER_NAME CMD ["/usr/bin/dumb-init", "../venv/bin/twistd", "--pidfile=", "-ny", "buildbot.tac"] diff --git a/contrib/ci-builders/Dockerfile-bbworker.centos8 b/contrib/ci-builders/Dockerfile-bbworker.centos8 index 79adaf860..fbc02101b 100644 --- a/contrib/ci-builders/Dockerfile-bbworker.centos8 +++ b/contrib/ci-builders/Dockerfile-bbworker.centos8 @@ -1,16 +1,4 @@ -FROM centos:8 - -RUN yum update -y \ - && dnf group install -y "Development Tools" \ - && yum install -y \ - python3 \ - python3-devel \ - wget - -RUN wget -O /usr/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.2/dumb-init_1.2.2_amd64 -RUN chmod +x /usr/bin/dumb-init -RUN alternatives --set python /usr/bin/python3 \ - && python3 -m pip install virtualenv +FROM electriccoinco/zcashd-build-centos8 # Buildbot user ARG BUILDBOT_USER=zcbbworker @@ -24,15 +12,11 @@ RUN useradd --home-dir /home/$BUILDBOT_USER \ USER $BUILDBOT_USER WORKDIR /home/$BUILDBOT_USER +ADD bbworker-requirements.txt requirements.txt RUN python3 -m venv venv \ && . venv/bin/activate \ - && python3 -m pip install \ - buildbot-worker \ - pyflakes \ - pyblake2 \ - pyzmq \ - requests \ - twisted[tls] + && python -m pip install wheel \ + && python -m pip install -r requirements.txt # Buildbot worker ARG BUILDBOT_WORKER_NAME=centos8-docker @@ -45,7 +29,7 @@ RUN venv/bin/buildbot-worker create-worker $BUILDBOT_WORKER_NAME \ $BUILDBOT_MASTER_HOST:$BUILDBOT_MASTER_PORT \ $BUILDBOT_WORKER_NAME $BUILDBOT_WORKER_PASS \ && echo "OS: Centos 8" > $BUILDBOT_WORKER_NAME/info/host -ADD buildbot.tac $BUILDBOT_WORKER_NAME/buildbot.tac +ADD bbworker-buildbot.tac $BUILDBOT_WORKER_NAME/buildbot.tac WORKDIR /home/$BUILDBOT_USER/$BUILDBOT_WORKER_NAME CMD ["/usr/bin/dumb-init", "../venv/bin/twistd", "--pidfile=", "-ny", "buildbot.tac"] diff --git a/contrib/ci-builders/Dockerfile-build-python.apt b/contrib/ci-builders/Dockerfile-build-python.apt new file mode 100644 index 000000000..8b73e9bb7 --- /dev/null +++ b/contrib/ci-builders/Dockerfile-build-python.apt @@ -0,0 +1,30 @@ +ARG FROMBASEOS +ARG FROMBASEOS_BUILD_TAG +FROM $FROMBASEOS:$FROMBASEOS_BUILD_TAG +ARG DEBIAN_FRONTEND=noninteractive + +ADD apt-package-list.txt /tmp/apt-package-list.txt +RUN apt-get update \ + && apt-get install -y $(tr "\n" " " < /tmp/apt-package-list.txt) \ + && update-alternatives --install /usr/bin/python python /usr/bin/python3 1 + +ARG PYTHONVERSION=3.7.5 +RUN apt-get install -y \ + build-essential \ + zlib1g-dev \ + libncurses5-dev \ + libgdbm-dev \ + libnss3-dev \ + libssl-dev \ + libreadline-dev \ + libffi-dev \ + curl +WORKDIR /tmp +RUN curl https://www.python.org/ftp/python/$PYTHONVERSION/Python-$PYTHONVERSION.tar.xz -o python.tar.xz \ + && tar -xf python.tar.xz \ + && cd Python-$PYTHONVERSION \ + && ./configure --enable-optimizations \ + && make -j 8 \ + && make altinstall \ + && update-alternatives --install /usr/bin/python python /usr/local/bin/python3.7 1 +RUN update-alternatives --install /usr/bin/python3 python3 /usr/local/bin/python3.7 1 \ No newline at end of file diff --git a/contrib/ci-builders/Dockerfile-build.arch b/contrib/ci-builders/Dockerfile-build.arch new file mode 100644 index 000000000..c75f5a190 --- /dev/null +++ b/contrib/ci-builders/Dockerfile-build.arch @@ -0,0 +1,23 @@ +ARG ARCHLINUX_TAG +FROM archlinux:$ARCHLINUX_TAG + +RUN pacman -Syyu --noconfirm \ + && pacman -S --noconfirm \ + base-devel \ + git \ + python3 \ + python-pip \ + wget + +RUN wget -O /usr/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.2/dumb-init_1.2.2_amd64 +RUN chmod +x /usr/bin/dumb-init +RUN python -m pip install virtualenv +# AUR for libtinfo5 Source: https://dev.to/cloudx/testing-our-package-build-in-the-docker-world-34p0 +RUN useradd builduser -m \ + && passwd -d builduser \ + && cd /home/builduser \ + && git clone "https://aur.archlinux.org/ncurses5-compat-libs.git" ncurses5-compat-libs \ + && chown builduser -R ncurses5-compat-libs \ + && (printf 'builduser ALL=(ALL) ALL\n' | tee -a /etc/sudoers) \ + && sudo -u builduser bash -c 'gpg --keyserver pool.sks-keyservers.net --recv-keys 702353E0F7E48EDB' \ + && sudo -u builduser bash -c 'cd ~/ncurses5-compat-libs && makepkg -si --noconfirm' \ No newline at end of file diff --git a/contrib/ci-builders/Dockerfile-build.centos8 b/contrib/ci-builders/Dockerfile-build.centos8 new file mode 100644 index 000000000..b5fb3eb20 --- /dev/null +++ b/contrib/ci-builders/Dockerfile-build.centos8 @@ -0,0 +1,14 @@ +FROM centos:8 + +RUN yum update -y \ + && dnf group install -y "Development Tools" \ + && yum install -y \ + ncurses-compat-libs \ + python3 \ + python3-devel \ + wget + +RUN wget -O /usr/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.2/dumb-init_1.2.2_amd64 +RUN chmod +x /usr/bin/dumb-init +RUN alternatives --set python /usr/bin/python3 \ + && python3 -m pip install virtualenv diff --git a/contrib/ci-builders/Dockerfile-gitian.apt b/contrib/ci-builders/Dockerfile-gitian.apt index 247262c17..5ca2a8757 100644 --- a/contrib/ci-builders/Dockerfile-gitian.apt +++ b/contrib/ci-builders/Dockerfile-gitian.apt @@ -1,6 +1,6 @@ ARG FROMBASEOS ARG FROMBASEOS_BUILD_TAG=latest -FROM electriccoinco/zcashd-build:$FROMBASEOS$FROMBASEOS_BUILD_TAG +FROM electriccoinco/zcashd-build-$FROMBASEOS$FROMBASEOS_BUILD_TAG RUN useradd -ms /bin/bash -U debian USER debian:debian diff --git a/contrib/ci-builders/apt-package-list.txt b/contrib/ci-builders/apt-package-list.txt index c1725b15a..c7813dfdc 100644 --- a/contrib/ci-builders/apt-package-list.txt +++ b/contrib/ci-builders/apt-package-list.txt @@ -3,11 +3,17 @@ bsdmainutils build-essential cmake curl +g++-aarch64-linux-gnu git lcov +libcap-dev libffi-dev +libtinfo5 libtool libssl-dev +libz-dev +libbz2-dev +mingw-w64 pkg-config python3 python3-dev diff --git a/contrib/ci-builders/docker-build.sh b/contrib/ci-builders/docker-build.sh old mode 100644 new mode 100755 index 6a37027af..232c5b9b1 --- a/contrib/ci-builders/docker-build.sh +++ b/contrib/ci-builders/docker-build.sh @@ -1,37 +1,49 @@ -# Debian -docker build . -f Dockerfile-build.apt --build-arg FROMBASEOS=debian --build-arg FROMBASEOS_BUILD_TAG=9 -t electriccoinco/zcashd-build:debian9 -docker build . -f Dockerfile-build.apt --build-arg FROMBASEOS=debian --build-arg FROMBASEOS_BUILD_TAG=10 -t electriccoinco/zcashd-build:debian10 +#!/bin/bash +set -exo pipefail +# Debian 9 +docker build . -f Dockerfile-build-python.apt --build-arg FROMBASEOS=debian --build-arg FROMBASEOS_BUILD_TAG=9 -t electriccoinco/zcashd-build-debian9 +docker push electriccoinco/zcashd-build-debian9 +docker build . -f Dockerfile-bbworker.apt --build-arg BASEOS=debian --build-arg FROMBASEOS=debian --build-arg FROMBASEOS_BUILD_TAG=9 -t electriccoinco/zcashd-bbworker-debian9 +docker push electriccoinco/zcashd-bbworker-debian9 -docker push electriccoinco/zcashd-build:debian9 -docker push electriccoinco/zcashd-build:debian10 +# Debian 10 +docker build . -f Dockerfile-build.apt --build-arg FROMBASEOS=debian --build-arg FROMBASEOS_BUILD_TAG=10 -t electriccoinco/zcashd-build-debian10 +docker push electriccoinco/zcashd-build-debian10 +docker build . -f Dockerfile-gitian.apt --build-arg FROMBASEOS=debian --build-arg FROMBASEOS_BUILD_TAG=10 -t electriccoinco/zcashd-gitian-debian10 +docker push electriccoinco/zcashd-gitian-debian10 +docker build . -f Dockerfile-bbworker.apt --build-arg BASEOS=debian --build-arg FROMBASEOS=debian --build-arg FROMBASEOS_BUILD_TAG=10 -t electriccoinco/zcashd-bbworker-debian10 +docker push electriccoinco/zcashd-bbworker-debian10 -docker build . -f Dockerfile-gitian.apt --build-arg FROMBASEOS=debian --build-arg FROMBASEOS_BUILD_TAG=9 -t electriccoinco/zcashd-gitian:debian9 -docker build . -f Dockerfile-gitian.apt --build-arg FROMBASEOS=debian --build-arg FROMBASEOS_BUILD_TAG=10 -t electriccoinco/zcashd-gitian:debian10 +# Ubuntu 16.04 +docker build . -f Dockerfile-build.apt --build-arg FROMBASEOS=ubuntu --build-arg FROMBASEOS_BUILD_TAG=16.04 -t electriccoinco/zcashd-build-ubuntu1604 +docker push electriccoinco/zcashd-build-ubuntu1604 +docker build . -f Dockerfile-gitian.apt --build-arg FROMBASEOS=ubuntu --build-arg FROMBASEOS_BUILD_TAG=1604 -t electriccoinco/zcashd-gitian-ubuntu1604 +docker push electriccoinco/zcashd-gitian-ubuntu1604 +docker build . -f Dockerfile-bbworker.apt --build-arg BASEOS=ubuntu --build-arg FROMBASEOS=ubuntu --build-arg FROMBASEOS_BUILD_TAG=1604 -t electriccoinco/zcashd-bbworker-ubuntu1604 +docker push electriccoinco/zcashd-bbworker-ubuntu1604 -docker push electriccoinco/zcashd-gitian:debian9 -docker push electriccoinco/zcashd-gitian:debian10 +# Ubuntu 18.04, 20.04 +docker build . -f Dockerfile-build.apt --build-arg FROMBASEOS=ubuntu --build-arg FROMBASEOS_BUILD_TAG=18.04 -t electriccoinco/zcashd-build-ubuntu1804 +docker build . -f Dockerfile-build.apt --build-arg FROMBASEOS=ubuntu --build-arg FROMBASEOS_BUILD_TAG=20.04 -t electriccoinco/zcashd-build-ubuntu2004 +docker push electriccoinco/zcashd-build-ubuntu1804 +docker push electriccoinco/zcashd-build-ubuntu2004 +docker build . -f Dockerfile-gitian.apt --build-arg FROMBASEOS=ubuntu --build-arg FROMBASEOS_BUILD_TAG=1804 -t electriccoinco/zcashd-gitian-ubuntu1804 +docker build . -f Dockerfile-gitian.apt --build-arg FROMBASEOS=ubuntu --build-arg FROMBASEOS_BUILD_TAG=2004 -t electriccoinco/zcashd-gitian-ubuntu2004 +docker push electriccoinco/zcashd-gitian-ubuntu1804 +docker push electriccoinco/zcashd-gitian-ubuntu2004 +docker build . -f Dockerfile-bbworker.apt --build-arg BASEOS=ubuntu --build-arg FROMBASEOS=ubuntu --build-arg FROMBASEOS_BUILD_TAG=1804 -t electriccoinco/zcashd-bbworker-ubuntu1804 +docker build . -f Dockerfile-bbworker.apt --build-arg BASEOS=ubuntu --build-arg FROMBASEOS=ubuntu --build-arg FROMBASEOS_BUILD_TAG=2004 -t electriccoinco/zcashd-bbworker-ubuntu2004 +docker push electriccoinco/zcashd-bbworker-ubuntu1804 +docker push electriccoinco/zcashd-bbworker-ubuntu2004 -docker build . -f Dockerfile-bbworker.apt --build-arg BASEOS=debian --build-arg FROMBASEOS=debian --build-arg FROMBASEOS_BUILD_TAG=9 -t electriccoinco/zcashd-bbworker:debian9 -docker build . -f Dockerfile-bbworker.apt --build-arg BASEOS=debian --build-arg FROMBASEOS=debian --build-arg FROMBASEOS_BUILD_TAG=10 -t electriccoinco/zcashd-bbworker:debian10 +# Centos8 +docker build . -f Dockerfile-build.centos8 -t electriccoinco/zcashd-build-centos8 +docker build . -f Dockerfile-bbworker.centos8 -t electriccoinco/zcashd-bbworker-centos8 +docker push electriccoinco/zcashd-build-centos8 +docker push electriccoinco/zcashd-bbworker-centos8 -docker push electriccoinco/zcashd-bbworker:debian9 -docker push electriccoinco/zcashd-bbworker:debian10 - -# Ubuntu -docker build . -f Dockerfile-build.apt --build-arg FROMBASEOS=ubuntu --build-arg FROMBASEOS_BUILD_TAG=18.04 -t electriccoinco/zcashd-build:ubuntu18.04 -docker build . -f Dockerfile-build.apt --build-arg FROMBASEOS=ubuntu --build-arg FROMBASEOS_BUILD_TAG=20.04 -t electriccoinco/zcashd-build:ubuntu20.04 - -docker push electriccoinco/zcashd-build:ubuntu18.04 -docker push electriccoinco/zcashd-build:ubuntu20.04 - -docker build . -f Dockerfile-gitian.apt --build-arg FROMBASEOS=ubuntu --build-arg FROMBASEOS_BUILD_TAG=18.04 -t electriccoinco/zcashd-gitian:ubuntu18.04 -docker build . -f Dockerfile-gitian.apt --build-arg FROMBASEOS=ubuntu --build-arg FROMBASEOS_BUILD_TAG=20.04 -t electriccoinco/zcashd-gitian:ubuntu20.04 - -docker push electriccoinco/zcashd-gitian:ubuntu18.04 -docker push electriccoinco/zcashd-gitian:ubuntu20.04 - -docker build . -f Dockerfile-bbworker.apt --build-arg BASEOS=ubuntu --build-arg FROMBASEOS=ubuntu --build-arg FROMBASEOS_BUILD_TAG=18.04 -t electriccoinco/zcashd-bbworker:ubuntu18.04 -docker build . -f Dockerfile-bbworker.apt --build-arg BASEOS=ubuntu --build-arg FROMBASEOS=ubuntu --build-arg FROMBASEOS_BUILD_TAG=20.04 -t electriccoinco/zcashd-bbworker:ubuntu20.04 - -docker push electriccoinco/zcashd-bbworker:ubuntu18.04 -docker push electriccoinco/zcashd-bbworker:ubuntu20.04 \ No newline at end of file +# Arch 20200908 +docker build . -f Dockerfile-build.arch --build-arg ARCHLINUX_TAG=20200908 -t electriccoinco/zcashd-build-arch +docker build . -f Dockerfile-bbworker.arch -t electriccoinco/zcashd-bbworker-arch +docker push electriccoinco/zcashd-build-arch +docker push electriccoinco/zcashd-bbworker-arch \ No newline at end of file