From 6db418c9827d2d205f6c1bde0229ff852b4e4dcc Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Tue, 4 Jul 2017 17:46:17 -0700 Subject: [PATCH 01/10] Ansible playbook for installing Zcash dependencies and Buildbot worker --- contrib/ci-workers/README.md | 4 +++ contrib/ci-workers/unix.yml | 56 +++++++++++++++++++++++++++++ contrib/ci-workers/vars/default.yml | 39 ++++++++++++++++++++ 3 files changed, 99 insertions(+) create mode 100644 contrib/ci-workers/README.md create mode 100644 contrib/ci-workers/unix.yml create mode 100644 contrib/ci-workers/vars/default.yml diff --git a/contrib/ci-workers/README.md b/contrib/ci-workers/README.md new file mode 100644 index 000000000..72b832fcb --- /dev/null +++ b/contrib/ci-workers/README.md @@ -0,0 +1,4 @@ +# Zcash CI workers + +This folder contains the Ansible playbooks for configuring a fresh OS +installation for use as a Buildbot worker in Zcash's CI. diff --git a/contrib/ci-workers/unix.yml b/contrib/ci-workers/unix.yml new file mode 100644 index 000000000..94e122447 --- /dev/null +++ b/contrib/ci-workers/unix.yml @@ -0,0 +1,56 @@ +--- +- name: Configure a Buildbot worker for Zcash CI + hosts: zcash-ci-worker-unix + become: true + gather_facts: False + + pre_tasks: + - set_fact: + python_raw: "test -e /usr/bin/python || test -e /usr/bin/python2 || test -e /usr/bin/python2.7 || test -e /usr/local/bin/python2.7 || (test -e /usr/bin/apt && apt -qqy update && apt install -qqy python) || (test -e /usr/bin/dnf && dnf install -qqy python2) || (test -e /usr/sbin/pkg && pkg install -qqy python2)" + - name: Install Python 2 for Ansible and Buildbot [bash] + raw: bash -c "'{{ python_raw }}'" + ignore_errors: true + register: output + changed_when: + - output.stdout != "" + - output.stdout != "\r\n" + - name: Install Python 2 for Ansible and Buildbot [tcsh] + raw: tcsh -c "{{ python_raw }}" + ignore_errors: true + register: output + changed_when: + - output.stdout != "" + - output.stdout != "\r\n" + - name: Gathering Facts + setup: + + tasks: + - name: Get default dependencies + include_vars: "vars/default.yml" + + - name: Get dependencies for distribution + include_vars: "{{ item }}" + with_first_found: + - files: + - "vars/{{ ansible_distribution }}-{{ ansible_distribution_version }}.yml" + - "vars/{{ ansible_distribution }}-{{ ansible_distribution_major_version | int }}.yml" + - "vars/{{ ansible_distribution }}.yml" + - "vars/{{ ansible_os_family }}.yml" + skip: true + + - name: Collate dependencies + set_fact: + package_deps: "{{ buildbot_deps + fetch_deps + conf_deps + build_deps + link_deps + dist_deps }}" + python_modules: "{{ buildbot_modules + rpc_test_modules }}" + + - name: Install required packages + package: + name: "{{ item }}" + state: present + with_items: "{{ package_deps }}" + + - name: Install required Python modules + pip: + name: "{{ item }}" + state: latest + with_items: "{{ python_modules }}" diff --git a/contrib/ci-workers/vars/default.yml b/contrib/ci-workers/vars/default.yml new file mode 100644 index 000000000..9aa79da5e --- /dev/null +++ b/contrib/ci-workers/vars/default.yml @@ -0,0 +1,39 @@ +--- +# These variables can be overridden in distribution files. + +# Dependencies required to install Buildbot +buildbot_deps: + - python-pip # So we can install Python modules + +# Dependencies required to download files +fetch_deps: + - git + - wget # For zcutil/fetch-params.sh + +# Dependencies required to configure Zcash +conf_deps: + - autoconf + - automake + - m4 + +# Dependencies required to compile Zcash +build_deps: + - g++ + - gcc + - make + +# Dependencies required to link Zcash +link_deps: + - libtool + +# Additional distribution-specific dependencies +dist_deps: [] + +# Python modules required for a Zcash Buildbot worker +buildbot_modules: + - pip # Needs to be updated first so Buildbot installs + - buildbot-worker + +# Python modules required to run the Zcash RPC test suite +rpc_test_modules: + - pyzmq From 6f4a1721f24bd0d5511895a9a89329ab83456640 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Tue, 4 Jul 2017 17:47:11 -0700 Subject: [PATCH 02/10] Variable overrides for Debian, Ubuntu and Fedora --- contrib/ci-workers/vars/Debian.yml | 6 ++++++ contrib/ci-workers/vars/Fedora.yml | 10 ++++++++++ contrib/ci-workers/vars/Ubuntu.yml | 5 +++++ 3 files changed, 21 insertions(+) create mode 100644 contrib/ci-workers/vars/Debian.yml create mode 100644 contrib/ci-workers/vars/Fedora.yml create mode 100644 contrib/ci-workers/vars/Ubuntu.yml diff --git a/contrib/ci-workers/vars/Debian.yml b/contrib/ci-workers/vars/Debian.yml new file mode 100644 index 000000000..992224721 --- /dev/null +++ b/contrib/ci-workers/vars/Debian.yml @@ -0,0 +1,6 @@ +--- +build_deps: + - build-essential # Depends on g++, libc6-dev, make +dist_deps: + - pkg-config # Required until b556beda264308e040f8d88aca4f2f386a0183d9 is pulled in + - python-dev diff --git a/contrib/ci-workers/vars/Fedora.yml b/contrib/ci-workers/vars/Fedora.yml new file mode 100644 index 000000000..1c6b0e0f3 --- /dev/null +++ b/contrib/ci-workers/vars/Fedora.yml @@ -0,0 +1,10 @@ +--- +build_deps: + - gcc + - gcc-c++ + - make + - patch +dist_deps: + - pkgconfig # Required until b556beda264308e040f8d88aca4f2f386a0183d9 is pulled in + - python-devel + - redhat-rpm-config diff --git a/contrib/ci-workers/vars/Ubuntu.yml b/contrib/ci-workers/vars/Ubuntu.yml new file mode 100644 index 000000000..4acca499b --- /dev/null +++ b/contrib/ci-workers/vars/Ubuntu.yml @@ -0,0 +1,5 @@ +--- +build_deps: + - build-essential # Depends on g++, libc6-dev, make +dist_deps: + - pkg-config # Required until b556beda264308e040f8d88aca4f2f386a0183d9 is pulled in From c73014a2e3b0c45b71343a5024dfe1ee532fef51 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Tue, 4 Jul 2017 17:47:34 -0700 Subject: [PATCH 03/10] Variable overrides for FreeBSD These are insufficient to enable FreeBSD to build. --- contrib/ci-workers/vars/FreeBSD.yml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 contrib/ci-workers/vars/FreeBSD.yml diff --git a/contrib/ci-workers/vars/FreeBSD.yml b/contrib/ci-workers/vars/FreeBSD.yml new file mode 100644 index 000000000..65909d71d --- /dev/null +++ b/contrib/ci-workers/vars/FreeBSD.yml @@ -0,0 +1,9 @@ +--- +buildbot_deps: + - py27-pip +build_deps: + - gcc + - gmake +dist_deps: + - bash + - pkgconf # Required until b556beda264308e040f8d88aca4f2f386a0183d9 is pulled in From 046c3d4f242f107a918f2ef821b68598aa199504 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Mon, 10 Jul 2017 10:30:23 -0500 Subject: [PATCH 04/10] Simplify Python installation, inform user if they need to manually configure --- contrib/ci-workers/unix.yml | 41 ++++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/contrib/ci-workers/unix.yml b/contrib/ci-workers/unix.yml index 94e122447..6fd56a0a8 100644 --- a/contrib/ci-workers/unix.yml +++ b/contrib/ci-workers/unix.yml @@ -4,30 +4,43 @@ become: true gather_facts: False + vars_files: + - vars/default.yml + pre_tasks: - - set_fact: - python_raw: "test -e /usr/bin/python || test -e /usr/bin/python2 || test -e /usr/bin/python2.7 || test -e /usr/local/bin/python2.7 || (test -e /usr/bin/apt && apt -qqy update && apt install -qqy python) || (test -e /usr/bin/dnf && dnf install -qqy python2) || (test -e /usr/sbin/pkg && pkg install -qqy python2)" - - name: Install Python 2 for Ansible and Buildbot [bash] - raw: bash -c "'{{ python_raw }}'" - ignore_errors: true + - name: Install Python 2 for Ansible and Buildbot + raw: test -e /usr/bin/python || test -e /usr/bin/python2 || test -e /usr/bin/python2.7 || test -e /usr/local/bin/python2.7 || (test -e /usr/bin/apt && apt -qqy update && apt install -qqy python) || (test -e /usr/bin/dnf && dnf install -qqy python2) || (test -e /usr/sbin/pkg && pkg install -qqy python2) register: output changed_when: - output.stdout != "" - output.stdout != "\r\n" - - name: Install Python 2 for Ansible and Buildbot [tcsh] - raw: tcsh -c "{{ python_raw }}" + + - name: Check if Python is in the configured location + raw: test -e {{ ansible_python_interpreter }} ignore_errors: true - register: output - changed_when: - - output.stdout != "" - - output.stdout != "\r\n" + register: python_check + when: ansible_python_interpreter is defined + + - name: Fail if configured Python is unavailable + fail: + msg: Python is not accessible at {{ ansible_python_interpreter }} on this host! Please set the inventory variable 'ansible_python_interpreter' to the location of the Python 2 binary. + when: ansible_python_interpreter is defined and python_check.rc == 1 + + - name: Check if Python is in the default location + raw: test -e /usr/bin/python + ignore_errors: true + register: python_check + when: ansible_python_interpreter is undefined + + - name: Fail if default Python is unavailable + fail: + msg: Python is not accessible at /usr/bin/python on this host! Please set the inventory variable 'ansible_python_interpreter' to the location of the Python 2 binary. + when: ansible_python_interpreter is undefined and python_check.rc == 1 + - name: Gathering Facts setup: tasks: - - name: Get default dependencies - include_vars: "vars/default.yml" - - name: Get dependencies for distribution include_vars: "{{ item }}" with_first_found: From 6ff5a13497a1276fc03518bbdaa46a0bcf43a508 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Mon, 10 Jul 2017 11:08:36 -0500 Subject: [PATCH 05/10] Add Buildbot worker setup to Ansible playbook Currently assumes the host uses systemd --- .../templates/buildbot-worker.service.j2 | 17 +++++ contrib/ci-workers/templates/host.j2 | 4 ++ contrib/ci-workers/unix.yml | 68 +++++++++++++++++++ contrib/ci-workers/vars/buildbot.yml | 4 ++ 4 files changed, 93 insertions(+) create mode 100644 contrib/ci-workers/templates/buildbot-worker.service.j2 create mode 100644 contrib/ci-workers/templates/host.j2 create mode 100644 contrib/ci-workers/vars/buildbot.yml diff --git a/contrib/ci-workers/templates/buildbot-worker.service.j2 b/contrib/ci-workers/templates/buildbot-worker.service.j2 new file mode 100644 index 000000000..1fee26aa3 --- /dev/null +++ b/contrib/ci-workers/templates/buildbot-worker.service.j2 @@ -0,0 +1,17 @@ +[Unit] +Description=Buildbot worker +Wants=network.target +After=network.target + +[Service] +Type=forking +PIDFile=/home/{{ buildbot_worker_user }}/{{ buildbot_worker_name }}/twistd.pid +WorkingDirectory=/home/{{ buildbot_worker_user }} +ExecStart=/usr/local/bin/buildbot-worker start {{ buildbot_worker_name }} +ExecReload=/usr/local/bin/buildbot-worker restart {{ buildbot_worker_name }} +ExecStop=/usr/local/bin/buildbot-worker stop {{ buildbot_worker_name }} +Restart=always +User={{ buildbot_worker_user }} + +[Install] +WantedBy=multi-user.target diff --git a/contrib/ci-workers/templates/host.j2 b/contrib/ci-workers/templates/host.j2 new file mode 100644 index 000000000..708baf67b --- /dev/null +++ b/contrib/ci-workers/templates/host.j2 @@ -0,0 +1,4 @@ +OS: {{ ansible_distribution }} {{ ansible_distribution_version }} +Memory: {{ ansible_memtotal_mb }} MB +CPU: {{ ansible_processor[1] }} +{{ buildbot_worker_version.stdout }} diff --git a/contrib/ci-workers/unix.yml b/contrib/ci-workers/unix.yml index 6fd56a0a8..ecc826480 100644 --- a/contrib/ci-workers/unix.yml +++ b/contrib/ci-workers/unix.yml @@ -6,6 +6,17 @@ vars_files: - vars/default.yml + - vars/buildbot.yml + + vars_prompt: + - name: "buildbot_worker_admin" + prompt: "Admin details" + default: "Zcash " + - name: "buildbot_worker_name" + prompt: "Buildbot worker name (provided by ZECC)" + private: no + - name: "buildbot_worker_password" + prompt: "Buildbot worker password (provided by ZECC)" pre_tasks: - name: Install Python 2 for Ansible and Buildbot @@ -67,3 +78,60 @@ name: "{{ item }}" state: latest with_items: "{{ python_modules }}" + notify: restart buildbot-worker + + - name: Get installed Buildbot version + command: buildbot-worker --version + register: buildbot_worker_version + + - name: Set up the Buildbot worker user + user: + name: "{{ buildbot_worker_user }}" + comment: Buildbot worker + shell: /bin/bash + state: present + + - name: Create Buildbot worker + command: > + buildbot-worker create-worker ~/{{ buildbot_worker_name }} + {{ buildbot_master_host }}:{{ buildbot_master_port }} + {{ buildbot_worker_name|quote }} {{ buildbot_worker_password|quote }} + args: + creates: "~/{{ buildbot_worker_name }}/buildbot.tac" + become_user: "{{ buildbot_worker_user }}" + + - name: Set admin details for Buildbot worker + copy: + content: "{{ buildbot_worker_admin }}" + dest: "~/{{ buildbot_worker_name }}/info/admin" + become_user: "{{ buildbot_worker_user }}" + + - name: Set host details for Buildbot worker + template: + src: templates/host.j2 + dest: "~/{{ buildbot_worker_name }}/info/host" + become_user: "{{ buildbot_worker_user }}" + + - name: Copy Buildbot worker systemd service unit + template: + src: templates/buildbot-worker.service.j2 + dest: "/etc/systemd/system/buildbot-worker.service" + owner: root + group: root + mode: "0644" + notify: reload systemd + + - name: Start Buildbot worker. + service: + name: buildbot-worker + state: started + enabled: yes + + handlers: + - name: restart buildbot-worker + service: + name: buildbot-worker + state: restarted + + - name: reload systemd + command: /bin/systemctl daemon-reload diff --git a/contrib/ci-workers/vars/buildbot.yml b/contrib/ci-workers/vars/buildbot.yml new file mode 100644 index 000000000..3d21f400f --- /dev/null +++ b/contrib/ci-workers/vars/buildbot.yml @@ -0,0 +1,4 @@ +--- +buildbot_worker_user: zcbbworker +buildbot_master_host: dev-ci.z.cash +buildbot_master_port: 9899 From 64363d135108117528cce62444a1f3187567d2fc Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Mon, 10 Jul 2017 16:34:48 -0500 Subject: [PATCH 06/10] Add steps for setting up a latent worker on Amazon EC2 --- contrib/ci-workers/README.md | 44 ++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/contrib/ci-workers/README.md b/contrib/ci-workers/README.md index 72b832fcb..cc3c85ac2 100644 --- a/contrib/ci-workers/README.md +++ b/contrib/ci-workers/README.md @@ -2,3 +2,47 @@ This folder contains the Ansible playbooks for configuring a fresh OS installation for use as a Buildbot worker in Zcash's CI. + +# Setting up a latent worker on Amazon EC2 + +- Add a regular (non-latent) worker to the master.cfg for dev-ci.z.cash, and + deploy the changes. + - This enables the Ansible playbook to run to completion, ending in the worker + connecting to the master. + +- Start a basic EC2 instance using the template AMI for the target OS. + - Choose the smallest instance size, it won't be used for building Zcash. + +- Figure out which user to log into the instance with. + - E.g. for the Ubuntu template, use "ubuntu" instead of "root" + - If you get an Ansible error later with a message like "Failed to connect to + the host via ssh: Received message too long 1349281121\r\n", that means the + instance is sending a text string in response to the SSH connection, and the + Ansible protocol is balking. Try manually logging in with the same + credentials to diagnose. + +- Create `inventory/hosts` containing the following: + + [zcash-ci-worker-unix] + some-name ansible_host= ansible_ssh_user= + +- Run `ansible-playbook -i inventory/hosts unix.yml`, passing in the worker's + Buildbot name and password. + - After a successful run, the worker should be connected to dev-ci.z.cash and + visible in its worker list. + +- Create an AMI from the instance. This is the worker AMI to put into the + master.cfg for dev-ci.z.cash. + - 16 GB of storage should be sufficient. + +- SSH into the instance, and edit the worker config to connect to ci.z.cash. + +- Create an AMI from the instance. This is the worker AMI to put into the + master.cfg for ci.z.cash. + - 16 GB of storage should be sufficient. + +- Delete the instance (it is no longer needed). + +- Edit the master.cfg to turn the new worker into a latent (using the new AMI + IDs), add it to the appropriate worker groups, set up new builders etc. + - Deploy this via the normal PR review process. From 3ba809b6d71359938c7dc67995f7ed05cc3132c4 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Thu, 13 Jul 2017 17:40:19 -0500 Subject: [PATCH 07/10] Add pyblake2 to required Python modules See #2533 for details. --- contrib/ci-workers/vars/default.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/ci-workers/vars/default.yml b/contrib/ci-workers/vars/default.yml index 9aa79da5e..9afb8c55f 100644 --- a/contrib/ci-workers/vars/default.yml +++ b/contrib/ci-workers/vars/default.yml @@ -36,4 +36,5 @@ buildbot_modules: # Python modules required to run the Zcash RPC test suite rpc_test_modules: + - pyblake2 - pyzmq From 8247fd954c57de22ce0a89bc73a4c7ac9a616ac4 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 14 Jul 2017 10:02:14 -0500 Subject: [PATCH 08/10] Remove Buildbot version from host file Buildbot master already obtains and publishes this in the UI. --- contrib/ci-workers/templates/host.j2 | 1 - contrib/ci-workers/unix.yml | 4 ---- 2 files changed, 5 deletions(-) diff --git a/contrib/ci-workers/templates/host.j2 b/contrib/ci-workers/templates/host.j2 index 708baf67b..3a5abb0c2 100644 --- a/contrib/ci-workers/templates/host.j2 +++ b/contrib/ci-workers/templates/host.j2 @@ -1,4 +1,3 @@ OS: {{ ansible_distribution }} {{ ansible_distribution_version }} Memory: {{ ansible_memtotal_mb }} MB CPU: {{ ansible_processor[1] }} -{{ buildbot_worker_version.stdout }} diff --git a/contrib/ci-workers/unix.yml b/contrib/ci-workers/unix.yml index ecc826480..a33fac21e 100644 --- a/contrib/ci-workers/unix.yml +++ b/contrib/ci-workers/unix.yml @@ -80,10 +80,6 @@ with_items: "{{ python_modules }}" notify: restart buildbot-worker - - name: Get installed Buildbot version - command: buildbot-worker --version - register: buildbot_worker_version - - name: Set up the Buildbot worker user user: name: "{{ buildbot_worker_user }}" From e5f8e6185a50f9e164c9d929f2590ae884beb752 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 14 Jul 2017 12:24:01 -0500 Subject: [PATCH 09/10] Add a separate Buildbot host info template for EC2 Latent workers are not usually created on the instance type that will be used, so memory and CPU info collected at AMI creation will likely be inaccurate. --- contrib/ci-workers/README.md | 4 ++-- contrib/ci-workers/templates/host.ec2.j2 | 1 + contrib/ci-workers/unix.yml | 2 +- contrib/ci-workers/vars/buildbot.yml | 1 + 4 files changed, 5 insertions(+), 3 deletions(-) create mode 100644 contrib/ci-workers/templates/host.ec2.j2 diff --git a/contrib/ci-workers/README.md b/contrib/ci-workers/README.md index cc3c85ac2..067c0cb5e 100644 --- a/contrib/ci-workers/README.md +++ b/contrib/ci-workers/README.md @@ -26,8 +26,8 @@ installation for use as a Buildbot worker in Zcash's CI. [zcash-ci-worker-unix] some-name ansible_host= ansible_ssh_user= -- Run `ansible-playbook -i inventory/hosts unix.yml`, passing in the worker's - Buildbot name and password. +- Run `ansible-playbook -e buildbot_worker_host_template=templates/host.ec2.j2 -i inventory/hosts unix.yml`, + passing in the worker's Buildbot name and password. - After a successful run, the worker should be connected to dev-ci.z.cash and visible in its worker list. diff --git a/contrib/ci-workers/templates/host.ec2.j2 b/contrib/ci-workers/templates/host.ec2.j2 new file mode 100644 index 000000000..dee692e02 --- /dev/null +++ b/contrib/ci-workers/templates/host.ec2.j2 @@ -0,0 +1 @@ +OS: {{ ansible_distribution }} {{ ansible_distribution_version }} diff --git a/contrib/ci-workers/unix.yml b/contrib/ci-workers/unix.yml index a33fac21e..42bcaafc2 100644 --- a/contrib/ci-workers/unix.yml +++ b/contrib/ci-workers/unix.yml @@ -104,7 +104,7 @@ - name: Set host details for Buildbot worker template: - src: templates/host.j2 + src: "{{ buildbot_worker_host_template }}" dest: "~/{{ buildbot_worker_name }}/info/host" become_user: "{{ buildbot_worker_user }}" diff --git a/contrib/ci-workers/vars/buildbot.yml b/contrib/ci-workers/vars/buildbot.yml index 3d21f400f..38e3fd25a 100644 --- a/contrib/ci-workers/vars/buildbot.yml +++ b/contrib/ci-workers/vars/buildbot.yml @@ -2,3 +2,4 @@ buildbot_worker_user: zcbbworker buildbot_master_host: dev-ci.z.cash buildbot_master_port: 9899 +buildbot_worker_host_template: templates/host.j2 From e0c696a1e7e09221ee20cb451ba13c6d5384d0b6 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 14 Jul 2017 12:25:15 -0500 Subject: [PATCH 10/10] Add pyflakes to required Python modules See #2494 for details. --- contrib/ci-workers/vars/default.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/ci-workers/vars/default.yml b/contrib/ci-workers/vars/default.yml index 9afb8c55f..13a04e7b4 100644 --- a/contrib/ci-workers/vars/default.yml +++ b/contrib/ci-workers/vars/default.yml @@ -33,6 +33,7 @@ dist_deps: [] buildbot_modules: - pip # Needs to be updated first so Buildbot installs - buildbot-worker + - pyflakes # Python modules required to run the Zcash RPC test suite rpc_test_modules: