Auto merge of #2509 - str4d:ci-workers, r=str4d

Ansible playbook for configuring CI workers

The playbook in this PR can be used for configuring common Unix variants to run a Buildbot worker for the Zcash CI, starting from a fresh install (or a VM template provided by a hosting service).

contrib/ci-workers/README.md

@@ -0,0 +1,48 @@
+# Zcash CI workers
+
+This folder contains the Ansible playbooks for configuring a fresh OS
+installation for use as a Buildbot worker in Zcash's CI.
+
+# Setting up a latent worker on Amazon EC2
+
+- Add a regular (non-latent) worker to the master.cfg for dev-ci.z.cash, and
+  deploy the changes.
+  - This enables the Ansible playbook to run to completion, ending in the worker
+    connecting to the master.
+
+- Start a basic EC2 instance using the template AMI for the target OS.
+  - Choose the smallest instance size, it won't be used for building Zcash.
+
+- Figure out which user to log into the instance with.
+  - E.g. for the Ubuntu template, use "ubuntu" instead of "root"
+  - If you get an Ansible error later with a message like "Failed to connect to
+    the host via ssh: Received message too long 1349281121\r\n", that means the
+    instance is sending a text string in response to the SSH connection, and the
+    Ansible protocol is balking. Try manually logging in with the same
+    credentials to diagnose.
+
+- Create `inventory/hosts` containing the following:
+
+    [zcash-ci-worker-unix]
+    some-name ansible_host=<INSTANCE_IP> ansible_ssh_user=<USERNAME>
+
+- Run `ansible-playbook -e buildbot_worker_host_template=templates/host.ec2.j2 -i inventory/hosts unix.yml`,
+  passing in the worker's Buildbot name and password.
+  - After a successful run, the worker should be connected to dev-ci.z.cash and
+    visible in its worker list.
+
+- Create an AMI from the instance. This is the worker AMI to put into the
+  master.cfg for dev-ci.z.cash.
+  - 16 GB of storage should be sufficient.
+
+- SSH into the instance, and edit the worker config to connect to ci.z.cash.
+
+- Create an AMI from the instance. This is the worker AMI to put into the
+  master.cfg for ci.z.cash.
+  - 16 GB of storage should be sufficient.
+
+- Delete the instance (it is no longer needed).
+
+- Edit the master.cfg to turn the new worker into a latent (using the new AMI
+  IDs), add it to the appropriate worker groups, set up new builders etc.
+  - Deploy this via the normal PR review process.

contrib/ci-workers/templates/buildbot-worker.service.j2

@@ -0,0 +1,17 @@
+[Unit]
+Description=Buildbot worker
+Wants=network.target
+After=network.target
+
+[Service]
+Type=forking
+PIDFile=/home/{{ buildbot_worker_user }}/{{ buildbot_worker_name }}/twistd.pid
+WorkingDirectory=/home/{{ buildbot_worker_user }}
+ExecStart=/usr/local/bin/buildbot-worker start {{ buildbot_worker_name }}
+ExecReload=/usr/local/bin/buildbot-worker restart {{ buildbot_worker_name }}
+ExecStop=/usr/local/bin/buildbot-worker stop {{ buildbot_worker_name }}
+Restart=always
+User={{ buildbot_worker_user }}
+
+[Install]
+WantedBy=multi-user.target

contrib/ci-workers/templates/host.ec2.j2

@@ -0,0 +1 @@
+OS: {{ ansible_distribution }} {{ ansible_distribution_version }}

contrib/ci-workers/templates/host.j2

@@ -0,0 +1,3 @@
+OS: {{ ansible_distribution }} {{ ansible_distribution_version }}
+Memory: {{ ansible_memtotal_mb }} MB
+CPU: {{ ansible_processor[1] }}

contrib/ci-workers/unix.yml

@@ -0,0 +1,133 @@
+---
+- name: Configure a Buildbot worker for Zcash CI
+  hosts: zcash-ci-worker-unix
+  become: true
+  gather_facts: False
+
+  vars_files:
+    - vars/default.yml
+    - vars/buildbot.yml
+
+  vars_prompt:
+    - name: "buildbot_worker_admin"
+      prompt: "Admin details"
+      default: "Zcash <sysadmin@z.cash>"
+    - name: "buildbot_worker_name"
+      prompt: "Buildbot worker name (provided by ZECC)"
+      private: no
+    - name: "buildbot_worker_password"
+      prompt: "Buildbot worker password (provided by ZECC)"
+
+  pre_tasks:
+    - name: Install Python 2 for Ansible and Buildbot
+      raw: test -e /usr/bin/python || test -e /usr/bin/python2 || test -e /usr/bin/python2.7 || test -e /usr/local/bin/python2.7 || (test -e /usr/bin/apt && apt -qqy update && apt install -qqy python) || (test -e /usr/bin/dnf && dnf install -qqy python2) || (test -e /usr/sbin/pkg && pkg install -qqy python2)
+      register: output
+      changed_when:
+        - output.stdout != ""
+        - output.stdout != "\r\n"
+
+    - name: Check if Python is in the configured location
+      raw: test -e {{ ansible_python_interpreter }}
+      ignore_errors: true
+      register: python_check
+      when: ansible_python_interpreter is defined
+
+    - name: Fail if configured Python is unavailable
+      fail:
+        msg: Python is not accessible at {{ ansible_python_interpreter }} on this host! Please set the inventory variable 'ansible_python_interpreter' to the location of the Python 2 binary.
+      when: ansible_python_interpreter is defined and python_check.rc == 1
+
+    - name: Check if Python is in the default location
+      raw: test -e /usr/bin/python
+      ignore_errors: true
+      register: python_check
+      when: ansible_python_interpreter is undefined
+
+    - name: Fail if default Python is unavailable
+      fail:
+        msg: Python is not accessible at /usr/bin/python on this host! Please set the inventory variable 'ansible_python_interpreter' to the location of the Python 2 binary.
+      when: ansible_python_interpreter is undefined and python_check.rc == 1
+
+    - name: Gathering Facts
+      setup:
+
+  tasks:
+    - name: Get dependencies for distribution
+      include_vars: "{{ item }}"
+      with_first_found:
+        - files:
+            - "vars/{{ ansible_distribution }}-{{ ansible_distribution_version }}.yml"
+            - "vars/{{ ansible_distribution }}-{{ ansible_distribution_major_version | int }}.yml"
+            - "vars/{{ ansible_distribution }}.yml"
+            - "vars/{{ ansible_os_family }}.yml"
+          skip: true
+
+    - name: Collate dependencies
+      set_fact:
+        package_deps: "{{ buildbot_deps + fetch_deps + conf_deps + build_deps + link_deps + dist_deps }}"
+        python_modules: "{{ buildbot_modules + rpc_test_modules }}"
+
+    - name: Install required packages
+      package:
+        name: "{{ item }}"
+        state: present
+      with_items: "{{ package_deps }}"
+
+    - name: Install required Python modules
+      pip:
+        name: "{{ item }}"
+        state: latest
+      with_items: "{{ python_modules }}"
+      notify: restart buildbot-worker
+
+    - name: Set up the Buildbot worker user
+      user:
+        name: "{{ buildbot_worker_user }}"
+        comment: Buildbot worker
+        shell: /bin/bash
+        state: present
+
+    - name: Create Buildbot worker
+      command: >
+        buildbot-worker create-worker ~/{{ buildbot_worker_name }}
+        {{ buildbot_master_host }}:{{ buildbot_master_port }}
+        {{ buildbot_worker_name|quote }} {{ buildbot_worker_password|quote }}
+      args:
+        creates: "~/{{ buildbot_worker_name }}/buildbot.tac"
+      become_user: "{{ buildbot_worker_user }}"
+
+    - name: Set admin details for Buildbot worker
+      copy:
+        content: "{{ buildbot_worker_admin }}"
+        dest: "~/{{ buildbot_worker_name }}/info/admin"
+      become_user: "{{ buildbot_worker_user }}"
+
+    - name: Set host details for Buildbot worker
+      template:
+        src: "{{ buildbot_worker_host_template }}"
+        dest: "~/{{ buildbot_worker_name }}/info/host"
+      become_user: "{{ buildbot_worker_user }}"
+
+    - name: Copy Buildbot worker systemd service unit
+      template:
+        src: templates/buildbot-worker.service.j2
+        dest: "/etc/systemd/system/buildbot-worker.service"
+        owner: root
+        group: root
+        mode: "0644"
+      notify: reload systemd
+
+    - name: Start Buildbot worker.
+      service:
+        name: buildbot-worker
+        state: started
+        enabled: yes
+
+  handlers:
+    - name: restart buildbot-worker
+      service:
+        name: buildbot-worker
+        state: restarted
+
+    - name: reload systemd
+      command: /bin/systemctl daemon-reload

contrib/ci-workers/vars/Debian.yml

@@ -0,0 +1,6 @@
+---
+build_deps:
+  - build-essential # Depends on g++, libc6-dev, make
+dist_deps:
+  - pkg-config # Required until b556beda264308e040f8d88aca4f2f386a0183d9 is pulled in
+  - python-dev

contrib/ci-workers/vars/Fedora.yml

@@ -0,0 +1,10 @@
+---
+build_deps:
+  - gcc
+  - gcc-c++
+  - make
+  - patch
+dist_deps:
+  - pkgconfig # Required until b556beda264308e040f8d88aca4f2f386a0183d9 is pulled in
+  - python-devel
+  - redhat-rpm-config

contrib/ci-workers/vars/FreeBSD.yml

@@ -0,0 +1,9 @@
+---
+buildbot_deps:
+  - py27-pip
+build_deps:
+  - gcc
+  - gmake
+dist_deps:
+  - bash
+  - pkgconf # Required until b556beda264308e040f8d88aca4f2f386a0183d9 is pulled in

contrib/ci-workers/vars/Ubuntu.yml

@@ -0,0 +1,5 @@
+---
+build_deps:
+  - build-essential # Depends on g++, libc6-dev, make
+dist_deps:
+  - pkg-config # Required until b556beda264308e040f8d88aca4f2f386a0183d9 is pulled in

contrib/ci-workers/vars/buildbot.yml

@@ -0,0 +1,5 @@
+---
+buildbot_worker_user: zcbbworker
+buildbot_master_host: dev-ci.z.cash
+buildbot_master_port: 9899
+buildbot_worker_host_template: templates/host.j2

contrib/ci-workers/vars/default.yml

@@ -0,0 +1,41 @@
+---
+# These variables can be overridden in distribution files.
+
+# Dependencies required to install Buildbot
+buildbot_deps:
+  - python-pip # So we can install Python modules
+
+# Dependencies required to download files
+fetch_deps:
+  - git
+  - wget # For zcutil/fetch-params.sh
+
+# Dependencies required to configure Zcash
+conf_deps:
+  - autoconf
+  - automake
+  - m4
+
+# Dependencies required to compile Zcash
+build_deps:
+  - g++
+  - gcc
+  - make
+
+# Dependencies required to link Zcash
+link_deps:
+  - libtool
+
+# Additional distribution-specific dependencies
+dist_deps: []
+
+# Python modules required for a Zcash Buildbot worker
+buildbot_modules:
+  - pip # Needs to be updated first so Buildbot installs
+  - buildbot-worker
+  - pyflakes
+
+# Python modules required to run the Zcash RPC test suite
+rpc_test_modules:
+  - pyblake2
+  - pyzmq