Auto merge of #2509 - str4d:ci-workers, r=str4d
Ansible playbook for configuring CI workers The playbook in this PR can be used for configuring common Unix variants to run a Buildbot worker for the Zcash CI, starting from a fresh install (or a VM template provided by a hosting service).
This commit is contained in:
commit
d3a976095f
|
@ -0,0 +1,48 @@
|
||||||
|
# Zcash CI workers
|
||||||
|
|
||||||
|
This folder contains the Ansible playbooks for configuring a fresh OS
|
||||||
|
installation for use as a Buildbot worker in Zcash's CI.
|
||||||
|
|
||||||
|
# Setting up a latent worker on Amazon EC2
|
||||||
|
|
||||||
|
- Add a regular (non-latent) worker to the master.cfg for dev-ci.z.cash, and
|
||||||
|
deploy the changes.
|
||||||
|
- This enables the Ansible playbook to run to completion, ending in the worker
|
||||||
|
connecting to the master.
|
||||||
|
|
||||||
|
- Start a basic EC2 instance using the template AMI for the target OS.
|
||||||
|
- Choose the smallest instance size, it won't be used for building Zcash.
|
||||||
|
|
||||||
|
- Figure out which user to log into the instance with.
|
||||||
|
- E.g. for the Ubuntu template, use "ubuntu" instead of "root"
|
||||||
|
- If you get an Ansible error later with a message like "Failed to connect to
|
||||||
|
the host via ssh: Received message too long 1349281121\r\n", that means the
|
||||||
|
instance is sending a text string in response to the SSH connection, and the
|
||||||
|
Ansible protocol is balking. Try manually logging in with the same
|
||||||
|
credentials to diagnose.
|
||||||
|
|
||||||
|
- Create `inventory/hosts` containing the following:
|
||||||
|
|
||||||
|
[zcash-ci-worker-unix]
|
||||||
|
some-name ansible_host=<INSTANCE_IP> ansible_ssh_user=<USERNAME>
|
||||||
|
|
||||||
|
- Run `ansible-playbook -e buildbot_worker_host_template=templates/host.ec2.j2 -i inventory/hosts unix.yml`,
|
||||||
|
passing in the worker's Buildbot name and password.
|
||||||
|
- After a successful run, the worker should be connected to dev-ci.z.cash and
|
||||||
|
visible in its worker list.
|
||||||
|
|
||||||
|
- Create an AMI from the instance. This is the worker AMI to put into the
|
||||||
|
master.cfg for dev-ci.z.cash.
|
||||||
|
- 16 GB of storage should be sufficient.
|
||||||
|
|
||||||
|
- SSH into the instance, and edit the worker config to connect to ci.z.cash.
|
||||||
|
|
||||||
|
- Create an AMI from the instance. This is the worker AMI to put into the
|
||||||
|
master.cfg for ci.z.cash.
|
||||||
|
- 16 GB of storage should be sufficient.
|
||||||
|
|
||||||
|
- Delete the instance (it is no longer needed).
|
||||||
|
|
||||||
|
- Edit the master.cfg to turn the new worker into a latent (using the new AMI
|
||||||
|
IDs), add it to the appropriate worker groups, set up new builders etc.
|
||||||
|
- Deploy this via the normal PR review process.
|
|
@ -0,0 +1,17 @@
|
||||||
|
[Unit]
|
||||||
|
Description=Buildbot worker
|
||||||
|
Wants=network.target
|
||||||
|
After=network.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=forking
|
||||||
|
PIDFile=/home/{{ buildbot_worker_user }}/{{ buildbot_worker_name }}/twistd.pid
|
||||||
|
WorkingDirectory=/home/{{ buildbot_worker_user }}
|
||||||
|
ExecStart=/usr/local/bin/buildbot-worker start {{ buildbot_worker_name }}
|
||||||
|
ExecReload=/usr/local/bin/buildbot-worker restart {{ buildbot_worker_name }}
|
||||||
|
ExecStop=/usr/local/bin/buildbot-worker stop {{ buildbot_worker_name }}
|
||||||
|
Restart=always
|
||||||
|
User={{ buildbot_worker_user }}
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
|
@ -0,0 +1 @@
|
||||||
|
OS: {{ ansible_distribution }} {{ ansible_distribution_version }}
|
|
@ -0,0 +1,3 @@
|
||||||
|
OS: {{ ansible_distribution }} {{ ansible_distribution_version }}
|
||||||
|
Memory: {{ ansible_memtotal_mb }} MB
|
||||||
|
CPU: {{ ansible_processor[1] }}
|
|
@ -0,0 +1,133 @@
|
||||||
|
---
|
||||||
|
- name: Configure a Buildbot worker for Zcash CI
|
||||||
|
hosts: zcash-ci-worker-unix
|
||||||
|
become: true
|
||||||
|
gather_facts: False
|
||||||
|
|
||||||
|
vars_files:
|
||||||
|
- vars/default.yml
|
||||||
|
- vars/buildbot.yml
|
||||||
|
|
||||||
|
vars_prompt:
|
||||||
|
- name: "buildbot_worker_admin"
|
||||||
|
prompt: "Admin details"
|
||||||
|
default: "Zcash <sysadmin@z.cash>"
|
||||||
|
- name: "buildbot_worker_name"
|
||||||
|
prompt: "Buildbot worker name (provided by ZECC)"
|
||||||
|
private: no
|
||||||
|
- name: "buildbot_worker_password"
|
||||||
|
prompt: "Buildbot worker password (provided by ZECC)"
|
||||||
|
|
||||||
|
pre_tasks:
|
||||||
|
- name: Install Python 2 for Ansible and Buildbot
|
||||||
|
raw: test -e /usr/bin/python || test -e /usr/bin/python2 || test -e /usr/bin/python2.7 || test -e /usr/local/bin/python2.7 || (test -e /usr/bin/apt && apt -qqy update && apt install -qqy python) || (test -e /usr/bin/dnf && dnf install -qqy python2) || (test -e /usr/sbin/pkg && pkg install -qqy python2)
|
||||||
|
register: output
|
||||||
|
changed_when:
|
||||||
|
- output.stdout != ""
|
||||||
|
- output.stdout != "\r\n"
|
||||||
|
|
||||||
|
- name: Check if Python is in the configured location
|
||||||
|
raw: test -e {{ ansible_python_interpreter }}
|
||||||
|
ignore_errors: true
|
||||||
|
register: python_check
|
||||||
|
when: ansible_python_interpreter is defined
|
||||||
|
|
||||||
|
- name: Fail if configured Python is unavailable
|
||||||
|
fail:
|
||||||
|
msg: Python is not accessible at {{ ansible_python_interpreter }} on this host! Please set the inventory variable 'ansible_python_interpreter' to the location of the Python 2 binary.
|
||||||
|
when: ansible_python_interpreter is defined and python_check.rc == 1
|
||||||
|
|
||||||
|
- name: Check if Python is in the default location
|
||||||
|
raw: test -e /usr/bin/python
|
||||||
|
ignore_errors: true
|
||||||
|
register: python_check
|
||||||
|
when: ansible_python_interpreter is undefined
|
||||||
|
|
||||||
|
- name: Fail if default Python is unavailable
|
||||||
|
fail:
|
||||||
|
msg: Python is not accessible at /usr/bin/python on this host! Please set the inventory variable 'ansible_python_interpreter' to the location of the Python 2 binary.
|
||||||
|
when: ansible_python_interpreter is undefined and python_check.rc == 1
|
||||||
|
|
||||||
|
- name: Gathering Facts
|
||||||
|
setup:
|
||||||
|
|
||||||
|
tasks:
|
||||||
|
- name: Get dependencies for distribution
|
||||||
|
include_vars: "{{ item }}"
|
||||||
|
with_first_found:
|
||||||
|
- files:
|
||||||
|
- "vars/{{ ansible_distribution }}-{{ ansible_distribution_version }}.yml"
|
||||||
|
- "vars/{{ ansible_distribution }}-{{ ansible_distribution_major_version | int }}.yml"
|
||||||
|
- "vars/{{ ansible_distribution }}.yml"
|
||||||
|
- "vars/{{ ansible_os_family }}.yml"
|
||||||
|
skip: true
|
||||||
|
|
||||||
|
- name: Collate dependencies
|
||||||
|
set_fact:
|
||||||
|
package_deps: "{{ buildbot_deps + fetch_deps + conf_deps + build_deps + link_deps + dist_deps }}"
|
||||||
|
python_modules: "{{ buildbot_modules + rpc_test_modules }}"
|
||||||
|
|
||||||
|
- name: Install required packages
|
||||||
|
package:
|
||||||
|
name: "{{ item }}"
|
||||||
|
state: present
|
||||||
|
with_items: "{{ package_deps }}"
|
||||||
|
|
||||||
|
- name: Install required Python modules
|
||||||
|
pip:
|
||||||
|
name: "{{ item }}"
|
||||||
|
state: latest
|
||||||
|
with_items: "{{ python_modules }}"
|
||||||
|
notify: restart buildbot-worker
|
||||||
|
|
||||||
|
- name: Set up the Buildbot worker user
|
||||||
|
user:
|
||||||
|
name: "{{ buildbot_worker_user }}"
|
||||||
|
comment: Buildbot worker
|
||||||
|
shell: /bin/bash
|
||||||
|
state: present
|
||||||
|
|
||||||
|
- name: Create Buildbot worker
|
||||||
|
command: >
|
||||||
|
buildbot-worker create-worker ~/{{ buildbot_worker_name }}
|
||||||
|
{{ buildbot_master_host }}:{{ buildbot_master_port }}
|
||||||
|
{{ buildbot_worker_name|quote }} {{ buildbot_worker_password|quote }}
|
||||||
|
args:
|
||||||
|
creates: "~/{{ buildbot_worker_name }}/buildbot.tac"
|
||||||
|
become_user: "{{ buildbot_worker_user }}"
|
||||||
|
|
||||||
|
- name: Set admin details for Buildbot worker
|
||||||
|
copy:
|
||||||
|
content: "{{ buildbot_worker_admin }}"
|
||||||
|
dest: "~/{{ buildbot_worker_name }}/info/admin"
|
||||||
|
become_user: "{{ buildbot_worker_user }}"
|
||||||
|
|
||||||
|
- name: Set host details for Buildbot worker
|
||||||
|
template:
|
||||||
|
src: "{{ buildbot_worker_host_template }}"
|
||||||
|
dest: "~/{{ buildbot_worker_name }}/info/host"
|
||||||
|
become_user: "{{ buildbot_worker_user }}"
|
||||||
|
|
||||||
|
- name: Copy Buildbot worker systemd service unit
|
||||||
|
template:
|
||||||
|
src: templates/buildbot-worker.service.j2
|
||||||
|
dest: "/etc/systemd/system/buildbot-worker.service"
|
||||||
|
owner: root
|
||||||
|
group: root
|
||||||
|
mode: "0644"
|
||||||
|
notify: reload systemd
|
||||||
|
|
||||||
|
- name: Start Buildbot worker.
|
||||||
|
service:
|
||||||
|
name: buildbot-worker
|
||||||
|
state: started
|
||||||
|
enabled: yes
|
||||||
|
|
||||||
|
handlers:
|
||||||
|
- name: restart buildbot-worker
|
||||||
|
service:
|
||||||
|
name: buildbot-worker
|
||||||
|
state: restarted
|
||||||
|
|
||||||
|
- name: reload systemd
|
||||||
|
command: /bin/systemctl daemon-reload
|
|
@ -0,0 +1,6 @@
|
||||||
|
---
|
||||||
|
build_deps:
|
||||||
|
- build-essential # Depends on g++, libc6-dev, make
|
||||||
|
dist_deps:
|
||||||
|
- pkg-config # Required until b556beda264308e040f8d88aca4f2f386a0183d9 is pulled in
|
||||||
|
- python-dev
|
|
@ -0,0 +1,10 @@
|
||||||
|
---
|
||||||
|
build_deps:
|
||||||
|
- gcc
|
||||||
|
- gcc-c++
|
||||||
|
- make
|
||||||
|
- patch
|
||||||
|
dist_deps:
|
||||||
|
- pkgconfig # Required until b556beda264308e040f8d88aca4f2f386a0183d9 is pulled in
|
||||||
|
- python-devel
|
||||||
|
- redhat-rpm-config
|
|
@ -0,0 +1,9 @@
|
||||||
|
---
|
||||||
|
buildbot_deps:
|
||||||
|
- py27-pip
|
||||||
|
build_deps:
|
||||||
|
- gcc
|
||||||
|
- gmake
|
||||||
|
dist_deps:
|
||||||
|
- bash
|
||||||
|
- pkgconf # Required until b556beda264308e040f8d88aca4f2f386a0183d9 is pulled in
|
|
@ -0,0 +1,5 @@
|
||||||
|
---
|
||||||
|
build_deps:
|
||||||
|
- build-essential # Depends on g++, libc6-dev, make
|
||||||
|
dist_deps:
|
||||||
|
- pkg-config # Required until b556beda264308e040f8d88aca4f2f386a0183d9 is pulled in
|
|
@ -0,0 +1,5 @@
|
||||||
|
---
|
||||||
|
buildbot_worker_user: zcbbworker
|
||||||
|
buildbot_master_host: dev-ci.z.cash
|
||||||
|
buildbot_master_port: 9899
|
||||||
|
buildbot_worker_host_template: templates/host.j2
|
|
@ -0,0 +1,41 @@
|
||||||
|
---
|
||||||
|
# These variables can be overridden in distribution files.
|
||||||
|
|
||||||
|
# Dependencies required to install Buildbot
|
||||||
|
buildbot_deps:
|
||||||
|
- python-pip # So we can install Python modules
|
||||||
|
|
||||||
|
# Dependencies required to download files
|
||||||
|
fetch_deps:
|
||||||
|
- git
|
||||||
|
- wget # For zcutil/fetch-params.sh
|
||||||
|
|
||||||
|
# Dependencies required to configure Zcash
|
||||||
|
conf_deps:
|
||||||
|
- autoconf
|
||||||
|
- automake
|
||||||
|
- m4
|
||||||
|
|
||||||
|
# Dependencies required to compile Zcash
|
||||||
|
build_deps:
|
||||||
|
- g++
|
||||||
|
- gcc
|
||||||
|
- make
|
||||||
|
|
||||||
|
# Dependencies required to link Zcash
|
||||||
|
link_deps:
|
||||||
|
- libtool
|
||||||
|
|
||||||
|
# Additional distribution-specific dependencies
|
||||||
|
dist_deps: []
|
||||||
|
|
||||||
|
# Python modules required for a Zcash Buildbot worker
|
||||||
|
buildbot_modules:
|
||||||
|
- pip # Needs to be updated first so Buildbot installs
|
||||||
|
- buildbot-worker
|
||||||
|
- pyflakes
|
||||||
|
|
||||||
|
# Python modules required to run the Zcash RPC test suite
|
||||||
|
rpc_test_modules:
|
||||||
|
- pyblake2
|
||||||
|
- pyzmq
|
Loading…
Reference in New Issue