Auto merge of #2509 - str4d:ci-workers, r=str4d
Ansible playbook for configuring CI workers The playbook in this PR can be used for configuring common Unix variants to run a Buildbot worker for the Zcash CI, starting from a fresh install (or a VM template provided by a hosting service).
This commit is contained in:
commit
d3a976095f
|
@ -0,0 +1,48 @@
|
|||
# Zcash CI workers
|
||||
|
||||
This folder contains the Ansible playbooks for configuring a fresh OS
|
||||
installation for use as a Buildbot worker in Zcash's CI.
|
||||
|
||||
# Setting up a latent worker on Amazon EC2
|
||||
|
||||
- Add a regular (non-latent) worker to the master.cfg for dev-ci.z.cash, and
|
||||
deploy the changes.
|
||||
- This enables the Ansible playbook to run to completion, ending in the worker
|
||||
connecting to the master.
|
||||
|
||||
- Start a basic EC2 instance using the template AMI for the target OS.
|
||||
- Choose the smallest instance size, it won't be used for building Zcash.
|
||||
|
||||
- Figure out which user to log into the instance with.
|
||||
- E.g. for the Ubuntu template, use "ubuntu" instead of "root"
|
||||
- If you get an Ansible error later with a message like "Failed to connect to
|
||||
the host via ssh: Received message too long 1349281121\r\n", that means the
|
||||
instance is sending a text string in response to the SSH connection, and the
|
||||
Ansible protocol is balking. Try manually logging in with the same
|
||||
credentials to diagnose.
|
||||
|
||||
- Create `inventory/hosts` containing the following:
|
||||
|
||||
[zcash-ci-worker-unix]
|
||||
some-name ansible_host=<INSTANCE_IP> ansible_ssh_user=<USERNAME>
|
||||
|
||||
- Run `ansible-playbook -e buildbot_worker_host_template=templates/host.ec2.j2 -i inventory/hosts unix.yml`,
|
||||
passing in the worker's Buildbot name and password.
|
||||
- After a successful run, the worker should be connected to dev-ci.z.cash and
|
||||
visible in its worker list.
|
||||
|
||||
- Create an AMI from the instance. This is the worker AMI to put into the
|
||||
master.cfg for dev-ci.z.cash.
|
||||
- 16 GB of storage should be sufficient.
|
||||
|
||||
- SSH into the instance, and edit the worker config to connect to ci.z.cash.
|
||||
|
||||
- Create an AMI from the instance. This is the worker AMI to put into the
|
||||
master.cfg for ci.z.cash.
|
||||
- 16 GB of storage should be sufficient.
|
||||
|
||||
- Delete the instance (it is no longer needed).
|
||||
|
||||
- Edit the master.cfg to turn the new worker into a latent (using the new AMI
|
||||
IDs), add it to the appropriate worker groups, set up new builders etc.
|
||||
- Deploy this via the normal PR review process.
|
|
@ -0,0 +1,17 @@
|
|||
[Unit]
|
||||
Description=Buildbot worker
|
||||
Wants=network.target
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=forking
|
||||
PIDFile=/home/{{ buildbot_worker_user }}/{{ buildbot_worker_name }}/twistd.pid
|
||||
WorkingDirectory=/home/{{ buildbot_worker_user }}
|
||||
ExecStart=/usr/local/bin/buildbot-worker start {{ buildbot_worker_name }}
|
||||
ExecReload=/usr/local/bin/buildbot-worker restart {{ buildbot_worker_name }}
|
||||
ExecStop=/usr/local/bin/buildbot-worker stop {{ buildbot_worker_name }}
|
||||
Restart=always
|
||||
User={{ buildbot_worker_user }}
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
|
@ -0,0 +1 @@
|
|||
OS: {{ ansible_distribution }} {{ ansible_distribution_version }}
|
|
@ -0,0 +1,3 @@
|
|||
OS: {{ ansible_distribution }} {{ ansible_distribution_version }}
|
||||
Memory: {{ ansible_memtotal_mb }} MB
|
||||
CPU: {{ ansible_processor[1] }}
|
|
@ -0,0 +1,133 @@
|
|||
---
|
||||
- name: Configure a Buildbot worker for Zcash CI
|
||||
hosts: zcash-ci-worker-unix
|
||||
become: true
|
||||
gather_facts: False
|
||||
|
||||
vars_files:
|
||||
- vars/default.yml
|
||||
- vars/buildbot.yml
|
||||
|
||||
vars_prompt:
|
||||
- name: "buildbot_worker_admin"
|
||||
prompt: "Admin details"
|
||||
default: "Zcash <sysadmin@z.cash>"
|
||||
- name: "buildbot_worker_name"
|
||||
prompt: "Buildbot worker name (provided by ZECC)"
|
||||
private: no
|
||||
- name: "buildbot_worker_password"
|
||||
prompt: "Buildbot worker password (provided by ZECC)"
|
||||
|
||||
pre_tasks:
|
||||
- name: Install Python 2 for Ansible and Buildbot
|
||||
raw: test -e /usr/bin/python || test -e /usr/bin/python2 || test -e /usr/bin/python2.7 || test -e /usr/local/bin/python2.7 || (test -e /usr/bin/apt && apt -qqy update && apt install -qqy python) || (test -e /usr/bin/dnf && dnf install -qqy python2) || (test -e /usr/sbin/pkg && pkg install -qqy python2)
|
||||
register: output
|
||||
changed_when:
|
||||
- output.stdout != ""
|
||||
- output.stdout != "\r\n"
|
||||
|
||||
- name: Check if Python is in the configured location
|
||||
raw: test -e {{ ansible_python_interpreter }}
|
||||
ignore_errors: true
|
||||
register: python_check
|
||||
when: ansible_python_interpreter is defined
|
||||
|
||||
- name: Fail if configured Python is unavailable
|
||||
fail:
|
||||
msg: Python is not accessible at {{ ansible_python_interpreter }} on this host! Please set the inventory variable 'ansible_python_interpreter' to the location of the Python 2 binary.
|
||||
when: ansible_python_interpreter is defined and python_check.rc == 1
|
||||
|
||||
- name: Check if Python is in the default location
|
||||
raw: test -e /usr/bin/python
|
||||
ignore_errors: true
|
||||
register: python_check
|
||||
when: ansible_python_interpreter is undefined
|
||||
|
||||
- name: Fail if default Python is unavailable
|
||||
fail:
|
||||
msg: Python is not accessible at /usr/bin/python on this host! Please set the inventory variable 'ansible_python_interpreter' to the location of the Python 2 binary.
|
||||
when: ansible_python_interpreter is undefined and python_check.rc == 1
|
||||
|
||||
- name: Gathering Facts
|
||||
setup:
|
||||
|
||||
tasks:
|
||||
- name: Get dependencies for distribution
|
||||
include_vars: "{{ item }}"
|
||||
with_first_found:
|
||||
- files:
|
||||
- "vars/{{ ansible_distribution }}-{{ ansible_distribution_version }}.yml"
|
||||
- "vars/{{ ansible_distribution }}-{{ ansible_distribution_major_version | int }}.yml"
|
||||
- "vars/{{ ansible_distribution }}.yml"
|
||||
- "vars/{{ ansible_os_family }}.yml"
|
||||
skip: true
|
||||
|
||||
- name: Collate dependencies
|
||||
set_fact:
|
||||
package_deps: "{{ buildbot_deps + fetch_deps + conf_deps + build_deps + link_deps + dist_deps }}"
|
||||
python_modules: "{{ buildbot_modules + rpc_test_modules }}"
|
||||
|
||||
- name: Install required packages
|
||||
package:
|
||||
name: "{{ item }}"
|
||||
state: present
|
||||
with_items: "{{ package_deps }}"
|
||||
|
||||
- name: Install required Python modules
|
||||
pip:
|
||||
name: "{{ item }}"
|
||||
state: latest
|
||||
with_items: "{{ python_modules }}"
|
||||
notify: restart buildbot-worker
|
||||
|
||||
- name: Set up the Buildbot worker user
|
||||
user:
|
||||
name: "{{ buildbot_worker_user }}"
|
||||
comment: Buildbot worker
|
||||
shell: /bin/bash
|
||||
state: present
|
||||
|
||||
- name: Create Buildbot worker
|
||||
command: >
|
||||
buildbot-worker create-worker ~/{{ buildbot_worker_name }}
|
||||
{{ buildbot_master_host }}:{{ buildbot_master_port }}
|
||||
{{ buildbot_worker_name|quote }} {{ buildbot_worker_password|quote }}
|
||||
args:
|
||||
creates: "~/{{ buildbot_worker_name }}/buildbot.tac"
|
||||
become_user: "{{ buildbot_worker_user }}"
|
||||
|
||||
- name: Set admin details for Buildbot worker
|
||||
copy:
|
||||
content: "{{ buildbot_worker_admin }}"
|
||||
dest: "~/{{ buildbot_worker_name }}/info/admin"
|
||||
become_user: "{{ buildbot_worker_user }}"
|
||||
|
||||
- name: Set host details for Buildbot worker
|
||||
template:
|
||||
src: "{{ buildbot_worker_host_template }}"
|
||||
dest: "~/{{ buildbot_worker_name }}/info/host"
|
||||
become_user: "{{ buildbot_worker_user }}"
|
||||
|
||||
- name: Copy Buildbot worker systemd service unit
|
||||
template:
|
||||
src: templates/buildbot-worker.service.j2
|
||||
dest: "/etc/systemd/system/buildbot-worker.service"
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
notify: reload systemd
|
||||
|
||||
- name: Start Buildbot worker.
|
||||
service:
|
||||
name: buildbot-worker
|
||||
state: started
|
||||
enabled: yes
|
||||
|
||||
handlers:
|
||||
- name: restart buildbot-worker
|
||||
service:
|
||||
name: buildbot-worker
|
||||
state: restarted
|
||||
|
||||
- name: reload systemd
|
||||
command: /bin/systemctl daemon-reload
|
|
@ -0,0 +1,6 @@
|
|||
---
|
||||
build_deps:
|
||||
- build-essential # Depends on g++, libc6-dev, make
|
||||
dist_deps:
|
||||
- pkg-config # Required until b556beda264308e040f8d88aca4f2f386a0183d9 is pulled in
|
||||
- python-dev
|
|
@ -0,0 +1,10 @@
|
|||
---
|
||||
build_deps:
|
||||
- gcc
|
||||
- gcc-c++
|
||||
- make
|
||||
- patch
|
||||
dist_deps:
|
||||
- pkgconfig # Required until b556beda264308e040f8d88aca4f2f386a0183d9 is pulled in
|
||||
- python-devel
|
||||
- redhat-rpm-config
|
|
@ -0,0 +1,9 @@
|
|||
---
|
||||
buildbot_deps:
|
||||
- py27-pip
|
||||
build_deps:
|
||||
- gcc
|
||||
- gmake
|
||||
dist_deps:
|
||||
- bash
|
||||
- pkgconf # Required until b556beda264308e040f8d88aca4f2f386a0183d9 is pulled in
|
|
@ -0,0 +1,5 @@
|
|||
---
|
||||
build_deps:
|
||||
- build-essential # Depends on g++, libc6-dev, make
|
||||
dist_deps:
|
||||
- pkg-config # Required until b556beda264308e040f8d88aca4f2f386a0183d9 is pulled in
|
|
@ -0,0 +1,5 @@
|
|||
---
|
||||
buildbot_worker_user: zcbbworker
|
||||
buildbot_master_host: dev-ci.z.cash
|
||||
buildbot_master_port: 9899
|
||||
buildbot_worker_host_template: templates/host.j2
|
|
@ -0,0 +1,41 @@
|
|||
---
|
||||
# These variables can be overridden in distribution files.
|
||||
|
||||
# Dependencies required to install Buildbot
|
||||
buildbot_deps:
|
||||
- python-pip # So we can install Python modules
|
||||
|
||||
# Dependencies required to download files
|
||||
fetch_deps:
|
||||
- git
|
||||
- wget # For zcutil/fetch-params.sh
|
||||
|
||||
# Dependencies required to configure Zcash
|
||||
conf_deps:
|
||||
- autoconf
|
||||
- automake
|
||||
- m4
|
||||
|
||||
# Dependencies required to compile Zcash
|
||||
build_deps:
|
||||
- g++
|
||||
- gcc
|
||||
- make
|
||||
|
||||
# Dependencies required to link Zcash
|
||||
link_deps:
|
||||
- libtool
|
||||
|
||||
# Additional distribution-specific dependencies
|
||||
dist_deps: []
|
||||
|
||||
# Python modules required for a Zcash Buildbot worker
|
||||
buildbot_modules:
|
||||
- pip # Needs to be updated first so Buildbot installs
|
||||
- buildbot-worker
|
||||
- pyflakes
|
||||
|
||||
# Python modules required to run the Zcash RPC test suite
|
||||
rpc_test_modules:
|
||||
- pyblake2
|
||||
- pyzmq
|
Loading…
Reference in New Issue