Auto merge of #2674 - str4d:2469-ci-workers-macos, r=str4d

[ci-workers] Add support for MacOSX

Also alters the `unix.yml` script to install all Python modules in a virtualenv, and adds curl and cmake dependencies.

Closes #2469.

contrib/ci-workers/README.md

@@ -5,29 +5,29 @@ installation for use as a Buildbot worker in Zcash's CI.
 
 # Criteria for Adding Workers
 
-a. Don't add workers until users complain about a problem on a platform
-   that doesn't yet have workers or if we anticipate many users will use
-   a platform, we may pre-emptively add an unsupported worker for it.
+a. Don't add workers until users complain about a problem on a platform that
+   doesn't yet have workers. However, if we anticipate many users will use a
+   platform, we may pre-emptively add an unsupported worker for it.
 
 b. Prioritize the platforms that seem to have the most users.
 
-c. When adding workers start by adding workers for the "most common"
-   variant of any distro, then if users later encounter problems with a
-   sub-variant, we can consider adding new workers at that point.
-   Example: add Ubuntu Desktop before Xubuntu, on the assumption the
-   former has a larger population base.
+c. When adding workers, start by adding workers for the "most common" variant of
+   any distro. Then if users later encounter problems with a sub-variant, we can
+   consider adding new workers at that point. Example: add Ubuntu Desktop before
+   Xubuntu, on the assumption the former has a larger population base, and the
+   latter only materially differs in the GUI.
 
 # Setting up a latent worker on Amazon EC2
 
-- Add a regular (non-latent) worker to the master.cfg for dev-ci.z.cash, and
-  deploy the changes.
+1. Add a regular (non-latent) worker to the master.cfg for dev-ci.z.cash, and
+   deploy the changes.
   - This enables the Ansible playbook to run to completion, ending in the worker
     connecting to the master.
 
-- Start a basic EC2 instance using the template AMI for the target OS.
+2. Start a basic EC2 instance using the template AMI for the target OS.
   - Choose the smallest instance size, it won't be used for building Zcash.
 
-- Figure out which user to log into the instance with.
+3. Figure out which user to log into the instance with.
   - E.g. for the Ubuntu template, use "ubuntu" instead of "root"
   - If you get an Ansible error later with a message like "Failed to connect to
     the host via ssh: Received message too long 1349281121\r\n", that means the
@@ -35,28 +35,28 @@ c. When adding workers start by adding workers for the "most common"
     Ansible protocol is balking. Try manually logging in with the same
     credentials to diagnose.
 
-- Create `inventory/hosts` containing the following:
+4. Create `inventory/hosts` containing the following:
 
     [zcash-ci-worker-unix]
     some-name ansible_host=<INSTANCE_IP> ansible_ssh_user=<USERNAME>
 
-- Run `ansible-playbook -e buildbot_worker_host_template=templates/host.ec2.j2 -i inventory/hosts unix.yml`,
-  passing in the worker's Buildbot name and password.
+5. Run `ansible-playbook -e buildbot_worker_host_template=templates/host.ec2.j2 -i inventory/hosts unix.yml`,
+   passing in the worker's Buildbot name and password.
   - After a successful run, the worker should be connected to dev-ci.z.cash and
     visible in its worker list.
 
-- Create an AMI from the instance. This is the worker AMI to put into the
-  master.cfg for dev-ci.z.cash.
+6. Create an AMI from the instance. This is the worker AMI to put into the
+   master.cfg for dev-ci.z.cash.
   - 16 GB of storage should be sufficient.
 
-- SSH into the instance, and edit the worker config to connect to ci.z.cash.
+7. SSH into the instance, and edit the worker config to connect to ci.z.cash.
 
-- Create an AMI from the instance. This is the worker AMI to put into the
-  master.cfg for ci.z.cash.
+8. Create an AMI from the instance. This is the worker AMI to put into the
+   master.cfg for ci.z.cash.
   - 16 GB of storage should be sufficient.
 
-- Delete the instance (it is no longer needed).
+9. Delete the instance (it is no longer needed).
 
-- Edit the master.cfg to turn the new worker into a latent (using the new AMI
-  IDs), add it to the appropriate worker groups, set up new builders etc.
+10. Edit the master.cfg to turn the new worker into a latent (using the new AMI
+    IDs), add it to the appropriate worker groups, set up new builders etc.
   - Deploy this via the normal PR review process.

contrib/ci-workers/files/bashrc

@@ -0,0 +1,2 @@
+export PATH=$HOME/venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+export EDITOR=vim

contrib/ci-workers/tasks/install-brew.yml

@@ -0,0 +1,10 @@
+---
+- name: Check if brew is installed
+  stat:
+    path: /usr/local/bin/brew
+  register: brew_check
+
+- name: Fail if brew is unavailable
+  fail:
+    msg: 'brew is not installed! Please install Homebrew: https://docs.brew.sh/Installation.html'
+  when: not brew_check.stat.exists

contrib/ci-workers/templates/buildbot-worker.plist.j2

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>Label</key>
+	<string>net.buildbot.worker</string>
+	<key>ProgramArguments</key>
+	<array>
+		<string>{{ buildbot_worker_dir }}/venv/bin/buildbot-worker</string>
+		<string>start</string>
+		<string>{{ buildbot_worker_name }}</string>
+	</array>
+	<key>WorkingDirectory</key>
+	<string>{{ buildbot_worker_dir }}</string>
+	<key>UserName</key>
+	<string>{{ buildbot_worker_user }}</string>
+	<key>KeepAlive</key>
+	<dict>
+		<key>NetworkState</key>
+		<true/>
+	</dict>
+</dict>
+</plist>

contrib/ci-workers/templates/buildbot-worker.service.j2

@@ -5,11 +5,11 @@ After=network.target
 
 [Service]
 Type=forking
-PIDFile=/home/{{ buildbot_worker_user }}/{{ buildbot_worker_name }}/twistd.pid
-WorkingDirectory=/home/{{ buildbot_worker_user }}
-ExecStart={{ pip_bin_dir }}/buildbot-worker start {{ buildbot_worker_name }}
-ExecReload={{ pip_bin_dir }}/buildbot-worker restart {{ buildbot_worker_name }}
-ExecStop={{ pip_bin_dir }}/buildbot-worker stop {{ buildbot_worker_name }}
+PIDFile={{ buildbot_worker_dir }}/{{ buildbot_worker_name }}/twistd.pid
+WorkingDirectory={{ buildbot_worker_dir }}
+ExecStart={{ buildbot_worker_dir }}/venv/bin/buildbot-worker start {{ buildbot_worker_name }}
+ExecReload={{ buildbot_worker_dir }}/venv/bin/buildbot-worker restart {{ buildbot_worker_name }}
+ExecStop={{ buildbot_worker_dir }}/venv/bin/buildbot-worker stop {{ buildbot_worker_name }}
 Restart=always
 User={{ buildbot_worker_user }}
 

contrib/ci-workers/templates/host.j2

@@ -1,3 +1,3 @@
 OS: {{ ansible_distribution }} {{ ansible_distribution_version }}
 Memory: {{ ansible_memtotal_mb }} MB
-CPU: {{ ansible_processor[1] }}
+CPU: {{ ansible_processor if ansible_processor is string else ansible_processor[1] }} ({{ ansible_processor_cores }} cores)

contrib/ci-workers/unix.yml

@@ -50,6 +50,7 @@
 
     - name: Gathering Facts
       setup:
+      tags: deps
 
     - name: Fail if Python is the wrong version
       fail:
@@ -66,34 +67,44 @@
             - "vars/{{ ansible_distribution }}.yml"
             - "vars/{{ ansible_os_family }}.yml"
           skip: true
+      tags: deps
 
     - name: Collate dependencies
       set_fact:
         package_deps: "{{ buildbot_deps + fetch_deps + conf_deps + build_deps + link_deps + dist_deps }}"
         python_modules: "{{ buildbot_modules + rpc_test_modules }}"
+      tags: deps
+
+    - name: Install Homebrew [MacOSX]
+      include: tasks/install-brew.yml
+      when: ansible_distribution == 'MacOSX'
+      tags: deps
 
     - name: Update rolling release [Arch Linux]
       pacman:
         update_cache: yes
         upgrade: yes
       when: ansible_distribution == 'Archlinux'
+      tags: deps
 
     - name: Install required packages
       package:
         name: "{{ item }}"
         state: present
       with_items: "{{ package_deps }}"
+      become_user: "{{ ansible_ssh_user if ansible_distribution == 'MacOSX' else 'root' }}"
+      tags: deps
 
-    - name: Install pip [CentOS]
+    - name: Install pip [CentOS, MacOSX]
       include: tasks/install-pip.yml
-      when: ansible_distribution == 'CentOS'
+      when: ansible_distribution in ['CentOS', 'MacOSX']
 
-    - name: Install required Python modules
+    - name: Install required Python system modules
       pip:
         name: "{{ item }}"
         state: latest
-      with_items: "{{ python_modules }}"
-      notify: restart buildbot-worker
+        executable: "{{ '/usr/local/bin/pip' if ansible_distribution == 'MacOSX' else omit }}"
+      with_items: "{{ system_modules }}"
 
     - name: Set up the Buildbot worker user
       user:
@@ -102,9 +113,28 @@
         shell: /bin/bash
         state: present
 
+    - name: Get absolute path to Buildbot worker home directory
+      command: echo ~
+      register: homedir
+      become_user: "{{ buildbot_worker_user }}"
+
+    - name: Save absolute path to Buildbot worker home directory
+      set_fact:
+        buildbot_worker_dir: "{{ homedir.stdout }}"
+
+    - name: Install required Python modules
+      pip:
+        name: "{{ item }}"
+        state: latest
+        virtualenv: "~{{ buildbot_worker_user }}/venv"
+        virtualenv_command: "{{ '/usr/local/bin/virtualenv' if ansible_distribution == 'MacOSX' else omit }}"
+      with_items: "{{ python_modules }}"
+      become_user: "{{ buildbot_worker_user }}"
+      notify: restart buildbot-worker
+
     - name: Create Buildbot worker
       command: >
-        buildbot-worker create-worker ~/{{ buildbot_worker_name }}
+        ~{{ buildbot_worker_user }}/venv/bin/buildbot-worker create-worker ~/{{ buildbot_worker_name }}
         {{ buildbot_master_host }}:{{ buildbot_master_port }}
         {{ buildbot_worker_name|quote }} {{ buildbot_worker_password|quote }}
       args:
@@ -116,7 +146,7 @@
         content: "{{ buildbot_worker_admin }}"
         dest: "~{{ buildbot_worker_user }}/{{ buildbot_worker_name }}/info/admin"
         owner: "{{ buildbot_worker_user }}"
-        group: "{{ buildbot_worker_user }}"
+        group: "{{ omit if ansible_distribution == 'MacOSX' else buildbot_worker_user }}"
         mode: "0644"
 
     - name: Set host details for Buildbot worker
@@ -124,7 +154,15 @@
         src: "{{ buildbot_worker_host_template }}"
         dest: "~{{ buildbot_worker_user }}/{{ buildbot_worker_name }}/info/host"
         owner: "{{ buildbot_worker_user }}"
-        group: "{{ buildbot_worker_user }}"
+        group: "{{ omit if ansible_distribution == 'MacOSX' else buildbot_worker_user }}"
+        mode: "0644"
+
+    - name: Install custom bashrc for virtualenv
+      copy:
+        src: bashrc
+        dest: "~{{ buildbot_worker_user }}/.bashrc"
+        owner: "{{ buildbot_worker_user }}"
+        group: "{{ omit if ansible_distribution == 'MacOSX' else buildbot_worker_user }}"
         mode: "0644"
 
     - name: Copy Buildbot worker systemd service unit
@@ -134,13 +172,32 @@
         owner: root
         group: root
         mode: "0644"
+      when: ansible_distribution != 'MacOSX'
       notify: reload systemd
 
-    - name: Start Buildbot worker.
+    - name: Copy Buildbot worker launchd service unit
+      template:
+        src: templates/buildbot-worker.plist.j2
+        dest: "/Library/LaunchDaemons/net.buildbot.worker.plist"
+        owner: root
+        group: wheel
+        mode: "0644"
+      when: ansible_distribution == 'MacOSX'
+
+    - name: Start Buildbot worker
       service:
         name: buildbot-worker
         state: started
         enabled: yes
+      when: ansible_distribution != 'MacOSX'
+
+    - name: Load Buildbot worker service [MacOSX]
+      command: launchctl load /Library/LaunchDaemons/net.buildbot.worker.plist
+      when: ansible_distribution == 'MacOSX'
+
+    - name: Start Buildbot worker [MacOSX]
+      command: launchctl start net.buildbot.worker
+      when: ansible_distribution == 'MacOSX'
 
   handlers:
     - name: restart buildbot-worker

contrib/ci-workers/vars/Archlinux.yml

@@ -2,6 +2,6 @@
 buildbot_deps:
   - python2-pip
 build_deps:
+  - cmake
   - multilib/gcc
   - make
-pip_bin_dir: /usr/bin

contrib/ci-workers/vars/CentOS.yml

@@ -2,6 +2,7 @@
 buildbot_deps: [] # Empty to remove python-pip
 build_deps:
   - bzip2
+  - cmake
   - gcc
   - gcc-c++
   - make
@@ -10,4 +11,3 @@ dist_deps:
   - pkgconfig # Required until b556beda264308e040f8d88aca4f2f386a0183d9 is pulled in
   - python-devel
   - redhat-rpm-config
-pip_bin_dir: /usr/bin

contrib/ci-workers/vars/Debian.yml

@@ -1,6 +1,6 @@
 ---
 build_deps:
   - build-essential # Depends on g++, libc6-dev, make
+  - cmake
 dist_deps:
-  - pkg-config # Required until b556beda264308e040f8d88aca4f2f386a0183d9 is pulled in
   - python-dev

contrib/ci-workers/vars/Fedora.yml

@@ -1,5 +1,6 @@
 ---
 build_deps:
+  - cmake
   - gcc
   - gcc-c++
   - make

contrib/ci-workers/vars/FreeBSD.yml

@@ -2,6 +2,7 @@
 buildbot_deps:
   - py27-pip
 build_deps:
+  - cmake
   - gcc
   - gmake
 dist_deps:

contrib/ci-workers/vars/MacOSX.yml

@@ -0,0 +1,6 @@
+---
+buildbot_deps:
+  - coreutils # For gnproc etc.
+# Most are already installed
+build_deps:
+  - cmake

contrib/ci-workers/vars/Ubuntu.yml

@@ -1,5 +1,4 @@
 ---
 build_deps:
   - build-essential # Depends on g++, libc6-dev, make
-dist_deps:
-  - pkg-config # Required until b556beda264308e040f8d88aca4f2f386a0183d9 is pulled in
+  - cmake

contrib/ci-workers/vars/default.yml

@@ -7,6 +7,7 @@ buildbot_deps:
 
 # Dependencies required to download files
 fetch_deps:
+  - curl # For depends/
   - git
   - wget # For zcutil/fetch-params.sh
 
@@ -15,9 +16,11 @@ conf_deps:
   - autoconf
   - automake
   - m4
+  - pkg-config
 
 # Dependencies required to compile Zcash
 build_deps:
+  - cmake
   - g++
   - gcc
   - make
@@ -34,6 +37,10 @@ grind_deps:
   - lcov
   - valgrind
 
+# Python modules required on the system
+system_modules:
+  - virtualenv
+
 # Python modules required for a Zcash Buildbot worker
 buildbot_modules:
   - pip # Needs to be updated first so Buildbot installs
@@ -44,6 +51,3 @@ buildbot_modules:
 rpc_test_modules:
   - pyblake2
   - pyzmq
-
-# Environment variables
-pip_bin_dir: /usr/local/bin