Merge pull request #146 from superbaud/parallel-build
This commit is contained in:
sasha
GitHub
commit
ae0cbf7f8c
|
|
@ -514,6 +514,8 @@ version.
|
|||
|
||||
## Provision a virtual machine
|
||||
|
||||
Edit the `Vagrantfile` so the `v.memory` and `v.cpus` values are a bit lower than the memory and CPU count in the actual physical machine. If they're set too high then you might badly wedge your machine.
|
||||
|
||||
From the project root directory, run:
|
||||
|
||||
```
|
||||
|
|
@ -598,6 +600,8 @@ Once in a shell session in the VM, we're ready to run the gitian build.
|
|||
vagrant@zcash-build:~$ ./gitian-build.sh
|
||||
```
|
||||
|
||||
If you want to run a parallel build, invoke `./gitian-parallel-build.sh` instead.
|
||||
|
||||
The output from `gbuild` is informative. There are some common warnings which can be ignored, e.g. if you get an intermittent privileges error related to LXC then just execute the script again. The most important thing is that one reaches the step which says `Running build script (log in var/build.log)`. If not, then something else is wrong and you should let us know.
|
||||
|
||||
Take a look at the variables near the top of `~/gitian-build.sh` and get familiar with its functioning, as it can handle most tasks.
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ Vagrant.configure(2) do |config|
|
|||
}
|
||||
|
||||
config.ssh.forward_agent = true
|
||||
config.disksize.size = '32GB'
|
||||
config.disksize.size = '64GB'
|
||||
config.vm.define 'zcash-build', autostart: false do |gitian|
|
||||
gitian.vm.box = "debian/bullseye64"
|
||||
gitian.vm.box_version = "11.20220328.1"
|
||||
|
|
@ -21,7 +21,8 @@ Vagrant.configure(2) do |config|
|
|||
gitian.vm.provider "virtualbox" do |v|
|
||||
v.name = "zcash-build"
|
||||
v.memory = 4096
|
||||
v.cpus = 2
|
||||
v.cpus = 4
|
||||
v.customize ["modifyvm", :id, "--nested-hw-virt", "on"]
|
||||
end
|
||||
|
||||
# Added to disable synced folders
|
||||
|
|
|
|||
|
|
@ -127,30 +127,6 @@
|
|||
force: yes
|
||||
become_user: "{{ gitian_user }}"
|
||||
|
||||
- name: Reboot.
|
||||
shell: sleep 3 && shutdown -r now "Rebooting..."
|
||||
async: 1
|
||||
poll: 0
|
||||
ignore_errors: true
|
||||
become: yes
|
||||
|
||||
- name: Figure out the Vagrant VM's SSH port number.
|
||||
local_action: "shell vagrant ssh-config zcash-build | grep Port | awk {'print $2'}"
|
||||
register: vagrant_ssh_port
|
||||
become: no
|
||||
|
||||
- name: Wait for virtual machine to come back.
|
||||
local_action: wait_for
|
||||
host={{ ansible_host | default('localhost') }}
|
||||
port={{ vagrant_ssh_port.stdout | int }}
|
||||
delay=30
|
||||
state=started
|
||||
become: no
|
||||
|
||||
- name: Wait extra time for VM to come back up.
|
||||
pause:
|
||||
seconds: 10
|
||||
|
||||
- name: Set Git username.
|
||||
command: "git config --global user.name \"{{ git_name }}\""
|
||||
become_user: "{{ gitian_user }}"
|
||||
|
|
@ -182,6 +158,15 @@
|
|||
mode: "0755"
|
||||
tags: script
|
||||
|
||||
- name: Copy Parallel Gitian build script.
|
||||
template:
|
||||
src: gitian-parallel-build.sh
|
||||
dest: "/home/{{ gitian_user }}/gitian-parallel-build.sh"
|
||||
owner: "{{ gitian_user }}"
|
||||
group: "{{ gitian_user }}"
|
||||
mode: "0755"
|
||||
tags: script
|
||||
|
||||
- name: Clean the apt cache to free up space.
|
||||
apt:
|
||||
autoclean: yes
|
||||
|
|
@ -196,3 +181,4 @@
|
|||
Finished bootstrapping the Gitian host VM!
|
||||
To enter the environment, run `vagrant ssh zcash-build`
|
||||
and then use `./gitian-build.sh` to kick off a build.
|
||||
For parallelism, use `./gitian-parallel-build.sh`.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,265 @@
|
|||
# Copyright (c) 2016 The Bitcoin Core developers
|
||||
# Distributed under the MIT software license, see the accompanying
|
||||
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
|
||||
|
||||
# What to do
|
||||
verify=false
|
||||
build=true
|
||||
|
||||
# Systems to build
|
||||
linux=true
|
||||
|
||||
# Other Basic variables
|
||||
SIGNER="{{ gpg_key_name }}"
|
||||
VERSION={{ zcash_version }}
|
||||
commit=false
|
||||
url={{ zcash_git_repo_url }}
|
||||
proc=$(nproc)
|
||||
mem=$(free -m | awk 'FNR==2 { print $2-512}')
|
||||
lxc=true
|
||||
scriptName=$(basename -- "$0")
|
||||
signProg="gpg --detach-sign"
|
||||
commitFiles=true
|
||||
|
||||
gitian_builder_repo_path=${HOME}/gitian-builder
|
||||
gitian_sigs_repo_path=${HOME}/gitian.sigs
|
||||
|
||||
zcash_repo_dir_path=${HOME}/zcash
|
||||
gitian_descriptor_path=${zcash_repo_dir_path}/contrib/gitian-descriptors/gitian-linux-parallel.yml
|
||||
|
||||
zcash_binaries_dir_path=${HOME}/zcash-binaries
|
||||
|
||||
build_dir_path=${gitian_builder_repo_path}/build
|
||||
suite_descriptors_dir_path=${gitian_builder_repo_path}/suites
|
||||
|
||||
# Help Message
|
||||
read -d '' usage <<- EOF
|
||||
Usage: $scriptName [-c|u|v|b|o|h|j|m|] signer version
|
||||
|
||||
Run this script from the directory containing the zcash, gitian-builder, and gitian.sigs repositories.
|
||||
|
||||
Arguments:
|
||||
signer GPG signer to sign each build assert file
|
||||
version Version number, commit, or branch to build. If building a commit or branch, the -c option must be specified
|
||||
|
||||
Options:
|
||||
-c|--commit Indicate that the version argument is for a commit or branch
|
||||
-u|--url Specify the URL of the repository. Default is {{ zcash_git_repo_url }}
|
||||
-v|--verify Verify the gitian build
|
||||
-b|--build Do a gitian build
|
||||
-j Number of processes to use. Default is $(nproc)
|
||||
-m Memory to allocate in MiB. Default is total memory minus 512 bytes
|
||||
--detach-sign Create the assert file for detached signing. Will not commit anything.
|
||||
--no-commit Do not commit anything to git
|
||||
-h|--help Print this help message
|
||||
EOF
|
||||
|
||||
# Get options and arguments
|
||||
while :; do
|
||||
case $1 in
|
||||
# Verify
|
||||
-v|--verify)
|
||||
verify=true
|
||||
;;
|
||||
# Build
|
||||
-b|--build)
|
||||
build=true
|
||||
;;
|
||||
# PGP Signer
|
||||
-S|--signer)
|
||||
if [ -n "$2" ]
|
||||
then
|
||||
SIGNER=$2
|
||||
shift
|
||||
else
|
||||
echo 'Error: "--signer" requires a non-empty argument.'
|
||||
exit 1
|
||||
fi
|
||||
;;
|
||||
# Help message
|
||||
-h|--help)
|
||||
echo "$usage"
|
||||
exit 0
|
||||
;;
|
||||
# Commit or branch
|
||||
-c|--commit)
|
||||
commit=true
|
||||
;;
|
||||
# Number of Processes
|
||||
-j)
|
||||
if [ -n "$2" ]
|
||||
then
|
||||
proc=$2
|
||||
shift
|
||||
else
|
||||
echo 'Error: "-j" requires an argument'
|
||||
exit 1
|
||||
fi
|
||||
;;
|
||||
# Memory to allocate
|
||||
-m)
|
||||
if [ -n "$2" ]
|
||||
then
|
||||
mem=$2
|
||||
shift
|
||||
else
|
||||
echo 'Error: "-m" requires an argument'
|
||||
exit 1
|
||||
fi
|
||||
;;
|
||||
# URL
|
||||
-u)
|
||||
if [ -n "$2" ]
|
||||
then
|
||||
url=$2
|
||||
shift
|
||||
else
|
||||
echo 'Error: "-u" requires an argument'
|
||||
exit 1
|
||||
fi
|
||||
;;
|
||||
# Detach sign
|
||||
--detach-sign)
|
||||
signProg="true"
|
||||
commitFiles=false
|
||||
;;
|
||||
# Commit files
|
||||
--no-commit)
|
||||
commitFiles=false
|
||||
;;
|
||||
*) # Default case: If no more options then break out of the loop.
|
||||
break
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
# Set up LXC
|
||||
if [[ $lxc = true ]]
|
||||
then
|
||||
source ~/.profile
|
||||
fi
|
||||
|
||||
# Get version
|
||||
if [[ -n "$1" ]]
|
||||
then
|
||||
VERSION=$1
|
||||
COMMIT=$VERSION
|
||||
shift
|
||||
fi
|
||||
|
||||
# Check that a signer is specified
|
||||
if [[ $SIGNER == "" ]]
|
||||
then
|
||||
echo "$scriptName: Missing signer."
|
||||
echo "Try $scriptName --help for more information"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check that a version is specified
|
||||
if [[ $VERSION == "" ]]
|
||||
then
|
||||
echo "$scriptName: Missing version."
|
||||
echo "Try $scriptName --help for more information"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Add a "v" if no -c
|
||||
if [[ $commit = false ]]
|
||||
then
|
||||
COMMIT="${VERSION}"
|
||||
fi
|
||||
echo ${COMMIT}
|
||||
|
||||
# Set up build
|
||||
pushd ${zcash_repo_dir_path}
|
||||
git fetch
|
||||
git checkout ${COMMIT}
|
||||
popd
|
||||
|
||||
|
||||
suites=$(explode_yaml_file.py ${gitian_descriptor_path} suites ${suite_descriptors_dir_path})
|
||||
|
||||
# Build
|
||||
if [[ $build = true ]]
|
||||
then
|
||||
# Make output folder
|
||||
mkdir -p ${zcash_binaries_dir_path}/${VERSION}
|
||||
|
||||
# Linux
|
||||
if [[ $linux = true ]]
|
||||
then
|
||||
for suite in ${suites} ; do
|
||||
echo "processing suite ${suite}"
|
||||
|
||||
suite_dir_path=${suite_descriptors_dir_path}/${suite}
|
||||
echo "suite_dir_path: ${suite_dir_path}"
|
||||
|
||||
# Build Dependencies
|
||||
echo ""
|
||||
echo "Building Dependencies"
|
||||
echo ""
|
||||
pushd ${gitian_builder_repo_path}
|
||||
mkdir -p inputs
|
||||
rm -rf ${gitian_builder_repo_path}/cache/* # Clear cache directory before each build
|
||||
|
||||
make -C ${zcash_repo_dir_path}/depends download SOURCES_PATH=${gitian_builder_repo_path}/cache/common
|
||||
|
||||
suite_image_path=${gitian_builder_repo_path}/base-${suite}-amd64
|
||||
echo "suite_image_path: ${suite_image_path}"
|
||||
|
||||
if [ ! -f ${suite_image_path} ]; then
|
||||
echo "Image not found for suite ${suite}; calling make-base-vm to build it"
|
||||
./bin/make-base-vm --lxc --arch amd64 --distro debian --suite ${suite}
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Compiling variant: ${VERSION}_${suite}"
|
||||
echo ""
|
||||
|
||||
./bin/gbuild --fetch-tags -j ${proc} -m ${mem} --commit zcash=${COMMIT} --url zcash=${url} ${suite_dir_path}/gitian-linux-parallel.yml
|
||||
./bin/gsign -p "$signProg" --signer "$SIGNER" --release ${VERSION}_${suite} --destination ${gitian_sigs_repo_path}/ ${suite_dir_path}/gitian-linux-parallel.yml
|
||||
|
||||
suite_binaries_dir_path=${zcash_binaries_dir_path}/${VERSION}/${suite}
|
||||
mkdir ${suite_binaries_dir_path}
|
||||
|
||||
mv ${build_dir_path}/out/zcash-*.tar.gz ${build_dir_path}/out/src/zcash-*.tar.gz ${suite_binaries_dir_path}
|
||||
|
||||
popd # pushd ${gitian_builder_repo_path}
|
||||
|
||||
|
||||
if [[ $commitFiles = true ]]
|
||||
then
|
||||
# Commit to gitian.sigs repo
|
||||
echo ""
|
||||
echo "Committing ${VERSION}_${suite} Signatures"
|
||||
echo ""
|
||||
pushd ${gitian_sigs_repo_path}
|
||||
git add ${VERSION}_${suite}/${SIGNER}
|
||||
git commit -a -m "Add ${VERSION}_${suite} signatures for ${SIGNER}"
|
||||
popd
|
||||
fi
|
||||
done
|
||||
fi
|
||||
fi
|
||||
|
||||
# Verify the build
|
||||
if [[ $verify = true ]]
|
||||
then
|
||||
# Linux
|
||||
pushd ${gitian_builder_repo_path}
|
||||
|
||||
for suite in ${suites} ; do
|
||||
echo "processing suite ${suite}"
|
||||
|
||||
suite_dir_path=${suite_descriptors_dir_path}/${suite}
|
||||
echo "suite_dir_path: ${suite_dir_path}"
|
||||
|
||||
echo ""
|
||||
echo "Verifying ${VERSION}_${suite} Linux"
|
||||
echo ""
|
||||
|
||||
./bin/gverify -v -d ${gitian_sigs_repo_path}/ -r ${VERSION}_${suite} ${suite_dir_path}/gitian-linux-parallel.yml
|
||||
done
|
||||
|
||||
popd
|
||||
fi
|
||||
Loading…
Reference in New Issue