Auto merge of #4148 - str4d:macos-cross-compile, r=mdr0id

Enable macOS cross-compilation

Includes code cherry-picked from the following upstream PRs:

- bitcoin/bitcoin#7809
  - The `native_cctools` fix.
- bitcoin/bitcoin#8210
  - The macOS toolchain bump.
- bitcoin/bitcoin#9891
- bitcoin/bitcoin#15581
  - The `tar` change.
- bitcoin/bitcoin#16049
  - The `native_cctools` change.

Build instructions:
- Fetch `MacOSX10.11.sdk` from e.g. https://github.com/phracker/MacOSX-SDKs/releases
- Extract it into `depends/SDKs` (creating that folder first)
- `HOST=x86_64-apple-darwin11 ./zcutil/build.sh`

configure.ac

@@ -494,7 +494,7 @@ if test x$use_hardening != xno; then
     HARDENED_CPPFLAGS="$HARDENED_CPPFLAGS -D_FORTIFY_SOURCE=2"
   ],[AC_MSG_ERROR(Cannot enable -D_FORTIFY_SOURCE=2)])
 
-  if test x$BUILD_OS = xdarwin || test x$TARGET_OS = xwindows; then
+  if test x$TARGET_OS = xdarwin || test x$TARGET_OS = xwindows; then
     # Xcode's ld (at least ld64-302.3) doesn't support -z
     # mingw-w64's ld (at least mingw-w64 4.0.4-2) also appears to not support -z
     AX_CHECK_LINK_FLAG([[-Wl,-z,relro]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-z,relro"],[AC_MSG_WARN(Cannot enable RELRO)])

depends/funcs.mk

@@ -91,8 +91,9 @@ $(1)_download_path_fixed=$(subst :,\:,$$($(1)_download_path))
 
 
 #default commands
+# The default behavior for tar will try to set ownership when running as uid 0 and may not succeed, --no-same-owner disables this behavior
 $(1)_fetch_cmds ?= $(call fetch_file,$(1),$(subst \:,:,$$($(1)_download_path_fixed)),$$($(1)_download_file),$($(1)_file_name),$($(1)_sha256_hash))
-$(1)_extract_cmds ?= mkdir -p $$($(1)_extract_dir) && echo "$$($(1)_sha256_hash)  $$($(1)_source)" > $$($(1)_extract_dir)/.$$($(1)_file_name).hash &&  $(build_SHA256SUM) -c $$($(1)_extract_dir)/.$$($(1)_file_name).hash && tar --strip-components=1 -xf $$($(1)_source)
+$(1)_extract_cmds ?= mkdir -p $$($(1)_extract_dir) && echo "$$($(1)_sha256_hash)  $$($(1)_source)" > $$($(1)_extract_dir)/.$$($(1)_file_name).hash &&  $(build_SHA256SUM) -c $$($(1)_extract_dir)/.$$($(1)_file_name).hash && tar --no-same-owner --strip-components=1 -xf $$($(1)_source)
 $(1)_preprocess_cmds ?=
 $(1)_build_cmds ?=
 $(1)_config_cmds ?=
@@ -193,7 +194,7 @@ $($(1)_preprocessed): | $($(1)_dependencies) $($(1)_extracted)
 	$(AT)touch $$@
 $($(1)_configured): | $($(1)_preprocessed)
 	$(AT)echo Configuring $(1)...
-	$(AT)rm -rf $(host_prefix); mkdir -p $(host_prefix)/lib; cd $(host_prefix); $(foreach package,$($(1)_all_dependencies), tar xf $($(package)_cached); )
+	$(AT)rm -rf $(host_prefix); mkdir -p $(host_prefix)/lib; cd $(host_prefix); $(foreach package,$($(1)_all_dependencies), tar --no-same-owner -xf $($(package)_cached); )
 	$(AT)mkdir -p $$(@D)
 	$(AT)+cd $$(@D); $($(1)_config_env) $(call $(1)_config_cmds, $(1))
 	$(AT)touch $$@

depends/packages.md

@@ -14,8 +14,9 @@ Each package is required to define at least these variables:
     placeholder such as 1.0 can be used.
 
     $(package)_download_path:
-    Location of the upstream source, without the file-name. Usually http or
-    ftp.
+    Location of the upstream source, without the file-name. Usually http, https
+    or ftp. Secure transmission options like https should be preferred if
+    available.
 
     $(package)_file_name:
     The upstream source filename available at the download path.

depends/packages/bdb.mk

@@ -10,6 +10,9 @@ $(package)_config_opts=--disable-shared --enable-cxx --disable-replication
 $(package)_config_opts_mingw32=--enable-mingw
 $(package)_config_opts_linux=--with-pic
 $(package)_config_opts_freebsd=--with-pic
+ifneq ($(build_os),darwin)
+$(package)_config_opts_darwin=--disable-atomicsupport
+endif
 $(package)_config_opts_aarch64=--disable-atomicsupport
 $(package)_cxxflags=-std=c++11
 endef

depends/packages/boost.mk

@@ -29,7 +29,7 @@ define $(package)_preprocess_cmds
 endef
 
 define $(package)_config_cmds
-  ./bootstrap.sh --without-icu --with-toolset=$($(package)_toolset_$(host_os)) --with-libraries=$($(package)_config_libraries) && \
+  ./bootstrap.sh --without-icu --with-libraries=$($(package)_config_libraries) && \
   sed -i -e "s|using gcc ;|using $(boost_toolset_$(host_os)) : : $($(package)_cxx) : <cxxflags>\"$($(package)_cxxflags) $($(package)_cppflags)\" <linkflags>\"$($(package)_ldflags)\" <archiver>\"$(boost_archiver_$(host_os))\" <striper>\"$(host_STRIP)\"  <ranlib>\"$(host_RANLIB)\" <rc>\"$(host_WINDRES)\" : ;|" project-config.jam
 endef
 

depends/packages/googletest.mk

@@ -13,7 +13,7 @@ endef
 
 define $(package)_build_cmds
   $(MAKE) -C googlemock/make CC="$($(package)_cc)" CXX="$($(package)_cxx)" AR="$($(package)_ar)" CXXFLAGS="$($(package)_cxxflags)" gmock.a && \
-  $(MAKE) -C googletest/make CC="$($(package)_cc)" CXX="$($(package)_cxx)" AR="$($(package)_ar)" CXXFLAGS="$($(package)_cxxflags)" gtest.a
+  $($(package)_build_env) $(MAKE) -C googletest/make CC="$($(package)_cc)" CXX="$($(package)_cxx)" AR="$($(package)_ar)" CXXFLAGS="$($(package)_cxxflags)" gtest.a
 endef
 
 define $(package)_stage_cmds

depends/packages/native_cctools.mk

@@ -0,0 +1,65 @@
+package=native_cctools
+$(package)_version=807d6fd1be5d2224872e381870c0a75387fe05e6
+$(package)_download_path=https://github.com/theuni/cctools-port/archive
+$(package)_file_name=$($(package)_version).tar.gz
+$(package)_sha256_hash=a09c9ba4684670a0375e42d9d67e7f12c1f62581a27f28f7c825d6d7032ccc6a
+$(package)_build_subdir=cctools
+$(package)_clang_version=3.7.1
+$(package)_clang_download_path=https://llvm.org/releases/$($(package)_clang_version)
+$(package)_clang_download_file=clang+llvm-$($(package)_clang_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz
+$(package)_clang_file_name=clang-llvm-$($(package)_clang_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz
+$(package)_clang_sha256_hash=99b28a6b48e793705228a390471991386daa33a9717cd9ca007fcdde69608fd9
+$(package)_extra_sources=$($(package)_clang_file_name)
+
+define $(package)_fetch_cmds
+$(call fetch_file,$(package),$($(package)_download_path),$($(package)_download_file),$($(package)_file_name),$($(package)_sha256_hash)) && \
+$(call fetch_file,$(package),$($(package)_clang_download_path),$($(package)_clang_download_file),$($(package)_clang_file_name),$($(package)_clang_sha256_hash))
+endef
+
+define $(package)_extract_cmds
+  mkdir -p $($(package)_extract_dir) && \
+  echo "$($(package)_sha256_hash)  $($(package)_source)" > $($(package)_extract_dir)/.$($(package)_file_name).hash && \
+  echo "$($(package)_clang_sha256_hash)  $($(package)_source_dir)/$($(package)_clang_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \
+  $(build_SHA256SUM) -c $($(package)_extract_dir)/.$($(package)_file_name).hash && \
+  mkdir -p toolchain/bin toolchain/lib/clang/3.5/include && \
+  tar --no-same-owner --strip-components=1 -C toolchain -xf $($(package)_source_dir)/$($(package)_clang_file_name) && \
+  rm -f toolchain/lib/libc++abi.so* && \
+  echo "#!/bin/sh" > toolchain/bin/$(host)-dsymutil && \
+  echo "exit 0" >> toolchain/bin/$(host)-dsymutil && \
+  chmod +x toolchain/bin/$(host)-dsymutil && \
+  tar --no-same-owner --strip-components=1 -xf $($(package)_source)
+endef
+
+define $(package)_set_vars
+$(package)_config_opts=--target=$(host) --disable-lto-support
+$(package)_ldflags+=-Wl,-rpath=\\$$$$$$$$\$$$$$$$$ORIGIN/../lib
+$(package)_cc=$($(package)_extract_dir)/toolchain/bin/clang
+$(package)_cxx=$($(package)_extract_dir)/toolchain/bin/clang++
+endef
+
+define $(package)_preprocess_cmds
+  cd $($(package)_build_subdir); ./autogen.sh && \
+  sed -i.old "/define HAVE_PTHREADS/d" ld64/src/ld/InputFiles.h
+endef
+
+define $(package)_config_cmds
+  $($(package)_autoconf)
+endef
+
+define $(package)_build_cmds
+  $(MAKE)
+endef
+
+define $(package)_stage_cmds
+  $(MAKE) DESTDIR=$($(package)_staging_dir) install && \
+  cd $($(package)_extract_dir)/toolchain && \
+  mkdir -p $($(package)_staging_prefix_dir)/lib/clang/$($(package)_clang_version)/include && \
+  mkdir -p $($(package)_staging_prefix_dir)/bin $($(package)_staging_prefix_dir)/include && \
+  cp bin/clang $($(package)_staging_prefix_dir)/bin/ &&\
+  cp -P bin/clang++ $($(package)_staging_prefix_dir)/bin/ &&\
+  cp lib/libLTO.so $($(package)_staging_prefix_dir)/lib/ && \
+  cp -rf lib/clang/$($(package)_clang_version)/include/* $($(package)_staging_prefix_dir)/lib/clang/$($(package)_clang_version)/include/ && \
+  cp bin/llvm-dsymutil $($(package)_staging_prefix_dir)/bin/$(host)-dsymutil && \
+  if `test -d include/c++/`; then cp -rf include/c++/ $($(package)_staging_prefix_dir)/include/; fi && \
+  if `test -d lib/c++/`; then cp -rf lib/c++/ $($(package)_staging_prefix_dir)/lib/; fi
+endef

depends/packages/packages.mk

@@ -72,3 +72,7 @@ packages := boost openssl libevent zeromq $(zcash_packages) googletest
 native_packages := native_ccache
 
 wallet_packages=bdb
+
+ifneq ($(build_os),darwin)
+darwin_native_packages=native_cctools
+endif

depends/packages/rust.mk

@@ -10,10 +10,12 @@ $(package)_sha256_hash_freebsd=eeeb1e9d0d7823c55f00f434789696e7249f465ba5966a5ab
 
 # Mapping from GCC canonical hosts to Rust targets
 # If a mapping is not present, we assume they are identical
+$(package)_rust_target_x86_64-apple-darwin11=x86_64-apple-darwin
 $(package)_rust_target_x86_64-w64-mingw32=x86_64-pc-windows-gnu
 
 # Mapping from Rust targets to SHA-256 hashes
 $(package)_rust_std_sha256_hash_aarch64-unknown-linux-gnu=22bfc32b5003c3d5259babb202f3f66be16fa6f3c75c20f429a16d7ef5eb1928
+$(package)_rust_std_sha256_hash_x86_64-apple-darwin=7c6806809e010e5fba1780007ecff5c31f0ad2fcac1b414b98ca3baa0fb41b36
 $(package)_rust_std_sha256_hash_x86_64-pc-windows-gnu=3657c361f5d6048c2cb814f1773fece07b840276d60f012d0bf70993fa95a77e
 
 ifneq ($(canonical_host),$(build))