Merge pull request #5465

e27d7cb docs: release process fixups (Cory Fields)
d7db4b6 depends: fix packages with hard-coded SOURCES_PATH (Cory Fields)
d546191 depends: Move source stamps to source dir and misc cleanups (Cory Fields)
422f873 depends: teach 'make download' to download sources for all hosts (Cory Fields)

depends/Makefile

@@ -31,6 +31,7 @@ endif
 
 base_build_dir=$(BASEDIR)/work/build
 base_staging_dir=$(BASEDIR)/work/staging
+base_download_dir=$(BASEDIR)/work/download
 canonical_host:=$(shell ./config.sub $(HOST))
 build:=$(shell ./config.sub $(BUILD))
 
@@ -122,5 +123,12 @@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_
 	$(AT)touch $@
 
 install: $(host_prefix)/share/config.site
-download: $(all_sources)
-.PHONY: install cached
+download-one: $(all_sources)
+download-osx:
+	@$(MAKE) -s HOST=x86_64-apple-darwin11 download-one
+download-linux:
+	@$(MAKE) -s HOST=x86_64-unknown-linux-gnu download-one
+download-win:
+	@$(MAKE) -s HOST=x86_64-w64-mingw32 download-one
+download: download-osx download-linux download-win
+.PHONY: install cached download-one download-osx download-linux download-win download

depends/README.usage

@@ -29,4 +29,7 @@ If some packages are not built, for example 'make NO_WALLET=1', the appropriate
 options will be passed to bitcoin's configure. In this case, --disable-wallet.
 
 Additional targets:
-download: run 'make download' to fetch sources without building them
+download: run 'make download' to fetch all sources without building them
+download-osx: run 'make download-osx' to fetch all sources needed for osx builds
+download-win: run 'make download-win' to fetch all sources needed for win builds
+download-linux: run 'make download-linux' to fetch all sources needed for linux builds

depends/funcs.mk

@@ -20,18 +20,19 @@ $(sort $(foreach dep,$(2),$(2) $(call int_get_all_dependencies,$(1),$($(dep)_dep
 endef
 
 define fetch_file
-(test -f $(SOURCES_PATH)/$(4) || \
-  ( mkdir -p $$($(1)_extract_dir) && \
-  ( $(build_DOWNLOAD) "$$($(1)_extract_dir)/$(4).temp" "$(2)/$(3)" || \
-    $(build_DOWNLOAD) "$$($(1)_extract_dir)/$(4).temp" "$(FALLBACK_DOWNLOAD_PATH)/$(3)" ) && \
-    echo "$(5)  $$($(1)_extract_dir)/$(4).temp" > $$($(1)_extract_dir)/.$(4).hash && \
-    $(build_SHA256SUM) -c $$($(1)_extract_dir)/.$(4).hash && \
-    mv $$($(1)_extract_dir)/$(4).temp $(SOURCES_PATH)/$(4) ))
+(test -f $$($(1)_source_dir)/$(4) || \
+  ( mkdir -p $$($(1)_download_dir) && echo Fetching $(1)... && \
+  ( $(build_DOWNLOAD) "$$($(1)_download_dir)/$(4).temp" "$(2)/$(3)" || \
+    $(build_DOWNLOAD) "$$($(1)_download_dir)/$(4).temp" "$(FALLBACK_DOWNLOAD_PATH)/$(3)" ) && \
+    echo "$(5)  $$($(1)_download_dir)/$(4).temp" > $$($(1)_download_dir)/.$(4).hash && \
+    $(build_SHA256SUM) -c $$($(1)_download_dir)/.$(4).hash && \
+    mv $$($(1)_download_dir)/$(4).temp $$($(1)_source_dir)/$(4) && \
+    rm -rf $$($(1)_download_dir) ))
 endef
 
 define int_get_build_recipe_hash
 $(eval $(1)_all_file_checksums:=$(shell $(build_SHA256SUM) $(meta_depends) packages/$(1).mk $(addprefix $(PATCHES_PATH)/$(1)/,$($(1)_patches))))
-$(eval $(1)_recipe_hash:=$(shell echo -n "$($(1)_all_file_checksums)" | $(build_SHA256SUM)))
+$(eval $(1)_recipe_hash:=$(shell echo -n "$($(1)_all_file_checksums)" | cut -d" " -f1 | $(build_SHA256SUM)))
 endef
 
 define int_get_build_id
@@ -45,17 +46,19 @@ final_build_id_long+=$($(package)_build_id_long)
 #compute package-specific paths
 $(1)_build_subdir?=.
 $(1)_download_file?=$($(1)_file_name)
-$(1)_source:=$(SOURCES_PATH)/$($(1)_file_name)
+$(1)_source_dir:=$(SOURCES_PATH)
+$(1)_source:=$$($(1)_source_dir)/$($(1)_file_name)
 $(1)_staging_dir=$(base_staging_dir)/$(host)/$(1)/$($(1)_version)-$($(1)_build_id)
 $(1)_staging_prefix_dir:=$$($(1)_staging_dir)$($($(1)_type)_prefix)
 $(1)_extract_dir:=$(base_build_dir)/$(host)/$(1)/$($(1)_version)-$($(1)_build_id)
+$(1)_download_dir:=$(base_download_dir)/$(1)-$($(1)_version)
 $(1)_build_dir:=$$($(1)_extract_dir)/$$($(1)_build_subdir)
 $(1)_patch_dir:=$(base_build_dir)/$(host)/$(1)/$($(1)_version)-$($(1)_build_id)/.patches-$($(1)_build_id)
 $(1)_prefixbin:=$($($(1)_type)_prefix)/bin/
 $(1)_cached:=$(BASE_CACHE)/$(host)/$(1)/$(1)-$($(1)_version)-$($(1)_build_id).tar.gz
 
 #stamps
-$(1)_fetched=$$($(1)_extract_dir)/.stamp_fetched
+$(1)_fetched=$$($(1)_source_dir)/download-stamps/.stamp_fetched-$(1)-$($(1)_file_name)
 $(1)_extracted=$$($(1)_extract_dir)/.stamp_extracted
 $(1)_preprocessed=$$($(1)_extract_dir)/.stamp_preprocessed
 $(1)_cleaned=$$($(1)_extract_dir)/.stamp_cleaned
@@ -150,7 +153,6 @@ endef
 
 define int_add_cmds
 $($(1)_fetched):
-	$(AT)echo Fetching $(1)...
 	$(AT)mkdir -p $$(@D) $(SOURCES_PATH)
 	$(AT)cd $$(@D); $(call $(1)_fetch_cmds,$(1))
 	$(AT)touch $$@

depends/packages/native_cctools.mk

@@ -39,11 +39,11 @@ $(package)_ldflags+=-m32 -Wl,-rpath=\\$$$$$$$$\$$$$$$$$ORIGIN/../lib
 $(package)_ldflags+=-L$$(native_cctools_extract_dir)/clang+llvm-3.2-x86-linux-ubuntu-12.04/lib
 endef
 define $(package)_extract_cmds
-  tar --strip-components=1 -xf $(SOURCES_PATH)/$($(package)_toolchain4_file_name) && \
+  tar --strip-components=1 -xf $($(package)_source_dir)/$($(package)_toolchain4_file_name) && \
   ln -sf $($(package)_source) cctools2odcctools/$($(package)_file_name) && \
-  ln -sf $(SOURCES_PATH)/$($(package)_ld64_file_name) cctools2odcctools/$($(package)_ld64_file_name) && \
-  ln -sf $(SOURCES_PATH)/$($(package)_dyld_file_name) cctools2odcctools/$($(package)_dyld_file_name) && \
-  tar xf $(SOURCES_PATH)/$($(package)_clang_file_name) && \
+  ln -sf $($(package)_source_dir)/$($(package)_ld64_file_name) cctools2odcctools/$($(package)_ld64_file_name) && \
+  ln -sf $($(package)_source_dir)/$($(package)_dyld_file_name) cctools2odcctools/$($(package)_dyld_file_name) && \
+  tar xf $($(package)_source_dir)/$($(package)_clang_file_name) && \
   mkdir -p $(SDK_PATH) sdks &&\
   cd sdks; ln -sf $(OSX_SDK) MacOSX$(OSX_SDK_VERSION).sdk
 endef

depends/packages/native_comparisontool.mk

@@ -17,5 +17,5 @@ endef
 
 define $(package)_stage_cmds
   mkdir -p $($(package)_staging_prefix_dir)/share/$($(package)_install_dirname) && \
-  mv $(SOURCES_PATH)/$($(package)_file_name) $($(package)_staging_prefix_dir)/share/$($(package)_install_dirname)/$($(package)_install_filename)
+  cp $($(package)_source) $($(package)_staging_prefix_dir)/share/$($(package)_install_dirname)/$($(package)_install_filename)
 endef

doc/release-process.md

@@ -40,7 +40,7 @@ Release Process
 
 ###fetch and build inputs: (first time, or when dependency versions change)
  
-	mkdir -p inputs; cd inputs/
+	mkdir -p inputs
 
  Register and download the Apple SDK: (see OSX Readme for details)
  
@@ -50,7 +50,15 @@ Release Process
  
 	tar -C /Volumes/Xcode/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/ -czf MacOSX10.7.sdk.tar.gz MacOSX10.7.sdk
 
- Build Bitcoin Core for Linux, Windows, and OS X:
+###Optional: Seed the Gitian sources cache
+
+  By default, gitian will fetch source files as needed. For offline builds, they can be fetched ahead of time:
+
+	make -C ../bitcoin/depends download SOURCES_PATH=`pwd`/cache/common
+
+  Only missing files will be fetched, so this is safe to re-run for each build.
+
+###Build Bitcoin Core for Linux, Windows, and OS X:
   
 	./bin/gbuild --commit bitcoin=v${VERSION} ../bitcoin/contrib/gitian-descriptors/gitian-linux.yml
 	./bin/gsign --signer $SIGNER --release ${VERSION}-linux --destination ../gitian.sigs/ ../bitcoin/contrib/gitian-descriptors/gitian-linux.yml
@@ -63,7 +71,6 @@ Release Process
 	mv build/out/bitcoin-*-unsigned.tar.gz inputs
 	mv build/out/bitcoin-*.tar.gz build/out/bitcoin-*.dmg ../
 	popd
-bitcoin-0.9.99-osx-unsigned.tar.gz
   Build output expected:
 
   1. source tarball (bitcoin-${VERSION}.tar.gz)
@@ -84,11 +91,12 @@ Commit your signature to gitian.sigs:
 	git push  # Assuming you can push to the gitian.sigs tree
 	popd
 
-Wait for OSX detached signature:
+  Wait for OSX detached signature:
 	Once the OSX build has 3 matching signatures, Gavin will sign it with the apple App-Store key.
 	He will then upload a detached signature to be combined with the unsigned app to create a signed binary.
 
-Create the signed OSX binary:
+  Create the signed OSX binary:
+
 	pushd ./gitian-builder
 	# Fetch the signature as instructed by Gavin
 	cp signature.tar.gz inputs/