From c897b1e732f524e525d1bd150326b7f0632c1b69 Mon Sep 17 00:00:00 2001 From: Cory Fields Date: Fri, 22 Aug 2014 15:41:38 -0400 Subject: [PATCH 1/2] depends: add a fallback path in case package sources go missing If a source url fails to download, try again at $FALLBACK_DOWNLOAD_PATH/file.name, where FALLBACK_DOWNLOAD_PATH can be overridden by the user. --- depends/Makefile | 1 + depends/README.usage | 1 + depends/funcs.mk | 14 ++++++++------ depends/packages/native_cctools.mk | 10 +++++----- depends/packages/native_comparisontool.mk | 12 ++++++------ 5 files changed, 21 insertions(+), 17 deletions(-) diff --git a/depends/Makefile b/depends/Makefile index bd5f0bf5..8075c66b 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -6,6 +6,7 @@ SDK_PATH ?= $(BASEDIR)/SDKs NO_QT ?= NO_WALLET ?= NO_UPNP ?= +FALLBACK_DOWNLOAD_PATH ?= https://bitcoincore.org/depends-sources BUILD = $(shell ./config.guess) HOST ?= $(BUILD) diff --git a/depends/README.usage b/depends/README.usage index f50714f1..0aacefbf 100644 --- a/depends/README.usage +++ b/depends/README.usage @@ -18,6 +18,7 @@ The following can be set when running make: make FOO=bar SOURCES_PATH: downloaded sources will be placed here BASE_CACHE: built packages will be placed here SDK_PATH: Path where sdk's can be found (used by OSX) +FALLBACK_DOWNLOAD_PATH: If a source file can't be fetched, try here before giving up NO_QT: Don't download/build/cache qt and its dependencies NO_WALLET: Don't download/build/cache libs needed to enable the wallet NO_UPNP: Don't download/build/cache packages needed for enabling upnp diff --git a/depends/funcs.mk b/depends/funcs.mk index b5d8b0ee..280706ef 100644 --- a/depends/funcs.mk +++ b/depends/funcs.mk @@ -20,11 +20,13 @@ $(sort $(foreach dep,$(2),$(2) $(call int_get_all_dependencies,$(1),$($(dep)_dep endef define fetch_file -(test -f $(SOURCES_PATH)/$(3) || \ - ( mkdir -p $$($(1)_extract_dir) && $(build_DOWNLOAD) "$$($(1)_extract_dir)/$(3).temp" "$(2)" && \ - echo "$(4) $$($(1)_extract_dir)/$(3).temp" > $$($(1)_extract_dir)/.$(3).hash && \ - $(build_SHA256SUM) -c $$($(1)_extract_dir)/.$(3).hash && \ - mv $$($(1)_extract_dir)/$(3).temp $(SOURCES_PATH)/$(3) )) +(test -f $(SOURCES_PATH)/$(4) || \ + ( mkdir -p $$($(1)_extract_dir) && \ + ( $(build_DOWNLOAD) "$$($(1)_extract_dir)/$(3).temp" "$(2)/$(3)" || \ + $(build_DOWNLOAD) "$$($(1)_extract_dir)/$(3).temp" "$(FALLBACK_DOWNLOAD_PATH)/$(3)" ) && \ + echo "$(5) $$($(1)_extract_dir)/$(4).temp" > $$($(1)_extract_dir)/.$(4).hash && \ + $(build_SHA256SUM) -c $$($(1)_extract_dir)/.$(4).hash && \ + mv $$($(1)_extract_dir)/$(4).temp $(SOURCES_PATH)/$(4) )) endef define int_get_build_recipe_hash @@ -65,7 +67,7 @@ $(1)_download_path_fixed=$(subst :,\:,$$($(1)_download_path)) #default commands -$(1)_fetch_cmds ?= $(call fetch_file,$(1),$(subst \:,:,$$($(1)_download_path_fixed)/$$($(1)_download_file)),$($(1)_file_name),$($(1)_sha256_hash)) +$(1)_fetch_cmds ?= $(call fetch_file,$(1),$(subst \:,:,$$($(1)_download_path_fixed)),$$($(1)_download_file),$($(1)_file_name),$($(1)_sha256_hash)) $(1)_extract_cmds ?= mkdir -p $$($(1)_extract_dir) && echo "$$($(1)_sha256_hash) $$($(1)_source)" > $$($(1)_extract_dir)/.$$($(1)_file_name).hash && $(build_SHA256SUM) -c $$($(1)_extract_dir)/.$$($(1)_file_name).hash && tar --strip-components=1 -xf $$($(1)_source) $(1)_preprocess_cmds ?= $(1)_build_cmds ?= diff --git a/depends/packages/native_cctools.mk b/depends/packages/native_cctools.mk index e3ba9685..ad989cb5 100644 --- a/depends/packages/native_cctools.mk +++ b/depends/packages/native_cctools.mk @@ -23,11 +23,11 @@ $(package)_clang_file_name=clang-llvm-3.2-x86-linux-ubuntu-12.04.tar.gz $(package)_clang_sha256_hash=b9d57a88f9514fa1f327a1a703756d0c1c960f4c58494a5bd80313245d13ffff define $(package)_fetch_cmds -$(call fetch_file,$(package),$($(package)_download_path)/$($(package)_download_file),$($(package)_file_name),$($(package)_sha256_hash)) && \ -$(call fetch_file,$(package),$($(package)_ld64_download_path)/$($(package)_ld64_download_file),$($(package)_ld64_file_name),$($(package)_ld64_sha256_hash)) && \ -$(call fetch_file,$(package),$($(package)_dyld_download_path)/$($(package)_dyld_download_file),$($(package)_dyld_file_name),$($(package)_dyld_sha256_hash)) && \ -$(call fetch_file,$(package),$($(package)_clang_download_path)/$($(package)_clang_download_file),$($(package)_clang_file_name),$($(package)_clang_sha256_hash)) && \ -$(call fetch_file,$(package),$($(package)_toolchain4_download_path)/$($(package)_toolchain4_download_file),$($(package)_toolchain4_file_name),$($(package)_toolchain4_sha256_hash)) +$(call fetch_file,$(package),$($(package)_download_path),$($(package)_download_file),$($(package)_file_name),$($(package)_sha256_hash)) && \ +$(call fetch_file,$(package),$($(package)_ld64_download_path),$($(package)_ld64_download_file),$($(package)_ld64_file_name),$($(package)_ld64_sha256_hash)) && \ +$(call fetch_file,$(package),$($(package)_dyld_download_path),$($(package)_dyld_download_file),$($(package)_dyld_file_name),$($(package)_dyld_sha256_hash)) && \ +$(call fetch_file,$(package),$($(package)_clang_download_path),$($(package)_clang_download_file),$($(package)_clang_file_name),$($(package)_clang_sha256_hash)) && \ +$(call fetch_file,$(package),$($(package)_toolchain4_download_path),$($(package)_toolchain4_download_file),$($(package)_toolchain4_file_name),$($(package)_toolchain4_sha256_hash)) endef define $(package)_set_vars diff --git a/depends/packages/native_comparisontool.mk b/depends/packages/native_comparisontool.mk index 5eef7305..003923d7 100644 --- a/depends/packages/native_comparisontool.mk +++ b/depends/packages/native_comparisontool.mk @@ -15,12 +15,12 @@ $(package)_slf4j-jdk14_file_name=slf4j-jdk14-1.6.4.jar $(package)_slf4j-jdk14_sha256_hash=064bd81796710f713f9f4a2309c0e032309934c2d2b4f7d3b6958325e584e13f define $(package)_fetch_cmds -$(call fetch_file,$(package),$($(package)_download_path)/$($(package)_file_name),$($(package)_file_name),$($(package)_sha256_hash)) && \ -$(call fetch_file,$(package),$($(package)_download_path)/$($(package)_guava_file_name),$($(package)_guava_file_name),$($(package)_guava_sha256_hash)) && \ -$(call fetch_file,$(package),$($(package)_download_path)/$($(package)_h2_file_name),$($(package)_h2_file_name),$($(package)_h2_sha256_hash)) && \ -$(call fetch_file,$(package),$($(package)_download_path)/$($(package)_sc-light-jdk15on_file_name),$($(package)_sc-light-jdk15on_file_name),$($(package)_sc-light-jdk15on_sha256_hash)) && \ -$(call fetch_file,$(package),$($(package)_download_path)/$($(package)_slf4j-api_file_name),$($(package)_slf4j-api_file_name),$($(package)_slf4j-api_sha256_hash)) && \ -$(call fetch_file,$(package),$($(package)_download_path)/$($(package)_slf4j-jdk14_file_name),$($(package)_slf4j-jdk14_file_name),$($(package)_slf4j-jdk14_sha256_hash)) +$(call fetch_file,$(package),$($(package)_download_path),$($(package)_file_name),$($(package)_file_name),$($(package)_sha256_hash)) && \ +$(call fetch_file,$(package),$($(package)_download_path),$($(package)_guava_file_name),$($(package)_guava_file_name),$($(package)_guava_sha256_hash)) && \ +$(call fetch_file,$(package),$($(package)_download_path),$($(package)_h2_file_name),$($(package)_h2_file_name),$($(package)_h2_sha256_hash)) && \ +$(call fetch_file,$(package),$($(package)_download_path),$($(package)_sc-light-jdk15on_file_name),$($(package)_sc-light-jdk15on_file_name),$($(package)_sc-light-jdk15on_sha256_hash)) && \ +$(call fetch_file,$(package),$($(package)_download_path),$($(package)_slf4j-api_file_name),$($(package)_slf4j-api_file_name),$($(package)_slf4j-api_sha256_hash)) && \ +$(call fetch_file,$(package),$($(package)_download_path),$($(package)_slf4j-jdk14_file_name),$($(package)_slf4j-jdk14_file_name),$($(package)_slf4j-jdk14_sha256_hash)) endef define $(package)_extract_cmds From df7565d99c4baca212db6bf1d75b097a7ec58475 Mon Sep 17 00:00:00 2001 From: Cory Fields Date: Wed, 27 Aug 2014 03:40:30 -0400 Subject: [PATCH 2/2] depends: add sensible download timeout/retry values --- depends/Makefile | 2 ++ depends/builders/darwin.mk | 2 +- depends/builders/linux.mk | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/depends/Makefile b/depends/Makefile index 8075c66b..f5fb5b86 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -13,6 +13,8 @@ HOST ?= $(BUILD) PATCHES_PATH = $(BASEDIR)/patches BASEDIR = $(CURDIR) HASH_LENGTH:=11 +DOWNLOAD_CONNECT_TIMEOUT:=10 +DOWNLOAD_RETRIES:=3 host:=$(BUILD) ifneq ($(HOST),) diff --git a/depends/builders/darwin.mk b/depends/builders/darwin.mk index 6b734648..b366460e 100644 --- a/depends/builders/darwin.mk +++ b/depends/builders/darwin.mk @@ -7,7 +7,7 @@ build_darwin_OTOOL: = $(shell xcrun -f otool) build_darwin_NM: = $(shell xcrun -f nm) build_darwin_INSTALL_NAME_TOOL:=$(shell xcrun -f install_name_tool) build_darwin_SHA256SUM = shasum -a 256 -build_darwin_DOWNLOAD = curl -L -o +build_darwin_DOWNLOAD = curl --connect-timeout $(DOWNLOAD_CONNECT_TIMEOUT) --retry $(DOWNLOAD_RETRIES) -L -o #darwin host on darwin builder. overrides darwin host preferences. darwin_CC=$(shell xcrun -f clang) -mmacosx-version-min=$(OSX_MIN_VERSION) diff --git a/depends/builders/linux.mk b/depends/builders/linux.mk index d98ba597..98d0e9de 100644 --- a/depends/builders/linux.mk +++ b/depends/builders/linux.mk @@ -1,2 +1,2 @@ build_linux_SHA256SUM = sha256sum -build_linux_DOWNLOAD = wget -nv -O +build_linux_DOWNLOAD = wget --timeout=$(DOWNLOAD_CONNECT_TIMEOUT) --tries=$(DOWNLOAD_RETRIES) -nv -O