Travis optimizations (#1496)

* add travis directorie cache for arm sdk

* Travis. Test if make all in one job would work. And how slow/fast.

* Travis. Check that the old method with a couple specific targets still works.

* Rebased master and build all targets again.

* Make verbosity level 0 even less verbose, was still too much for travis in one mega-jumbo-all-target build.

* Even less output from make in verbosity level 0. Some Makefile symbol name cleanup too.

* arm_sdk_install did not check if allready downloaded and installed. Added checks and deps. Travis caching might work better now...

* Build all 56 targets to test Travis caching perf.

* Build all 56 targets to test Travis caching perf. Again. Now with correct yml arrsy syntax.

* Added installation of ccache and also to Travis cache. Caching the cache.

* Travis. Remove some unused APT libs, to see if this affects caching and build times.

* Travis. YAPT (Yet another perf trial). Remove all apt package install.

* Disaster. Put back installation if build-essential.

* Disaster still. Put back installation if zliblg-dev.

* Disaster, again. Put back installation of libc6 .

* Travis. Minimizing APT installation as it not be cached, issue 5876 on travis-ci github.

* Travis. Added comment on target specification.

.travis.sh

@@ -50,7 +50,7 @@ elif [ $PUBLISHMETA ] ; then
 		curl -k "${CURL_BASEOPTS[@]}" "${CURL_PUB_BASEOPTS[@]}" --form "recent_commits=${RECENT_COMMITS}" ${PUBLISH_URL} || true
 	fi
 
-else
+elif [ $TARGET ] ; then
 	if [ $PUBLISH_URL ] ; then
 		make -j2 $MAKEFILE
 		if   [ -f ${TARGET_FILE}.bin ] ; then
@@ -67,4 +67,7 @@ else
 	else
 		make -j2 $MAKEFILE
 	fi
+else
+# No target specified, build all with very low verbosity.
+    make V=0 all
 fi

.travis.yml

@@ -1,20 +1,23 @@
+
 env:
+# Specify target(s) to build, or none to build all.
 #  - RUNTESTS=True
 #  - PUBLISHMETA=True
 #  - PUBLISHDOCS=True
 #  - TARGET=AFROMINI
-#  - TARGET=BEEBRAIN
 #  - TARGET=AIORACERF3
 #  - TARGET=AIR32
+#  - TARGET=AIRBOTF4
 #  - TARGET=AIRHEROF3
 #  - TARGET=ALIENFLIGHTF1
-  - TARGET=ALIENFLIGHTF3
-  - TARGET=ALIENFLIGHTF4
-  - TARGET=ANYFCF7
-  - TARGET=BETAFLIGHTF3
-  - TARGET=BLUEJAYF4
-  - TARGET=CC3D
-  - TARGET=CC3D_OPBL
+#  - TARGET=ALIENFLIGHTF3
+#  - TARGET=ALIENFLIGHTF4
+#  - TARGET=ANYFCF7
+#  - TARGET=BEEBRAIN
+#  - TARGET=BETAFLIGHTF3
+#  - TARGET=BLUEJAYF4
+#  - TARGET=CC3D
+#  - TARGET=CC3D_OPBL
 #  - TARGET=CHEBUZZF3
 #  - TARGET=CJMCU
 #  - TARGET=COLIBRI
@@ -23,50 +26,51 @@ env:
 #  - TARGET=DOGE
 #  - TARGET=F4BY
 #  - TARGET=FURYF3
-  - TARGET=FURYF4
+#  - TARGET=FURYF4
+#  - TARGET=FURYF7
+#  - TARGET=IMPULSERCF3
 #  - TARGET=IRCFUSIONF3
 #  - TARGET=ISHAPEDF3
 #  - TARGET=KISSFC
+#  - TARGET=LUXV2_RACE
 #  - TARGET=LUX_RACE
 #  - TARGET=MICROSCISKY
 #  - TARGET=MOTOLAB
-  - TARGET=NAZE
+#  - TARGET=NAZE
 #  - TARGET=OMNIBUS
 #  - TARGET=OMNIBUSF4
 #  - TARGET=PIKOBLX
 #  - TARGET=RACEBASE
-  - TARGET=REVO
+#  - TARGET=RCEXPLORERF3
+#  - TARGET=REVO
+#  - TARGET=REVOLT
 #  - TARGET=REVONANO
 #  - TARGET=REVO_OPBL
 #  - TARGET=RMDO
 #  - TARGET=SINGULARITY
-  - TARGET=SIRINFPV
-  - TARGET=SPARKY
+#  - TARGET=SIRINFPV
+#  - TARGET=SOULF4
+#  - TARGET=SPARKY
 #  - TARGET=SPARKY2
-#  - TARGET=SPARKY_OPBL
-  - TARGET=SPRACINGF3
-  - TARGET=SPRACINGF3EVO
+#  - TARGET=SPRACINGF3
+#  - TARGET=SPRACINGF3EVO
 #  - TARGET=SPRACINGF3MINI
-  - TARGET=STM32F3DISCOVERY
+#  - TARGET=STM32F3DISCOVERY
 #  - TARGET=VRRACE
 #  - TARGET=X_RACERSPI
+#  - TARGET=YUPIF4
 #  - TARGET=ZCOREF3
-#  - TARGET=RCEXPLORERF3
 
 # use new docker environment
 sudo: false
 
+git:
+  depth: 5
+
 addons:
   apt:
     packages:
-      - build-essential
-      - git
       - libc6-i386
-      - zlib1g-dev
-      - libssl-dev
-      - wkhtmltopdf
-      - libxml2-dev
-      - libxslt-dev
 
 # We use cpp for unit tests, and c for the main project.
 language: cpp
@@ -78,8 +82,11 @@ install:
 before_script: tools/gcc-arm-none-eabi-5_4-2016q3/bin/arm-none-eabi-gcc --version
 script: ./.travis.sh
 
-cache: apt
-
+cache: 
+  directories:
+  - downloads
+  - tools
+    
 #notifications:
 #  irc: "chat.freenode.net#cleanflight"
 #  use_notice: true

Makefile

@@ -45,10 +45,16 @@ export AT := @
 ifndef V
 export V0    :=
 export V1    := $(AT)
+export STDOUT   :=
 else ifeq ($(V), 0)
 export V0    := $(AT)
 export V1    := $(AT)
+export STDOUT:= "> /dev/null"
+export MAKE  := $(MAKE) --no-print-directory
 else ifeq ($(V), 1)
+export V0    :=
+export V1    :=
+export STDOUT   :=
 endif
 
 ###############################################################################
@@ -813,25 +819,25 @@ $(TARGET_BIN): $(TARGET_ELF)
 	$(V0) $(OBJCOPY) -O binary $< $@
 
 $(TARGET_ELF):  $(TARGET_OBJS)
-	$(V1) echo LD $(notdir $@)
+	$(V1) echo Linking $(TARGET)
 	$(V1) $(CC) -o $@ $^ $(LDFLAGS)
 	$(V0) $(SIZE) $(TARGET_ELF)
 
 # Compile
 $(OBJECT_DIR)/$(TARGET)/%.o: %.c
 	$(V1) mkdir -p $(dir $@)
-	$(V1) echo %% $(notdir $<)
+	$(V1) echo "%% $(notdir $<)" "$(STDOUT)"
 	$(V1) $(CC) -c -o $@ $(CFLAGS) $<
 
 # Assemble
 $(OBJECT_DIR)/$(TARGET)/%.o: %.s
 	$(V1) mkdir -p $(dir $@)
-	$(V1) echo %% $(notdir $<)
+	$(V1) echo "%% $(notdir $<)" "$(STDOUT)"
 	$(V1) $(CC) -c -o $@ $(ASFLAGS) $<
 
 $(OBJECT_DIR)/$(TARGET)/%.o: %.S
 	$(V1) mkdir -p $(dir $@)
-	$(V1) echo %% $(notdir $<)
+	$(V1) echo "%% $(notdir $<)" "$(STDOUT)"
 	$(V1) $(CC) -c -o $@ $(ASFLAGS) $<
 
 ## sample            : Build all sample (travis) targets

make/tools.mk

@@ -25,36 +25,47 @@ ARM_SDK_URL_BASE  := https://launchpad.net/gcc-arm-embedded/5.0/5-2016-q3-update
 
 # source: https://launchpad.net/gcc-arm-embedded/5.0/
 ifdef LINUX
-  arm_sdk_install: ARM_SDK_URL  := $(ARM_SDK_URL_BASE)-linux.tar.bz2
+  ARM_SDK_URL  := $(ARM_SDK_URL_BASE)-linux.tar.bz2
 endif
 
 ifdef MACOSX
-  arm_sdk_install: ARM_SDK_URL  := $(ARM_SDK_URL_BASE)-mac.tar.bz2
+  ARM_SDK_URL  := $(ARM_SDK_URL_BASE)-mac.tar.bz2
 endif
 
 ifdef WINDOWS
-  arm_sdk_install: ARM_SDK_URL  := $(ARM_SDK_URL_BASE)-win32.zip
+  ARM_SDK_URL  := $(ARM_SDK_URL_BASE)-win32.zip
 endif
 
-arm_sdk_install: ARM_SDK_FILE := $(notdir $(ARM_SDK_URL))
-# order-only prereq on directory existance:
-arm_sdk_install: | $(DL_DIR) $(TOOLS_DIR)
-arm_sdk_install: arm_sdk_clean
-ifneq ($(OSFAMILY), windows)
-        # download the source only if it's newer than what we already have
-	$(V1) curl -L -k -o "$(DL_DIR)/$(ARM_SDK_FILE)" "$(ARM_SDK_URL)"
+ARM_SDK_FILE := $(notdir $(ARM_SDK_URL))
 
+SDK_INSTALL_MARKER := $(ARM_SDK_DIR)/bin/arm-none-eabi-gcc-$(GCC_REQUIRED_VERSION)
+
+# order-only prereq on directory existance:
+arm_sdk_install: | $(TOOLS_DIR)
+
+arm_sdk_install: arm_sdk_download $(SDK_INSTALL_MARKER)
+
+$(SDK_INSTALL_MARKER):
+ifneq ($(OSFAMILY), windows)
         # binary only release so just extract it
 	$(V1) tar -C $(TOOLS_DIR) -xjf "$(DL_DIR)/$(ARM_SDK_FILE)"
 else
-	$(V1) curl -L -k -o "$(DL_DIR)/$(ARM_SDK_FILE)" "$(ARM_SDK_URL)"
 	$(V1) unzip -q -d $(ARM_SDK_DIR) "$(DL_DIR)/$(ARM_SDK_FILE)"
 endif
 
+.PHONY: arm_sdk_download
+arm_sdk_download: | $(DL_DIR)
+arm_sdk_download: $(DL_DIR)/$(ARM_SDK_FILE)
+$(DL_DIR)/$(ARM_SDK_FILE):
+        # download the source only if it's newer than what we already have
+	$(V1) curl -L -k -o "$(DL_DIR)/$(ARM_SDK_FILE)" -z "$(DL_DIR)/$(ARM_SDK_FILE)" "$(ARM_SDK_URL)"
+
+
 ## arm_sdk_clean     : Uninstall Arm SDK
 .PHONY: arm_sdk_clean
 arm_sdk_clean:
 	$(V1) [ ! -d "$(ARM_SDK_DIR)" ] || $(RM) -r $(ARM_SDK_DIR)
+	$(V1) [ ! -d "$(DL_DIR)" ] || $(RM) -r $(DL_DIR)
 
 .PHONY: openocd_win_install