diff --git a/.gitignore b/.gitignore
index 2646684e8d041c675eb00e1389580813e456b784..aff65bfd8996070f1927efa916783292fd349528 100644
--- a/.gitignore
+++ b/.gitignore
@@ -47,3 +47,4 @@ Makefile.local
.ycm_extra_conf.py
.ycm_extra_conf.pyc
__pycache__
+.dlcache
diff --git a/Makefile.include b/Makefile.include
index fbc3a77c482bb57ca4ea6ce5bccd1c882269201d..52b8669e3532936223e07df3fb05357b148a1717 100644
--- a/Makefile.include
+++ b/Makefile.include
@@ -16,6 +16,8 @@ APPDIR ?= $(CURDIR)
BINDIRBASE ?= $(APPDIR)/bin
BINDIR ?= $(BINDIRBASE)/$(BOARD)
PKGDIRBASE ?= $(BINDIRBASE)/pkg/$(BOARD)
+DLCACHE ?= $(RIOTBASE)/dist/tools/dlcache/dlcache.sh
+DLCACHE_DIR ?= $(RIOTBASE)/.dlcache
__DIRECTORY_VARIABLES := \
RIOTBASE \
@@ -30,6 +32,7 @@ __DIRECTORY_VARIABLES := \
CCACHE_BASEDIR \
GITCACHE \
PKGDIRBASE \
+ DLCACHE_DIR \
#
# Make all paths absolute.
@@ -45,6 +48,7 @@ override APPDIR := $(abspath $(APPDIR))
override BINDIRBASE := $(abspath $(BINDIRBASE))
override BINDIR := $(abspath $(BINDIR))
override PKGDIRBASE := $(abspath $(PKGDIRBASE))
+override DLCACHE_DIR := $(abspath $(DLCACHE_DIR))
# Ensure that all directories are set and don't contain spaces.
ifneq (, $(filter-out 1, $(foreach v,${__DIRECTORY_VARIABLES},$(words ${${v}}))))
diff --git a/boards/x86-multiboot-common/Makefile.include b/boards/x86-multiboot-common/Makefile.include
index 9662795cbfb6f11687664b21d5c351816d6588f8..4133f13305ac18c4f560dcd6763c02d520a2fde3 100644
--- a/boards/x86-multiboot-common/Makefile.include
+++ b/boards/x86-multiboot-common/Makefile.include
@@ -5,11 +5,12 @@ endif
ifneq (0, $(shell test -e "$(NEWLIB_BASE)/lib/libc.a" && echo $$?))
NEWLIB_PRECOMPILED_NAME := i586-newlib_2.2.0.20150623_tlsf-3337.tar.bz2
NEWLIB_PRECOMPILED := http://download.riot-os.org/$(NEWLIB_PRECOMPILED_NAME)
+ NEWLIB_PRECOMPILED_MD5 := 8090a768cf0294b259fe7ad904d085c7
$(warning Precompiled newlib is missing in $(NEWLIB_BASE))
$(warning Downloading from $(NEWLIB_PRECOMPILED))
- $(shell cd $(RIOTBASE) && $(DOWNLOAD_TO_STDOUT) "$(NEWLIB_PRECOMPILED)" | tar xj)
+ $(shell cd $(RIOTBASE) && $(DLCACHE) "$(NEWLIB_PRECOMPILED)" 8090a768cf0294b259fe7ad904d085c7 - | tar xj)
endif
ifeq (,$(BUILD_INCLUDE_BASE))
diff --git a/dist/tools/dlcache/README.md b/dist/tools/dlcache/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..059f19212000eb0c682955ecc9eb5543468d87d5
--- /dev/null
+++ b/dist/tools/dlcache/README.md
@@ -0,0 +1,11 @@
+# Introduction
+
+This script aims to cache http(s)-downloads
+
+## How it works
+
+- if a file with the right name and md5 exists, nothing happens
+- if a file with the right name and md5 is in cache, use that
+- if a file with the right name but wrong md5 is in cache, redownload
+ download to cache
+- after download, check md5, then copy to target
diff --git a/dist/tools/dlcache/dlcache.sh b/dist/tools/dlcache/dlcache.sh
new file mode 100755
index 0000000000000000000000000000000000000000..08cf7d6de4698880fc41cb3d3a98bdcd85008991
--- /dev/null
+++ b/dist/tools/dlcache/dlcache.sh
@@ -0,0 +1,108 @@
+#!/bin/sh
+
+DLCACHE_DIR=${DLCACHE_DIR:-~/.dlcache}
+
+mkdir -p "$DLCACHE_DIR"
+
+_echo() {
+ echo "$@" 1>&2
+}
+
+if [ "$(uname)" = Darwin ]; then
+ _locked() {
+ local lockfile="$1"
+ shift
+
+ while ! shlock -p $$ -f $lockfile; do
+ sleep 0.2
+ done
+
+ $*
+
+ rm $lockfile
+ }
+else
+ _locked() {
+ local lockfile="$1"
+ shift
+
+ (
+ flock -w 600 9 || exit 1
+ $*
+ ) 9>"$lockfile"
+ }
+fi
+
+if [ "$(uname)" = Darwin ]; then
+ MD5="md5 -r"
+else
+ MD5=md5sum
+fi
+
+calcmd5() {
+ local file="$1"
+ local md5="$2"
+
+ local file_md5=$(${MD5} "$file" | cut -d\ -f1)
+
+ test "$md5" = "$file_md5"
+}
+
+downloader() {
+ if [ -n "$(command -v wget)" ]; then
+ wget -nv "$1" -O $2
+ elif [ -n "$(command -v curl)" ]; then
+ curl -L $1 -o $2
+ else
+ _echo "$0: neither wget nor curl available!"
+ return 1
+ fi
+}
+
+download() {
+ local url="$1"
+ local _md5="$2"
+ local basename_url=$(basename ${url})
+ local target="${3:-${basename_url}}"
+
+ [ -f "$target" ] && {
+ # if our target file exists, check it's md5.
+ calcmd5 "$target" "$_md5" && {
+ _echo "$0: target exists, md5 matches."
+ exit 0
+ }
+ }
+
+ local filename="$(basename $url)"
+ [ -f "$DLCACHE_DIR/$filename" ] && {
+ # if the file exists in cache, check it's md5 and possibly remove it.
+ if calcmd5 "$DLCACHE_DIR/$filename" "$_md5"; then
+ _echo "$0: getting \"$url\" from cache"
+ else
+ _echo "$0: \"$DLCACHE_DIR/$filename\" has wrong checksum, re-downloading"
+ rm "$DLCACHE_DIR/$filename"
+ fi
+ }
+
+ [ ! -f "$DLCACHE_DIR/$filename" ] && {
+ _echo "$0: downloading \"$url\"..."
+ downloader "$url" "$DLCACHE_DIR/$filename" || {
+ _echo "$0: error downloading $url to $DLCACHE_DIR/$filename!"
+ exit 1
+ }
+ _echo "$0: done downloading \"$url\""
+ }
+
+ calcmd5 "$DLCACHE_DIR/$filename" "$_md5" || {
+ _echo "$0: checksum mismatch!"
+ exit 1
+ }
+
+ if [ "$target" = "-" ]; then
+ cat "$DLCACHE_DIR/$filename"
+ else
+ cp "$DLCACHE_DIR/$filename" "$target"
+ fi
+}
+
+_locked "$DLCACHE_DIR/$(basename $1).locked" download "$@"
diff --git a/makefiles/vars.inc.mk b/makefiles/vars.inc.mk
index a6404a347d8bfe518ec35c7a926fdad23e0beb37..23cb5f63a8e1d63dcaedda84d7adea38338e1719 100644
--- a/makefiles/vars.inc.mk
+++ b/makefiles/vars.inc.mk
@@ -68,6 +68,7 @@ export RESET_FLAGS # The parameters to supply to RESET.
export CCACHE_BASEDIR # ccache basedir, allows multiple riot build
# directories to share a ccache directory
+export DLCACHE # directory used to cache http downloads
export DOWNLOAD_TO_FILE # Use `$(DOWNLOAD_TO_FILE) $(DESTINATION) $(URL)` to download `$(URL)` to `$(DESTINATION)`.
export DOWNLOAD_TO_STDOUT # Use `$(DOWNLOAD_TO_STDOUT) $(URL)` to download `$(URL)` output `$(URL)` to stdout, e.g. to be piped into `tar xz`.
export UNZIP_HERE # Use `cd $(SOME_FOLDER) && $(UNZIP_HERE) $(SOME_FILE)` to extract the contents of the zip file `$(SOME_FILE)` into `$(SOME_FOLDER)`.
diff --git a/pkg/nordic_softdevice_ble/Makefile b/pkg/nordic_softdevice_ble/Makefile
index 9c2d71447ae85b9f031cebe93b7d2d40e4194b4f..27e380fec58d25273b4f8468084e311c89df53f4 100644
--- a/pkg/nordic_softdevice_ble/Makefile
+++ b/pkg/nordic_softdevice_ble/Makefile
@@ -2,6 +2,7 @@ PKG_NAME = nordic_softdevice_ble
PKG_VERSION = 3288530
PKG_FILE = nrf5_iot_sdk_$(PKG_VERSION).zip
PKG_URL = https://developer.nordicsemi.com/nRF5_IoT_SDK/nRF5_IoT_SDK_v0.9.x/$(PKG_FILE)
+PKG_MD5 = 291c875df69c51e22491e6112128bd9c
PKG_LICENSE = nordic-bsd
PKG_DIR=$(CURDIR)
PKG_BUILDDIR=$(PKGDIRBASE)/$(PKG_NAME)
@@ -41,7 +42,7 @@ $(PKG_SRCDIR)/.extracted: $(PKG_BUILDDIR)/$(PKG_FILE)
$(PKG_BUILDDIR)/$(PKG_FILE):
@mkdir -p $(@D)
- $(Q)$(DOWNLOAD_TO_FILE) $@ $(PKG_URL)
+ $(Q)$(DLCACHE) $(PKG_URL) $(PKG_MD5) $@
clean::
rm -rf $(PKG_SRCDIR)/
diff --git a/pkg/tlsf/Makefile b/pkg/tlsf/Makefile
index 480e2ff606976da153b9f72c24727e6bbe7275fa..cc32ba1ff8f2b77a8a74778510cb2afe9556e62e 100644
--- a/pkg/tlsf/Makefile
+++ b/pkg/tlsf/Makefile
@@ -2,6 +2,7 @@ PKG_NAME = tlsf
PKG_VERSION = 3.0
PKG_FILE = tlsf-$(PKG_VERSION).zip
PKG_URL = http://download.riot-os.org/$(PKG_FILE)
+PKG_MD5 = 176965d58af525347e582e1630ea9604
PKG_LICENSE = PD
PKG_DIR=$(CURDIR)
PKG_BUILDDIR=$(PKGDIRBASE)/$(PKG_NAME)
@@ -22,7 +23,7 @@ $(PKG_SRCDIR)/Makefile: $(PKG_BUILDDIR)/$(PKG_FILE) $(CURDIR)/patch.txt
$(PKG_BUILDDIR)/$(PKG_FILE):
@mkdir -p $(@D)
- $(Q)$(DOWNLOAD_TO_FILE) $@ $(PKG_URL)
+ $(Q)$(DLCACHE) $(PKG_URL) $(PKG_MD5) $@
clean::
rm -rf $(PKG_SRCDIR)/