Merge pull request #6643 from kaspar030/add_http_dl_cache

dist/tools: add http download with caching

.gitignore

@@ -47,3 +47,4 @@ Makefile.local
 .ycm_extra_conf.py
 .ycm_extra_conf.pyc
 __pycache__
+.dlcache

Makefile.include

@@ -16,6 +16,8 @@ APPDIR         ?= $(CURDIR)
 BINDIRBASE     ?= $(APPDIR)/bin
 BINDIR         ?= $(BINDIRBASE)/$(BOARD)
 PKGDIRBASE     ?= $(BINDIRBASE)/pkg/$(BOARD)
+DLCACHE        ?= $(RIOTBASE)/dist/tools/dlcache/dlcache.sh
+DLCACHE_DIR    ?= $(RIOTBASE)/.dlcache
 
 __DIRECTORY_VARIABLES := \
   RIOTBASE \
@@ -30,6 +32,7 @@ __DIRECTORY_VARIABLES := \
   CCACHE_BASEDIR \
   GITCACHE \
   PKGDIRBASE \
+  DLCACHE_DIR \
   #
 
 # Make all paths absolute.
@@ -45,6 +48,7 @@ override APPDIR         := $(abspath $(APPDIR))
 override BINDIRBASE     := $(abspath $(BINDIRBASE))
 override BINDIR         := $(abspath $(BINDIR))
 override PKGDIRBASE     := $(abspath $(PKGDIRBASE))
+override DLCACHE_DIR    := $(abspath $(DLCACHE_DIR))
 
 # Ensure that all directories are set and don't contain spaces.
 ifneq (, $(filter-out 1, $(foreach v,${__DIRECTORY_VARIABLES},$(words ${${v}}))))

boards/x86-multiboot-common/Makefile.include

@@ -5,11 +5,12 @@ endif
 ifneq (0, $(shell test -e "$(NEWLIB_BASE)/lib/libc.a" && echo $$?))
   NEWLIB_PRECOMPILED_NAME := i586-newlib_2.2.0.20150623_tlsf-3337.tar.bz2
   NEWLIB_PRECOMPILED := http://download.riot-os.org/$(NEWLIB_PRECOMPILED_NAME)
+  NEWLIB_PRECOMPILED_MD5 := 8090a768cf0294b259fe7ad904d085c7
 
   $(warning Precompiled newlib is missing in $(NEWLIB_BASE))
   $(warning Downloading from $(NEWLIB_PRECOMPILED))
 
-  $(shell cd $(RIOTBASE) && $(DOWNLOAD_TO_STDOUT) "$(NEWLIB_PRECOMPILED)" | tar xj)
+  $(shell cd $(RIOTBASE) && $(DLCACHE) "$(NEWLIB_PRECOMPILED)" 8090a768cf0294b259fe7ad904d085c7 - | tar xj)
 endif
 
 ifeq (,$(BUILD_INCLUDE_BASE))

dist/tools/dlcache/README.md

+# Introduction
+
+This script aims to cache http(s)-downloads
+
+## How it works
+
+- if a file with the right name and md5 exists, nothing happens
+- if a file with the right name and md5 is in cache, use that
+- if a file with the right name but wrong md5 is in cache, redownload
+  download to cache
+- after download, check md5, then copy to target

dist/tools/dlcache/dlcache.sh

+#!/bin/sh
+
+DLCACHE_DIR=${DLCACHE_DIR:-~/.dlcache}
+
+mkdir -p "$DLCACHE_DIR"
+
+_echo() {
+    echo "$@" 1>&2
+}
+
+if [ "$(uname)" = Darwin ]; then
+    _locked() {
+        local lockfile="$1"
+        shift
+
+        while ! shlock -p $$ -f $lockfile; do
+            sleep 0.2
+        done
+
+        $*
+
+        rm $lockfile
+    }
+else
+    _locked() {
+        local lockfile="$1"
+        shift
+
+        (
+        flock -w 600 9 || exit 1
+        $*
+        ) 9>"$lockfile"
+    }
+fi
+
+if [ "$(uname)" = Darwin ]; then
+    MD5="md5 -r"
+else
+    MD5=md5sum
+fi
+
+calcmd5() {
+    local file="$1"
+    local md5="$2"
+
+    local file_md5=$(${MD5} "$file" | cut -d\  -f1)
+
+    test "$md5" = "$file_md5"
+}
+
+downloader() {
+    if [ -n "$(command -v wget)" ]; then
+        wget -nv "$1" -O $2
+    elif [ -n "$(command -v curl)" ]; then
+        curl -L $1 -o $2
+    else
+        _echo "$0: neither wget nor curl available!"
+        return 1
+    fi
+}
+
+download() {
+    local url="$1"
+    local _md5="$2"
+    local basename_url=$(basename ${url})
+    local target="${3:-${basename_url}}"
+
+    [ -f "$target" ] && {
+        # if our target file exists, check it's md5.
+        calcmd5 "$target" "$_md5" && {
+            _echo "$0: target exists, md5 matches."
+            exit 0
+        }
+    }
+
+    local filename="$(basename $url)"
+    [ -f "$DLCACHE_DIR/$filename" ] && {
+        # if the file exists in cache, check it's md5 and possibly remove it.
+        if calcmd5 "$DLCACHE_DIR/$filename" "$_md5"; then
+            _echo "$0: getting \"$url\" from cache"
+        else
+            _echo "$0: \"$DLCACHE_DIR/$filename\" has wrong checksum, re-downloading"
+            rm "$DLCACHE_DIR/$filename"
+        fi
+    }
+
+    [ ! -f "$DLCACHE_DIR/$filename" ] && {
+        _echo "$0: downloading \"$url\"..."
+        downloader "$url" "$DLCACHE_DIR/$filename" || {
+            _echo "$0: error downloading $url to $DLCACHE_DIR/$filename!"
+            exit 1
+        }
+        _echo "$0: done downloading \"$url\""
+    }
+
+    calcmd5 "$DLCACHE_DIR/$filename" "$_md5" || {
+        _echo "$0: checksum mismatch!"
+        exit 1
+    }
+
+    if [ "$target" = "-" ]; then
+        cat "$DLCACHE_DIR/$filename"
+    else
+        cp "$DLCACHE_DIR/$filename" "$target"
+    fi
+}
+
+_locked "$DLCACHE_DIR/$(basename $1).locked" download "$@"

makefiles/vars.inc.mk

@@ -68,6 +68,7 @@ export RESET_FLAGS           # The parameters to supply to RESET.
 export CCACHE_BASEDIR        # ccache basedir, allows multiple riot build
                              # directories to share a ccache directory
 
+export DLCACHE               # directory used to cache http downloads
 export DOWNLOAD_TO_FILE      # Use `$(DOWNLOAD_TO_FILE) $(DESTINATION) $(URL)` to download `$(URL)` to `$(DESTINATION)`.
 export DOWNLOAD_TO_STDOUT    # Use `$(DOWNLOAD_TO_STDOUT) $(URL)` to download `$(URL)` output `$(URL)` to stdout, e.g. to be piped into `tar xz`.
 export UNZIP_HERE            # Use `cd $(SOME_FOLDER) && $(UNZIP_HERE) $(SOME_FILE)` to extract the contents of the zip file `$(SOME_FILE)` into `$(SOME_FOLDER)`.

pkg/nordic_softdevice_ble/Makefile

@@ -2,6 +2,7 @@ PKG_NAME = nordic_softdevice_ble
 PKG_VERSION = 3288530
 PKG_FILE = nrf5_iot_sdk_$(PKG_VERSION).zip
 PKG_URL = https://developer.nordicsemi.com/nRF5_IoT_SDK/nRF5_IoT_SDK_v0.9.x/$(PKG_FILE)
+PKG_MD5 = 291c875df69c51e22491e6112128bd9c
 PKG_LICENSE = nordic-bsd
 PKG_DIR=$(CURDIR)
 PKG_BUILDDIR=$(PKGDIRBASE)/$(PKG_NAME)
@@ -41,7 +42,7 @@ $(PKG_SRCDIR)/.extracted: $(PKG_BUILDDIR)/$(PKG_FILE)
 
 $(PKG_BUILDDIR)/$(PKG_FILE):
 	@mkdir -p $(@D)
-	$(Q)$(DOWNLOAD_TO_FILE) $@ $(PKG_URL)
+	$(Q)$(DLCACHE) $(PKG_URL) $(PKG_MD5) $@
 
 clean::
 	rm -rf $(PKG_SRCDIR)/

pkg/tlsf/Makefile

@@ -2,6 +2,7 @@ PKG_NAME = tlsf
 PKG_VERSION = 3.0
 PKG_FILE = tlsf-$(PKG_VERSION).zip
 PKG_URL = http://download.riot-os.org/$(PKG_FILE)
+PKG_MD5 = 176965d58af525347e582e1630ea9604
 PKG_LICENSE = PD
 PKG_DIR=$(CURDIR)
 PKG_BUILDDIR=$(PKGDIRBASE)/$(PKG_NAME)
@@ -22,7 +23,7 @@ $(PKG_SRCDIR)/Makefile: $(PKG_BUILDDIR)/$(PKG_FILE) $(CURDIR)/patch.txt
 
 $(PKG_BUILDDIR)/$(PKG_FILE):
 	@mkdir -p $(@D)
-	$(Q)$(DOWNLOAD_TO_FILE) $@ $(PKG_URL)
+	$(Q)$(DLCACHE) $(PKG_URL) $(PKG_MD5) $@
 
 clean::
 	rm -rf $(PKG_SRCDIR)/