diff --git a/.gitignore b/.gitignore index 2646684e8d041c675eb00e1389580813e456b784..aff65bfd8996070f1927efa916783292fd349528 100644 --- a/.gitignore +++ b/.gitignore @@ -47,3 +47,4 @@ Makefile.local .ycm_extra_conf.py .ycm_extra_conf.pyc __pycache__ +.dlcache diff --git a/Makefile.include b/Makefile.include index fbc3a77c482bb57ca4ea6ce5bccd1c882269201d..52b8669e3532936223e07df3fb05357b148a1717 100644 --- a/Makefile.include +++ b/Makefile.include @@ -16,6 +16,8 @@ APPDIR ?= $(CURDIR) BINDIRBASE ?= $(APPDIR)/bin BINDIR ?= $(BINDIRBASE)/$(BOARD) PKGDIRBASE ?= $(BINDIRBASE)/pkg/$(BOARD) +DLCACHE ?= $(RIOTBASE)/dist/tools/dlcache/dlcache.sh +DLCACHE_DIR ?= $(RIOTBASE)/.dlcache __DIRECTORY_VARIABLES := \ RIOTBASE \ @@ -30,6 +32,7 @@ __DIRECTORY_VARIABLES := \ CCACHE_BASEDIR \ GITCACHE \ PKGDIRBASE \ + DLCACHE_DIR \ # # Make all paths absolute. @@ -45,6 +48,7 @@ override APPDIR := $(abspath $(APPDIR)) override BINDIRBASE := $(abspath $(BINDIRBASE)) override BINDIR := $(abspath $(BINDIR)) override PKGDIRBASE := $(abspath $(PKGDIRBASE)) +override DLCACHE_DIR := $(abspath $(DLCACHE_DIR)) # Ensure that all directories are set and don't contain spaces. ifneq (, $(filter-out 1, $(foreach v,${__DIRECTORY_VARIABLES},$(words ${${v}})))) diff --git a/dist/tools/dlcache/README.md b/dist/tools/dlcache/README.md new file mode 100644 index 0000000000000000000000000000000000000000..059f19212000eb0c682955ecc9eb5543468d87d5 --- /dev/null +++ b/dist/tools/dlcache/README.md @@ -0,0 +1,11 @@ +# Introduction + +This script aims to cache http(s)-downloads + +## How it works + +- if a file with the right name and md5 exists, nothing happens +- if a file with the right name and md5 is in cache, use that +- if a file with the right name but wrong md5 is in cache, redownload + download to cache +- after download, check md5, then copy to target diff --git a/dist/tools/dlcache/dlcache.sh b/dist/tools/dlcache/dlcache.sh new file mode 100755 index 0000000000000000000000000000000000000000..08cf7d6de4698880fc41cb3d3a98bdcd85008991 --- /dev/null +++ b/dist/tools/dlcache/dlcache.sh @@ -0,0 +1,108 @@ +#!/bin/sh + +DLCACHE_DIR=${DLCACHE_DIR:-~/.dlcache} + +mkdir -p "$DLCACHE_DIR" + +_echo() { + echo "$@" 1>&2 +} + +if [ "$(uname)" = Darwin ]; then + _locked() { + local lockfile="$1" + shift + + while ! shlock -p $$ -f $lockfile; do + sleep 0.2 + done + + $* + + rm $lockfile + } +else + _locked() { + local lockfile="$1" + shift + + ( + flock -w 600 9 || exit 1 + $* + ) 9>"$lockfile" + } +fi + +if [ "$(uname)" = Darwin ]; then + MD5="md5 -r" +else + MD5=md5sum +fi + +calcmd5() { + local file="$1" + local md5="$2" + + local file_md5=$(${MD5} "$file" | cut -d\ -f1) + + test "$md5" = "$file_md5" +} + +downloader() { + if [ -n "$(command -v wget)" ]; then + wget -nv "$1" -O $2 + elif [ -n "$(command -v curl)" ]; then + curl -L $1 -o $2 + else + _echo "$0: neither wget nor curl available!" + return 1 + fi +} + +download() { + local url="$1" + local _md5="$2" + local basename_url=$(basename ${url}) + local target="${3:-${basename_url}}" + + [ -f "$target" ] && { + # if our target file exists, check it's md5. + calcmd5 "$target" "$_md5" && { + _echo "$0: target exists, md5 matches." + exit 0 + } + } + + local filename="$(basename $url)" + [ -f "$DLCACHE_DIR/$filename" ] && { + # if the file exists in cache, check it's md5 and possibly remove it. + if calcmd5 "$DLCACHE_DIR/$filename" "$_md5"; then + _echo "$0: getting \"$url\" from cache" + else + _echo "$0: \"$DLCACHE_DIR/$filename\" has wrong checksum, re-downloading" + rm "$DLCACHE_DIR/$filename" + fi + } + + [ ! -f "$DLCACHE_DIR/$filename" ] && { + _echo "$0: downloading \"$url\"..." + downloader "$url" "$DLCACHE_DIR/$filename" || { + _echo "$0: error downloading $url to $DLCACHE_DIR/$filename!" + exit 1 + } + _echo "$0: done downloading \"$url\"" + } + + calcmd5 "$DLCACHE_DIR/$filename" "$_md5" || { + _echo "$0: checksum mismatch!" + exit 1 + } + + if [ "$target" = "-" ]; then + cat "$DLCACHE_DIR/$filename" + else + cp "$DLCACHE_DIR/$filename" "$target" + fi +} + +_locked "$DLCACHE_DIR/$(basename $1).locked" download "$@" diff --git a/makefiles/vars.inc.mk b/makefiles/vars.inc.mk index a6404a347d8bfe518ec35c7a926fdad23e0beb37..23cb5f63a8e1d63dcaedda84d7adea38338e1719 100644 --- a/makefiles/vars.inc.mk +++ b/makefiles/vars.inc.mk @@ -68,6 +68,7 @@ export RESET_FLAGS # The parameters to supply to RESET. export CCACHE_BASEDIR # ccache basedir, allows multiple riot build # directories to share a ccache directory +export DLCACHE # directory used to cache http downloads export DOWNLOAD_TO_FILE # Use `$(DOWNLOAD_TO_FILE) $(DESTINATION) $(URL)` to download `$(URL)` to `$(DESTINATION)`. export DOWNLOAD_TO_STDOUT # Use `$(DOWNLOAD_TO_STDOUT) $(URL)` to download `$(URL)` output `$(URL)` to stdout, e.g. to be piped into `tar xz`. export UNZIP_HERE # Use `cd $(SOME_FOLDER) && $(UNZIP_HERE) $(SOME_FILE)` to extract the contents of the zip file `$(SOME_FILE)` into `$(SOME_FOLDER)`.