Merge pull request #5112 from kaspar030/use_git_cache

make: pkg: introduce git-cache

Makefile.include

@@ -20,6 +20,8 @@ RIOTPKG ?= $(RIOTBASE)/pkg
 RIOTPROJECT ?= $(shell git rev-parse --show-toplevel 2>/dev/null || pwd)
 RIOTPROJECT := $(abspath $(RIOTPROJECT))
 
+GITCACHE:=$(RIOTBASE)/dist/tools/git/git-cache
+
 # Include Docker settings near the top because we need to build the environment
 # command line before some of the variable origins are overwritten below when
 # using abspath, strip etc.

Makefile.vars

@@ -43,6 +43,7 @@ export SIZE                  # The command to read to size of the ELF sections.
 export UNDEF                 # Set by the BOARD's and CPU's Makefile.include, this contains object files with must not be used in the ELFFILE even if the if no call to the functions.
 export WERROR                # Treat all compiler warnings as errors if set to 1 (see -Werror flag in GCC manual)
 
+export GITCACHE              # path to git-cache executable
 export FLASHER               # The command to call on "make flash".
 export FFLAGS                # The parameters to supply to FLASHER.
 export TERMPROG              # The command to call on "make term".

dist/tools/git/README.md

+# Overview
+
+This directory contains some git tools used by RIOT's build system
+
+## git-cache
+
+Simple git caching script, from https://github.com/kaspar030/git-cache
+If git-cache is unconfigured, the script pulls from the given remote location.
+
+In order to set up the cache, do:
+
+- install the git-cache binary into path.
+  This will make the script available as "git cache ...".
+  Alternatively, directly execute it.
+- run "git cache init", which initializes a git cache in ${HOME}/.gitcache.
+  The used path can be overridden using the "GIT_CACHE_DIR" environment
+  variable.
+  The cache repository will be used to cache multiple remote repositories.
+- add a repository to the cache: "git cache add \<name\> \<URL\>
+- whenever needed (at least once after adding a repository),
+  run "git cache update"

dist/tools/git/git-cache

+#!/bin/sh
+
+git_cache() {
+    git -C "${GIT_CACHE_DIR}" $*
+}
+
+init() {
+    set -ex
+    test -d "${GIT_CACHE_DIR}/.git" || {
+        mkdir -p "${GIT_CACHE_DIR}"
+
+        git_cache init --bare
+        git_cache config core.compression 1
+    }
+    set +ex
+}
+
+add() {
+    set -ex
+    git_cache remote add $1 $2
+    set +ex
+}
+
+update() {
+    set -ex
+    local REMOTE=${1:---all}
+    git_cache fetch $REMOTE
+    set +ex
+}
+
+list() {
+    local REMOTES="$(git_cache remote show)"
+    for remote in $REMOTES; do
+        echo "${remote}: $(git_cache remote get-url $remote)"
+    done
+}
+
+drop() {
+    set -ex
+    local REMOTE=${1}
+    [ -z "$REMOTE" ] && {
+        echo "usage: git cache drop <name>"
+        exit 1
+    }
+    git_cache remote remove $REMOTE
+    set +ex
+}
+
+_check_commit() {
+    git_cache cat-file -e ${1}^{commit}
+}
+
+clone() {
+    set -ex
+    local REMOTE="${1}"
+    local SHA1="${2}"
+    local REMOTE_NAME="$(basename $REMOTE)"
+    local TARGET_PATH="${3:-${REMOTE_NAME}}"
+
+    if _check_commit $2 2>&1; then
+        git init "${TARGET_PATH}"
+        git_cache tag commit$SHA1 $SHA1 || true # ignore possibly already existing tag
+        git -C "${TARGET_PATH}" fetch --depth=1 "${GIT_CACHE_DIR}" refs/tags/commit$SHA1
+        git -C "${TARGET_PATH}" checkout FETCH_HEAD
+    else
+        git clone "${REMOTE}" "${TARGET_PATH}"
+        git -C "${TARGET_PATH}" checkout $SHA1
+    fi
+    set +ex
+}
+
+usage() {
+    echo "git cache uses a bare git repository containing all objects from multiple"
+    echo "upstream git repositories."
+    echo ""
+    echo "usage:"
+    echo ""
+    echo "    git cache init                initialize git cache"
+    echo "    git cache add <name> <url>    add repository <url> with name <name>"
+    echo "    git cache list                list cached repositories"
+    echo "    git cache drop <name>         drop repo from cache"
+    echo "    git cache update [<name>]     fetch repo named <name> (or all)"
+    echo "    git cache clone <url> <SHA1>  clone repository <url> from cache"
+    echo "    git cache show-path           print's the path that can be used as "
+    echo "                                  '--reference' parameter"
+    echo ""
+    echo "To retrieve objects from cache (will use remote repository if needed):"
+    echo '    git clone --reference $(git cache show-path) <repo>'
+}
+
+ACTION=$1
+shift
+
+export GIT_CACHE_DIR=${GIT_CACHE_DIR:-${HOME}/.gitcache}
+
+case $ACTION in
+    init)
+        init $*
+        ;;
+    add)
+        add $*
+        ;;
+    update)
+        update $*
+        ;;
+    list)
+        list $*
+        ;;
+    drop)
+        drop $*
+        ;;
+    show-path)
+        echo ${GIT_CACHE_DIR}
+        ;;
+    clone)
+        clone $*
+        ;;
+    *)
+        usage
+        ;;
+esac

pkg/ccn-lite/Makefile

@@ -16,10 +16,7 @@ all: $(PKG_DIR)/Makefile
 $(PKG_DIR)/Makefile: $(PKG_DIR)/.git/config
 
 $(PKG_DIR)/.git/config:
-	test -d "$(PKG_DIR)" || git clone "$(PKG_URL)" "$(PKG_DIR)"; \
-		cd "$(PKG_DIR)" && \
-		git remote set-url origin "$(PKG_URL)" && \
-		git fetch && git checkout -f "$(PKG_VERSION)"
+	test -d "$(PKG_DIR)" || $(GITCACHE) clone "$(PKG_URL)" "$(PKG_VERSION)" "$(PKG_DIR)"
 
 clean::
 	@echo "Cleaning up CCN-Lite package..."

pkg/cmsis-dsp/Makefile

@@ -16,12 +16,7 @@ $(PKG_DIR)/Makefile: $(PKG_DIR)/.git/config
 	@
 
 $(PKG_DIR)/.git/config:
-	test -d "$(PKG_DIR)" || git clone "$(PKG_URL)" "$(PKG_DIR)"; \
-		cd "$(PKG_DIR)" && git checkout -f "$(PKG_VERSION)"
-
-$(CURDIR)/$(PKG_NAME) $(PKG_NAME):
-	# Get PKG_VERSION of package from PKG_URL
-	git clone '$(PKG_URL)' '$(PKG_NAME)' && cd '$(PKG_NAME)' && git reset --hard '$(PKG_VERSION)'
+	test -d "$(PKG_DIR)" || $(GITCACHE) clone "$(PKG_URL)" "$(PKG_VERSION)" "$(PKG_DIR)"
 
 clean::
 	@echo "Cleaning up $(PKG_NAME) package..."

pkg/libcoap/Makefile

@@ -19,8 +19,7 @@ $(PKG_DIR)/Makefile: $(PKG_DIR)/.git/config
 	cd "$(PKG_DIR)" && git am --ignore-whitespace "$(CURDIR)"/*.patch
 
 $(PKG_DIR)/.git/config:
-	test -d "$(PKG_DIR)" || git clone "$(PKG_URL)" "$(PKG_DIR)"; \
-		cd "$(PKG_DIR)" && git checkout -f "$(PKG_VERSION)"
+	test -d "$(PKG_DIR)" || $(GITCACHE) clone "$(PKG_URL)" "$(PKG_VERSION)" "$(PKG_DIR)"
 
 clean::
 	@echo "Cleaning up libcoap package..."

pkg/micro-ecc/Makefile

@@ -26,8 +26,7 @@ $(PKG_BUILDDIR)/Makefile: $(PKG_BUILDDIR)
 
 $(PKG_BUILDDIR):
 	mkdir -p $(BINDIR)/pkg && \
-	git clone $(PKG_URL) $@ && \
-		cd $@ && git reset --hard $(PKG_VERSION)
+	$(GITCACHE) clone "$(PKG_URL)" "$(PKG_VERSION)" "$(PKG_BUILDDIR)"
 
 clean::
 	rm -Rf $(PKG_BUILDDIR)

pkg/microcoap/Makefile

@@ -19,8 +19,7 @@ $(PKG_DIR)/Makefile: $(PKG_DIR)/.git/config
 	cd "$(PKG_DIR)" && git am --ignore-whitespace "$(CURDIR)"/*.patch
 
 $(PKG_DIR)/.git/config:
-	test -d "$(PKG_DIR)" || git clone "$(PKG_URL)" "$(PKG_DIR)"; \
-		cd "$(PKG_DIR)" && git checkout -f "$(PKG_VERSION)"
+	test -d "$(PKG_DIR)" || $(GITCACHE) clone "$(PKG_URL)" "$(PKG_VERSION)" "$(PKG_DIR)"
 
 clean::
 	@echo "Cleaning up $(PKG_NAME) package..."

pkg/oonf_api/Makefile

@@ -22,8 +22,7 @@ $(PKG_DIR)/Makefile: $(PKG_DIR)/.git/config
 	cd "$(PKG_DIR)" && git am --ignore-whitespace "$(CURDIR)"/*.patch
 
 $(PKG_DIR)/.git/config:
-	test -d "$(PKG_DIR)" || git clone "$(PKG_URL)" "$(PKG_DIR)"; \
-		cd "$(PKG_DIR)" && git checkout -f "$(PKG_VERSION)"
+	test -d "$(PKG_DIR)" || $(GITCACHE) clone "$(PKG_URL)" "$(PKG_VERSION)" "$(PKG_DIR)"
 
 clean::
 	@echo "Cleaning up oonf_api package..."

pkg/openwsn/Makefile

@@ -14,8 +14,7 @@ $(PKG_DIR)/Makefile: $(PKG_DIR)/.git/config
 	cd "$(PKG_DIR)" && git am --ignore-whitespace $(CURDIR)/*.patch
 
 $(PKG_DIR)/.git/config:
-	test -d "$(PKG_DIR)" || git clone "$(PKG_URL)" "$(PKG_DIR)"; \
-		cd "$(PKG_DIR)" && git checkout -f "$(PKG_VERSION)"
+	test -d "$(PKG_DIR)" || $(GITCACHE) clone "$(PKG_URL)" "$(PKG_VERSION)" "$(PKG_DIR)"
 
 clean::
 	@echo "Cleaning up OpenWSN package..."
@@ -29,4 +28,4 @@ distclean::
 	rm -rf "$(PKG_DIR)"
 
 Makefile.include:
-	@true
\ No newline at end of file
+	@true

pkg/relic/Makefile

@@ -23,8 +23,7 @@ $(PKG_DIR)/comp-options.cmake: $(PKG_DIR)/.git/config
 $(PKG_DIR)/Makefile: $(PKG_DIR)/comp-options.cmake
 	cd "$(PKG_DIR)" && COMP="$(filter-out -Werror=old-style-definition -Werror=strict-prototypes, $(CFLAGS) ) " cmake -DCMAKE_TOOLCHAIN_FILE=comp-options.cmake -DCHECK=off -DTESTS=0 -DBENCH=0 -DSHLIB=off -Wno-dev $(RELIC_CONFIG_FLAGS) .
 $(PKG_DIR)/.git/config:
-	test -d "$(PKG_DIR)" || git clone "$(PKG_URL)" "$(PKG_DIR)"; \
-		cd "$(PKG_DIR)" && git checkout -f "$(PKG_VERSION)"
+	test -d "$(PKG_DIR)" || $(GITCACHE) clone "$(PKG_URL)" "$(PKG_VERSION)" "$(PKG_DIR)"
 	cd "$(PKG_DIR)" && git am --ignore-whitespace $(CURDIR)/*.patch
 	./fix-util_print_wo_args.sh .
 	./fix-old-style-definitions.sh .

pkg/wakaama/Makefile

@@ -15,9 +15,7 @@ $(PKG_DIR)/Makefile: $(PKG_TEMP_DIR)/.git/config
 	echo 'include $$(RIOTBASE)/Makefile.base' > $(PKG_DIR)/Makefile
 
 $(PKG_TEMP_DIR)/.git/config:
-	test -d "$(PKG_TEMP_DIR)" || git clone "$(PKG_URL)" "$(PKG_TEMP_DIR)"; \
-		cd "$(PKG_TEMP_DIR)" && git checkout -f "$(PKG_VERSION)"; \
-		cd "$(PKG_TEMP_DIR)" && git am --ignore-whitespace "$(CURDIR)"/*.patch; \
+	test -d "$(PKG_TEMP_DIR)" || $(GITCACHE) clone "$(PKG_URL)" "$(PKG_VERSION)" "$(PKG_TEMP_DIR)" ; \
 		mkdir -p "$(PKG_DIR)" ; \
 		cp $(PKG_TEMP_DIR)/core/*.c $(PKG_TEMP_DIR)/core/*.h $(PKG_DIR); \
 		cp $(PKG_TEMP_DIR)/core/er-coap-13/*.c $(PKG_TEMP_DIR)/core/er-coap-13/*.h $(PKG_DIR); \