Backport parallel download locking support

Fixes #733

patches/openwrt/0018-tools-add-tar-host-build-required-for-sort-name-which-was-only-added-in-the-latest-version.patch

@@ -0,0 +1,81 @@
+From: Matthias Schiffer <[email protected]>
+Date: Sun, 24 Apr 2016 08:44:30 +0200
+Subject: tools: add tar host build, required for --sort=name which was only added in the latest version
+
+Signed-off-by: Felix Fietkau <[email protected]>
+
+Backport of r46876
+
+diff --git a/tools/Makefile b/tools/Makefile
+index c6cded8..98ce8c5 100644
+--- a/tools/Makefile
++++ b/tools/Makefile
+@@ -81,6 +81,9 @@ endif
+ # dependency for tools which have patches directory
+ $(foreach tool, $(tools-y), $(if $(wildcard $(curdir)/$(tool)/patches),$(eval $(curdir)/$(tool)/compile += $(curdir)/patch/install)))
+
++$(foreach tool, $(tools-y), $(eval $(curdir)/$(tool)/compile += $(curdir)/tar/install))
++tools-y += tar
++
+ $(curdir)/builddirs := $(tools-y) $(tools-dep) $(tools-)
+ $(curdir)/builddirs-default := $(tools-y)
+
+diff --git a/tools/tar/Makefile b/tools/tar/Makefile
+new file mode 100644
+index 0000000..975e783
+--- /dev/null
++++ b/tools/tar/Makefile
+@@ -0,0 +1,30 @@
++#
++# Copyright (C) 2015 OpenWrt.org
++#
++# This is free software, licensed under the GNU General Public License v2.
++# See /LICENSE for more information.
++#
++include $(TOPDIR)/rules.mk
++
++PKG_NAME:=tar
++PKG_VERSION:=1.28
++
++PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.xz
++PKG_SOURCE_URL:=@GNU/tar
++PKG_MD5SUM:=49b6306167724fe48f419a33a5beb857
++
++HOST_BUILD_PARALLEL := 1
++
++include $(INCLUDE_DIR)/host-build.mk
++
++HOSTCC := $(HOSTCC_NOCACHE)
++HOSTCXX := $(HOSTCXX_NOCACHE)
++
++HOST_CONFIGURE_ARGS += \
++	--without-posix-acls \
++	--without-selinux \
++	--without-xattrs \
++	--disable-acl \
++	--disable-nls
++
++$(eval $(call HostBuild))
+diff --git a/tools/tar/patches/100-fix_xattr_disable.patch b/tools/tar/patches/100-fix_xattr_disable.patch
+new file mode 100644
+index 0000000..5735bd2
+--- /dev/null
++++ b/tools/tar/patches/100-fix_xattr_disable.patch
+@@ -0,0 +1,17 @@
++--- a/lib/xattr-at.c
+++++ b/lib/xattr-at.c
++@@ -18,6 +18,8 @@
++ 
++ #include <config.h>
++ 
+++#ifdef HAVE_XATTRS
+++
++ #include "xattr-at.h"
++ #include "openat.h"
++ 
++@@ -108,3 +110,5 @@
++ #undef AT_FUNC_RESULT
++ #undef AT_FUNC_POST_FILE_PARAM_DECLS
++ #undef AT_FUNC_POST_FILE_ARGS
+++
+++#endif

patches/openwrt/0019-tools-compile-flock-before-everything-else.patch

@@ -0,0 +1,31 @@
+From: Matthias Schiffer <[email protected]>
+Date: Sun, 24 Apr 2016 08:44:47 +0200
+Subject: tools: compile flock before everything else
+
+Signed-off-by: Felix Fietkau <[email protected]>
+
+Backport of r48413
+
+diff --git a/tools/Makefile b/tools/Makefile
+index 98ce8c5..3402c08 100644
+--- a/tools/Makefile
++++ b/tools/Makefile
+@@ -26,7 +26,7 @@ endif
+ tools-$(BUILD_TOOLCHAIN) += gmp mpfr mpc libelf
+ tools-y += m4 libtool autoconf automake flex bison pkg-config sed mklibs
+ tools-y += sstrip make-ext4fs e2fsprogs mtd-utils mkimage
+-tools-y += firmware-utils patch-image patch quilt yaffs2 flock padjffs2
++tools-y += firmware-utils patch-image patch quilt yaffs2 padjffs2
+ tools-y += mm-macros missing-macros xz cmake scons bc findutils gengetopt patchelf
+ tools-$(CONFIG_TARGET_orion_generic) += wrt350nv2-builder upslug2
+ tools-$(CONFIG_powerpc) += upx
+@@ -84,6 +84,9 @@ $(foreach tool, $(tools-y), $(if $(wildcard $(curdir)/$(tool)/patches),$(eval $(
+ $(foreach tool, $(tools-y), $(eval $(curdir)/$(tool)/compile += $(curdir)/tar/install))
+ tools-y += tar
+
++$(curdir)/tar/compile := $(curdir)/flock/install
++tools-y += flock
++
+ $(curdir)/builddirs := $(tools-y) $(tools-dep) $(tools-)
+ $(curdir)/builddirs-default := $(tools-y)
+

patches/openwrt/0020-rules.mk-make-the-locked-template-available-even-if-flock-has-not-been-built-yet-fall-back-to-unlocked-shell-command.patch

@@ -0,0 +1,32 @@
+From: Matthias Schiffer <[email protected]>
+Date: Sun, 24 Apr 2016 08:45:44 +0200
+Subject: rules.mk: make the locked template available even if flock has not been built yet (fall back to unlocked shell command)
+
+Signed-off-by: Felix Fietkau <[email protected]>
+
+Backport of r48414
+
+diff --git a/rules.mk b/rules.mk
+index 9d0134d..ece5026 100644
+--- a/rules.mk
++++ b/rules.mk
+@@ -310,12 +310,16 @@ endef
+ # Execute commands under flock
+ # $(1) => The shell expression.
+ # $(2) => The lock name. If not given, the global lock will be used.
+-define locked
++ifneq ($(wildcard $(STAGING_DIR_HOST)/bin/flock),)
++  define locked
+ 	SHELL= \
+-	$(STAGING_DIR_HOST)/bin/flock \
++	flock \
+ 		$(TMP_DIR)/.$(if $(2),$(strip $(2)),global).flock \
+ 		-c '$(subst ','\'',$(1))'
+-endef
++  endef
++else
++  locked=$(1)
++endif
+
+ # Recursively copy paths into another directory, purge dangling
+ # symlinks before.

patches/openwrt/0021-build-add-locking-for-downloads-fixes-race-conditions-with-multiple-variants.patch

@@ -0,0 +1,20 @@
+From: Matthias Schiffer <[email protected]>
+Date: Sun, 24 Apr 2016 08:48:45 +0200
+Subject: build: add locking for downloads (fixes race conditions with multiple variants)
+
+Signed-off-by: Felix Fietkau <[email protected]>
+
+Backport of r48416
+
+diff --git a/include/download.mk b/include/download.mk
+index e518cce..9176b11 100644
+--- a/include/download.mk
++++ b/include/download.mk
+@@ -182,6 +182,6 @@ define Download
+
+   $(DL_DIR)/$(FILE):
+ 	mkdir -p $(DL_DIR)
+-	$(if $(DownloadMethod/$(call dl_method,$(URL),$(PROTO))),$(DownloadMethod/$(call dl_method,$(URL),$(PROTO))),$(DownloadMethod/unknown))
++	$(call locked,$(if $(DownloadMethod/$(call dl_method,$(URL),$(PROTO))),$(DownloadMethod/$(call dl_method,$(URL),$(PROTO))),$(DownloadMethod/unknown)),$(FILE))
+
+ endef

patches/openwrt/0022-download.mk-fix-packed-checkout-mirroring-support.patch

@@ -0,0 +1,26 @@
+From: Matthias Schiffer <[email protected]>
+Date: Sun, 24 Apr 2016 08:49:27 +0200
+Subject: download.mk: fix packed checkout mirroring support
+
+Changeset r48416  broke the downloading of mirrored, packed scm checkouts.
+
+Fix this by removing the "@" sign in front of the download command which is
+now executed as part of a larger shell command under flock.
+
+Signed-off-by: Jo-Philipp Wich <[email protected]>
+
+Backport of r48733
+
+diff --git a/include/download.mk b/include/download.mk
+index 9176b11..130bbc5 100644
+--- a/include/download.mk
++++ b/include/download.mk
+@@ -48,7 +48,7 @@ define DownloadMethod/default
+ endef
+
+ define wrap_mirror
+-	$(if $(if $(MIRROR),$(filter-out x,$(MIRROR_MD5SUM))),@$(SCRIPT_DIR)/download.pl "$(DL_DIR)" "$(FILE)" "$(MIRROR_MD5SUM)" "" || ( $(1) ),$(1))
++$(if $(if $(MIRROR),$(filter-out x,$(MIRROR_MD5SUM))),$(SCRIPT_DIR)/download.pl "$(DL_DIR)" "$(FILE)" "$(MIRROR_MD5SUM)" "" || ( $(1) ),$(1))
+ endef
+
+ define DownloadMethod/cvs