[WIP] Predownload wheel sources

.github/actions/download.sh

@@ -6,9 +6,11 @@
 #
 # Functions:
 # - Download all referenced native and cross source files for packages to build.
+# - Download all referenced python wheels needed to build.
 
 set -o pipefail
 
+# Download regular cross/* sources
 if [ -z "${DOWNLOAD_PACKAGES}" ]; then
     echo "===> No packages to download. <==="
 else
@@ -19,3 +21,20 @@ else
         make -C ${download} download
     done
 fi
+
+echo ""
+
+# Download any python wheel sources
+build_packages="${NOARCH_PACKAGES} ${ARCH_PACKAGES}"
+
+if [ -z "${build_packages}" ]; then
+    echo "===> No wheels to download. <==="
+else
+    for package in ${build_packages}
+    do
+        echo "===> Download wheels: ${package}"
+        make -C spk/${package} download_wheel
+    done
+fi
+
+echo ""

.github/workflows/build.yml

@@ -85,6 +85,8 @@ jobs:
         run: ./.github/actions/download.sh
         env:
           DOWNLOAD_PACKAGES: ${{ needs.prepare.outputs.download_packages }}
+          ARCH_PACKAGES: ${{ needs.prepare.outputs.arch_packages }}
+          NOARCH_PACKAGES: ${{ needs.prepare.outputs.noarch_packages }}
 
   build:
     name: Build

Dockerfile

@@ -83,6 +83,8 @@ RUN pip2 install virtualenv httpie
 RUN wget https://bootstrap.pypa.io/get-pip.py -O - | python3
 # Install meson cross-platform build system
 RUN pip3 install meson==0.56.0
+# Install poetry to allow pre-downloading wheels
+RUN pip3 install poetry==1.1.13
 
 # Volume pointing to spksrc sources
 VOLUME /spksrc

mk/spksrc.common.mk

@@ -17,12 +17,26 @@ RUN = cd $(WORK_DIR)/$(PKG_DIR) && env $(ENV)
 PIP ?= pip
 
 # System default pip outside from build environment
-PIP_SYSTEM = $(shell which pip)
+PIP_HOST = $(shell which pip)
 
 # Why ask for the same thing twice? Always cache downloads
-PIP_CACHE_OPT ?= --cache-dir $(PIP_DIR)
+PIP_CACHE_OPT ?= --no-cache-dir --find-links $(BASE_DISTRIB_DIR)
+
+# Define default wheel & download pip options
 PIP_WHEEL_ARGS = wheel --disable-pip-version-check --no-binary :all: $(PIP_CACHE_OPT) --no-deps --wheel-dir $(WHEELHOUSE)
 
+# If we're only building cross-compiled wheels
+# all packages will be pre-downloaded, do not
+# check online index to ensure using local files
+ifeq ($(filter 1 ON TRUE,$(WHEELS_PURE_PYTHON_PACKAGING_ENABLE)),)
+PIP_WHEEL_ARGS += --no-index
+endif
+
+# BROKEN: https://github.com/pypa/pip/issues/1884
+# Current implementation is a work-around for the
+# lack of proper source download support from pip
+PIP_DOWNLOAD_ARGS = download --disable-pip-version-check --no-binary :all: $(PIP_CACHE_OPT) --no-deps --dest $(BASE_DISTRIB_DIR) --no-build-isolation --exists-action w
+
 # Available languages
 LANGUAGES = chs cht csy dan enu fre ger hun ita jpn krn nld nor plk ptb ptg rus spn sve trk
 

mk/spksrc.wheel.mk

@@ -35,6 +35,48 @@ else
 $(POST_WHEEL_TARGET): $(WHEEL_TARGET)
 endif
 
+download_wheel: SHELL:=/bin/bash
+download_wheel:
+	@if [ -n "$(WHEELS)" ] ; then \
+		for wheel in $(WHEELS) ; \
+		do \
+			if [ -f $$wheel -a $$(basename $$wheel) != $(WHEELS_PURE_PYTHON) ] ; then \
+				$(MSG) "Downloading wheels from $$wheel ..." ; \
+				# BROKEN: https://github.com/pypa/pip/issues/1884 ; \
+				# xargs -n 1 $(PIP_HOST) $(PIP_DOWNLOAD_ARGS) 2>/dev/null < $$wheel || true ; \
+				while IFS= read -r requirement ; \
+				do \
+					if [ "$$(grep -s egg <<< $${requirement})" ] ; then \
+						name=$$(echo $${requirement#*egg=} | cut -f1 -d=) ; \
+						url=$${requirement} ; \
+					else \
+						name=$${requirement%%[<>=]=*} ; \
+						url="" ; \
+					fi ; \
+					version=$${requirement#*[<>=]=} ; \
+					$(MSG) pip download [$${name}], version [$${version}]$$([ "$${url}" ] && echo ", URL: [$${url}] ") ; \
+					if [ "$$(grep -s egg <<< $${requirement})" ] ; then \
+						$(PIP) $(PIP_DOWNLOAD_ARGS) --verbose $${requirement} ; \
+					else \
+						query="curl -s https://pypi.org/pypi/$${requirement%%=*}/json" ; \
+						query+=" | jq -r '.releases[][]" ; \
+						query+=" | select(.packagetype==\"sdist\")" ; \
+						query+=" | select((.filename|test(\"-$${requirement##*=}.tar.gz\")) or (.filename|test(\"-$${requirement##*=}.zip\"))) | .url'" ; \
+						localFile=$$(basename $$(eval $${query} 2>/dev/null) 2</dev/null) ; \
+						if [ "$${localFile}" = "" ]; then \
+							echo "ERROR: Invalid package name [$${requirement%%=*}]" ; \
+						elif [ -s $(DISTRIB_DIR)/$${localFile} ]; then \
+							echo "INFO: File already exists [$${localFile}]" ; \
+						else \
+							echo "wget --secure-protocol=TLSv1_2 -nv -O $(DISTRIB_DIR)/$${localFile}.part -nc $$(eval $${query})" ; \
+							wget --secure-protocol=TLSv1_2 -nv -O $(DISTRIB_DIR)/$${localFile}.part -nc $$(eval $${query}) ; \
+							mv $(DISTRIB_DIR)/$${localFile}.part $(DISTRIB_DIR)/$${localFile} ; \
+						fi ; \
+					fi ; \
+				done < <(grep -v  -e "^\#" -e "^\$$" $$wheel) || true ; \
+			fi ; \
+		done \
+	fi
 
 wheel_msg_target:
 	@$(MSG) "Processing wheels of $(NAME)"
@@ -43,7 +85,7 @@ wheel_msg_target:
 # PIP_CACHE_OPT is default "--cache-dir $(PIP_DIR)", PIP_DIR defaults to $(DISTRIB_DIR)/pip, so
 # will move if the user chooses a custom persistent distribution dir for caching downloads between
 # containers and builds.
-pre_wheel_target: wheel_msg_target
+pre_wheel_target: wheel_msg_target download_wheel
 ifneq ($(strip $(WHEELS)),)
 	@if [ -n "$(PIP_CACHE_OPT)" ] ; then \
 	   mkdir -p $(PIP_DIR) ; \
@@ -93,15 +135,21 @@ ifneq ($(strip $(WHEELS)),)
 	   while IFS= read -r requirement ; do \
 	      wheel=$${requirement#*:} ; \
 	      file=$$(basename $${requirement%%:*}) ; \
+	      version=$${wheel#*[<>=]=} ; \
 	      [ "$${file}" = "$(WHEELS_LIMITED_API)" ] && abi3="--build-option=--py-limited-api=$(PYTHON_LIMITED_API)" || abi3="" ; \
-	      [ "$$(grep -s egg <<< $${wheel})" ] && name=$${wheel#*egg=} || name=$${wheel%%[<>=]=*} ; \
+	      if [ "$$(grep -s egg <<< $${wheel})" ] ; then \
+	              name=$$(echo $${wheel#*egg=} | cut -f1 -d=) ; \
+	              wheel="$${name}==$${version}" ; \
+	      else \
+	              name=$${wheel%%[<>=]=*} ; \
+	      fi ; \
 	      options=($$(echo $(WHEELS_BUILD_ARGS) | sed -e 's/ \[/\n\[/g' | grep -i $${name} | cut -f2 -d] | xargs)) ; \
 	      [ "$${options}" ] && global_option=$$(printf "\x2D\x2Dglobal-option=%s " "$${options[@]}") || global_option="" ; \
 	      localCFLAGS=($$(echo $(WHEELS_CFLAGS) | sed -e 's/ \[/\n\[/g' | grep -i $${name} | cut -f2 -d] | xargs)) ; \
 	      localLDFLAGS=($$(echo $(WHEELS_LDFLAGS) | sed -e 's/ \[/\n\[/g' | grep -i $${name} | cut -f2 -d] | xargs)) ; \
 	      localCPPFLAGS=($$(echo $(WHEELS_CPPFLAGS) | sed -e 's/ \[/\n\[/g' | grep -i $${name} | cut -f2 -d] | xargs)) ; \
 	      localCXXFLAGS=($$(echo $(WHEELS_CXXFLAGS) | sed -e 's/ \[/\n\[/g' | grep -i $${name} | cut -f2 -d] | xargs)) ; \
-	      $(MSG) [$${name}] $$([ "$${localCFLAGS[@]}" ] && echo "CFLAGS=$${localCFLAGS[@]} ")$$([ "$${localLDFLAGS[@]}" ] && echo "LDFLAGS=$${localLDFLAGS[@]} ")$$([ "$${localCPPFLAGS[@]}" ] && echo "CPPFLAGS=$${localCPPFLAGS[@]} ")$$([ "$${localCXXFLAGS[@]}" ] && echo "CXXFLAGS=$${localCXXFLAGS[@]} ")$$([ "$${abi3}" ] && echo "$${abi3} ")"$${global_option}" ; \
+	      $(MSG) pip wheel [$${name}], version [$${version}] $$([ "$${localCFLAGS[@]}" ] && echo "CFLAGS=$${localCFLAGS[@]} ")$$([ "$${localLDFLAGS[@]}" ] && echo "LDFLAGS=$${localLDFLAGS[@]} ")$$([ "$${localCPPFLAGS[@]}" ] && echo "CPPFLAGS=$${localCPPFLAGS[@]} ")$$([ "$${localCXXFLAGS[@]}" ] && echo "CXXFLAGS=$${localCXXFLAGS[@]} ")$$([ "$${abi3}" ] && echo "$${abi3} ")"$${global_option}" ; \
 	      $(RUN) \
 	         _PYTHON_HOST_PLATFORM="$(TC_TARGET)" \
 	         CFLAGS="$(CFLAGS) -I$(STAGING_INSTALL_PREFIX)/$(PYTHON_INC_DIR) $${localCFLAGS[@]}" \

spk/bazarr/Makefile

@@ -1,7 +1,7 @@
 SPK_NAME = bazarr
 
 SPK_VERS = 1.0.2
-SPK_REV = 3
+SPK_REV = 4
 
 SPK_ICON = src/bazarr.png
 

spk/beets/Makefile

@@ -1,6 +1,6 @@
 SPK_NAME = beets
 SPK_VERS = 1.6.0
-SPK_REV = 7
+SPK_REV = 8
 SPK_ICON = src/beets.png
 
 BUILD_DEPENDS = cross/python310

spk/homeassistant/Makefile

@@ -1,6 +1,6 @@
 SPK_NAME = homeassistant
 SPK_VERS = 2021.9.7
-SPK_REV = 17
+SPK_REV = 18
 SPK_ICON = src/${SPK_NAME}.png
 
 SPK_DEPENDS = "python310"

spk/mercurial/Makefile

@@ -1,6 +1,6 @@
 SPK_NAME = mercurial
 SPK_VERS = 6.0
-SPK_REV = 8
+SPK_REV = 9
 SPK_ICON = src/mercurial.png
 
 BUILD_DEPENDS = cross/python310