framework: accelerate build, cache upstream pip/wheel downloads (#3620)

* Accelerate build: ensure upstream pip/wheel downloads are cached
* Move mkdir to wheel.mk as suggested

Author: chickenandpork

mk/spksrc.common.mk

@@ -15,7 +15,9 @@ RUN = cd $(WORK_DIR)/$(PKG_DIR) && env $(ENV)
 
 # Pip command
 PIP ?= pip
-PIP_WHEEL = $(PIP) wheel --no-binary :all: --no-deps --requirement $(WORK_DIR)/wheelhouse/requirements.txt --wheel-dir $(WORK_DIR)/wheelhouse --build-dir $(WORK_DIR)/wheelbuild
+# Why ask for the same thing twice?  Always cache downloads
+PIP_CACHE_OPT ?= --cache-dir $(PIP_DIR)
+PIP_WHEEL = $(PIP) wheel --no-binary :all: $(PIP_CACHE_OPT) --no-deps --requirement $(WORK_DIR)/wheelhouse/requirements.txt --wheel-dir $(WORK_DIR)/wheelhouse --build-dir $(WORK_DIR)/wheelbuild
 
 # Available languages
 LANGUAGES = chs cht csy dan enu fre ger hun ita jpn krn nld nor plk ptb ptg rus spn sve trk

mk/spksrc.directories.mk

@@ -12,9 +12,9 @@
 PWD := $(shell pwd)
 
 DISTRIB_DIR  = $(PWD)/../../distrib
-PIP_DIR = $(PWD)/../../distrib/pip
-TOOLCHAINS_DIR = $(PWD)/../../distrib/toolchains
-KERNELS_DIR = $(PWD)/../../distrib/kernels
+PIP_DIR = $(DISTRIB_DIR)/pip
+TOOLCHAINS_DIR = $(DISTRIB_DIR)/toolchains
+KERNELS_DIR = $(DISTRIB_DIR)/kernels
 PACKAGES_DIR = $(PWD)/../../packages
 
 ifndef WORK_DIR

mk/spksrc.wheel.mk

@@ -34,8 +34,15 @@ endif
 wheel_msg_target:
 	@$(MSG) "Processing wheels of $(NAME)"
 
+# PIP distributions caching requires that the user running it owns the cache directory.
+# PIP_CACHE_OPT is default "--cache-dir $(PIP_DIR)", PIP_DIR defaults to $(DISTRIB_DIR)/pip, so
+# will move if the user chooses a custom persistent distribution dir for caching downloads between
+# containers and builds.
 pre_wheel_target: wheel_msg_target
 	@if [ ! -z "$(WHEELS)" ] ; then \
+		if [ ! -z "$(PIP_CACHE_OPT)" ] ; then \
+			mkdir -p $(PIP_DIR) ; \
+		fi; \
 		mkdir -p $(WORK_DIR)/wheelhouse ; \
 		if [ -f "$(WHEELS)" ] ; then \
 			$(MSG) "Using existing requirements file" ; \