tools/mpremote: Add support for relative urls in package.json files.

URLs in `package.json` may now be specified relative to the base URL of the
`package.json` file.

Relative URLs wil work for `package.json` files installed from the web as
well as local file paths.

Docs: update `docs/reference/packages.rst` to add documentation for:

- Installing packages from local filesystems (PR micropython#12476); and
- Using relative URLs in the `package.json` file (PR micropython#12477);
- Update the packaging example to encourage relative URLs as the default
  in `package.json`.

Add `tools/mpremote/tests/test_mip_local_install.sh` to test the
installation of a package from local files using relative URLs in the
`package.json`.

Signed-off-by: Glenn Moloney <[email protected]>

docs/reference/packages.rst

@@ -96,6 +96,18 @@ The ``--target=path``, ``--no-mpy``, and ``--index`` arguments can be set::
     $ mpremote mip install --no-mpy pkgname
     $ mpremote mip install --index https://host/pi pkgname
 
+:term:`mpremote` can also install packages from files stored on the host's local
+filesystem::
+
+    $ mpremote mip install path/to/pkg.py
+    $ mpremote mip install path/to/app/package.json
+    $ mpremote mip install \\path\\to\\pkg.py
+
+This is especially useful for testing packages during development and for
+installing packages from local clones of GitHub repositories. Note that URLs in
+``package.json`` files must use forward slashes ("/") as directory separators,
+even on Windows, so that they are compatible with installing from the web.
+
 Installing packages manually
 ----------------------------
 
@@ -116,12 +128,25 @@ To write a "self-hosted" package that can be downloaded by ``mip`` or
 ``mpremote``, you need a static webserver (or GitHub) to host either a
 single .py file, or a ``package.json`` file alongside your .py files.
 
-A typical ``package.json`` for an example ``mlx90640`` library looks like::
+An example ``mlx90640`` library hosted on GitHub could be installed with::
+
+    $ mpremote mip install github:org/micropython-mlx90640
+
+The layout for the package on GitHub might look like::
+
+    https://github.com/org/micropython-mlx90640/
+        package.json
+        mlx90640/
+            __init__.py
+            utils.py
+
+The ``package.json`` specifies the location of files to be installed and other
+dependencies::
 
     {
       "urls": [
-        ["mlx90640/__init__.py", "github:org/micropython-mlx90640/mlx90640/__init__.py"],
-        ["mlx90640/utils.py", "github:org/micropython-mlx90640/mlx90640/utils.py"]
+        ["mlx90640/__init__.py", "mlx90640/__init__.py"],
+        ["mlx90640/utils.py", "mlx90640/utils.py"]
       ],
       "deps": [
         ["collections-defaultdict", "latest"],
@@ -132,9 +157,20 @@ A typical ``package.json`` for an example ``mlx90640`` library looks like::
       "version": "0.2"
     }
 
-This includes two files, hosted at a GitHub repo named
-``org/micropython-mlx90640``, which install into the ``mlx90640`` directory on
-the device. It depends on ``collections-defaultdict`` and ``os-path`` which will
+The ``urls`` list specifies the files to be installed according to::
+
+    "urls": [
+        [destination_path, source_url]
+        ...
+
+where ``destination_path`` is the location and name of the file to be installed
+on the device and ``source_url`` is the URL of the file to be installed. The
+source URL would usually be specified relative to the directory containing the
+``package.json`` file, but can also be an absolute URL, eg::
+
+    ["mlx90640/utils.py", "github:org/micropython-mlx90640/mlx90640/utils.py"]
+
+The package depends on ``collections-defaultdict`` and ``os-path`` which will
 be installed automatically from the :term:`micropython-lib`. The third
 dependency installs the content as defined by the ``package.json`` file of the
 ``main`` branch of the GitHub repo ``org/micropython-additions``.

tools/mpremote/mpremote/mip.py

@@ -7,6 +7,7 @@
 import json
 import tempfile
 import os
+import os.path
 
 from .commands import CommandError, show_progress_bar
 
@@ -64,22 +65,33 @@ def _rewrite_url(url, branch=None):
 
 
 def _download_file(transport, url, dest):
-    try:
-        with urllib.request.urlopen(url) as src:
-            data = src.read()
-            print("Installing:", dest)
-            _ensure_path_exists(transport, dest)
-            transport.fs_writefile(dest, data, progress_callback=show_progress_bar)
-    except urllib.error.HTTPError as e:
-        if e.status == 404:
-            raise CommandError(f"File not found: {url}")
-        else:
-            raise CommandError(f"Error {e.status} requesting {url}")
-    except urllib.error.URLError as e:
-        raise CommandError(f"{e.reason} requesting {url}")
+    if url.startswith(allowed_mip_url_prefixes):
+        try:
+            with urllib.request.urlopen(url) as src:
+                data = src.read()
+        except urllib.error.HTTPError as e:
+            if e.status == 404:
+                raise CommandError(f"File not found: {url}")
+            else:
+                raise CommandError(f"Error {e.status} requesting {url}")
+        except urllib.error.URLError as e:
+            raise CommandError(f"{e.reason} requesting {url}")
+    else:
+        if "\\" in url:
+            raise CommandError(f'Use "/" instead of "\\" in file URLs: {url!r}\n')
+        try:
+            with open(url, "rb") as f:
+                data = f.read()
+        except OSError as e:
+            raise CommandError(f"{e.strerror} opening {url}")
+
+    print("Installing:", dest)
+    _ensure_path_exists(transport, dest)
+    transport.fs_writefile(dest, data, progress_callback=show_progress_bar)
 
 
 def _install_json(transport, package_json_url, index, target, version, mpy):
+    base_url = ""
     if package_json_url.startswith(allowed_mip_url_prefixes):
         try:
             with urllib.request.urlopen(_rewrite_url(package_json_url, version)) as response:
@@ -91,12 +103,14 @@ def _install_json(transport, package_json_url, index, target, version, mpy):
                 raise CommandError(f"Error {e.status} requesting {package_json_url}")
         except urllib.error.URLError as e:
             raise CommandError(f"{e.reason} requesting {package_json_url}")
+        base_url = package_json_url.rpartition("/")[0]
     elif package_json_url.endswith(".json"):
         try:
             with open(package_json_url, "r") as f:
                 package_json = json.load(f)
         except OSError:
             raise CommandError(f"Error opening {package_json_url}")
+        base_url = os.path.dirname(package_json_url)
     else:
         raise CommandError(f"Invalid url for package: {package_json_url}")
     for target_path, short_hash in package_json.get("hashes", ()):
@@ -105,6 +119,8 @@ def _install_json(transport, package_json_url, index, target, version, mpy):
         _download_file(transport, file_url, fs_target_path)
     for target_path, url in package_json.get("urls", ()):
         fs_target_path = target + "/" + target_path
+        if base_url and not url.startswith(allowed_mip_url_prefixes):
+            url = f"{base_url}/{url}"  # Relative URLs
         _download_file(transport, _rewrite_url(url, version), fs_target_path)
     for dep, dep_version in package_json.get("deps", ()):
         _install_package(transport, dep, index, target, dep_version, mpy)

tools/mpremote/tests/test_mip_local_install.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+
+# This test the "mpremote mip install" from local files. It creates a package
+# and "mip installs" it into a ramdisk. The package is then imported and
+# executed. The package is a simple "Hello, world!" example.
+
+set -e
+
+PACKAGE=mip_example
+PACKAGE_DIR=${TMP}/example
+MODULE_DIR=${PACKAGE_DIR}/${PACKAGE}
+
+target=/__ramdisk
+block_size=512
+num_blocks=50
+
+# Create the smallest permissible ramdisk.
+cat << EOF > "${TMP}/ramdisk.py"
+class RAMBlockDev:
+    def __init__(self, block_size, num_blocks):
+        self.block_size = block_size
+        self.data = bytearray(block_size * num_blocks)
+
+    def readblocks(self, block_num, buf):
+        for i in range(len(buf)):
+            buf[i] = self.data[block_num * self.block_size + i]
+
+    def writeblocks(self, block_num, buf):
+        for i in range(len(buf)):
+            self.data[block_num * self.block_size + i] = buf[i]
+
+    def ioctl(self, op, arg):
+        if op == 4: # get number of blocks
+            return len(self.data) // self.block_size
+        if op == 5: # get block size
+            return self.block_size
+
+import os
+
+bdev = RAMBlockDev(${block_size}, ${num_blocks})
+os.VfsFat.mkfs(bdev)
+os.mount(bdev, '${target}')
+EOF
+
+echo ----- Setup
+mkdir -p ${MODULE_DIR}
+echo "def hello(): print('Hello, world!')" > ${MODULE_DIR}/hello.py
+echo "from .hello import hello" > ${MODULE_DIR}/__init__.py
+cat > ${PACKAGE_DIR}/package.json <<EOF
+{
+    "urls": [
+        ["${PACKAGE}/__init__.py", "${PACKAGE}/__init__.py"],
+        ["${PACKAGE}/hello.py", "${PACKAGE}/hello.py"]
+    ],
+    "version": "0.2"
+}
+EOF
+
+$MPREMOTE run "${TMP}/ramdisk.py"
+$MPREMOTE resume mkdir ${target}/lib
+echo
+echo ---- Install package
+$MPREMOTE resume mip install --target=${target}/lib ${PACKAGE_DIR}/package.json
+echo
+echo ---- Test package
+$MPREMOTE resume exec "import sys; sys.path.append(\"${target}/lib\")"
+$MPREMOTE resume exec "import ${PACKAGE}; ${PACKAGE}.hello()"

tools/mpremote/tests/test_mip_local_install.sh.exp

@@ -0,0 +1,11 @@
+----- Setup
+mkdir :/__ramdisk/lib
+
+---- Install package
+Install ${TMP}/example/package.json
+Installing: /__ramdisk/lib/mip_example/__init__.py
+Installing: /__ramdisk/lib/mip_example/hello.py
+Done
+
+---- Test package
+Hello, world!