diff --git a/docs/reference/packages.rst b/docs/reference/packages.rst index 86b8efaa1953d..5b5f626d45232 100644 --- a/docs/reference/packages.rst +++ b/docs/reference/packages.rst @@ -96,6 +96,18 @@ The ``--target=path``, ``--no-mpy``, and ``--index`` arguments can be set:: $ mpremote mip install --no-mpy pkgname $ mpremote mip install --index https://host/pi pkgname +:term:`mpremote` can also install packages from files stored on the host's local +filesystem:: + + $ mpremote mip install path/to/pkg.py + $ mpremote mip install path/to/app/package.json + $ mpremote mip install \\path\\to\\pkg.py + +This is especially useful for testing packages during development and for +installing packages from local clones of GitHub repositories. Note that URLs in +``package.json`` files must use forward slashes ("/") as directory separators, +even on Windows, so that they are compatible with installing from the web. + Installing packages manually ---------------------------- @@ -116,12 +128,25 @@ To write a "self-hosted" package that can be downloaded by ``mip`` or ``mpremote``, you need a static webserver (or GitHub) to host either a single .py file, or a ``package.json`` file alongside your .py files. -A typical ``package.json`` for an example ``mlx90640`` library looks like:: +An example ``mlx90640`` library hosted on GitHub could be installed with:: + + $ mpremote mip install github:org/micropython-mlx90640 + +The layout for the package on GitHub might look like:: + + https://github.com/org/micropython-mlx90640/ + package.json + mlx90640/ + __init__.py + utils.py + +The ``package.json`` specifies the location of files to be installed and other +dependencies:: { "urls": [ - ["mlx90640/__init__.py", "github:org/micropython-mlx90640/mlx90640/__init__.py"], - ["mlx90640/utils.py", "github:org/micropython-mlx90640/mlx90640/utils.py"] + ["mlx90640/__init__.py", "mlx90640/__init__.py"], + ["mlx90640/utils.py", "mlx90640/utils.py"] ], "deps": [ ["collections-defaultdict", "latest"], @@ -132,9 +157,20 @@ A typical ``package.json`` for an example ``mlx90640`` library looks like:: "version": "0.2" } -This includes two files, hosted at a GitHub repo named -``org/micropython-mlx90640``, which install into the ``mlx90640`` directory on -the device. It depends on ``collections-defaultdict`` and ``os-path`` which will +The ``urls`` list specifies the files to be installed according to:: + + "urls": [ + [destination_path, source_url] + ... + +where ``destination_path`` is the location and name of the file to be installed +on the device and ``source_url`` is the URL of the file to be installed. The +source URL would usually be specified relative to the directory containing the +``package.json`` file, but can also be an absolute URL, eg:: + + ["mlx90640/utils.py", "github:org/micropython-mlx90640/mlx90640/utils.py"] + +The package depends on ``collections-defaultdict`` and ``os-path`` which will be installed automatically from the :term:`micropython-lib`. The third dependency installs the content as defined by the ``package.json`` file of the ``main`` branch of the GitHub repo ``org/micropython-additions``. diff --git a/tools/mpremote/mpremote/mip.py b/tools/mpremote/mpremote/mip.py index 4ce62b7f77bbb..858b7933d11d8 100644 --- a/tools/mpremote/mpremote/mip.py +++ b/tools/mpremote/mpremote/mip.py @@ -7,6 +7,7 @@ import json import tempfile import os +import os.path from .commands import CommandError, show_progress_bar @@ -64,22 +65,33 @@ def _rewrite_url(url, branch=None): def _download_file(transport, url, dest): - try: - with urllib.request.urlopen(url) as src: - data = src.read() - print("Installing:", dest) - _ensure_path_exists(transport, dest) - transport.fs_writefile(dest, data, progress_callback=show_progress_bar) - except urllib.error.HTTPError as e: - if e.status == 404: - raise CommandError(f"File not found: {url}") - else: - raise CommandError(f"Error {e.status} requesting {url}") - except urllib.error.URLError as e: - raise CommandError(f"{e.reason} requesting {url}") + if url.startswith(allowed_mip_url_prefixes): + try: + with urllib.request.urlopen(url) as src: + data = src.read() + except urllib.error.HTTPError as e: + if e.status == 404: + raise CommandError(f"File not found: {url}") + else: + raise CommandError(f"Error {e.status} requesting {url}") + except urllib.error.URLError as e: + raise CommandError(f"{e.reason} requesting {url}") + else: + if "\\" in url: + raise CommandError(f'Use "/" instead of "\\" in file URLs: {url!r}\n') + try: + with open(url, "rb") as f: + data = f.read() + except OSError as e: + raise CommandError(f"{e.strerror} opening {url}") + + print("Installing:", dest) + _ensure_path_exists(transport, dest) + transport.fs_writefile(dest, data, progress_callback=show_progress_bar) def _install_json(transport, package_json_url, index, target, version, mpy): + base_url = "" if package_json_url.startswith(allowed_mip_url_prefixes): try: with urllib.request.urlopen(_rewrite_url(package_json_url, version)) as response: @@ -91,12 +103,14 @@ def _install_json(transport, package_json_url, index, target, version, mpy): raise CommandError(f"Error {e.status} requesting {package_json_url}") except urllib.error.URLError as e: raise CommandError(f"{e.reason} requesting {package_json_url}") + base_url = package_json_url.rpartition("/")[0] elif package_json_url.endswith(".json"): try: with open(package_json_url, "r") as f: package_json = json.load(f) except OSError: raise CommandError(f"Error opening {package_json_url}") + base_url = os.path.dirname(package_json_url) else: raise CommandError(f"Invalid url for package: {package_json_url}") for target_path, short_hash in package_json.get("hashes", ()): @@ -105,6 +119,8 @@ def _install_json(transport, package_json_url, index, target, version, mpy): _download_file(transport, file_url, fs_target_path) for target_path, url in package_json.get("urls", ()): fs_target_path = target + "/" + target_path + if base_url and not url.startswith(allowed_mip_url_prefixes): + url = f"{base_url}/{url}" # Relative URLs _download_file(transport, _rewrite_url(url, version), fs_target_path) for dep, dep_version in package_json.get("deps", ()): _install_package(transport, dep, index, target, dep_version, mpy) diff --git a/tools/mpremote/tests/test_mip_local_install.sh b/tools/mpremote/tests/test_mip_local_install.sh new file mode 100755 index 0000000000000..fb8c597bd1484 --- /dev/null +++ b/tools/mpremote/tests/test_mip_local_install.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +# This test the "mpremote mip install" from local files. It creates a package +# and "mip installs" it into a ramdisk. The package is then imported and +# executed. The package is a simple "Hello, world!" example. + +set -e + +PACKAGE=mip_example +PACKAGE_DIR=${TMP}/example +MODULE_DIR=${PACKAGE_DIR}/${PACKAGE} + +target=/__ramdisk +block_size=512 +num_blocks=50 + +# Create the smallest permissible ramdisk. +cat << EOF > "${TMP}/ramdisk.py" +class RAMBlockDev: + def __init__(self, block_size, num_blocks): + self.block_size = block_size + self.data = bytearray(block_size * num_blocks) + + def readblocks(self, block_num, buf): + for i in range(len(buf)): + buf[i] = self.data[block_num * self.block_size + i] + + def writeblocks(self, block_num, buf): + for i in range(len(buf)): + self.data[block_num * self.block_size + i] = buf[i] + + def ioctl(self, op, arg): + if op == 4: # get number of blocks + return len(self.data) // self.block_size + if op == 5: # get block size + return self.block_size + +import os + +bdev = RAMBlockDev(${block_size}, ${num_blocks}) +os.VfsFat.mkfs(bdev) +os.mount(bdev, '${target}') +EOF + +echo ----- Setup +mkdir -p ${MODULE_DIR} +echo "def hello(): print('Hello, world!')" > ${MODULE_DIR}/hello.py +echo "from .hello import hello" > ${MODULE_DIR}/__init__.py +cat > ${PACKAGE_DIR}/package.json <