Add python opener from rasterio 1.4 (#1331)

* Add rasterio's python opener from 1.4a1 and supporting things

* Modern _path module and updated usage
* A setup.cfg for editable installs
* A Dockerfile and Makefile for testing

Resolves #1328

* Add two more virtual filesystem tests

Dockerfile

@@ -0,0 +1,30 @@
+ARG GDAL=ubuntu-small-3.6.4
+FROM ghcr.io/osgeo/gdal:${GDAL} AS gdal
+ARG PYTHON_VERSION=3.9
+ENV LANG="C.UTF-8" LC_ALL="C.UTF-8"
+RUN apt-get update && apt-get install -y software-properties-common
+RUN add-apt-repository -y ppa:deadsnakes/ppa
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+    g++ \
+    gdb \
+    make \
+    python3-pip \
+    python${PYTHON_VERSION} \
+    python${PYTHON_VERSION}-dev \
+    python${PYTHON_VERSION}-venv \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+COPY requirements*.txt ./
+RUN python${PYTHON_VERSION} -m venv /venv && \
+    /venv/bin/python -m pip install -U build pip && \
+    /venv/bin/python -m pip install -r requirements-dev.txt && \
+    /venv/bin/python -m pip list
+
+FROM gdal
+COPY . .
+RUN /venv/bin/python -m build -o wheels
+RUN /venv/bin/python -m pip install --no-index -f wheels fiona[test]
+ENTRYPOINT ["/venv/bin/fio"]
+CMD ["--help"]

Makefile

@@ -0,0 +1,54 @@
+PYTHON_VERSION ?= 3.9
+GDAL ?= ubuntu-small-3.6.4
+all: deps clean install test
+
+.PHONY: docs
+
+install:
+	python setup.py build_ext
+	pip install -e .[all]
+
+deps:
+	pip install -r requirements-dev.txt
+
+clean:
+	pip uninstall -y fiona || echo "no need to uninstall"
+	python setup.py clean --all
+	find . -name '__pycache__' -delete -print -o -name '*.pyc' -delete -print
+	touch fiona/*.pyx
+
+sdist:
+	python setup.py sdist
+
+test:
+	python -m pytest --maxfail 1 -v --cov fiona --cov-report html --pdb tests
+
+docs:
+	cd docs && make apidocs && make html
+
+doctest:
+	py.test --doctest-modules fiona --doctest-glob='*.rst' docs/*.rst
+
+dockertestimage:
+	docker build --target gdal --build-arg GDAL=$(GDAL) --build-arg PYTHON_VERSION=$(PYTHON_VERSION) -t fiona:$(GDAL)-py$(PYTHON_VERSION) .
+
+dockertest: dockertestimage
+	docker run -it -v $(shell pwd):/app -v /tmp:/tmp --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY --entrypoint=/bin/bash fiona:$(GDAL)-py$(PYTHON_VERSION) -c '/venv/bin/python -m pip install --editable . --no-build-isolation && /venv/bin/python -B -m pytest -m "not wheel" --cov fiona --cov-report term-missing $(OPTS)'
+
+dockershell: dockertestimage
+	docker run -it -v $(shell pwd):/app --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY --entrypoint=/bin/bash fiona:$(GDAL)-py$(PYTHON_VERSION) -c '/venv/bin/python -m pip install --editable . --no-build-isolation && /bin/bash'
+
+dockersdist: dockertestimage
+	docker run -it -v $(shell pwd):/app --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY --entrypoint=/bin/bash fiona:$(GDAL)-py$(PYTHON_VERSION) -c '/venv/bin/python -m build --sdist'
+
+dockergdb: dockertestimage
+	docker run -it -v $(shell pwd):/app --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY --entrypoint=/bin/bash fiona:$(GDAL)-py$(PYTHON_VERSION) -c '/venv/bin/python -m pip install --editable . --no-build-isolation && gdb -ex=r --args /venv/bin/python -B -m pytest -m "not wheel" --cov fiona --cov-report term-missing $(OPTS)'
+
+dockerdocs: dockertestimage
+	docker run -it -v $(shell pwd):/app --entrypoint=/bin/bash fiona:$(GDAL)-py$(PYTHON_VERSION) -c 'source /venv/bin/activate && cd docs && make clean && make html'
+
+dockertestimage-amd64:
+	docker build --platform linux/amd64 --target gdal --build-arg GDAL=$(GDAL) --build-arg PYTHON_VERSION=$(PYTHON_VERSION) -t fiona-amd64:$(GDAL)-py$(PYTHON_VERSION) .
+
+dockertest-amd64: dockertestimage-amd64
+	docker run -it -v $(shell pwd):/app -v /tmp:/tmp --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY --entrypoint=/bin/bash fiona-amd64:$(GDAL)-py$(PYTHON_VERSION) -c '/venv/bin/python -m pip install --editable . --no-build-isolation && /venv/bin/python -B -m pytest -m "not wheel" --cov fiona --cov-report term-missing $(OPTS)'

fiona/__init__.py

@@ -20,6 +20,7 @@
 
 """
 
+from contextlib import ExitStack
 import glob
 import logging
 import os
@@ -46,6 +47,7 @@
 )
 from fiona._env import driver_count
 from fiona._show_versions import show_versions
+from fiona._vsiopener import _opener_registration
 from fiona.collection import BytesCollection, Collection
 from fiona.drvsupport import supported_drivers
 from fiona.env import ensure_env_with_credentials, Env
@@ -60,7 +62,7 @@
     _remove,
     _remove_layer,
 )
-from fiona.path import ParsedPath, parse_path, vsi_path
+from fiona._path import _ParsedPath, _UnparsedPath, _parse_path, _vsi_path
 from fiona.vfs import parse_paths as vfs_parse_paths
 
 # These modules are imported by fiona.ogrext, but are also import here to
@@ -82,7 +84,7 @@
     "remove",
 ]
 
-__version__ = "2.0dev"
+__version__ = "1.10dev"
 __gdal_version__ = get_gdal_release_name()
 
 gdal_version = get_gdal_version_tuple()
@@ -104,6 +106,7 @@ def open(
     enabled_drivers=None,
     crs_wkt=None,
     allow_unsupported_drivers=False,
+    opener=None,
     **kwargs
 ):
     """Open a collection for read, append, or write
@@ -191,6 +194,19 @@ def open(
         Defaults to GDAL's default (WKT1_GDAL for GDAL 3).
     allow_unsupported_drivers : bool
         If set to true do not limit GDAL drivers to set set of known working.
+    opener : callable or obj, optional
+        A custom dataset opener which can serve GDAL's virtual
+        filesystem machinery via Python file-like objects. The
+        underlying file-like object is obtained by calling *opener* with
+        (*fp*, *mode*) or (*fp*, *mode* + "b") depending on the format
+        driver's native mode. *opener* must return a Python file-like
+        object that provides read, seek, tell, and close methods. Note:
+        only one opener at a time per fp, mode pair is allowed.
+
+        Alternatively, opener may be a filesystem object from a package
+        like fsspec that provides the following methods: isdir(),
+        isfile(), ls(), mtime(), open(), and size(). The exact interface
+        is defined in the fiona._vsiopener._AbstractOpener class.
     kwargs : mapping
         Other driver-specific parameters that will be interpreted by
         the OGR library as layer creation or opening options.
@@ -273,49 +289,67 @@ def func(*args, **kwds):
     # At this point, the fp argument is a string or path-like object
     # which can be converted to a string.
     else:
-        # If a pathlib.Path instance is given, convert it to a string path.
-        if isinstance(fp, Path):
-            fp = str(fp)
+        stack = ExitStack()
 
-        if vfs:
-            warnings.warn(
-                "The vfs keyword argument is deprecated and will be removed in version 2.0.0. Instead, pass a URL that uses a zip or tar (for example) scheme.",
-                FionaDeprecationWarning,
-                stacklevel=2,
-            )
-            path, scheme, archive = vfs_parse_paths(fp, vfs=vfs)
-            path = ParsedPath(path, archive, scheme)
+        if hasattr(fp, "path") and hasattr(fp, "fs"):
+            log.debug("Detected fp is an OpenFile: fp=%r", fp)
+            raw_dataset_path = fp.path
+            opener = fp.fs.open
         else:
-            path = parse_path(fp)
-
-        if mode in ("a", "r"):
-            colxn = Collection(
-                path,
-                mode,
-                driver=driver,
-                encoding=encoding,
-                layer=layer,
-                enabled_drivers=enabled_drivers,
-                allow_unsupported_drivers=allow_unsupported_drivers,
-                **kwargs
-            )
-        elif mode == "w":
-            colxn = Collection(
-                path,
-                mode,
-                crs=crs,
-                driver=driver,
-                schema=schema,
-                encoding=encoding,
-                layer=layer,
-                enabled_drivers=enabled_drivers,
-                crs_wkt=crs_wkt,
-                allow_unsupported_drivers=allow_unsupported_drivers,
-                **kwargs
-            )
-        else:
-            raise ValueError("mode string must be one of {'r', 'w', 'a'}")
-
+            raw_dataset_path = os.fspath(fp)
+
+        try:
+            if opener:
+                log.debug("Registering opener: raw_dataset_path=%r, mode=%r, opener=%r", raw_dataset_path, mode, opener)
+                vsi_path_ctx = _opener_registration(raw_dataset_path, mode[0], opener)
+                registered_vsi_path = stack.enter_context(vsi_path_ctx)
+                log.debug("Registered vsi path: registered_vsi_path%r", registered_vsi_path)
+                path = _UnparsedPath(registered_vsi_path)
+            else:
+                if vfs:
+                    warnings.warn(
+                        "The vfs keyword argument is deprecated and will be removed in version 2.0.0. Instead, pass a URL that uses a zip or tar (for example) scheme.",
+                        FionaDeprecationWarning,
+                        stacklevel=2,
+                    )
+                    path, scheme, archive = vfs_parse_paths(fp, vfs=vfs)
+                    path = _ParsedPath(path, archive, scheme)
+                else:
+                    path = _parse_path(fp)
+
+            if mode in ("a", "r"):
+                colxn = Collection(
+                    path,
+                    mode,
+                    driver=driver,
+                    encoding=encoding,
+                    layer=layer,
+                    enabled_drivers=enabled_drivers,
+                    allow_unsupported_drivers=allow_unsupported_drivers,
+                    **kwargs
+                )
+            elif mode == "w":
+                colxn = Collection(
+                    path,
+                    mode,
+                    crs=crs,
+                    driver=driver,
+                    schema=schema,
+                    encoding=encoding,
+                    layer=layer,
+                    enabled_drivers=enabled_drivers,
+                    crs_wkt=crs_wkt,
+                    allow_unsupported_drivers=allow_unsupported_drivers,
+                    **kwargs
+                )
+            else:
+                raise ValueError("mode string must be one of {'r', 'w', 'a'}")
+
+        except Exception:
+            stack.close()
+            raise
+
+        colxn._env = stack
         return colxn
 
 
@@ -392,8 +426,8 @@ def listdir(fp):
     if not isinstance(fp, str):
         raise TypeError("invalid path: %r" % fp)
 
-    pobj = parse_path(fp)
-    return _listdir(vsi_path(pobj))
+    pobj = _parse_path(fp)
+    return _listdir(_vsi_path(pobj))
 
 
 @ensure_env_with_credentials
@@ -442,13 +476,13 @@ def listlayers(fp, vfs=None, **kwargs):
                 FionaDeprecationWarning,
                 stacklevel=2,
             )
-            pobj_vfs = parse_path(vfs)
-            pobj_path = parse_path(fp)
-            pobj = ParsedPath(pobj_path.path, pobj_vfs.path, pobj_vfs.scheme)
+            pobj_vfs = _parse_path(vfs)
+            pobj_path = _parse_path(fp)
+            pobj = _ParsedPath(pobj_path.path, pobj_vfs.path, pobj_vfs.scheme)
         else:
-            pobj = parse_path(fp)
+            pobj = _parse_path(fp)
 
-        return _listlayers(vsi_path(pobj), **kwargs)
+        return _listlayers(_vsi_path(pobj), **kwargs)
 
 
 def prop_width(val):

fiona/_env.pyx

@@ -17,6 +17,7 @@ import threading
 
 from fiona._err cimport exc_wrap_int, exc_wrap_ogrerr
 from fiona._err import CPLE_BaseError
+from fiona._vsiopener cimport install_pyopener_plugin
 from fiona.errors import EnvError
 
 level_map = {
@@ -60,7 +61,7 @@ except ImportError:
     pass
 
 
-
+cdef VSIFilesystemPluginCallbacksStruct* pyopener_plugin = NULL
 cdef bint is_64bit = sys.maxsize > 2 ** 32
 
 cdef void set_proj_search_path(object path):
@@ -408,6 +409,7 @@ cdef class GDALEnv(ConfigEnv):
 
                     GDALAllRegister()
                     OGRRegisterAll()
+                    install_pyopener_plugin(pyopener_plugin)
 
                     if 'GDAL_DATA' in os.environ:
                         log.debug("GDAL_DATA found in environment.")