feat: pass kwargs for io operations

CHANGELOG.md

@@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Include kwargs in io `read_text` for use with internal `fsspec.open` call. ([372](https://github.com/stac-utils/stactools/pull/372))
+
 ### Changed
 
 ### Fixed

src/stactools/core/io/__init__.py

@@ -1,9 +1,11 @@
 """Input/output utility functions and definitions."""
 
-from typing import Callable, Optional
+import os
+from typing import Any, Callable, Optional
 
 import fsspec
-from pystac.stac_io import DefaultStacIO, StacIO
+from pystac.link import HREF
+from pystac.stac_io import StacIO
 
 from stactools.core import utils
 
@@ -14,7 +16,11 @@
 """
 
 
-def read_text(href: str, read_href_modifier: Optional[ReadHrefModifier] = None) -> str:
+def read_text(
+    href: str,
+    read_href_modifier: Optional[ReadHrefModifier] = None,
+    **kwargs: Any,
+) -> str:
     """Reads a string from an href.
 
     If ``read_href_modifier`` is provided, then ``href`` will be passed through
@@ -26,32 +32,44 @@ def read_text(href: str, read_href_modifier: Optional[ReadHrefModifier] = None)
         read_href_modifier (ReadHrefModifier, optional):
             A function to modify
             the provided href. Defaults to None.
+        **kwargs :
+            Arbitrary keyword arguments that may be utilized by the concrete
+            implementation.
 
     Returns:
         str: The text as read from the href.
     """
     if read_href_modifier is None:
-        return StacIO.default().read_text(href)
+        return StacIO.default().read_text(href, **kwargs)
     else:
-        return StacIO.default().read_text(read_href_modifier(href))
+        return StacIO.default().read_text(read_href_modifier(href), **kwargs)
 
 
-class FsspecStacIO(DefaultStacIO):
+class FsspecStacIO(StacIO):
     """A subclass of :py:class:`pystac.DefaultStacIO` that uses `fsspec
     <https://filesystem-spec.readthedocs.io/en/latest/>`_ for reads and writes.
     """
 
-    def read_text_from_href(self, href: str) -> str:
+    def read_text(self, source: HREF, *args: Any, **kwargs: Any) -> str:
+        """A concrete implementation of :meth:`StacIO.read_text
+        <pystac.StacIO.read_text>`. Converts the ``source`` argument to a string (if it
+        is not already) and delegates to :meth:`FsspecStacIO.read_text_from_href` for
+        opening and reading the file."""
+        href = str(os.fspath(source))
+        return self.read_text_from_href(href, **kwargs)
+
+    def read_text_from_href(self, href: str, **kwargs: Any) -> str:
         """Reads a file as a utf-8 string using `fsspec
         <https://filesystem-spec.readthedocs.io/en/latest/>`_
 
         Args:
             href (str): The href to read.
+            **kwargs: Additional keyword arguments to be passed to fsspec.open.
 
         Returns:
             str: The read text, decoded as utf-8 if necessary.
         """
-        with fsspec.open(href, "r") as f:
+        with fsspec.open(href, "r", **kwargs) as f:
             s = f.read()
             if isinstance(s, str):
                 return s
@@ -60,6 +78,14 @@ def read_text_from_href(self, href: str) -> str:
             else:
                 raise ValueError(f"Unable to decode data loaded from HREF: {href}")
 
+    def write_text(self, dest: HREF, txt: str, *args: Any, **kwargs: Any) -> None:
+        """A concrete implementation of :meth:`StacIO.write_text
+        <pystac.StacIO.write_text>`. Converts the ``dest`` argument to a string (if it
+        is not already) and delegates to :meth:`FsspecStacIO.write_text_from_href` for
+        opening and reading the file."""
+        href = str(os.fspath(dest))
+        return self.write_text_to_href(href, txt, **kwargs)
+
     def write_text_from_href(self, href: str, txt: str) -> None:
         utils.deprecate(
             "FsspecStacIO.write_text_from_href",
@@ -68,14 +94,15 @@ def write_text_from_href(self, href: str, txt: str) -> None:
         )
         return self.write_text_to_href(href, txt)
 
-    def write_text_to_href(self, href: str, txt: str) -> None:
+    def write_text_to_href(self, href: str, txt: str, **kwargs: Any) -> None:
         """Writes text to an href using fsspec.
 
         Args:
             href (str): The href to write to.
             txt (str): The text to write.
+            **kwargs: Additional keyword arguments to be passed to fsspec.open.
         """
-        with fsspec.open(href, "w") as destination:
+        with fsspec.open(href, "w", **kwargs) as destination:
             destination.write(txt)
 
 

src/stactools/core/io/xml.py

@@ -1,5 +1,5 @@
 from functools import lru_cache
-from typing import Callable, List, Optional, cast
+from typing import Any, Callable, List, Optional, cast
 
 from lxml import etree
 from lxml.etree import _Element as lxmlElement
@@ -159,7 +159,10 @@ def get_attr(self, attr: str) -> Optional[str]:
 
     @classmethod
     def from_file(
-        cls, href: str, read_href_modifier: Optional[ReadHrefModifier] = None
+        cls,
+        href: str,
+        read_href_modifier: Optional[ReadHrefModifier] = None,
+        **kwargs: Any,
     ) -> "XmlElement":
         """Reads an XmlElement from an href.
 
@@ -175,5 +178,5 @@ def from_file(
         Returns:
             XmlElement: The read XmlElement.
         """
-        text = read_text(href, read_href_modifier)
+        text = read_text(href, read_href_modifier, **kwargs)
         return cls(etree.fromstring(bytes(text, encoding="utf-8")))

tests/core/test_io.py

@@ -1,9 +1,12 @@
+import io
 import os
 import unittest
 from tempfile import TemporaryDirectory
+from unittest.mock import create_autospec, patch
 
 import pystac
 
+import stactools.core.io
 from stactools.core import use_fsspec
 
 
@@ -26,3 +29,13 @@ def test_fsspec_io(self):
             cat2 = pystac.read_file(os.path.join(tmp_dir, "catalog.json"))
             col2 = cat2.get_child("country-1")
             self.assertEqual(len(list(col2.get_children())), 2)
+
+    @patch("stactools.core.io.fsspec.open")
+    def test_fsspec_kwargs(self, mock_open):
+        open_file = create_autospec(io.TextIOBase)
+        open_file.read.return_value = "string"
+        use_fsspec()
+        url = "url"
+        mock_open.return_value.__enter__.return_value = open_file
+        stactools.core.io.read_text(url, requester_pays=True)
+        mock_open.assert_called_with(url, "r", requester_pays=True)