From 7c3408aaf2f8495542084c30c173730bf189196a Mon Sep 17 00:00:00 2001 From: Luis Antonio Obis Aparicio <35803280+lobis@users.noreply.github.com> Date: Thu, 7 Dec 2023 14:29:02 -0600 Subject: [PATCH] fix: correctly handle local files with colons (`:`) in name (#1452) --------- Co-authored-by: Martin Durant --- fsspec/core.py | 2 +- fsspec/generic.py | 7 +++++-- fsspec/implementations/tests/test_local.py | 21 ++++++++++++++++++++- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/fsspec/core.py b/fsspec/core.py index a1e15b2eb..8bc8e8e9b 100644 --- a/fsspec/core.py +++ b/fsspec/core.py @@ -521,7 +521,7 @@ def split_protocol(urlpath): if len(protocol) > 1: # excludes Windows paths return protocol, path - if ":" in urlpath and urlpath.find(":") > 1: + if urlpath.startswith("data:"): return urlpath.split(":", 1) return None, urlpath diff --git a/fsspec/generic.py b/fsspec/generic.py index 290bb436a..20534cf40 100644 --- a/fsspec/generic.py +++ b/fsspec/generic.py @@ -250,9 +250,12 @@ async def _pipe_file( return fs.pipe_file(path, value, **kwargs) async def _rm(self, url, **kwargs): - fs = _resolve_fs(url, self.method) + urls = url + if isinstance(urls, str): + urls = [urls] + fs = _resolve_fs(urls[0], self.method) if fs.async_impl: - await fs._rm(url, **kwargs) + await fs._rm(urls, **kwargs) else: fs.rm(url, **kwargs) diff --git a/fsspec/implementations/tests/test_local.py b/fsspec/implementations/tests/test_local.py index 82eba0f02..2980eb7c7 100644 --- a/fsspec/implementations/tests/test_local.py +++ b/fsspec/implementations/tests/test_local.py @@ -33,7 +33,6 @@ ), } - csv_files = { ".test.fakedata.1.csv": (b"a,b\n" b"1,2\n"), ".test.fakedata.2.csv": (b"a,b\n" b"3,4\n"), @@ -56,6 +55,8 @@ def filetexts(d, open=open, mode="t"): try: os.chdir(dirname) for filename, text in d.items(): + if dirname := os.path.dirname(filename): + os.makedirs(dirname, exist_ok=True) f = open(filename, f"w{mode}") try: f.write(text) @@ -991,3 +992,21 @@ def test_cp_two_files(tmpdir): make_path_posix(os.path.join(target, "file0")), make_path_posix(os.path.join(target, "file1")), ] + + +@pytest.mark.skipif(WIN, reason="Windows does not support colons in filenames") +def test_issue_1447(): + files_with_colons = { + ".local:file:with:colons.txt": b"content1", + ".colons-after-extension.txt:after": b"content2", + ".colons-after-extension/file:colon.txt:before/after": b"content3", + } + with filetexts(files_with_colons, mode="b"): + for file, contents in files_with_colons.items(): + with fsspec.filesystem("file").open(file, "rb") as f: + assert f.read() == contents + + fs, urlpath = fsspec.core.url_to_fs(file) + assert isinstance(fs, fsspec.implementations.local.LocalFileSystem) + with fs.open(urlpath, "rb") as f: + assert f.read() == contents