From ce36af899e6c0f40ac59d4e1515c00970db7677f Mon Sep 17 00:00:00 2001 From: Heiko Mueller Date: Tue, 11 May 2021 14:30:45 +0200 Subject: [PATCH 1/4] Avoid PermissionError for temporary log file on MS Windows --- drama/process.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/drama/process.py b/drama/process.py index e944aef..649a390 100644 --- a/drama/process.py +++ b/drama/process.py @@ -1,5 +1,7 @@ import json +import os import tempfile + from abc import abstractmethod from datetime import datetime from enum import Enum @@ -62,7 +64,11 @@ def __init__( # logging self.logger = get_logger(__name__) - self.logging_file = tempfile.NamedTemporaryFile(dir=storage.temp_dir) + # Note that opening a NamedTemporaryFile multiple times will cause a + # PermissionError on Windows. To avoid this, one solution is to set the + # delete flag to False. This, however, requires to delete the file manually + # when it is closed. Base on: https://stackoverflow.com/questions/23212435. + self.logging_file = tempfile.NamedTemporaryFile(dir=storage.temp_dir, delete=False) @abstractmethod def to_downstream(self, data: DataType) -> Message: @@ -302,8 +308,11 @@ def close(self, force_interruption: bool = False, remove_local_dir: bool = False # send log to remote storage logging_remote = self.storage.put_file(self.logging_file.name, rename=logging_filename) - # once uploaded, delete named temporal file in local temp dir + # once uploaded, delete named temporal file in local temp dir. self.logging_file.close() + # Delete the temporary file explicitly since it was opened with the + # delete flag set to False. + os.unlink(self.logging_file.name) if remove_local_dir: # remove local directory without deleting the logging file (always kept for debugging purposes) From 41ce782c74a337092d80a684a7e1e191532e39ad Mon Sep 17 00:00:00 2001 From: Heiko Mueller Date: Tue, 11 May 2021 14:31:20 +0200 Subject: [PATCH 2/4] Avoid OSError by isfile for URLs on MS Windows --- drama/storage/backend/local.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drama/storage/backend/local.py b/drama/storage/backend/local.py index 0d87819..3e72af3 100644 --- a/drama/storage/backend/local.py +++ b/drama/storage/backend/local.py @@ -1,3 +1,4 @@ +import os import shutil from pathlib import Path from typing import List, Optional, Union @@ -27,7 +28,11 @@ def put_file(self, file_path: Union[str, Path], rename: Optional[str] = None) -> return LocalResource(resource=str(file_path)) def get_file(self, data_file: str) -> str: - if not Path(data_file).is_file(): + # Use os.path.isfile(data_file) instead of Path(data_file).is_file() + # if the data_file may contain a URL. On WIndows systems, the latter + # will raise a OSError: [WinError 123]. As an alternative, catch the + # OSError and raise a FileNotFoundError. + if not os.path.isfile(data_file): raise FileNotFoundError() return data_file From 9e46d519cb1e4e3f98386d13c4605e68322d6375 Mon Sep 17 00:00:00 2001 From: Heiko Mueller Date: Tue, 11 May 2021 14:31:36 +0200 Subject: [PATCH 3/4] More expressive error message for invalid file names --- drama/storage/backend/minio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drama/storage/backend/minio.py b/drama/storage/backend/minio.py index b85e62d..8e9aee2 100644 --- a/drama/storage/backend/minio.py +++ b/drama/storage/backend/minio.py @@ -80,7 +80,7 @@ def put_file(self, file_path: Union[str, Path], rename: Optional[str] = None) -> def get_file(self, data_file: str) -> str: if not data_file.startswith("minio://"): - raise NotValidScheme("Object file prefix is invalid: expected `minio://`") + raise NotValidScheme(f"Object file prefix for '{data_file}' is invalid: expected `minio://`") # remove scheme and deconstruct path bucket_name, object_name = data_file[len("minio://") :].split("/", 1) From 7b6a0745d43d7358af889141113eca192674038b Mon Sep 17 00:00:00 2001 From: Heiko Mueller Date: Tue, 11 May 2021 14:31:55 +0200 Subject: [PATCH 4/4] Ignore storage volumes created when running dev environment using Docker --- .gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 5c8b508..8eba07c 100644 --- a/.gitignore +++ b/.gitignore @@ -46,4 +46,7 @@ dmypy.json # Log history *.log -*.log.* \ No newline at end of file +*.log.* + +# Storage volumes +volumes/