Merge pull request #478 from djhoese/bugfix-output-dir-creation

Allow writers to create output directories if they don't exist

doc/source/conf.py

@@ -51,6 +51,7 @@ def __getattr__(cls, name):
     sys.modules[mod_name] = Mock()
 
 autodoc_mock_imports = ['h5netcdf', 'pyninjotiff', 'pygac', 'cf', 'glymur', 'pyhdf', 'osgeo', 'mipp']
+autoclass_content = 'both'  # append class __init__ docstring to the class docstring
 
 # -- General configuration -----------------------------------------------------
 
@@ -243,4 +244,6 @@ def __getattr__(cls, name):
     'xarray': ('https://xarray.pydata.org/en/stable', None),
     'dask': ('https://dask.pydata.org/en/latest', None),
     'pyresample': ('https://pyresample.readthedocs.io/en/stable', None),
+    'trollsift': ('https://trollsift.readthedocs.io/en/stable', None),
+    'trollimage': ('https://trollimage.readthedocs.io/en/stable', None),
 }

satpy/plugin_base.py

@@ -23,38 +23,39 @@
 """
 
 import logging
-import os
 import yaml
 
 from satpy.config import config_search_paths, get_environ_config_dir, recursive_dict_update
 
-try:
-    import configparser
-except ImportError:
-    from six.moves import configparser
-
 LOG = logging.getLogger(__name__)
 
 
 class Plugin(object):
+    """Base plugin class for all dynamically loaded and configured objects."""
+
+    def __init__(self, ppp_config_dir=None, default_config_filename=None, config_files=None, **kwargs):
+        """Load configuration files related to this plugin.
+
+        This initializes a `self.config` dictionary that can be used to customize the subclass.
 
-    """The base plugin class. It is not to be used as is, it has to be
-    inherited by other classes.
-    """
+        Args:
+            ppp_config_dir (str): Base "etc" directory for all configuration
+                files.
+            default_config_filename (str): Configuration filename to use if
+                no other files have been specified with `config_files`.
+            config_files (list or str): Configuration files to load instead
+                of those automatically found in `ppp_config_dir` and other
+                default configuration locations.
+            kwargs (dict): Unused keyword arguments.
 
-    def __init__(self,
-                 ppp_config_dir=None,
-                 default_config_filename=None,
-                 config_files=None,
-                 **kwargs):
+        """
         self.ppp_config_dir = ppp_config_dir or get_environ_config_dir()
 
         self.default_config_filename = default_config_filename
         self.config_files = config_files
         if self.config_files is None and self.default_config_filename is not None:
             # Specify a default
-            self.config_files = config_search_paths(
-                self.default_config_filename, self.ppp_config_dir)
+            self.config_files = config_search_paths(self.default_config_filename, self.ppp_config_dir)
         if not isinstance(self.config_files, (list, tuple)):
             self.config_files = [self.config_files]
 
@@ -64,5 +65,6 @@ def __init__(self,
                 self.load_yaml_config(config_file)
 
     def load_yaml_config(self, conf):
+        """Load a YAML configuration file and recursively update the overall configuration."""
         with open(conf) as fd:
             self.config = recursive_dict_update(self.config, yaml.load(fd))

satpy/tests/test_writers.py

@@ -273,7 +273,7 @@ def test_enhance_with_sensor_entry2(self):
 class TestYAMLFiles(unittest.TestCase):
     """Test and analyze the writer configuration files."""
 
-    def test_filename_matches_reader_name(self):
+    def test_filename_matches_writer_name(self):
         """Test that every writer filename matches the name in the YAML."""
         import yaml
 
@@ -432,6 +432,64 @@ def test_mixed(self):
         self.assertTrue(os.path.isfile(fname2))
 
 
+class TestBaseWriter(unittest.TestCase):
+    """Test the base writer class."""
+
+    def setUp(self):
+        """Set up tests."""
+        import tempfile
+        from datetime import datetime
+
+        from satpy.scene import Scene
+        import dask.array as da
+
+        ds1 = xr.DataArray(
+            da.zeros((100, 200), chunks=50),
+            dims=('y', 'x'),
+            attrs={'name': 'test',
+                   'start_time': datetime(2018, 1, 1, 0, 0, 0)}
+        )
+        self.scn = Scene()
+        self.scn['test'] = ds1
+
+        # Temp dir
+        self.base_dir = tempfile.mkdtemp()
+
+    def tearDown(self):
+        """Remove the temporary directory created for a test"""
+        try:
+            shutil.rmtree(self.base_dir, ignore_errors=True)
+        except OSError:
+            pass
+
+    def test_save_dataset_static_filename(self):
+        """Test saving a dataset with a static filename specified."""
+        self.scn.save_datasets(base_dir=self.base_dir, filename='geotiff.tif')
+        self.assertTrue(os.path.isfile(os.path.join(self.base_dir, 'geotiff.tif')))
+
+    def test_save_dataset_dynamic_filename(self):
+        """Test saving a dataset with a format filename specified."""
+        fmt_fn = 'geotiff_{name}_{start_time:%Y%m%d_%H%M%S}.tif'
+        exp_fn = 'geotiff_test_20180101_000000.tif'
+        self.scn.save_datasets(base_dir=self.base_dir, filename=fmt_fn)
+        self.assertTrue(os.path.isfile(os.path.join(self.base_dir, exp_fn)))
+
+    def test_save_dataset_dynamic_filename_with_dir(self):
+        """Test saving a dataset with a format filename that includes a directory."""
+        fmt_fn = os.path.join('{start_time:%Y%m%d}', 'geotiff_{name}_{start_time:%Y%m%d_%H%M%S}.tif')
+        exp_fn = os.path.join('20180101', 'geotiff_test_20180101_000000.tif')
+        self.scn.save_datasets(base_dir=self.base_dir, filename=fmt_fn)
+        self.assertTrue(os.path.isfile(os.path.join(self.base_dir, exp_fn)))
+
+        # change the filename pattern but keep the same directory
+        fmt_fn2 = os.path.join('{start_time:%Y%m%d}', 'geotiff_{name}_{start_time:%Y%m%d_%H}.tif')
+        exp_fn2 = os.path.join('20180101', 'geotiff_test_20180101_00.tif')
+        self.scn.save_datasets(base_dir=self.base_dir, filename=fmt_fn2)
+        self.assertTrue(os.path.isfile(os.path.join(self.base_dir, exp_fn2)))
+        # the original file should still exist
+        self.assertTrue(os.path.isfile(os.path.join(self.base_dir, exp_fn)))
+
+
 def suite():
     """The test suite for test_writers."""
     loader = unittest.TestLoader()
@@ -441,5 +499,6 @@ def suite():
     my_suite.addTest(loader.loadTestsFromTestCase(TestEnhancerUserConfigs))
     my_suite.addTest(loader.loadTestsFromTestCase(TestYAMLFiles))
     my_suite.addTest(loader.loadTestsFromTestCase(TestComputeWriterResults))
+    my_suite.addTest(loader.loadTestsFromTestCase(TestBaseWriter))
 
     return my_suite

satpy/writers/__init__.py

@@ -432,16 +432,36 @@ def compute_writer_results(results):
 
 
 class Writer(Plugin):
+    """Base Writer class for all other writers.
 
-    """Writer plugins. They must implement the *save_image* method. This is an
-    abstract class to be inherited.
+    A minimal writer subclass should implement the `save_dataset` method.
     """
 
-    def __init__(self,
-                 name=None,
-                 filename=None,
-                 base_dir=None,
-                 **kwargs):
+    def __init__(self, name=None, filename=None, base_dir=None, **kwargs):
+        """Initialize the writer object.
+
+        Args:
+            name (str): A name for this writer for log and error messages.
+                If this writer is configured in a YAML file its name should
+                match the name of the YAML file. Writer names may also appear
+                in output file attributes.
+            filename (str): Filename to save data to. This filename can and
+                should specify certain python string formatting fields to
+                differentiate between data written to the files. Any
+                attributes provided by the ``.attrs`` of a DataArray object
+                may be included. Format and conversion specifiers provided by
+                the :class:`trollsift <trollsift.parser.StringFormatter>`
+                package may also be used. Any directories in the provided
+                pattern will be created if they do not exist. Example::
+
+                    {platform_name}_{sensor}_{name}_{start_time:%Y%m%d_%H%M%S.tif
+
+            base_dir (str):
+                Base destination directories for all created files.
+            kwargs (dict): Additional keyword arguments to pass to the
+                :class:`~satpy.plugin_base.Plugin` class.
+
+            """
         # Load the config
         Plugin.__init__(self, **kwargs)
         self.info = self.config['writer']
@@ -456,10 +476,8 @@ def __init__(self,
             filename = kwargs.pop('file_pattern')
 
         # Use options from the config file if they weren't passed as arguments
-        self.name = self.info.get("name",
-                                  None) if name is None else name
-        self.file_pattern = self.info.get(
-            "filename", None) if filename is None else filename
+        self.name = self.info.get("name", None) if name is None else name
+        self.file_pattern = self.info.get("filename", None) if filename is None else filename
 
         if self.name is None:
             raise ValueError("Writer 'name' not provided")
@@ -468,6 +486,20 @@ def __init__(self,
 
     @classmethod
     def separate_init_kwargs(cls, kwargs):
+        """Helper class method to separate arguments between init and save methods.
+
+        Currently the :class:`~satpy.scene.Scene` is passed one set of
+        arguments to represent the Writer creation and saving steps. This is
+        not preferred for Writer structure, but provides a simpler interface
+        to users. This method splits the provided keyword arguments between
+        those needed for initialization and those needed for the ``save_dataset``
+        and ``save_datasets`` method calls.
+
+        Writer subclasses should try to prefer keyword arguments only for the
+        save methods only and leave the init keyword arguments to the base
+        classes when possible.
+
+        """
         # FUTURE: Don't pass Scene.save_datasets kwargs to init and here
         init_kwargs = {}
         kwargs = kwargs.copy()
@@ -477,6 +509,7 @@ def separate_init_kwargs(cls, kwargs):
         return init_kwargs, kwargs
 
     def create_filename_parser(self, base_dir):
+        """Create a :class:`trollsift.parser.Parser` object for later use."""
         # just in case a writer needs more complex file patterns
         # Set a way to create filenames if we were given a pattern
         if base_dir and self.file_pattern:
@@ -486,10 +519,21 @@ def create_filename_parser(self, base_dir):
         return parser.Parser(file_pattern) if file_pattern else None
 
     def get_filename(self, **kwargs):
+        """Create a filename where output data will be saved.
+
+        Args:
+            kwargs (dict): Attributes and other metadata to use for formatting
+                the previously provided `filename`.
+
+        """
         if self.filename_parser is None:
-            raise RuntimeError(
-                "No filename pattern or specific filename provided")
-        return self.filename_parser.compose(kwargs)
+            raise RuntimeError("No filename pattern or specific filename provided")
+        output_filename = self.filename_parser.compose(kwargs)
+        dirname = os.path.dirname(output_filename)
+        if dirname and not os.path.isdir(dirname):
+            LOG.info("Creating output directory: {}".format(dirname))
+            os.makedirs(dirname)
+        return output_filename
 
     def save_datasets(self, datasets, compute=True, **kwargs):
         """Save all datasets to one or more files.
@@ -575,17 +619,10 @@ def save_dataset(self, dataset, filename=None, fill_value=None,
 
 class ImageWriter(Writer):
 
-    def __init__(self,
-                 name=None,
-                 filename=None,
-                 enhancement_config=None,
-                 base_dir=None,
-                 **kwargs):
-        Writer.__init__(self, name, filename, base_dir,
-                        **kwargs)
+    def __init__(self, name=None, filename=None, enhancement_config=None, base_dir=None, **kwargs):
+        Writer.__init__(self, name, filename, base_dir, **kwargs)
         enhancement_config = self.info.get(
-            "enhancement_config",
-            None) if enhancement_config is None else enhancement_config
+            "enhancement_config", None) if enhancement_config is None else enhancement_config
 
         self.enhancer = Enhancer(ppp_config_dir=self.ppp_config_dir,
                                  enhancement_config_file=enhancement_config)

setup.py

@@ -31,7 +31,7 @@
 
 from setuptools import find_packages, setup
 
-requires = ['numpy >=1.4.1', 'pillow', 'pyresample >=1.10.0', 'trollsift',
+requires = ['numpy >=1.12', 'pillow', 'pyresample >=1.10.0', 'trollsift',
             'trollimage >=1.5.1', 'pykdtree', 'six', 'pyyaml', 'xarray >=0.10.1',
             'dask[array] >=0.17.1']
 
@@ -69,6 +69,8 @@
     'mitiff': ['libtiff'],
     # MultiScene:
     'animations': ['imageio'],
+    # Documentation:
+    'doc': ['sphinx'],
 }
 all_extras = []
 for extra_deps in extras_require.values():