hrit reader bugs

[hazbottles]

Describe the bug
Errors reading/loading HRIT files

To Reproduce

Have a directory of JMA HRIT files

filenames=find_files_and_readers(base_dir='./DK01_201101010030/', reader='hrit_jma')
scn = Scene(filenames=filenames)
scn.load(['B03'])

Expected behavior
Images are loaded into memory

Actual results

Trying to load some JMA HRIT files:

>>> filenames=find_files_and_readers(base_dir='./DK01_201101010030/', reader='hrit_jma')
>>> scn = Scene(filenames=filenames)

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-2-5fb0dc98e3f1> in <module>()
      1 filenames=find_files_and_readers(base_dir='./DK01_201101010030/', reader='hrit_jma')
----> 2 scn = Scene(filenames=filenames)

/mnt/c/Users/harry/gdrive/work/2018-02_rerun/data_sources/mtsat/ext/satpy/satpy/scene.py in __init__(self, filenames, reader, filter_parameters, reader_kwargs, ppp_config_dir, base_dir, sensor, start_time, end_time, area)
    148         self.readers = self.create_reader_instances(filenames=filenames,
    149                                                     reader=reader,
--> 150                                                     reader_kwargs=reader_kwargs)
    151         self.attrs.update(self._compute_metadata_from_readers())
    152         self.datasets = DatasetDict()

/mnt/c/Users/harry/gdrive/work/2018-02_rerun/data_sources/mtsat/ext/satpy/satpy/scene.py in create_reader_instances(self, filenames, reader, reader_kwargs)
    193                             reader=reader,
    194                             reader_kwargs=reader_kwargs,
--> 195                             ppp_config_dir=self.ppp_config_dir)
    196 
    197     @property

/mnt/c/Users/harry/gdrive/work/2018-02_rerun/data_sources/mtsat/ext/satpy/satpy/readers/__init__.py in load_readers(filenames, reader, reader_kwargs, ppp_config_dir)
    587             loadables = reader_instance.select_files_from_pathnames(readers_files)
    588         if loadables:
--> 589             reader_instance.create_filehandlers(loadables)
    590             reader_instances[reader_instance.name] = reader_instance
    591             remaining_filenames -= set(loadables)

/mnt/c/Users/harry/gdrive/work/2018-02_rerun/data_sources/mtsat/ext/satpy/satpy/readers/yaml_reader.py in create_filehandlers(self, filenames)
    485         for filetype, filetype_info in self.sorted_filetype_items():
    486             filehandlers = self.new_filehandlers_for_filetype(filetype_info,
--> 487                                                               filename_set)
    488 
    489             filename_set -= set([fhd.filename for fhd in filehandlers])

/mnt/c/Users/harry/gdrive/work/2018-02_rerun/data_sources/mtsat/ext/satpy/satpy/readers/yaml_reader.py in new_filehandlers_for_filetype(self, filetype_info, filenames)
    472                                                           filename_iter)
    473         filtered_iter = self.filter_fh_by_metadata(filehandler_iter)
--> 474         return list(filtered_iter)
    475 
    476     def create_filehandlers(self, filenames):

/mnt/c/Users/harry/gdrive/work/2018-02_rerun/data_sources/mtsat/ext/satpy/satpy/readers/yaml_reader.py in filter_fh_by_metadata(self, filehandlers)
    445     def filter_fh_by_metadata(self, filehandlers):
    446         """Filter out filehandlers using provide filter parameters."""
--> 447         for filehandler in filehandlers:
    448             filehandler.metadata['start_time'] = filehandler.start_time
    449             filehandler.metadata['end_time'] = filehandler.end_time

/mnt/c/Users/harry/gdrive/work/2018-02_rerun/data_sources/mtsat/ext/satpy/satpy/readers/yaml_reader.py in new_filehandler_instances(self, filetype_info, filename_items)
    389                 continue
    390 
--> 391             yield filetype_cls(filename, filename_info, filetype_info, *req_fh)
    392 
    393     def time_matches(self, fstart, fend):

/mnt/c/Users/harry/gdrive/work/2018-02_rerun/data_sources/mtsat/ext/satpy/satpy/readers/hrit_jma.py in __init__(self, filename, filename_info, filetype_info)
    116         self.mda['planned_start_segment_number'] = 1
    117 
--> 118         items = self.mda['image_data_function'].split('\r')
    119         if items[0].startswith('$HALFTONE'):
    120             self.calibration_table = []

TypeError: a bytes-like object is required, not 'str'

Upon fixing the above, this error would crop up:

/mnt/c/Users/harry/gdrive/work/2018-02_rerun/data_sources/mtsat/ext/satpy/satpy/readers/hrit_jma.py in __init__(self, filename, filename_info, filetype_info)
    134             self.calibration_table = np.array(self.calibration_table)
    135 
--> 136         sublon = float(self.mda['projection_name'].strip().split('(')[1][:-1])
    137         self.mda['projection_parameters']['SSP_longitude'] = sublon
    138 

TypeError: a bytes-like object is required, not 'str'

Upon trying to load a band:

>>> scn.load(['B03'])

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-3-6b4b661f8be8> in <module>()
----> 1 scn.load(['B03'])

/mnt/c/Users/harry/gdrive/work/2018-02_rerun/data_sources/mtsat/ext/satpy/satpy/scene.py in load(self, wishlist, calibration, resolution, polarization, level, generate, unload, **kwargs)
    845             raise KeyError("Unknown datasets: {}".format(unknown_str))
    846 
--> 847         self.read(**kwargs)
    848         if generate:
    849             keepables = self.generate_composites()

/mnt/c/Users/harry/gdrive/work/2018-02_rerun/data_sources/mtsat/ext/satpy/satpy/scene.py in read(self, nodes, **kwargs)
    753             required_nodes = self.wishlist - set(self.datasets.keys())
    754             nodes = self.dep_tree.leaves(nodes=required_nodes)
--> 755         return self._read_datasets(nodes, **kwargs)
    756 
    757     def generate_composites(self, nodes=None):

/mnt/c/Users/harry/gdrive/work/2018-02_rerun/data_sources/mtsat/ext/satpy/satpy/scene.py in _read_datasets(self, dataset_nodes, **kwargs)
    612         for reader_name, ds_ids in reader_datasets.items():
    613             reader_instance = self.readers[reader_name]
--> 614             new_datasets = reader_instance.load(ds_ids, **kwargs)
    615             loaded_datasets.update(new_datasets)
    616         self.datasets.update(loaded_datasets)

/mnt/c/Users/harry/gdrive/work/2018-02_rerun/data_sources/mtsat/ext/satpy/satpy/readers/yaml_reader.py in load(self, dataset_keys, previous_datasets)
    775             coords = [all_datasets.get(cid, None)
    776                       for cid in coordinates.get(dsid, [])]
--> 777             ds = self._load_dataset_with_area(dsid, coords)
    778             if ds is not None:
    779                 all_datasets[dsid] = ds

/mnt/c/Users/harry/gdrive/work/2018-02_rerun/data_sources/mtsat/ext/satpy/satpy/readers/yaml_reader.py in _load_dataset_with_area(self, dsid, coords)
    719 
    720         try:
--> 721             ds = self._load_dataset_data(file_handlers, dsid, **slice_kwargs)
    722         except (KeyError, ValueError) as err:
    723             logger.exception("Could not load dataset '%s': %s", dsid, str(err))

/mnt/c/Users/harry/gdrive/work/2018-02_rerun/data_sources/mtsat/ext/satpy/satpy/readers/yaml_reader.py in _load_dataset_data(self, file_handlers, dsid, xslice, yslice)
    573                            yslice=slice(None)):
    574         ds_info = self.ids[dsid]
--> 575         proj = self._load_dataset(dsid, ds_info, file_handlers)
    576         # FIXME: areas could be concatenated here
    577         # Update the metadata

/mnt/c/Users/harry/gdrive/work/2018-02_rerun/data_sources/mtsat/ext/satpy/satpy/readers/yaml_reader.py in _load_dataset(dsid, ds_info, file_handlers, dim)
    545         for fh in file_handlers:
    546             try:
--> 547                 projectable = fh.get_dataset(dsid, ds_info)
    548                 if projectable is not None:
    549                     slice_list.append(projectable)

/mnt/c/Users/harry/gdrive/work/2018-02_rerun/data_sources/mtsat/ext/satpy/satpy/readers/hrit_jma.py in get_dataset(self, key, info)
    184     def get_dataset(self, key, info):
    185         """Get the dataset designated by *key*."""
--> 186         res = super(HRITJMAFileHandler, self).get_dataset(key, info)
    187 
    188         res = self.calibrate(res, key.calibration)

/mnt/c/Users/harry/gdrive/work/2018-02_rerun/data_sources/mtsat/ext/satpy/satpy/readers/hrit_base.py in get_dataset(self, key, info)
    167         """Load a dataset."""
    168         # Read bands
--> 169         data = self.read_band(key, info)
    170 
    171         # Convert to xarray

/mnt/c/Users/harry/gdrive/work/2018-02_rerun/data_sources/mtsat/ext/satpy/satpy/readers/hrit_base.py in read_band(self, key, info)
    253                          offset=self.mda['total_header_length'],
    254                          dtype=dtype,
--> 255                          shape=shape)
    256         data = da.from_array(data, chunks=shape[0])
    257         if self.mda['number_of_bits_per_pixel'] == 10:

~/miniconda3/lib/python3.6/site-packages/numpy/core/memmap.py in __new__(subtype, filename, dtype, mode, offset, shape, order)
    265 
    266         self = ndarray.__new__(subtype, shape, dtype=descr, buffer=mm,
--> 267                                offset=array_offset, order=order)
    268         self._mmap = mm
    269         self.offset = offset

TypeError: 'float' object cannot be interpreted as an integer

Environment Info:

• OS: Linux
• SatPy Version: 0.90
• PyResample Version: 1.9.3