add KITTI detection and segmentation

datumaro/plugins/kitti_format/converter.py

@@ -31,7 +31,7 @@ class KittiConverter(Converter):
 
     @staticmethod
     def _split_tasks_string(s):
-        return [KittiTask[i.strip()] for i in s.split(',')]
+        return [KittiTask[i.strip().lower()] for i in s.split(',')]
 
     @staticmethod
     def _get_labelmap(s):
@@ -92,7 +92,7 @@ def apply(self):
                 os.makedirs(osp.join(self._save_dir, subset_name,
                     KittiPath.INSTANCES_DIR), exist_ok=True)
             for item in subset:
-                image_name = item.id+KittiPath.IMAGE_EXT
+                image_name = self._make_image_filename(item)
                 image_path = osp.join(subset_name, KittiPath.IMAGES_DIR,
                     image_name)
                 if self._save_images:
@@ -105,20 +105,20 @@ def apply(self):
                         instance_labels=[self._label_id_mapping(m.label)
                             for m in masks])
                     color_mask_path = osp.join(subset_name,
-                        KittiPath.SEMANTIC_RGB_DIR, image_name)
+                        KittiPath.SEMANTIC_RGB_DIR, item.id + KittiPath.MASK_EXT)
                     self.save_mask(osp.join(self._save_dir, color_mask_path),
                         compiled_class_mask.class_mask)
 
                     labelids_mask_path = osp.join(subset_name,
-                        KittiPath.SEMANTIC_DIR, image_name)
+                        KittiPath.SEMANTIC_DIR, item.id + KittiPath.MASK_EXT)
                     self.save_mask(osp.join(self._save_dir, labelids_mask_path),
                         compiled_class_mask.class_mask, apply_colormap=False,
                         dtype=np.int32)
 
                     compiled_instance_mask = CompiledMask.from_instance_masks(masks,
                         instance_labels=[(m.label << 8) + m.id for m in masks])
                     inst_path = osp.join(subset_name,
-                        KittiPath.INSTANCES_DIR, image_name)
+                        KittiPath.INSTANCES_DIR, item.id + KittiPath.MASK_EXT)
                     self.save_mask(osp.join(self._save_dir, inst_path),
                         compiled_instance_mask.class_mask, apply_colormap=False,
                         dtype=np.int32)

datumaro/plugins/kitti_format/extractor.py

@@ -10,7 +10,7 @@
 from datumaro.components.extractor import (SourceExtractor,
     AnnotationType, DatasetItem, Mask, Bbox
 )
-from datumaro.util.image import load_image
+from datumaro.util.image import load_image, find_images
 
 from .format import (
     KittiTask, KittiPath, KittiLabelMap, parse_label_map,
@@ -50,25 +50,23 @@ def _load_categories_segmentation(self, path):
     def _load_items(self):
         items = {}
 
-        for image_path in iglob(
-                osp.join(self._path, KittiPath.IMAGES_DIR, '**',
-                '*%s' % KittiPath.IMAGE_EXT),
-                recursive=True):
+        for image_path in find_images(osp.join(self._path, KittiPath.IMAGES_DIR),
+                                      recursive=True):
             image_name = osp.relpath(image_path, osp.join(self._path,
                 KittiPath.IMAGES_DIR))
-            sample_id = image_name.replace(KittiPath.IMAGE_EXT, '')
+            sample_id = osp.splitext(image_name)[0]
             anns = []
 
             instances_path = osp.join(self._path, KittiPath.INSTANCES_DIR,
-                image_name)
+                sample_id + KittiPath.MASK_EXT)
             if self._task == KittiTask.segmentation and \
                 osp.isfile(instances_path):
                 instances_mask = load_image(instances_path, dtype=np.int32)
                 segm_ids = np.unique(instances_mask)
                 for segm_id in segm_ids:
                     semantic_id = segm_id >> 8
                     ann_id = segm_id % 256
-                    isCrowd = bool(ann_id == 0)
+                    isCrowd = (ann_id == 0)
                     anns.append(Mask(
                         image=self._lazy_extract_mask(instances_mask, segm_id),
                         label=semantic_id, id=ann_id,

datumaro/plugins/kitti_format/format.py

@@ -74,6 +74,7 @@ class KittiPath:
     SEMANTIC_RGB_DIR = 'semantic_rgb'
     SEMANTIC_DIR = 'semantic'
     IMAGE_EXT = '.png'
+    MASK_EXT = '.png'
 
     LABELMAP_FILE = 'label_colors.txt'
 

datumaro/plugins/kitti_format/importer.py

@@ -75,3 +75,11 @@ def find_sources(path):
                 subsets.setdefault(subset_name, {})[task] = path
 
         return subsets
+
+class KittiDetectionImporter(KittiImporter):
+    _TASK = KittiTask.detection
+    _TASKS = { _TASK: KittiImporter._TASKS[_TASK] }
+
+class KittiSegmentationImporter(KittiImporter):
+    _TASK = KittiTask.segmentation
+    _TASKS = { _TASK: KittiImporter._TASKS[_TASK] }

docs/formats/cityscapes_user_manual.md

@@ -139,7 +139,7 @@ particular problems with Cityscapes dataset:
 datum create -o project
 datum add path -p project -f cityscapes ./Cityscapes/
 datum stats -p project
-datum export -p final_project -o dataset -f voc --overwrite  -- --save-images
+datum export -p final_project -o dataset -f voc -- --save-images
 ```
 
 ### Example 2. How to create custom Cityscapes-like dataset

docs/formats/kitti_user_manual.md

@@ -51,13 +51,13 @@ KITTI segmentation dataset directory should have the following structure:
 └─ Dataset/
     ├── testing/
     │   └── image_2/
-    │       ├── <name_1>.png
-    │       ├── <name_2>.png
+    │       ├── <name_1>.<img_ext>
+    │       ├── <name_2>.<img_ext>
     │       └── ...
     ├── training/
     │   ├── image_2/ # left color camera images
-    │   │   ├── <name_1>.png
-    │   │   ├── <name_2>.png
+    │   │   ├── <name_1>.<img_ext>
+    │   │   ├── <name_2>.<img_ext>
     │   │   └── ...
     │   ├── label_2/ # left color camera label files
     │   │   ├── <name_1>.txt
@@ -174,7 +174,7 @@ particular problems with KITTI dataset:
 datum create -o project
 datum add path -p project -f kitti ./KITTI/
 datum stats -p project
-datum export -p final_project -o dataset -f cityscapes --overwrite -- --save-images
+datum export -p final_project -o dataset -f cityscapes -- --save-images
 ```
 
 ### Example 2. How to create custom KITTI-like dataset

tests/test_kitti_format.py

@@ -14,11 +14,14 @@
     make_kitti_categories, make_kitti_detection_categories,
     parse_label_map, write_label_map,
 )
-from datumaro.plugins.kitti_format.importer import KittiImporter
+from datumaro.plugins.kitti_format.importer import (KittiImporter,
+    KittiDetectionImporter, KittiSegmentationImporter)
 from datumaro.util.image import Image
 from datumaro.util.test_utils import (TestDir, compare_datasets,
     test_save_and_load)
 
+from .requirements import Requirements, mark_requirement
+
 DUMMY_DATASET_DIR = osp.join(osp.dirname(__file__), 'assets',
     'kitti_dataset')
 
@@ -96,8 +99,30 @@ def test_can_import_detection(self):
 
         compare_datasets(self, source_dataset, parsed_dataset)
 
+    @mark_requirement(Requirements.DATUM_GENERAL_REQ)
     def test_can_detect_kitti(self):
-        self.assertTrue(KittiImporter.detect(DUMMY_DATASET_DIR))
+        matrix = [
+            # Whole dataset
+            (DUMMY_DATASET_DIR, KittiImporter),
+
+            # Subformats
+            (DUMMY_DATASET_DIR, KittiSegmentationImporter),
+            (DUMMY_DATASET_DIR, KittiDetectionImporter),
+
+            # Subsets of subformats
+            (osp.join(DUMMY_DATASET_DIR, 'kitti_detection'),
+                KittiDetectionImporter),
+            (osp.join(DUMMY_DATASET_DIR, 'kitti_detection', 'training'),
+                KittiDetectionImporter),
+            (osp.join(DUMMY_DATASET_DIR, 'kitti_segmentation'),
+                KittiSegmentationImporter),
+            (osp.join(DUMMY_DATASET_DIR, 'kitti_segmentation', 'training'),
+                KittiSegmentationImporter),
+        ]
+
+        for path, subtask in matrix:
+            with self.subTest(path=path, task=subtask):
+                self.assertTrue(subtask.detect(path))
 
 
 class TestExtractorBase(Extractor):
@@ -277,6 +302,7 @@ def __iter__(self):
                 partial(KittiConverter.convert, label_map='kitti',
                 save_images=True), test_dir)
 
+    @mark_requirement(Requirements.DATUM_GENERAL_REQ)
     def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self):
         class TestExtractor(TestExtractorBase):
             def __iter__(self):
@@ -409,6 +435,7 @@ def categories(self):
                 partial(KittiConverter.convert, label_map='source',
                 save_images=True), test_dir, target_dataset=DstExtractor())
 
+    @mark_requirement(Requirements.DATUM_GENERAL_REQ)
     def test_can_save_and_load_image_with_arbitrary_extension(self):
         class TestExtractor(TestExtractorBase):
             def __iter__(self):