sync with upstream (#79)

plugins/openvino_plugin
components/annotation.py
components/shift_analyzer.py
components/dataset_base.py
components/dataset_item_storage.py
components/filter.py
components/dataset_storage.py
components/annotations (extracting from components/operations.py)
components/comparator.py (extracting from components/operations.py)
components/merge (extracting from components/operations.py)

src/datumaro/cli/commands/diff.py

@@ -8,8 +8,8 @@
 import os.path as osp
 from enum import Enum, auto
 
+from datumaro.components.comparator import DistanceComparator, EqualityComparator
 from datumaro.components.errors import ProjectNotFoundError
-from datumaro.components.operations import DistanceComparator, ExactComparator
 from datumaro.util import dump_json_file
 from datumaro.util.os_util import rmtree
 from datumaro.util.scope import on_error_do, scope_add, scoped
@@ -221,37 +221,21 @@ def diff_command(args):
     if args.method is ComparisonMethod.equality:
         if args.ignore_field:
             args.ignore_field = eq_default_if
-        comparator = ExactComparator(
+        comparator = EqualityComparator(
             match_images=args.match_images,
             ignored_fields=args.ignore_field,
             ignored_attrs=args.ignore_attr,
             ignored_item_attrs=args.ignore_item_attr,
+            all=args.all,
         )
-        matches, mismatches, a_extra, b_extra, errors = comparator.compare_datasets(
-            first_dataset, second_dataset
-        )
-
-        output = {
-            "mismatches": mismatches,
-            "a_extra_items": sorted(a_extra),
-            "b_extra_items": sorted(b_extra),
-            "errors": errors,
-        }
-        if args.all:
-            output["matches"] = matches
+        output = comparator.compare_datasets(first_dataset, second_dataset)
 
         output_file = osp.join(
             dst_dir, generate_next_file_name("diff", ext=".json", basedir=dst_dir)
         )
         log.info("Saving diff to '%s'" % output_file)
         dump_json_file(output_file, output, indent=True)
 
-        print("Found:")
-        print("The first project has %s unmatched items" % len(a_extra))
-        print("The second project has %s unmatched items" % len(b_extra))
-        print("%s item conflicts" % len(errors))
-        print("%s matching annotations" % len(matches))
-        print("%s mismatching annotations" % len(mismatches))
     elif args.method is ComparisonMethod.distance:
         comparator = DistanceComparator(iou_threshold=args.iou_thresh)
 

src/datumaro/cli/commands/merge.py

@@ -8,10 +8,10 @@
 import os.path as osp
 from collections import OrderedDict
 
-from datumaro.components.dataset import DEFAULT_FORMAT
+from datumaro.components.dataset import DEFAULT_FORMAT, Dataset
 from datumaro.components.environment import Environment
 from datumaro.components.errors import DatasetMergeError, DatasetQualityError, ProjectNotFoundError
-from datumaro.components.operations import IntersectMerge
+from datumaro.components.merge.intersect_merge import IntersectMerge
 from datumaro.components.project import ProjectBuildTargets
 from datumaro.util import dump_json_file
 from datumaro.util.scope import scope_add, scoped
@@ -230,7 +230,7 @@ def merge_command(args):
             quorum=args.quorum,
         )
     )
-    merged_dataset = merger(source_datasets)
+    merged_dataset = Dataset(source=merger(*source_datasets))
 
     merged_dataset.export(save_dir=dst_dir, format=converter, **export_args)
 

src/datumaro/components/abstracts/merger.py

@@ -6,7 +6,7 @@
 from typing import Dict, Optional, Sequence, Type
 
 from datumaro.components.annotation import Annotation
-from datumaro.components.dataset_base import IDataset
+from datumaro.components.dataset_base import DatasetInfo, IDataset
 from datumaro.components.dataset_item_storage import (
     DatasetItemStorage,
     DatasetItemStorageDatasetView,
@@ -23,16 +23,18 @@ def get_any_label_name(self, ann: Annotation, label_id: int) -> str:
 
 
 class IMergerContext(IMatcherContext):
+    @staticmethod
     @abstractmethod
-    def merge_infos(self, sources: Sequence[IDataset]) -> Dict:
+    def merge_infos(sources: Sequence[DatasetInfo]) -> Dict:
         raise NotImplementedError
 
     @abstractmethod
     def merge_categories(self, sources: Sequence[IDataset]) -> Dict:
         raise NotImplementedError
 
+    @staticmethod
     @abstractmethod
-    def merge_media_types(self, sources: Sequence[IDataset]) -> Optional[Type[MediaElement]]:
+    def merge_media_types(sources: Sequence[IDataset]) -> Optional[Type[MediaElement]]:
         raise NotImplementedError
 
     @abstractmethod