From ca82f54cf6aa3f93228194855e5f490d0270d830 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B8=D1=85=D0=B0=D0=B8=D0=BB?= Date: Sun, 3 Nov 2024 14:59:19 +0300 Subject: [PATCH] v1.6.5rc1 (#48) * exclude examples * update load_json * add keypoints_crowd * format * fix hook * fix linter * upd history * bump version --- .pre-commit-config.yaml | 4 +- Makefile | 2 +- examples/crowdpose_example.ipynb | 182 +++++ faster_coco_eval/core/coco.py | 47 +- faster_coco_eval/core/cocoeval.py | 126 ++-- faster_coco_eval/core/faster_eval_api.py | 78 ++- faster_coco_eval/version.py | 2 +- history.md | 6 + pyproject.toml | 1 + tests/dataset/example_crowdpose_preds.json | 737 +++++++++++++++++++++ tests/dataset/example_crowdpose_val.json | 677 +++++++++++++++++++ tests/test_basic.py | 14 +- tests/test_boundary.py | 6 +- tests/test_crowdpose.py | 53 ++ 14 files changed, 1828 insertions(+), 107 deletions(-) create mode 100644 examples/crowdpose_example.ipynb create mode 100644 tests/dataset/example_crowdpose_preds.json create mode 100644 tests/dataset/example_crowdpose_val.json create mode 100644 tests/test_crowdpose.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 57a0db6..4cbc616 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -59,10 +59,8 @@ repos: hooks: # try to fix what is possible - id: ruff - args: ["--fix", "--exclude", "examples"] + args: ["--fix"] # perform formatting updates - id: ruff-format - args: ["--exclude", "examples"] # validate if all is fine with preview mode - id: ruff - args: ["--exclude", "examples"] diff --git a/Makefile b/Makefile index 197fcaa..9f01277 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ format: pre-commit run --all-files linter: - pre-commit check --all-files + ruff check --force-exclude clean: rm -rf build diff --git a/examples/crowdpose_example.ipynb b/examples/crowdpose_example.ipynb new file mode 100644 index 0000000..24c4a0c --- /dev/null +++ b/examples/crowdpose_example.ipynb @@ -0,0 +1,182 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "faster_coco_eval.__version__='1.6.4'\n" + ] + } + ], + "source": [ + "import logging\n", + "import numpy as np\n", + "import faster_coco_eval\n", + "from faster_coco_eval import COCO, COCOeval_faster\n", + "\n", + "print(f\"{faster_coco_eval.__version__=}\")\n", + "\n", + "logging.root.setLevel(\"INFO\")\n", + "logging.debug(\"Запись.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "gt_file = '../tests/dataset/example_crowdpose_val.json'\n", + "preds = '../tests/dataset/example_crowdpose_preds.json'" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "sigmas = np.array([\n", + " .79, .79, .72, .72, .62, .62, 1.07, 1.07, .87, .87, .89, .89, .79,\n", + " .79\n", + " ]) / 10.0" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:faster_coco_eval.core.cocoeval:Evaluate annotation type *keypoints_crowd*\n", + "INFO:faster_coco_eval.core.cocoeval:COCOeval_opt.evaluate() finished...\n", + "INFO:faster_coco_eval.core.cocoeval:DONE (t=0.00s).\n", + "INFO:faster_coco_eval.core.cocoeval:Accumulating evaluation results...\n", + "INFO:faster_coco_eval.core.cocoeval:COCOeval_opt.accumulate() finished...\n", + "INFO:faster_coco_eval.core.cocoeval:DONE (t=0.00s).\n", + "INFO:faster_coco_eval.core.cocoeval: Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets= 20 ] = 0.788\n", + "INFO:faster_coco_eval.core.cocoeval: Average Precision (AP) @[ IoU=0.50 | area= all | maxDets= 20 ] = 0.988\n", + "INFO:faster_coco_eval.core.cocoeval: Average Precision (AP) @[ IoU=0.75 | area= all | maxDets= 20 ] = 0.731\n", + "INFO:faster_coco_eval.core.cocoeval: Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 20 ] = 0.822\n", + "INFO:faster_coco_eval.core.cocoeval: Average Recall (AR) @[ IoU=0.50 | area= all | maxDets= 20 ] = 1.000\n", + "INFO:faster_coco_eval.core.cocoeval: Average Recall (AR) @[ IoU=0.75 | area= all | maxDets= 20 ] = 0.778\n", + "INFO:faster_coco_eval.core.cocoeval: Average Precision (AP) @[ IoU=0.50:0.95 | type= easy | maxDets= 20 ] = 1.000\n", + "INFO:faster_coco_eval.core.cocoeval: Average Precision (AP) @[ IoU=0.50:0.95 | type=medium | maxDets= 20 ] = 0.980\n", + "INFO:faster_coco_eval.core.cocoeval: Average Precision (AP) @[ IoU=0.50:0.95 | type= hard | maxDets= 20 ] = 0.412\n" + ] + }, + { + "data": { + "text/plain": [ + "{'AP_all': 0.7877215935879303,\n", + " 'AP_50': 0.9881188118811886,\n", + " 'AP_75': 0.7314356435643564,\n", + " 'AR_all': 0.8222222222222223,\n", + " 'AR_50': 1.0,\n", + " 'AR_75': 0.7777777777777778,\n", + " 'AP_easy': 1.0,\n", + " 'AP_medium': 0.9802,\n", + " 'AP_hard': 0.4116}" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cocoGt = COCO(gt_file)\n", + "cocoDt = cocoGt.loadRes(preds)\n", + "cocoEval = COCOeval_faster(cocoGt, cocoDt, 'keypoints_crowd', kpt_oks_sigmas=sigmas, use_area=False)\n", + "cocoEval.evaluate()\n", + "cocoEval.accumulate()\n", + "cocoEval.summarize()\n", + "\n", + "cocoEval.stats_as_dict" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Orig Code\n", + "\n", + "```py\n", + "from xtcocotools.coco import COCO\n", + "from xtcocotools.cocoeval import COCOeval\n", + "\n", + "\n", + "cocoGt = COCO(gt_file)\n", + "cocoDt = cocoGt.loadRes(preds)\n", + "cocoEval = COCOeval(cocoGt, cocoDt, 'keypoints_crowd', sigmas, use_area=False)\n", + "cocoEval.evaluate()\n", + "cocoEval.accumulate()\n", + "cocoEval.summarize()\n", + "```\n", + "\n", + "## Orig result\n", + "\n", + "```text\n", + "loading annotations into memory...\n", + "Done (t=0.00s)\n", + "creating index...\n", + "index created!\n", + "Loading and preparing results...\n", + "DONE (t=0.00s)\n", + "creating index...\n", + "index created!\n", + "Running per image evaluation...\n", + "Evaluate annotation type *keypoints_crowd*\n", + "DONE (t=0.00s).\n", + "Accumulating evaluation results...\n", + "DONE (t=0.00s).\n", + " Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets= 20 ] = 0.788\n", + " Average Precision (AP) @[ IoU=0.50 | area= all | maxDets= 20 ] = 0.988\n", + " Average Precision (AP) @[ IoU=0.75 | area= all | maxDets= 20 ] = 0.731\n", + " Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 20 ] = 0.822\n", + " Average Recall (AR) @[ IoU=0.50 | area= all | maxDets= 20 ] = 1.000\n", + " Average Recall (AR) @[ IoU=0.75 | area= all | maxDets= 20 ] = 0.778\n", + " Average Precision (AP) @[ IoU=0.50:0.95 | type= easy | maxDets= 20 ] = 1.000\n", + " Average Precision (AP) @[ IoU=0.50:0.95 | type=medium | maxDets= 20 ] = 0.980\n", + " Average Precision (AP) @[ IoU=0.50:0.95 | type= hard | maxDets= 20 ] = 0.412\n", + "```" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/faster_coco_eval/core/coco.py b/faster_coco_eval/core/coco.py index 4a82ab5..9df86f8 100644 --- a/faster_coco_eval/core/coco.py +++ b/faster_coco_eval/core/coco.py @@ -9,7 +9,7 @@ import time import warnings from collections import defaultdict -from typing import List, Optional, Union +from typing import Callable, List, Optional, Union import numpy as np @@ -28,6 +28,7 @@ def __init__( self, annotation_file: Optional[Union[str, dict, os.PathLike, pathlib.PosixPath]] = None, use_deepcopy: bool = False, + print_function: Callable = logger.debug, ): """Constructor of Microsoft COCO helper class. @@ -45,21 +46,17 @@ def __init__( {}, ) self.imgToAnns, self.catToImgs = defaultdict(list), defaultdict(list) - self.print_function = logger.debug + self.print_function = print_function self.use_deepcopy = use_deepcopy + self.annotation_file = annotation_file if annotation_file is not None: self._print_function("loading annotations into memory...") tic = time.time() - if type(annotation_file) in [str, os.PathLike, pathlib.PosixPath]: - self.dataset = self.load_json(annotation_file) - elif type(annotation_file) is dict: - if self.use_deepcopy: - self.dataset = copy.deepcopy(annotation_file) - else: - self.dataset = annotation_file.copy() + if type(annotation_file) in [str, os.PathLike, pathlib.PosixPath, dict, list]: + self.dataset = self.load_json(annotation_file, self.use_deepcopy) else: - self.dataset = None + raise TypeError(f"type {type(annotation_file)} is not supported") assert type(self.dataset) is dict, f"annotation file format {type(self.dataset)} not supported" self.print_function(f"Done (t={time.time() - tic:0.2f}s)") @@ -290,29 +287,36 @@ def loadImgs(self, ids: Union[List[int], int] = []) -> List[dict]: return [self.imgs[ids]] @staticmethod - def load_json(json_file: Optional[Union[str, os.PathLike]]) -> dict: + def load_json( + json_file: Union[str, os.PathLike, pathlib.PosixPath, dict, list], use_deepcopy: Optional[bool] = False + ) -> dict: """Load a json file. Args: - json_file (str or os.PathLike): Path to the json file + json_file (str or os.PathLike or dict or list): Path to the json file or data dict Return: data (dict): Loaded json data """ - - with open(json_file) as io: - _data = json.load(io) + if type(json_file) in [str, os.PathLike]: + with open(json_file) as io: + _data = json.load(io) + else: + if use_deepcopy: + return copy.deepcopy(json_file) + else: + return json_file.copy() return _data def loadRes( self, - resFile: Union[str, os.PathLike, np.ndarray], + resFile: Union[str, os.PathLike, pathlib.PosixPath, dict, list, np.ndarray], min_score: float = 0.0, ) -> "COCO": """Load result file and return a result api object. Args: - resFile (str) : file name of result file + resFile (str or os.PathLike or dict or list) : file name of result file min_score (float) : minimum score to consider a result Return: @@ -324,15 +328,12 @@ def loadRes( self.print_function("Loading and preparing results...") tic = time.time() - if type(resFile) in [str, os.PathLike]: - anns = self.load_json(resFile) + if type(resFile) in [str, os.PathLike, pathlib.PosixPath, dict, list]: + anns = self.load_json(resFile, self.use_deepcopy) elif type(resFile) is np.ndarray: anns = self.loadNumpyAnnotations(resFile) else: - if self.use_deepcopy: - anns = copy.deepcopy(resFile) - else: - anns = resFile.copy() + raise TypeError(f"type {type(resFile)} is not supported") assert type(anns) is list, "results in not an array of objects" diff --git a/faster_coco_eval/core/cocoeval.py b/faster_coco_eval/core/cocoeval.py index 52386a1..0e217c4 100644 --- a/faster_coco_eval/core/cocoeval.py +++ b/faster_coco_eval/core/cocoeval.py @@ -10,7 +10,7 @@ if sys.version_info >= (3, 8): from typing import Literal - iouTypeT = Literal["segm", "bbox", "keypoints", "boundary"] + iouTypeT = Literal["segm", "bbox", "keypoints", "keypoints_crowd", "boundary"] else: iouTypeT = str @@ -30,9 +30,10 @@ def __init__( cocoGt: Optional[COCO] = None, cocoDt: Optional[COCO] = None, iouType: iouTypeT = "segm", - print_function: Callable = logger.debug, + print_function: Callable = logger.info, extra_calc: bool = False, kpt_oks_sigmas: Optional[List[float]] = None, + use_area: Optional[bool] = True, lvis_style: bool = False, separate_eval: bool = False, boundary_dilation_ratio: float = 0.02, @@ -53,6 +54,8 @@ def __init__( whether to perform extra calculations, defaults to False kpt_oks_sigmas (None or list): list of sigmas for keypoint evaluation, defaults to None + use_area (bool): + If gt annotations (eg. CrowdPose) do not have 'area', please set use_area=False. lvis_style (bool): whether to use LVIS style evaluation, defaults to False separate_eval (bool): @@ -81,6 +84,7 @@ def __init__( self.separate_eval = separate_eval self.boundary_dilation_ratio = boundary_dilation_ratio self.boundary_cpu_count = boundary_cpu_count + self.use_area = use_area if iouType == "keypoints" and self.lvis_style: logger.warning("lvis_style not supported for keypoint evaluation") @@ -128,8 +132,8 @@ def _prepare(self): for gt in gts: gt["ignore"] = gt["ignore"] if "ignore" in gt else 0 gt["ignore"] = "iscrowd" in gt and gt["iscrowd"] - if p.iouType == "keypoints": - gt["ignore"] = (gt.get("num_keypoints") == 0) or gt["ignore"] + if "keypoints" in p.iouType: + gt["ignore"] = (gt.get("num_keypoints", 0) == 0) or gt["ignore"] img_pl = defaultdict(set) # per image list of categories present in image img_nl = {} # per image map of categories not present in image @@ -159,7 +163,6 @@ def get_img_size_by_id(image_id, dataset: COCO) -> tuple: if img_sizes.get(image_id) is None: t = dataset.imgs[image_id] img_sizes[image_id] = t["height"], t["width"] - return img_sizes[image_id] for gt in gts: if p.compute_rle: @@ -328,7 +331,13 @@ def computeOks(self, imgId: int, catId: int) -> np.ndarray: z = np.zeros(k) dx = np.max((z, x0 - xd), axis=0) + np.max((z, xd - x1), axis=0) dy = np.max((z, y0 - yd), axis=0) + np.max((z, yd - y1), axis=0) - e = (dx**2 + dy**2) / vars / (gt["area"] + np.spacing(1)) / 2 + + if self.use_area: + e = (dx**2 + dy**2) / vars / (gt["area"] + np.spacing(1)) / 2 + else: + tmparea = gt["bbox"][3] * gt["bbox"][2] * 0.53 + e = (dx**2 + dy**2) / vars / (tmparea + np.spacing(1)) / 2 + if k1 > 0: e = e[vg > 0] ious[i, j] = np.sum(np.exp(-e)) / e.shape[0] @@ -362,30 +371,12 @@ def _summarize( iouStr = f"{p.iouThrs[0]:0.2f}:{p.iouThrs[-1]:0.2f}" if iouThr is None else f"{iouThr:0.2f}" if catIds is not None: - # catNames = [ - # self.cocoGt.cats[p.catIds[used_cat_idx]].get( - # "name", used_cat_idx - # ) - # for used_cat_idx in catIds - # ] freq_str = f"catIds=={str(catIds):>3s}" if self.lvis_style and (freq_group_idx is not None): catIds = self.freq_groups[freq_group_idx] freq_str = f"catIds={p.imgCountLbl[freq_group_idx]:>3s}" - # In my case, C++ too slow... - # mean_s = _C._summarize( - # ap, - # (iouThr if iouThr else -1), - # areaRng, - # maxDets, - # catIds, - # p, - # self.eval["counts"], - # self.eval["precision"] if ap == 1 else self.eval["recall"], - # ) - aind = p.areaRngLbl.index(areaRng) mind = p.maxDets.index(maxDets) @@ -440,6 +431,7 @@ def summarize(self): def _summarizeDets(): _count = 17 if self.lvis_style else 14 stats = np.zeros((_count,)) + stats[0] = self._summarize(1, maxDets=self.params.maxDets[-1]) # AP_all stats[1] = self._summarize(1, iouThr=0.5, maxDets=self.params.maxDets[-1]) # AP_50 stats[2] = self._summarize(1, iouThr=0.75, maxDets=self.params.maxDets[-1]) # AP_75 @@ -452,8 +444,7 @@ def _summarizeDets(): stats[15] = self._summarize(1, maxDets=self.params.maxDets[-1], freq_group_idx=1) # APc stats[16] = self._summarize(1, maxDets=self.params.maxDets[-1], freq_group_idx=2) # APf - # AR_first or AR_all - stats[6] = self._summarize(0, maxDets=self.params.maxDets[0]) + stats[6] = self._summarize(0, maxDets=self.params.maxDets[0]) # AR_first or AR_all if len(self.params.maxDets) >= 2: stats[7] = self._summarize(0, maxDets=self.params.maxDets[1]) # AR_second if len(self.params.maxDets) >= 3: @@ -470,16 +461,42 @@ def _summarizeDets(): def _summarizeKps(): stats = np.zeros((10,)) - stats[0] = self._summarize(1, maxDets=self.params.maxDets[-1]) - stats[1] = self._summarize(1, maxDets=self.params.maxDets[-1], iouThr=0.5) - stats[2] = self._summarize(1, maxDets=self.params.maxDets[-1], iouThr=0.75) - stats[3] = self._summarize(1, maxDets=self.params.maxDets[-1], areaRng="medium") - stats[4] = self._summarize(1, maxDets=self.params.maxDets[-1], areaRng="large") - stats[5] = self._summarize(0, maxDets=self.params.maxDets[-1]) - stats[6] = self._summarize(0, maxDets=self.params.maxDets[-1], iouThr=0.5) - stats[7] = self._summarize(0, maxDets=self.params.maxDets[-1], iouThr=0.75) - stats[8] = self._summarize(0, maxDets=self.params.maxDets[-1], areaRng="medium") - stats[9] = self._summarize(0, maxDets=self.params.maxDets[-1], areaRng="large") + stats[0] = self._summarize(1, maxDets=self.params.maxDets[-1]) # AP_all + stats[1] = self._summarize(1, maxDets=self.params.maxDets[-1], iouThr=0.5) # AP_50 + stats[2] = self._summarize(1, maxDets=self.params.maxDets[-1], iouThr=0.75) # AP_75 + stats[3] = self._summarize(1, maxDets=self.params.maxDets[-1], areaRng="medium") # AP_medium + stats[4] = self._summarize(1, maxDets=self.params.maxDets[-1], areaRng="large") # AP_large + stats[5] = self._summarize(0, maxDets=self.params.maxDets[-1]) # AR_all + stats[6] = self._summarize(0, maxDets=self.params.maxDets[-1], iouThr=0.5) # AR_50 + stats[7] = self._summarize(0, maxDets=self.params.maxDets[-1], iouThr=0.75) # AR_75 + stats[8] = self._summarize(0, maxDets=self.params.maxDets[-1], areaRng="medium") # AR_medium + stats[9] = self._summarize(0, maxDets=self.params.maxDets[-1], areaRng="large") # AR_large + return stats + + def _summarizeKps_crowd(): + stats = np.zeros((9,)) + stats[0] = self._summarize(1, maxDets=self.params.maxDets[-1]) # AP_all + stats[1] = self._summarize(1, maxDets=self.params.maxDets[-1], iouThr=0.5) # AP_50 + stats[2] = self._summarize(1, maxDets=self.params.maxDets[-1], iouThr=0.75) # AP_75 + stats[3] = self._summarize(0, maxDets=self.params.maxDets[-1]) # AR_all + stats[4] = self._summarize(0, maxDets=self.params.maxDets[-1], iouThr=0.5) # AR_50 + stats[5] = self._summarize(0, maxDets=self.params.maxDets[-1], iouThr=0.75) # AR_75 + type_result = self.get_type_result(first=0.2, second=0.8) + + p = self.params + iStr = " {:<18} {} @[ IoU={:<9} | type={:>6s} | maxDets={:>3d} ] = {:0.3f}" + titleStr = "Average Precision" + typeStr = "(AP)" + iouStr = f"{p.iouThrs[0]:0.2f}:{p.iouThrs[-1]:0.2f}" + self.print_function(iStr.format(titleStr, typeStr, iouStr, "easy", self.params.maxDets[-1], type_result[0])) + self.print_function( + iStr.format(titleStr, typeStr, iouStr, "medium", self.params.maxDets[-1], type_result[1]) + ) + self.print_function(iStr.format(titleStr, typeStr, iouStr, "hard", self.params.maxDets[-1], type_result[2])) + stats[6] = type_result[0] # AP_easy + stats[7] = type_result[1] # AP_medium + stats[8] = type_result[2] # AP_hard + return stats if not self.eval: @@ -491,12 +508,45 @@ def _summarizeKps(): summarize = _summarizeDets elif iouType == "keypoints": summarize = _summarizeKps + elif iouType == "keypoints_crowd": + summarize = _summarizeKps_crowd else: - ValueError(f"iouType must be bbox, segm, boundary or keypoints. Get {iouType}") + ValueError(f"iouType must be bbox, segm, boundary or keypoints or keypoints_crowd. Get {iouType}") self.all_stats = summarize() self.stats = self.all_stats[:12] + def get_type_result(self, first=0.01, second=0.85): + easy, mid, hard = self.split(self.cocoGt.annotation_file, first, second) + res = [] + prev_print = self.print_function + self.print_function = lambda *args, **kwargs: None + + for curr_type in [easy, mid, hard]: + curr_list = curr_type + self.params.imgIds = curr_list + self.evaluate() + self.accumulate() + score = self.eval["precision"][:, :, :, 0, :] + res.append(round(np.mean(score), 4)) + + self.print_function = prev_print + return res + + def split(self, gt_file, first=0.01, second=0.85): + data = COCO.load_json(gt_file, use_deepcopy=True) + easy = [] + mid = [] + hard = [] + for item in data["images"]: + if item["crowdIndex"] < first: + easy.append(item["id"]) + elif item["crowdIndex"] < second: + mid.append(item["id"]) + else: + hard.append(item["id"]) + return easy, mid, hard + def __str__(self): self.summarize() return str(self.__repr__()) @@ -574,7 +624,7 @@ def __init__( if iouType in set(["segm", "bbox", "boundary"]): self.setDetParams() - elif iouType == "keypoints": + elif "keypoints" in iouType: self.setKpParams() if kpt_sigmas is not None: self.kpt_oks_sigmas = np.array(kpt_sigmas) diff --git a/faster_coco_eval/core/faster_eval_api.py b/faster_coco_eval/core/faster_eval_api.py index 01d7868..2059d78 100644 --- a/faster_coco_eval/core/faster_eval_api.py +++ b/faster_coco_eval/core/faster_eval_api.py @@ -48,7 +48,7 @@ def evaluate(self): if p.iouType in set(["segm", "bbox", "boundary"]): computeIoU = self.computeIoU - elif p.iouType == "keypoints": + elif "keypoints" in p.iouType: computeIoU = self.computeOks else: raise ValueError(f"p.iouType must be segm, bbox, boundary or keypoints. Get {p.iouType}") @@ -178,28 +178,54 @@ def run(self): @property def stats_as_dict(self): - labels = [ - "AP_all", - "AP_50", - "AP_75", - "AP_small", - "AP_medium", - "AP_large", - "AR_all", - "AR_second", - "AR_third", - "AR_small", - "AR_medium", - "AR_large", - ] - if self.params.iouType in set(["segm", "bbox", "boundary"]): - labels += ["AR_50", "AR_75"] - else: - labels = [label for label in labels if "small" not in label] + labels = [ + "AP_all", + "AP_50", + "AP_75", + "AP_small", + "AP_medium", + "AP_large", + "AR_all", + "AR_second", + "AR_third", + "AR_small", + "AR_medium", + "AR_large", + "AR_50", + "AR_75", + ] - if self.lvis_style: - labels += ["APr", "APc", "APf"] + if self.lvis_style: + labels += ["APr", "APc", "APf"] + + elif self.params.iouType == "keypoints": + labels = [ + "AP_all", + "AP_50", + "AP_75", + "AP_medium", + "AP_large", + "AR_all", + "AR_50", + "AR_75", + "AR_medium", + "AR_large", + ] + elif self.params.iouType == "keypoints_crowd": + labels = [ + "AP_all", + "AP_50", + "AP_75", + "AR_all", + "AR_50", + "AR_75", + "AP_easy", + "AP_medium", + "AP_hard", + ] + else: + ValueError(f"iouType must be bbox, segm, boundary or keypoints. Get {self.params.iouType}") if self.matched: labels += [ @@ -207,16 +233,6 @@ def stats_as_dict(self): "mAUC_" + str(int(self.params.iouThrs[0] * 100)), ] - maxDets = self.params.maxDets - if len(maxDets) > 1: - labels[6] = f"AR_{maxDets[0]}" - - if len(maxDets) >= 2: - labels[7] = f"AR_{maxDets[1]}" - - if len(maxDets) >= 3: - labels[8] = f"AR_{maxDets[2]}" - return {_label: float(self.all_stats[i]) for i, _label in enumerate(labels)} @staticmethod diff --git a/faster_coco_eval/version.py b/faster_coco_eval/version.py index 65fffdf..f2d9dfb 100644 --- a/faster_coco_eval/version.py +++ b/faster_coco_eval/version.py @@ -1,2 +1,2 @@ -__version__ = "1.6.4" +__version__ = "1.6.5rc1" __author__ = "MiXaiLL76" diff --git a/history.md b/history.md index 2f213d6..a712c84 100644 --- a/history.md +++ b/history.md @@ -1,5 +1,11 @@ # history +## v1.6.5 - dev + +- [x] Support CrowdPose Dataset + - [x] add CrowdPose simple test +- [x] ImageEvaluation pickleable https://github.com/MiXaiLL76/faster_coco_eval/issues/45 + ## v1.6.4 - [x] fix windows mask api. diff --git a/pyproject.toml b/pyproject.toml index 7cf6d11..7b052d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,3 +33,4 @@ lint.select = [ lint.mccabe.max-complexity = 10 # Use Google-style docstrings. lint.pydocstyle.convention = "google" +exclude = ["examples"] diff --git a/tests/dataset/example_crowdpose_preds.json b/tests/dataset/example_crowdpose_preds.json new file mode 100644 index 0000000..1bb7b0b --- /dev/null +++ b/tests/dataset/example_crowdpose_preds.json @@ -0,0 +1,737 @@ +[ + { + "image_id": 114203, + "category_id": 1, + "keypoints": [ + 351.26873779296875, + 265.73907470703125, + 0.8180673122406006, + 166.99375915527344, + 290.30908203125, + 0.7561607360839844, + 443.40625, + 364.01904296875, + 0.7576647996902466, + 99.42626953125, + 376.3040771484375, + 0.7234059572219849, + 455.69122314453125, + 462.299072265625, + 0.555873692035675, + 56.42875671386719, + 380.91094970703125, + 0.7035706043243408, + 363.5537414550781, + 516.0459594726562, + 0.8000281453132629, + 259.1312561035156, + 540.6159057617188, + 0.7953501343727112, + 486.40374755859375, + 638.8959350585938, + 0.8609960079193115, + 228.41876220703125, + 744.8540649414062, + 0.7511023879051208, + 517.1162719726562, + 798.6009521484375, + 0.815504789352417, + 228.41876220703125, + 896.8809814453125, + 0.8421406149864197, + 259.1312561035156, + 110.64093017578125, + 0.8916172385215759, + 259.1312561035156, + 241.1690673828125, + 0.8564639687538147 + ], + "score": 1.6721848377159665 + }, + { + "image_id": 114203, + "category_id": 1, + "keypoints": [ + 723.1052856445312, + 302.38336181640625, + 0.6796568632125854, + 568.1421508789062, + 295.2091064453125, + 0.850772500038147, + 728.8446655273438, + 352.6029052734375, + 0.9064595699310303, + 470.57281494140625, + 352.6029052734375, + 0.7553263902664185, + 694.408447265625, + 267.94708251953125, + 0.7736124992370605, + 395.9609375, + 371.255859375, + 0.8428546190261841, + 665.7115478515625, + 554.9158935546875, + 0.7592582702636719, + 579.6209106445312, + 524.7841186523438, + 0.7476921081542969, + 665.7115478515625, + 715.6183471679688, + 0.8434103727340698, + 505.009033203125, + 520.4796142578125, + 0.7899116277694702, + 671.4509887695312, + 910.7571411132812, + 0.8141639232635498, + 390.2215576171875, + 593.6566772460938, + 0.9809013605117798, + 619.7965698242188, + 191.900390625, + 0.8087974786758423, + 648.4934692382812, + 279.42584228515625, + 0.7796021103858948 + ], + "score": 1.7903599100453513 + }, + { + "image_id": 114203, + "category_id": 1, + "keypoints": [ + 293.3525085449219, + 331.036865234375, + 0.3973623514175415, + 149.83253479003906, + 309.7331237792969, + 9.999999747378752e-06, + 311.29248046875, + 435.3131103515625, + 0.31791025400161743, + 109.467529296875, + 417.37310791015625, + 9.999999747378752e-06, + 311.29248046875, + 525.0131225585938, + 0.30083543062210083, + 82.55754089355469, + 516.0430908203125, + 0.8508201837539673, + 248.5025177001953, + 555.286865234375, + 0.6527189016342163, + 176.74253845214844, + 546.31689453125, + 0.8271096348762512, + 226.07752990722656, + 695.443115234375, + 0.26841261982917786, + 172.2575225830078, + 668.5330810546875, + 0.794041633605957, + 235.0475311279297, + 806.4468994140625, + 0.12073180824518204, + 185.71253967285156, + 794.1130981445312, + 0.7710750102996826, + 140.86253356933594, + 282.8231201171875, + 0.08674563467502594, + 244.0175323486328, + 255.91311645507812, + 9.999999747378752e-06 + ], + "score": 1.2356625739653933 + }, + { + "image_id": 114203, + "category_id": 1, + "keypoints": [ + 355.60467529296875, + 268.69708251953125, + 9.999999747378752e-06, + 172.10406494140625, + 292.3746032714844, + 9.999999747378752e-06, + 444.3952941894531, + 363.4071044921875, + 9.999999747378752e-06, + 89.2328109741211, + 375.245849609375, + 9.999999747378752e-06, + 391.1209411621094, + 367.84661865234375, + 0.3527158498764038, + 65.55531311035156, + 379.68536376953125, + 9.999999747378752e-06, + 367.44342041015625, + 517.3108520507812, + 9.999999747378752e-06, + 254.97531127929688, + 540.9883422851562, + 9.999999747378752e-06, + 485.8309631347656, + 640.137939453125, + 9.999999747378752e-06, + 225.37843322753906, + 742.2470703125, + 9.999999747378752e-06, + 515.4278564453125, + 794.0416259765625, + 9.999999747378752e-06, + 231.29782104492188, + 900.59033203125, + 9.999999747378752e-06, + 260.89471435546875, + 114.79336547851562, + 9.999999747378752e-06, + 254.97531127929688, + 237.62039184570312, + 9.999999747378752e-06 + ], + "score": 0.3779191248673409 + }, + { + "image_id": 114203, + "category_id": 1, + "keypoints": [ + 353.15472412109375, + 267.5152282714844, + 9.999999747378752e-06, + 166.45469665527344, + 290.85272216796875, + 9.999999747378752e-06, + 440.67034912109375, + 360.865234375, + 9.999999747378752e-06, + 102.27656555175781, + 376.9097595214844, + 9.999999747378752e-06, + 393.995361328125, + 376.9097595214844, + 0.6492501497268677, + 84.77345275878906, + 500.8902282714844, + 0.36296719312667847, + 253.97032165527344, + 535.896484375, + 0.19618932902812958, + 172.28907775878906, + 528.603515625, + 0.2696319818496704, + 464.0078125, + 621.9534912109375, + 9.999999747378752e-06, + 166.45469665527344, + 675.9215087890625, + 0.39008715748786926, + 178.1234588623047, + 792.6090087890625, + 9.999999747378752e-06, + 189.7921905517578, + 792.6090087890625, + 0.4075685143470764, + 259.8046875, + 115.82148742675781, + 9.999999747378752e-06, + 253.97032165527344, + 236.884765625, + 9.999999747378752e-06 + ], + "score": 0.8118054586957442 + }, + { + "image_id": 107292, + "category_id": 1, + "keypoints": [ + 421.375, + 220.09375, + 0.5270644426345825, + 383.125, + 229.09375, + 0.49643799662590027, + 403.375, + 258.90625, + 0.2977980077266693, + 380.875, + 267.90625, + 0.23715302348136902, + 380.875, + 272.40625, + 0.36040812730789185, + 353.875, + 285.90625, + 0.17865687608718872, + 446.125, + 265.09375, + 0.5074470043182373, + 412.375, + 274.09375, + 0.5964010953903198, + 448.375, + 269.59375, + 0.1952536255121231, + 371.875, + 290.40625, + 0.7809563279151917, + 448.375, + 296.59375, + 0.5803224444389343, + 320.125, + 317.40625, + 0.510760486125946, + 392.125, + 182.40625, + 0.4422689378261566, + 398.875, + 215.59375, + 0.4545695185661316 + ], + "score": 1.221349036054952 + }, + { + "image_id": 107292, + "category_id": 1, + "keypoints": [ + 67.08515930175781, + 246.1501922607422, + 9.999999747378752e-06, + 92.64140319824219, + 241.8908233642578, + 0.10620707273483276, + 90.51171875, + 241.8908233642578, + 9.999999747378752e-06, + 122.45703125, + 271.7064514160156, + 0.06404098123311996, + 126.7164077758789, + 233.3720703125, + 0.08939849585294724, + 122.45703125, + 237.63143920898438, + 0.12724143266677856, + 113.93827819824219, + 321.2216796875, + 0.09325428307056427, + 62.825782775878906, + 376.59356689453125, + 0.09064708650112152, + 175.69921875, + 339.8564453125, + 0.1935565173625946, + 173.5695343017578, + 346.7779235839844, + 0.23823411762714386, + 154.40234375, + 403.7470703125, + 0.06936991959810257, + 158.66171264648438, + 414.92791748046875, + 0.06923064589500427, + 22.361719131469727, + 178.0001983642578, + 0.22966018319129944, + 26.62109375, + 240.29354858398438, + 0.3429762125015259 + ], + "score": 0.4653931373754209 + }, + { + "image_id": 107292, + "category_id": 1, + "keypoints": [ + 346.0953369140625, + 192.79177856445312, + 0.7804575562477112, + 306.97344970703125, + 184.76678466796875, + 0.6301885843276978, + 346.0953369140625, + 230.158203125, + 0.8623130321502686, + 303.9640808105469, + 224.13943481445312, + 0.5270564556121826, + 343.0859375, + 261.0043029785156, + 0.3253191113471985, + 318.0078125, + 260.251953125, + 0.09123627096414566, + 309.9828186035156, + 220.87928771972656, + 0.4797114133834839, + 284.9046936035156, + 216.86679077148438, + 0.5754569172859192, + 306.97344970703125, + 260.251953125, + 0.051026735454797745, + 290.9234313964844, + 267.0230407714844, + 0.09640863537788391, + 312.9921875, + 288.3394470214844, + 9.999999747378752e-06, + 307.9765625, + 276.30194091796875, + 0.08239077031612396, + 344.0890808105469, + 156.67929077148438, + 0.911925196647644, + 328.0390625, + 184.76678466796875, + 0.818507194519043 + ], + "score": 1.3570686161929189 + }, + { + "image_id": 107292, + "category_id": 1, + "keypoints": [ + 77.42811584472656, + 224.5382843017578, + 9.999999747378752e-06, + 95.61561584472656, + 230.44921875, + 9.999999747378752e-06, + 113.80311584472656, + 237.72421264648438, + 9.999999747378752e-06, + 197.46563720703125, + 250.0007781982422, + 0.10605712234973907, + 168.36561584472656, + 235.45077514648438, + 9.999999747378752e-06, + 224.74688720703125, + 275.4632873535156, + 9.999999747378752e-06, + 135.62811279296875, + 266.82421875, + 0.07180708646774292, + 128.35311889648438, + 277.7367248535156, + 0.07542550563812256, + 184.734375, + 332.2992248535156, + 9.999999747378752e-06, + 164.72811889648438, + 333.66326904296875, + 9.999999747378752e-06, + 177.45938110351562, + 314.1117248535156, + 0.0735294371843338, + 177.45938110351562, + 311.8382873535156, + 9.999999747378752e-06, + 239.296875, + 162.70077514648438, + 9.999999747378752e-06, + 228.38436889648438, + 199.07577514648438, + 0.17402568459510803 + ], + "score": 0.20980674432602037 + }, + { + "image_id": 107292, + "category_id": 1, + "keypoints": [ + 71.8499984741211, + 244.61251831054688, + 9.999999747378752e-06, + 94.94999694824219, + 237.7875213623047, + 9.999999747378752e-06, + 84.44999694824219, + 232.01251220703125, + 9.999999747378752e-06, + 195.75, + 253.01251220703125, + 9.999999747378752e-06, + 120.1500015258789, + 232.01251220703125, + 9.999999747378752e-06, + 223.0500030517578, + 275.5875244140625, + 9.999999747378752e-06, + 113.8499984741211, + 321.78753662109375, + 9.999999747378752e-06, + 84.44999694824219, + 359.5875244140625, + 9.999999747378752e-06, + 174.75, + 341.2125244140625, + 9.999999747378752e-06, + 155.85000610351562, + 383.2125244140625, + 0.2424914389848709, + 164.25, + 391.61248779296875, + 9.999999747378752e-06, + 168.45001220703125, + 395.8125, + 9.999999747378752e-06, + 21.450000762939453, + 181.61251831054688, + 9.999999747378752e-06, + 27.75, + 232.01251220703125, + 9.999999747378752e-06 + ], + "score": 0.25982154176926997 + }, + { + "image_id": 107292, + "category_id": 1, + "keypoints": [ + 420.9765625, + 221.623046875, + 0.15918737649917603, + 383.8515625, + 228.970703125, + 9.999999747378752e-06, + 403.9609375, + 258.748046875, + 0.16403470933437347, + 386.9453125, + 261.841796875, + 9.999999747378752e-06, + 377.6640625, + 277.310546875, + 9.999999747378752e-06, + 306.5078125, + 269.189453125, + 0.07442621886730194, + 433.3515625, + 271.123046875, + 9.999999747378752e-06, + 410.1484375, + 275.376953125, + 9.999999747378752e-06, + 373.0234375, + 290.845703125, + 0.47673648595809937, + 371.4765625, + 287.751953125, + 9.999999747378752e-06, + 326.6171875, + 318.689453125, + 0.22087916731834412, + 306.5078125, + 281.564453125, + 9.999999747378752e-06, + 342.0859375, + 156.654296875, + 0.41441068053245544, + 329.7109375, + 185.658203125, + 0.3491266071796417 + ], + "score": 0.6095130035072154 + }, + { + "image_id": 107292, + "category_id": 1, + "keypoints": [ + 227.96249389648438, + 209.296875, + 0.2940279543399811, + 307.01251220703125, + 184.7531280517578, + 9.999999747378752e-06, + 345.26251220703125, + 230.6531219482422, + 9.999999747378752e-06, + 304.4624938964844, + 225.55313110351562, + 9.999999747378752e-06, + 351.63751220703125, + 271.453125, + 0.26815545558929443, + 226.6875, + 276.5531311035156, + 0.23028942942619324, + 250.91250610351562, + 261.2531433105469, + 0.0709555447101593, + 241.98748779296875, + 267.9468688964844, + 0.15146148204803467, + 291.7124938964844, + 256.15313720703125, + 0.1067427322268486, + 318.48748779296875, + 227.14688110351562, + 0.24569344520568848, + 313.38751220703125, + 284.203125, + 0.34412580728530884, + 309.5625, + 286.7531433105469, + 9.999999747378752e-06, + 241.98748779296875, + 164.35311889648438, + 0.429085373878479, + 226.6875, + 197.50311279296875, + 0.43026989698410034 + ], + "score": 0.6139018342478916 + }, + { + "image_id": 104173, + "category_id": 1, + "keypoints": [ + 302.93658447265625, + 179.1063232421875, + 0.6510030627250671, + 223.6296844482422, + 220.48382568359375, + 0.6930370330810547, + 351.2103271484375, + 192.89883422851562, + 0.6112078428268433, + 175.35592651367188, + 257.5511779785156, + 0.7341751456260681, + 361.5546875, + 220.48382568359375, + 0.7591686248779297, + 120.1859359741211, + 248.06883239746094, + 0.8202218413352966, + 289.14404296875, + 344.6163330078125, + 0.6980067491531372, + 223.6296844482422, + 344.6163330078125, + 0.7276822328567505, + 320.17718505859375, + 436.85369873046875, + 0.7203179001808167, + 175.35592651367188, + 450.64617919921875, + 0.6364948749542236, + 320.17718505859375, + 565.2963256835938, + 0.7633028030395508, + 199.49281311035156, + 505.816162109375, + 0.6130881309509277, + 258.1109313964844, + 137.7288360595703, + 0.7092248201370239, + 261.5590515136719, + 188.58868408203125, + 0.7168369293212891 + ], + "score": 1.5240624121257238 + }, + { + "image_id": 104173, + "category_id": 1, + "keypoints": [ + 182.91156005859375, + 250.5772705078125, + 0.5185858607292175, + 94.50341033935547, + 270.2235107421875, + 0.6907219886779785, + 241.85031127929688, + 302.96728515625, + 0.2605689764022827, + 150.16778564453125, + 313.6090087890625, + 0.7091324329376221, + 176.36279296875, + 300.5115051269531, + 0.06252239644527435, + 222.20404052734375, + 307.06024169921875, + 0.35872799158096313, + 209.10655212402344, + 440.4910583496094, + 0.718409538269043, + 140.34466552734375, + 440.4910583496094, + 0.7442821860313416, + 258.22216796875, + 569.01025390625, + 0.691102921962738, + 104.3265380859375, + 538.7222290039062, + 0.7298852205276489, + 202.55780029296875, + 499.4297790527344, + 0.17385971546173096, + 199.28343200683594, + 499.4297790527344, + 9.999999747378752e-06, + 107.60092163085938, + 171.9922637939453, + 0.6459799408912659, + 127.24716186523438, + 241.57273864746094, + 0.7564504146575928 + ], + "score": 1.260753242127196 + }, + { + "image_id": 115625, + "category_id": 1, + "keypoints": [ + 285.61871337890625, + 254.3703155517578, + 0.9053182601928711, + 193.59373474121094, + 247.9796905517578, + 0.8513275384902954, + 280.5062255859375, + 346.39532470703125, + 0.7143865823745728, + 183.3687286376953, + 356.62030029296875, + 0.8726442456245422, + 326.51873779296875, + 356.62030029296875, + 0.812847375869751, + 208.9312286376953, + 442.25469970703125, + 0.8677387237548828, + 280.5062255859375, + 452.47967529296875, + 0.835115909576416, + 229.38121032714844, + 452.47967529296875, + 0.8401076793670654, + 270.28125, + 616.0797119140625, + 0.7448552846908569, + 224.2687225341797, + 612.2453002929688, + 0.8154234886169434, + 249.8312225341797, + 745.1702880859375, + 0.6500316262245178, + 198.7062225341797, + 759.2296752929688, + 0.792145848274231, + 254.94371032714844, + 125.2796859741211, + 0.8769934773445129, + 244.71873474121094, + 223.6953125, + 0.9096454977989197 + ], + "score": 1.7302584648132324 + } +] diff --git a/tests/dataset/example_crowdpose_val.json b/tests/dataset/example_crowdpose_val.json new file mode 100644 index 0000000..d5df992 --- /dev/null +++ b/tests/dataset/example_crowdpose_val.json @@ -0,0 +1,677 @@ +{ + "categories": [ + { + "supercategory": "person", + "id": 1, + "name": "person", + "keypoints": [ + "left_shoulder", + "right_shoulder", + "left_elbow", + "right_elbow", + "left_wrist", + "right_wrist", + "left_hip", + "right_hip", + "left_knee", + "right_knee", + "left_ankle", + "right_ankle", + "head", + "neck" + ], + "skeleton": [ + [ + 16, + 14 + ], + [ + 14, + 12 + ], + [ + 17, + 15 + ], + [ + 15, + 13 + ], + [ + 12, + 13 + ], + [ + 6, + 12 + ], + [ + 7, + 13 + ], + [ + 6, + 7 + ], + [ + 6, + 8 + ], + [ + 7, + 9 + ], + [ + 8, + 10 + ], + [ + 9, + 11 + ] + ] + } + ], + "images": [ + { + "file_name": "114203.jpg", + "id": 114203, + "height": 1000, + "width": 795, + "crowdIndex": 0.76 + }, + { + "file_name": "104173.jpg", + "id": 104173, + "height": 640, + "width": 427, + "crowdIndex": 0.76 + }, + { + "file_name": "107292.jpg", + "id": 107292, + "height": 423, + "width": 640, + "crowdIndex": 0.82 + }, + { + "file_name": "115625.jpg", + "id": 115625, + "height": 835, + "width": 550, + "crowdIndex": 0.0 + } + ], + "annotations": [ + { + "num_keypoints": 12, + "iscrowd": 0, + "keypoints": [ + 739, + 301, + 1, + 569, + 287, + 1, + 730, + 361, + 1, + 503, + 340, + 1, + 699, + 276, + 1, + 413, + 371, + 1, + 672, + 553, + 1, + 583, + 519, + 1, + 666, + 714, + 1, + 508, + 513, + 1, + 675, + 908, + 1, + 400, + 595, + 1, + 617, + 192, + 1, + 651, + 283, + 1 + ], + "image_id": 114203, + "bbox": [ + 290, + 166, + 482, + 791 + ], + "category_id": 1, + "id": 165388 + }, + { + "num_keypoints": 12, + "iscrowd": 0, + "keypoints": [ + 353, + 257, + 1, + 171, + 296, + 1, + 447, + 360, + 1, + 93, + 367, + 1, + 457, + 466, + 1, + 63, + 379, + 1, + 375, + 525, + 1, + 258, + 570, + 1, + 492, + 644, + 1, + 230, + 759, + 1, + 519, + 803, + 1, + 236, + 907, + 1, + 260, + 112, + 1, + 253, + 242, + 1 + ], + "image_id": 114203, + "bbox": [ + 7, + 89, + 608, + 858 + ], + "category_id": 1, + "id": 181177 + }, + { + "num_keypoints": 12, + "iscrowd": 0, + "keypoints": [ + 295, + 318, + 1, + 156, + 318, + 1, + 289, + 421, + 1, + 114, + 421, + 1, + 338, + 487, + 1, + 81, + 504, + 1, + 260, + 537, + 1, + 176, + 536, + 1, + 245, + 678, + 1, + 173, + 673, + 1, + 228, + 808, + 1, + 185, + 798, + 1, + 138, + 281, + 1, + 239, + 261, + 1 + ], + "image_id": 114203, + "bbox": [ + 54, + 195, + 184, + 675 + ], + "category_id": 1, + "id": 183967 + }, + { + "num_keypoints": 12, + "iscrowd": 0, + "keypoints": [ + 308, + 172, + 2, + 222, + 221, + 2, + 352, + 185, + 2, + 174, + 261, + 2, + 364, + 215, + 2, + 123, + 250, + 2, + 295, + 331, + 2, + 219, + 328, + 2, + 325, + 437, + 2, + 173, + 453, + 2, + 325, + 574, + 2, + 207, + 510, + 2, + 254, + 133, + 1, + 261, + 182, + 1 + ], + "image_id": 104173, + "bbox": [ + 86.29, + 129.2, + 304.9, + 496.18 + ], + "category_id": 1, + "id": 121125 + }, + { + "num_keypoints": 8, + "iscrowd": 0, + "keypoints": [ + 203, + 250, + 1, + 107, + 262, + 1, + 0, + 0, + 0, + 154, + 309, + 2, + 0, + 0, + 0, + 230, + 306, + 1, + 210, + 441, + 1, + 141, + 448, + 2, + 254, + 569, + 2, + 101, + 537, + 2, + 0, + 0, + 0, + 0, + 0, + 0, + 108, + 173, + 1, + 134, + 242, + 1 + ], + "image_id": 104173, + "bbox": [ + 66.16, + 163.68, + 231.55, + 468.85 + ], + "category_id": 1, + "id": 124249 + }, + { + "num_keypoints": 0, + "iscrowd": 0, + "keypoints": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "image_id": 104173, + "bbox": [ + 327.41, + 242.02, + 29.61, + 69.83 + ], + "category_id": 1, + "id": 148054 + }, + { + "num_keypoints": 12, + "iscrowd": 0, + "keypoints": [ + 422, + 212, + 2, + 386, + 225, + 2, + 400, + 262, + 2, + 385, + 266, + 2, + 367, + 280, + 2, + 361, + 282, + 2, + 439, + 260, + 2, + 409, + 275, + 2, + 450, + 285, + 2, + 370, + 287, + 2, + 457, + 345, + 2, + 317, + 320, + 2, + 390, + 181, + 1, + 398, + 217, + 1 + ], + "image_id": 107292, + "bbox": [ + 257.82, + 175.98, + 214.93, + 197.61 + ], + "category_id": 1, + "id": 122714 + }, + { + "num_keypoints": 11, + "iscrowd": 0, + "keypoints": [ + 347, + 190, + 2, + 306, + 187, + 2, + 346, + 232, + 2, + 308, + 232, + 1, + 352, + 274, + 1, + 315, + 274, + 1, + 313, + 216, + 2, + 289, + 216, + 2, + 353, + 196, + 2, + 299, + 257, + 1, + 334, + 256, + 1, + 0, + 0, + 0, + 343, + 156, + 1, + 328, + 186, + 1 + ], + "image_id": 107292, + "bbox": [ + 269.98, + 151.27, + 90.24, + 94.8 + ], + "category_id": 1, + "id": 123425 + }, + { + "num_keypoints": 10, + "iscrowd": 0, + "keypoints": [ + 236, + 218, + 2, + 218, + 209, + 2, + 0, + 0, + 0, + 198, + 255, + 2, + 0, + 0, + 0, + 227, + 277, + 2, + 251, + 278, + 2, + 231, + 267, + 2, + 299, + 261, + 2, + 320, + 233, + 2, + 315, + 298, + 1, + 379, + 289, + 1, + 240, + 163, + 1, + 228, + 200, + 1 + ], + "image_id": 107292, + "bbox": [ + 182.61, + 160.09, + 195.31, + 155.29 + ], + "category_id": 1, + "id": 123805 + }, + { + "num_keypoints": 12, + "iscrowd": 0, + "keypoints": [ + 288, + 258, + 1, + 195, + 253, + 1, + 287, + 350, + 1, + 185, + 367, + 1, + 320, + 375, + 1, + 218, + 447, + 1, + 279, + 476, + 1, + 232, + 473, + 1, + 271, + 637, + 1, + 226, + 634, + 1, + 247, + 760, + 1, + 207, + 757, + 1, + 252, + 127, + 1, + 247, + 223, + 1 + ], + "image_id": 115625, + "bbox": [ + 135, + 102, + 251, + 720 + ], + "category_id": 1, + "id": 162705 + } + ] +} diff --git a/tests/test_basic.py b/tests/test_basic.py index b91bae5..9042b57 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -68,7 +68,7 @@ def test_params(self): self.assertEqual(kp_params.iou_type, "segm") def test_bad_coco_set(self): - with self.assertRaises(AssertionError): + with self.assertRaises(TypeError): COCO(1) def test_bad_iou_type(self): @@ -86,9 +86,9 @@ def test_ignore_coco_eval(self): "AP_small": -1.0, "AP_medium": 0.7099009900990099, "AP_large": -1.0, - "AR_1": 0.0, - "AR_10": 0.45384615384615384, - "AR_100": 0.7153846153846154, + "AR_all": 0.0, + "AR_second": 0.45384615384615384, + "AR_third": 0.7153846153846154, "AR_small": -1.0, "AR_medium": 0.7153846153846154, "AR_large": -1.0, @@ -119,9 +119,9 @@ def test_coco_eval(self): "AP_small": -1.0, "AP_medium": 0.7367986798679867, "AP_large": 0.0, - "AR_1": 0.6666666666666666, - "AR_10": 0.75, - "AR_100": 0.75, + "AR_all": 0.6666666666666666, + "AR_second": 0.75, + "AR_third": 0.75, "AR_small": -1.0, "AR_medium": 0.7916666666666666, "AR_large": 0.0, diff --git a/tests/test_boundary.py b/tests/test_boundary.py index f89733e..eab5f92 100644 --- a/tests/test_boundary.py +++ b/tests/test_boundary.py @@ -103,9 +103,9 @@ def test_boundary_eval(self): "AP_small": -1.0, "AP_medium": 0.7367986798679867, "AP_large": 0.0, - "AR_1": 0.6666666666666666, - "AR_10": 0.75, - "AR_100": 0.75, + "AR_all": 0.6666666666666666, + "AR_second": 0.75, + "AR_third": 0.75, "AR_small": -1.0, "AR_medium": 0.7916666666666666, "AR_large": 0.0, diff --git a/tests/test_crowdpose.py b/tests/test_crowdpose.py new file mode 100644 index 0000000..c977473 --- /dev/null +++ b/tests/test_crowdpose.py @@ -0,0 +1,53 @@ +#!/usr/bin/python3 +import os +import unittest + +import numpy as np + +from faster_coco_eval import COCO, COCOeval_faster + + +class TestCrowdpose(unittest.TestCase): + """Test Crowdpose functionality.""" + + def setUp(self): + self.gt_file = os.path.join("dataset", "example_crowdpose_val.json") + self.dt_file = os.path.join("dataset", "example_crowdpose_preds.json") + self.sigmas = ( + np.array([0.79, 0.79, 0.72, 0.72, 0.62, 0.62, 1.07, 1.07, 0.87, 0.87, 0.89, 0.89, 0.79, 0.79]) / 10.0 + ) + + if not os.path.exists(self.gt_file): + self.gt_file = os.path.join(os.path.dirname(__file__), self.gt_file) + self.dt_file = os.path.join(os.path.dirname(__file__), self.dt_file) + + def test_crowdpose_eval(self): + stats_as_dict = { + "AP_all": 0.7877215935879303, + "AP_50": 0.9881188118811886, + "AP_75": 0.7314356435643564, + "AR_all": 0.8222222222222223, + "AR_50": 1.0, + "AR_75": 0.7777777777777778, + "AP_easy": 1.0, + "AP_medium": 0.9802, + "AP_hard": 0.4116, + } + + cocoGt = COCO(self.gt_file) + + cocoDt = cocoGt.loadRes(self.dt_file) + + cocoEval = COCOeval_faster( + cocoGt, cocoDt, iouType="keypoints_crowd", kpt_oks_sigmas=self.sigmas, use_area=False + ) + + cocoEval.evaluate() + cocoEval.accumulate() + cocoEval.summarize() + + self.assertAlmostEqual(cocoEval.stats_as_dict, stats_as_dict, places=10) + + +if __name__ == "__main__": + unittest.main()