diff --git a/.vscode/launch.json b/.vscode/launch.json index ea067150536b..3768b701befd 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -125,6 +125,26 @@ ], "justMyCode": false, }, + { + "name": "REST API tests: Attach to RQ consensus worker", + "type": "debugpy", + "request": "attach", + "connect": { + "host": "127.0.0.1", + "port": 9096 + }, + "pathMappings": [ + { + "localRoot": "${workspaceFolder}", + "remoteRoot": "/home/django/" + }, + { + "localRoot": "${workspaceFolder}/.env", + "remoteRoot": "/opt/venv", + } + ], + "justMyCode": false, + }, { "type": "pwa-chrome", "request": "launch", @@ -383,6 +403,28 @@ }, "console": "internalConsole" }, + { + "name": "server: RQ - consensus", + "type": "debugpy", + "request": "launch", + "stopOnEntry": false, + "justMyCode": false, + "python": "${command:python.interpreterPath}", + "program": "${workspaceRoot}/manage.py", + "args": [ + "rqworker", + "consensus", + "--worker-class", + "cvat.rqworker.SimpleWorker" + ], + "django": true, + "cwd": "${workspaceFolder}", + "env": { + "DJANGO_LOG_SERVER_HOST": "localhost", + "DJANGO_LOG_SERVER_PORT": "8282" + }, + "console": "internalConsole" + }, { "name": "server: migrate", "type": "debugpy", @@ -566,6 +608,7 @@ "server: RQ - analytics reports", "server: RQ - cleaning", "server: RQ - chunks", + "server: RQ - consensus", ] } ] diff --git a/changelog.d/20250213_182204_mzhiltso_consensus_simple_merging.md b/changelog.d/20250213_182204_mzhiltso_consensus_simple_merging.md new file mode 100644 index 000000000000..99dd48f1651e --- /dev/null +++ b/changelog.d/20250213_182204_mzhiltso_consensus_simple_merging.md @@ -0,0 +1,10 @@ +### Added + +- Simple merging for consensus-enabled tasks + () + +### Changed + +- Hidden points in skeletons now also contribute to the skeleton similarity + in quality computations and in consensus merging + () diff --git a/cvat-core/src/api-implementation.ts b/cvat-core/src/api-implementation.ts index 559cb6f9f4b3..b6d1b55dc3ad 100644 --- a/cvat-core/src/api-implementation.ts +++ b/cvat-core/src/api-implementation.ts @@ -32,7 +32,7 @@ import Webhook from './webhook'; import { ArgumentError } from './exceptions'; import { AnalyticsReportFilter, QualityConflictsFilter, QualityReportsFilter, - QualitySettingsFilter, SerializedAsset, + QualitySettingsFilter, SerializedAsset, ConsensusSettingsFilter, } from './server-response-types'; import QualityReport from './quality-report'; import AboutData from './about'; @@ -40,6 +40,7 @@ import QualityConflict, { ConflictSeverity } from './quality-conflict'; import QualitySettings from './quality-settings'; import { getFramesMeta } from './frames'; import AnalyticsReport from './analytics-report'; +import ConsensusSettings from './consensus-settings'; import { callAction, listActions, registerAction, runAction, } from './annotations-actions/annotations-actions'; @@ -415,6 +416,20 @@ export default function implementAPI(cvat: CVATCore): CVATCore { return webhooks; }); + implementationMixin(cvat.consensus.settings.get, async (filter: ConsensusSettingsFilter) => { + checkFilter(filter, { + taskID: isInteger, + }); + + const params = fieldsToSnakeCase(filter); + + const settings = await serverProxy.consensus.settings.get(params); + const schema = await getServerAPISchema(); + const descriptions = convertDescriptions(schema.components.schemas.ConsensusSettings.properties); + + return new ConsensusSettings({ ...settings, descriptions }); + }); + implementationMixin(cvat.analytics.quality.reports, async (filter: QualityReportsFilter) => { checkFilter(filter, { page: isInteger, diff --git a/cvat-core/src/api.ts b/cvat-core/src/api.ts index 48789cce0fff..c3e338133e4f 100644 --- a/cvat-core/src/api.ts +++ b/cvat-core/src/api.ts @@ -378,6 +378,14 @@ function build(): CVATCore { return result; }, }, + consensus: { + settings: { + async get(filter = {}) { + const result = await PluginRegistry.apiWrapper(cvat.consensus.settings.get, filter); + return result; + }, + }, + }, analytics: { performance: { async reports(filter = {}) { @@ -482,6 +490,7 @@ function build(): CVATCore { cvat.cloudStorages = Object.freeze(cvat.cloudStorages); cvat.organizations = Object.freeze(cvat.organizations); cvat.webhooks = Object.freeze(cvat.webhooks); + cvat.consensus = Object.freeze(cvat.consensus); cvat.analytics = Object.freeze(cvat.analytics); cvat.classes = Object.freeze(cvat.classes); cvat.utils = Object.freeze(cvat.utils); diff --git a/cvat-core/src/consensus-settings.ts b/cvat-core/src/consensus-settings.ts new file mode 100644 index 000000000000..070414b8debb --- /dev/null +++ b/cvat-core/src/consensus-settings.ts @@ -0,0 +1,87 @@ +// Copyright (C) 2024 CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import { SerializedConsensusSettingsData } from './server-response-types'; +import PluginRegistry from './plugins'; +import serverProxy from './server-proxy'; +import { convertDescriptions, getServerAPISchema } from './server-schema'; + +export default class ConsensusSettings { + #id: number; + #task: number; + #iouThreshold: number; + #quorum: number; + #descriptions: Record; + + constructor(initialData: SerializedConsensusSettingsData) { + this.#id = initialData.id; + this.#task = initialData.task; + this.#iouThreshold = initialData.iou_threshold; + this.#quorum = initialData.quorum; + this.#descriptions = initialData.descriptions; + } + + get id(): number { + return this.#id; + } + + get task(): number { + return this.#task; + } + + get iouThreshold(): number { + return this.#iouThreshold; + } + + set iouThreshold(newVal: number) { + this.#iouThreshold = newVal; + } + + get quorum(): number { + return this.#quorum; + } + + set quorum(newVal: number) { + this.#quorum = newVal; + } + + get descriptions(): Record { + const descriptions: Record = Object.keys(this.#descriptions).reduce((acc, key) => { + const camelCaseKey = _.camelCase(key); + acc[camelCaseKey] = this.#descriptions[key]; + return acc; + }, {}); + + return descriptions; + } + + public toJSON(): SerializedConsensusSettingsData { + const result: SerializedConsensusSettingsData = { + iou_threshold: this.#iouThreshold, + quorum: this.#quorum, + }; + + return result; + } + + public async save(): Promise { + const result = await PluginRegistry.apiWrapper.call(this, ConsensusSettings.prototype.save); + return result; + } +} + +Object.defineProperties(ConsensusSettings.prototype.save, { + implementation: { + writable: false, + enumerable: false, + value: async function implementation(): Promise { + const result = await serverProxy.consensus.settings.update( + this.id, this.toJSON(), + ); + const schema = await getServerAPISchema(); + const descriptions = convertDescriptions(schema.components.schemas.ConsensusSettings.properties); + return new ConsensusSettings({ ...result, descriptions }); + }, + }, +}); diff --git a/cvat-core/src/index.ts b/cvat-core/src/index.ts index 52f755ce04ad..92db2d414d83 100644 --- a/cvat-core/src/index.ts +++ b/cvat-core/src/index.ts @@ -4,6 +4,7 @@ import { AnalyticsReportFilter, QualityConflictsFilter, QualityReportsFilter, QualitySettingsFilter, + ConsensusSettingsFilter, } from './server-response-types'; import PluginRegistry from './plugins'; import serverProxy from './server-proxy'; @@ -30,6 +31,7 @@ import Webhook from './webhook'; import QualityReport from './quality-report'; import QualityConflict from './quality-conflict'; import QualitySettings from './quality-settings'; +import ConsensusSettings from './consensus-settings'; import AnalyticsReport from './analytics-report'; import AnnotationGuide from './guide'; import { JobValidationLayout, TaskValidationLayout } from './validation-layout'; @@ -140,6 +142,11 @@ export default interface CVATCore { webhooks: { get: any; }; + consensus: { + settings: { + get: (filter: ConsensusSettingsFilter) => Promise; + }; + } analytics: { quality: { reports: (filter: QualityReportsFilter) => Promise>; diff --git a/cvat-core/src/server-proxy.ts b/cvat-core/src/server-proxy.ts index 614b594b54f0..7e45a251d4f7 100644 --- a/cvat-core/src/server-proxy.ts +++ b/cvat-core/src/server-proxy.ts @@ -19,6 +19,7 @@ import { SerializedInvitationData, SerializedCloudStorage, SerializedFramesMetaData, SerializedCollection, SerializedQualitySettingsData, APIQualitySettingsFilter, SerializedQualityConflictData, APIQualityConflictsFilter, SerializedQualityReportData, APIQualityReportsFilter, SerializedAnalyticsReport, APIAnalyticsReportFilter, + SerializedConsensusSettingsData, APIConsensusSettingsFilter, SerializedRequest, SerializedJobValidationLayout, SerializedTaskValidationLayout, } from './server-response-types'; import { PaginatedResource, UpdateStatusData } from './core-types'; @@ -767,6 +768,41 @@ async function deleteTask(id: number, organizationID: string | null = null): Pro } } +async function mergeConsensusJobs(id: number, instanceType: string): Promise { + const { backendAPI } = config; + const url = `${backendAPI}/consensus/merges`; + const params = { + rq_id: null, + }; + const requestBody = { + task_id: undefined, + job_id: undefined, + }; + + if (instanceType === 'task') requestBody.task_id = id; + else requestBody.job_id = id; + + return new Promise((resolve, reject) => { + async function request() { + try { + const response = await Axios.post(url, requestBody, { params }); + params.rq_id = response.data.rq_id; + const { status } = response; + if (status === 202) { + setTimeout(request, 3000); + } else if (status === 201) { + resolve(); + } else { + reject(generateError(response)); + } + } catch (errorData) { + reject(generateError(errorData)); + } + } + setTimeout(request); + }); +} + async function getLabels(filter: { job_id?: number, task_id?: number, @@ -2182,6 +2218,42 @@ async function updateQualitySettings( } } +async function getConsensusSettings( + filter: APIConsensusSettingsFilter, +): Promise { + const { backendAPI } = config; + + try { + const response = await Axios.get(`${backendAPI}/consensus/settings`, { + params: { + ...filter, + }, + }); + + return response.data.results[0]; + } catch (errorData) { + throw generateError(errorData); + } +} + +async function updateConsensusSettings( + settingsID: number, + settingsData: SerializedConsensusSettingsData, +): Promise { + const params = enableOrganization(); + const { backendAPI } = config; + + try { + const response = await Axios.patch(`${backendAPI}/consensus/settings/${settingsID}`, settingsData, { + params, + }); + + return response.data; + } catch (errorData) { + throw generateError(errorData); + } +} + async function getQualityConflicts( filter: APIQualityConflictsFilter, ): Promise { @@ -2411,6 +2483,7 @@ export default Object.freeze({ backup: backupTask, restore: restoreTask, validationLayout: validationLayout('tasks'), + mergeConsensusJobs, }), labels: Object.freeze({ @@ -2427,6 +2500,7 @@ export default Object.freeze({ delete: deleteJob, exportDataset: exportDataset('jobs'), validationLayout: validationLayout('jobs'), + mergeConsensusJobs, }), users: Object.freeze({ @@ -2531,6 +2605,13 @@ export default Object.freeze({ }), }), + consensus: Object.freeze({ + settings: Object.freeze({ + get: getConsensusSettings, + update: updateConsensusSettings, + }), + }), + requests: Object.freeze({ list: getRequestsList, status: getRequestStatus, diff --git a/cvat-core/src/server-response-types.ts b/cvat-core/src/server-response-types.ts index 90bea042d905..524ef7117893 100644 --- a/cvat-core/src/server-response-types.ts +++ b/cvat-core/src/server-response-types.ts @@ -242,8 +242,15 @@ export interface SerializedOrganization { export interface APIQualitySettingsFilter extends APICommonFilterParams { task_id?: number; } + export type QualitySettingsFilter = Camelized; +export interface APIConsensusSettingsFilter extends APICommonFilterParams { + task_id?: number; +} + +export type ConsensusSettingsFilter = Camelized; + export interface SerializedQualitySettingsData { id?: number; task?: number; @@ -332,6 +339,14 @@ export interface SerializedQualityReportData { }; } +export interface SerializedConsensusSettingsData { + id?: number; + task?: number; + quorum?: number; + iou_threshold?: number; + descriptions?: Record; +} + export interface SerializedDataEntry { date?: string; value?: number | Record diff --git a/cvat-core/src/session-implementation.ts b/cvat-core/src/session-implementation.ts index aaa9d1ed1e24..a96c04746d4b 100644 --- a/cvat-core/src/session-implementation.ts +++ b/cvat-core/src/session-implementation.ts @@ -616,6 +616,14 @@ export function implementJob(Job: typeof JobClass): typeof JobClass { }, }); + Object.defineProperty(Job.prototype.mergeConsensusJobs, 'implementation', { + value: function mergeConsensusJobsImplementation( + this: JobClass, + ): ReturnType { + return serverProxy.jobs.mergeConsensusJobs(this.id, 'job'); + }, + }); + return Job; } @@ -817,6 +825,14 @@ export function implementTask(Task: typeof TaskClass): typeof TaskClass { }, }); + Object.defineProperty(Task.prototype.mergeConsensusJobs, 'implementation', { + value: function mergeConsensusJobsImplementation( + this: TaskClass, + ): ReturnType { + return serverProxy.tasks.mergeConsensusJobs(this.id, 'task'); + }, + }); + Object.defineProperty(Task.prototype.issues, 'implementation', { value: function issuesImplementation( this: TaskClass, diff --git a/cvat-core/src/session.ts b/cvat-core/src/session.ts index 50c7df31fd0e..8deff9694ad4 100644 --- a/cvat-core/src/session.ts +++ b/cvat-core/src/session.ts @@ -737,6 +737,11 @@ export class Job extends Session { const result = await PluginRegistry.apiWrapper.call(this, Job.prototype.delete); return result; } + + async mergeConsensusJobs(): Promise { + const result = await PluginRegistry.apiWrapper.call(this, Job.prototype.mergeConsensusJobs); + return result; + } } export class Task extends Session { @@ -1199,6 +1204,11 @@ export class Task extends Session { return result; } + async mergeConsensusJobs(): Promise { + const result = await PluginRegistry.apiWrapper.call(this, Task.prototype.mergeConsensusJobs); + return result; + } + async backup(targetStorage: Storage, useDefaultSettings: boolean, fileName?: string): Promise { const result = await PluginRegistry.apiWrapper.call( this, diff --git a/cvat-ui/src/actions/common.ts b/cvat-ui/src/actions/common.ts new file mode 100644 index 000000000000..8acff094483b --- /dev/null +++ b/cvat-ui/src/actions/common.ts @@ -0,0 +1,24 @@ +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import { getCore, ProjectOrTaskOrJob } from 'cvat-core-wrapper'; +import { RequestInstanceType } from './requests-actions'; + +const core = getCore(); + +export function getInstanceType(instance: ProjectOrTaskOrJob | RequestInstanceType): 'project' | 'task' | 'job' { + if (instance instanceof core.classes.Project) { + return 'project'; + } + + if (instance instanceof core.classes.Task) { + return 'task'; + } + + if (instance instanceof core.classes.Job) { + return 'job'; + } + + return instance.type; +} diff --git a/cvat-ui/src/actions/consensus-actions.ts b/cvat-ui/src/actions/consensus-actions.ts new file mode 100644 index 000000000000..a06de6ef3bdc --- /dev/null +++ b/cvat-ui/src/actions/consensus-actions.ts @@ -0,0 +1,40 @@ +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import { ActionUnion, createAction, ThunkAction } from 'utils/redux'; +import { Project, ProjectOrTaskOrJob } from 'cvat-core-wrapper'; + +export enum ConsensusActionTypes { + MERGE_CONSENSUS_JOBS = 'MERGE_CONSENSUS_JOBS', + MERGE_CONSENSUS_JOBS_SUCCESS = 'MERGE_CONSENSUS_JOBS_SUCCESS', + MERGE_CONSENSUS_JOBS_FAILED = 'MERGE_CONSENSUS_JOBS_FAILED', +} + +export const consensusActions = { + mergeConsensusJobs: (instance: Exclude) => ( + createAction(ConsensusActionTypes.MERGE_CONSENSUS_JOBS, { instance }) + ), + mergeConsensusJobsSuccess: (instance: Exclude) => ( + createAction(ConsensusActionTypes.MERGE_CONSENSUS_JOBS_SUCCESS, { instance }) + ), + mergeConsensusJobsFailed: (instance: Exclude, error: any) => ( + createAction(ConsensusActionTypes.MERGE_CONSENSUS_JOBS_FAILED, { instance, error }) + ), +}; + +export const mergeConsensusJobsAsync = ( + instance: Exclude, +): ThunkAction => async (dispatch) => { + try { + dispatch(consensusActions.mergeConsensusJobs(instance)); + await instance.mergeConsensusJobs(); + } catch (error) { + dispatch(consensusActions.mergeConsensusJobsFailed(instance, error)); + return; + } + + dispatch(consensusActions.mergeConsensusJobsSuccess(instance)); +}; + +export type ConsensusActions = ActionUnion; diff --git a/cvat-ui/src/actions/export-actions.ts b/cvat-ui/src/actions/export-actions.ts index 8903bca35817..c433a9d9769f 100644 --- a/cvat-ui/src/actions/export-actions.ts +++ b/cvat-ui/src/actions/export-actions.ts @@ -8,9 +8,10 @@ import { Storage, ProjectOrTaskOrJob, Job, getCore, StorageLocation, } from 'cvat-core-wrapper'; import { - getInstanceType, RequestInstanceType, listen, + RequestInstanceType, listen, RequestsActions, updateRequestProgress, } from './requests-actions'; +import { getInstanceType } from './common'; export enum ExportActionTypes { OPEN_EXPORT_DATASET_MODAL = 'OPEN_EXPORT_DATASET_MODAL', diff --git a/cvat-ui/src/actions/import-actions.ts b/cvat-ui/src/actions/import-actions.ts index 6977b26b4915..98d34f74e80c 100644 --- a/cvat-ui/src/actions/import-actions.ts +++ b/cvat-ui/src/actions/import-actions.ts @@ -10,9 +10,10 @@ import { import { getProjectsAsync } from './projects-actions'; import { AnnotationActionTypes, fetchAnnotationsAsync } from './annotation-actions'; import { - getInstanceType, listen, RequestInstanceType, + listen, RequestInstanceType, RequestsActions, updateRequestProgress, } from './requests-actions'; +import { getInstanceType } from './common'; const core = getCore(); diff --git a/cvat-ui/src/actions/requests-actions.ts b/cvat-ui/src/actions/requests-actions.ts index b03cda983fcd..e2504a340e18 100644 --- a/cvat-ui/src/actions/requests-actions.ts +++ b/cvat-ui/src/actions/requests-actions.ts @@ -4,7 +4,7 @@ import { ActionUnion, createAction } from 'utils/redux'; import { CombinedState, RequestsQuery } from 'reducers'; -import { Request, ProjectOrTaskOrJob, getCore } from 'cvat-core-wrapper'; +import { Request, getCore } from 'cvat-core-wrapper'; import { Store } from 'redux'; import { getCVATStore } from 'cvat-store'; @@ -64,22 +64,6 @@ export interface RequestInstanceType { type: 'project' | 'task' | 'job'; } -export function getInstanceType(instance: ProjectOrTaskOrJob | RequestInstanceType): 'project' | 'task' | 'job' { - if (instance instanceof core.classes.Project) { - return 'project'; - } - - if (instance instanceof core.classes.Task) { - return 'task'; - } - - if (instance instanceof core.classes.Job) { - return 'job'; - } - - return instance.type; -} - export function updateRequestProgress(request: Request, dispatch: (action: RequestsActions) => void): void { dispatch( requestsActions.getRequestStatusSuccess(request), diff --git a/cvat-ui/src/base.scss b/cvat-ui/src/base.scss index 5f8e8f82a3db..23ba40f5d85f 100644 --- a/cvat-ui/src/base.scss +++ b/cvat-ui/src/base.scss @@ -64,3 +64,22 @@ $scroll-breakpoint: 1300px; align-items: center; } } + +.cvat-scrollbar { + overflow-y: auto; + + &::-webkit-scrollbar { + background-color: #fff; + width: $grid-unit-size * 2; + } + + &::-webkit-scrollbar-track { + background-color: #fff; + } + + &::-webkit-scrollbar-thumb { + background-color: #babac0; + border-radius: $border-radius-base * 2; + border: 6px solid #fff; + } +} diff --git a/cvat-ui/src/components/actions-menu/actions-menu.tsx b/cvat-ui/src/components/actions-menu/actions-menu.tsx index 9bf89f5b1d7c..51b4d48c580a 100644 --- a/cvat-ui/src/components/actions-menu/actions-menu.tsx +++ b/cvat-ui/src/components/actions-menu/actions-menu.tsx @@ -6,10 +6,12 @@ import './styles.scss'; import React, { useCallback } from 'react'; import Modal from 'antd/lib/modal'; +import { LoadingOutlined } from '@ant-design/icons'; import { DimensionType, CVATCore } from 'cvat-core-wrapper'; import Menu, { MenuInfo } from 'components/dropdown-menu'; import { usePlugins } from 'utils/hooks'; import { CombinedState } from 'reducers'; +import { useSelector } from 'react-redux'; type AnnotationFormats = Awaited>; @@ -22,6 +24,7 @@ interface Props { dumpers: AnnotationFormats['dumpers']; inferenceIsActive: boolean; taskDimension: DimensionType; + consensusEnabled: boolean; onClickMenu: (params: MenuInfo) => void; } @@ -35,6 +38,8 @@ export enum Actions { BACKUP_TASK = 'backup_task', VIEW_ANALYTICS = 'view_analytics', QUALITY_CONTROL = 'quality_control', + CONSENSUS_MANAGEMENT = 'consensus_management', + MERGE_CONSENSUS_JOBS = 'merge_consensus_jobs', } function ActionsMenuComponent(props: Props): JSX.Element { @@ -43,11 +48,15 @@ function ActionsMenuComponent(props: Props): JSX.Element { projectID, bugTracker, inferenceIsActive, + consensusEnabled, onClickMenu, } = props; const plugins = usePlugins((state: CombinedState) => state.plugins.components.taskActions.items, props); + const mergingConsensus = useSelector((state: CombinedState) => state.consensus.actions.merging); + const isTaskInMergingConsensus = mergingConsensus[`task_${taskID}`]; + const onClickMenuWrapper = useCallback( (params: MenuInfo) => { if (!params) { @@ -68,6 +77,20 @@ function ActionsMenuComponent(props: Props): JSX.Element { }, okText: 'Delete', }); + } else if (params.key === Actions.MERGE_CONSENSUS_JOBS) { + Modal.confirm({ + title: 'The consensus jobs will be merged', + content: 'Existing annotations in parent jobs will be updated. Continue?', + className: 'cvat-modal-confirm-consensus-merge-task', + onOk: () => { + onClickMenu(params); + }, + okButtonProps: { + type: 'primary', + danger: true, + }, + okText: 'Merge', + }); } else { onClickMenu(params); } @@ -120,6 +143,25 @@ function ActionsMenuComponent(props: Props): JSX.Element { ), 60]); + if (consensusEnabled) { + menuItems.push([( + + Consensus management + + ), 55]); + menuItems.push([( + } + > + Merge consensus jobs + + ), 60]); + } + if (projectID === null) { menuItems.push([( Move to project diff --git a/cvat-ui/src/components/analytics-page/styles.scss b/cvat-ui/src/components/analytics-page/styles.scss index dd30fed01756..c7e40d1240e8 100644 --- a/cvat-ui/src/components/analytics-page/styles.scss +++ b/cvat-ui/src/components/analytics-page/styles.scss @@ -75,3 +75,12 @@ width: 100%; height: 100%; } + +.cvat-task-analytics-tabs { + width: 100%; +} + + +.cvat-analytics-card-holder { + min-height: $grid-unit-size * 19; +} diff --git a/cvat-ui/src/components/analytics-page/views/analytics-card.tsx b/cvat-ui/src/components/analytics-page/views/analytics-card.tsx index a072596539c7..1dc64dc66eff 100644 --- a/cvat-ui/src/components/analytics-page/views/analytics-card.tsx +++ b/cvat-ui/src/components/analytics-page/views/analytics-card.tsx @@ -30,7 +30,7 @@ function AnalyticsCard(props: Props): JSX.Element { return ( - + diff --git a/cvat-ui/src/components/consensus-management-page/consensus-management-page.tsx b/cvat-ui/src/components/consensus-management-page/consensus-management-page.tsx new file mode 100644 index 000000000000..0443330c2f8e --- /dev/null +++ b/cvat-ui/src/components/consensus-management-page/consensus-management-page.tsx @@ -0,0 +1,292 @@ +// Copyright (C) 2024 CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import './styles.scss'; + +import React, { + useCallback, useEffect, useState, useReducer, +} from 'react'; +import { useParams } from 'react-router'; +import { Link } from 'react-router-dom'; +import { Row, Col } from 'antd/lib/grid'; +import Title from 'antd/lib/typography/Title'; +import notification from 'antd/lib/notification'; +import { ConsensusSettings, Task, getCore } from 'cvat-core-wrapper'; +import GoBackButton from 'components/common/go-back-button'; + +import Tabs, { TabsProps } from 'antd/lib/tabs'; +import Result from 'antd/lib/result'; + +import CVATLoadingSpinner from 'components/common/loading-spinner'; +import { ActionUnion, createAction } from 'utils/redux'; +import { fetchTask } from 'utils/fetch'; +import ConsensusSettingsTab from './consensus-settings-tab'; + +const core = getCore(); + +function getTabFromHash(supportedTabs: string[]): string { + const tab = window.location.hash.slice(1); + return supportedTabs.includes(tab) ? tab : supportedTabs[0]; +} + +enum TabName { + settings = 'settings', +} + +const DEFAULT_TAB = TabName.settings; + +interface State { + fetching: boolean; + reportRefreshingStatus: string | null; + error: Error | null; + consensusSettings: { + settings: ConsensusSettings | null; + fetching: boolean; + }; +} + +enum ReducerActionType { + SET_FETCHING = 'SET_FETCHING', + SET_CONSENSUS_SETTINGS = 'SET_CONSENSUS_SETTINGS', + SET_CONSENSUS_SETTINGS_FETCHING = 'SET_CONSENSUS_SETTINGS_FETCHING', + SET_REPORT_REFRESHING_STATUS = 'SET_REPORT_REFRESHING_STATUS', + SET_ERROR = 'SET_ERROR', +} + +export const reducerActions = { + setFetching: (fetching: boolean) => ( + createAction(ReducerActionType.SET_FETCHING, { fetching }) + ), + setConsensusSettings: (consensusSettings: ConsensusSettings) => ( + createAction(ReducerActionType.SET_CONSENSUS_SETTINGS, { consensusSettings }) + ), + setConsensusSettingsFetching: (fetching: boolean) => ( + createAction(ReducerActionType.SET_CONSENSUS_SETTINGS_FETCHING, { fetching }) + ), + setError: (error: Error) => ( + createAction(ReducerActionType.SET_ERROR, { error }) + ), +}; + +const reducer = (state: State, action: ActionUnion): State => { + if (action.type === ReducerActionType.SET_FETCHING) { + return { + ...state, + fetching: action.payload.fetching, + }; + } + + if (action.type === ReducerActionType.SET_CONSENSUS_SETTINGS) { + return { + ...state, + consensusSettings: { + ...state.consensusSettings, + settings: action.payload.consensusSettings, + }, + }; + } + + if (action.type === ReducerActionType.SET_CONSENSUS_SETTINGS_FETCHING) { + return { + ...state, + consensusSettings: { + ...state.consensusSettings, + fetching: action.payload.fetching, + }, + }; + } + + if (action.type === ReducerActionType.SET_ERROR) { + return { + ...state, + error: action.payload.error, + }; + } + + return state; +}; + +function ConsensusManagementPage(): JSX.Element { + const supportedTabs = Object.values(TabName); + const [state, dispatch] = useReducer(reducer, { + fetching: true, + reportRefreshingStatus: null, + error: null, + consensusSettings: { + settings: null, + fetching: false, + }, + }); + + const requestedInstanceID = +useParams<{ tid: string }>().tid; + + const [activeTab, setActiveTab] = useState(getTabFromHash(supportedTabs)); + const [instance, setInstance] = useState(null); + + const initializeData = async (id: number): Promise => { + try { + const taskInstance = await fetchTask(id); + + setInstance(taskInstance); + try { + dispatch(reducerActions.setConsensusSettingsFetching(true)); + const settings = await core.consensus.settings.get({ taskID: taskInstance.id }); + dispatch(reducerActions.setConsensusSettings(settings)); + } finally { + dispatch(reducerActions.setConsensusSettingsFetching(false)); + } + } catch (error: unknown) { + dispatch(reducerActions.setError(error instanceof Error ? error : new Error('Unknown error'))); + } finally { + dispatch(reducerActions.setFetching(false)); + } + }; + + const onSaveConsensusSettings = useCallback(async (values) => { + try { + const { settings } = state.consensusSettings; + if (settings) { + settings.quorum = values.quorum / 100; + settings.iouThreshold = values.iouThreshold / 100; + + try { + dispatch(reducerActions.setConsensusSettingsFetching(true)); + const responseSettings = await settings.save(); + dispatch(reducerActions.setConsensusSettings(responseSettings)); + notification.info({ message: 'Settings have been updated' }); + } catch (error: unknown) { + notification.error({ + message: 'Could not save consensus settings', + description: typeof Error === 'object' ? (error as object).toString() : '', + }); + throw error; + } finally { + dispatch(reducerActions.setConsensusSettingsFetching(false)); + } + } + return settings; + } catch (e) { + return false; + } + }, [state.consensusSettings.settings]); + + useEffect(() => { + initializeData(requestedInstanceID); + }, [requestedInstanceID]); + + useEffect(() => { + window.addEventListener('hashchange', () => { + const hash = getTabFromHash(supportedTabs); + setActiveTab(hash); + }); + }, []); + + useEffect(() => { + window.location.hash = activeTab; + }, [activeTab]); + + const onTabKeyChange = useCallback((key: string): void => { + setActiveTab(key); + }, []); + + const backNavigation: JSX.Element | null = ( + + + + + + ); + let title: JSX.Element | null = null; + let tabs: JSX.Element | null = null; + + const { + fetching, + error, + consensusSettings: { + settings: consensusSettings, + fetching: consensusSettingsFetching, + }, + } = state; + + if (error) { + return ( +
+
+ +
+
+ ); + } + + if (fetching || consensusSettingsFetching) { + return ( +
+
+ +
+
+ ); + } + + if (instance) { + title = ( + + + Consensus management for + <Link to={`/tasks/${instance.id}`}>{` Task #${instance.id}`}</Link> + + + ); + + const tabsItems: NonNullable[0][] = []; + + if (consensusSettings) { + tabsItems.push({ + key: TabName.settings, + label: 'Settings', + children: ( + + ), + }); + } + + tabs = ( + + ); + } + + return ( +
+ + + {backNavigation} + + + {title} + {tabs} + + + + +
+ ); +} + +export default React.memo(ConsensusManagementPage); diff --git a/cvat-ui/src/components/consensus-management-page/consensus-settings-tab.tsx b/cvat-ui/src/components/consensus-management-page/consensus-settings-tab.tsx new file mode 100644 index 000000000000..856529b32075 --- /dev/null +++ b/cvat-ui/src/components/consensus-management-page/consensus-settings-tab.tsx @@ -0,0 +1,54 @@ +// Copyright (C) 2023-2024 CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import React, { useCallback } from 'react'; +import Text from 'antd/lib/typography/Text'; +import Form from 'antd/lib/form'; +import { ConsensusSettings } from 'cvat-core-wrapper'; +import CVATLoadingSpinner from 'components/common/loading-spinner'; +import ConsensusSettingsForm from './task-consensus/consensus-settings-form'; + +interface Props { + fetching: boolean; + settings: ConsensusSettings | null; + setSettings: (settings: ConsensusSettings) => void; +} + +function ConsensusSettingsTab(props: Readonly): JSX.Element | null { + const { + fetching, + settings, + setSettings, + } = props; + + const [form] = Form.useForm(); + const onSave = useCallback(async () => { + const values = await form.validateFields(); + setSettings(values); + }, [form, setSettings]); + + if (fetching) { + return ( +
+
+ +
+
+ ); + } + + return ( +
+ { settings ? ( + + ) : No consensus settings found } +
+ ); +} + +export default React.memo(ConsensusSettingsTab); diff --git a/cvat-ui/src/components/consensus-management-page/styles.scss b/cvat-ui/src/components/consensus-management-page/styles.scss new file mode 100644 index 000000000000..7de7e0043011 --- /dev/null +++ b/cvat-ui/src/components/consensus-management-page/styles.scss @@ -0,0 +1,54 @@ +// Copyright (C) 2024 CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +@import 'base'; + +.cvat-consensus-management-inner { + background: $background-color-1; + padding: $grid-unit-size * 4; + padding-bottom: $grid-unit-size; + padding-top: 0; + border-radius: $border-radius-base; +} + +.cvat-consensus-settings-title { + margin-bottom: $grid-unit-size * 2; + align-items: center; +} + +.cvat-consensus-settings-form { + display: block; + position: relative; + + .cvat-consensus-settings-save-btn { + position: sticky; + z-index: 1; + top: 0; + height: 0; + } + + .ant-divider-horizontal { + margin: $grid-unit-size 0; + } +} + + +.cvat-consensus-management-loading, .cvat-consensus-management-page-error { + position: absolute; + top: 50%; + left: 50%; + transform: translate(-50%, -50%); +} + +.cvat-consensus-management-overview-tab { + min-height: 50vh; +} + +.cvat-task-control-tabs { + .ant-tabs-tabpane { + @extend .cvat-scrollbar; + + height: calc(100vh - $grid-unit-size * 33); + } +} diff --git a/cvat-ui/src/components/consensus-management-page/task-consensus/consensus-settings-form.tsx b/cvat-ui/src/components/consensus-management-page/task-consensus/consensus-settings-form.tsx new file mode 100644 index 000000000000..a77c67dd10fa --- /dev/null +++ b/cvat-ui/src/components/consensus-management-page/task-consensus/consensus-settings-form.tsx @@ -0,0 +1,119 @@ +// Copyright (C) 2024 CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import React from 'react'; +import { QuestionCircleOutlined } from '@ant-design/icons/lib/icons'; +import Text from 'antd/lib/typography/Text'; +import InputNumber from 'antd/lib/input-number'; +import { Col, Row } from 'antd/lib/grid'; +import Divider from 'antd/lib/divider'; +import Form, { FormInstance } from 'antd/lib/form'; +import Button from 'antd/lib/button'; +import CVATTooltip from 'components/common/cvat-tooltip'; +import { ConsensusSettings } from 'cvat-core-wrapper'; + +interface Props { + form: FormInstance; + settings: ConsensusSettings; + onSave: () => void; +} + +export default function ConsensusSettingsForm(props: Readonly): JSX.Element | null { + const { form, settings, onSave } = props; + + const initialValues = { + quorum: settings.quorum * 100, + iouThreshold: settings.iouThreshold * 100, + }; + + const makeTooltipFragment = (metric: string, description: string): JSX.Element => ( +
+ {`${metric}:`} + + {description} + +
+ ); + + const makeTooltip = (jsx: JSX.Element): JSX.Element => ( +
+ {jsx} +
+ ); + + const generalTooltip = makeTooltip( + <> + {makeTooltipFragment('Quorum', settings.descriptions.quorum.replace( + 'required share of', + 'required percent of', + ))} + , + ); + + const shapeComparisonTooltip = makeTooltip( + <> + {makeTooltipFragment('Min overlap threshold (IoU)', settings.descriptions.iouThreshold)} + , + ); + + return ( +
+ + + + + + + General + + + + + + + + + + + + + + Shape comparison + + + + + + + + + + + + + ); +} diff --git a/cvat-ui/src/components/cvat-app.tsx b/cvat-ui/src/components/cvat-app.tsx index 602235e87db9..a603cfb05fc0 100644 --- a/cvat-ui/src/components/cvat-app.tsx +++ b/cvat-ui/src/components/cvat-app.tsx @@ -81,6 +81,7 @@ import IncorrectEmailConfirmationPage from './email-confirmation-pages/incorrect import CreateJobPage from './create-job-page/create-job-page'; import AnalyticsPage from './analytics-page/analytics-page'; import QualityControlPage from './quality-control/quality-control-page'; +import ConsensusManagementPage from './consensus-management-page/consensus-management-page'; import InvitationWatcher from './invitation-watcher/invitation-watcher'; interface CVATAppProps { @@ -510,6 +511,7 @@ class CVATApplication extends React.PureComponent + diff --git a/cvat-ui/src/components/job-item/job-actions-menu.tsx b/cvat-ui/src/components/job-item/job-actions-menu.tsx index 6ed839c6fb42..33430bb4a1f5 100644 --- a/cvat-ui/src/components/job-item/job-actions-menu.tsx +++ b/cvat-ui/src/components/job-item/job-actions-menu.tsx @@ -3,18 +3,22 @@ // SPDX-License-Identifier: MIT import React, { useCallback } from 'react'; -import { useDispatch } from 'react-redux'; +import { useDispatch, useSelector } from 'react-redux'; import { useHistory } from 'react-router'; import Modal from 'antd/lib/modal'; - +import { LoadingOutlined } from '@ant-design/icons'; import { exportActions } from 'actions/export-actions'; import { deleteJobAsync } from 'actions/jobs-actions'; import { importActions } from 'actions/import-actions'; import { Job, JobType } from 'cvat-core-wrapper'; import Menu, { MenuInfo } from 'components/dropdown-menu'; +import { mergeConsensusJobsAsync } from 'actions/consensus-actions'; +import { CombinedState } from 'reducers'; +import { makeKey } from 'reducers/consensus-reducer'; interface Props { job: Job; + consensusJobsPresent?: boolean; } export enum Actions { @@ -24,11 +28,12 @@ export enum Actions { IMPORT_JOB = 'import_job', EXPORT_JOB = 'export_job', VIEW_ANALYTICS = 'view_analytics', + MERGE_SPECIFIC_CONSENSUS_JOBS = 'merge_specific_consensus_jobs', DELETE = 'delete', } function JobActionsMenu(props: Props): JSX.Element { - const { job } = props; + const { job, consensusJobsPresent } = props; const dispatch = useDispatch(); const history = useHistory(); @@ -49,6 +54,20 @@ function JobActionsMenu(props: Props): JSX.Element { dispatch(exportActions.openExportDatasetModal(job)); } else if (action.key === Actions.VIEW_ANALYTICS) { history.push(`/tasks/${job.taskId}/jobs/${job.id}/analytics`); + } else if (action.key === Actions.MERGE_SPECIFIC_CONSENSUS_JOBS) { + Modal.confirm({ + title: 'The consensus job will be merged', + content: 'Existing annotations in the parent job will be updated. Continue?', + className: 'cvat-modal-confirm-consensus-merge-job', + onOk: () => { + dispatch(mergeConsensusJobsAsync(job)); + }, + okButtonProps: { + type: 'primary', + danger: true, + }, + okText: 'Merge', + }); } else if (action.key === Actions.DELETE) { Modal.confirm({ title: `The job ${job.id} will be deleted`, @@ -68,6 +87,9 @@ function JobActionsMenu(props: Props): JSX.Element { [job], ); + const mergingConsensus = useSelector((state: CombinedState) => state.consensus.actions.merging); + const isInMergingConsensus = mergingConsensus[makeKey(job)]; + return ( Import annotations Export annotations View analytics + {consensusJobsPresent && job.parentJobId === null && ( + } + > + Merge consensus job + + )} } + overlay={( + 0} + /> + )} > diff --git a/cvat-ui/src/components/jobs-page/job-card.tsx b/cvat-ui/src/components/jobs-page/job-card.tsx index c43810d95034..df634be50796 100644 --- a/cvat-ui/src/components/jobs-page/job-card.tsx +++ b/cvat-ui/src/components/jobs-page/job-card.tsx @@ -95,7 +95,9 @@ function JobCardComponent(props: Props): JSX.Element { } + overlay={( + + )} > diff --git a/cvat-ui/src/components/quality-control/quality-control-page.tsx b/cvat-ui/src/components/quality-control/quality-control-page.tsx index 4f800dab7ae5..b0b8c369658f 100644 --- a/cvat-ui/src/components/quality-control/quality-control-page.tsx +++ b/cvat-ui/src/components/quality-control/quality-control-page.tsx @@ -22,6 +22,7 @@ import { import CVATLoadingSpinner from 'components/common/loading-spinner'; import GoBackButton from 'components/common/go-back-button'; import { ActionUnion, createAction } from 'utils/redux'; +import { fetchTask } from 'utils/fetch'; import QualityOverviewTab from './task-quality/quality-overview-tab'; import QualityManagementTab from './task-quality/quality-magement-tab'; import QualitySettingsTab from './quality-settings-tab'; @@ -180,12 +181,7 @@ function QualityControlPage(): JSX.Element { const initializeData = async (id: number): Promise => { try { - let taskInstance = null; - try { - [taskInstance] = await core.tasks.get({ id }); - } catch (error: unknown) { - throw new Error('The task was not found on the server'); - } + const taskInstance = await fetchTask(id); setInstance(taskInstance); try { @@ -398,7 +394,9 @@ function QualityControlPage(): JSX.Element { ), }); } + } + if (qualitySettings) { tabsItems.push({ key: 'settings', label: 'Settings', diff --git a/cvat-ui/src/components/quality-control/styles.scss b/cvat-ui/src/components/quality-control/styles.scss index 1ff2c7d5d13d..2bbde7028dc4 100644 --- a/cvat-ui/src/components/quality-control/styles.scss +++ b/cvat-ui/src/components/quality-control/styles.scss @@ -4,25 +4,6 @@ @import 'base'; -.cvat-quality-scrollbar { - overflow: hidden auto; - - &::-webkit-scrollbar { - background-color: #fff; - width: $grid-unit-size * 2; - } - - &::-webkit-scrollbar-track { - background-color: #fff; - } - - &::-webkit-scrollbar-thumb { - background-color: #babac0; - border-radius: $border-radius-base * 2; - border: 6px solid #fff; - } -} - .cvat-quality-control-inner { height: 100%; display: flex; @@ -37,7 +18,7 @@ } .cvat-quality-settings-form { - @extend .cvat-quality-scrollbar; + @extend .cvat-scrollbar; display: block; position: relative; diff --git a/cvat-ui/src/components/requests-page/styles.scss b/cvat-ui/src/components/requests-page/styles.scss index 518eab18e261..a08e62b5305e 100644 --- a/cvat-ui/src/components/requests-page/styles.scss +++ b/cvat-ui/src/components/requests-page/styles.scss @@ -4,23 +4,6 @@ @import 'base'; -.cvat-requests-scrollbar { - &::-webkit-scrollbar { - background-color: #fff; - width: $grid-unit-size * 2; - } - - &::-webkit-scrollbar-track { - background-color: #fff; - } - - &::-webkit-scrollbar-thumb { - background-color: #babac0; - border-radius: $border-radius-base * 2; - border: 6px solid #fff; - } -} - .cvat-requests-page { height: 100%; padding-top: $grid-unit-size * 2; @@ -88,7 +71,7 @@ } .cvat-request-item-progress-message { - @extend .cvat-requests-scrollbar; + @extend .cvat-scrollbar; max-height: $grid-unit-size * 8; overflow: hidden auto; diff --git a/cvat-ui/src/components/task-page/job-list.tsx b/cvat-ui/src/components/task-page/job-list.tsx index 4e74ab01355f..788e80dedcb7 100644 --- a/cvat-ui/src/components/task-page/job-list.tsx +++ b/cvat-ui/src/components/task-page/job-list.tsx @@ -107,11 +107,20 @@ function JobListComponent(props: Props): JSX.Element { }, [taskInstance]); const [uncollapsedJobs, setUncollapsedJobs] = useState>({}); + useEffect(() => { + const savedState = localStorage.getItem('uncollapsedJobs'); + if (savedState) { + setUncollapsedJobs(JSON.parse(savedState)); + } + }, []); const onCollapseChange = useCallback((jobId: number) => { - setUncollapsedJobs((prevState) => ({ - ...prevState, - [jobId]: !prevState[jobId], - })); + setUncollapsedJobs((prevState) => { + const newState = { ...prevState }; + newState[jobId] = !prevState[jobId]; + + localStorage.setItem('uncollapsedJobs', JSON.stringify(newState)); + return newState; + }); }, []); const [query, setQuery] = useState(updatedQuery); diff --git a/cvat-ui/src/components/task-page/top-bar.tsx b/cvat-ui/src/components/task-page/top-bar.tsx index 266d10fece0a..fd0189e578a0 100644 --- a/cvat-ui/src/components/task-page/top-bar.tsx +++ b/cvat-ui/src/components/task-page/top-bar.tsx @@ -29,6 +29,10 @@ export default function DetailsComponent(props: DetailsComponentProps): JSX.Elem history.push(`/tasks/${taskInstance.id}/quality-control`); }; + const onViewConsensusManagement = (): void => { + history.push(`/tasks/${taskInstance.id}/consensus`); + }; + return ( @@ -63,6 +67,7 @@ export default function DetailsComponent(props: DetailsComponentProps): JSX.Elem taskInstance={taskInstance} onViewAnalytics={onViewAnalytics} onViewQualityControl={onViewQualityControl} + onViewConsensusManagement={onViewConsensusManagement} /> )} > diff --git a/cvat-ui/src/components/tasks-page/task-item.tsx b/cvat-ui/src/components/tasks-page/task-item.tsx index 4e8aec06a206..f2895afba5aa 100644 --- a/cvat-ui/src/components/tasks-page/task-item.tsx +++ b/cvat-ui/src/components/tasks-page/task-item.tsx @@ -242,6 +242,10 @@ class TaskItemComponent extends React.PureComponent { + history.push(`/tasks/${taskInstance.id}/consensus`); + }; + return ( @@ -271,6 +275,7 @@ class TaskItemComponent extends React.PureComponent )} > diff --git a/cvat-ui/src/containers/actions-menu/actions-menu.tsx b/cvat-ui/src/containers/actions-menu/actions-menu.tsx index 879b9ec5853f..9df4155a9da9 100644 --- a/cvat-ui/src/containers/actions-menu/actions-menu.tsx +++ b/cvat-ui/src/containers/actions-menu/actions-menu.tsx @@ -17,12 +17,14 @@ import { } from 'actions/tasks-actions'; import { exportActions } from 'actions/export-actions'; import { importActions } from 'actions/import-actions'; -import { RQStatus } from 'cvat-core-wrapper'; +import { mergeConsensusJobsAsync } from 'actions/consensus-actions'; +import { RQStatus, Task } from 'cvat-core-wrapper'; interface OwnProps { taskInstance: any; onViewAnalytics: () => void; onViewQualityControl: () => void; + onViewConsensusManagement: () => void; } interface StateToProps { @@ -36,6 +38,7 @@ interface DispatchToProps { openRunModelWindow: (taskInstance: any) => void; deleteTask: (taskInstance: any) => void; openMoveTaskToProjectWindow: (taskInstance: any) => void; + mergeConsensusJobs: (taskInstance: any) => void; } function mapStateToProps(state: CombinedState, own: OwnProps): StateToProps { @@ -77,6 +80,9 @@ function mapDispatchToProps(dispatch: any): DispatchToProps { openMoveTaskToProjectWindow: (taskId: number): void => { dispatch(switchMoveTaskModalVisible(true, taskId)); }, + mergeConsensusJobs: (taskInstance: Task): void => { + dispatch(mergeConsensusJobsAsync(taskInstance)); + }, }; } @@ -92,6 +98,8 @@ function ActionsMenuContainer(props: OwnProps & StateToProps & DispatchToProps): openMoveTaskToProjectWindow, onViewAnalytics, onViewQualityControl, + onViewConsensusManagement, + mergeConsensusJobs, } = props; const onClickMenu = (params: MenuInfo): void | JSX.Element => { const [action] = params.keyPath; @@ -113,6 +121,10 @@ function ActionsMenuContainer(props: OwnProps & StateToProps & DispatchToProps): onViewAnalytics(); } else if (action === Actions.QUALITY_CONTROL) { onViewQualityControl(); + } else if (action === Actions.CONSENSUS_MANAGEMENT) { + onViewConsensusManagement(); + } else if (action === Actions.MERGE_CONSENSUS_JOBS) { + mergeConsensusJobs(taskInstance); } }; @@ -127,6 +139,7 @@ function ActionsMenuContainer(props: OwnProps & StateToProps & DispatchToProps): inferenceIsActive={inferenceIsActive} onClickMenu={onClickMenu} taskDimension={taskInstance.dimension} + consensusEnabled={taskInstance.consensusEnabled} /> ); } diff --git a/cvat-ui/src/cvat-core-wrapper.ts b/cvat-ui/src/cvat-core-wrapper.ts index 5dab7ab1d0fd..c70479ad18ba 100644 --- a/cvat-ui/src/cvat-core-wrapper.ts +++ b/cvat-ui/src/cvat-core-wrapper.ts @@ -22,6 +22,7 @@ import Project from 'cvat-core/src/project'; import QualityReport, { QualitySummary } from 'cvat-core/src/quality-report'; import QualityConflict, { AnnotationConflict, ConflictSeverity } from 'cvat-core/src/quality-conflict'; import QualitySettings, { TargetMetric } from 'cvat-core/src/quality-settings'; +import ConsensusSettings from 'cvat-core/src/consensus-settings'; import { FramesMetaData, FrameData } from 'cvat-core/src/frames'; import { ServerError, RequestError } from 'cvat-core/src/exceptions'; import { @@ -98,6 +99,7 @@ export { QualityReport, QualityConflict, QualitySettings, + ConsensusSettings, TargetMetric, AnnotationConflict, ConflictSeverity, diff --git a/cvat-ui/src/reducers/consensus-reducer.ts b/cvat-ui/src/reducers/consensus-reducer.ts new file mode 100644 index 000000000000..8e9ea00489f2 --- /dev/null +++ b/cvat-ui/src/reducers/consensus-reducer.ts @@ -0,0 +1,78 @@ +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import { Project, ProjectOrTaskOrJob } from 'cvat-core-wrapper'; +import { ConsensusActions, ConsensusActionTypes } from 'actions/consensus-actions'; +import { getInstanceType } from 'actions/common'; +import { ConsensusState } from '.'; + +const defaultState: ConsensusState = { + taskInstance: null, + jobInstance: null, + fetching: true, + consensusSettings: null, + actions: { + merging: {}, + }, +}; + +export function makeKey(instance: Exclude): string { + return `${getInstanceType(instance)}_${instance.id}`; +} + +export default (state: ConsensusState = defaultState, action: ConsensusActions): ConsensusState => { + switch (action.type) { + case ConsensusActionTypes.MERGE_CONSENSUS_JOBS: { + const { instance } = action.payload; + const { merging } = state.actions; + + merging[makeKey(instance)] = true; + + return { + ...state, + actions: { + ...state.actions, + merging: { + ...merging, + }, + }, + }; + } + + case ConsensusActionTypes.MERGE_CONSENSUS_JOBS_SUCCESS: { + const { instance } = action.payload; + const { merging } = state.actions; + + merging[makeKey(instance)] = false; + + return { + ...state, + actions: { + ...state.actions, + merging: { + ...merging, + }, + }, + }; + } + case ConsensusActionTypes.MERGE_CONSENSUS_JOBS_FAILED: { + const { instance } = action.payload; + const { merging } = state.actions; + + delete merging[makeKey(instance)]; + + return { + ...state, + actions: { + ...state.actions, + merging: { + ...merging, + }, + }, + }; + } + default: + return state; + } +}; diff --git a/cvat-ui/src/reducers/import-reducer.ts b/cvat-ui/src/reducers/import-reducer.ts index 24af643ce3cb..65d6a35b368a 100644 --- a/cvat-ui/src/reducers/import-reducer.ts +++ b/cvat-ui/src/reducers/import-reducer.ts @@ -4,8 +4,9 @@ // SPDX-License-Identifier: MIT import { ImportActions, ImportActionTypes } from 'actions/import-actions'; -import { getInstanceType, RequestInstanceType } from 'actions/requests-actions'; +import { RequestInstanceType } from 'actions/requests-actions'; import { ProjectOrTaskOrJob } from 'cvat-core-wrapper'; +import { getInstanceType } from 'actions/common'; import { ImportState } from '.'; const defaultProgress = 0.0; diff --git a/cvat-ui/src/reducers/index.ts b/cvat-ui/src/reducers/index.ts index 370a468e191a..dfcecd55427b 100644 --- a/cvat-ui/src/reducers/index.ts +++ b/cvat-ui/src/reducers/index.ts @@ -9,7 +9,7 @@ import { Webhook, MLModel, Organization, Job, Task, Project, Label, User, QualityConflict, FramesMetaData, RQStatus, Event, Invitation, SerializedAPISchema, Request, JobValidationLayout, QualitySettings, TaskValidationLayout, ObjectState, - AboutData, + ConsensusSettings, AboutData, } from 'cvat-core-wrapper'; import { IntelligentScissors } from 'utils/opencv-wrapper/intelligent-scissors'; import { KeyMap, KeyMapItem } from 'utils/mousetrap-react'; @@ -171,6 +171,18 @@ export interface ImportState { instanceType: 'project' | 'task' | 'job' | null; } +export interface ConsensusState { + fetching: boolean; + consensusSettings: ConsensusSettings | null; + taskInstance: Task | null; + jobInstance: Job | null; + actions: { + merging: { + [instanceKey: string]: boolean; + }; + } +} + export interface FormatsState { annotationFormats: any; fetching: boolean; @@ -477,6 +489,7 @@ export interface NotificationsState { exporting: null | ErrorState; importing: null | ErrorState; moving: null | ErrorState; + mergingConsensus: null | ErrorState; }; jobs: { updating: null | ErrorState; @@ -600,6 +613,7 @@ export interface NotificationsState { loadingDone: null | NotificationState; importingDone: null | NotificationState; movingDone: null | NotificationState; + mergingConsensusDone: null | NotificationState; }; models: { inferenceDone: null | NotificationState; @@ -1019,6 +1033,7 @@ export interface CombinedState { review: ReviewState; export: ExportState; import: ImportState; + consensus: ConsensusState; cloudStorages: CloudStoragesState; organizations: OrganizationState; invitations: InvitationsState; diff --git a/cvat-ui/src/reducers/notifications-reducer.ts b/cvat-ui/src/reducers/notifications-reducer.ts index 75ec88647192..c1878fee19ac 100644 --- a/cvat-ui/src/reducers/notifications-reducer.ts +++ b/cvat-ui/src/reducers/notifications-reducer.ts @@ -23,9 +23,11 @@ import { JobsActionTypes } from 'actions/jobs-actions'; import { WebhooksActionsTypes } from 'actions/webhooks-actions'; import { InvitationsActionTypes } from 'actions/invitations-actions'; import { ServerAPIActionTypes } from 'actions/server-actions'; -import { RequestsActionsTypes, getInstanceType } from 'actions/requests-actions'; +import { RequestsActionsTypes } from 'actions/requests-actions'; import { ImportActionTypes } from 'actions/import-actions'; import { ExportActionTypes } from 'actions/export-actions'; +import { ConsensusActionTypes } from 'actions/consensus-actions'; +import { getInstanceType } from 'actions/common'; import config from 'config'; import { NotificationsState } from '.'; @@ -72,6 +74,7 @@ const defaultState: NotificationsState = { exporting: null, importing: null, moving: null, + mergingConsensus: null, }, jobs: { updating: null, @@ -195,6 +198,7 @@ const defaultState: NotificationsState = { loadingDone: null, importingDone: null, movingDone: null, + mergingConsensusDone: null, }, models: { inferenceDone: null, @@ -734,6 +738,55 @@ export default function (state = defaultState, action: AnyAction): Notifications }, }; } + case ConsensusActionTypes.MERGE_CONSENSUS_JOBS_SUCCESS: { + const { instance } = action.payload; + let message = ''; + const instanceType = getInstanceType(instance); + if (instanceType === 'job') { + message = + `Consensus [job #${instance.id}](/tasks/${instance.taskId}/jobs/${instance.id}) has been merged`; + } else if (instanceType === 'task') { + message = `Consensus jobs in the [task #${instance.id}](/tasks/${instance.id}) have been merged`; + } + return { + ...state, + messages: { + ...state.messages, + tasks: { + ...state.messages.tasks, + mergingConsensusDone: { + message, + }, + }, + }, + }; + } + case ConsensusActionTypes.MERGE_CONSENSUS_JOBS_FAILED: { + const { instance } = action.payload; + let message = ''; + const instanceType = getInstanceType(instance); + if (instanceType === 'job') { + message = + `Could not merge the [job #${instance.id}](/tasks/${instance.taskId}/jobs/${instance.id})`; + } else if (instanceType === 'task') { + message = `Could not merge the [task ${instance.id}](/tasks/${instance.id})`; + } + return { + ...state, + errors: { + ...state.errors, + tasks: { + ...state.errors.tasks, + mergingConsensus: { + message, + reason: action.payload.error, + shouldLog: !(action.payload.error instanceof ServerError), + className: 'cvat-notification-notice-consensus-merge-task-failed', + }, + }, + }, + }; + } case TasksActionTypes.CREATE_TASK_FAILED: { return { ...state, diff --git a/cvat-ui/src/reducers/root-reducer.ts b/cvat-ui/src/reducers/root-reducer.ts index 96b94f28c46d..9072a5d95fe4 100644 --- a/cvat-ui/src/reducers/root-reducer.ts +++ b/cvat-ui/src/reducers/root-reducer.ts @@ -20,6 +20,7 @@ import userAgreementsReducer from './useragreements-reducer'; import reviewReducer from './review-reducer'; import exportReducer from './export-reducer'; import importReducer from './import-reducer'; +import consensusReducer from './consensus-reducer'; import cloudStoragesReducer from './cloud-storages-reducer'; import organizationsReducer from './organizations-reducer'; import webhooksReducer from './webhooks-reducer'; @@ -46,6 +47,7 @@ export default function createRootReducer(): Reducer { review: reviewReducer, export: exportReducer, import: importReducer, + consensus: consensusReducer, cloudStorages: cloudStoragesReducer, organizations: organizationsReducer, webhooks: webhooksReducer, diff --git a/cvat-ui/src/utils/fetch.ts b/cvat-ui/src/utils/fetch.ts new file mode 100644 index 000000000000..a50524824525 --- /dev/null +++ b/cvat-ui/src/utils/fetch.ts @@ -0,0 +1,17 @@ +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import { getCore, Task } from 'cvat-core-wrapper'; + +const core = getCore(); + +export async function fetchTask(id: number): Promise { + let taskInstance = null; + try { + [taskInstance] = await core.tasks.get({ id }); + } catch (error: unknown) { + throw new Error('The task was not found on the server'); + } + return taskInstance; +} diff --git a/cvat/apps/consensus/__init__.py b/cvat/apps/consensus/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/cvat/apps/consensus/apps.py b/cvat/apps/consensus/apps.py new file mode 100644 index 000000000000..5c37ca073fcb --- /dev/null +++ b/cvat/apps/consensus/apps.py @@ -0,0 +1,18 @@ +# Copyright (C) CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +from django.apps import AppConfig + + +class ConsensusConfig(AppConfig): + name = "cvat.apps.consensus" + + def ready(self) -> None: + + from cvat.apps.iam.permissions import load_app_permissions + + load_app_permissions(self) + + # Required to define signals in the application + from . import signals # pylint: disable=unused-import diff --git a/cvat/apps/consensus/intersect_merge.py b/cvat/apps/consensus/intersect_merge.py new file mode 100644 index 000000000000..5064b885b536 --- /dev/null +++ b/cvat/apps/consensus/intersect_merge.py @@ -0,0 +1,501 @@ +# Copyright (C) CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +from __future__ import annotations + +import itertools +from abc import ABCMeta, abstractmethod +from collections.abc import Collection +from typing import ClassVar, Iterable, Sequence + +import attrs +import datumaro as dm +import datumaro.components.merge.intersect_merge +from datumaro.components.errors import FailedLabelVotingError +from datumaro.util.annotation_util import mean_bbox +from datumaro.util.attrs_util import ensure_cls + +from cvat.apps.quality_control.quality_reports import ( + ComparisonParameters, + DistanceComparator, + segment_iou, +) + + +@attrs.define(kw_only=True, slots=False) +class IntersectMerge(datumaro.components.merge.intersect_merge.IntersectMerge): + @attrs.define(kw_only=True, slots=False) + class Conf: + pairwise_dist: float = 0.5 + sigma: float = 0.1 + + output_conf_thresh: float = 0 + quorum: int = 1 + ignored_attributes: Collection[str] = attrs.field(factory=tuple) + torso_r: float = 0.01 + + groups: Collection[Collection[str]] = attrs.field(factory=tuple) + close_distance: float = 0 # disabled + + included_annotation_types: Collection[dm.AnnotationType] = None + + def __attrs_post_init__(self): + if self.included_annotation_types is None: + self.included_annotation_types = ComparisonParameters.included_annotation_types + + conf: Conf = attrs.field(converter=ensure_cls(Conf), factory=Conf) + + def __call__(self, *datasets): + return dm.Dataset(super().__call__(*datasets)) + + def _find_cluster_attrs(self, cluster, ann): + merged_attributes = super()._find_cluster_attrs(cluster, ann) + merged_attributes["source"] = "consensus" + return merged_attributes + + def _check_annotation_distance(self, t, merged_clusters): + return # disabled, need to clarify how to compare merged instances correctly + + def get_ann_dataset_id(self, ann_id: int) -> int: + return self._dataset_map[self.get_ann_source(ann_id)][1] + + def get_ann_source_item(self, ann_id: int) -> dm.DatasetItem: + return self._item_map[self._ann_map[ann_id][1]][0] + + def get_item_media_dims(self, ann_id: int) -> tuple[int, int]: + return self.get_ann_source_item(ann_id).media_as(dm.Image).size + + def get_label_id(self, label: str) -> int: + return self._get_label_id(label) + + def get_src_label_name(self, ann: dm.Annotation, label_id: int) -> str: + return self._get_src_label_name(ann, label_id) + + def get_dataset_source_id(self, dataset_id: int) -> int: + return self._dataset_map[dataset_id][1] + + def dataset_count(self) -> int: + return len(self._dataset_map) + + def _make_mergers(self, sources): + def _make(c, **kwargs): + kwargs.update(attrs.asdict(self.conf)) + fields = attrs.fields_dict(c) + return c(**{k: v for k, v in kwargs.items() if k in fields}, context=self) + + def _for_type(t: dm.AnnotationType, **kwargs) -> AnnotationMatcher: + if t is dm.AnnotationType.label: + return _make(LabelMerger, **kwargs) + elif t is dm.AnnotationType.bbox: + return _make(BboxMerger, **kwargs) + elif t is dm.AnnotationType.mask: + return _make(MaskMerger, **kwargs) + elif t is dm.AnnotationType.polygon or t is dm.AnnotationType.mask: + return _make(PolygonMerger, **kwargs) + elif t is dm.AnnotationType.polyline: + return _make(LineMerger, **kwargs) + elif t is dm.AnnotationType.points: + return _make(PointsMerger, **kwargs) + elif t is dm.AnnotationType.skeleton: + return _make(SkeletonMerger, **kwargs) + else: + raise AssertionError(f"Annotation type {t} is not supported") + + self._mergers = { + t: _for_type(t, categories=self._categories) + for t in self.conf.included_annotation_types + } + + +@attrs.define(kw_only=True, slots=False) +class AnnotationMatcher(metaclass=ABCMeta): + _context: IntersectMerge + + @abstractmethod + def match_annotations( + self, sources: Sequence[Sequence[dm.Annotation]] + ) -> Sequence[Sequence[dm.Annotation]]: + "Matches annotations from different sources and produces annotation groups" + + @abstractmethod + def distance(self, a: dm.Annotation, b: dm.Annotation) -> float: + """ + Computes distance (actually similarity) between 2 annotations. + The output value is in the range [0; 1]. + + distance(a, b) == 0 => annotations are different + + distance(a, b) == 1 => annotations are same + """ + + +@attrs.define(kw_only=True, slots=False) +class LabelMatcher(AnnotationMatcher): + def distance(self, a: dm.Label, b: dm.Label) -> bool: + a_label = self._context.get_any_label_name(a, a.label) + b_label = self._context.get_any_label_name(b, b.label) + return a_label == b_label + + def match_annotations(self, sources): + return [list(itertools.chain.from_iterable(sources))] + + +CacheKey = tuple[int, int] + + +@attrs.define +class CachedSimilarityFunction: + cache: dict[CacheKey, float] = attrs.field(factory=dict, kw_only=True) + + def __call__(self, a_ann: dm.Annotation, b_ann: dm.Annotation) -> float: + a_ann_id = id(a_ann) + b_ann_id = id(b_ann) + + if a_ann_id == b_ann_id: + return 1 + + key = ( + a_ann_id, + b_ann_id, + ) # make sure the annotations have stable ids before calling this + key = self._sort_key(key) + return self.cache[key] + + @staticmethod + def _sort_key(key: CacheKey) -> CacheKey: + return tuple(sorted(key)) + + def pop(self, key: CacheKey) -> float: + return self.cache.pop(self._sort_key(key), None) + + def set(self, key: CacheKey, value: float): + self.cache[self._sort_key(key)] = value + + def keys(self) -> Iterable[CacheKey]: + return self.cache.keys() + + def clear_cache(self): + self.cache.clear() + + +@attrs.define(kw_only=True, slots=False) +class ShapeMatcher(AnnotationMatcher, metaclass=ABCMeta): + pairwise_dist: float = 0.9 + cluster_dist: float | None = None + categories: dm.CategoriesInfo + _comparator: DistanceComparator = attrs.field(init=False) + _distance: CachedSimilarityFunction = attrs.field(init=False) + + def __attrs_post_init__(self): + if self.cluster_dist is None: + self.cluster_dist = self.pairwise_dist + + self._comparator = DistanceComparator( + categories=self.categories, + return_distances=True, + iou_threshold=self.pairwise_dist, + oks_sigma=self._context.conf.sigma, + line_torso_radius=self._context.conf.torso_r, + panoptic_comparison=False, + # allow_groups=True is not supported. Requires significant logic changes in + # the whole merging algorithm, as it's likely to produce clusters with annotations + # from the same source or some new annotations (instances). + allow_groups=False, + ) + + self._distance = CachedSimilarityFunction() + + def distance(self, a, b): + return self._distance(a, b) + + def _match_annotations_between_two_sources( + self, source_a: list[dm.Annotation], source_b: list[dm.Annotation] + ) -> list[tuple[dm.Annotation, dm.Annotation]]: + if not source_a and not source_b: + return [] + + item_a = self._context.get_ann_source_item(id(source_a[0])) + item_b = self._context.get_ann_source_item(id(source_b[0])) + return self._match_annotations_between_two_items(item_a, item_b) + + def _match_annotations_between_two_items( + self, item_a: dm.DatasetItem, item_b: dm.DatasetItem + ) -> list[tuple[dm.Annotation, dm.Annotation]]: + matches, distances = self.match_annotations_between_two_items(item_a, item_b) + + # Remember distances + for (p_a_id, p_b_id), dist in distances.items(): + self._distance.set((p_a_id, p_b_id), dist) + + return matches + + @abstractmethod + def match_annotations_between_two_items( + self, item_a: dm.DatasetItem, item_b: dm.DatasetItem + ) -> tuple[list[tuple[dm.Annotation, dm.Annotation]], dict[CacheKey, float]]: ... + + def match_annotations(self, sources): + distance = self.distance + cluster_dist = self.cluster_dist + + id_segm = {id(ann): (ann, id(source)) for source in sources for ann in source} + + def _is_close_enough(cluster, extra_id): + # check if whole cluster IoU will not be broken + # when this segment is added + b = id_segm[extra_id][0] + for a_id in cluster: + a = id_segm[a_id][0] + if distance(a, b) < cluster_dist: + return False + return True + + def _has_same_source(cluster, extra_id): + b = id_segm[extra_id][1] + for a_id in cluster: + a = id_segm[a_id][1] + if a == b: + return True + return False + + # match segments in sources, pairwise + adjacent = {i: [] for i in id_segm} # id(sgm) -> [id(adj_sgm1), ...] + for a_idx, src_a in enumerate(sources): + # matches further sources of same frame for matching annotations + for src_b in sources[a_idx + 1 :]: + # an annotation can be adjacent to multiple annotations + matches = self._match_annotations_between_two_sources(src_a, src_b) + for a, b in matches: + adjacent[id(a)].append(id(b)) + + # join all segments into matching clusters + clusters = [] + visited = set() + for cluster_idx in adjacent: + if cluster_idx in visited: + continue + + cluster = set() + to_visit = {cluster_idx} + while to_visit: + c = to_visit.pop() + cluster.add(c) + visited.add(c) + + for i in adjacent[c]: + if i in visited: + continue + + if _has_same_source(cluster, i): + continue + + if 0 < cluster_dist and not _is_close_enough(cluster, i): + # if positive, cluster_dist and this annotation isn't close enough + # with other annotations in the cluster + continue + + to_visit.add(i) # check annotations adjacent to the new one in the cluster + + clusters.append([id_segm[i][0] for i in cluster]) + + return clusters + + +@attrs.define(kw_only=True, slots=False) +class BboxMatcher(ShapeMatcher): + def match_annotations_between_two_items(self, item_a, item_b): + matches, _, _, _, distances = self._comparator.match_boxes(item_a, item_b) + return matches, distances + + +@attrs.define(kw_only=True, slots=False) +class PolygonMatcher(ShapeMatcher): + _annotation_type: ClassVar[dm.AnnotationType] = dm.AnnotationType.polygon + + def match_annotations_between_two_items(self, item_a, item_b): + item_a = item_a.wrap( + annotations=[a for a in item_a.annotations if a.type == self._annotation_type] + ) + item_b = item_b.wrap( + annotations=[a for a in item_b.annotations if a.type == self._annotation_type] + ) + matches, _, _, _, distances = self._comparator.match_segmentations(item_a, item_b) + return matches, distances + + +@attrs.define(kw_only=True, slots=False) +class MaskMatcher(PolygonMatcher): + _annotation_type: ClassVar[dm.AnnotationType] = dm.AnnotationType.mask + + +@attrs.define(kw_only=True, slots=False) +class PointsMatcher(ShapeMatcher): + def match_annotations_between_two_items(self, item_a, item_b): + matches, _, _, _, distances = self._comparator.match_points(item_a, item_b) + return matches, distances + + +@attrs.define(kw_only=True, slots=False) +class SkeletonMatcher(ShapeMatcher): + def match_annotations_between_two_items(self, item_a, item_b): + matches, _, _, _, distances = self._comparator.match_skeletons(item_a, item_b) + return matches, distances + + +@attrs.define(kw_only=True, slots=False) +class LineMatcher(ShapeMatcher): + def match_annotations_between_two_items(self, item_a, item_b): + matches, _, _, _, distances = self._comparator.match_lines(item_a, item_b) + return matches, distances + + +@attrs.define(kw_only=True, slots=False) +class AnnotationMerger(AnnotationMatcher, metaclass=ABCMeta): + @abstractmethod + def merge_clusters( + self, clusters: Sequence[Sequence[dm.Annotation]] + ) -> Sequence[dm.Annotation]: + "Merges annotations in each cluster into a single annotation" + + +@attrs.define(kw_only=True, slots=False) +class LabelMerger(AnnotationMerger, LabelMatcher): + quorum: int = 0 + + def merge_clusters(self, clusters): + assert len(clusters) <= 1 + if len(clusters) == 0: + return [] + + votes = {} # label -> score + for ann in clusters[0]: + label = self._context.get_src_label_name(ann, ann.label) + votes[label] = 1 + votes.get(label, 0) + + merged = [] + for label, count in votes.items(): + if count < self.quorum: + sources = set( + self._context.get_ann_source(id(a)) + for a in clusters[0] + if label not in [self._context.get_src_label_name(l, l.label) for l in a] + ) + sources = [self._context.get_dataset_source_id(s) for s in sources] + self._context.add_item_error(FailedLabelVotingError, votes, sources=sources) + continue + + merged.append( + dm.Label( + self._context.get_label_id(label), + attributes={"score": count / self._context.dataset_count()}, + ) + ) + + return merged + + +@attrs.define(kw_only=True, slots=False) +class ShapeMerger(AnnotationMerger, ShapeMatcher): + quorum = attrs.field(converter=int, default=0) + + def merge_clusters(self, clusters): + return list(filter(lambda x: x is not None, map(self.merge_cluster, clusters))) + + def find_cluster_label(self, cluster: Sequence[dm.Annotation]) -> tuple[int | None, float]: + votes = {} + for s in cluster: + label = self._context.get_src_label_name(s, s.label) + state = votes.setdefault(label, [0, 0]) + state[0] += s.attributes.get("score", 1.0) + state[1] += 1 + + label, (score, count) = max(votes.items(), key=lambda e: e[1][0]) + if count < self.quorum: + self._context.add_item_error(FailedLabelVotingError, votes) + label = None + else: + label = self._context.get_label_id(label) + + score = score / self._context.dataset_count() + return label, score + + def _merge_cluster_shape_mean_box_nearest( + self, cluster: Sequence[dm.Annotation] + ) -> dm.Annotation: + mbbox = dm.Bbox(*mean_bbox(cluster)) + img_h, img_w = self._context.get_item_media_dims(id(cluster[0])) + + dist = [] + for s in cluster: + if isinstance(s, (dm.Points, dm.PolyLine)): + s = self._comparator.to_polygon(dm.Bbox(*s.get_bbox())) + elif isinstance(s, dm.Bbox): + s = self._comparator.to_polygon(s) + dist.append( + segment_iou(self._comparator.to_polygon(mbbox), s, img_h=img_h, img_w=img_w) + ) + nearest_pos, _ = max(enumerate(dist), key=lambda e: e[1]) + return cluster[nearest_pos] + + def merge_cluster_shape_mean_nearest(self, cluster: Sequence[dm.Annotation]) -> dm.Annotation: + return self._merge_cluster_shape_mean_box_nearest(cluster) + + def merge_cluster_shape(self, cluster: Sequence[dm.Annotation]) -> tuple[dm.Annotation, float]: + shape = self.merge_cluster_shape_mean_nearest(cluster) + shape_score = sum(max(0, self.distance(shape, s)) for s in cluster) / len(cluster) + return shape, shape_score + + def merge_cluster(self, cluster): + label, label_score = self.find_cluster_label(cluster) + + # when the merged annotation is rejected due to quorum constraint + if label is None: + return None + + shape, shape_score = self.merge_cluster_shape(cluster) + shape.z_order = max(cluster, key=lambda a: a.z_order).z_order + shape.label = label + shape.attributes["score"] = label_score * shape_score + + return shape + + +@attrs.define(kw_only=True, slots=False) +class BboxMerger(ShapeMerger, BboxMatcher): + pass + + +@attrs.define(kw_only=True, slots=False) +class PolygonMerger(ShapeMerger, PolygonMatcher): + pass + + +@attrs.define(kw_only=True, slots=False) +class MaskMerger(ShapeMerger, MaskMatcher): + pass + + +@attrs.define(kw_only=True, slots=False) +class PointsMerger(ShapeMerger, PointsMatcher): + pass + + +@attrs.define(kw_only=True, slots=False) +class LineMerger(ShapeMerger, LineMatcher): + pass + + +@attrs.define(kw_only=True, slots=False) +class SkeletonMerger(ShapeMerger, SkeletonMatcher): + def merge_cluster_shape_mean_nearest(self, cluster): + dist = {} + for idx, a in enumerate(cluster): + a_cluster_distance = 0 + for b in cluster: + # (1 - x) because it's actually a similarity function + a_cluster_distance += 1 - self.distance(a, b) + + dist[idx] = a_cluster_distance / len(cluster) + + return cluster[min(dist, key=dist.get)] diff --git a/cvat/apps/consensus/merging_manager.py b/cvat/apps/consensus/merging_manager.py new file mode 100644 index 000000000000..34cec8cc063b --- /dev/null +++ b/cvat/apps/consensus/merging_manager.py @@ -0,0 +1,248 @@ +# Copyright (C) CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +import math +from typing import Type + +import datumaro as dm +import django_rq +from django.conf import settings +from django.db import transaction +from django_rq.queues import DjangoRQ as RqQueue +from rq.job import Job as RqJob +from rq.job import JobStatus as RqJobStatus + +from cvat.apps.consensus.intersect_merge import IntersectMerge +from cvat.apps.consensus.models import ConsensusSettings +from cvat.apps.dataset_manager.bindings import import_dm_annotations +from cvat.apps.dataset_manager.task import PatchAction, patch_job_data +from cvat.apps.engine.models import ( + DimensionType, + Job, + JobType, + StageChoice, + StateChoice, + Task, + User, + clear_annotations_in_jobs, +) +from cvat.apps.engine.types import ExtendedRequest +from cvat.apps.engine.utils import define_dependent_job, get_rq_job_meta, get_rq_lock_by_user +from cvat.apps.profiler import silk_profile +from cvat.apps.quality_control.quality_reports import ComparisonParameters, JobDataProvider + + +class _TaskMerger: + _task: Task + _jobs: dict[int, list[tuple[int, User]]] + _parent_jobs: list[Job] + _settings: ConsensusSettings + + def check_merging_available(self, *, parent_job_id: int | None = None): + if not self._task.consensus_replicas: + raise MergingNotAvailable("Consensus is not enabled in this task") + + if self._task.dimension != DimensionType.DIM_2D: + raise MergingNotAvailable("Merging is only supported in 2d tasks") + + if self._jobs is None: + self._init_jobs() + + if not self._jobs: + raise MergingNotAvailable( + f"No {JobType.ANNOTATION} jobs in the {StageChoice.ANNOTATION} stage or " + f"no {JobType.CONSENSUS_REPLICA} jobs " + f"not in the {StageChoice.ANNOTATION} - {StateChoice.NEW} state found" + ) + + if parent_job_id: + parent_job_info = self._jobs.get(parent_job_id) + if not parent_job_info: + raise MergingNotAvailable( + f"No annotated consensus jobs found for parent job {parent_job_id}. " + f"Make sure at least one consensus job is not " + f"in the {StageChoice.ANNOTATION} - {StateChoice.NEW} state" + ) + + def __init__(self, task: int | Task) -> None: + if not isinstance(task, Task): + task = Task.objects.get(pk=task) + self._task = task + + self._init_jobs() + + self._settings = ConsensusSettings.objects.get_or_create(task=task)[0] + + def _init_jobs(self) -> None: + job_map = {} # parent_job_id -> [(consensus_job_id, assignee)] + parent_jobs: dict[int, Job] = {} + for job in ( + Job.objects.prefetch_related("segment", "parent_job", "assignee") + .filter( + segment__task=self._task, + type=JobType.CONSENSUS_REPLICA, + parent_job__stage=StageChoice.ANNOTATION, + parent_job__isnull=False, + ) + .exclude(stage=StageChoice.ANNOTATION, state=StateChoice.NEW) + ): + job_map.setdefault(job.parent_job_id, []).append((job.id, job.assignee)) + parent_jobs.setdefault(job.parent_job_id, job.parent_job) + + self._jobs = job_map + self._parent_jobs = list(parent_jobs.values()) + + @staticmethod + def _get_annotations(job_id: int) -> dm.Dataset: + return JobDataProvider(job_id).dm_dataset + + def _merge_consensus_jobs(self, parent_job_id: int): + self.check_merging_available(parent_job_id=parent_job_id) + + consensus_job_info = self._jobs[parent_job_id] + + consensus_job_ids = [consensus_job_id for consensus_job_id, _ in consensus_job_info] + + consensus_job_data_providers = list(map(JobDataProvider, consensus_job_ids)) + consensus_datasets = [ + consensus_job_data_provider.dm_dataset + for consensus_job_data_provider in consensus_job_data_providers + ] + + comparison_parameters = ComparisonParameters() + merger = IntersectMerge( + conf=IntersectMerge.Conf( + pairwise_dist=self._settings.iou_threshold, + quorum=math.ceil(self._settings.quorum * len(consensus_datasets)), + sigma=comparison_parameters.oks_sigma, + torso_r=comparison_parameters.line_thickness, + included_annotation_types=comparison_parameters.included_annotation_types, + ) + ) + merged_dataset = merger(*consensus_datasets) + + # Delete the existing annotations in the job. + # If we don't delete existing annotations, the imported annotations + # will be appended to the existing annotations, and thus updated annotation + # would have both existing + imported annotations, but we only want the + # imported annotations + clear_annotations_in_jobs([parent_job_id]) + + parent_job_data_provider = JobDataProvider(parent_job_id) + + # imports the annotations in the `parent_job.job_data` instance + import_dm_annotations(merged_dataset, parent_job_data_provider.job_data) + + # updates the annotations in the job + patch_job_data( + parent_job_id, parent_job_data_provider.job_data.data.serialize(), PatchAction.UPDATE + ) + + for parent_job in self._parent_jobs: + if parent_job.id == parent_job_id and parent_job.type == JobType.ANNOTATION.value: + parent_job.state = StateChoice.COMPLETED.value + parent_job.save() + + @transaction.atomic + def merge_all_consensus_jobs(self) -> None: + for parent_job_id in self._jobs.keys(): + self._merge_consensus_jobs(parent_job_id) + + @transaction.atomic + def merge_single_consensus_job(self, parent_job_id: int) -> None: + self._merge_consensus_jobs(parent_job_id) + + +class MergingNotAvailable(Exception): + pass + + +class JobAlreadyExists(MergingNotAvailable): + def __init__(self, instance: Task | Job): + super().__init__() + self.instance = instance + + def __str__(self): + return f"Merging for this {type(self.instance).__name__.lower()} already enqueued" + + +class MergingManager: + _QUEUE_CUSTOM_JOB_PREFIX = "consensus-merge-" + _JOB_RESULT_TTL = 300 + + def _get_queue(self) -> RqQueue: + return django_rq.get_queue(settings.CVAT_QUEUES.CONSENSUS.value) + + def _make_job_id(self, task_id: int, job_id: int | None, user_id: int) -> str: + key = f"{self._QUEUE_CUSTOM_JOB_PREFIX}task-{task_id}" + if job_id: + key += f"-job-{job_id}" + key += f"-user-{user_id}" # TODO: remove user id, add support for non owners to get status + return key + + def _check_merging_available(self, task: Task, job: Job | None): + _TaskMerger(task=task).check_merging_available(parent_job_id=job.id if job else None) + + def schedule_merge(self, target: Task | Job, *, request: ExtendedRequest) -> str: + if isinstance(target, Job): + target_task = target.segment.task + target_job = target + else: + target_task = target + target_job = None + + self._check_merging_available(target_task, target_job) + + queue = self._get_queue() + + user_id = request.user.id + with get_rq_lock_by_user(queue, user_id=user_id): + rq_id = self._make_job_id( + task_id=target_task.id, + job_id=target_job.id if target_job else None, + user_id=user_id, + ) + rq_job = queue.fetch_job(rq_id) + if rq_job: + if rq_job.get_status(refresh=False) in ( + RqJobStatus.QUEUED, + RqJobStatus.STARTED, + RqJobStatus.SCHEDULED, + RqJobStatus.DEFERRED, + ): + raise JobAlreadyExists(target) + + rq_job.delete() + + dependency = define_dependent_job( + queue, user_id=user_id, rq_id=rq_id, should_be_dependent=True + ) + + queue.enqueue( + self._merge, + target_type=type(target), + target_id=target.id, + job_id=rq_id, + meta=get_rq_job_meta(request=request, db_obj=target), + result_ttl=self._JOB_RESULT_TTL, + failure_ttl=self._JOB_RESULT_TTL, + depends_on=dependency, + ) + + return rq_id + + def get_job(self, rq_id: str) -> RqJob | None: + queue = self._get_queue() + return queue.fetch_job(rq_id) + + @classmethod + @silk_profile() + def _merge(cls, *, target_type: Type[Task | Job], target_id: int) -> int: + if issubclass(target_type, Task): + return _TaskMerger(task=target_id).merge_all_consensus_jobs() + elif issubclass(target_type, Job): + job = Job.objects.get(pk=target_id) + return _TaskMerger(task=job.get_task_id()).merge_single_consensus_job(target_id) + else: + assert False diff --git a/cvat/apps/consensus/migrations/0001_initial.py b/cvat/apps/consensus/migrations/0001_initial.py new file mode 100644 index 000000000000..898903ebc061 --- /dev/null +++ b/cvat/apps/consensus/migrations/0001_initial.py @@ -0,0 +1,54 @@ +# Generated by Django 4.2.18 on 2025-02-10 11:29 + +import django.db.models.deletion +from django.db import migrations, models + + +def init_consensus_settings_in_existing_consensus_tasks(apps, schema_editor): + Task = apps.get_model("engine", "Task") + ConsensusSettings = apps.get_model("consensus", "ConsensusSettings") + + tasks_with_consensus = Task.objects.filter( + segment__job__type="consensus_replica", consensus_settings__isnull=True + ).distinct() + ConsensusSettings.objects.bulk_create( + [ConsensusSettings(task=t) for t in tasks_with_consensus], + batch_size=10000, + ) + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ("engine", "0088_consensus_jobs"), + ] + + operations = [ + migrations.CreateModel( + name="ConsensusSettings", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("quorum", models.FloatField(default=0.5)), + ("iou_threshold", models.FloatField(default=0.4)), + ( + "task", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="consensus_settings", + to="engine.task", + ), + ), + ], + ), + migrations.RunPython( + init_consensus_settings_in_existing_consensus_tasks, + reverse_code=migrations.RunPython.noop, + ), + ] diff --git a/cvat/apps/consensus/migrations/__init__.py b/cvat/apps/consensus/migrations/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/cvat/apps/consensus/models.py b/cvat/apps/consensus/models.py new file mode 100644 index 000000000000..d555d8a0ba7a --- /dev/null +++ b/cvat/apps/consensus/models.py @@ -0,0 +1,20 @@ +# Copyright (C) CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +from __future__ import annotations + +from django.db import models + +import cvat.apps.quality_control.quality_reports as qc +from cvat.apps.engine.models import Task + + +class ConsensusSettings(models.Model): + task = models.ForeignKey(Task, on_delete=models.CASCADE, related_name="consensus_settings") + quorum = models.FloatField(default=0.5) + iou_threshold = models.FloatField(default=qc.DatasetComparator.DEFAULT_SETTINGS.iou_threshold) + + @property + def organization_id(self): + return self.task.organization_id diff --git a/cvat/apps/consensus/permissions.py b/cvat/apps/consensus/permissions.py new file mode 100644 index 000000000000..3eb24f59cb09 --- /dev/null +++ b/cvat/apps/consensus/permissions.py @@ -0,0 +1,250 @@ +# Copyright (C) CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +from typing import cast + +from django.conf import settings +from rest_framework.exceptions import PermissionDenied, ValidationError + +from cvat.apps.engine.models import Job, Project, Task +from cvat.apps.engine.permissions import TaskPermission +from cvat.apps.engine.types import ExtendedRequest +from cvat.apps.iam.permissions import OpenPolicyAgentPermission, StrEnum, get_iam_context + +from .models import ConsensusSettings + + +class ConsensusMergePermission(OpenPolicyAgentPermission): + rq_job_owner_id: int | None + task_id: int | None + + class Scopes(StrEnum): + CREATE = "create" + VIEW_STATUS = "view:status" + + @classmethod + def create_scope_check_status( + cls, request: ExtendedRequest, rq_job_owner_id: int, iam_context=None + ): + if not iam_context and request: + iam_context = get_iam_context(request, None) + return cls(**iam_context, scope=cls.Scopes.VIEW_STATUS, rq_job_owner_id=rq_job_owner_id) + + @classmethod + def create(cls, request, view, obj, iam_context): + Scopes = __class__.Scopes + + permissions = [] + if view.basename == "consensus_merges": + for scope in cls.get_scopes(request, view, obj): + if scope == Scopes.CREATE: + # Note: POST /api/consensus/merges is used to initiate report creation + # and to check the operation status + rq_id = request.query_params.get("rq_id") + task_id = request.data.get("task_id") + job_id = request.data.get("job_id") + + if not (task_id or job_id or rq_id): + raise PermissionDenied( + "Either task_id or job_id or rq_id must be specified" + ) + + if rq_id: + # There will be another check for this case during request processing + continue + + # merge is always at least at the task level, even for specific jobs + if task_id is not None or job_id is not None: + if job_id: + try: + job = Job.objects.select_related("segment").get(id=job_id) + except Job.DoesNotExist: + raise ValidationError("The specified job does not exist") + + task_id = job.get_task_id() + + # The request may have a different org or org unset + # Here we need to retrieve iam_context for this user, based on the task_id + try: + task = Task.objects.get(id=task_id) + except Task.DoesNotExist: + raise ValidationError("The specified task does not exist") + + iam_context = get_iam_context(request, task) + + permissions.append( + cls.create_base_perm( + request, + view, + scope, + iam_context, + obj, + task_id=task_id, + ) + ) + + else: + permissions.append(cls.create_base_perm(request, view, scope, iam_context, obj)) + + return permissions + + def __init__(self, **kwargs): + if "rq_job_owner_id" in kwargs: + self.rq_job_owner_id = int(kwargs.pop("rq_job_owner_id")) + + super().__init__(**kwargs) + self.url = settings.IAM_OPA_DATA_URL + "/consensus_merges/allow" + + @staticmethod + def get_scopes(request, view, obj): + Scopes = __class__.Scopes + return [ + { + "create": Scopes.CREATE, + }[view.action] + ] + + def get_resource(self): + data = None + + if self.scope == self.Scopes.CREATE: + task: Task | None = None + project: Project | None = None + + if self.scope == self.Scopes.CREATE and self.task_id: + try: + task = Task.objects.get(id=self.task_id) + except Task.DoesNotExist: + raise ValidationError("The specified task does not exist") + + if task and task.project: + project = task.project + organization_id = task.project.organization_id + else: + organization_id = task.organization_id + + data = { + "organization": {"id": organization_id}, + "task": ( + { + "owner": {"id": task.owner_id}, + "assignee": {"id": task.assignee_id}, + } + if task + else None + ), + "project": ( + { + "owner": {"id": project.owner_id}, + "assignee": {"id": project.assignee_id}, + } + if project + else None + ), + } + elif self.scope == self.Scopes.VIEW_STATUS: + data = {"owner": {"id": self.rq_job_owner_id}} + + return data + + +class ConsensusSettingPermission(OpenPolicyAgentPermission): + obj: ConsensusSettings | None + + class Scopes(StrEnum): + LIST = "list" + VIEW = "view" + UPDATE = "update" + + @classmethod + def create(cls, request, view, obj, iam_context): + Scopes = __class__.Scopes + + permissions = [] + if view.basename == "consensus_settings": + for scope in cls.get_scopes(request, view, obj): + if scope in [Scopes.VIEW, Scopes.UPDATE]: + obj = cast(ConsensusSettings, obj) + + if scope == Scopes.VIEW: + task_scope = TaskPermission.Scopes.VIEW + elif scope == Scopes.UPDATE: + task_scope = TaskPermission.Scopes.UPDATE_DESC + else: + assert False + + # Access rights are the same as in the owning task + # This component doesn't define its own rules in this case + permissions.append( + TaskPermission.create_base_perm( + request, + view, + iam_context=iam_context, + scope=task_scope, + obj=obj.task, + ) + ) + elif scope == cls.Scopes.LIST: + if task_id := request.query_params.get("task_id", None): + permissions.append( + TaskPermission.create_scope_view( + request, + int(task_id), + iam_context=iam_context, + ) + ) + + permissions.append(cls.create_scope_list(request, iam_context)) + else: + permissions.append(cls.create_base_perm(request, view, scope, iam_context, obj)) + + return permissions + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.url = settings.IAM_OPA_DATA_URL + "/consensus_settings/allow" + + @staticmethod + def get_scopes(request, view, obj): + Scopes = __class__.Scopes + return [ + { + "list": Scopes.LIST, + "retrieve": Scopes.VIEW, + "partial_update": Scopes.UPDATE, + }.get(view.action, None) + ] + + def get_resource(self): + data = None + + if self.obj: + task = self.obj.task + if task.project: + organization_id = task.project.organization_id + else: + organization_id = task.organization_id + + data = { + "id": self.obj.id, + "organization": {"id": organization_id}, + "task": ( + { + "owner": {"id": task.owner_id}, + "assignee": {"id": task.assignee_id}, + } + if task + else None + ), + "project": ( + { + "owner": {"id": task.project.owner_id}, + "assignee": {"id": task.project.assignee_id}, + } + if task.project + else None + ), + } + + return data diff --git a/cvat/apps/consensus/rules/consensus_merges.rego b/cvat/apps/consensus/rules/consensus_merges.rego new file mode 100644 index 000000000000..113ff7885595 --- /dev/null +++ b/cvat/apps/consensus/rules/consensus_merges.rego @@ -0,0 +1,145 @@ +package consensus_merges + +import rego.v1 + +import data.utils +import data.organizations +import data.quality_utils + +# input: { +# "scope": <"create"|"view"|"view:status"|"list"> or null, +# "auth": { +# "user": { +# "id": , +# "privilege": <"admin"|"user"|"worker"> or null +# }, +# "organization": { +# "id": , +# "owner": { +# "id": +# }, +# "user": { +# "role": <"owner"|"maintainer"|"supervisor"|"worker"> or null +# } +# } or null, +# }, +# "resource": { +# "owner": { "id": }, +# "organization": { "id": } or null, +# "task": { +# "id": , +# "owner": { "id": }, +# "assignee": { "id": }, +# "organization": { "id": } or null, +# } or null, +# "project": { +# "id": , +# "owner": { "id": }, +# "assignee": { "id": }, +# "organization": { "id": } or null, +# } or null, +# } +# } + +default allow := false + +allow if { + utils.is_admin +} + +allow if { + input.scope == utils.LIST + utils.is_sandbox +} + +allow if { + input.scope == utils.LIST + organizations.is_member +} + +allow if { + input.scope == utils.VIEW_STATUS + utils.is_resource_owner +} + +allow if { + input.scope in {utils.CREATE, utils.VIEW} + utils.is_sandbox + quality_utils.is_task_staff(input.resource.task, input.resource.project, input.auth) + utils.has_perm(utils.WORKER) +} + +allow if { + input.scope in {utils.CREATE, utils.VIEW} + input.auth.organization.id == input.resource.organization.id + utils.has_perm(utils.USER) + organizations.has_perm(organizations.MAINTAINER) +} + +allow if { + input.scope in {utils.CREATE, utils.VIEW} + quality_utils.is_task_staff(input.resource.task, input.resource.project, input.auth) + input.auth.organization.id == input.resource.organization.id + utils.has_perm(utils.WORKER) + organizations.has_perm(organizations.WORKER) +} + +filter := [] if { # Django Q object to filter list of entries + utils.is_admin + utils.is_sandbox +} else := qobject if { + utils.is_admin + utils.is_organization + org := input.auth.organization + qobject := [ + {"job__segment__task__organization": org.id}, + {"job__segment__task__project__organization": org.id}, "|", + {"task__organization": org.id}, "|", + {"task__project__organization": org.id}, "|", + ] +} else := qobject if { + utils.is_sandbox + user := input.auth.user + qobject := [ + {"job__segment__task__owner_id": user.id}, + {"job__segment__task__assignee_id": user.id}, "|", + {"job__segment__task__project__owner_id": user.id}, "|", + {"job__segment__task__project__assignee_id": user.id}, "|", + {"task__owner_id": user.id}, "|", + {"task__assignee_id": user.id}, "|", + {"task__project__owner_id": user.id}, "|", + {"task__project__assignee_id": user.id}, "|", + ] +} else := qobject if { + utils.is_organization + utils.has_perm(utils.USER) + organizations.has_perm(organizations.MAINTAINER) + org := input.auth.organization + qobject := [ + {"job__segment__task__organization": org.id}, + {"job__segment__task__project__organization": org.id}, "|", + {"task__organization": org.id}, "|", + {"task__project__organization": org.id}, "|", + ] +} else := qobject if { + organizations.has_perm(organizations.WORKER) + user := input.auth.user + org := input.auth.organization + qobject := [ + {"job__segment__task__organization": org.id}, + {"job__segment__task__project__organization": org.id}, "|", + {"task__organization": org.id}, "|", + {"task__project__organization": org.id}, "|", + + {"job__segment__task__owner_id": user.id}, + {"job__segment__task__assignee_id": user.id}, "|", + {"job__segment__task__project__owner_id": user.id}, "|", + {"job__segment__task__project__assignee_id": user.id}, "|", + {"task__owner_id": user.id}, "|", + {"task__assignee_id": user.id}, "|", + {"task__project__owner_id": user.id}, "|", + {"task__project__assignee_id": user.id}, "|", + + "&" + ] +} diff --git a/cvat/apps/consensus/rules/consensus_settings.rego b/cvat/apps/consensus/rules/consensus_settings.rego new file mode 100644 index 000000000000..fd67061c6e1a --- /dev/null +++ b/cvat/apps/consensus/rules/consensus_settings.rego @@ -0,0 +1,104 @@ +package consensus_settings + +import rego.v1 + +import data.utils +import data.organizations + +# input: { +# "scope": <"view"> or null, +# "auth": { +# "user": { +# "id": , +# "privilege": <"admin"|"business"|"user"|"worker"> or null +# }, +# "organization": { +# "id": , +# "owner": { +# "id": +# }, +# "user": { +# "role": <"owner"|"maintainer"|"supervisor"|"worker"> or null +# } +# } or null, +# }, +# "resource": { +# "id": , +# "owner": { "id": }, +# "organization": { "id": } or null, +# "task": { +# "id": , +# "owner": { "id": }, +# "assignee": { "id": }, +# "organization": { "id": } or null, +# } or null, +# "project": { +# "id": , +# "owner": { "id": }, +# "assignee": { "id": }, +# "organization": { "id": } or null, +# } or null, +# } +# } + +default allow := false + +allow if { + utils.is_admin +} + +allow if { + input.scope == utils.LIST + utils.is_sandbox +} + +allow if { + input.scope == utils.LIST + organizations.is_member +} + +filter := [] if { # Django Q object to filter list of entries + utils.is_admin + utils.is_sandbox +} else := qobject if { + utils.is_admin + utils.is_organization + org := input.auth.organization + qobject := [ + {"task__organization": org.id}, + {"task__project__organization": org.id}, "|", + ] +} else := qobject if { + utils.is_sandbox + user := input.auth.user + qobject := [ + {"task__owner_id": user.id}, + {"task__assignee_id": user.id}, "|", + {"task__project__owner_id": user.id}, "|", + {"task__project__assignee_id": user.id}, "|", + ] +} else := qobject if { + utils.is_organization + utils.has_perm(utils.USER) + organizations.has_perm(organizations.MAINTAINER) + org := input.auth.organization + qobject := [ + {"task__organization": org.id}, + {"task__project__organization": org.id}, "|", + ] +} else := qobject if { + organizations.has_perm(organizations.WORKER) + user := input.auth.user + org := input.auth.organization + qobject := [ + {"task__organization": org.id}, + {"task__project__organization": org.id}, "|", + + {"task__owner_id": user.id}, + {"task__assignee_id": user.id}, "|", + {"task__project__owner_id": user.id}, "|", + {"task__project__assignee_id": user.id}, "|", + + "&" + ] +} diff --git a/cvat/apps/consensus/serializers.py b/cvat/apps/consensus/serializers.py new file mode 100644 index 000000000000..0a37ac0df2b1 --- /dev/null +++ b/cvat/apps/consensus/serializers.py @@ -0,0 +1,51 @@ +# Copyright (C) CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +import textwrap + +from rest_framework import serializers + +from cvat.apps.consensus import models +from cvat.apps.engine import field_validation + + +class ConsensusMergeCreateSerializer(serializers.Serializer): + task_id = serializers.IntegerField(write_only=True, required=False) + job_id = serializers.IntegerField(write_only=True, required=False) + + def validate(self, attrs): + field_validation.require_one_of_fields(attrs, ["task_id", "job_id"]) + return super().validate(attrs) + + +class ConsensusSettingsSerializer(serializers.ModelSerializer): + class Meta: + model = models.ConsensusSettings + fields = ( + "id", + "task_id", + "iou_threshold", + "quorum", + ) + read_only_fields = ( + "id", + "task_id", + ) + + extra_kwargs = {} + + for field_name, help_text in { + "iou_threshold": "Pairwise annotation matching IoU threshold", + "quorum": """ + Minimum required share of sources having an annotation for it to be accepted + """, + }.items(): + extra_kwargs.setdefault(field_name, {}).setdefault( + "help_text", textwrap.dedent(help_text.lstrip("\n")) + ) + + for field_name in fields: + if field_name.endswith("_threshold") or field_name == "quorum": + extra_kwargs.setdefault(field_name, {}).setdefault("min_value", 0) + extra_kwargs.setdefault(field_name, {}).setdefault("max_value", 1) diff --git a/cvat/apps/consensus/signals.py b/cvat/apps/consensus/signals.py new file mode 100644 index 000000000000..9193adba6173 --- /dev/null +++ b/cvat/apps/consensus/signals.py @@ -0,0 +1,22 @@ +# Copyright (C) CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +from django.db.models.signals import post_save +from django.dispatch import receiver + +from cvat.apps.consensus.models import ConsensusSettings +from cvat.apps.engine.models import Task + + +@receiver( + post_save, + sender=Task, + dispatch_uid=__name__ + ".save_task-initialize_consensus_settings", +) +def __save_task__initialize_consensus_settings(instance: Task, created, **kwargs): + # Initializes default quality settings for the task + # this is done in a signal to decouple this component from the engine app + + if created and instance.consensus_replicas: + ConsensusSettings.objects.get_or_create(task=instance) diff --git a/cvat/apps/consensus/urls.py b/cvat/apps/consensus/urls.py new file mode 100644 index 000000000000..f140d7c0f5e1 --- /dev/null +++ b/cvat/apps/consensus/urls.py @@ -0,0 +1,17 @@ +# Copyright (C) CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +from django.urls import include, path +from rest_framework import routers + +from cvat.apps.consensus import views + +router = routers.DefaultRouter(trailing_slash=False) +router.register("merges", views.ConsensusMergesViewSet, basename="consensus_merges") +router.register("settings", views.ConsensusSettingsViewSet, basename="consensus_settings") + +urlpatterns = [ + # entry point for API + path("consensus/", include(router.urls)), +] diff --git a/cvat/apps/consensus/views.py b/cvat/apps/consensus/views.py new file mode 100644 index 000000000000..c1e804eddacb --- /dev/null +++ b/cvat/apps/consensus/views.py @@ -0,0 +1,207 @@ +# Copyright (C) CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +import textwrap + +from drf_spectacular.utils import ( + OpenApiParameter, + OpenApiResponse, + OpenApiTypes, + extend_schema, + extend_schema_view, +) +from rest_framework import mixins, status, viewsets +from rest_framework.exceptions import NotFound, ValidationError +from rest_framework.response import Response +from rq.job import JobStatus as RqJobStatus + +from cvat.apps.consensus import merging_manager as merging +from cvat.apps.consensus.models import ConsensusSettings +from cvat.apps.consensus.permissions import ConsensusMergePermission, ConsensusSettingPermission +from cvat.apps.consensus.serializers import ( + ConsensusMergeCreateSerializer, + ConsensusSettingsSerializer, +) +from cvat.apps.engine.mixins import PartialUpdateModelMixin +from cvat.apps.engine.models import Job, Task +from cvat.apps.engine.rq_job_handler import RQJobMetaField +from cvat.apps.engine.serializers import RqIdSerializer +from cvat.apps.engine.types import ExtendedRequest +from cvat.apps.engine.utils import process_failed_job + + +@extend_schema(tags=["consensus"]) +class ConsensusMergesViewSet(viewsets.GenericViewSet): + CREATE_MERGE_RQ_ID_PARAMETER = "rq_id" + + @extend_schema( + operation_id="consensus_create_merge", + summary="Create a consensus merge", + parameters=[ + OpenApiParameter( + CREATE_MERGE_RQ_ID_PARAMETER, + type=str, + description=textwrap.dedent( + """\ + The consensus merge request id. Can be specified to check operation status. + """ + ), + ) + ], + request=ConsensusMergeCreateSerializer(required=False), + responses={ + "201": None, + "202": OpenApiResponse( + RqIdSerializer, + description=textwrap.dedent( + """\ + A consensus merge request has been enqueued, the request id is returned. + The request status can be checked at this endpoint by passing the {} + as the query parameter. If the request id is specified, this response + means the consensus merge request is queued or is being processed. + """.format( + CREATE_MERGE_RQ_ID_PARAMETER + ) + ), + ), + "400": OpenApiResponse( + description="Invalid or failed request, check the response data for details" + ), + }, + ) + def create(self, request: ExtendedRequest, *args, **kwargs): + rq_id = request.query_params.get(self.CREATE_MERGE_RQ_ID_PARAMETER, None) + + if rq_id is None: + input_serializer = ConsensusMergeCreateSerializer(data=request.data) + input_serializer.is_valid(raise_exception=True) + + task_id = input_serializer.validated_data.get("task_id", None) + job_id = input_serializer.validated_data.get("job_id", None) + if task_id: + try: + instance = Task.objects.get(pk=task_id) + except Task.DoesNotExist as ex: + raise NotFound(f"Task {task_id} does not exist") from ex + elif job_id: + try: + instance = Job.objects.select_related("segment").get(pk=job_id) + except Job.DoesNotExist as ex: + raise NotFound(f"Jobs {job_id} do not exist") from ex + + try: + manager = merging.MergingManager() + rq_id = manager.schedule_merge(instance, request=request) + serializer = RqIdSerializer({"rq_id": rq_id}) + return Response(serializer.data, status=status.HTTP_202_ACCEPTED) + except merging.MergingNotAvailable as ex: + raise ValidationError(str(ex)) + else: + serializer = RqIdSerializer(data={"rq_id": rq_id}) + serializer.is_valid(raise_exception=True) + rq_id = serializer.validated_data["rq_id"] + + manager = merging.MergingManager() + rq_job = manager.get_job(rq_id) + if ( + not rq_job + or not ConsensusMergePermission.create_scope_check_status( + request, rq_job_owner_id=rq_job.meta[RQJobMetaField.USER]["id"] + ) + .check_access() + .allow + ): + # We should not provide job existence information to unauthorized users + raise NotFound("Unknown request id") + + rq_job_status = rq_job.get_status(refresh=False) + if rq_job_status == RqJobStatus.FAILED: + exc_info = process_failed_job(rq_job) + + exc_name_pattern = f"{merging.MergingNotAvailable.__name__}: " + if (exc_pos := exc_info.find(exc_name_pattern)) != -1: + return Response( + data=exc_info[exc_pos + len(exc_name_pattern) :].strip(), + status=status.HTTP_400_BAD_REQUEST, + ) + + return Response(data=str(exc_info), status=status.HTTP_500_INTERNAL_SERVER_ERROR) + elif rq_job_status in ( + RqJobStatus.QUEUED, + RqJobStatus.STARTED, + RqJobStatus.SCHEDULED, + RqJobStatus.DEFERRED, + ): + return Response(serializer.data, status=status.HTTP_202_ACCEPTED) + elif rq_job_status == RqJobStatus.FINISHED: + rq_job.delete() + return Response(status=status.HTTP_201_CREATED) + + raise AssertionError(f"Unexpected rq job '{rq_id}' status '{rq_job_status}'") + + +@extend_schema(tags=["consensus"]) +@extend_schema_view( + list=extend_schema( + summary="List consensus settings instances", + responses={ + "200": ConsensusSettingsSerializer(many=True), + }, + ), + retrieve=extend_schema( + summary="Get consensus settings instance details", + parameters=[ + OpenApiParameter( + "id", + type=OpenApiTypes.INT, + location="path", + description="An id of a consensus settings instance", + ) + ], + responses={ + "200": ConsensusSettingsSerializer, + }, + ), + partial_update=extend_schema( + summary="Update a consensus settings instance", + parameters=[ + OpenApiParameter( + "id", + type=OpenApiTypes.INT, + location="path", + description="An id of a consensus settings instance", + ) + ], + request=ConsensusSettingsSerializer(partial=True), + responses={ + "200": ConsensusSettingsSerializer, + }, + ), +) +class ConsensusSettingsViewSet( + viewsets.GenericViewSet, + mixins.ListModelMixin, + mixins.RetrieveModelMixin, + PartialUpdateModelMixin, +): + queryset = ConsensusSettings.objects + + iam_organization_field = "task__organization" + + search_fields = [] + filter_fields = ["id", "task_id"] + simple_filters = ["task_id"] + ordering_fields = ["id"] + ordering = "id" + + serializer_class = ConsensusSettingsSerializer + + def get_queryset(self): + queryset = super().get_queryset() + + if self.action == "list": + permissions = ConsensusSettingPermission.create_scope_list(self.request) + queryset = permissions.filter(queryset) + + return queryset diff --git a/cvat/apps/engine/utils.py b/cvat/apps/engine/utils.py index 27044bb37efe..4b29e863845d 100644 --- a/cvat/apps/engine/utils.py +++ b/cvat/apps/engine/utils.py @@ -151,7 +151,7 @@ def parse_exception_message(msg: str) -> str: pass return parsed_msg -def process_failed_job(rq_job: RQJob): +def process_failed_job(rq_job: RQJob) -> str: exc_info = str(rq_job.exc_info or '') rq_job.delete() diff --git a/cvat/apps/quality_control/models.py b/cvat/apps/quality_control/models.py index fb6930c5520e..a993879aeceb 100644 --- a/cvat/apps/quality_control/models.py +++ b/cvat/apps/quality_control/models.py @@ -269,4 +269,4 @@ def to_dict(self): @property def organization_id(self): - return getattr(self.task.organization, "id", None) + return self.task.organization_id diff --git a/cvat/apps/quality_control/quality_reports.py b/cvat/apps/quality_control/quality_reports.py index 8be628a5bedb..500d9c5b3b6d 100644 --- a/cvat/apps/quality_control/quality_reports.py +++ b/cvat/apps/quality_control/quality_reports.py @@ -770,7 +770,9 @@ def oks(a, b, sigma=0.1, bbox=None, scale=None, visibility_a=None, visibility_b= dists = np.linalg.norm(p1 - p2, axis=1) return np.sum( - visibility_a * visibility_b * np.exp(-(dists**2) / (2 * scale * (2 * sigma) ** 2)) + visibility_a + * visibility_b + * np.exp((visibility_a == visibility_b) * (-(dists**2) / (2 * scale * (2 * sigma) ** 2))) ) / np.sum(visibility_a | visibility_b, dtype=float) @@ -993,6 +995,7 @@ def __init__( compare_line_orientation: bool = False, line_torso_radius: float = 0.01, panoptic_comparison: bool = False, + allow_groups: bool = True, ): super().__init__(iou_threshold=iou_threshold) self.categories = categories @@ -1016,6 +1019,12 @@ def __init__( self.panoptic_comparison = panoptic_comparison "Compare only the visible parts of polygons and masks" + self.allow_groups = allow_groups + """ + When comparing grouped annotations, consider all the group elements with the same label + as the same annotation, if applicable. Affects polygons, masks, and points + """ + def instance_bbox( self, instance_anns: Sequence[dm.Annotation] ) -> tuple[float, float, float, float]: @@ -1149,12 +1158,18 @@ def _get_segmentations(item): img_h, img_w = item_a.media_as(dm.Image).size def _find_instances(annotations): - # Group instance annotations by label. - # Annotations with the same label and group will be merged, - # and considered a single object in comparison instances = [] instance_map = {} # ann id -> instance id - for ann_group in datumaro.util.annotation_util.find_instances(annotations): + + if self.allow_groups: + # Group instance annotations by label. + # Annotations with the same label and group will be merged, + # and considered a single object in comparison + groups = datumaro.util.annotation_util.find_instances(annotations) + else: + groups = [[a] for a in annotations] # ignore groups + + for ann_group in groups: ann_group = sorted(ann_group, key=lambda a: a.label) for _, label_group in itertools.groupby(ann_group, key=lambda a: a.label): label_group = list(label_group) @@ -1314,9 +1329,24 @@ def match_points(self, item_a: dm.DatasetItem, item_b: dm.DatasetItem): a_points = self._get_ann_type(dm.AnnotationType.points, item_a) b_points = self._get_ann_type(dm.AnnotationType.points, item_b) + if not a_points and not b_points: + results = [[], [], [], []] + + if self.return_distances: + results.append({}) + + return tuple(results) + instance_map = {} # points id -> (instance group, instance bbox) for source_anns in [item_a.annotations, item_b.annotations]: - source_instances = datumaro.util.annotation_util.find_instances(source_anns) + if self.allow_groups: + # Group instance annotations by label. + # Annotations with the same label and group will be merged, + # and considered a single object in comparison + source_instances = datumaro.util.annotation_util.find_instances(source_anns) + else: + source_instances = [[a] for a in source_anns] # ignore groups + for instance_group in source_instances: instance_bbox = self.instance_bbox(instance_group) @@ -1416,7 +1446,12 @@ def match_skeletons(self, item_a, item_b): a_skeletons = self._get_ann_type(dm.AnnotationType.skeleton, item_a) b_skeletons = self._get_ann_type(dm.AnnotationType.skeleton, item_b) if not a_skeletons and not b_skeletons: - return [], [], [], [] + results = [[], [], [], []] + + if self.return_distances: + results.append({}) + + return tuple(results) # Convert skeletons to point lists for comparison # This is required to compute correct per-instance distance diff --git a/cvat/apps/quality_control/rules/quality_utils.rego b/cvat/apps/quality_control/rules/quality_utils.rego index 3e8a7196fe4b..0c4e5d10cd07 100644 --- a/cvat/apps/quality_control/rules/quality_utils.rego +++ b/cvat/apps/quality_control/rules/quality_utils.rego @@ -3,6 +3,10 @@ package quality_utils import rego.v1 +is_job_assignee(job_data, auth_data) if { + job_data.assignee.id == auth_data.user.id +} + is_task_owner(task_data, auth_data) if { task_data.owner.id == auth_data.user.id } @@ -38,3 +42,11 @@ is_task_staff(task_data, project_data, auth_data) if { is_task_staff(task_data, project_data, auth_data) if { is_task_assignee(task_data, auth_data) } + +is_job_staff(job_data, task_data, project_data, auth_data) if { + is_task_staff(task_data, project_data, auth_data) +} + +is_job_staff(job_data, task_data, project_data, auth_data) if { + is_job_assignee(job_data, auth_data) +} diff --git a/cvat/apps/quality_control/serializers.py b/cvat/apps/quality_control/serializers.py index 684e25aed26f..0fa3068b2cc6 100644 --- a/cvat/apps/quality_control/serializers.py +++ b/cvat/apps/quality_control/serializers.py @@ -177,10 +177,7 @@ class Meta: "help_text", textwrap.dedent(help_text.lstrip("\n")) ) - def validate(self, attrs): - for k, v in attrs.items(): - if k.endswith("_threshold") or k in ["oks_sigma", "line_thickness"]: - if not 0 <= v <= 1: - raise serializers.ValidationError(f"{k} must be in the range [0; 1]") - - return super().validate(attrs) + for field_name in fields: + if field_name.endswith("_threshold") or field_name in ["oks_sigma", "line_thickness"]: + extra_kwargs.setdefault(field_name, {}).setdefault("min_value", 0) + extra_kwargs.setdefault(field_name, {}).setdefault("max_value", 1) diff --git a/cvat/schema.yml b/cvat/schema.yml index fe92ac2d6c76..48fd07776512 100644 --- a/cvat/schema.yml +++ b/cvat/schema.yml @@ -1049,6 +1049,176 @@ paths: responses: '204': description: The comment has been deleted + /api/consensus/merges: + post: + operationId: consensus_create_merge + summary: Create a consensus merge + parameters: + - in: query + name: rq_id + schema: + type: string + description: | + The consensus merge request id. Can be specified to check operation status. + tags: + - consensus + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/ConsensusMergeCreateRequest' + security: + - sessionAuth: [] + csrfAuth: [] + tokenAuth: [] + - signatureAuth: [] + - basicAuth: [] + responses: + '201': + description: No response body + '202': + content: + application/vnd.cvat+json: + schema: + $ref: '#/components/schemas/RqId' + description: | + A consensus merge request has been enqueued, the request id is returned. + The request status can be checked at this endpoint by passing the rq_id + as the query parameter. If the request id is specified, this response + means the consensus merge request is queued or is being processed. + '400': + description: Invalid or failed request, check the response data for details + /api/consensus/settings: + get: + operationId: consensus_list_settings + summary: List consensus settings instances + parameters: + - name: X-Organization + in: header + description: Organization unique slug + schema: + type: string + - name: filter + required: false + in: query + description: |2- + + JSON Logic filter. This filter can be used to perform complex filtering by grouping rules. + + For example, using such a filter you can get all resources created by you: + + - {"and":[{"==":[{"var":"owner"},""]}]} + + Details about the syntax used can be found at the link: https://jsonlogic.com/ + + Available filter_fields: ['id', 'task_id']. + schema: + type: string + - name: org + in: query + description: Organization unique slug + schema: + type: string + - name: org_id + in: query + description: Organization identifier + schema: + type: integer + - name: page + required: false + in: query + description: A page number within the paginated result set. + schema: + type: integer + - name: page_size + required: false + in: query + description: Number of results to return per page. + schema: + type: integer + - name: sort + required: false + in: query + description: 'Which field to use when ordering the results. Available ordering_fields: + [''id'']' + schema: + type: string + - name: task_id + in: query + description: A simple equality filter for the task_id field + schema: + type: integer + tags: + - consensus + security: + - sessionAuth: [] + csrfAuth: [] + tokenAuth: [] + - signatureAuth: [] + - basicAuth: [] + responses: + '200': + content: + application/vnd.cvat+json: + schema: + $ref: '#/components/schemas/PaginatedConsensusSettingsList' + description: '' + /api/consensus/settings/{id}: + get: + operationId: consensus_retrieve_settings + summary: Get consensus settings instance details + parameters: + - in: path + name: id + schema: + type: integer + description: An id of a consensus settings instance + required: true + tags: + - consensus + security: + - sessionAuth: [] + csrfAuth: [] + tokenAuth: [] + - signatureAuth: [] + - basicAuth: [] + responses: + '200': + content: + application/vnd.cvat+json: + schema: + $ref: '#/components/schemas/ConsensusSettings' + description: '' + patch: + operationId: consensus_partial_update_settings + summary: Update a consensus settings instance + parameters: + - in: path + name: id + schema: + type: integer + description: An id of a consensus settings instance + required: true + tags: + - consensus + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/PatchedConsensusSettingsRequest' + security: + - sessionAuth: [] + csrfAuth: [] + tokenAuth: [] + - signatureAuth: [] + - basicAuth: [] + responses: + '200': + content: + application/vnd.cvat+json: + schema: + $ref: '#/components/schemas/ConsensusSettings' + description: '' /api/events: get: operationId: events_list @@ -7452,6 +7622,37 @@ components: type: string format: uri readOnly: true + ConsensusMergeCreateRequest: + type: object + properties: + task_id: + type: integer + writeOnly: true + job_id: + type: integer + writeOnly: true + ConsensusSettings: + type: object + properties: + id: + type: integer + readOnly: true + task_id: + type: integer + readOnly: true + iou_threshold: + type: number + format: double + maximum: 1 + minimum: 0 + description: Pairwise annotation matching IoU threshold + quorum: + type: number + format: double + maximum: 1 + minimum: 0 + description: | + Minimum required share of sources having an annotation for it to be accepted CredentialsTypeEnum: enum: - KEY_SECRET_KEY_PAIR @@ -9126,6 +9327,29 @@ components: type: array items: $ref: '#/components/schemas/CommentRead' + PaginatedConsensusSettingsList: + type: object + required: + - count + - results + properties: + count: + type: integer + example: 123 + next: + type: string + nullable: true + format: uri + example: http://api.example.org/accounts/?page=4 + previous: + type: string + nullable: true + format: uri + example: http://api.example.org/accounts/?page=2 + results: + type: array + items: + $ref: '#/components/schemas/ConsensusSettings' PaginatedInvitationReadList: type: object required: @@ -9564,6 +9788,22 @@ components: message: type: string minLength: 1 + PatchedConsensusSettingsRequest: + type: object + properties: + iou_threshold: + type: number + format: double + maximum: 1 + minimum: 0 + description: Pairwise annotation matching IoU threshold + quorum: + type: number + format: double + maximum: 1 + minimum: 0 + description: | + Minimum required share of sources having an annotation for it to be accepted PatchedDataMetaWriteRequest: type: object properties: @@ -9767,6 +10007,8 @@ components: target_metric_threshold: type: number format: double + maximum: 1 + minimum: 0 description: | Defines the minimal quality requirements in terms of the selected target metric. max_validations_per_job: @@ -9780,10 +10022,14 @@ components: iou_threshold: type: number format: double + maximum: 1 + minimum: 0 description: Used for distinction between matched / unmatched shapes oks_sigma: type: number format: double + maximum: 1 + minimum: 0 description: | Like IoU threshold, but for points. The percent of the bbox side, used as the radius of the circle around the GT point, @@ -9811,6 +10057,8 @@ components: line_thickness: type: number format: double + maximum: 1 + minimum: 0 description: | Thickness of polylines, relatively to the (image area) ^ 0.5. The distance to the boundary around the GT line, @@ -9818,6 +10066,8 @@ components: low_overlap_threshold: type: number format: double + maximum: 1 + minimum: 0 description: | Used for distinction between strong / weak (low_overlap) matches compare_line_orientation: @@ -9826,6 +10076,8 @@ components: line_orientation_threshold: type: number format: double + maximum: 1 + minimum: 0 description: | The minimal gain in the GT IoU between the given and reversed line directions to consider the line inverted. @@ -9836,6 +10088,8 @@ components: group_match_threshold: type: number format: double + maximum: 1 + minimum: 0 description: | Minimal IoU for groups to be considered matching. Only used when the 'compare_groups' parameter is true @@ -9846,6 +10100,8 @@ components: object_visibility_threshold: type: number format: double + maximum: 1 + minimum: 0 description: | Minimal visible area percent of the spatial annotations (polygons, masks) for reporting covered annotations. @@ -10274,6 +10530,8 @@ components: target_metric_threshold: type: number format: double + maximum: 1 + minimum: 0 description: | Defines the minimal quality requirements in terms of the selected target metric. max_validations_per_job: @@ -10287,10 +10545,14 @@ components: iou_threshold: type: number format: double + maximum: 1 + minimum: 0 description: Used for distinction between matched / unmatched shapes oks_sigma: type: number format: double + maximum: 1 + minimum: 0 description: | Like IoU threshold, but for points. The percent of the bbox side, used as the radius of the circle around the GT point, @@ -10318,6 +10580,8 @@ components: line_thickness: type: number format: double + maximum: 1 + minimum: 0 description: | Thickness of polylines, relatively to the (image area) ^ 0.5. The distance to the boundary around the GT line, @@ -10325,6 +10589,8 @@ components: low_overlap_threshold: type: number format: double + maximum: 1 + minimum: 0 description: | Used for distinction between strong / weak (low_overlap) matches compare_line_orientation: @@ -10333,6 +10599,8 @@ components: line_orientation_threshold: type: number format: double + maximum: 1 + minimum: 0 description: | The minimal gain in the GT IoU between the given and reversed line directions to consider the line inverted. @@ -10343,6 +10611,8 @@ components: group_match_threshold: type: number format: double + maximum: 1 + minimum: 0 description: | Minimal IoU for groups to be considered matching. Only used when the 'compare_groups' parameter is true @@ -10353,6 +10623,8 @@ components: object_visibility_threshold: type: number format: double + maximum: 1 + minimum: 0 description: | Minimal visible area percent of the spatial annotations (polygons, masks) for reporting covered annotations. diff --git a/cvat/settings/base.py b/cvat/settings/base.py index cc4b5d9b01b5..96125b20ae18 100644 --- a/cvat/settings/base.py +++ b/cvat/settings/base.py @@ -120,6 +120,7 @@ def generate_secret_key(): 'cvat.apps.quality_control', 'cvat.apps.analytics_report', 'cvat.apps.redis_handler', + 'cvat.apps.consensus', ] SITE_ID = 1 @@ -280,6 +281,7 @@ class CVAT_QUEUES(Enum): ANALYTICS_REPORTS = 'analytics_reports' CLEANING = 'cleaning' CHUNKS = 'chunks' + CONSENSUS = 'consensus' redis_inmem_host = os.getenv('CVAT_REDIS_INMEM_HOST', 'localhost') redis_inmem_port = os.getenv('CVAT_REDIS_INMEM_PORT', 6379) @@ -329,6 +331,10 @@ class CVAT_QUEUES(Enum): **REDIS_INMEM_SETTINGS, 'DEFAULT_TIMEOUT': '5m', }, + CVAT_QUEUES.CONSENSUS.value: { + **REDIS_INMEM_SETTINGS, + 'DEFAULT_TIMEOUT': '1h', + }, } NUCLIO = { diff --git a/cvat/urls.py b/cvat/urls.py index ca62b7cb03a3..770a5d0f3d43 100644 --- a/cvat/urls.py +++ b/cvat/urls.py @@ -51,3 +51,6 @@ if apps.is_installed("cvat.apps.analytics_report"): urlpatterns.append(path("api/", include("cvat.apps.analytics_report.urls"))) + +if apps.is_installed("cvat.apps.consensus"): + urlpatterns.append(path("api/", include("cvat.apps.consensus.urls"))) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index a7a13d931f78..05757bbc2da8 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -78,6 +78,18 @@ services: ports: - '9095:9095' + cvat_worker_consensus: + environment: + # For debugging, make sure to set 1 process + # Due to the supervisord specifics, the extra processes will fail and + # after few attempts supervisord will give up restarting, leaving only 1 process + # NUMPROCS: 1 + CVAT_DEBUG_ENABLED: '${CVAT_DEBUG_ENABLED:-no}' + CVAT_DEBUG_PORT: '9096' + COVERAGE_PROCESS_START: + ports: + - '9096:9096' + cvat_worker_annotation: environment: # For debugging, make sure to set 1 process diff --git a/docker-compose.external_db.yml b/docker-compose.external_db.yml index e99f3ee8518a..24d94e2f256b 100644 --- a/docker-compose.external_db.yml +++ b/docker-compose.external_db.yml @@ -28,6 +28,7 @@ services: cvat_worker_quality_reports: *backend-settings cvat_worker_webhooks: *backend-settings cvat_worker_chunks: *backend-settings + cvat_worker_consensus: *backend-settings secrets: postgres_password: diff --git a/docker-compose.yml b/docker-compose.yml index 1d83ad44754a..b018134666cc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -242,6 +242,22 @@ services: networks: - cvat + cvat_worker_consensus: + container_name: cvat_worker_consensus + image: cvat/server:${CVAT_VERSION:-dev} + restart: always + depends_on: *backend-deps + environment: + <<: *backend-env + NUMPROCS: 1 + command: run worker.consensus + volumes: + - cvat_data:/home/django/data + - cvat_keys:/home/django/keys + - cvat_logs:/home/django/logs + networks: + - cvat + cvat_ui: container_name: cvat_ui image: cvat/ui:${CVAT_VERSION:-dev} diff --git a/helm-chart/templates/cvat_backend/worker_consensus/deployment.yml b/helm-chart/templates/cvat_backend/worker_consensus/deployment.yml new file mode 100644 index 000000000000..c96b5aa2e96a --- /dev/null +++ b/helm-chart/templates/cvat_backend/worker_consensus/deployment.yml @@ -0,0 +1,85 @@ +{{- $localValues := .Values.cvat.backend.worker.consensus -}} + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-backend-worker-consensus + namespace: {{ .Release.Namespace }} + labels: + app: cvat-app + tier: backend + component: worker-consensus + {{- include "cvat.labels" . | nindent 4 }} + {{- with merge $localValues.labels .Values.cvat.backend.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with merge $localValues.annotations .Values.cvat.backend.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: {{ $localValues.replicas }} + strategy: + type: Recreate + selector: + matchLabels: + {{- include "cvat.labels" . | nindent 6 }} + {{- with merge $localValues.labels .Values.cvat.backend.labels }} + {{- toYaml . | nindent 6 }} + {{- end }} + app: cvat-app + tier: backend + component: worker-consensus + template: + metadata: + labels: + app: cvat-app + tier: backend + component: worker-consensus + {{- include "cvat.labels" . | nindent 8 }} + {{- with merge $localValues.labels .Values.cvat.backend.labels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with merge $localValues.annotations .Values.cvat.backend.annotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "cvat.backend.serviceAccountName" . }} + containers: + - name: cvat-backend + image: {{ .Values.cvat.backend.image }}:{{ .Values.cvat.backend.tag }} + imagePullPolicy: {{ .Values.cvat.backend.imagePullPolicy }} + {{- with merge $localValues.resources .Values.cvat.backend.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + args: ["run", "worker.consensus"] + env: + {{ include "cvat.sharedBackendEnv" . | indent 10 }} + {{- with concat .Values.cvat.backend.additionalEnv $localValues.additionalEnv }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- $probeArgs := list "consensus" -}} + {{- $probeConfig := dict "args" $probeArgs "livenessProbe" $.Values.cvat.backend.worker.livenessProbe -}} + {{ include "cvat.backend.worker.livenessProbe" $probeConfig | indent 10 }} + {{- with concat .Values.cvat.backend.additionalVolumeMounts $localValues.additionalVolumeMounts }} + volumeMounts: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with merge $localValues.affinity .Values.cvat.backend.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with concat .Values.cvat.backend.tolerations $localValues.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with concat .Values.cvat.backend.additionalVolumes $localValues.additionalVolumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/helm-chart/values.yaml b/helm-chart/values.yaml index 1ea296d3dd78..f05f03a1137a 100644 --- a/helm-chart/values.yaml +++ b/helm-chart/values.yaml @@ -127,6 +127,16 @@ cvat: additionalEnv: [] additionalVolumes: [] additionalVolumeMounts: [] + consensus: + replicas: 1 + labels: {} + annotations: {} + resources: {} + affinity: {} + tolerations: [] + additionalEnv: [] + additionalVolumes: [] + additionalVolumeMounts: [] utils: replicas: 1 labels: {} diff --git a/supervisord/worker.consensus.conf b/supervisord/worker.consensus.conf new file mode 100644 index 000000000000..1072226b7786 --- /dev/null +++ b/supervisord/worker.consensus.conf @@ -0,0 +1,27 @@ +[unix_http_server] +file = /tmp/supervisord/supervisor.sock + +[supervisorctl] +serverurl = unix:///tmp/supervisord/supervisor.sock + + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisord] +nodaemon=true +logfile=%(ENV_HOME)s/logs/supervisord.log ; supervisord log file +logfile_maxbytes=50MB ; maximum size of logfile before rotation +logfile_backups=10 ; number of backed up logfiles +loglevel=debug ; info, debug, warn, trace +pidfile=/tmp/supervisord/supervisord.pid ; pidfile location +childlogdir=%(ENV_HOME)s/logs/ ; where child log files will live + +[program:rqworker-consensus] +command=%(ENV_HOME)s/wait_for_deps.sh + python3 %(ENV_HOME)s/manage.py rqworker -v 3 consensus + --worker-class cvat.rqworker.DefaultWorker +environment=VECTOR_EVENT_HANDLER="SynchronousLogstashHandler",CVAT_POSTGRES_APPLICATION_NAME="cvat:worker:consensus" +numprocs=%(ENV_NUMPROCS)s +process_name=%(program_name)s-%(process_num)d +autorestart=true diff --git a/tests/python/rest_api/test_check_objects_integrity.py b/tests/python/rest_api/test_check_objects_integrity.py index f11ce4ff598b..39bb00c5bb3d 100644 --- a/tests/python/rest_api/test_check_objects_integrity.py +++ b/tests/python/rest_api/test_check_objects_integrity.py @@ -18,7 +18,12 @@ class TestGetResources: def test_check_objects_integrity(self, path: Path): with open(path) as f: endpoint = path.stem - if endpoint in ["quality_settings", "quality_reports", "quality_conflicts"]: + if endpoint in [ + "quality_settings", + "quality_reports", + "quality_conflicts", + "consensus_settings", + ]: endpoint = "/".join(endpoint.split("_")) if endpoint == "annotations": diff --git a/tests/python/rest_api/test_consensus.py b/tests/python/rest_api/test_consensus.py new file mode 100644 index 000000000000..2294990ee601 --- /dev/null +++ b/tests/python/rest_api/test_consensus.py @@ -0,0 +1,725 @@ +# Copyright (C) CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +import json +from copy import deepcopy +from functools import partial +from http import HTTPStatus +from typing import Any, Dict, Optional, Tuple + +import pytest +import urllib3 +from cvat_sdk.api_client import exceptions, models +from cvat_sdk.api_client.api_client import ApiClient, Endpoint +from cvat_sdk.core.helpers import get_paginated_collection +from deepdiff import DeepDiff + +from shared.utils.config import make_api_client + +from .utils import CollectionSimpleFilterTestBase, compare_annotations + + +class _PermissionTestBase: + def merge( + self, + *, + task_id: Optional[int] = None, + job_id: Optional[int] = None, + user: str, + raise_on_error: bool = True, + wait_result: bool = True, + ) -> urllib3.HTTPResponse: + assert task_id is not None or job_id is not None + + kwargs = {} + if task_id is not None: + kwargs["task_id"] = task_id + if job_id is not None: + kwargs["job_id"] = job_id + + with make_api_client(user) as api_client: + (_, response) = api_client.consensus_api.create_merge( + consensus_merge_create_request=models.ConsensusMergeCreateRequest(**kwargs), + _parse_response=False, + _check_status=raise_on_error, + ) + + if not raise_on_error and response.status != HTTPStatus.ACCEPTED: + return response + assert response.status == HTTPStatus.ACCEPTED + + rq_id = json.loads(response.data)["rq_id"] + + while wait_result: + (_, response) = api_client.consensus_api.create_merge( + rq_id=rq_id, _parse_response=False + ) + assert response.status in [HTTPStatus.CREATED, HTTPStatus.ACCEPTED] + + if response.status == HTTPStatus.CREATED: + break + + return response + + def request_merge( + self, + *, + task_id: Optional[int] = None, + job_id: Optional[int] = None, + user: str, + ) -> str: + response = self.merge(user=user, task_id=task_id, job_id=job_id, wait_result=False) + return json.loads(response.data)["rq_id"] + + @pytest.fixture + def find_sandbox_task(self, tasks, jobs, users, is_task_staff): + def _find( + is_staff: bool, *, has_consensus_jobs: Optional[bool] = None + ) -> tuple[dict[str, Any], dict[str, Any]]: + task = next( + t + for t in tasks + if t["organization"] is None + and not users[t["owner"]["id"]]["is_superuser"] + and ( + has_consensus_jobs is None + or has_consensus_jobs + == any( + j + for j in jobs + if j["task_id"] == t["id"] and j["type"] == "consensus_replica" + ) + ) + ) + + if is_staff: + user = task["owner"] + else: + user = next(u for u in users if not is_task_staff(u["id"], task["id"])) + + return task, user + + return _find + + @pytest.fixture + def find_sandbox_task_with_consensus(self, find_sandbox_task): + return partial(find_sandbox_task, has_consensus_jobs=True) + + @pytest.fixture + def find_org_task( + self, restore_db_per_function, tasks, jobs, users, is_org_member, is_task_staff, admin_user + ): + def _find( + is_staff: bool, user_org_role: str, *, has_consensus_jobs: Optional[bool] = None + ) -> tuple[dict[str, Any], dict[str, Any]]: + for user in users: + if user["is_superuser"]: + continue + + task = next( + ( + t + for t in tasks + if t["organization"] is not None + and is_task_staff(user["id"], t["id"]) == is_staff + and is_org_member(user["id"], t["organization"], role=user_org_role) + and ( + has_consensus_jobs is None + or has_consensus_jobs + == any( + j + for j in jobs + if j["task_id"] == t["id"] and j["type"] == "consensus_replica" + ) + ) + ), + None, + ) + if task is not None: + break + + if not task: + task = next( + t + for t in tasks + if t["organization"] is not None + if has_consensus_jobs is None or has_consensus_jobs == t["consensus_enabled"] + ) + user = next( + u + for u in users + if is_org_member(u["id"], task["organization"], role=user_org_role) + ) + + if is_staff: + with make_api_client(admin_user) as api_client: + api_client.tasks_api.partial_update( + task["id"], + patched_task_write_request=models.PatchedTaskWriteRequest( + assignee_id=user["id"] + ), + ) + + return task, user + + return _find + + @pytest.fixture + def find_org_task_with_consensus(self, find_org_task): + return partial(find_org_task, has_consensus_jobs=True) + + _default_sandbox_cases = ("is_staff, allow", [(True, True), (False, False)]) + + _default_org_cases = ( + "org_role, is_staff, allow", + [ + ("owner", True, True), + ("owner", False, True), + ("maintainer", True, True), + ("maintainer", False, True), + ("supervisor", True, True), + ("supervisor", False, False), + ("worker", True, True), + ("worker", False, False), + ], + ) + + _default_org_roles = ("owner", "maintainer", "supervisor", "worker") + + +@pytest.mark.usefixtures("restore_db_per_function") +@pytest.mark.usefixtures("restore_redis_inmem_per_function") +class TestPostConsensusMerge(_PermissionTestBase): + def test_can_merge_task_with_consensus_jobs(self, admin_user, tasks): + task_id = next(t["id"] for t in tasks if t["consensus_enabled"]) + + assert self.merge(user=admin_user, task_id=task_id).status == HTTPStatus.CREATED + + def test_can_merge_consensus_job(self, admin_user, jobs): + job_id = next( + j["id"] for j in jobs if j["type"] == "annotation" and j["consensus_replicas"] > 0 + ) + + assert self.merge(user=admin_user, job_id=job_id).status == HTTPStatus.CREATED + + def test_cannot_merge_task_without_consensus_jobs(self, admin_user, tasks): + task_id = next(t["id"] for t in tasks if not t["consensus_enabled"]) + + with pytest.raises(exceptions.ApiException) as capture: + self.merge(user=admin_user, task_id=task_id) + + assert "Consensus is not enabled in this task" in capture.value.body + + def test_cannot_merge_task_without_mergeable_parent_jobs(self, admin_user, tasks, jobs): + task_id = next(t["id"] for t in tasks if t["consensus_enabled"]) + + for j in jobs: + if (j["stage"] != "annotation" or j["state"] != "new") and ( + j["task_id"] == task_id and j["type"] in ("annotation", "consensus_replica") + ): + with make_api_client(admin_user) as api_client: + api_client.jobs_api.partial_update( + j["id"], + patched_job_write_request=models.PatchedJobWriteRequest( + state="new", stage="annotation" + ), + ) + + with pytest.raises(exceptions.ApiException) as capture: + self.merge(user=admin_user, task_id=task_id) + + assert "No annotation jobs in the annotation stage" in capture.value.body + + def test_cannot_merge_replica_job(self, admin_user, tasks, jobs): + job_id = next( + j["id"] + for j in jobs + if j["type"] == "consensus_replica" + if tasks.map[j["task_id"]]["consensus_enabled"] + ) + + with pytest.raises(exceptions.ApiException) as capture: + self.merge(user=admin_user, job_id=job_id) + + assert "No annotated consensus jobs found for parent job" in capture.value.body + + def _test_merge_200( + self, user: str, *, task_id: Optional[int] = None, job_id: Optional[int] = None + ): + return self.merge(user=user, task_id=task_id, job_id=job_id) + + def _test_merge_403( + self, user: str, *, task_id: Optional[int] = None, job_id: Optional[int] = None + ): + response = self.merge(user=user, task_id=task_id, job_id=job_id, raise_on_error=False) + assert response.status == HTTPStatus.FORBIDDEN + return response + + @pytest.mark.parametrize(*_PermissionTestBase._default_sandbox_cases) + def test_user_merge_in_sandbox_task(self, is_staff, allow, find_sandbox_task_with_consensus): + task, user = find_sandbox_task_with_consensus(is_staff) + + if allow: + self._test_merge_200(user["username"], task_id=task["id"]) + else: + self._test_merge_403(user["username"], task_id=task["id"]) + + @pytest.mark.parametrize(*_PermissionTestBase._default_org_cases) + def test_user_merge_in_org_task( + self, + find_org_task_with_consensus, + org_role, + is_staff, + allow, + ): + task, user = find_org_task_with_consensus(is_staff, org_role) + + if allow: + self._test_merge_200(user["username"], task_id=task["id"]) + else: + self._test_merge_403(user["username"], task_id=task["id"]) + + # only rq job owner or admin now has the right to check status of report creation + def _test_check_merge_status_by_non_rq_job_owner( + self, + rq_id: str, + *, + staff_user: str, + other_user: str, + ): + with make_api_client(other_user) as api_client: + (_, response) = api_client.consensus_api.create_merge( + rq_id=rq_id, _parse_response=False, _check_status=False + ) + assert response.status == HTTPStatus.NOT_FOUND + assert json.loads(response.data)["detail"] == "Unknown request id" + + with make_api_client(staff_user) as api_client: + (_, response) = api_client.consensus_api.create_merge( + rq_id=rq_id, _parse_response=False, _check_status=False + ) + assert response.status in {HTTPStatus.ACCEPTED, HTTPStatus.CREATED} + + def test_non_rq_job_owner_cannot_check_status_of_merge_in_sandbox( + self, + find_sandbox_task_with_consensus, + users, + ): + task, task_staff = find_sandbox_task_with_consensus(is_staff=True) + + other_user = next( + u + for u in users + if ( + u["id"] != task_staff["id"] + and not u["is_superuser"] + and u["id"] != task["owner"]["id"] + ) + ) + + rq_id = self.request_merge(task_id=task["id"], user=task_staff["username"]) + self._test_check_merge_status_by_non_rq_job_owner( + rq_id, staff_user=task_staff["username"], other_user=other_user["username"] + ) + + @pytest.mark.parametrize("role", _PermissionTestBase._default_org_roles) + def test_non_rq_job_owner_cannot_check_status_of_merge_in_org( + self, + find_org_task_with_consensus, + find_users, + role: str, + ): + task, task_staff = find_org_task_with_consensus(is_staff=True, user_org_role="supervisor") + + other_user = next( + u + for u in find_users(role=role, org=task["organization"]) + if ( + u["id"] != task_staff["id"] + and not u["is_superuser"] + and u["id"] != task["owner"]["id"] + ) + ) + rq_id = self.request_merge(task_id=task["id"], user=task_staff["username"]) + self._test_check_merge_status_by_non_rq_job_owner( + rq_id, staff_user=task_staff["username"], other_user=other_user["username"] + ) + + @pytest.mark.parametrize("is_sandbox", (True, False)) + def test_admin_can_check_status_of_merge( + self, + find_org_task_with_consensus, + find_sandbox_task_with_consensus, + users, + is_sandbox: bool, + ): + if is_sandbox: + task, task_staff = find_sandbox_task_with_consensus(is_staff=True) + else: + task, task_staff = find_org_task_with_consensus(is_staff=True, user_org_role="owner") + + admin = next( + u + for u in users + if ( + u["is_superuser"] + and u["id"] != task_staff["id"] + and u["id"] != task["owner"]["id"] + and u["id"] != (task["assignee"] or {}).get("id") + ) + ) + + rq_id = self.request_merge(task_id=task["id"], user=task_staff["username"]) + + with make_api_client(admin["username"]) as api_client: + (_, response) = api_client.consensus_api.create_merge( + rq_id=rq_id, _parse_response=False + ) + assert response.status in {HTTPStatus.ACCEPTED, HTTPStatus.CREATED} + + +class TestSimpleConsensusSettingsFilters(CollectionSimpleFilterTestBase): + @pytest.fixture(autouse=True) + def setup(self, admin_user, consensus_settings): + self.user = admin_user + self.samples = consensus_settings + + def _get_endpoint(self, api_client: ApiClient) -> Endpoint: + return api_client.consensus_api.list_settings_endpoint + + @pytest.mark.parametrize("field", ("task_id",)) + def test_can_use_simple_filter_for_object_list(self, field): + return super()._test_can_use_simple_filter_for_object_list(field) + + +class TestListSettings(_PermissionTestBase): + def _test_list_settings_200( + self, user: str, task_id: int, *, expected_data: Optional[Dict[str, Any]] = None, **kwargs + ): + with make_api_client(user) as api_client: + actual = get_paginated_collection( + api_client.consensus_api.list_settings_endpoint, + task_id=task_id, + **kwargs, + return_json=True, + ) + + if expected_data is not None: + assert DeepDiff(expected_data, actual, ignore_order=True) == {} + + def _test_list_settings_403(self, user: str, task_id: int, **kwargs): + with make_api_client(user) as api_client: + (_, response) = api_client.consensus_api.list_settings( + task_id=task_id, **kwargs, _parse_response=False, _check_status=False + ) + assert response.status == HTTPStatus.FORBIDDEN + + return response + + @pytest.mark.parametrize(*_PermissionTestBase._default_sandbox_cases) + def test_user_list_settings_in_sandbox_task( + self, is_staff, allow, find_sandbox_task_with_consensus, consensus_settings + ): + task, user = find_sandbox_task_with_consensus(is_staff) + settings = next(s for s in consensus_settings if s["task_id"] == task["id"]) + + if allow: + self._test_list_settings_200(user["username"], task["id"], expected_data=[settings]) + else: + self._test_list_settings_403(user["username"], task["id"]) + + @pytest.mark.parametrize(*_PermissionTestBase._default_org_cases) + def test_user_list_settings_in_org_task( + self, + consensus_settings, + find_org_task_with_consensus, + org_role: str, + is_staff, + allow: bool, + ): + task, user = find_org_task_with_consensus(is_staff, org_role) + settings = next(s for s in consensus_settings if s["task_id"] == task["id"]) + org_id = task["organization"] + + if allow: + self._test_list_settings_200( + user["username"], task["id"], expected_data=[settings], org_id=org_id + ) + else: + self._test_list_settings_403(user["username"], task["id"], org_id=org_id) + + +class TestGetSettings(_PermissionTestBase): + def _test_get_settings_200( + self, user: str, obj_id: int, *, expected_data: Optional[Dict[str, Any]] = None, **kwargs + ): + with make_api_client(user) as api_client: + (_, response) = api_client.consensus_api.retrieve_settings(obj_id, **kwargs) + assert response.status == HTTPStatus.OK + + if expected_data is not None: + assert DeepDiff(expected_data, json.loads(response.data), ignore_order=True) == {} + + return response + + def _test_get_settings_403(self, user: str, obj_id: int, **kwargs): + with make_api_client(user) as api_client: + (_, response) = api_client.consensus_api.retrieve_settings( + obj_id, **kwargs, _parse_response=False, _check_status=False + ) + assert response.status == HTTPStatus.FORBIDDEN + + return response + + def test_can_get_settings(self, admin_user, consensus_settings): + settings = next(iter(consensus_settings)) + self._test_get_settings_200(admin_user, settings["id"], expected_data=settings) + + @pytest.mark.parametrize(*_PermissionTestBase._default_sandbox_cases) + def test_user_get_settings_in_sandbox_task( + self, is_staff, allow, find_sandbox_task_with_consensus, consensus_settings + ): + task, user = find_sandbox_task_with_consensus(is_staff) + settings = next(s for s in consensus_settings if s["task_id"] == task["id"]) + + if allow: + self._test_get_settings_200(user["username"], settings["id"], expected_data=settings) + else: + self._test_get_settings_403(user["username"], settings["id"]) + + @pytest.mark.parametrize(*_PermissionTestBase._default_org_cases) + def test_user_get_settings_in_org_task( + self, + consensus_settings, + find_org_task_with_consensus, + org_role: str, + is_staff, + allow: bool, + ): + task, user = find_org_task_with_consensus(is_staff, org_role) + settings = next(s for s in consensus_settings if s["task_id"] == task["id"]) + + if allow: + self._test_get_settings_200(user["username"], settings["id"], expected_data=settings) + else: + self._test_get_settings_403(user["username"], settings["id"]) + + +@pytest.mark.usefixtures("restore_db_per_function") +class TestPatchSettings(_PermissionTestBase): + def _test_patch_settings_200( + self, + user: str, + obj_id: int, + data: Dict[str, Any], + *, + expected_data: Optional[Dict[str, Any]] = None, + **kwargs, + ): + with make_api_client(user) as api_client: + (_, response) = api_client.consensus_api.partial_update_settings( + obj_id, patched_consensus_settings_request=data, **kwargs + ) + assert response.status == HTTPStatus.OK + + if expected_data is not None: + assert DeepDiff(expected_data, json.loads(response.data), ignore_order=True) == {} + + return response + + def _test_patch_settings_403(self, user: str, obj_id: int, data: Dict[str, Any], **kwargs): + with make_api_client(user) as api_client: + (_, response) = api_client.consensus_api.partial_update_settings( + obj_id, + patched_consensus_settings_request=data, + **kwargs, + _parse_response=False, + _check_status=False, + ) + assert response.status == HTTPStatus.FORBIDDEN + + return response + + def _get_request_data(self, data: Dict[str, Any]) -> Tuple[Dict[str, Any], Dict[str, Any]]: + patched_data = deepcopy(data) + + for field, value in data.items(): + if isinstance(value, bool): + patched_data[field] = not value + elif isinstance(value, float): + patched_data[field] = 1 - value + + expected_data = deepcopy(patched_data) + + return patched_data, expected_data + + def test_can_patch_settings(self, admin_user, consensus_settings): + settings = next(iter(consensus_settings)) + data, expected_data = self._get_request_data(settings) + self._test_patch_settings_200(admin_user, settings["id"], data, expected_data=expected_data) + + @pytest.mark.parametrize(*_PermissionTestBase._default_sandbox_cases) + def test_user_patch_settings_in_sandbox_task( + self, consensus_settings, find_sandbox_task_with_consensus, is_staff: bool, allow: bool + ): + task, user = find_sandbox_task_with_consensus(is_staff) + settings = next(s for s in consensus_settings if s["task_id"] == task["id"]) + request_data, expected_data = self._get_request_data(settings) + + if allow: + self._test_patch_settings_200( + user["username"], settings["id"], request_data, expected_data=expected_data + ) + else: + self._test_patch_settings_403(user["username"], settings["id"], request_data) + + @pytest.mark.parametrize( + "org_role, is_staff, allow", + [ + ("owner", True, True), + ("owner", False, True), + ("maintainer", True, True), + ("maintainer", False, True), + ("supervisor", True, True), + ("supervisor", False, False), + ("worker", True, True), + ("worker", False, False), + ], + ) + def test_user_patch_settings_in_org_task( + self, + consensus_settings, + find_org_task_with_consensus, + org_role: str, + is_staff: bool, + allow: bool, + ): + task, user = find_org_task_with_consensus(is_staff, org_role) + settings = next(s for s in consensus_settings if s["task_id"] == task["id"]) + request_data, expected_data = self._get_request_data(settings) + + if allow: + self._test_patch_settings_200( + user["username"], settings["id"], request_data, expected_data=expected_data + ) + else: + self._test_patch_settings_403(user["username"], settings["id"], request_data) + + +@pytest.mark.usefixtures("restore_db_per_function") +@pytest.mark.usefixtures("restore_redis_inmem_per_function") +class TestMerging(_PermissionTestBase): + @pytest.mark.parametrize("task_id", [31]) + def test_quorum_is_applied(self, admin_user, jobs, labels, consensus_settings, task_id: int): + task_labels = [l for l in labels if l.get("task_id") == task_id] + settings = next(s for s in consensus_settings if s["task_id"] == task_id) + + task_jobs = [j for j in jobs if j["task_id"] == task_id] + parent_job = next( + j for j in task_jobs if j["type"] == "annotation" if j["consensus_replicas"] > 0 + ) + replicas = [ + j + for j in task_jobs + if j["type"] == "consensus_replica" + if j["parent_job_id"] == parent_job["id"] + ] + assert len(replicas) == 2 + + with make_api_client(admin_user) as api_client: + api_client.tasks_api.destroy_annotations(task_id) + + for replica in replicas: + api_client.jobs_api.destroy_annotations(replica["id"]) + + api_client.consensus_api.partial_update_settings( + settings["id"], + patched_consensus_settings_request=models.PatchedConsensusSettingsRequest( + quorum=0.6 + ), + ) + + # Should be used > quorum times, must be present in the resulting dataset + bbox1 = models.LabeledShapeRequest( + type="rectangle", + frame=parent_job["start_frame"], + label_id=task_labels[0]["id"], + points=[0, 0, 2, 2], + attributes=[ + {"spec_id": attr["id"], "value": attr["default_value"]} + for attr in task_labels[0]["attributes"] + ], + rotation=0, + z_order=0, + occluded=False, + outside=False, + group=0, + ) + + # Should be used < quorum times + bbox2 = models.LabeledShapeRequest( + type="rectangle", + frame=parent_job["start_frame"], + label_id=task_labels[0]["id"], + points=[4, 0, 6, 2], + ) + + api_client.jobs_api.update_annotations( + replicas[0]["id"], + job_annotations_update_request=models.JobAnnotationsUpdateRequest(shapes=[bbox1]), + ) + api_client.jobs_api.update_annotations( + replicas[1]["id"], + job_annotations_update_request=models.JobAnnotationsUpdateRequest( + shapes=[bbox1, bbox2] + ), + ) + + self.merge(job_id=parent_job["id"], user=admin_user) + + merged_annotations = json.loads( + api_client.jobs_api.retrieve_annotations(parent_job["id"])[1].data + ) + assert ( + compare_annotations( + merged_annotations, + {"version": 0, "tags": [], "shapes": [bbox1.to_dict()], "tracks": []}, + ) + == {} + ) + + @pytest.mark.parametrize("job_id", [42]) + def test_unmodified_job_produces_same_annotations(self, admin_user, annotations, job_id: int): + old_annotations = annotations["job"][str(job_id)] + + self.merge(job_id=job_id, user=admin_user) + + with make_api_client(admin_user) as api_client: + new_annotations = json.loads(api_client.jobs_api.retrieve_annotations(job_id)[1].data) + + assert compare_annotations(old_annotations, new_annotations) == {} + + @pytest.mark.parametrize("job_id", [42]) + def test_modified_job_produces_different_annotations( + self, admin_user, annotations, jobs, consensus_settings, job_id: int + ): + settings = next( + s for s in consensus_settings if s["task_id"] == jobs.map[job_id]["task_id"] + ) + old_annotations = annotations["job"][str(job_id)] + + with make_api_client(admin_user) as api_client: + api_client.consensus_api.partial_update_settings( + settings["id"], + patched_consensus_settings_request=models.PatchedConsensusSettingsRequest( + quorum=0.6 + ), + ) + + self.merge(job_id=job_id, user=admin_user) + + with make_api_client(admin_user) as api_client: + new_annotations = json.loads(api_client.jobs_api.retrieve_annotations(job_id)[1].data) + + assert compare_annotations(old_annotations, new_annotations) != {} diff --git a/tests/python/shared/assets/annotations.json b/tests/python/shared/assets/annotations.json index e477df05882c..77c5ffc13272 100644 --- a/tests/python/shared/assets/annotations.json +++ b/tests/python/shared/assets/annotations.json @@ -5251,287 +5251,2895 @@ { "attributes": [ { - "spec_id": 16, - "value": "j1 frame 1 (32) ann 1" + "spec_id": 17, + "value": "" } ], "elements": [], "frame": 0, "group": 0, - "id": 199, - "label_id": 78, + "id": 240, + "label_id": 79, "occluded": false, "outside": false, "points": [ - 260.20437641345416, - 64.96839991694105, - 339.7988160670684, - 103.23495744271713 + 96.42, + 78.7, + 145.41, + 134.61 ], "rotation": 0.0, - "source": "manual", + "source": "consensus", "type": "rectangle", "z_order": 0 - } - ], - "tags": [], - "tracks": [], - "version": 0 - }, - "43": { - "shapes": [ + }, { "attributes": [ { "spec_id": 16, - "value": "j1 r1 frame 1 (32) ann 1" + "value": "" } ], "elements": [], "frame": 0, "group": 0, - "id": 200, + "id": 241, "label_id": 78, "occluded": false, "outside": false, "points": [ - 236.47911074747208, - 63.43773761591001, - 335.20682916397527, - 123.1335673561207 + 189.79, + 99.41, + 224.23, + 98.64, + 208.16, + 130.02 ], "rotation": 0.0, - "source": "manual", - "type": "rectangle", + "source": "consensus", + "type": "polygon", "z_order": 0 - } - ], - "tags": [], - "tracks": [], - "version": 0 - }, - "44": { - "shapes": [], - "tags": [], - "tracks": [], - "version": 0 - }, - "45": { - "shapes": [], - "tags": [], - "tracks": [], - "version": 0 - }, - "46": { - "shapes": [ + }, { "attributes": [ { "spec_id": 16, - "value": "j2 frame 2 (32) ann 1" + "value": "" } ], "elements": [], - "frame": 6, + "frame": 0, "group": 0, - "id": 194, + "id": 242, "label_id": 78, "occluded": false, "outside": false, "points": [ - 472.58377068151276, - 63.05507204065361, - 535.3409250237855, - 110.50560337261595 + 179.45, + 160.21, + 229.95, + 141.91, + 212.38, + 167.15 ], "rotation": 0.0, - "source": "manual", - "type": "rectangle", + "source": "consensus", + "type": "polygon", "z_order": 0 - } - ], - "tags": [], - "tracks": [], - "version": 0 - }, - "47": { - "shapes": [ + }, { "attributes": [ { "spec_id": 16, - "value": "j2 r1 frame 1 (35) ann 1" + "value": "" } ], "elements": [], - "frame": 5, + "frame": 0, "group": 0, - "id": 195, + "id": 243, "label_id": 78, "occluded": false, "outside": false, "points": [ - 183.389531668141, - 181.09253910664302, - 247.88201512555497, - 226.14893165908325 + 271.7, + 119.3, + 304.6, + 104.0, + 310.72, + 128.49, + 339.8, + 107.06 ], "rotation": 0.0, - "source": "manual", - "type": "rectangle", + "source": "consensus", + "type": "polyline", "z_order": 0 }, { "attributes": [ { "spec_id": 16, - "value": "j2 r1 frame 1 (35) ann 2" + "value": "" } ], "elements": [], - "frame": 5, + "frame": 0, "group": 0, - "id": 196, + "id": 244, "label_id": 78, "occluded": false, "outside": false, "points": [ - 344.1790109729245, - 311.8444233764694, - 391.8857795578606, - 376.3369068338834 + 288.53, + 139.6, + 307.66, + 164.85, + 336.75, + 143.42 ], "rotation": 0.0, - "source": "manual", - "type": "rectangle", + "source": "consensus", + "type": "polyline", "z_order": 0 - } - ], - "tags": [], - "tracks": [], - "version": 0 - }, - "48": { - "shapes": [], - "tags": [], - "tracks": [], - "version": 0 - }, - "49": { - "shapes": [ + }, { "attributes": [ { "spec_id": 16, - "value": "j2 r3 frame 1 (35) ann 1" + "value": "" } ], "elements": [], - "frame": 5, + "frame": 0, "group": 0, - "id": 197, + "id": 245, "label_id": 78, "occluded": false, "outside": false, "points": [ - 169.25419282815892, - 210.24667546410456, - 239.0474283505664, - 239.4008118215661 + 277.04, + 68.03, + 290.82, + 38.95, + 307.65, + 68.03, + 322.2, + 40.48 ], "rotation": 0.0, - "source": "manual", - "type": "rectangle", + "source": "consensus", + "type": "points", "z_order": 0 }, { "attributes": [ { - "spec_id": 16, - "value": "j2 r3 frame 1 (35) ann 2" + "spec_id": 17, + "value": "" } ], "elements": [], - "frame": 5, + "frame": 0, "group": 0, - "id": 198, - "label_id": 78, + "id": 246, + "label_id": 79, "occluded": false, "outside": false, "points": [ - 336.22788287543517, - 361.3181093164021, - 398.9534489778507, - 424.9271340963169 + 216.21, + 45.84, + 246.82, + 45.85, + 261.36, + 18.29 ], "rotation": 0.0, - "source": "manual", - "type": "rectangle", + "source": "consensus", + "type": "points", "z_order": 0 - } - ], - "tags": [], - "tracks": [], - "version": 0 - }, - "50": { - "shapes": [ + }, { "attributes": [ { "spec_id": 16, - "value": "gt job frame 1 (32) ann 1" + "value": "" } ], "elements": [], - "frame": 8, + "frame": 0, "group": 0, - "id": 191, + "id": 247, "label_id": 78, "occluded": false, "outside": false, "points": [ - 160.71132684643453, - 59.61108186333331, - 254.8470583598446, - 111.65360009838878 + 17.0, + 9.0, + 32.0, + 13.0, + 28.0, + 17.0, + 25.0, + 19.0, + 23.0, + 22.0, + 19.0, + 25.0, + 17.0, + 27.0, + 16.0, + 27.0, + 15.0, + 29.0, + 13.0, + 31.0, + 12.0, + 31.0, + 11.0, + 33.0, + 9.0, + 35.0, + 8.0, + 35.0, + 7.0, + 37.0, + 6.0, + 37.0, + 5.0, + 39.0, + 4.0, + 39.0, + 4.0, + 39.0, + 3.0, + 41.0, + 2.0, + 41.0, + 2.0, + 41.0, + 2.0, + 41.0, + 2.0, + 41.0, + 1.0, + 559.0, + 1.0, + 41.0, + 2.0, + 41.0, + 2.0, + 41.0, + 2.0, + 41.0, + 3.0, + 39.0, + 4.0, + 39.0, + 4.0, + 39.0, + 4.0, + 39.0, + 5.0, + 37.0, + 6.0, + 37.0, + 7.0, + 35.0, + 8.0, + 35.0, + 9.0, + 33.0, + 11.0, + 31.0, + 12.0, + 31.0, + 13.0, + 29.0, + 15.0, + 27.0, + 16.0, + 27.0, + 17.0, + 25.0, + 20.0, + 22.0, + 22.0, + 19.0, + 25.0, + 17.0, + 28.0, + 13.0, + 32.0, + 9.0, + 17.0, + 379.0, + 78.0, + 421.0, + 138.0 ], "rotation": 0.0, - "source": "Ground truth", - "type": "rectangle", + "source": "consensus", + "type": "mask", "z_order": 0 }, { "attributes": [ { "spec_id": 16, - "value": "gt job frame 1 (32) ann 2" + "value": "" } ], "elements": [], - "frame": 8, + "frame": 0, "group": 0, - "id": 192, + "id": 248, "label_id": 78, "occluded": false, "outside": false, "points": [ - 434.6998787309949, - 78.74436062622226, - 502.0490199763608, - 127.72555425921564 - ], - "rotation": 0.0, - "source": "Ground truth", - "type": "rectangle", - "z_order": 0 - }, - { - "attributes": [ - { - "spec_id": 16, - "value": "gt job frame 2 (35) ann 1" - } + 127.0, + 2.0, + 151.0, + 9.0, + 143.0, + 17.0, + 136.0, + 24.0, + 129.0, + 31.0, + 122.0, + 38.0, + 115.0, + 46.0, + 56.0, + 4.0, + 49.0, + 51.0, + 53.0, + 7.0, + 48.0, + 52.0, + 51.0, + 10.0, + 47.0, + 52.0, + 48.0, + 14.0, + 46.0, + 52.0, + 46.0, + 17.0, + 45.0, + 52.0, + 43.0, + 21.0, + 44.0, + 52.0, + 41.0, + 24.0, + 43.0, + 52.0, + 38.0, + 27.0, + 43.0, + 52.0, + 36.0, + 30.0, + 42.0, + 52.0, + 33.0, + 34.0, + 42.0, + 51.0, + 31.0, + 37.0, + 41.0, + 51.0, + 28.0, + 41.0, + 40.0, + 52.0, + 25.0, + 43.0, + 40.0, + 52.0, + 23.0, + 46.0, + 39.0, + 52.0, + 20.0, + 50.0, + 38.0, + 52.0, + 18.0, + 53.0, + 37.0, + 52.0, + 15.0, + 57.0, + 36.0, + 52.0, + 13.0, + 60.0, + 35.0, + 52.0, + 10.0, + 63.0, + 35.0, + 52.0, + 8.0, + 66.0, + 34.0, + 52.0, + 5.0, + 70.0, + 34.0, + 51.0, + 3.0, + 73.0, + 33.0, + 51.0, + 1.0, + 76.0, + 32.0, + 48.0, + 2.0, + 79.0, + 31.0, + 46.0, + 4.0, + 79.0, + 31.0, + 44.0, + 5.0, + 81.0, + 30.0, + 41.0, + 8.0, + 82.0, + 29.0, + 39.0, + 9.0, + 84.0, + 28.0, + 37.0, + 10.0, + 86.0, + 27.0, + 34.0, + 13.0, + 87.0, + 26.0, + 32.0, + 14.0, + 88.0, + 26.0, + 30.0, + 16.0, + 51.0, + 1.0, + 34.0, + 29.0, + 26.0, + 18.0, + 50.0, + 3.0, + 33.0, + 30.0, + 24.0, + 19.0, + 49.0, + 4.0, + 32.0, + 32.0, + 22.0, + 21.0, + 47.0, + 6.0, + 30.0, + 34.0, + 19.0, + 23.0, + 46.0, + 7.0, + 30.0, + 35.0, + 17.0, + 25.0, + 44.0, + 9.0, + 28.0, + 37.0, + 15.0, + 26.0, + 43.0, + 10.0, + 27.0, + 39.0, + 12.0, + 28.0, + 42.0, + 12.0, + 26.0, + 40.0, + 10.0, + 30.0, + 41.0, + 12.0, + 25.0, + 42.0, + 8.0, + 31.0, + 40.0, + 14.0, + 24.0, + 43.0, + 5.0, + 34.0, + 38.0, + 15.0, + 23.0, + 46.0, + 2.0, + 35.0, + 37.0, + 17.0, + 21.0, + 84.0, + 36.0, + 18.0, + 21.0, + 85.0, + 34.0, + 20.0, + 19.0, + 141.0, + 18.0, + 141.0, + 17.0, + 143.0, + 15.0, + 144.0, + 15.0, + 145.0, + 13.0, + 146.0, + 12.0, + 148.0, + 11.0, + 148.0, + 10.0, + 150.0, + 9.0, + 150.0, + 8.0, + 152.0, + 6.0, + 153.0, + 6.0, + 154.0, + 4.0, + 155.0, + 4.0, + 156.0, + 2.0, + 157.0, + 1.0, + 87.0, + 375.0, + 13.0, + 533.0, + 80.0 + ], + "rotation": 0.0, + "source": "consensus", + "type": "mask", + "z_order": 0 + }, + { + "attributes": [ + { + "spec_id": 17, + "value": "" + } + ], + "elements": [], + "frame": 0, + "group": 0, + "id": 249, + "label_id": 79, + "occluded": false, + "outside": false, + "points": [ + 127.0, + 3.0, + 151.0, + 10.0, + 143.0, + 18.0, + 136.0, + 25.0, + 129.0, + 32.0, + 122.0, + 39.0, + 115.0, + 47.0, + 109.0, + 52.0, + 108.0, + 53.0, + 107.0, + 54.0, + 107.0, + 54.0, + 107.0, + 54.0, + 107.0, + 54.0, + 107.0, + 54.0, + 38.0, + 1.0, + 68.0, + 54.0, + 36.0, + 3.0, + 68.0, + 54.0, + 33.0, + 5.0, + 69.0, + 54.0, + 31.0, + 6.0, + 71.0, + 53.0, + 28.0, + 9.0, + 71.0, + 54.0, + 25.0, + 10.0, + 72.0, + 54.0, + 23.0, + 11.0, + 73.0, + 54.0, + 20.0, + 14.0, + 35.0, + 2.0, + 36.0, + 54.0, + 18.0, + 15.0, + 34.0, + 5.0, + 35.0, + 54.0, + 15.0, + 17.0, + 34.0, + 7.0, + 34.0, + 54.0, + 13.0, + 19.0, + 32.0, + 10.0, + 33.0, + 54.0, + 10.0, + 21.0, + 32.0, + 11.0, + 33.0, + 54.0, + 8.0, + 22.0, + 31.0, + 14.0, + 32.0, + 54.0, + 5.0, + 25.0, + 29.0, + 17.0, + 31.0, + 54.0, + 3.0, + 26.0, + 29.0, + 19.0, + 31.0, + 53.0, + 1.0, + 27.0, + 28.0, + 22.0, + 30.0, + 81.0, + 26.0, + 25.0, + 29.0, + 49.0, + 2.0, + 29.0, + 26.0, + 26.0, + 29.0, + 47.0, + 4.0, + 29.0, + 24.0, + 29.0, + 28.0, + 45.0, + 5.0, + 29.0, + 24.0, + 31.0, + 27.0, + 42.0, + 8.0, + 28.0, + 24.0, + 33.0, + 26.0, + 40.0, + 9.0, + 29.0, + 23.0, + 35.0, + 25.0, + 38.0, + 10.0, + 29.0, + 23.0, + 37.0, + 24.0, + 35.0, + 13.0, + 28.0, + 23.0, + 38.0, + 24.0, + 33.0, + 14.0, + 29.0, + 22.0, + 38.0, + 25.0, + 31.0, + 16.0, + 28.0, + 23.0, + 36.0, + 28.0, + 27.0, + 18.0, + 28.0, + 25.0, + 34.0, + 29.0, + 25.0, + 19.0, + 29.0, + 24.0, + 33.0, + 31.0, + 23.0, + 21.0, + 28.0, + 25.0, + 31.0, + 33.0, + 20.0, + 23.0, + 28.0, + 25.0, + 31.0, + 34.0, + 18.0, + 25.0, + 28.0, + 25.0, + 29.0, + 36.0, + 16.0, + 26.0, + 28.0, + 25.0, + 28.0, + 38.0, + 13.0, + 28.0, + 28.0, + 26.0, + 27.0, + 39.0, + 11.0, + 30.0, + 28.0, + 25.0, + 26.0, + 41.0, + 9.0, + 31.0, + 28.0, + 26.0, + 25.0, + 42.0, + 6.0, + 34.0, + 27.0, + 26.0, + 24.0, + 45.0, + 3.0, + 35.0, + 28.0, + 26.0, + 22.0, + 84.0, + 28.0, + 26.0, + 22.0, + 85.0, + 28.0, + 26.0, + 20.0, + 141.0, + 19.0, + 141.0, + 18.0, + 143.0, + 16.0, + 144.0, + 16.0, + 145.0, + 14.0, + 146.0, + 13.0, + 148.0, + 12.0, + 148.0, + 11.0, + 150.0, + 10.0, + 150.0, + 9.0, + 152.0, + 7.0, + 153.0, + 7.0, + 154.0, + 5.0, + 155.0, + 5.0, + 156.0, + 3.0, + 157.0, + 2.0, + 87.0, + 616.0, + 13.0, + 775.0, + 81.0 + ], + "rotation": 0.0, + "source": "consensus", + "type": "mask", + "z_order": 0 + } + ], + "tags": [ + { + "attributes": [ + { + "spec_id": 16, + "value": "" + } + ], + "frame": 0, + "group": 0, + "id": 13, + "label_id": 78, + "source": "manual" + } + ], + "tracks": [], + "version": 0 + }, + "43": { + "shapes": [ + { + "attributes": [ + { + "spec_id": 17, + "value": "" + } + ], + "elements": [], + "frame": 0, + "group": 0, + "id": 218, + "label_id": 79, + "occluded": false, + "outside": false, + "points": [ + 96.423828125, + 78.70000000000073, + 145.40502175799338, + 134.6134376525879 + ], + "rotation": 0.0, + "source": "manual", + "type": "rectangle", + "z_order": 0 + }, + { + "attributes": [ + { + "spec_id": 16, + "value": "" + } + ], + "elements": [], + "frame": 0, + "group": 0, + "id": 219, + "label_id": 78, + "occluded": false, + "outside": false, + "points": [ + 189.0, + 113.20000000000073, + 224.23381233922373, + 98.64297053962582, + 208.16185817839687, + 130.0215477107613 + ], + "rotation": 0.0, + "source": "manual", + "type": "polygon", + "z_order": 0 + }, + { + "attributes": [ + { + "spec_id": 16, + "value": "" + } + ], + "elements": [], + "frame": 0, + "group": 0, + "id": 220, + "label_id": 78, + "occluded": false, + "outside": false, + "points": [ + 271.7000000000007, + 119.30000000000109, + 304.60000000000036, + 104.0, + 310.7162323474786, + 128.49088540973025, + 339.7988160670684, + 107.06161319529565 + ], + "rotation": 0.0, + "source": "manual", + "type": "polyline", + "z_order": 0 + }, + { + "attributes": [ + { + "spec_id": 16, + "value": "" + } + ], + "elements": [], + "frame": 0, + "group": 0, + "id": 221, + "label_id": 78, + "occluded": false, + "outside": false, + "points": [ + 277.0419921875, + 68.029296875, + 290.817622434075, + 38.94714079941332, + 307.6549077454165, + 68.02972451900314 + ], + "rotation": 0.0, + "source": "manual", + "type": "points", + "z_order": 0 + }, + { + "attributes": [ + { + "spec_id": 16, + "value": "" + } + ], + "elements": [], + "frame": 0, + "group": 0, + "id": 222, + "label_id": 78, + "occluded": false, + "outside": false, + "points": [ + 399.884765625, + 108.27509765625109, + 420.931640625, + 78.45000000000073 + ], + "rotation": 0.0, + "source": "manual", + "type": "ellipse", + "z_order": 0 + }, + { + "attributes": [ + { + "spec_id": 16, + "value": "" + } + ], + "elements": [], + "frame": 0, + "group": 0, + "id": 223, + "label_id": 78, + "occluded": false, + "outside": false, + "points": [ + 502.049255907923, + 101.79330723501334, + 502.049255907923, + 137.6748346606273, + 537.2541861748086, + 101.73114849558442, + 537.2541861748086, + 137.6748346606455, + 563.2122646078951, + 89.3489492958688, + 563.2122646078951, + 124.59863263951047, + 528.6418953937591, + 89.43091015684695, + 528.6418953937591, + 124.6208099212563 + ], + "rotation": 0.0, + "source": "manual", + "type": "cuboid", + "z_order": 0 + }, + { + "attributes": [ + { + "spec_id": 16, + "value": "" + } + ], + "elements": [], + "frame": 0, + "group": 0, + "id": 224, + "label_id": 78, + "occluded": false, + "outside": false, + "points": [ + 127.0, + 2.0, + 151.0, + 9.0, + 143.0, + 17.0, + 136.0, + 24.0, + 129.0, + 31.0, + 122.0, + 38.0, + 115.0, + 46.0, + 56.0, + 4.0, + 49.0, + 51.0, + 53.0, + 7.0, + 48.0, + 52.0, + 51.0, + 10.0, + 47.0, + 52.0, + 48.0, + 14.0, + 46.0, + 52.0, + 46.0, + 17.0, + 45.0, + 52.0, + 43.0, + 21.0, + 44.0, + 52.0, + 41.0, + 24.0, + 43.0, + 52.0, + 38.0, + 27.0, + 43.0, + 52.0, + 36.0, + 30.0, + 42.0, + 52.0, + 33.0, + 34.0, + 42.0, + 51.0, + 31.0, + 37.0, + 41.0, + 51.0, + 28.0, + 41.0, + 40.0, + 52.0, + 25.0, + 43.0, + 40.0, + 52.0, + 23.0, + 46.0, + 39.0, + 52.0, + 20.0, + 50.0, + 38.0, + 52.0, + 18.0, + 53.0, + 37.0, + 52.0, + 15.0, + 57.0, + 36.0, + 52.0, + 13.0, + 60.0, + 35.0, + 52.0, + 10.0, + 63.0, + 35.0, + 52.0, + 8.0, + 66.0, + 34.0, + 52.0, + 5.0, + 70.0, + 34.0, + 51.0, + 3.0, + 73.0, + 33.0, + 51.0, + 1.0, + 76.0, + 32.0, + 48.0, + 2.0, + 79.0, + 31.0, + 46.0, + 4.0, + 79.0, + 31.0, + 44.0, + 5.0, + 81.0, + 30.0, + 41.0, + 8.0, + 82.0, + 29.0, + 39.0, + 9.0, + 84.0, + 28.0, + 37.0, + 10.0, + 86.0, + 27.0, + 34.0, + 13.0, + 87.0, + 26.0, + 32.0, + 14.0, + 88.0, + 26.0, + 30.0, + 16.0, + 51.0, + 1.0, + 34.0, + 29.0, + 26.0, + 18.0, + 50.0, + 3.0, + 33.0, + 30.0, + 24.0, + 19.0, + 49.0, + 4.0, + 32.0, + 32.0, + 22.0, + 21.0, + 47.0, + 6.0, + 30.0, + 34.0, + 19.0, + 23.0, + 46.0, + 7.0, + 30.0, + 35.0, + 17.0, + 25.0, + 44.0, + 9.0, + 28.0, + 37.0, + 15.0, + 26.0, + 43.0, + 10.0, + 27.0, + 39.0, + 12.0, + 28.0, + 42.0, + 12.0, + 26.0, + 40.0, + 10.0, + 30.0, + 41.0, + 12.0, + 25.0, + 42.0, + 8.0, + 31.0, + 40.0, + 14.0, + 24.0, + 43.0, + 5.0, + 34.0, + 38.0, + 15.0, + 23.0, + 46.0, + 2.0, + 35.0, + 37.0, + 17.0, + 21.0, + 84.0, + 36.0, + 18.0, + 21.0, + 85.0, + 34.0, + 20.0, + 19.0, + 141.0, + 18.0, + 141.0, + 17.0, + 143.0, + 15.0, + 144.0, + 15.0, + 145.0, + 13.0, + 146.0, + 12.0, + 148.0, + 11.0, + 148.0, + 10.0, + 150.0, + 9.0, + 150.0, + 8.0, + 152.0, + 6.0, + 153.0, + 6.0, + 154.0, + 4.0, + 155.0, + 4.0, + 156.0, + 2.0, + 157.0, + 1.0, + 87.0, + 375.0, + 13.0, + 533.0, + 80.0 + ], + "rotation": 0.0, + "source": "manual", + "type": "mask", + "z_order": 0 + } + ], + "tags": [ + { + "attributes": [ + { + "spec_id": 16, + "value": "" + } + ], + "frame": 0, + "group": 0, + "id": 11, + "label_id": 78, + "source": "manual" + } + ], + "tracks": [], + "version": 0 + }, + "44": { + "shapes": [ + { + "attributes": [ + { + "spec_id": 16, + "value": "" + } + ], + "elements": [], + "frame": 0, + "group": 0, + "id": 212, + "label_id": 78, + "occluded": false, + "outside": false, + "points": [ + 189.7939453125, + 99.408203125, + 219.60000000000036, + 108.60000000000036, + 208.16185817839687, + 130.0215477107613 + ], + "rotation": 0.0, + "source": "manual", + "type": "polygon", + "z_order": 0 + }, + { + "attributes": [ + { + "spec_id": 16, + "value": "" + } + ], + "elements": [], + "frame": 0, + "group": 0, + "id": 213, + "label_id": 78, + "occluded": false, + "outside": false, + "points": [ + 267.90000000000146, + 123.90000000000146, + 291.58295358458963, + 103.23495744271895, + 310.7162323474786, + 128.49088540973025, + 323.0, + 103.20000000000073 + ], + "rotation": 0.0, + "source": "manual", + "type": "polyline", + "z_order": 0 + }, + { + "attributes": [ + { + "spec_id": 16, + "value": "" + } + ], + "elements": [], + "frame": 0, + "group": 0, + "id": 214, + "label_id": 78, + "occluded": false, + "outside": false, + "points": [ + 277.0419921875, + 68.029296875, + 290.817622434075, + 38.94714079941332, + 322.1961996052123, + 40.47780310044436 + ], + "rotation": 0.0, + "source": "manual", + "type": "points", + "z_order": 0 + }, + { + "attributes": [ + { + "spec_id": 16, + "value": "" + } + ], + "elements": [], + "frame": 0, + "group": 0, + "id": 215, + "label_id": 78, + "occluded": false, + "outside": false, + "points": [ + 412.1189453125007, + 126.21103515624964, + 445.40000000000146, + 96.72207031249854 + ], + "rotation": 0.0, + "source": "manual", + "type": "ellipse", + "z_order": 0 + }, + { + "attributes": [ + { + "spec_id": 16, + "value": "" + } + ], + "elements": [], + "frame": 0, + "group": 0, + "id": 216, + "label_id": 78, + "occluded": false, + "outside": false, + "points": [ + 502.049255907923, + 93.39047190463316, + 502.049255907923, + 152.18125991462875, + 559.4494986748086, + 93.26539073985441, + 559.4494986748086, + 152.22223330696215, + 585.0063249908471, + 81.0335845571608, + 585.0063249908471, + 138.85086779521043, + 528.6418953937846, + 81.19004210502862, + 528.6418953937846, + 138.84761929236265 + ], + "rotation": 0.0, + "source": "manual", + "type": "cuboid", + "z_order": 0 + }, + { + "attributes": [ + { + "spec_id": 17, + "value": "" + } + ], + "elements": [], + "frame": 0, + "group": 0, + "id": 211, + "label_id": 79, + "occluded": false, + "outside": false, + "points": [ + 96.42351020313072, + 99.40830169014043, + 145.4047038361241, + 134.61353461385443 + ], + "rotation": 0.0, + "source": "manual", + "type": "rectangle", + "z_order": 0 + }, + { + "attributes": [ + { + "spec_id": 16, + "value": "" + } + ], + "elements": [], + "frame": 0, + "group": 0, + "id": 217, + "label_id": 78, + "occluded": false, + "outside": false, + "points": [ + 127.0, + 2.0, + 151.0, + 9.0, + 143.0, + 17.0, + 136.0, + 24.0, + 129.0, + 31.0, + 122.0, + 38.0, + 115.0, + 46.0, + 56.0, + 4.0, + 49.0, + 51.0, + 53.0, + 7.0, + 48.0, + 52.0, + 51.0, + 10.0, + 47.0, + 52.0, + 48.0, + 14.0, + 46.0, + 52.0, + 46.0, + 17.0, + 45.0, + 52.0, + 43.0, + 21.0, + 44.0, + 52.0, + 41.0, + 24.0, + 43.0, + 52.0, + 38.0, + 27.0, + 43.0, + 52.0, + 36.0, + 30.0, + 42.0, + 52.0, + 33.0, + 34.0, + 42.0, + 51.0, + 31.0, + 37.0, + 41.0, + 51.0, + 28.0, + 41.0, + 40.0, + 52.0, + 25.0, + 43.0, + 40.0, + 52.0, + 23.0, + 46.0, + 39.0, + 52.0, + 20.0, + 50.0, + 38.0, + 52.0, + 18.0, + 53.0, + 37.0, + 52.0, + 15.0, + 57.0, + 36.0, + 52.0, + 13.0, + 60.0, + 35.0, + 52.0, + 10.0, + 63.0, + 35.0, + 52.0, + 8.0, + 66.0, + 34.0, + 52.0, + 5.0, + 70.0, + 34.0, + 51.0, + 3.0, + 73.0, + 33.0, + 51.0, + 1.0, + 76.0, + 32.0, + 48.0, + 2.0, + 79.0, + 31.0, + 46.0, + 4.0, + 79.0, + 31.0, + 44.0, + 5.0, + 81.0, + 30.0, + 41.0, + 8.0, + 82.0, + 29.0, + 39.0, + 9.0, + 84.0, + 28.0, + 37.0, + 10.0, + 86.0, + 27.0, + 34.0, + 13.0, + 87.0, + 26.0, + 32.0, + 14.0, + 88.0, + 26.0, + 30.0, + 16.0, + 51.0, + 1.0, + 34.0, + 29.0, + 26.0, + 18.0, + 50.0, + 3.0, + 33.0, + 30.0, + 24.0, + 19.0, + 49.0, + 4.0, + 32.0, + 32.0, + 22.0, + 21.0, + 47.0, + 6.0, + 30.0, + 34.0, + 19.0, + 23.0, + 46.0, + 7.0, + 30.0, + 35.0, + 17.0, + 25.0, + 44.0, + 9.0, + 28.0, + 37.0, + 15.0, + 26.0, + 43.0, + 10.0, + 27.0, + 39.0, + 12.0, + 28.0, + 42.0, + 12.0, + 26.0, + 40.0, + 10.0, + 30.0, + 41.0, + 12.0, + 25.0, + 42.0, + 8.0, + 31.0, + 40.0, + 14.0, + 24.0, + 43.0, + 5.0, + 34.0, + 38.0, + 15.0, + 23.0, + 46.0, + 2.0, + 35.0, + 37.0, + 17.0, + 21.0, + 84.0, + 36.0, + 18.0, + 21.0, + 85.0, + 34.0, + 20.0, + 19.0, + 141.0, + 18.0, + 141.0, + 17.0, + 143.0, + 15.0, + 144.0, + 15.0, + 145.0, + 13.0, + 146.0, + 12.0, + 148.0, + 11.0, + 148.0, + 10.0, + 150.0, + 9.0, + 150.0, + 8.0, + 152.0, + 6.0, + 153.0, + 6.0, + 154.0, + 4.0, + 155.0, + 4.0, + 156.0, + 2.0, + 157.0, + 1.0, + 87.0, + 375.0, + 13.0, + 533.0, + 80.0 + ], + "rotation": 0.0, + "source": "manual", + "type": "mask", + "z_order": 0 + } + ], + "tags": [ + { + "attributes": [ + { + "spec_id": 16, + "value": "" + } + ], + "frame": 0, + "group": 0, + "id": 10, + "label_id": 78, + "source": "manual" + } + ], + "tracks": [], + "version": 0 + }, + "45": { + "shapes": [ + { + "attributes": [ + { + "spec_id": 17, + "value": "" + } + ], + "elements": [], + "frame": 0, + "group": 0, + "id": 204, + "label_id": 79, + "occluded": false, + "outside": false, + "points": [ + 96.42351020313072, + 99.40830169014043, + 145.4047038361241, + 134.61353461385443 + ], + "rotation": 0.0, + "source": "manual", + "type": "rectangle", + "z_order": 0 + }, + { + "attributes": [ + { + "spec_id": 16, + "value": "" + } + ], + "elements": [], + "frame": 0, + "group": 0, + "id": 205, + "label_id": 78, + "occluded": false, + "outside": false, + "points": [ + 189.7939453125, + 99.408203125, + 224.23381233922373, + 98.64297053962582, + 208.16185817839687, + 130.0215477107613 + ], + "rotation": 0.0, + "source": "manual", + "type": "polygon", + "z_order": 0 + }, + { + "attributes": [ + { + "spec_id": 16, + "value": "" + } + ], + "elements": [], + "frame": 0, + "group": 0, + "id": 206, + "label_id": 78, + "occluded": false, + "outside": false, + "points": [ + 263.265625, + 116.2451171875, + 291.58295358458963, + 103.23495744271895, + 310.7162323474786, + 128.49088540973025, + 339.7988160670684, + 107.06161319529565 + ], + "rotation": 0.0, + "source": "manual", + "type": "polyline", + "z_order": 0 + }, + { + "attributes": [ + { + "spec_id": 16, + "value": "" + } + ], + "elements": [], + "frame": 0, + "group": 0, + "id": 207, + "label_id": 78, + "occluded": false, + "outside": false, + "points": [ + 277.0419921875, + 68.029296875, + 290.817622434075, + 38.94714079941332, + 307.6549077454165, + 68.02972451900314, + 322.1961996052123, + 40.47780310044436 + ], + "rotation": 0.0, + "source": "manual", + "type": "points", + "z_order": 0 + }, + { + "attributes": [ + { + "spec_id": 16, + "value": "" + } + ], + "elements": [], + "frame": 0, + "group": 0, + "id": 208, + "label_id": 78, + "occluded": false, + "outside": false, + "points": [ + 399.884765625, + 117.7861328125, + 420.931640625, + 97.1220703125 + ], + "rotation": 0.0, + "source": "manual", + "type": "ellipse", + "z_order": 0 + }, + { + "attributes": [ + { + "spec_id": 16, + "value": "" + } + ], + "elements": [], + "frame": 0, + "group": 0, + "id": 209, + "label_id": 78, + "occluded": false, + "outside": false, + "points": [ + 502.049255907923, + 93.39047190463862, + 502.049255907923, + 137.6748346606255, + 545.6731314873086, + 93.29541089258055, + 545.6731314873086, + 137.6748346606455, + 571.4791182468543, + 81.07113365704572, + 571.4791182468543, + 124.59332935967359, + 528.6418953937628, + 81.1900421050359, + 528.6418953937628, + 124.62080992126721 + ], + "rotation": 0.0, + "source": "manual", + "type": "cuboid", + "z_order": 0 + }, + { + "attributes": [ + { + "spec_id": 16, + "value": "" + } + ], + "elements": [], + "frame": 0, + "group": 0, + "id": 210, + "label_id": 78, + "occluded": false, + "outside": false, + "points": [ + 127.0, + 2.0, + 151.0, + 9.0, + 143.0, + 17.0, + 136.0, + 24.0, + 129.0, + 31.0, + 122.0, + 38.0, + 115.0, + 46.0, + 56.0, + 4.0, + 49.0, + 51.0, + 53.0, + 7.0, + 48.0, + 52.0, + 51.0, + 10.0, + 47.0, + 52.0, + 48.0, + 14.0, + 46.0, + 52.0, + 46.0, + 17.0, + 45.0, + 52.0, + 43.0, + 21.0, + 44.0, + 52.0, + 41.0, + 24.0, + 43.0, + 52.0, + 38.0, + 27.0, + 43.0, + 52.0, + 36.0, + 30.0, + 42.0, + 52.0, + 33.0, + 34.0, + 42.0, + 51.0, + 31.0, + 37.0, + 41.0, + 51.0, + 28.0, + 41.0, + 40.0, + 52.0, + 25.0, + 43.0, + 40.0, + 52.0, + 23.0, + 46.0, + 39.0, + 52.0, + 20.0, + 50.0, + 38.0, + 52.0, + 18.0, + 53.0, + 37.0, + 52.0, + 15.0, + 57.0, + 36.0, + 52.0, + 13.0, + 60.0, + 35.0, + 52.0, + 10.0, + 63.0, + 35.0, + 52.0, + 8.0, + 66.0, + 34.0, + 52.0, + 5.0, + 70.0, + 34.0, + 51.0, + 3.0, + 73.0, + 33.0, + 51.0, + 1.0, + 76.0, + 32.0, + 48.0, + 2.0, + 79.0, + 31.0, + 46.0, + 4.0, + 79.0, + 31.0, + 44.0, + 5.0, + 81.0, + 30.0, + 41.0, + 8.0, + 82.0, + 29.0, + 39.0, + 9.0, + 84.0, + 28.0, + 37.0, + 10.0, + 86.0, + 27.0, + 34.0, + 13.0, + 87.0, + 26.0, + 32.0, + 14.0, + 88.0, + 26.0, + 30.0, + 16.0, + 51.0, + 1.0, + 34.0, + 29.0, + 26.0, + 18.0, + 50.0, + 3.0, + 33.0, + 30.0, + 24.0, + 19.0, + 49.0, + 4.0, + 32.0, + 32.0, + 22.0, + 21.0, + 47.0, + 6.0, + 30.0, + 34.0, + 19.0, + 23.0, + 46.0, + 7.0, + 30.0, + 35.0, + 17.0, + 25.0, + 44.0, + 9.0, + 28.0, + 37.0, + 15.0, + 26.0, + 43.0, + 10.0, + 27.0, + 39.0, + 12.0, + 28.0, + 42.0, + 12.0, + 26.0, + 40.0, + 10.0, + 30.0, + 41.0, + 12.0, + 25.0, + 42.0, + 8.0, + 31.0, + 40.0, + 14.0, + 24.0, + 43.0, + 5.0, + 34.0, + 38.0, + 15.0, + 23.0, + 46.0, + 2.0, + 35.0, + 37.0, + 17.0, + 21.0, + 84.0, + 36.0, + 18.0, + 21.0, + 85.0, + 34.0, + 20.0, + 19.0, + 141.0, + 18.0, + 141.0, + 17.0, + 143.0, + 15.0, + 144.0, + 15.0, + 145.0, + 13.0, + 146.0, + 12.0, + 148.0, + 11.0, + 148.0, + 10.0, + 150.0, + 9.0, + 150.0, + 8.0, + 152.0, + 6.0, + 153.0, + 6.0, + 154.0, + 4.0, + 155.0, + 4.0, + 156.0, + 2.0, + 157.0, + 1.0, + 87.0, + 375.0, + 13.0, + 533.0, + 80.0 + ], + "rotation": 0.0, + "source": "manual", + "type": "mask", + "z_order": 0 + }, + { + "attributes": [ + { + "spec_id": 16, + "value": "" + } + ], + "elements": [], + "frame": 0, + "group": 0, + "id": 225, + "label_id": 78, + "occluded": false, + "outside": false, + "points": [ + 179.44667968750036, + 160.21328125000036, + 229.94667968750036, + 141.9132812500011, + 212.37865505339687, + 167.1504539607613 + ], + "rotation": 0.0, + "source": "manual", + "type": "polygon", + "z_order": 0 + }, + { + "attributes": [ + { + "spec_id": 16, + "value": "" + } + ], + "elements": [], + "frame": 0, + "group": 0, + "id": 226, + "label_id": 78, + "occluded": false, + "outside": false, + "points": [ + 288.52924264708963, + 139.59823869271895, + 307.6625214099786, + 164.85416665973025, + 336.7451051295684, + 143.42489444529565 + ], + "rotation": 0.0, + "source": "manual", + "type": "polyline", + "z_order": 0 + }, + { + "attributes": [ + { + "spec_id": 17, + "value": "" + } + ], + "elements": [], + "frame": 0, + "group": 0, + "id": 227, + "label_id": 79, + "occluded": false, + "outside": false, + "points": [ + 216.205078125, + 45.8447265625, + 246.8179936829165, + 45.84515420650314, + 261.3592855427123, + 18.293232787944362 + ], + "rotation": 0.0, + "source": "manual", + "type": "points", + "z_order": 0 + }, + { + "attributes": [ + { + "spec_id": 17, + "value": "" + } + ], + "elements": [], + "frame": 0, + "group": 0, + "id": 228, + "label_id": 79, + "occluded": false, + "outside": false, + "points": [ + 127.0, + 3.0, + 151.0, + 10.0, + 143.0, + 18.0, + 136.0, + 25.0, + 129.0, + 32.0, + 122.0, + 39.0, + 115.0, + 47.0, + 109.0, + 52.0, + 108.0, + 53.0, + 107.0, + 54.0, + 107.0, + 54.0, + 107.0, + 54.0, + 107.0, + 54.0, + 107.0, + 54.0, + 38.0, + 1.0, + 68.0, + 54.0, + 36.0, + 3.0, + 68.0, + 54.0, + 33.0, + 5.0, + 69.0, + 54.0, + 31.0, + 6.0, + 71.0, + 53.0, + 28.0, + 9.0, + 71.0, + 54.0, + 25.0, + 10.0, + 72.0, + 54.0, + 23.0, + 11.0, + 73.0, + 54.0, + 20.0, + 14.0, + 35.0, + 2.0, + 36.0, + 54.0, + 18.0, + 15.0, + 34.0, + 5.0, + 35.0, + 54.0, + 15.0, + 17.0, + 34.0, + 7.0, + 34.0, + 54.0, + 13.0, + 19.0, + 32.0, + 10.0, + 33.0, + 54.0, + 10.0, + 21.0, + 32.0, + 11.0, + 33.0, + 54.0, + 8.0, + 22.0, + 31.0, + 14.0, + 32.0, + 54.0, + 5.0, + 25.0, + 29.0, + 17.0, + 31.0, + 54.0, + 3.0, + 26.0, + 29.0, + 19.0, + 31.0, + 53.0, + 1.0, + 27.0, + 28.0, + 22.0, + 30.0, + 81.0, + 26.0, + 25.0, + 29.0, + 49.0, + 2.0, + 29.0, + 26.0, + 26.0, + 29.0, + 47.0, + 4.0, + 29.0, + 24.0, + 29.0, + 28.0, + 45.0, + 5.0, + 29.0, + 24.0, + 31.0, + 27.0, + 42.0, + 8.0, + 28.0, + 24.0, + 33.0, + 26.0, + 40.0, + 9.0, + 29.0, + 23.0, + 35.0, + 25.0, + 38.0, + 10.0, + 29.0, + 23.0, + 37.0, + 24.0, + 35.0, + 13.0, + 28.0, + 23.0, + 38.0, + 24.0, + 33.0, + 14.0, + 29.0, + 22.0, + 38.0, + 25.0, + 31.0, + 16.0, + 28.0, + 23.0, + 36.0, + 28.0, + 27.0, + 18.0, + 28.0, + 25.0, + 34.0, + 29.0, + 25.0, + 19.0, + 29.0, + 24.0, + 33.0, + 31.0, + 23.0, + 21.0, + 28.0, + 25.0, + 31.0, + 33.0, + 20.0, + 23.0, + 28.0, + 25.0, + 31.0, + 34.0, + 18.0, + 25.0, + 28.0, + 25.0, + 29.0, + 36.0, + 16.0, + 26.0, + 28.0, + 25.0, + 28.0, + 38.0, + 13.0, + 28.0, + 28.0, + 26.0, + 27.0, + 39.0, + 11.0, + 30.0, + 28.0, + 25.0, + 26.0, + 41.0, + 9.0, + 31.0, + 28.0, + 26.0, + 25.0, + 42.0, + 6.0, + 34.0, + 27.0, + 26.0, + 24.0, + 45.0, + 3.0, + 35.0, + 28.0, + 26.0, + 22.0, + 84.0, + 28.0, + 26.0, + 22.0, + 85.0, + 28.0, + 26.0, + 20.0, + 141.0, + 19.0, + 141.0, + 18.0, + 143.0, + 16.0, + 144.0, + 16.0, + 145.0, + 14.0, + 146.0, + 13.0, + 148.0, + 12.0, + 148.0, + 11.0, + 150.0, + 10.0, + 150.0, + 9.0, + 152.0, + 7.0, + 153.0, + 7.0, + 154.0, + 5.0, + 155.0, + 5.0, + 156.0, + 3.0, + 157.0, + 2.0, + 87.0, + 616.0, + 13.0, + 775.0, + 81.0 + ], + "rotation": 0.0, + "source": "manual", + "type": "mask", + "z_order": 0 + } + ], + "tags": [ + { + "attributes": [ + { + "spec_id": 16, + "value": "" + } + ], + "frame": 0, + "group": 0, + "id": 9, + "label_id": 78, + "source": "manual" + } + ], + "tracks": [], + "version": 0 + }, + "46": { + "shapes": [ + { + "attributes": [ + { + "spec_id": 16, + "value": "j2 frame 2 (32) ann 1" + } + ], + "elements": [], + "frame": 6, + "group": 0, + "id": 194, + "label_id": 78, + "occluded": false, + "outside": false, + "points": [ + 472.58377068151276, + 63.05507204065361, + 535.3409250237855, + 110.50560337261595 + ], + "rotation": 0.0, + "source": "manual", + "type": "rectangle", + "z_order": 0 + } + ], + "tags": [], + "tracks": [], + "version": 0 + }, + "47": { + "shapes": [ + { + "attributes": [ + { + "spec_id": 16, + "value": "j2 r1 frame 1 (35) ann 1" + } + ], + "elements": [], + "frame": 5, + "group": 0, + "id": 195, + "label_id": 78, + "occluded": false, + "outside": false, + "points": [ + 183.389531668141, + 181.09253910664302, + 247.88201512555497, + 226.14893165908325 + ], + "rotation": 0.0, + "source": "manual", + "type": "rectangle", + "z_order": 0 + }, + { + "attributes": [ + { + "spec_id": 16, + "value": "j2 r1 frame 1 (35) ann 2" + } + ], + "elements": [], + "frame": 5, + "group": 0, + "id": 196, + "label_id": 78, + "occluded": false, + "outside": false, + "points": [ + 344.1790109729245, + 311.8444233764694, + 391.8857795578606, + 376.3369068338834 + ], + "rotation": 0.0, + "source": "manual", + "type": "rectangle", + "z_order": 0 + } + ], + "tags": [], + "tracks": [], + "version": 0 + }, + "48": { + "shapes": [], + "tags": [], + "tracks": [], + "version": 0 + }, + "49": { + "shapes": [ + { + "attributes": [ + { + "spec_id": 16, + "value": "j2 r3 frame 1 (35) ann 1" + } + ], + "elements": [], + "frame": 5, + "group": 0, + "id": 197, + "label_id": 78, + "occluded": false, + "outside": false, + "points": [ + 169.25419282815892, + 210.24667546410456, + 239.0474283505664, + 239.4008118215661 + ], + "rotation": 0.0, + "source": "manual", + "type": "rectangle", + "z_order": 0 + }, + { + "attributes": [ + { + "spec_id": 16, + "value": "j2 r3 frame 1 (35) ann 2" + } + ], + "elements": [], + "frame": 5, + "group": 0, + "id": 198, + "label_id": 78, + "occluded": false, + "outside": false, + "points": [ + 336.22788287543517, + 361.3181093164021, + 398.9534489778507, + 424.9271340963169 + ], + "rotation": 0.0, + "source": "manual", + "type": "rectangle", + "z_order": 0 + } + ], + "tags": [], + "tracks": [], + "version": 0 + }, + "50": { + "shapes": [ + { + "attributes": [ + { + "spec_id": 16, + "value": "gt job frame 1 (32) ann 1" + } + ], + "elements": [], + "frame": 8, + "group": 0, + "id": 191, + "label_id": 78, + "occluded": false, + "outside": false, + "points": [ + 160.71132684643453, + 59.61108186333331, + 254.8470583598446, + 111.65360009838878 + ], + "rotation": 0.0, + "source": "Ground truth", + "type": "rectangle", + "z_order": 0 + }, + { + "attributes": [ + { + "spec_id": 16, + "value": "gt job frame 1 (32) ann 2" + } + ], + "elements": [], + "frame": 8, + "group": 0, + "id": 192, + "label_id": 78, + "occluded": false, + "outside": false, + "points": [ + 434.6998787309949, + 78.74436062622226, + 502.0490199763608, + 127.72555425921564 + ], + "rotation": 0.0, + "source": "Ground truth", + "type": "rectangle", + "z_order": 0 + }, + { + "attributes": [ + { + "spec_id": 16, + "value": "gt job frame 2 (35) ann 1" + } ], "elements": [], "frame": 9, @@ -5541,13 +8149,108 @@ "occluded": false, "outside": false, "points": [ - 245.67336843180783, - 162.98163621791718, - 334.90269485918907, - 215.98915686784676 + 245.67336843180783, + 162.98163621791718, + 334.90269485918907, + 215.98915686784676 + ], + "rotation": 0.0, + "source": "Ground truth", + "type": "rectangle", + "z_order": 0 + } + ], + "tags": [], + "tracks": [], + "version": 0 + }, + "51": { + "shapes": [], + "tags": [], + "tracks": [], + "version": 0 + }, + "52": { + "shapes": [ + { + "attributes": [ + { + "spec_id": 18, + "value": "j2 f1 (30) a1" + } + ], + "elements": [], + "frame": 0, + "group": 0, + "id": 201, + "label_id": 80, + "occluded": false, + "outside": false, + "points": [ + 295.39537490125076, + 129.95137884321593, + 379.50821517094846, + 194.85735007494986 + ], + "rotation": 0.0, + "source": "manual", + "type": "rectangle", + "z_order": 0 + } + ], + "tags": [], + "tracks": [], + "version": 0 + }, + "53": { + "shapes": [ + { + "attributes": [ + { + "spec_id": 18, + "value": "j1 f1 (30) a1" + } + ], + "elements": [], + "frame": 0, + "group": 0, + "id": 202, + "label_id": 80, + "occluded": false, + "outside": false, + "points": [ + 339.7698654372325, + 132.6006021587964, + 397.3904725511202, + 175.6504810369879 + ], + "rotation": 0.0, + "source": "manual", + "type": "rectangle", + "z_order": 0 + }, + { + "attributes": [ + { + "spec_id": 18, + "value": "j2 f1 (30) a2" + } + ], + "elements": [], + "frame": 0, + "group": 0, + "id": 203, + "label_id": 80, + "occluded": false, + "outside": false, + "points": [ + 529.1893325012752, + 65.70771344037712, + 597.4068328774865, + 119.35448558089229 ], "rotation": 0.0, - "source": "Ground truth", + "source": "manual", "type": "rectangle", "z_order": 0 } @@ -8506,71 +11209,339 @@ "attributes": [ { "spec_id": 15, - "value": "j1 frame1 n1" + "value": "j1 frame1 n1" + } + ], + "elements": [], + "frame": 0, + "group": 0, + "id": 169, + "label_id": 77, + "occluded": false, + "outside": false, + "points": [ + 19.650000000003274, + 13.100000000002183, + 31.850000000004002, + 18.900000000001455 + ], + "rotation": 0.0, + "source": "manual", + "type": "rectangle", + "z_order": 0 + }, + { + "attributes": [ + { + "spec_id": 15, + "value": "j1 frame2 n1" + } + ], + "elements": [], + "frame": 1, + "group": 0, + "id": 170, + "label_id": 77, + "occluded": false, + "outside": false, + "points": [ + 18.650000000003274, + 10.500000000001819, + 28.650000000003274, + 15.200000000002547 + ], + "rotation": 0.0, + "source": "manual", + "type": "rectangle", + "z_order": 0 + }, + { + "attributes": [ + { + "spec_id": 15, + "value": "j1 frame2 n2" + } + ], + "elements": [], + "frame": 1, + "group": 0, + "id": 171, + "label_id": 77, + "occluded": false, + "outside": false, + "points": [ + 18.850000000002183, + 19.50000000000182, + 27.05000000000291, + 24.900000000001455 + ], + "rotation": 0.0, + "source": "manual", + "type": "rectangle", + "z_order": 0 + }, + { + "attributes": [ + { + "spec_id": 15, + "value": "j1 frame6 n1" + } + ], + "elements": [], + "frame": 5, + "group": 0, + "id": 172, + "label_id": 77, + "occluded": false, + "outside": false, + "points": [ + 26.25000000000182, + 16.50000000000182, + 40.95000000000255, + 23.900000000001455 + ], + "rotation": 0.0, + "source": "manual", + "type": "rectangle", + "z_order": 0 + }, + { + "attributes": [ + { + "spec_id": 15, + "value": "j2 frame1 n1" + } + ], + "elements": [], + "frame": 8, + "group": 0, + "id": 173, + "label_id": 77, + "occluded": false, + "outside": false, + "points": [ + 14.650000000003274, + 10.000000000001819, + 25.750000000003638, + 17.30000000000109 + ], + "rotation": 0.0, + "source": "manual", + "type": "rectangle", + "z_order": 0 + }, + { + "attributes": [ + { + "spec_id": 15, + "value": "j2 frame1 n2" + } + ], + "elements": [], + "frame": 8, + "group": 0, + "id": 174, + "label_id": 77, + "occluded": false, + "outside": false, + "points": [ + 30.350000000002183, + 18.700000000002547, + 43.05000000000291, + 26.400000000003274 + ], + "rotation": 0.0, + "source": "manual", + "type": "rectangle", + "z_order": 0 + }, + { + "attributes": [ + { + "spec_id": 15, + "value": "j2 frame2 n1" + } + ], + "elements": [], + "frame": 9, + "group": 0, + "id": 175, + "label_id": 77, + "occluded": false, + "outside": false, + "points": [ + 9.200000000002547, + 34.35000000000218, + 21.900000000003274, + 38.55000000000291 + ], + "rotation": 0.0, + "source": "manual", + "type": "rectangle", + "z_order": 0 + }, + { + "attributes": [ + { + "spec_id": 15, + "value": "j2 frame2 n2" + } + ], + "elements": [], + "frame": 9, + "group": 0, + "id": 176, + "label_id": 77, + "occluded": false, + "outside": false, + "points": [ + 40.900390625, + 29.0498046875, + 48.80000000000291, + 30.350000000002183, + 45.10000000000218, + 39.25000000000182, + 45.70000000000255, + 24.450000000002547 + ], + "rotation": 0.0, + "source": "manual", + "type": "points", + "z_order": 0 + }, + { + "attributes": [ + { + "spec_id": 15, + "value": "j2 frame5 n1" + } + ], + "elements": [], + "frame": 12, + "group": 0, + "id": 177, + "label_id": 77, + "occluded": false, + "outside": false, + "points": [ + 16.791015625, + 32.8505859375, + 27.858705213058784, + 37.01258996859542, + 21.633141273523506, + 39.77950727505595 + ], + "rotation": 0.0, + "source": "manual", + "type": "points", + "z_order": 0 + }, + { + "attributes": [ + { + "spec_id": 15, + "value": "j3 frame1 n1" + } + ], + "elements": [], + "frame": 16, + "group": 0, + "id": 178, + "label_id": 77, + "occluded": false, + "outside": false, + "points": [ + 29.0498046875, + 14.2998046875, + 30.350000000002183, + 22.00000000000182, + 20.650000000003274, + 21.600000000002183, + 20.650000000003274, + 11.30000000000291 + ], + "rotation": 0.0, + "source": "manual", + "type": "polygon", + "z_order": 0 + }, + { + "attributes": [ + { + "spec_id": 15, + "value": "j3 frame2 n1" } ], "elements": [], - "frame": 0, + "frame": 17, "group": 0, - "id": 169, + "id": 179, "label_id": 77, "occluded": false, "outside": false, "points": [ - 19.650000000003274, - 13.100000000002183, - 31.850000000004002, - 18.900000000001455 + 51.2001953125, + 10.900390625, + 56.60000000000218, + 15.700000000002547, + 48.400000000003274, + 20.400000000003274 ], "rotation": 0.0, "source": "manual", - "type": "rectangle", + "type": "polygon", "z_order": 0 }, { "attributes": [ { "spec_id": 15, - "value": "j1 frame2 n1" + "value": "j3 frame5 n1" } ], "elements": [], - "frame": 1, + "frame": 20, "group": 0, - "id": 170, + "id": 180, "label_id": 77, "occluded": false, "outside": false, "points": [ - 18.650000000003274, - 10.500000000001819, - 28.650000000003274, - 15.200000000002547 + 37.2998046875, + 7.7001953125, + 42.400000000003274, + 11.900000000003274, + 35.80000000000291, + 17.200000000002547, + 28.400000000003274, + 8.80000000000291, + 37.400000000003274, + 12.100000000002183 ], "rotation": 0.0, "source": "manual", - "type": "rectangle", + "type": "polygon", "z_order": 0 }, { "attributes": [ { "spec_id": 15, - "value": "j1 frame2 n2" + "value": "j3 frame5 n2" } ], "elements": [], - "frame": 1, + "frame": 20, "group": 0, - "id": 171, + "id": 181, "label_id": 77, "occluded": false, "outside": false, "points": [ - 18.850000000002183, - 19.50000000000182, - 27.05000000000291, - 24.900000000001455 + 17.600000000002183, + 14.900000000003274, + 27.200000000002547, + 21.600000000004002 ], "rotation": 0.0, "source": "manual", @@ -8581,21 +11552,21 @@ "attributes": [ { "spec_id": 15, - "value": "j1 frame6 n1" + "value": "j3 frame6 n1" } ], "elements": [], - "frame": 5, + "frame": 21, "group": 0, - "id": 172, + "id": 182, "label_id": 77, "occluded": false, "outside": false, "points": [ - 26.25000000000182, - 16.50000000000182, - 40.95000000000255, - 23.900000000001455 + 43.15465253950242, + 24.59525439814206, + 55.395253809205315, + 35.071444674014856 ], "rotation": 0.0, "source": "manual", @@ -8606,21 +11577,21 @@ "attributes": [ { "spec_id": 15, - "value": "j2 frame1 n1" + "value": "j3 frame7 n1" } ], "elements": [], - "frame": 8, + "frame": 22, "group": 0, - "id": 173, + "id": 183, "label_id": 77, "occluded": false, "outside": false, "points": [ - 14.650000000003274, - 10.000000000001819, - 25.750000000003638, - 17.30000000000109 + 38.50000000000182, + 9.600000000002183, + 51.80000000000109, + 17.100000000002183 ], "rotation": 0.0, "source": "manual", @@ -8631,21 +11602,21 @@ "attributes": [ { "spec_id": 15, - "value": "j2 frame1 n2" + "value": "j3 frame7 n2" } ], "elements": [], - "frame": 8, + "frame": 22, "group": 0, - "id": 174, + "id": 184, "label_id": 77, "occluded": false, "outside": false, "points": [ - 30.350000000002183, - 18.700000000002547, - 43.05000000000291, - 26.400000000003274 + 52.10000000000218, + 17.30000000000291, + 59.400000000001455, + 21.500000000003638 ], "rotation": 0.0, "source": "manual", @@ -8656,24 +11627,24 @@ "attributes": [ { "spec_id": 15, - "value": "j2 frame2 n1" + "value": "gt frame1 n1" } ], "elements": [], - "frame": 9, + "frame": 23, "group": 0, - "id": 175, + "id": 185, "label_id": 77, "occluded": false, "outside": false, "points": [ - 9.200000000002547, - 34.35000000000218, - 21.900000000003274, - 38.55000000000291 + 17.650000000003274, + 11.30000000000291, + 30.55000000000291, + 21.700000000002547 ], "rotation": 0.0, - "source": "manual", + "source": "Ground truth", "type": "rectangle", "z_order": 0 }, @@ -8681,408 +11652,1027 @@ "attributes": [ { "spec_id": 15, - "value": "j2 frame2 n2" + "value": "gt frame2 n2" } ], "elements": [], - "frame": 9, + "frame": 24, "group": 0, - "id": 176, + "id": 186, "label_id": 77, "occluded": false, "outside": false, "points": [ - 40.900390625, - 29.0498046875, - 48.80000000000291, - 30.350000000002183, - 45.10000000000218, - 39.25000000000182, - 45.70000000000255, - 24.450000000002547 + 18.850000000002183, + 12.000000000001819, + 25.850000000002183, + 19.50000000000182 ], "rotation": 0.0, - "source": "manual", - "type": "points", + "source": "Ground truth", + "type": "rectangle", "z_order": 0 }, { "attributes": [ { "spec_id": 15, - "value": "j2 frame5 n1" + "value": "gt frame2 n1" } ], "elements": [], - "frame": 12, + "frame": 24, "group": 0, - "id": 177, + "id": 187, "label_id": 77, "occluded": false, "outside": false, "points": [ - 16.791015625, - 32.8505859375, - 27.858705213058784, - 37.01258996859542, - 21.633141273523506, - 39.77950727505595 + 26.150000000003274, + 25.00000000000182, + 34.150000000003274, + 34.50000000000182 ], "rotation": 0.0, - "source": "manual", - "type": "points", + "source": "Ground truth", + "type": "rectangle", "z_order": 0 }, { "attributes": [ { "spec_id": 15, - "value": "j3 frame1 n1" + "value": "gt frame3 n1" } ], "elements": [], - "frame": 16, + "frame": 25, "group": 0, - "id": 178, + "id": 188, "label_id": 77, "occluded": false, "outside": false, "points": [ - 29.0498046875, - 14.2998046875, - 30.350000000002183, - 22.00000000000182, - 20.650000000003274, - 21.600000000002183, - 20.650000000003274, - 11.30000000000291 + 24.600000000002183, + 11.500000000001819, + 37.10000000000218, + 18.700000000002547 ], "rotation": 0.0, - "source": "manual", - "type": "polygon", + "source": "Ground truth", + "type": "rectangle", "z_order": 0 }, { "attributes": [ { "spec_id": 15, - "value": "j3 frame2 n1" + "value": "gt frame5 n1" } ], "elements": [], - "frame": 17, + "frame": 27, "group": 0, - "id": 179, + "id": 189, "label_id": 77, "occluded": false, "outside": false, "points": [ - 51.2001953125, - 10.900390625, - 56.60000000000218, - 15.700000000002547, - 48.400000000003274, - 20.400000000003274 + 17.863216443472993, + 36.43614886308387, + 41.266725327279346, + 42.765472201610464 ], "rotation": 0.0, - "source": "manual", - "type": "polygon", + "source": "Ground truth", + "type": "rectangle", "z_order": 0 }, { "attributes": [ { "spec_id": 15, - "value": "j3 frame5 n1" + "value": "gt frame5 n2" } ], "elements": [], - "frame": 20, + "frame": 27, "group": 0, - "id": 180, + "id": 190, "label_id": 77, "occluded": false, "outside": false, "points": [ - 37.2998046875, - 7.7001953125, - 42.400000000003274, - 11.900000000003274, - 35.80000000000291, - 17.200000000002547, - 28.400000000003274, - 8.80000000000291, - 37.400000000003274, - 12.100000000002183 + 34.349609375, + 52.806640625, + 27.086274131672326, + 63.1830161588623, + 40.229131337355284, + 67.44868033965395, + 48.87574792004307, + 59.03264019917333, + 45.53238950807099, + 53.3835173651496 + ], + "rotation": 0.0, + "source": "Ground truth", + "type": "polygon", + "z_order": 0 + } + ], + "tags": [], + "tracks": [], + "version": 0 + }, + "30": { + "shapes": [ + { + "attributes": [ + { + "spec_id": 17, + "value": "" + } + ], + "elements": [], + "frame": 0, + "group": 0, + "id": 240, + "label_id": 79, + "occluded": false, + "outside": false, + "points": [ + 96.42, + 78.7, + 145.41, + 134.61 ], "rotation": 0.0, - "source": "manual", - "type": "polygon", + "source": "consensus", + "type": "rectangle", "z_order": 0 }, { "attributes": [ { - "spec_id": 15, - "value": "j3 frame5 n2" + "spec_id": 16, + "value": "" } ], "elements": [], - "frame": 20, + "frame": 0, "group": 0, - "id": 181, - "label_id": 77, + "id": 241, + "label_id": 78, "occluded": false, "outside": false, "points": [ - 17.600000000002183, - 14.900000000003274, - 27.200000000002547, - 21.600000000004002 + 189.79, + 99.41, + 224.23, + 98.64, + 208.16, + 130.02 ], "rotation": 0.0, - "source": "manual", - "type": "rectangle", + "source": "consensus", + "type": "polygon", "z_order": 0 }, { "attributes": [ { - "spec_id": 15, - "value": "j3 frame6 n1" + "spec_id": 16, + "value": "" } ], "elements": [], - "frame": 21, + "frame": 0, "group": 0, - "id": 182, - "label_id": 77, + "id": 242, + "label_id": 78, "occluded": false, "outside": false, "points": [ - 43.15465253950242, - 24.59525439814206, - 55.395253809205315, - 35.071444674014856 + 179.45, + 160.21, + 229.95, + 141.91, + 212.38, + 167.15 ], "rotation": 0.0, - "source": "manual", - "type": "rectangle", + "source": "consensus", + "type": "polygon", "z_order": 0 }, { "attributes": [ { - "spec_id": 15, - "value": "j3 frame7 n1" + "spec_id": 16, + "value": "" } ], "elements": [], - "frame": 22, + "frame": 0, "group": 0, - "id": 183, - "label_id": 77, + "id": 243, + "label_id": 78, "occluded": false, "outside": false, "points": [ - 38.50000000000182, - 9.600000000002183, - 51.80000000000109, - 17.100000000002183 + 271.7, + 119.3, + 304.6, + 104.0, + 310.72, + 128.49, + 339.8, + 107.06 ], "rotation": 0.0, - "source": "manual", - "type": "rectangle", + "source": "consensus", + "type": "polyline", "z_order": 0 }, { "attributes": [ { - "spec_id": 15, - "value": "j3 frame7 n2" + "spec_id": 16, + "value": "" } ], "elements": [], - "frame": 22, + "frame": 0, "group": 0, - "id": 184, - "label_id": 77, + "id": 244, + "label_id": 78, "occluded": false, "outside": false, "points": [ - 52.10000000000218, - 17.30000000000291, - 59.400000000001455, - 21.500000000003638 + 288.53, + 139.6, + 307.66, + 164.85, + 336.75, + 143.42 ], "rotation": 0.0, - "source": "manual", - "type": "rectangle", + "source": "consensus", + "type": "polyline", "z_order": 0 }, { "attributes": [ { - "spec_id": 15, - "value": "gt frame1 n1" + "spec_id": 16, + "value": "" } ], "elements": [], - "frame": 23, + "frame": 0, "group": 0, - "id": 185, - "label_id": 77, + "id": 245, + "label_id": 78, "occluded": false, "outside": false, "points": [ - 17.650000000003274, - 11.30000000000291, - 30.55000000000291, - 21.700000000002547 + 277.04, + 68.03, + 290.82, + 38.95, + 307.65, + 68.03, + 322.2, + 40.48 ], "rotation": 0.0, - "source": "Ground truth", - "type": "rectangle", + "source": "consensus", + "type": "points", "z_order": 0 }, { "attributes": [ { - "spec_id": 15, - "value": "gt frame2 n2" + "spec_id": 17, + "value": "" } ], "elements": [], - "frame": 24, + "frame": 0, "group": 0, - "id": 186, - "label_id": 77, + "id": 246, + "label_id": 79, "occluded": false, "outside": false, "points": [ - 18.850000000002183, - 12.000000000001819, - 25.850000000002183, - 19.50000000000182 + 216.21, + 45.84, + 246.82, + 45.85, + 261.36, + 18.29 ], "rotation": 0.0, - "source": "Ground truth", - "type": "rectangle", + "source": "consensus", + "type": "points", "z_order": 0 }, { "attributes": [ { - "spec_id": 15, - "value": "gt frame2 n1" + "spec_id": 16, + "value": "" } ], "elements": [], - "frame": 24, + "frame": 0, "group": 0, - "id": 187, - "label_id": 77, + "id": 247, + "label_id": 78, "occluded": false, "outside": false, "points": [ - 26.150000000003274, - 25.00000000000182, - 34.150000000003274, - 34.50000000000182 + 17.0, + 9.0, + 32.0, + 13.0, + 28.0, + 17.0, + 25.0, + 19.0, + 23.0, + 22.0, + 19.0, + 25.0, + 17.0, + 27.0, + 16.0, + 27.0, + 15.0, + 29.0, + 13.0, + 31.0, + 12.0, + 31.0, + 11.0, + 33.0, + 9.0, + 35.0, + 8.0, + 35.0, + 7.0, + 37.0, + 6.0, + 37.0, + 5.0, + 39.0, + 4.0, + 39.0, + 4.0, + 39.0, + 3.0, + 41.0, + 2.0, + 41.0, + 2.0, + 41.0, + 2.0, + 41.0, + 2.0, + 41.0, + 1.0, + 559.0, + 1.0, + 41.0, + 2.0, + 41.0, + 2.0, + 41.0, + 2.0, + 41.0, + 3.0, + 39.0, + 4.0, + 39.0, + 4.0, + 39.0, + 4.0, + 39.0, + 5.0, + 37.0, + 6.0, + 37.0, + 7.0, + 35.0, + 8.0, + 35.0, + 9.0, + 33.0, + 11.0, + 31.0, + 12.0, + 31.0, + 13.0, + 29.0, + 15.0, + 27.0, + 16.0, + 27.0, + 17.0, + 25.0, + 20.0, + 22.0, + 22.0, + 19.0, + 25.0, + 17.0, + 28.0, + 13.0, + 32.0, + 9.0, + 17.0, + 379.0, + 78.0, + 421.0, + 138.0 ], "rotation": 0.0, - "source": "Ground truth", - "type": "rectangle", + "source": "consensus", + "type": "mask", "z_order": 0 }, { "attributes": [ { - "spec_id": 15, - "value": "gt frame3 n1" + "spec_id": 16, + "value": "" } ], "elements": [], - "frame": 25, + "frame": 0, "group": 0, - "id": 188, - "label_id": 77, + "id": 248, + "label_id": 78, "occluded": false, "outside": false, "points": [ - 24.600000000002183, - 11.500000000001819, - 37.10000000000218, - 18.700000000002547 + 127.0, + 2.0, + 151.0, + 9.0, + 143.0, + 17.0, + 136.0, + 24.0, + 129.0, + 31.0, + 122.0, + 38.0, + 115.0, + 46.0, + 56.0, + 4.0, + 49.0, + 51.0, + 53.0, + 7.0, + 48.0, + 52.0, + 51.0, + 10.0, + 47.0, + 52.0, + 48.0, + 14.0, + 46.0, + 52.0, + 46.0, + 17.0, + 45.0, + 52.0, + 43.0, + 21.0, + 44.0, + 52.0, + 41.0, + 24.0, + 43.0, + 52.0, + 38.0, + 27.0, + 43.0, + 52.0, + 36.0, + 30.0, + 42.0, + 52.0, + 33.0, + 34.0, + 42.0, + 51.0, + 31.0, + 37.0, + 41.0, + 51.0, + 28.0, + 41.0, + 40.0, + 52.0, + 25.0, + 43.0, + 40.0, + 52.0, + 23.0, + 46.0, + 39.0, + 52.0, + 20.0, + 50.0, + 38.0, + 52.0, + 18.0, + 53.0, + 37.0, + 52.0, + 15.0, + 57.0, + 36.0, + 52.0, + 13.0, + 60.0, + 35.0, + 52.0, + 10.0, + 63.0, + 35.0, + 52.0, + 8.0, + 66.0, + 34.0, + 52.0, + 5.0, + 70.0, + 34.0, + 51.0, + 3.0, + 73.0, + 33.0, + 51.0, + 1.0, + 76.0, + 32.0, + 48.0, + 2.0, + 79.0, + 31.0, + 46.0, + 4.0, + 79.0, + 31.0, + 44.0, + 5.0, + 81.0, + 30.0, + 41.0, + 8.0, + 82.0, + 29.0, + 39.0, + 9.0, + 84.0, + 28.0, + 37.0, + 10.0, + 86.0, + 27.0, + 34.0, + 13.0, + 87.0, + 26.0, + 32.0, + 14.0, + 88.0, + 26.0, + 30.0, + 16.0, + 51.0, + 1.0, + 34.0, + 29.0, + 26.0, + 18.0, + 50.0, + 3.0, + 33.0, + 30.0, + 24.0, + 19.0, + 49.0, + 4.0, + 32.0, + 32.0, + 22.0, + 21.0, + 47.0, + 6.0, + 30.0, + 34.0, + 19.0, + 23.0, + 46.0, + 7.0, + 30.0, + 35.0, + 17.0, + 25.0, + 44.0, + 9.0, + 28.0, + 37.0, + 15.0, + 26.0, + 43.0, + 10.0, + 27.0, + 39.0, + 12.0, + 28.0, + 42.0, + 12.0, + 26.0, + 40.0, + 10.0, + 30.0, + 41.0, + 12.0, + 25.0, + 42.0, + 8.0, + 31.0, + 40.0, + 14.0, + 24.0, + 43.0, + 5.0, + 34.0, + 38.0, + 15.0, + 23.0, + 46.0, + 2.0, + 35.0, + 37.0, + 17.0, + 21.0, + 84.0, + 36.0, + 18.0, + 21.0, + 85.0, + 34.0, + 20.0, + 19.0, + 141.0, + 18.0, + 141.0, + 17.0, + 143.0, + 15.0, + 144.0, + 15.0, + 145.0, + 13.0, + 146.0, + 12.0, + 148.0, + 11.0, + 148.0, + 10.0, + 150.0, + 9.0, + 150.0, + 8.0, + 152.0, + 6.0, + 153.0, + 6.0, + 154.0, + 4.0, + 155.0, + 4.0, + 156.0, + 2.0, + 157.0, + 1.0, + 87.0, + 375.0, + 13.0, + 533.0, + 80.0 ], "rotation": 0.0, - "source": "Ground truth", - "type": "rectangle", + "source": "consensus", + "type": "mask", "z_order": 0 }, { "attributes": [ { - "spec_id": 15, - "value": "gt frame5 n1" + "spec_id": 17, + "value": "" } ], "elements": [], - "frame": 27, + "frame": 0, "group": 0, - "id": 189, - "label_id": 77, + "id": 249, + "label_id": 79, "occluded": false, "outside": false, "points": [ - 17.863216443472993, - 36.43614886308387, - 41.266725327279346, - 42.765472201610464 + 127.0, + 3.0, + 151.0, + 10.0, + 143.0, + 18.0, + 136.0, + 25.0, + 129.0, + 32.0, + 122.0, + 39.0, + 115.0, + 47.0, + 109.0, + 52.0, + 108.0, + 53.0, + 107.0, + 54.0, + 107.0, + 54.0, + 107.0, + 54.0, + 107.0, + 54.0, + 107.0, + 54.0, + 38.0, + 1.0, + 68.0, + 54.0, + 36.0, + 3.0, + 68.0, + 54.0, + 33.0, + 5.0, + 69.0, + 54.0, + 31.0, + 6.0, + 71.0, + 53.0, + 28.0, + 9.0, + 71.0, + 54.0, + 25.0, + 10.0, + 72.0, + 54.0, + 23.0, + 11.0, + 73.0, + 54.0, + 20.0, + 14.0, + 35.0, + 2.0, + 36.0, + 54.0, + 18.0, + 15.0, + 34.0, + 5.0, + 35.0, + 54.0, + 15.0, + 17.0, + 34.0, + 7.0, + 34.0, + 54.0, + 13.0, + 19.0, + 32.0, + 10.0, + 33.0, + 54.0, + 10.0, + 21.0, + 32.0, + 11.0, + 33.0, + 54.0, + 8.0, + 22.0, + 31.0, + 14.0, + 32.0, + 54.0, + 5.0, + 25.0, + 29.0, + 17.0, + 31.0, + 54.0, + 3.0, + 26.0, + 29.0, + 19.0, + 31.0, + 53.0, + 1.0, + 27.0, + 28.0, + 22.0, + 30.0, + 81.0, + 26.0, + 25.0, + 29.0, + 49.0, + 2.0, + 29.0, + 26.0, + 26.0, + 29.0, + 47.0, + 4.0, + 29.0, + 24.0, + 29.0, + 28.0, + 45.0, + 5.0, + 29.0, + 24.0, + 31.0, + 27.0, + 42.0, + 8.0, + 28.0, + 24.0, + 33.0, + 26.0, + 40.0, + 9.0, + 29.0, + 23.0, + 35.0, + 25.0, + 38.0, + 10.0, + 29.0, + 23.0, + 37.0, + 24.0, + 35.0, + 13.0, + 28.0, + 23.0, + 38.0, + 24.0, + 33.0, + 14.0, + 29.0, + 22.0, + 38.0, + 25.0, + 31.0, + 16.0, + 28.0, + 23.0, + 36.0, + 28.0, + 27.0, + 18.0, + 28.0, + 25.0, + 34.0, + 29.0, + 25.0, + 19.0, + 29.0, + 24.0, + 33.0, + 31.0, + 23.0, + 21.0, + 28.0, + 25.0, + 31.0, + 33.0, + 20.0, + 23.0, + 28.0, + 25.0, + 31.0, + 34.0, + 18.0, + 25.0, + 28.0, + 25.0, + 29.0, + 36.0, + 16.0, + 26.0, + 28.0, + 25.0, + 28.0, + 38.0, + 13.0, + 28.0, + 28.0, + 26.0, + 27.0, + 39.0, + 11.0, + 30.0, + 28.0, + 25.0, + 26.0, + 41.0, + 9.0, + 31.0, + 28.0, + 26.0, + 25.0, + 42.0, + 6.0, + 34.0, + 27.0, + 26.0, + 24.0, + 45.0, + 3.0, + 35.0, + 28.0, + 26.0, + 22.0, + 84.0, + 28.0, + 26.0, + 22.0, + 85.0, + 28.0, + 26.0, + 20.0, + 141.0, + 19.0, + 141.0, + 18.0, + 143.0, + 16.0, + 144.0, + 16.0, + 145.0, + 14.0, + 146.0, + 13.0, + 148.0, + 12.0, + 148.0, + 11.0, + 150.0, + 10.0, + 150.0, + 9.0, + 152.0, + 7.0, + 153.0, + 7.0, + 154.0, + 5.0, + 155.0, + 5.0, + 156.0, + 3.0, + 157.0, + 2.0, + 87.0, + 616.0, + 13.0, + 775.0, + 81.0 ], "rotation": 0.0, - "source": "Ground truth", - "type": "rectangle", + "source": "consensus", + "type": "mask", "z_order": 0 }, - { - "attributes": [ - { - "spec_id": 15, - "value": "gt frame5 n2" - } - ], - "elements": [], - "frame": 27, - "group": 0, - "id": 190, - "label_id": 77, - "occluded": false, - "outside": false, - "points": [ - 34.349609375, - 52.806640625, - 27.086274131672326, - 63.1830161588623, - 40.229131337355284, - 67.44868033965395, - 48.87574792004307, - 59.03264019917333, - 45.53238950807099, - 53.3835173651496 - ], - "rotation": 0.0, - "source": "Ground truth", - "type": "polygon", - "z_order": 0 - } - ], - "tags": [], - "tracks": [], - "version": 0 - }, - "30": { - "shapes": [ { "attributes": [ { @@ -9108,31 +12698,6 @@ "type": "rectangle", "z_order": 0 }, - { - "attributes": [ - { - "spec_id": 16, - "value": "j1 frame 1 (32) ann 1" - } - ], - "elements": [], - "frame": 0, - "group": 0, - "id": 199, - "label_id": 78, - "occluded": false, - "outside": false, - "points": [ - 260.20437641345416, - 64.96839991694105, - 339.7988160670684, - 103.23495744271713 - ], - "rotation": 0.0, - "source": "manual", - "type": "rectangle", - "z_order": 0 - }, { "attributes": [ { @@ -9209,6 +12774,26 @@ "z_order": 0 } ], + "tags": [ + { + "attributes": [ + { + "spec_id": 16, + "value": "" + } + ], + "frame": 0, + "group": 0, + "id": 13, + "label_id": 78, + "source": "manual" + } + ], + "tracks": [], + "version": 0 + }, + "31": { + "shapes": [], "tags": [], "tracks": [], "version": 0 diff --git a/tests/python/shared/assets/consensus_settings.json b/tests/python/shared/assets/consensus_settings.json new file mode 100644 index 000000000000..62a2727b7d2c --- /dev/null +++ b/tests/python/shared/assets/consensus_settings.json @@ -0,0 +1,19 @@ +{ + "count": 2, + "next": null, + "previous": null, + "results": [ + { + "id": 1, + "iou_threshold": 0.4, + "quorum": 0.5, + "task_id": 30 + }, + { + "id": 2, + "iou_threshold": 0.4, + "quorum": 0.5, + "task_id": 31 + } + ] +} \ No newline at end of file diff --git a/tests/python/shared/assets/cvat_db/cvat_data.tar.bz2 b/tests/python/shared/assets/cvat_db/cvat_data.tar.bz2 index a4c28196e9a5..c125c975d98c 100644 Binary files a/tests/python/shared/assets/cvat_db/cvat_data.tar.bz2 and b/tests/python/shared/assets/cvat_db/cvat_data.tar.bz2 differ diff --git a/tests/python/shared/assets/cvat_db/data.json b/tests/python/shared/assets/cvat_db/data.json index f59fba333657..dc1c35d7db04 100644 --- a/tests/python/shared/assets/cvat_db/data.json +++ b/tests/python/shared/assets/cvat_db/data.json @@ -1398,6 +1398,25 @@ "deleted_frames": "[]" } }, +{ + "model": "engine.data", + "pk": 30, + "fields": { + "chunk_size": 72, + "size": 3, + "image_quality": 70, + "start_frame": 0, + "stop_frame": 2, + "frame_filter": "", + "compressed_chunk_type": "imageset", + "original_chunk_type": "imageset", + "storage_method": "cache", + "storage": "local", + "cloud_storage": null, + "sorting_method": "lexicographical", + "deleted_frames": "[]" + } +}, { "model": "engine.video", "pk": 1, @@ -3758,6 +3777,45 @@ "real_frame": 0 } }, +{ + "model": "engine.image", + "pk": 533, + "fields": { + "data": 30, + "path": "30.png", + "frame": 0, + "width": 810, + "height": 399, + "is_placeholder": false, + "real_frame": 0 + } +}, +{ + "model": "engine.image", + "pk": 534, + "fields": { + "data": 30, + "path": "31.png", + "frame": 1, + "width": 916, + "height": 158, + "is_placeholder": false, + "real_frame": 0 + } +}, +{ + "model": "engine.image", + "pk": 535, + "fields": { + "data": 30, + "path": "32.png", + "frame": 2, + "width": 936, + "height": 182, + "is_placeholder": false, + "real_frame": 0 + } +}, { "model": "engine.project", "pk": 1, @@ -4754,6 +4812,33 @@ "consensus_replicas": 3 } }, +{ + "model": "engine.task", + "pk": 31, + "fields": { + "created_date": "2025-01-27T16:42:31.302Z", + "updated_date": "2025-01-27T16:43:36.188Z", + "project": null, + "name": "sandbox task with consensus", + "mode": "annotation", + "owner": [ + "user1" + ], + "assignee": null, + "assignee_updated_date": null, + "bug_tracker": "", + "overlap": 0, + "segment_size": 3, + "status": "annotation", + "data": 30, + "dimension": "2d", + "subset": "", + "organization": null, + "source_storage": 53, + "target_storage": 54, + "consensus_replicas": 2 + } +}, { "model": "engine.clientfile", "pk": 131, @@ -5730,6 +5815,30 @@ "file": "/home/django/data/data/29/raw/35.png" } }, +{ + "model": "engine.clientfile", + "pk": 470, + "fields": { + "data": 30, + "file": "/home/django/data/data/30/raw/30.png" + } +}, +{ + "model": "engine.clientfile", + "pk": 471, + "fields": { + "data": 30, + "file": "/home/django/data/data/30/raw/31.png" + } +}, +{ + "model": "engine.clientfile", + "pk": 472, + "fields": { + "data": 30, + "file": "/home/django/data/data/30/raw/32.png" + } +}, { "model": "engine.relatedfile", "pk": 1, @@ -6185,6 +6294,18 @@ "frames": "[]" } }, +{ + "model": "engine.segment", + "pk": 45, + "fields": { + "task": 31, + "start_frame": 0, + "stop_frame": 2, + "chunks_updated_date": "2025-01-27T16:42:31.655Z", + "type": "range", + "frames": "[]" + } +}, { "model": "engine.job", "pk": 2, @@ -6744,13 +6865,13 @@ "pk": 42, "fields": { "created_date": "2025-01-14T17:19:04.062Z", - "updated_date": "2025-01-14T17:25:16.549Z", + "updated_date": "2025-02-03T15:25:42.655Z", "segment": 42, "assignee": null, "assignee_updated_date": null, "status": "annotation", "stage": "annotation", - "state": "in progress", + "state": "completed", "type": "annotation", "parent_job": null } @@ -6760,7 +6881,7 @@ "pk": 43, "fields": { "created_date": "2025-01-14T17:19:04.077Z", - "updated_date": "2025-01-14T17:26:01.233Z", + "updated_date": "2025-02-03T15:25:32.413Z", "segment": 42, "assignee": null, "assignee_updated_date": null, @@ -6792,13 +6913,13 @@ "pk": 45, "fields": { "created_date": "2025-01-14T17:19:04.086Z", - "updated_date": "2025-01-14T17:19:51.388Z", + "updated_date": "2025-02-03T15:22:52.241Z", "segment": 42, "assignee": null, "assignee_updated_date": null, "status": "annotation", "stage": "annotation", - "state": "new", + "state": "in progress", "type": "consensus_replica", "parent_job": 42 } @@ -6883,6 +7004,54 @@ "parent_job": null } }, +{ + "model": "engine.job", + "pk": 51, + "fields": { + "created_date": "2025-01-27T16:42:31.660Z", + "updated_date": "2025-01-27T16:42:31.660Z", + "segment": 45, + "assignee": null, + "assignee_updated_date": null, + "status": "annotation", + "stage": "annotation", + "state": "new", + "type": "annotation", + "parent_job": null + } +}, +{ + "model": "engine.job", + "pk": 52, + "fields": { + "created_date": "2025-01-27T16:42:31.672Z", + "updated_date": "2025-01-27T16:43:07.696Z", + "segment": 45, + "assignee": null, + "assignee_updated_date": null, + "status": "annotation", + "stage": "annotation", + "state": "in progress", + "type": "consensus_replica", + "parent_job": 51 + } +}, +{ + "model": "engine.job", + "pk": 53, + "fields": { + "created_date": "2025-01-27T16:42:31.676Z", + "updated_date": "2025-01-27T16:43:47.462Z", + "segment": 45, + "assignee": null, + "assignee_updated_date": null, + "status": "annotation", + "stage": "annotation", + "state": "completed", + "type": "consensus_replica", + "parent_job": 51 + } +}, { "model": "engine.label", "pk": 3, @@ -7747,6 +7916,18 @@ "parent": null } }, +{ + "model": "engine.label", + "pk": 80, + "fields": { + "task": 31, + "project": null, + "name": "cat", + "color": "#6080c0", + "type": "any", + "parent": null + } +}, { "model": "engine.skeleton", "pk": 1, @@ -7995,6 +8176,18 @@ "values": "" } }, +{ + "model": "engine.attributespec", + "pk": 18, + "fields": { + "label": 80, + "name": "src", + "mutable": false, + "input_type": "text", + "default_value": "", + "values": "" + } +}, { "model": "engine.labeledimage", "pk": 1, @@ -8061,6 +8254,50 @@ "source": "manual" } }, +{ + "model": "engine.labeledimage", + "pk": 9, + "fields": { + "job": 45, + "label": 78, + "frame": 0, + "group": 0, + "source": "manual" + } +}, +{ + "model": "engine.labeledimage", + "pk": 10, + "fields": { + "job": 44, + "label": 78, + "frame": 0, + "group": 0, + "source": "manual" + } +}, +{ + "model": "engine.labeledimage", + "pk": 11, + "fields": { + "job": 43, + "label": 78, + "frame": 0, + "group": 0, + "source": "manual" + } +}, +{ + "model": "engine.labeledimage", + "pk": 13, + "fields": { + "job": 42, + "label": 78, + "frame": 0, + "group": 0, + "source": "manual" + } +}, { "model": "engine.labeledimageattributeval", "pk": 1, @@ -8097,6 +8334,42 @@ "image": 8 } }, +{ + "model": "engine.labeledimageattributeval", + "pk": 5, + "fields": { + "spec": 16, + "value": "", + "image": 9 + } +}, +{ + "model": "engine.labeledimageattributeval", + "pk": 6, + "fields": { + "spec": 16, + "value": "", + "image": 10 + } +}, +{ + "model": "engine.labeledimageattributeval", + "pk": 7, + "fields": { + "spec": 16, + "value": "", + "image": 11 + } +}, +{ + "model": "engine.labeledimageattributeval", + "pk": 9, + "fields": { + "spec": 16, + "value": "", + "image": 13 + } +}, { "model": "engine.labeledshape", "pk": 1, @@ -11213,10 +11486,10 @@ }, { "model": "engine.labeledshape", - "pk": 199, + "pk": 201, "fields": { - "job": 42, - "label": 78, + "job": 52, + "label": 80, "frame": 0, "group": 0, "source": "manual", @@ -11224,17 +11497,17 @@ "occluded": false, "outside": false, "z_order": 0, - "points": "260.20437641345416,64.96839991694105,339.7988160670684,103.23495744271713", + "points": "295.39537490125076,129.95137884321593,379.50821517094846,194.85735007494986", "rotation": 0.0, "parent": null } }, { "model": "engine.labeledshape", - "pk": 200, + "pk": 202, "fields": { - "job": 43, - "label": 78, + "job": 53, + "label": 80, "frame": 0, "group": 0, "source": "manual", @@ -11242,1440 +11515,2412 @@ "occluded": false, "outside": false, "z_order": 0, - "points": "236.47911074747208,63.43773761591001,335.20682916397527,123.1335673561207", + "points": "339.7698654372325,132.6006021587964,397.3904725511202,175.6504810369879", "rotation": 0.0, "parent": null } }, { - "model": "engine.labeledshapeattributeval", - "pk": 1, + "model": "engine.labeledshape", + "pk": 203, "fields": { - "spec": 2, - "value": "white", - "shape": 36 + "job": 53, + "label": 80, + "frame": 0, + "group": 0, + "source": "manual", + "type": "rectangle", + "occluded": false, + "outside": false, + "z_order": 0, + "points": "529.1893325012752,65.70771344037712,597.4068328774865,119.35448558089229", + "rotation": 0.0, + "parent": null } }, { - "model": "engine.labeledshapeattributeval", - "pk": 2, + "model": "engine.labeledshape", + "pk": 204, + "fields": { + "job": 45, + "label": 79, + "frame": 0, + "group": 0, + "source": "manual", + "type": "rectangle", + "occluded": false, + "outside": false, + "z_order": 0, + "points": "96.42351020313072,99.40830169014043,145.4047038361241,134.61353461385443", + "rotation": 0.0, + "parent": null + } +}, +{ + "model": "engine.labeledshape", + "pk": 205, + "fields": { + "job": 45, + "label": 78, + "frame": 0, + "group": 0, + "source": "manual", + "type": "polygon", + "occluded": false, + "outside": false, + "z_order": 0, + "points": "189.7939453125,99.408203125,224.23381233922373,98.64297053962582,208.16185817839687,130.0215477107613", + "rotation": 0.0, + "parent": null + } +}, +{ + "model": "engine.labeledshape", + "pk": 206, + "fields": { + "job": 45, + "label": 78, + "frame": 0, + "group": 0, + "source": "manual", + "type": "polyline", + "occluded": false, + "outside": false, + "z_order": 0, + "points": "263.265625,116.2451171875,291.58295358458963,103.23495744271895,310.7162323474786,128.49088540973025,339.7988160670684,107.06161319529565", + "rotation": 0.0, + "parent": null + } +}, +{ + "model": "engine.labeledshape", + "pk": 207, + "fields": { + "job": 45, + "label": 78, + "frame": 0, + "group": 0, + "source": "manual", + "type": "points", + "occluded": false, + "outside": false, + "z_order": 0, + "points": "277.0419921875,68.029296875,290.817622434075,38.94714079941332,307.6549077454165,68.02972451900314,322.1961996052123,40.47780310044436", + "rotation": 0.0, + "parent": null + } +}, +{ + "model": "engine.labeledshape", + "pk": 208, + "fields": { + "job": 45, + "label": 78, + "frame": 0, + "group": 0, + "source": "manual", + "type": "ellipse", + "occluded": false, + "outside": false, + "z_order": 0, + "points": "399.884765625,117.7861328125,420.931640625,97.1220703125", + "rotation": 0.0, + "parent": null + } +}, +{ + "model": "engine.labeledshape", + "pk": 209, + "fields": { + "job": 45, + "label": 78, + "frame": 0, + "group": 0, + "source": "manual", + "type": "cuboid", + "occluded": false, + "outside": false, + "z_order": 0, + "points": "502.049255907923,93.39047190463862,502.049255907923,137.6748346606255,545.6731314873086,93.29541089258055,545.6731314873086,137.6748346606455,571.4791182468543,81.07113365704572,571.4791182468543,124.59332935967359,528.6418953937628,81.1900421050359,528.6418953937628,124.62080992126721", + "rotation": 0.0, + "parent": null + } +}, +{ + "model": "engine.labeledshape", + "pk": 210, + "fields": { + "job": 45, + "label": 78, + "frame": 0, + "group": 0, + "source": "manual", + "type": "mask", + "occluded": false, + "outside": false, + "z_order": 0, + "points": "127,2,151,9,143,17,136,24,129,31,122,38,115,46,56,4,49,51,53,7,48,52,51,10,47,52,48,14,46,52,46,17,45,52,43,21,44,52,41,24,43,52,38,27,43,52,36,30,42,52,33,34,42,51,31,37,41,51,28,41,40,52,25,43,40,52,23,46,39,52,20,50,38,52,18,53,37,52,15,57,36,52,13,60,35,52,10,63,35,52,8,66,34,52,5,70,34,51,3,73,33,51,1,76,32,48,2,79,31,46,4,79,31,44,5,81,30,41,8,82,29,39,9,84,28,37,10,86,27,34,13,87,26,32,14,88,26,30,16,51,1,34,29,26,18,50,3,33,30,24,19,49,4,32,32,22,21,47,6,30,34,19,23,46,7,30,35,17,25,44,9,28,37,15,26,43,10,27,39,12,28,42,12,26,40,10,30,41,12,25,42,8,31,40,14,24,43,5,34,38,15,23,46,2,35,37,17,21,84,36,18,21,85,34,20,19,141,18,141,17,143,15,144,15,145,13,146,12,148,11,148,10,150,9,150,8,152,6,153,6,154,4,155,4,156,2,157,1,87,375,13,533,80", + "rotation": 0.0, + "parent": null + } +}, +{ + "model": "engine.labeledshape", + "pk": 211, + "fields": { + "job": 44, + "label": 79, + "frame": 0, + "group": 0, + "source": "manual", + "type": "rectangle", + "occluded": false, + "outside": false, + "z_order": 0, + "points": "96.42351020313072,99.40830169014043,145.4047038361241,134.61353461385443", + "rotation": 0.0, + "parent": null + } +}, +{ + "model": "engine.labeledshape", + "pk": 212, + "fields": { + "job": 44, + "label": 78, + "frame": 0, + "group": 0, + "source": "manual", + "type": "polygon", + "occluded": false, + "outside": false, + "z_order": 0, + "points": "189.7939453125,99.408203125,219.60000000000036,108.60000000000036,208.16185817839687,130.0215477107613", + "rotation": 0.0, + "parent": null + } +}, +{ + "model": "engine.labeledshape", + "pk": 213, + "fields": { + "job": 44, + "label": 78, + "frame": 0, + "group": 0, + "source": "manual", + "type": "polyline", + "occluded": false, + "outside": false, + "z_order": 0, + "points": "267.90000000000146,123.90000000000146,291.58295358458963,103.23495744271895,310.7162323474786,128.49088540973025,323,103.20000000000073", + "rotation": 0.0, + "parent": null + } +}, +{ + "model": "engine.labeledshape", + "pk": 214, + "fields": { + "job": 44, + "label": 78, + "frame": 0, + "group": 0, + "source": "manual", + "type": "points", + "occluded": false, + "outside": false, + "z_order": 0, + "points": "277.0419921875,68.029296875,290.817622434075,38.94714079941332,322.1961996052123,40.47780310044436", + "rotation": 0.0, + "parent": null + } +}, +{ + "model": "engine.labeledshape", + "pk": 215, + "fields": { + "job": 44, + "label": 78, + "frame": 0, + "group": 0, + "source": "manual", + "type": "ellipse", + "occluded": false, + "outside": false, + "z_order": 0, + "points": "412.1189453125007,126.21103515624964,445.40000000000146,96.72207031249854", + "rotation": 0.0, + "parent": null + } +}, +{ + "model": "engine.labeledshape", + "pk": 216, + "fields": { + "job": 44, + "label": 78, + "frame": 0, + "group": 0, + "source": "manual", + "type": "cuboid", + "occluded": false, + "outside": false, + "z_order": 0, + "points": "502.049255907923,93.39047190463316,502.049255907923,152.18125991462875,559.4494986748086,93.26539073985441,559.4494986748086,152.22223330696215,585.0063249908471,81.0335845571608,585.0063249908471,138.85086779521043,528.6418953937846,81.19004210502862,528.6418953937846,138.84761929236265", + "rotation": 0.0, + "parent": null + } +}, +{ + "model": "engine.labeledshape", + "pk": 217, + "fields": { + "job": 44, + "label": 78, + "frame": 0, + "group": 0, + "source": "manual", + "type": "mask", + "occluded": false, + "outside": false, + "z_order": 0, + "points": "127,2,151,9,143,17,136,24,129,31,122,38,115,46,56,4,49,51,53,7,48,52,51,10,47,52,48,14,46,52,46,17,45,52,43,21,44,52,41,24,43,52,38,27,43,52,36,30,42,52,33,34,42,51,31,37,41,51,28,41,40,52,25,43,40,52,23,46,39,52,20,50,38,52,18,53,37,52,15,57,36,52,13,60,35,52,10,63,35,52,8,66,34,52,5,70,34,51,3,73,33,51,1,76,32,48,2,79,31,46,4,79,31,44,5,81,30,41,8,82,29,39,9,84,28,37,10,86,27,34,13,87,26,32,14,88,26,30,16,51,1,34,29,26,18,50,3,33,30,24,19,49,4,32,32,22,21,47,6,30,34,19,23,46,7,30,35,17,25,44,9,28,37,15,26,43,10,27,39,12,28,42,12,26,40,10,30,41,12,25,42,8,31,40,14,24,43,5,34,38,15,23,46,2,35,37,17,21,84,36,18,21,85,34,20,19,141,18,141,17,143,15,144,15,145,13,146,12,148,11,148,10,150,9,150,8,152,6,153,6,154,4,155,4,156,2,157,1,87,375,13,533,80", + "rotation": 0.0, + "parent": null + } +}, +{ + "model": "engine.labeledshape", + "pk": 218, + "fields": { + "job": 43, + "label": 79, + "frame": 0, + "group": 0, + "source": "manual", + "type": "rectangle", + "occluded": false, + "outside": false, + "z_order": 0, + "points": "96.423828125,78.70000000000073,145.40502175799338,134.6134376525879", + "rotation": 0.0, + "parent": null + } +}, +{ + "model": "engine.labeledshape", + "pk": 219, + "fields": { + "job": 43, + "label": 78, + "frame": 0, + "group": 0, + "source": "manual", + "type": "polygon", + "occluded": false, + "outside": false, + "z_order": 0, + "points": "189,113.20000000000073,224.23381233922373,98.64297053962582,208.16185817839687,130.0215477107613", + "rotation": 0.0, + "parent": null + } +}, +{ + "model": "engine.labeledshape", + "pk": 220, + "fields": { + "job": 43, + "label": 78, + "frame": 0, + "group": 0, + "source": "manual", + "type": "polyline", + "occluded": false, + "outside": false, + "z_order": 0, + "points": "271.7000000000007,119.30000000000109,304.60000000000036,104,310.7162323474786,128.49088540973025,339.7988160670684,107.06161319529565", + "rotation": 0.0, + "parent": null + } +}, +{ + "model": "engine.labeledshape", + "pk": 221, + "fields": { + "job": 43, + "label": 78, + "frame": 0, + "group": 0, + "source": "manual", + "type": "points", + "occluded": false, + "outside": false, + "z_order": 0, + "points": "277.0419921875,68.029296875,290.817622434075,38.94714079941332,307.6549077454165,68.02972451900314", + "rotation": 0.0, + "parent": null + } +}, +{ + "model": "engine.labeledshape", + "pk": 222, + "fields": { + "job": 43, + "label": 78, + "frame": 0, + "group": 0, + "source": "manual", + "type": "ellipse", + "occluded": false, + "outside": false, + "z_order": 0, + "points": "399.884765625,108.27509765625109,420.931640625,78.45000000000073", + "rotation": 0.0, + "parent": null + } +}, +{ + "model": "engine.labeledshape", + "pk": 223, + "fields": { + "job": 43, + "label": 78, + "frame": 0, + "group": 0, + "source": "manual", + "type": "cuboid", + "occluded": false, + "outside": false, + "z_order": 0, + "points": "502.049255907923,101.79330723501334,502.049255907923,137.6748346606273,537.2541861748086,101.73114849558442,537.2541861748086,137.6748346606455,563.2122646078951,89.3489492958688,563.2122646078951,124.59863263951047,528.6418953937591,89.43091015684695,528.6418953937591,124.6208099212563", + "rotation": 0.0, + "parent": null + } +}, +{ + "model": "engine.labeledshape", + "pk": 224, + "fields": { + "job": 43, + "label": 78, + "frame": 0, + "group": 0, + "source": "manual", + "type": "mask", + "occluded": false, + "outside": false, + "z_order": 0, + "points": "127,2,151,9,143,17,136,24,129,31,122,38,115,46,56,4,49,51,53,7,48,52,51,10,47,52,48,14,46,52,46,17,45,52,43,21,44,52,41,24,43,52,38,27,43,52,36,30,42,52,33,34,42,51,31,37,41,51,28,41,40,52,25,43,40,52,23,46,39,52,20,50,38,52,18,53,37,52,15,57,36,52,13,60,35,52,10,63,35,52,8,66,34,52,5,70,34,51,3,73,33,51,1,76,32,48,2,79,31,46,4,79,31,44,5,81,30,41,8,82,29,39,9,84,28,37,10,86,27,34,13,87,26,32,14,88,26,30,16,51,1,34,29,26,18,50,3,33,30,24,19,49,4,32,32,22,21,47,6,30,34,19,23,46,7,30,35,17,25,44,9,28,37,15,26,43,10,27,39,12,28,42,12,26,40,10,30,41,12,25,42,8,31,40,14,24,43,5,34,38,15,23,46,2,35,37,17,21,84,36,18,21,85,34,20,19,141,18,141,17,143,15,144,15,145,13,146,12,148,11,148,10,150,9,150,8,152,6,153,6,154,4,155,4,156,2,157,1,87,375,13,533,80", + "rotation": 0.0, + "parent": null + } +}, +{ + "model": "engine.labeledshape", + "pk": 225, + "fields": { + "job": 45, + "label": 78, + "frame": 0, + "group": 0, + "source": "manual", + "type": "polygon", + "occluded": false, + "outside": false, + "z_order": 0, + "points": "179.44667968750036,160.21328125000036,229.94667968750036,141.9132812500011,212.37865505339687,167.1504539607613", + "rotation": 0.0, + "parent": null + } +}, +{ + "model": "engine.labeledshape", + "pk": 226, + "fields": { + "job": 45, + "label": 78, + "frame": 0, + "group": 0, + "source": "manual", + "type": "polyline", + "occluded": false, + "outside": false, + "z_order": 0, + "points": "288.52924264708963,139.59823869271895,307.6625214099786,164.85416665973025,336.7451051295684,143.42489444529565", + "rotation": 0.0, + "parent": null + } +}, +{ + "model": "engine.labeledshape", + "pk": 227, + "fields": { + "job": 45, + "label": 79, + "frame": 0, + "group": 0, + "source": "manual", + "type": "points", + "occluded": false, + "outside": false, + "z_order": 0, + "points": "216.205078125,45.8447265625,246.8179936829165,45.84515420650314,261.3592855427123,18.293232787944362", + "rotation": 0.0, + "parent": null + } +}, +{ + "model": "engine.labeledshape", + "pk": 228, + "fields": { + "job": 45, + "label": 79, + "frame": 0, + "group": 0, + "source": "manual", + "type": "mask", + "occluded": false, + "outside": false, + "z_order": 0, + "points": "127,3,151,10,143,18,136,25,129,32,122,39,115,47,109,52,108,53,107,54,107,54,107,54,107,54,107,54,38,1,68,54,36,3,68,54,33,5,69,54,31,6,71,53,28,9,71,54,25,10,72,54,23,11,73,54,20,14,35,2,36,54,18,15,34,5,35,54,15,17,34,7,34,54,13,19,32,10,33,54,10,21,32,11,33,54,8,22,31,14,32,54,5,25,29,17,31,54,3,26,29,19,31,53,1,27,28,22,30,81,26,25,29,49,2,29,26,26,29,47,4,29,24,29,28,45,5,29,24,31,27,42,8,28,24,33,26,40,9,29,23,35,25,38,10,29,23,37,24,35,13,28,23,38,24,33,14,29,22,38,25,31,16,28,23,36,28,27,18,28,25,34,29,25,19,29,24,33,31,23,21,28,25,31,33,20,23,28,25,31,34,18,25,28,25,29,36,16,26,28,25,28,38,13,28,28,26,27,39,11,30,28,25,26,41,9,31,28,26,25,42,6,34,27,26,24,45,3,35,28,26,22,84,28,26,22,85,28,26,20,141,19,141,18,143,16,144,16,145,14,146,13,148,12,148,11,150,10,150,9,152,7,153,7,154,5,155,5,156,3,157,2,87,616,13,775,81", + "rotation": 0.0, + "parent": null + } +}, +{ + "model": "engine.labeledshape", + "pk": 240, + "fields": { + "job": 42, + "label": 79, + "frame": 0, + "group": 0, + "source": "consensus", + "type": "rectangle", + "occluded": false, + "outside": false, + "z_order": 0, + "points": "96.42,78.7,145.41,134.61", + "rotation": 0.0, + "parent": null + } +}, +{ + "model": "engine.labeledshape", + "pk": 241, + "fields": { + "job": 42, + "label": 78, + "frame": 0, + "group": 0, + "source": "consensus", + "type": "polygon", + "occluded": false, + "outside": false, + "z_order": 0, + "points": "189.79,99.41,224.23,98.64,208.16,130.02", + "rotation": 0.0, + "parent": null + } +}, +{ + "model": "engine.labeledshape", + "pk": 242, + "fields": { + "job": 42, + "label": 78, + "frame": 0, + "group": 0, + "source": "consensus", + "type": "polygon", + "occluded": false, + "outside": false, + "z_order": 0, + "points": "179.45,160.21,229.95,141.91,212.38,167.15", + "rotation": 0.0, + "parent": null + } +}, +{ + "model": "engine.labeledshape", + "pk": 243, + "fields": { + "job": 42, + "label": 78, + "frame": 0, + "group": 0, + "source": "consensus", + "type": "polyline", + "occluded": false, + "outside": false, + "z_order": 0, + "points": "271.7,119.3,304.6,104.0,310.72,128.49,339.8,107.06", + "rotation": 0.0, + "parent": null + } +}, +{ + "model": "engine.labeledshape", + "pk": 244, + "fields": { + "job": 42, + "label": 78, + "frame": 0, + "group": 0, + "source": "consensus", + "type": "polyline", + "occluded": false, + "outside": false, + "z_order": 0, + "points": "288.53,139.6,307.66,164.85,336.75,143.42", + "rotation": 0.0, + "parent": null + } +}, +{ + "model": "engine.labeledshape", + "pk": 245, + "fields": { + "job": 42, + "label": 78, + "frame": 0, + "group": 0, + "source": "consensus", + "type": "points", + "occluded": false, + "outside": false, + "z_order": 0, + "points": "277.04,68.03,290.82,38.95,307.65,68.03,322.2,40.48", + "rotation": 0.0, + "parent": null + } +}, +{ + "model": "engine.labeledshape", + "pk": 246, + "fields": { + "job": 42, + "label": 79, + "frame": 0, + "group": 0, + "source": "consensus", + "type": "points", + "occluded": false, + "outside": false, + "z_order": 0, + "points": "216.21,45.84,246.82,45.85,261.36,18.29", + "rotation": 0.0, + "parent": null + } +}, +{ + "model": "engine.labeledshape", + "pk": 247, + "fields": { + "job": 42, + "label": 78, + "frame": 0, + "group": 0, + "source": "consensus", + "type": "mask", + "occluded": false, + "outside": false, + "z_order": 0, + "points": "17,9,32,13,28,17,25,19,23,22,19,25,17,27,16,27,15,29,13,31,12,31,11,33,9,35,8,35,7,37,6,37,5,39,4,39,4,39,3,41,2,41,2,41,2,41,2,41,1,559,1,41,2,41,2,41,2,41,3,39,4,39,4,39,4,39,5,37,6,37,7,35,8,35,9,33,11,31,12,31,13,29,15,27,16,27,17,25,20,22,22,19,25,17,28,13,32,9,17,379,78,421,138", + "rotation": 0.0, + "parent": null + } +}, +{ + "model": "engine.labeledshape", + "pk": 248, + "fields": { + "job": 42, + "label": 78, + "frame": 0, + "group": 0, + "source": "consensus", + "type": "mask", + "occluded": false, + "outside": false, + "z_order": 0, + "points": "127,2,151,9,143,17,136,24,129,31,122,38,115,46,56,4,49,51,53,7,48,52,51,10,47,52,48,14,46,52,46,17,45,52,43,21,44,52,41,24,43,52,38,27,43,52,36,30,42,52,33,34,42,51,31,37,41,51,28,41,40,52,25,43,40,52,23,46,39,52,20,50,38,52,18,53,37,52,15,57,36,52,13,60,35,52,10,63,35,52,8,66,34,52,5,70,34,51,3,73,33,51,1,76,32,48,2,79,31,46,4,79,31,44,5,81,30,41,8,82,29,39,9,84,28,37,10,86,27,34,13,87,26,32,14,88,26,30,16,51,1,34,29,26,18,50,3,33,30,24,19,49,4,32,32,22,21,47,6,30,34,19,23,46,7,30,35,17,25,44,9,28,37,15,26,43,10,27,39,12,28,42,12,26,40,10,30,41,12,25,42,8,31,40,14,24,43,5,34,38,15,23,46,2,35,37,17,21,84,36,18,21,85,34,20,19,141,18,141,17,143,15,144,15,145,13,146,12,148,11,148,10,150,9,150,8,152,6,153,6,154,4,155,4,156,2,157,1,87,375,13,533,80", + "rotation": 0.0, + "parent": null + } +}, +{ + "model": "engine.labeledshape", + "pk": 249, + "fields": { + "job": 42, + "label": 79, + "frame": 0, + "group": 0, + "source": "consensus", + "type": "mask", + "occluded": false, + "outside": false, + "z_order": 0, + "points": "127,3,151,10,143,18,136,25,129,32,122,39,115,47,109,52,108,53,107,54,107,54,107,54,107,54,107,54,38,1,68,54,36,3,68,54,33,5,69,54,31,6,71,53,28,9,71,54,25,10,72,54,23,11,73,54,20,14,35,2,36,54,18,15,34,5,35,54,15,17,34,7,34,54,13,19,32,10,33,54,10,21,32,11,33,54,8,22,31,14,32,54,5,25,29,17,31,54,3,26,29,19,31,53,1,27,28,22,30,81,26,25,29,49,2,29,26,26,29,47,4,29,24,29,28,45,5,29,24,31,27,42,8,28,24,33,26,40,9,29,23,35,25,38,10,29,23,37,24,35,13,28,23,38,24,33,14,29,22,38,25,31,16,28,23,36,28,27,18,28,25,34,29,25,19,29,24,33,31,23,21,28,25,31,33,20,23,28,25,31,34,18,25,28,25,29,36,16,26,28,25,28,38,13,28,28,26,27,39,11,30,28,25,26,41,9,31,28,26,25,42,6,34,27,26,24,45,3,35,28,26,22,84,28,26,22,85,28,26,20,141,19,141,18,143,16,144,16,145,14,146,13,148,12,148,11,150,10,150,9,152,7,153,7,154,5,155,5,156,3,157,2,87,616,13,775,81", + "rotation": 0.0, + "parent": null + } +}, +{ + "model": "engine.labeledshapeattributeval", + "pk": 1, + "fields": { + "spec": 2, + "value": "white", + "shape": 36 + } +}, +{ + "model": "engine.labeledshapeattributeval", + "pk": 2, + "fields": { + "spec": 3, + "value": "val1", + "shape": 39 + } +}, +{ + "model": "engine.labeledshapeattributeval", + "pk": 3, + "fields": { + "spec": 1, + "value": "mazda", + "shape": 42 + } +}, +{ + "model": "engine.labeledshapeattributeval", + "pk": 4, + "fields": { + "spec": 7, + "value": "non-default", + "shape": 54 + } +}, +{ + "model": "engine.labeledshapeattributeval", + "pk": 5, + "fields": { + "spec": 8, + "value": "black", + "shape": 55 + } +}, +{ + "model": "engine.labeledshapeattributeval", + "pk": 6, + "fields": { + "spec": 9, + "value": "non-default", + "shape": 56 + } +}, +{ + "model": "engine.labeledshapeattributeval", + "pk": 7, + "fields": { + "spec": 10, + "value": "black", + "shape": 57 + } +}, +{ + "model": "engine.labeledshapeattributeval", + "pk": 8, + "fields": { + "spec": 13, + "value": "yy", + "shape": 64 + } +}, +{ + "model": "engine.labeledshapeattributeval", + "pk": 9, + "fields": { + "spec": 14, + "value": "1", + "shape": 64 + } +}, +{ + "model": "engine.labeledshapeattributeval", + "pk": 10, + "fields": { + "spec": 13, + "value": "yz", + "shape": 65 + } +}, +{ + "model": "engine.labeledshapeattributeval", + "pk": 11, + "fields": { + "spec": 14, + "value": "2", + "shape": 65 + } +}, +{ + "model": "engine.labeledshapeattributeval", + "pk": 12, + "fields": { + "spec": 13, + "value": "yy", + "shape": 66 + } +}, +{ + "model": "engine.labeledshapeattributeval", + "pk": 13, + "fields": { + "spec": 14, + "value": "1", + "shape": 66 + } +}, +{ + "model": "engine.labeledshapeattributeval", + "pk": 14, + "fields": { + "spec": 13, + "value": "yy", + "shape": 67 + } +}, +{ + "model": "engine.labeledshapeattributeval", + "pk": 15, + "fields": { + "spec": 14, + "value": "1", + "shape": 67 + } +}, +{ + "model": "engine.labeledshapeattributeval", + "pk": 16, + "fields": { + "spec": 13, + "value": "yy", + "shape": 68 + } +}, +{ + "model": "engine.labeledshapeattributeval", + "pk": 17, + "fields": { + "spec": 14, + "value": "1", + "shape": 68 + } +}, +{ + "model": "engine.labeledshapeattributeval", + "pk": 18, + "fields": { + "spec": 13, + "value": "yy", + "shape": 69 + } +}, +{ + "model": "engine.labeledshapeattributeval", + "pk": 19, + "fields": { + "spec": 14, + "value": "1", + "shape": 69 + } +}, +{ + "model": "engine.labeledshapeattributeval", + "pk": 20, + "fields": { + "spec": 13, + "value": "yy", + "shape": 71 + } +}, +{ + "model": "engine.labeledshapeattributeval", + "pk": 21, + "fields": { + "spec": 14, + "value": "1", + "shape": 71 + } +}, +{ + "model": "engine.labeledshapeattributeval", + "pk": 22, "fields": { - "spec": 3, - "value": "val1", - "shape": 39 + "spec": 13, + "value": "yy", + "shape": 72 } }, { "model": "engine.labeledshapeattributeval", - "pk": 3, + "pk": 23, "fields": { - "spec": 1, - "value": "mazda", - "shape": 42 + "spec": 14, + "value": "1", + "shape": 72 } }, { "model": "engine.labeledshapeattributeval", - "pk": 4, + "pk": 24, "fields": { - "spec": 7, - "value": "non-default", - "shape": 54 + "spec": 13, + "value": "yy", + "shape": 75 } }, { "model": "engine.labeledshapeattributeval", - "pk": 5, + "pk": 25, "fields": { - "spec": 8, - "value": "black", - "shape": 55 + "spec": 14, + "value": "1", + "shape": 75 } }, { "model": "engine.labeledshapeattributeval", - "pk": 6, + "pk": 26, "fields": { - "spec": 9, - "value": "non-default", - "shape": 56 + "spec": 13, + "value": "yy", + "shape": 76 } }, { "model": "engine.labeledshapeattributeval", - "pk": 7, + "pk": 27, "fields": { - "spec": 10, - "value": "black", - "shape": 57 + "spec": 14, + "value": "1", + "shape": 76 } }, { "model": "engine.labeledshapeattributeval", - "pk": 8, + "pk": 28, "fields": { "spec": 13, "value": "yy", - "shape": 64 + "shape": 77 } }, { "model": "engine.labeledshapeattributeval", - "pk": 9, + "pk": 29, "fields": { "spec": 14, "value": "1", - "shape": 64 + "shape": 77 } }, { "model": "engine.labeledshapeattributeval", - "pk": 10, + "pk": 30, "fields": { "spec": 13, - "value": "yz", - "shape": 65 + "value": "yy", + "shape": 78 } }, { "model": "engine.labeledshapeattributeval", - "pk": 11, + "pk": 31, "fields": { "spec": 14, - "value": "2", - "shape": 65 + "value": "1", + "shape": 78 } }, { "model": "engine.labeledshapeattributeval", - "pk": 12, + "pk": 32, "fields": { "spec": 13, "value": "yy", - "shape": 66 + "shape": 79 } }, { "model": "engine.labeledshapeattributeval", - "pk": 13, + "pk": 33, "fields": { "spec": 14, "value": "1", - "shape": 66 + "shape": 79 } }, { "model": "engine.labeledshapeattributeval", - "pk": 14, + "pk": 34, "fields": { "spec": 13, "value": "yy", - "shape": 67 + "shape": 80 } }, { "model": "engine.labeledshapeattributeval", - "pk": 15, + "pk": 35, "fields": { "spec": 14, "value": "1", - "shape": 67 + "shape": 80 } }, { "model": "engine.labeledshapeattributeval", - "pk": 16, + "pk": 36, "fields": { "spec": 13, "value": "yy", - "shape": 68 + "shape": 81 } }, { "model": "engine.labeledshapeattributeval", - "pk": 17, + "pk": 37, "fields": { "spec": 14, "value": "1", - "shape": 68 + "shape": 81 } }, { "model": "engine.labeledshapeattributeval", - "pk": 18, + "pk": 38, "fields": { "spec": 13, "value": "yy", - "shape": 69 + "shape": 82 } }, { "model": "engine.labeledshapeattributeval", - "pk": 19, + "pk": 39, "fields": { "spec": 14, "value": "1", - "shape": 69 + "shape": 82 } }, { "model": "engine.labeledshapeattributeval", - "pk": 20, + "pk": 40, "fields": { "spec": 13, "value": "yy", - "shape": 71 + "shape": 83 } }, { "model": "engine.labeledshapeattributeval", - "pk": 21, + "pk": 41, "fields": { "spec": 14, "value": "1", - "shape": 71 + "shape": 83 } }, { "model": "engine.labeledshapeattributeval", - "pk": 22, + "pk": 42, + "fields": { + "spec": 13, + "value": "yy", + "shape": 84 + } +}, +{ + "model": "engine.labeledshapeattributeval", + "pk": 43, + "fields": { + "spec": 14, + "value": "1", + "shape": 84 + } +}, +{ + "model": "engine.labeledshapeattributeval", + "pk": 44, + "fields": { + "spec": 13, + "value": "yy", + "shape": 85 + } +}, +{ + "model": "engine.labeledshapeattributeval", + "pk": 45, + "fields": { + "spec": 14, + "value": "1", + "shape": 85 + } +}, +{ + "model": "engine.labeledshapeattributeval", + "pk": 46, + "fields": { + "spec": 13, + "value": "yy", + "shape": 86 + } +}, +{ + "model": "engine.labeledshapeattributeval", + "pk": 47, + "fields": { + "spec": 14, + "value": "1", + "shape": 86 + } +}, +{ + "model": "engine.labeledshapeattributeval", + "pk": 48, + "fields": { + "spec": 13, + "value": "yy", + "shape": 87 + } +}, +{ + "model": "engine.labeledshapeattributeval", + "pk": 49, + "fields": { + "spec": 14, + "value": "1", + "shape": 87 + } +}, +{ + "model": "engine.labeledshapeattributeval", + "pk": 50, + "fields": { + "spec": 13, + "value": "yy", + "shape": 88 + } +}, +{ + "model": "engine.labeledshapeattributeval", + "pk": 51, + "fields": { + "spec": 14, + "value": "1", + "shape": 88 + } +}, +{ + "model": "engine.labeledshapeattributeval", + "pk": 52, + "fields": { + "spec": 13, + "value": "yy", + "shape": 90 + } +}, +{ + "model": "engine.labeledshapeattributeval", + "pk": 53, + "fields": { + "spec": 14, + "value": "1", + "shape": 90 + } +}, +{ + "model": "engine.labeledshapeattributeval", + "pk": 54, + "fields": { + "spec": 13, + "value": "yy", + "shape": 91 + } +}, +{ + "model": "engine.labeledshapeattributeval", + "pk": 55, + "fields": { + "spec": 14, + "value": "1", + "shape": 91 + } +}, +{ + "model": "engine.labeledshapeattributeval", + "pk": 56, + "fields": { + "spec": 13, + "value": "yy", + "shape": 93 + } +}, +{ + "model": "engine.labeledshapeattributeval", + "pk": 57, + "fields": { + "spec": 14, + "value": "1", + "shape": 93 + } +}, +{ + "model": "engine.labeledshapeattributeval", + "pk": 58, "fields": { "spec": 13, "value": "yy", - "shape": 72 + "shape": 94 } }, { "model": "engine.labeledshapeattributeval", - "pk": 23, + "pk": 59, "fields": { "spec": 14, "value": "1", - "shape": 72 + "shape": 94 } }, { "model": "engine.labeledshapeattributeval", - "pk": 24, + "pk": 60, "fields": { "spec": 13, "value": "yy", - "shape": 75 + "shape": 95 } }, { "model": "engine.labeledshapeattributeval", - "pk": 25, + "pk": 61, "fields": { "spec": 14, "value": "1", - "shape": 75 + "shape": 95 } }, { "model": "engine.labeledshapeattributeval", - "pk": 26, + "pk": 62, "fields": { "spec": 13, "value": "yy", - "shape": 76 + "shape": 96 } }, { "model": "engine.labeledshapeattributeval", - "pk": 27, + "pk": 63, "fields": { "spec": 14, "value": "1", - "shape": 76 + "shape": 96 } }, { "model": "engine.labeledshapeattributeval", - "pk": 28, + "pk": 64, "fields": { "spec": 13, "value": "yy", - "shape": 77 + "shape": 98 } }, { "model": "engine.labeledshapeattributeval", - "pk": 29, + "pk": 65, "fields": { "spec": 14, "value": "1", - "shape": 77 + "shape": 98 } }, { "model": "engine.labeledshapeattributeval", - "pk": 30, + "pk": 66, "fields": { "spec": 13, "value": "yy", - "shape": 78 + "shape": 99 } }, { "model": "engine.labeledshapeattributeval", - "pk": 31, + "pk": 67, "fields": { "spec": 14, "value": "1", - "shape": 78 + "shape": 99 } }, { "model": "engine.labeledshapeattributeval", - "pk": 32, + "pk": 68, "fields": { "spec": 13, "value": "yy", - "shape": 79 + "shape": 100 } }, { "model": "engine.labeledshapeattributeval", - "pk": 33, + "pk": 69, "fields": { "spec": 14, "value": "1", - "shape": 79 + "shape": 100 } }, { "model": "engine.labeledshapeattributeval", - "pk": 34, + "pk": 70, "fields": { "spec": 13, "value": "yy", - "shape": 80 + "shape": 101 } }, { "model": "engine.labeledshapeattributeval", - "pk": 35, + "pk": 71, "fields": { "spec": 14, "value": "1", - "shape": 80 + "shape": 101 } }, { "model": "engine.labeledshapeattributeval", - "pk": 36, + "pk": 72, "fields": { "spec": 13, "value": "yy", - "shape": 81 + "shape": 102 } }, { "model": "engine.labeledshapeattributeval", - "pk": 37, + "pk": 73, "fields": { "spec": 14, "value": "1", - "shape": 81 + "shape": 102 } }, { "model": "engine.labeledshapeattributeval", - "pk": 38, + "pk": 74, "fields": { "spec": 13, "value": "yy", - "shape": 82 + "shape": 103 } }, { "model": "engine.labeledshapeattributeval", - "pk": 39, + "pk": 75, "fields": { "spec": 14, "value": "1", - "shape": 82 + "shape": 103 } }, { "model": "engine.labeledshapeattributeval", - "pk": 40, + "pk": 76, "fields": { "spec": 13, "value": "yy", - "shape": 83 + "shape": 104 } }, { "model": "engine.labeledshapeattributeval", - "pk": 41, + "pk": 77, "fields": { "spec": 14, "value": "1", - "shape": 83 + "shape": 104 } }, { "model": "engine.labeledshapeattributeval", - "pk": 42, + "pk": 78, "fields": { "spec": 13, "value": "yy", - "shape": 84 + "shape": 105 } }, { "model": "engine.labeledshapeattributeval", - "pk": 43, + "pk": 79, "fields": { "spec": 14, "value": "1", - "shape": 84 + "shape": 105 } }, { "model": "engine.labeledshapeattributeval", - "pk": 44, + "pk": 80, "fields": { "spec": 13, "value": "yy", - "shape": 85 + "shape": 106 } }, { "model": "engine.labeledshapeattributeval", - "pk": 45, + "pk": 81, "fields": { "spec": 14, "value": "1", - "shape": 85 + "shape": 106 } }, { "model": "engine.labeledshapeattributeval", - "pk": 46, + "pk": 82, "fields": { "spec": 13, "value": "yy", - "shape": 86 + "shape": 107 } }, { "model": "engine.labeledshapeattributeval", - "pk": 47, + "pk": 83, "fields": { "spec": 14, "value": "1", - "shape": 86 + "shape": 107 } }, { "model": "engine.labeledshapeattributeval", - "pk": 48, + "pk": 84, "fields": { "spec": 13, "value": "yy", - "shape": 87 + "shape": 108 } }, { "model": "engine.labeledshapeattributeval", - "pk": 49, + "pk": 85, "fields": { "spec": 14, "value": "1", - "shape": 87 + "shape": 108 } }, { "model": "engine.labeledshapeattributeval", - "pk": 50, + "pk": 86, "fields": { "spec": 13, "value": "yy", - "shape": 88 + "shape": 109 } }, { "model": "engine.labeledshapeattributeval", - "pk": 51, + "pk": 87, "fields": { "spec": 14, "value": "1", - "shape": 88 + "shape": 109 } }, { "model": "engine.labeledshapeattributeval", - "pk": 52, + "pk": 88, "fields": { "spec": 13, "value": "yy", - "shape": 90 + "shape": 110 } }, { "model": "engine.labeledshapeattributeval", - "pk": 53, + "pk": 89, "fields": { "spec": 14, "value": "1", - "shape": 90 + "shape": 110 } }, { "model": "engine.labeledshapeattributeval", - "pk": 54, + "pk": 90, "fields": { "spec": 13, "value": "yy", - "shape": 91 + "shape": 111 } }, { "model": "engine.labeledshapeattributeval", - "pk": 55, + "pk": 91, "fields": { "spec": 14, "value": "1", - "shape": 91 + "shape": 111 } }, { "model": "engine.labeledshapeattributeval", - "pk": 56, + "pk": 92, "fields": { "spec": 13, "value": "yy", - "shape": 93 + "shape": 112 } }, { "model": "engine.labeledshapeattributeval", - "pk": 57, + "pk": 93, "fields": { "spec": 14, "value": "1", - "shape": 93 + "shape": 112 } }, { "model": "engine.labeledshapeattributeval", - "pk": 58, + "pk": 94, "fields": { "spec": 13, "value": "yy", - "shape": 94 + "shape": 113 } }, { "model": "engine.labeledshapeattributeval", - "pk": 59, + "pk": 95, "fields": { "spec": 14, "value": "1", - "shape": 94 + "shape": 113 } }, { "model": "engine.labeledshapeattributeval", - "pk": 60, + "pk": 96, "fields": { "spec": 13, "value": "yy", - "shape": 95 + "shape": 114 } }, { "model": "engine.labeledshapeattributeval", - "pk": 61, + "pk": 97, "fields": { "spec": 14, "value": "1", - "shape": 95 + "shape": 114 } }, { "model": "engine.labeledshapeattributeval", - "pk": 62, + "pk": 98, "fields": { "spec": 13, "value": "yy", - "shape": 96 + "shape": 115 } }, { "model": "engine.labeledshapeattributeval", - "pk": 63, + "pk": 99, "fields": { "spec": 14, "value": "1", - "shape": 96 + "shape": 115 } }, { "model": "engine.labeledshapeattributeval", - "pk": 64, + "pk": 100, "fields": { "spec": 13, "value": "yy", - "shape": 98 + "shape": 116 } }, { "model": "engine.labeledshapeattributeval", - "pk": 65, + "pk": 101, "fields": { "spec": 14, "value": "1", - "shape": 98 + "shape": 116 } }, { "model": "engine.labeledshapeattributeval", - "pk": 66, + "pk": 102, "fields": { "spec": 13, "value": "yy", - "shape": 99 + "shape": 117 } }, { "model": "engine.labeledshapeattributeval", - "pk": 67, + "pk": 103, "fields": { "spec": 14, "value": "1", - "shape": 99 + "shape": 117 } }, { "model": "engine.labeledshapeattributeval", - "pk": 68, + "pk": 104, "fields": { "spec": 13, "value": "yy", - "shape": 100 + "shape": 118 } }, { "model": "engine.labeledshapeattributeval", - "pk": 69, + "pk": 105, "fields": { "spec": 14, "value": "1", - "shape": 100 + "shape": 118 } }, { "model": "engine.labeledshapeattributeval", - "pk": 70, + "pk": 106, "fields": { "spec": 13, "value": "yy", - "shape": 101 + "shape": 119 } }, { "model": "engine.labeledshapeattributeval", - "pk": 71, + "pk": 107, "fields": { "spec": 14, "value": "1", - "shape": 101 + "shape": 119 } }, { "model": "engine.labeledshapeattributeval", - "pk": 72, + "pk": 108, "fields": { "spec": 13, "value": "yy", - "shape": 102 + "shape": 120 } }, { "model": "engine.labeledshapeattributeval", - "pk": 73, + "pk": 109, "fields": { "spec": 14, "value": "1", - "shape": 102 + "shape": 120 } }, { "model": "engine.labeledshapeattributeval", - "pk": 74, + "pk": 110, "fields": { "spec": 13, "value": "yy", - "shape": 103 + "shape": 121 } }, { "model": "engine.labeledshapeattributeval", - "pk": 75, + "pk": 111, "fields": { "spec": 14, "value": "1", - "shape": 103 + "shape": 121 } }, { "model": "engine.labeledshapeattributeval", - "pk": 76, + "pk": 112, "fields": { "spec": 13, "value": "yy", - "shape": 104 + "shape": 123 } }, { "model": "engine.labeledshapeattributeval", - "pk": 77, + "pk": 113, "fields": { "spec": 14, "value": "1", - "shape": 104 + "shape": 123 } }, { "model": "engine.labeledshapeattributeval", - "pk": 78, + "pk": 114, "fields": { "spec": 13, "value": "yy", - "shape": 105 + "shape": 124 } }, { "model": "engine.labeledshapeattributeval", - "pk": 79, + "pk": 115, "fields": { "spec": 14, "value": "1", - "shape": 105 + "shape": 124 } }, { "model": "engine.labeledshapeattributeval", - "pk": 80, + "pk": 116, "fields": { "spec": 13, "value": "yy", - "shape": 106 + "shape": 125 } }, { "model": "engine.labeledshapeattributeval", - "pk": 81, + "pk": 117, "fields": { "spec": 14, "value": "1", - "shape": 106 + "shape": 125 } }, { "model": "engine.labeledshapeattributeval", - "pk": 82, + "pk": 118, "fields": { "spec": 13, "value": "yy", - "shape": 107 + "shape": 126 } }, { "model": "engine.labeledshapeattributeval", - "pk": 83, + "pk": 119, "fields": { "spec": 14, "value": "1", - "shape": 107 + "shape": 126 } }, { "model": "engine.labeledshapeattributeval", - "pk": 84, + "pk": 120, "fields": { "spec": 13, "value": "yy", - "shape": 108 + "shape": 127 } }, { "model": "engine.labeledshapeattributeval", - "pk": 85, + "pk": 121, "fields": { "spec": 14, "value": "1", - "shape": 108 + "shape": 127 } }, { "model": "engine.labeledshapeattributeval", - "pk": 86, + "pk": 122, "fields": { "spec": 13, "value": "yy", - "shape": 109 + "shape": 128 } }, { "model": "engine.labeledshapeattributeval", - "pk": 87, + "pk": 123, "fields": { "spec": 14, "value": "1", - "shape": 109 + "shape": 128 } }, { "model": "engine.labeledshapeattributeval", - "pk": 88, + "pk": 124, "fields": { "spec": 13, "value": "yy", - "shape": 110 + "shape": 129 } }, { "model": "engine.labeledshapeattributeval", - "pk": 89, + "pk": 125, "fields": { "spec": 14, "value": "1", - "shape": 110 + "shape": 129 } }, { "model": "engine.labeledshapeattributeval", - "pk": 90, + "pk": 126, "fields": { "spec": 13, "value": "yy", - "shape": 111 + "shape": 130 } }, { "model": "engine.labeledshapeattributeval", - "pk": 91, + "pk": 127, "fields": { "spec": 14, "value": "1", - "shape": 111 + "shape": 130 } }, { "model": "engine.labeledshapeattributeval", - "pk": 92, + "pk": 128, "fields": { - "spec": 13, - "value": "yy", - "shape": 112 + "spec": 15, + "value": "j1 frame1 n1", + "shape": 169 } }, { "model": "engine.labeledshapeattributeval", - "pk": 93, + "pk": 129, "fields": { - "spec": 14, - "value": "1", - "shape": 112 + "spec": 15, + "value": "j1 frame2 n1", + "shape": 170 } }, { "model": "engine.labeledshapeattributeval", - "pk": 94, + "pk": 130, "fields": { - "spec": 13, - "value": "yy", - "shape": 113 + "spec": 15, + "value": "j1 frame2 n2", + "shape": 171 } }, { "model": "engine.labeledshapeattributeval", - "pk": 95, + "pk": 131, "fields": { - "spec": 14, - "value": "1", - "shape": 113 + "spec": 15, + "value": "j1 frame6 n1", + "shape": 172 } }, { "model": "engine.labeledshapeattributeval", - "pk": 96, + "pk": 132, "fields": { - "spec": 13, - "value": "yy", - "shape": 114 + "spec": 15, + "value": "j2 frame1 n1", + "shape": 173 } }, { "model": "engine.labeledshapeattributeval", - "pk": 97, + "pk": 133, "fields": { - "spec": 14, - "value": "1", - "shape": 114 + "spec": 15, + "value": "j2 frame1 n2", + "shape": 174 } }, { "model": "engine.labeledshapeattributeval", - "pk": 98, + "pk": 134, "fields": { - "spec": 13, - "value": "yy", - "shape": 115 + "spec": 15, + "value": "j2 frame2 n1", + "shape": 175 } }, { "model": "engine.labeledshapeattributeval", - "pk": 99, + "pk": 135, "fields": { - "spec": 14, - "value": "1", - "shape": 115 + "spec": 15, + "value": "j2 frame2 n2", + "shape": 176 } }, { "model": "engine.labeledshapeattributeval", - "pk": 100, + "pk": 136, "fields": { - "spec": 13, - "value": "yy", - "shape": 116 + "spec": 15, + "value": "j2 frame5 n1", + "shape": 177 } }, { "model": "engine.labeledshapeattributeval", - "pk": 101, + "pk": 137, "fields": { - "spec": 14, - "value": "1", - "shape": 116 + "spec": 15, + "value": "j3 frame1 n1", + "shape": 178 } }, { "model": "engine.labeledshapeattributeval", - "pk": 102, + "pk": 138, "fields": { - "spec": 13, - "value": "yy", - "shape": 117 + "spec": 15, + "value": "j3 frame2 n1", + "shape": 179 } }, { "model": "engine.labeledshapeattributeval", - "pk": 103, + "pk": 139, "fields": { - "spec": 14, - "value": "1", - "shape": 117 + "spec": 15, + "value": "j3 frame5 n1", + "shape": 180 } }, { "model": "engine.labeledshapeattributeval", - "pk": 104, + "pk": 140, "fields": { - "spec": 13, - "value": "yy", - "shape": 118 + "spec": 15, + "value": "j3 frame5 n2", + "shape": 181 } }, { "model": "engine.labeledshapeattributeval", - "pk": 105, + "pk": 141, "fields": { - "spec": 14, - "value": "1", - "shape": 118 + "spec": 15, + "value": "j3 frame6 n1", + "shape": 182 } }, { "model": "engine.labeledshapeattributeval", - "pk": 106, + "pk": 142, "fields": { - "spec": 13, - "value": "yy", - "shape": 119 + "spec": 15, + "value": "j3 frame7 n1", + "shape": 183 } }, { "model": "engine.labeledshapeattributeval", - "pk": 107, + "pk": 143, "fields": { - "spec": 14, - "value": "1", - "shape": 119 + "spec": 15, + "value": "j3 frame7 n2", + "shape": 184 } }, { "model": "engine.labeledshapeattributeval", - "pk": 108, + "pk": 144, "fields": { - "spec": 13, - "value": "yy", - "shape": 120 + "spec": 15, + "value": "gt frame1 n1", + "shape": 185 } }, { "model": "engine.labeledshapeattributeval", - "pk": 109, + "pk": 145, "fields": { - "spec": 14, - "value": "1", - "shape": 120 + "spec": 15, + "value": "gt frame2 n2", + "shape": 186 } }, { "model": "engine.labeledshapeattributeval", - "pk": 110, + "pk": 147, "fields": { - "spec": 13, - "value": "yy", - "shape": 121 + "spec": 15, + "value": "gt frame3 n1", + "shape": 188 } }, { "model": "engine.labeledshapeattributeval", - "pk": 111, + "pk": 148, "fields": { - "spec": 14, - "value": "1", - "shape": 121 + "spec": 15, + "value": "gt frame5 n1", + "shape": 189 } }, { "model": "engine.labeledshapeattributeval", - "pk": 112, + "pk": 149, "fields": { - "spec": 13, - "value": "yy", - "shape": 123 + "spec": 15, + "value": "gt frame5 n2", + "shape": 190 } }, { "model": "engine.labeledshapeattributeval", - "pk": 113, + "pk": 150, "fields": { - "spec": 14, - "value": "1", - "shape": 123 + "spec": 15, + "value": "gt frame2 n1", + "shape": 187 } }, { "model": "engine.labeledshapeattributeval", - "pk": 114, + "pk": 151, "fields": { - "spec": 13, - "value": "yy", - "shape": 124 + "spec": 16, + "value": "gt job frame 1 (32) ann 1", + "shape": 191 } }, { "model": "engine.labeledshapeattributeval", - "pk": 115, + "pk": 152, "fields": { - "spec": 14, - "value": "1", - "shape": 124 + "spec": 16, + "value": "gt job frame 1 (32) ann 2", + "shape": 192 } }, { "model": "engine.labeledshapeattributeval", - "pk": 116, + "pk": 153, "fields": { - "spec": 13, - "value": "yy", - "shape": 125 + "spec": 16, + "value": "gt job frame 2 (35) ann 1", + "shape": 193 } }, { "model": "engine.labeledshapeattributeval", - "pk": 117, + "pk": 155, "fields": { - "spec": 14, - "value": "1", - "shape": 125 + "spec": 16, + "value": "j2 r1 frame 1 (35) ann 1", + "shape": 195 } }, { "model": "engine.labeledshapeattributeval", - "pk": 118, + "pk": 156, "fields": { - "spec": 13, - "value": "yy", - "shape": 126 + "spec": 16, + "value": "j2 r1 frame 1 (35) ann 2", + "shape": 196 } }, { "model": "engine.labeledshapeattributeval", - "pk": 119, + "pk": 157, "fields": { - "spec": 14, - "value": "1", - "shape": 126 + "spec": 16, + "value": "j2 frame 2 (32) ann 1", + "shape": 194 } }, { "model": "engine.labeledshapeattributeval", - "pk": 120, + "pk": 158, "fields": { - "spec": 13, - "value": "yy", - "shape": 127 + "spec": 16, + "value": "j2 r3 frame 1 (35) ann 1", + "shape": 197 } }, { "model": "engine.labeledshapeattributeval", - "pk": 121, + "pk": 159, "fields": { - "spec": 14, - "value": "1", - "shape": 127 + "spec": 16, + "value": "j2 r3 frame 1 (35) ann 2", + "shape": 198 } }, { "model": "engine.labeledshapeattributeval", - "pk": 122, + "pk": 162, "fields": { - "spec": 13, - "value": "yy", - "shape": 128 + "spec": 18, + "value": "j2 f1 (30) a1", + "shape": 201 } }, { "model": "engine.labeledshapeattributeval", - "pk": 123, + "pk": 163, "fields": { - "spec": 14, - "value": "1", - "shape": 128 + "spec": 18, + "value": "j1 f1 (30) a1", + "shape": 202 } }, { "model": "engine.labeledshapeattributeval", - "pk": 124, + "pk": 164, "fields": { - "spec": 13, - "value": "yy", - "shape": 129 + "spec": 18, + "value": "j2 f1 (30) a2", + "shape": 203 } }, { "model": "engine.labeledshapeattributeval", - "pk": 125, + "pk": 165, "fields": { - "spec": 14, - "value": "1", - "shape": 129 + "spec": 17, + "value": "", + "shape": 204 } }, { "model": "engine.labeledshapeattributeval", - "pk": 126, + "pk": 166, "fields": { - "spec": 13, - "value": "yy", - "shape": 130 + "spec": 16, + "value": "", + "shape": 205 } }, { "model": "engine.labeledshapeattributeval", - "pk": 127, + "pk": 167, "fields": { - "spec": 14, - "value": "1", - "shape": 130 + "spec": 16, + "value": "", + "shape": 206 } }, { "model": "engine.labeledshapeattributeval", - "pk": 128, + "pk": 168, "fields": { - "spec": 15, - "value": "j1 frame1 n1", - "shape": 169 + "spec": 16, + "value": "", + "shape": 207 } }, { "model": "engine.labeledshapeattributeval", - "pk": 129, + "pk": 169, "fields": { - "spec": 15, - "value": "j1 frame2 n1", - "shape": 170 + "spec": 16, + "value": "", + "shape": 208 } }, { "model": "engine.labeledshapeattributeval", - "pk": 130, + "pk": 170, "fields": { - "spec": 15, - "value": "j1 frame2 n2", - "shape": 171 + "spec": 16, + "value": "", + "shape": 209 } }, { "model": "engine.labeledshapeattributeval", - "pk": 131, + "pk": 171, "fields": { - "spec": 15, - "value": "j1 frame6 n1", - "shape": 172 + "spec": 16, + "value": "", + "shape": 210 } }, { "model": "engine.labeledshapeattributeval", - "pk": 132, + "pk": 172, "fields": { - "spec": 15, - "value": "j2 frame1 n1", - "shape": 173 + "spec": 17, + "value": "", + "shape": 211 } }, { "model": "engine.labeledshapeattributeval", - "pk": 133, + "pk": 178, "fields": { - "spec": 15, - "value": "j2 frame1 n2", - "shape": 174 + "spec": 16, + "value": "", + "shape": 217 } }, { "model": "engine.labeledshapeattributeval", - "pk": 134, + "pk": 185, "fields": { - "spec": 15, - "value": "j2 frame2 n1", - "shape": 175 + "spec": 16, + "value": "", + "shape": 224 } }, { "model": "engine.labeledshapeattributeval", - "pk": 135, + "pk": 186, "fields": { - "spec": 15, - "value": "j2 frame2 n2", - "shape": 176 + "spec": 16, + "value": "", + "shape": 212 } }, { "model": "engine.labeledshapeattributeval", - "pk": 136, + "pk": 187, "fields": { - "spec": 15, - "value": "j2 frame5 n1", - "shape": 177 + "spec": 16, + "value": "", + "shape": 213 } }, { "model": "engine.labeledshapeattributeval", - "pk": 137, + "pk": 188, "fields": { - "spec": 15, - "value": "j3 frame1 n1", - "shape": 178 + "spec": 16, + "value": "", + "shape": 214 } }, { "model": "engine.labeledshapeattributeval", - "pk": 138, + "pk": 189, "fields": { - "spec": 15, - "value": "j3 frame2 n1", - "shape": 179 + "spec": 16, + "value": "", + "shape": 215 } }, { "model": "engine.labeledshapeattributeval", - "pk": 139, + "pk": 190, "fields": { - "spec": 15, - "value": "j3 frame5 n1", - "shape": 180 + "spec": 16, + "value": "", + "shape": 216 } }, { "model": "engine.labeledshapeattributeval", - "pk": 140, + "pk": 191, "fields": { - "spec": 15, - "value": "j3 frame5 n2", - "shape": 181 + "spec": 17, + "value": "", + "shape": 218 } }, { "model": "engine.labeledshapeattributeval", - "pk": 141, + "pk": 192, "fields": { - "spec": 15, - "value": "j3 frame6 n1", - "shape": 182 + "spec": 16, + "value": "", + "shape": 219 } }, { "model": "engine.labeledshapeattributeval", - "pk": 142, + "pk": 193, "fields": { - "spec": 15, - "value": "j3 frame7 n1", - "shape": 183 + "spec": 16, + "value": "", + "shape": 220 } }, { "model": "engine.labeledshapeattributeval", - "pk": 143, + "pk": 194, "fields": { - "spec": 15, - "value": "j3 frame7 n2", - "shape": 184 + "spec": 16, + "value": "", + "shape": 221 } }, { "model": "engine.labeledshapeattributeval", - "pk": 144, + "pk": 195, "fields": { - "spec": 15, - "value": "gt frame1 n1", - "shape": 185 + "spec": 16, + "value": "", + "shape": 222 } }, { "model": "engine.labeledshapeattributeval", - "pk": 145, + "pk": 196, "fields": { - "spec": 15, - "value": "gt frame2 n2", - "shape": 186 + "spec": 16, + "value": "", + "shape": 223 } }, { "model": "engine.labeledshapeattributeval", - "pk": 147, + "pk": 197, "fields": { - "spec": 15, - "value": "gt frame3 n1", - "shape": 188 + "spec": 16, + "value": "", + "shape": 225 } }, { "model": "engine.labeledshapeattributeval", - "pk": 148, + "pk": 198, "fields": { - "spec": 15, - "value": "gt frame5 n1", - "shape": 189 + "spec": 16, + "value": "", + "shape": 226 } }, { "model": "engine.labeledshapeattributeval", - "pk": 149, + "pk": 199, "fields": { - "spec": 15, - "value": "gt frame5 n2", - "shape": 190 + "spec": 17, + "value": "", + "shape": 227 } }, { "model": "engine.labeledshapeattributeval", - "pk": 150, + "pk": 200, "fields": { - "spec": 15, - "value": "gt frame2 n1", - "shape": 187 + "spec": 17, + "value": "", + "shape": 228 } }, { "model": "engine.labeledshapeattributeval", - "pk": 151, + "pk": 212, "fields": { - "spec": 16, - "value": "gt job frame 1 (32) ann 1", - "shape": 191 + "spec": 17, + "value": "", + "shape": 240 } }, { "model": "engine.labeledshapeattributeval", - "pk": 152, + "pk": 213, "fields": { "spec": 16, - "value": "gt job frame 1 (32) ann 2", - "shape": 192 + "value": "", + "shape": 241 } }, { "model": "engine.labeledshapeattributeval", - "pk": 153, + "pk": 214, "fields": { "spec": 16, - "value": "gt job frame 2 (35) ann 1", - "shape": 193 + "value": "", + "shape": 242 } }, { "model": "engine.labeledshapeattributeval", - "pk": 155, + "pk": 215, "fields": { "spec": 16, - "value": "j2 r1 frame 1 (35) ann 1", - "shape": 195 + "value": "", + "shape": 243 } }, { "model": "engine.labeledshapeattributeval", - "pk": 156, + "pk": 216, "fields": { "spec": 16, - "value": "j2 r1 frame 1 (35) ann 2", - "shape": 196 + "value": "", + "shape": 244 } }, { "model": "engine.labeledshapeattributeval", - "pk": 157, + "pk": 217, "fields": { "spec": 16, - "value": "j2 frame 2 (32) ann 1", - "shape": 194 + "value": "", + "shape": 245 } }, { "model": "engine.labeledshapeattributeval", - "pk": 158, + "pk": 218, "fields": { - "spec": 16, - "value": "j2 r3 frame 1 (35) ann 1", - "shape": 197 + "spec": 17, + "value": "", + "shape": 246 } }, { "model": "engine.labeledshapeattributeval", - "pk": 159, + "pk": 219, "fields": { "spec": 16, - "value": "j2 r3 frame 1 (35) ann 2", - "shape": 198 + "value": "", + "shape": 247 } }, { "model": "engine.labeledshapeattributeval", - "pk": 160, + "pk": 220, "fields": { "spec": 16, - "value": "j1 frame 1 (32) ann 1", - "shape": 199 + "value": "", + "shape": 248 } }, { "model": "engine.labeledshapeattributeval", - "pk": 161, + "pk": 221, "fields": { - "spec": 16, - "value": "j1 r1 frame 1 (32) ann 1", - "shape": 200 + "spec": 17, + "value": "", + "shape": 249 } }, { @@ -14044,6 +15289,22 @@ "cloud_storage": null } }, +{ + "model": "engine.storage", + "pk": 53, + "fields": { + "location": "local", + "cloud_storage": null + } +}, +{ + "model": "engine.storage", + "pk": 54, + "fields": { + "location": "local", + "cloud_storage": null + } +}, { "model": "webhooks.webhook", "pk": 1, @@ -19279,5 +20540,47 @@ "target_metric_threshold": 0.7, "max_validations_per_job": 0 } +}, +{ + "model": "quality_control.qualitysettings", + "pk": 26, + "fields": { + "task": 31, + "iou_threshold": 0.4, + "oks_sigma": 0.09, + "line_thickness": 0.01, + "low_overlap_threshold": 0.8, + "point_size_base": "group_bbox_size", + "compare_line_orientation": true, + "line_orientation_threshold": 0.1, + "compare_groups": true, + "group_match_threshold": 0.5, + "check_covered_annotations": true, + "object_visibility_threshold": 0.05, + "panoptic_comparison": true, + "compare_attributes": true, + "empty_is_annotated": false, + "target_metric": "accuracy", + "target_metric_threshold": 0.7, + "max_validations_per_job": 0 + } +}, +{ + "model": "consensus.consensussettings", + "pk": 1, + "fields": { + "task": 30, + "quorum": 0.5, + "iou_threshold": 0.4 + } +}, +{ + "model": "consensus.consensussettings", + "pk": 2, + "fields": { + "task": 31, + "quorum": 0.5, + "iou_threshold": 0.4 + } } ] diff --git a/tests/python/shared/assets/jobs.json b/tests/python/shared/assets/jobs.json index 71fad5510b52..850af4921f51 100644 --- a/tests/python/shared/assets/jobs.json +++ b/tests/python/shared/assets/jobs.json @@ -1,8 +1,140 @@ { - "count": 43, + "count": 46, "next": null, "previous": null, "results": [ + { + "assignee": null, + "assignee_updated_date": null, + "bug_tracker": null, + "consensus_replicas": 0, + "created_date": "2025-01-27T16:42:31.676000Z", + "data_chunk_size": 72, + "data_compressed_chunk_type": "imageset", + "data_original_chunk_type": "imageset", + "dimension": "2d", + "frame_count": 3, + "guide_id": null, + "id": 53, + "issues": { + "count": 0, + "url": "http://localhost:8080/api/issues?job_id=53" + }, + "labels": { + "url": "http://localhost:8080/api/labels?job_id=53" + }, + "mode": "annotation", + "organization": null, + "parent_job_id": 51, + "project_id": null, + "source_storage": { + "cloud_storage_id": null, + "id": 53, + "location": "local" + }, + "stage": "annotation", + "start_frame": 0, + "state": "completed", + "status": "annotation", + "stop_frame": 2, + "target_storage": { + "cloud_storage_id": null, + "id": 54, + "location": "local" + }, + "task_id": 31, + "type": "consensus_replica", + "updated_date": "2025-01-27T16:43:47.462000Z", + "url": "http://localhost:8080/api/jobs/53" + }, + { + "assignee": null, + "assignee_updated_date": null, + "bug_tracker": null, + "consensus_replicas": 0, + "created_date": "2025-01-27T16:42:31.672000Z", + "data_chunk_size": 72, + "data_compressed_chunk_type": "imageset", + "data_original_chunk_type": "imageset", + "dimension": "2d", + "frame_count": 3, + "guide_id": null, + "id": 52, + "issues": { + "count": 0, + "url": "http://localhost:8080/api/issues?job_id=52" + }, + "labels": { + "url": "http://localhost:8080/api/labels?job_id=52" + }, + "mode": "annotation", + "organization": null, + "parent_job_id": 51, + "project_id": null, + "source_storage": { + "cloud_storage_id": null, + "id": 53, + "location": "local" + }, + "stage": "annotation", + "start_frame": 0, + "state": "in progress", + "status": "annotation", + "stop_frame": 2, + "target_storage": { + "cloud_storage_id": null, + "id": 54, + "location": "local" + }, + "task_id": 31, + "type": "consensus_replica", + "updated_date": "2025-01-27T16:43:07.696000Z", + "url": "http://localhost:8080/api/jobs/52" + }, + { + "assignee": null, + "assignee_updated_date": null, + "bug_tracker": null, + "consensus_replicas": 2, + "created_date": "2025-01-27T16:42:31.660000Z", + "data_chunk_size": 72, + "data_compressed_chunk_type": "imageset", + "data_original_chunk_type": "imageset", + "dimension": "2d", + "frame_count": 3, + "guide_id": null, + "id": 51, + "issues": { + "count": 0, + "url": "http://localhost:8080/api/issues?job_id=51" + }, + "labels": { + "url": "http://localhost:8080/api/labels?job_id=51" + }, + "mode": "annotation", + "organization": null, + "parent_job_id": null, + "project_id": null, + "source_storage": { + "cloud_storage_id": null, + "id": 53, + "location": "local" + }, + "stage": "annotation", + "start_frame": 0, + "state": "new", + "status": "annotation", + "stop_frame": 2, + "target_storage": { + "cloud_storage_id": null, + "id": 54, + "location": "local" + }, + "task_id": 31, + "type": "annotation", + "updated_date": "2025-01-27T16:42:31.660000Z", + "url": "http://localhost:8080/api/jobs/51" + }, { "assignee": null, "assignee_updated_date": null, @@ -254,7 +386,7 @@ }, "stage": "annotation", "start_frame": 0, - "state": "new", + "state": "in progress", "status": "annotation", "stop_frame": 4, "target_storage": { @@ -264,7 +396,7 @@ }, "task_id": 30, "type": "consensus_replica", - "updated_date": "2025-01-14T17:19:51.388000Z", + "updated_date": "2025-02-03T15:22:52.241000Z", "url": "http://localhost:8080/api/jobs/45" }, { @@ -352,7 +484,7 @@ }, "task_id": 30, "type": "consensus_replica", - "updated_date": "2025-01-14T17:26:01.233000Z", + "updated_date": "2025-02-03T15:25:32.413000Z", "url": "http://localhost:8080/api/jobs/43" }, { @@ -386,7 +518,7 @@ }, "stage": "annotation", "start_frame": 0, - "state": "in progress", + "state": "completed", "status": "annotation", "stop_frame": 4, "target_storage": { @@ -396,7 +528,7 @@ }, "task_id": 30, "type": "annotation", - "updated_date": "2025-01-14T17:25:16.549000Z", + "updated_date": "2025-02-03T15:25:42.655000Z", "url": "http://localhost:8080/api/jobs/42" }, { diff --git a/tests/python/shared/assets/labels.json b/tests/python/shared/assets/labels.json index 91563de0375e..07e281ddd12a 100644 --- a/tests/python/shared/assets/labels.json +++ b/tests/python/shared/assets/labels.json @@ -1,5 +1,5 @@ { - "count": 46, + "count": 47, "next": null, "previous": null, "results": [ @@ -916,6 +916,28 @@ "sublabels": [], "task_id": 30, "type": "any" + }, + { + "attributes": [ + { + "default_value": "", + "id": 18, + "input_type": "text", + "mutable": false, + "name": "src", + "values": [ + "" + ] + } + ], + "color": "#6080c0", + "has_parent": false, + "id": 80, + "name": "cat", + "parent_id": null, + "sublabels": [], + "task_id": 31, + "type": "any" } ] } \ No newline at end of file diff --git a/tests/python/shared/assets/quality_settings.json b/tests/python/shared/assets/quality_settings.json index 1594c48ff8d9..e6764f5a204b 100644 --- a/tests/python/shared/assets/quality_settings.json +++ b/tests/python/shared/assets/quality_settings.json @@ -1,5 +1,5 @@ { - "count": 25, + "count": 26, "next": null, "previous": null, "results": [ @@ -527,6 +527,27 @@ "target_metric": "accuracy", "target_metric_threshold": 0.7, "task_id": 30 + }, + { + "check_covered_annotations": true, + "compare_attributes": true, + "compare_groups": true, + "compare_line_orientation": true, + "empty_is_annotated": false, + "group_match_threshold": 0.5, + "id": 26, + "iou_threshold": 0.4, + "line_orientation_threshold": 0.1, + "line_thickness": 0.01, + "low_overlap_threshold": 0.8, + "max_validations_per_job": 0, + "object_visibility_threshold": 0.05, + "oks_sigma": 0.09, + "panoptic_comparison": true, + "point_size_base": "group_bbox_size", + "target_metric": "accuracy", + "target_metric_threshold": 0.7, + "task_id": 31 } ] } \ No newline at end of file diff --git a/tests/python/shared/assets/tasks.json b/tests/python/shared/assets/tasks.json index 500439f5b709..b3bf31dedd56 100644 --- a/tests/python/shared/assets/tasks.json +++ b/tests/python/shared/assets/tasks.json @@ -1,8 +1,61 @@ { - "count": 25, + "count": 26, "next": null, "previous": null, "results": [ + { + "assignee": null, + "assignee_updated_date": null, + "bug_tracker": "", + "consensus_enabled": true, + "created_date": "2025-01-27T16:42:31.302000Z", + "data": 30, + "data_chunk_size": 72, + "data_compressed_chunk_type": "imageset", + "data_original_chunk_type": "imageset", + "dimension": "2d", + "guide_id": null, + "id": 31, + "image_quality": 70, + "jobs": { + "completed": 0, + "count": 3, + "url": "http://localhost:8080/api/jobs?task_id=31", + "validation": 0 + }, + "labels": { + "url": "http://localhost:8080/api/labels?task_id=31" + }, + "mode": "annotation", + "name": "sandbox task with consensus", + "organization": null, + "overlap": 0, + "owner": { + "first_name": "User", + "id": 2, + "last_name": "First", + "url": "http://localhost:8080/api/users/2", + "username": "user1" + }, + "project_id": null, + "segment_size": 3, + "size": 3, + "source_storage": { + "cloud_storage_id": null, + "id": 53, + "location": "local" + }, + "status": "annotation", + "subset": "", + "target_storage": { + "cloud_storage_id": null, + "id": 54, + "location": "local" + }, + "updated_date": "2025-01-27T16:43:36.188000Z", + "url": "http://localhost:8080/api/tasks/31", + "validation_mode": null + }, { "assignee": null, "assignee_updated_date": null, diff --git a/tests/python/shared/fixtures/data.py b/tests/python/shared/fixtures/data.py index c3e23ef4dafe..eb8f7393cd8c 100644 --- a/tests/python/shared/fixtures/data.py +++ b/tests/python/shared/fixtures/data.py @@ -233,6 +233,12 @@ def quality_settings(): return Container(json.load(f)["results"]) +@pytest.fixture(scope="session") +def consensus_settings(): + with open(ASSETS_DIR / "consensus_settings.json") as f: + return Container(json.load(f)["results"]) + + @pytest.fixture(scope="session") def users_by_name(users): return {user["username"]: user for user in users} diff --git a/tests/python/shared/utils/dump_objects.py b/tests/python/shared/utils/dump_objects.py index 54d1eab87eb1..26e2fd538ce7 100644 --- a/tests/python/shared/utils/dump_objects.py +++ b/tests/python/shared/utils/dump_objects.py @@ -53,6 +53,7 @@ def clean_list_response(data: dict[str, Any]) -> dict[str, Any]: "quality/report", "quality/conflict", "quality/setting", + "consensus/setting", ]: response = get_method("admin1", f"{obj}s", page_size="all")