From 85b46beb4cda3eb6d854cdf5984801c90a7f0ea6 Mon Sep 17 00:00:00 2001 From: Laurent Bonnet <146674147+labo-flg@users.noreply.github.com> Date: Thu, 21 Dec 2023 14:14:10 +0100 Subject: [PATCH] [backend/frontend] Invalidate empty CSV mappers (#4644) --- .../data/csvMapper/CsvMapperForm.tsx | 3 +- .../internal/csvMapper/csvMapper-utils.ts | 5 +++ .../01-unit/domain/csv-mapper-utils-test.ts | 37 ++++++++++++++++ .../05-parser/csv-parser-test.js | 42 +++++++++---------- ...v-mapper-mock-simple-different-entities.ts | 11 ++--- 5 files changed, 68 insertions(+), 30 deletions(-) create mode 100644 opencti-platform/opencti-graphql/tests/01-unit/domain/csv-mapper-utils-test.ts rename opencti-platform/opencti-graphql/tests/{02-integration/05-parser/simple-different-entities-test => data}/csv-mapper-mock-simple-different-entities.ts (75%) diff --git a/opencti-platform/opencti-front/src/private/components/data/csvMapper/CsvMapperForm.tsx b/opencti-platform/opencti-front/src/private/components/data/csvMapper/CsvMapperForm.tsx index 8afdf85c790e..3bf00724faad 100644 --- a/opencti-platform/opencti-front/src/private/components/data/csvMapper/CsvMapperForm.tsx +++ b/opencti-platform/opencti-front/src/private/components/data/csvMapper/CsvMapperForm.tsx @@ -140,7 +140,8 @@ const CsvMapperForm: FunctionComponent = ({ csvMapper, onSub }; // -- ERRORS -- - const [hasError, setHasError] = useState(false); + // on edit mode, csvMapper.errors might be set; on create mode backend validation is not done yet so error is null + const [hasError, setHasError] = useState(!!csvMapper.errors?.length || csvMapper.representations.length === 0); let errors: Map = new Map(); const handleRepresentationErrors = (key: string, value: boolean) => { errors = { ...errors, [key]: value }; diff --git a/opencti-platform/opencti-graphql/src/modules/internal/csvMapper/csvMapper-utils.ts b/opencti-platform/opencti-graphql/src/modules/internal/csvMapper/csvMapper-utils.ts index 0eafa4f720b4..f0e74f94ebe4 100644 --- a/opencti-platform/opencti-graphql/src/modules/internal/csvMapper/csvMapper-utils.ts +++ b/opencti-platform/opencti-graphql/src/modules/internal/csvMapper/csvMapper-utils.ts @@ -34,6 +34,11 @@ export const isValidTargetType = (representation: CsvMapperRepresentation) => { }; export const validate = async (context: AuthContext, mapper: BasicStoreEntityCsvMapper) => { + // consider empty csv mapper as invalid to avoid being used in the importer + if (mapper.representations.length === 0) { + throw Error(`CSV Mapper '${mapper.name}' has no representation`); + } + await Promise.all(Array.from(mapper.representations.entries()).map(async ([idx, representation]) => { // Validate target type isValidTargetType(representation); diff --git a/opencti-platform/opencti-graphql/tests/01-unit/domain/csv-mapper-utils-test.ts b/opencti-platform/opencti-graphql/tests/01-unit/domain/csv-mapper-utils-test.ts new file mode 100644 index 000000000000..d386823dacf3 --- /dev/null +++ b/opencti-platform/opencti-graphql/tests/01-unit/domain/csv-mapper-utils-test.ts @@ -0,0 +1,37 @@ +import { assert, describe, expect, it } from 'vitest'; +import { csvMapperMockSimpleDifferentEntities } from '../../data/csv-mapper-mock-simple-different-entities'; +import { validate } from '../../../src/modules/internal/csvMapper/csvMapper-utils'; +import { testContext } from '../../utils/testQuery'; +import type { BasicStoreEntityCsvMapper } from '../../../src/modules/internal/csvMapper/csvMapper-types'; + +describe('CSV Mapper', () => { + it('validate a valid mapper', async () => { + await validate(testContext, { + ...csvMapperMockSimpleDifferentEntities as BasicStoreEntityCsvMapper, + name: 'Valid Mapper' + }); + assert(true); + }); + it('invalidate a invalid mapper', async () => { + const mapper = csvMapperMockSimpleDifferentEntities as BasicStoreEntityCsvMapper; + await expect(() => validate(testContext, { + ...mapper, + name: 'Invalid Mapper', + representations: [], // cannot have 0 representations + })).rejects.toThrowError('CSV Mapper \'Invalid Mapper\' has no representation'); + + await expect(() => validate(testContext, { + ...mapper, + name: 'Invalid Mapper', + representations: [ + { + ...mapper.representations[0], + attributes: [], // missing attribute + }, + mapper.representations[1], + ] + })).rejects.toThrowError(/missing values for required attribute : name/); + + // TODO: cover more validation tests + }); +}); diff --git a/opencti-platform/opencti-graphql/tests/02-integration/05-parser/csv-parser-test.js b/opencti-platform/opencti-graphql/tests/02-integration/05-parser/csv-parser-test.js index f0d5e842bd32..76631deca069 100644 --- a/opencti-platform/opencti-graphql/tests/02-integration/05-parser/csv-parser-test.js +++ b/opencti-platform/opencti-graphql/tests/02-integration/05-parser/csv-parser-test.js @@ -1,19 +1,17 @@ import { describe, expect, it } from 'vitest'; -import { csvMapperMockSimpleEntity } from "./simple-entity-test/csv-mapper-mock-simple-entity"; -import { isNotEmptyField } from "../../../src/database/utils"; +import { csvMapperMockSimpleEntity } from './simple-entity-test/csv-mapper-mock-simple-entity'; +import { isNotEmptyField } from '../../../src/database/utils'; import '../../../src/modules'; -import { csvMapperMockSimpleRelationship } from "./simple-relationship-test/csv-mapper-mock-simple-relationship"; -import { csvMapperMockSimpleEntityWithRef } from "./simple-entity-with-ref-test/csv-mapper-mock-simple-entity-with-ref"; -import { columnNameToIdx } from "../../../src/parser/csv-helper"; -import { csvMapperMockRealUseCase } from "./real-use-case/csv-mapper-mock-real-use-case"; -import { - csvMapperMockSimpleDifferentEntities -} from "./simple-different-entities-test/csv-mapper-mock-simple-different-entities"; -import { csvMapperMockSimpleSighting } from "./simple-sighting-test/csv-mapper-mock-simple-sighting"; -import { bundleProcess } from "../../../src/parser/csv-bundler"; -import { ADMIN_USER, testContext } from "../../utils/testQuery"; -import { csvMapperMockSimpleSkipLine } from "./simple-skip-line-test/csv-mapper-mock-simple-skip-line"; +import { csvMapperMockSimpleRelationship } from './simple-relationship-test/csv-mapper-mock-simple-relationship'; +import { csvMapperMockSimpleEntityWithRef } from './simple-entity-with-ref-test/csv-mapper-mock-simple-entity-with-ref'; +import { columnNameToIdx } from '../../../src/parser/csv-helper'; +import { csvMapperMockRealUseCase } from './real-use-case/csv-mapper-mock-real-use-case'; +import { csvMapperMockSimpleDifferentEntities } from '../../data/csv-mapper-mock-simple-different-entities'; +import { csvMapperMockSimpleSighting } from './simple-sighting-test/csv-mapper-mock-simple-sighting'; +import { bundleProcess } from '../../../src/parser/csv-bundler'; +import { ADMIN_USER, testContext } from '../../utils/testQuery'; +import { csvMapperMockSimpleSkipLine } from './simple-skip-line-test/csv-mapper-mock-simple-skip-line'; describe('CSV-HELPER', () => { it('Column name to idx', async () => { @@ -35,7 +33,7 @@ describe('CSV-HELPER', () => { idx = columnNameToIdx('AJD'); expect(idx) .toBe(939); - }) + }); }); describe('CSV-PARSER', () => { @@ -43,7 +41,7 @@ describe('CSV-PARSER', () => { const filPath = './tests/02-integration/05-parser/simple-entity-test/Threat-Actor-Group_list.csv'; const bundle = await bundleProcess(testContext, ADMIN_USER, filPath, csvMapperMockSimpleEntity); - const objects = bundle.objects; + const { objects } = bundle; expect(objects.length) .toBe(5); expect(objects.filter((o) => isNotEmptyField(o.name)).length) @@ -59,7 +57,7 @@ describe('CSV-PARSER', () => { const filPath = './tests/02-integration/05-parser/simple-relationship-test/Threat-Actor-Group_PART-OF_list.csv'; const bundle = await bundleProcess(testContext, ADMIN_USER, filPath, csvMapperMockSimpleRelationship); - const objects = bundle.objects; + const { objects } = bundle; expect(objects.length) .toBe(6); expect(objects.filter((o) => o.type === 'threat-actor').length) @@ -71,7 +69,7 @@ describe('CSV-PARSER', () => { const filPath = './tests/02-integration/05-parser/simple-sighting-test/Threat-Actor-Group_SIGHTING_org.csv'; const bundle = await bundleProcess(testContext, ADMIN_USER, filPath, csvMapperMockSimpleSighting); - const objects = bundle.objects; + const { objects } = bundle; expect(objects.length) .toBe(3); expect(objects.filter((o) => o.type === 'threat-actor').length) @@ -85,7 +83,7 @@ describe('CSV-PARSER', () => { const filPath = './tests/02-integration/05-parser/simple-entity-with-ref-test/Threat-Actor-Group_with-ref.csv'; const bundle = await bundleProcess(testContext, ADMIN_USER, filPath, csvMapperMockSimpleEntityWithRef); - const objects = bundle.objects; + const { objects } = bundle; expect(objects.length) .toBe(3); const label = objects.filter((o) => o.type === 'label')[0]; @@ -110,7 +108,7 @@ describe('CSV-PARSER', () => { const filPath = './tests/02-integration/05-parser/simple-different-entities-test/Threat-Actor-Group_or_Organization.csv'; const bundle = await bundleProcess(testContext, ADMIN_USER, filPath, csvMapperMockSimpleDifferentEntities); - const objects = bundle.objects; + const { objects } = bundle; expect(objects.length) .toBe(2); expect(objects.filter((o) => o.type === 'threat-actor').length) @@ -122,7 +120,7 @@ describe('CSV-PARSER', () => { const filPath = './tests/02-integration/05-parser/real-use-case/schema incidents.csv'; const bundle = await bundleProcess(testContext, ADMIN_USER, filPath, csvMapperMockRealUseCase); - const objects = bundle.objects; + const { objects } = bundle; const incidents = objects.filter((o) => o.type === 'incident'); expect(incidents.length) .toBe(118); @@ -148,7 +146,7 @@ describe('CSV-PARSER', () => { it('Parse CSV - Simple skip line test on Simple entity ', async () => { const filPath = './tests/02-integration/05-parser/simple-skip-line-test/Threat-Actor-Group_list_skip_line.csv'; const bundle = await bundleProcess(testContext, ADMIN_USER, filPath, csvMapperMockSimpleSkipLine); - const objects = bundle.objects; + const { objects } = bundle; expect(objects.length) .toBe(5); expect(objects.filter((o) => isNotEmptyField(o.name)).length) @@ -160,4 +158,4 @@ describe('CSV-PARSER', () => { expect(threatActorWithTypes.threat_actor_types.length) .toBe(2); }); -}) +}); diff --git a/opencti-platform/opencti-graphql/tests/02-integration/05-parser/simple-different-entities-test/csv-mapper-mock-simple-different-entities.ts b/opencti-platform/opencti-graphql/tests/data/csv-mapper-mock-simple-different-entities.ts similarity index 75% rename from opencti-platform/opencti-graphql/tests/02-integration/05-parser/simple-different-entities-test/csv-mapper-mock-simple-different-entities.ts rename to opencti-platform/opencti-graphql/tests/data/csv-mapper-mock-simple-different-entities.ts index 7dafce0da85f..dec6d7e386cc 100644 --- a/opencti-platform/opencti-graphql/tests/02-integration/05-parser/simple-different-entities-test/csv-mapper-mock-simple-different-entities.ts +++ b/opencti-platform/opencti-graphql/tests/data/csv-mapper-mock-simple-different-entities.ts @@ -1,9 +1,6 @@ -import { ENTITY_TYPE_THREAT_ACTOR_GROUP } from '../../../../src/schema/stixDomainObject'; -import { ENTITY_TYPE_IDENTITY_ORGANIZATION } from '../../../../src/modules/organization/organization-types'; -import { - type BasicStoreEntityCsvMapper, - CsvMapperRepresentationType, Operator -} from '../../../../src/modules/internal/csvMapper/csvMapper-types'; +import { ENTITY_TYPE_THREAT_ACTOR_GROUP } from '../../src/schema/stixDomainObject'; +import { ENTITY_TYPE_IDENTITY_ORGANIZATION } from '../../src/modules/organization/organization-types'; +import { type BasicStoreEntityCsvMapper, CsvMapperRepresentationType, Operator } from '../../src/modules/internal/csvMapper/csvMapper-types'; export const csvMapperMockSimpleDifferentEntities: Partial = { id: 'mapper-mock-simple-different-entities', @@ -51,4 +48,4 @@ export const csvMapperMockSimpleDifferentEntities: Partial