Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Policy preview import performance #3530

Merged
merged 3 commits into from
Apr 24, 2024
Merged

Policy preview import performance #3530

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
eac9d80
Policy Preview/Import performance optimisations
Pyatakov Apr 10, 2024
f4ac86a
fixed fields parsing for compare
Stepan-Kirjakov Apr 11, 2024
c5521e6
parseConditions function improvements
Pyatakov Apr 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 18 additions & 40 deletions common/src/import-export/policy.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -207,32 +207,28 @@ export class PolicyImportExport {
throw new Error('Zip file is not a policy');
}
const policyString = await content.files[PolicyImportExport.policyFileName].async('string');
const tokensStringArray = await Promise.all(Object.entries(content.files)
.filter(file => !file[1].dir)
.filter(file => /^tokens\/.+/.test(file[0]))
.map(file => file[1].async('string')));

const schemasStringArray = await Promise.all(Object.entries(content.files)
.filter(file => !file[1].dir)
.filter(file => /^schem[a,e]s\/.+/.test(file[0]))
.map(file => file[1].async('string')));

const toolsStringArray = await Promise.all(Object.entries(content.files)
.filter(file => !file[1].dir)
.filter(file => /^tools\/.+/.test(file[0]))
.map(file => file[1].async('string')));

const metaDataFile = (Object.entries(content.files)
.find(file => file[0] === 'artifacts/metadata.json'));
const policy = JSON.parse(policyString);

const fileEntries = Object.entries(content.files).filter(file => !file[1].dir);
const [tokensStringArray, schemasStringArray, toolsStringArray, tagsStringArray] = await Promise.all([
Promise.all(fileEntries.filter(file => /^tokens\/.+/.test(file[0])).map(file => file[1].async('string'))),
Promise.all(fileEntries.filter(file => /^schem[a,e]s\/.+/.test(file[0])).map(file => file[1].async('string'))),
Promise.all(fileEntries.filter(file => /^tools\/.+/.test(file[0])).map(file => file[1].async('string'))),
Promise.all(fileEntries.filter(file => /^tags\/.+/.test(file[0])).map(file => file[1].async('string')))
]);

const tokens = tokensStringArray.map(item => JSON.parse(item));
const schemas = schemasStringArray.map(item => JSON.parse(item));
const tools = toolsStringArray.map(item => JSON.parse(item));
const tags = tagsStringArray.map(item => JSON.parse(item));

const metaDataFile = (Object.entries(content.files).find(file => file[0] === 'artifacts/metadata.json'));
const metaDataString = metaDataFile && await metaDataFile[1].async('string') || '[]';
const metaDataBody: any[] = JSON.parse(metaDataString);

let artifacts: any;
if (includeArtifactsData) {
const data = Object.entries(content.files)
.filter(file => !file[1].dir)
.filter(file => /^artifacts\/.+/.test(file[0]) && file[0] !== 'artifacts/metadata.json')
.map(async file => {
const data = fileEntries.filter(file => /^artifacts\/.+/.test(file[0]) && file[0] !== 'artifacts/metadata.json').map(async file => {
const uuid = file[0].split('/')[1];
const artifactMetaData = metaDataBody.find(item => item.uuid === uuid);
return {
Expand All @@ -254,31 +250,13 @@ export class PolicyImportExport {
});
}

const tagsStringArray = await Promise.all(Object.entries(content.files)
.filter(file => !file[1].dir)
.filter(file => /^tags\/.+/.test(file[0]))
.map(file => file[1].async('string')));

const policy = JSON.parse(policyString);
const tokens = tokensStringArray.map(item => JSON.parse(item));
const schemas = schemasStringArray.map(item => JSON.parse(item));
const tools = toolsStringArray.map(item => JSON.parse(item));
const tags = tagsStringArray.map(item => JSON.parse(item));

if (policy.categoriesExport?.length) {
const allCategories = await DatabaseServer.getPolicyCategories();
policy.categories = PolicyImportExport.parsePolicyCategories(policy, allCategories);
policy.categoriesExport = [];
}

return {
policy,
tokens,
schemas,
artifacts,
tags,
tools
};
return { policy, tokens, schemas, artifacts, tags, tools };
}

/**
Expand Down
24 changes: 11 additions & 13 deletions guardian-service/src/analytics/compare/comparators/hash-comparator.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,21 +57,20 @@ export class HashComparator {
const { policy, schemas, tokens, artifacts } = file;

//Policy
const policyModel = (new PolicyModel(policy, HashComparator.options));
const policyModel = new PolicyModel(policy, HashComparator.options);

//Schemas
const schemaModels: SchemaModel[] = [];
for (const schema of schemas) {
const schemaModels = schemas.map(schema => {
const m = new SchemaModel(schema, HashComparator.options);
m.setPolicy(policy);
m.update(HashComparator.options);
schemaModels.push(m);
}
return m;
});

policyModel.setSchemas(schemaModels);

//Tokens
const tokenModels: TokenModel[] = [];
for (const row of tokens) {
const tokenModels = tokens.map(row => {
const token: any = {
tokenId: row.tokenId,
tokenName: row.tokenName,
Expand All @@ -86,17 +85,16 @@ export class HashComparator {
}
const t = new TokenModel(token, HashComparator.options);
t.update(HashComparator.options);
tokenModels.push(t);
}
return t;
});
policyModel.setTokens(tokenModels);

//Artifacts
const artifactsModels: FileModel[] = [];
for (const artifact of artifacts) {
const artifactsModels = artifacts.map(artifact => {
const f = new FileModel(artifact, artifact.data, HashComparator.options);
f.update(HashComparator.options);
artifactsModels.push(f);
}
return f;
});
policyModel.setArtifacts(artifactsModels);

//Compare
Expand Down
10 changes: 1 addition & 9 deletions guardian-service/src/analytics/compare/models/condition.model.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,6 @@ import { FieldModel } from './field.model.js';
* Condition Model
*/
export class ConditionModel {
/**
* Condition index
* @public
*/
public readonly index: number;

/**
* Field name
* @public
Expand Down Expand Up @@ -50,10 +44,8 @@ export class ConditionModel {
field: FieldModel,
fieldValue: any,
thenFields: FieldModel[],
elseFields: FieldModel[],
index: number
elseFields: FieldModel[]
) {
this.index = index;
this.field = field;
this.fieldValue = fieldValue;
this.thenFields = thenFields;
Expand Down
3 changes: 1 addition & 2 deletions guardian-service/src/analytics/compare/models/field.model.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -180,8 +180,7 @@ export class FieldModel implements IWeightModel {
constructor(
name: string,
prop: any,
required: boolean,
index: number
required: boolean
) {
let _property = prop;
if (_property.oneOf && _property.oneOf.length) {
Expand Down
105 changes: 62 additions & 43 deletions guardian-service/src/analytics/compare/models/schema-document.model.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { ConditionModel } from './condition.model.js';
import { CompareOptions } from '../interfaces/compare-options.interface.js';
import MurmurHash3 from 'imurmurhash';
import { ComparePolicyUtils } from '../utils/compare-policy-utils.js';
import { ISchemaDocument } from '@guardian/interfaces';

/**
* Schema Model
Expand All @@ -26,47 +27,46 @@ export class SchemaDocumentModel {
*/
private _weight: string;

constructor(document: any, index: number, defs?: any) {
constructor(
document: ISchemaDocument,
defs: { [x: string]: ISchemaDocument },
cache: Map<string, SchemaDocumentModel>
) {
this._weight = '';
this.fields = this.parseFields(document, index + 1, defs);
this.conditions = this.parseConditions(document, index + 1, this.fields, defs);
this.fields = this.parseFields(document, defs, cache);
this.conditions = this.parseConditions(document, this.fields, defs, cache);
this.fields = this.updateConditions();
}

/**
* Parse fields
* @param document
* @param contextURL
* @param defs
* @param cache
* @private
*/
private parseFields(document: any, index: number, defs?: any): FieldModel[] {
const fields: FieldModel[] = [];

if (!document || !document.properties) {
return fields;
}

const required = {};
if (document.required) {
for (const element of document.required) {
required[element] = true;
}
private parseFields(
document: ISchemaDocument,
defs: { [x: string]: ISchemaDocument },
cache: Map<string, SchemaDocumentModel> = new Map()
): FieldModel[] {
if (!document?.properties) {
return [];
}

const properties = Object.keys(document.properties);
for (const name of properties) {
if (name === '@context' || name === 'type') {
continue;
}

const property = document.properties[name];

const field = new FieldModel(name, property, !!required[name], index);
const required = new Set(document.required || []);
const subSchemas = defs || document.$defs || {};
const fields: FieldModel[] = [];
const properties = Object.entries(document.properties).filter(([name]) => name !== '@context' && name !== 'type');
for (const [name, property] of properties) {
const field = new FieldModel(name, property, required.has(name));
if (field.isRef) {
const subSchemas = defs || document.$defs;
const subDocument = subSchemas[field.type];
const subSchema = new SchemaDocumentModel(subDocument, index + 1, subSchemas);
let subSchema = cache.get(field.type);
if (!subSchema) {
const subDocument = subSchemas[field.type];
subSchema = new SchemaDocumentModel(subDocument, subSchemas, cache);
cache.set(field.type, subSchema);
}
field.setSubSchema(subSchema);
}
fields.push(field);
Expand All @@ -78,38 +78,47 @@ export class SchemaDocumentModel {
/**
* Parse conditions
* @param document
* @param index
* @param fields
* @param defs
* @param cache
* @private
*/
private parseConditions(
document: any,
index: number,
document: ISchemaDocument,
fields: FieldModel[],
defs: any = null
defs: { [x: string]: ISchemaDocument },
cache?: Map<string, SchemaDocumentModel>
): ConditionModel[] {
const conditions: ConditionModel[] = [];
if (!document || !document.allOf) {
return conditions;
return [];
}
const allOf = Object.keys(document.allOf);
for (const oneOf of allOf) {
const condition = document.allOf[oneOf];

const conditions: ConditionModel[] = [];
const fieldsMap = new Map(fields.map(field => [field.name, field]));
const combinedDefs = document.$defs || defs;

for (const condition of document.allOf) {
if (!condition.if) {
continue;
}
const ifConditionFieldName = Object.keys(condition.if.properties)[0];
const field = fieldsMap.get(ifConditionFieldName);
if (!field) {
continue;
}
const ifFieldValue = condition.if.properties[ifConditionFieldName].const;
const thenFields = this.parseFields(condition.then, combinedDefs, cache);
const elseFields = this.parseFields(condition.else, combinedDefs, cache);
conditions.push(new ConditionModel(
fields.find(field => field.name === ifConditionFieldName),
condition.if.properties[ifConditionFieldName].const,
this.parseFields(condition.then, index, document.$defs || defs),
this.parseFields(condition.else, index, document.$defs || defs),
index
field,
ifFieldValue,
thenFields,
elseFields
));
}
return conditions;
}

/**
* Update conditions
* @private
Expand Down Expand Up @@ -206,4 +215,14 @@ export class SchemaDocumentModel {

return Math.floor(total / rates.length);
}
}

/**
* Create model
* @param document
* @public
*/
public static from(document: ISchemaDocument): SchemaDocumentModel {
const cache = new Map<string, SchemaDocumentModel>();
return new SchemaDocumentModel(document, document?.$defs, cache);
}
}
Loading
Loading