Skip to content

Commit

Permalink
Dynamically assert Rule Details based on titles
Browse files Browse the repository at this point in the history 
Rule Details are unfortunately unstructured: they're an array of <dt>s
and <dd>s without any hierarchy. To address this, tests
were previously hardcoding the order of these fields, and assertions
were performed by querying for all <dd>s and then indexing with the
hardcoded number (e.g. ABOUT_FALSE_POSITIVES).

However, in addition to being unstructured, these fields are also
_dynamic_, and will be present/absent depending on the data of the given
rule. Thus, we started needing multiple orderings for the different
combinations of rule fields/rule types.

In the absence of refactoring how we build rule details, I'm introducing
a simple helper function to fetch the relevant <dd> by the corresponding
<dt>s text. This should be more robust to change and more declarative.
  • Loading branch information
@rylnd
rylnd committed Sep 11, 2020
1 parent b174f44 commit 569980b
Show file tree
Hide file tree
Showing 5 changed files with 177 additions and 248 deletions.
94 changes: 45 additions & 49 deletions x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules_custom.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,26 +16,16 @@ import {
SHOWING_RULES_TEXT,
} from '../screens/alerts_detection_rules';
import {
ABOUT_FALSE_POSITIVES,
ABOUT_INVESTIGATION_NOTES,
ABOUT_MITRE,
ABOUT_RISK,
ABOUT_RULE_DESCRIPTION,
ABOUT_SEVERITY,
ABOUT_STEP,
ABOUT_TAGS,
ABOUT_URLS,
DEFINITION_CUSTOM_QUERY,
DEFINITION_INDEX_PATTERNS,
DEFINITION_TIMELINE,
DEFINITION_STEP,
INVESTIGATION_NOTES_MARKDOWN,
INVESTIGATION_NOTES_TOGGLE,
RULE_ABOUT_DETAILS_HEADER_TOGGLE,
RULE_NAME_HEADER,
SCHEDULE_LOOPBACK,
SCHEDULE_RUNS,
SCHEDULE_STEP,
getDescriptionForTitle,
ABOUT_DETAILS,
DEFINITION_DETAILS,
SCHEDULE_DETAILS,
} from '../screens/rule_details';

import {
Expand Down Expand Up @@ -173,32 +163,35 @@ describe('Detection rules, custom', () => {
cy.get(RULE_NAME_HEADER).invoke('text').should('eql', `${newRule.name} Beta`);

cy.get(ABOUT_RULE_DESCRIPTION).invoke('text').should('eql', newRule.description);
cy.get(ABOUT_STEP).eq(ABOUT_SEVERITY).invoke('text').should('eql', newRule.severity);
cy.get(ABOUT_STEP).eq(ABOUT_RISK).invoke('text').should('eql', newRule.riskScore);
cy.get(ABOUT_STEP).eq(ABOUT_URLS).invoke('text').should('eql', expectedUrls);
cy.get(ABOUT_STEP)
.eq(ABOUT_FALSE_POSITIVES)
.invoke('text')
.should('eql', expectedFalsePositives);
cy.get(ABOUT_STEP).eq(ABOUT_MITRE).invoke('text').should('eql', expectedMitre);
cy.get(ABOUT_STEP).eq(ABOUT_TAGS).invoke('text').should('eql', expectedTags);
cy.get(ABOUT_DETAILS).within(() => {
getDescriptionForTitle('Severity').invoke('text').should('eql', newRule.severity);
getDescriptionForTitle('Risk score').invoke('text').should('eql', newRule.riskScore);
getDescriptionForTitle('Reference URLs').invoke('text').should('eql', expectedUrls);
getDescriptionForTitle('False positive examples')
.invoke('text')
.should('eql', expectedFalsePositives);
getDescriptionForTitle('MITRE ATT&CK').invoke('text').should('eql', expectedMitre);
getDescriptionForTitle('Tags').invoke('text').should('eql', expectedTags);
});

cy.get(RULE_ABOUT_DETAILS_HEADER_TOGGLE).eq(INVESTIGATION_NOTES_TOGGLE).click({ force: true });
cy.get(ABOUT_INVESTIGATION_NOTES).invoke('text').should('eql', INVESTIGATION_NOTES_MARKDOWN);

cy.get(DEFINITION_INDEX_PATTERNS).then((patterns) => {
cy.wrap(patterns).each((pattern, index) => {
cy.wrap(pattern).invoke('text').should('eql', expectedIndexPatterns[index]);
});
cy.get(DEFINITION_DETAILS).within(() => {
getDescriptionForTitle('Index patterns')
.invoke('text')
.should('eql', expectedIndexPatterns.join(''));
getDescriptionForTitle('Custom query')
.invoke('text')
.should('eql', `${newRule.customQuery} `);
getDescriptionForTitle('Rule type').invoke('text').should('eql', 'Query');
getDescriptionForTitle('Timeline template').invoke('text').should('eql', 'None');
});

cy.get(SCHEDULE_DETAILS).within(() => {
getDescriptionForTitle('Runs every').invoke('text').should('eql', '5m');
getDescriptionForTitle('Additional look-back time').invoke('text').should('eql', '1m');
});
cy.get(DEFINITION_STEP)
.eq(DEFINITION_CUSTOM_QUERY)
.invoke('text')
.should('eql', `${newRule.customQuery} `);
cy.get(DEFINITION_STEP).eq(DEFINITION_TIMELINE).invoke('text').should('eql', 'None');

cy.get(SCHEDULE_STEP).eq(SCHEDULE_RUNS).invoke('text').should('eql', '5m');
cy.get(SCHEDULE_STEP).eq(SCHEDULE_LOOPBACK).invoke('text').should('eql', '1m');
});
});

Expand Down Expand Up @@ -328,27 +321,30 @@ describe('Deletes custom rules', () => {
cy.get(RULE_NAME_HEADER).invoke('text').should('eql', `${editedRule.name} Beta`);

cy.get(ABOUT_RULE_DESCRIPTION).invoke('text').should('eql', editedRule.description);
cy.get(ABOUT_STEP).eq(ABOUT_SEVERITY).invoke('text').should('eql', editedRule.severity);
cy.get(ABOUT_STEP).eq(ABOUT_RISK).invoke('text').should('eql', editedRule.riskScore);
cy.get(ABOUT_STEP).eq(2).invoke('text').should('eql', expectedTags);
cy.get(ABOUT_DETAILS).within(() => {
getDescriptionForTitle('Severity').invoke('text').should('eql', editedRule.severity);
getDescriptionForTitle('Risk score').invoke('text').should('eql', editedRule.riskScore);
getDescriptionForTitle('Tags').invoke('text').should('eql', expectedTags);
});

cy.get(RULE_ABOUT_DETAILS_HEADER_TOGGLE).eq(INVESTIGATION_NOTES_TOGGLE).click({ force: true });
cy.get(ABOUT_INVESTIGATION_NOTES).invoke('text').should('eql', editedRule.note);

cy.get(DEFINITION_INDEX_PATTERNS).then((patterns) => {
cy.wrap(patterns).each((pattern, index) => {
cy.wrap(pattern).invoke('text').should('eql', expectedIndexPatterns[index]);
});
cy.get(DEFINITION_DETAILS).within(() => {
getDescriptionForTitle('Index patterns')
.invoke('text')
.should('eql', expectedIndexPatterns.join(''));
getDescriptionForTitle('Custom query')
.invoke('text')
.should('eql', `${editedRule.customQuery} `);
getDescriptionForTitle('Rule type').invoke('text').should('eql', 'Query');
getDescriptionForTitle('Timeline template').invoke('text').should('eql', 'None');
});
cy.get(DEFINITION_STEP)
.eq(DEFINITION_CUSTOM_QUERY)
.invoke('text')
.should('eql', `${editedRule.customQuery} `);
cy.get(DEFINITION_STEP).eq(2).invoke('text').should('eql', 'Query');
cy.get(DEFINITION_STEP).eq(DEFINITION_TIMELINE).invoke('text').should('eql', 'None');

if (editedRule.interval) {
cy.get(SCHEDULE_STEP).eq(SCHEDULE_RUNS).invoke('text').should('eql', editedRule.interval);
cy.get(SCHEDULE_DETAILS).within(() => {
getDescriptionForTitle('Runs every').invoke('text').should('eql', editedRule.interval);
});
}
});
});
81 changes: 36 additions & 45 deletions x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules_ml.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,24 +16,14 @@ import {
SEVERITY,
} from '../screens/alerts_detection_rules';
import {
ABOUT_FALSE_POSITIVES,
ABOUT_MITRE,
ABOUT_RISK,
ABOUT_RULE_DESCRIPTION,
ABOUT_SEVERITY,
ABOUT_STEP,
ABOUT_TAGS,
ABOUT_URLS,
ANOMALY_SCORE,
DEFINITION_TIMELINE,
DEFINITION_STEP,
MACHINE_LEARNING_JOB_ID,
MACHINE_LEARNING_JOB_STATUS,
RULE_NAME_HEADER,
SCHEDULE_LOOPBACK,
SCHEDULE_RUNS,
SCHEDULE_STEP,
RULE_TYPE,
getDescriptionForTitle,
ABOUT_DETAILS,
DEFINITION_DETAILS,
SCHEDULE_DETAILS,
} from '../screens/rule_details';

import {
Expand Down Expand Up @@ -126,36 +116,37 @@ describe('Detection rules, machine learning', () => {
cy.get(RULE_NAME_HEADER).invoke('text').should('eql', `${machineLearningRule.name} Beta`);

cy.get(ABOUT_RULE_DESCRIPTION).invoke('text').should('eql', machineLearningRule.description);
cy.get(ABOUT_STEP)
.eq(ABOUT_SEVERITY)
.invoke('text')
.should('eql', machineLearningRule.severity);
cy.get(ABOUT_STEP).eq(ABOUT_RISK).invoke('text').should('eql', machineLearningRule.riskScore);
cy.get(ABOUT_STEP).eq(ABOUT_URLS).invoke('text').should('eql', expectedUrls);
cy.get(ABOUT_STEP)
.eq(ABOUT_FALSE_POSITIVES)
.invoke('text')
.should('eql', expectedFalsePositives);
cy.get(ABOUT_STEP).eq(ABOUT_MITRE).invoke('text').should('eql', expectedMitre);
cy.get(ABOUT_STEP).eq(ABOUT_TAGS).invoke('text').should('eql', expectedTags);

cy.get(DEFINITION_STEP).eq(RULE_TYPE).invoke('text').should('eql', 'Machine Learning');
cy.get(DEFINITION_STEP)
.eq(ANOMALY_SCORE)
.invoke('text')
.should('eql', machineLearningRule.anomalyScoreThreshold);
cy.get(DEFINITION_STEP)
.get(MACHINE_LEARNING_JOB_STATUS)
.invoke('text')
.should('eql', 'Stopped');
cy.get(DEFINITION_STEP)
.get(MACHINE_LEARNING_JOB_ID)
.invoke('text')
.should('eql', machineLearningRule.machineLearningJob);

cy.get(DEFINITION_STEP).eq(DEFINITION_TIMELINE).invoke('text').should('eql', 'None');

cy.get(SCHEDULE_STEP).eq(SCHEDULE_RUNS).invoke('text').should('eql', '5m');
cy.get(SCHEDULE_STEP).eq(SCHEDULE_LOOPBACK).invoke('text').should('eql', '1m');
cy.get(ABOUT_DETAILS).within(() => {
getDescriptionForTitle('Severity').invoke('text').should('eql', machineLearningRule.severity);
getDescriptionForTitle('Risk score')
.invoke('text')
.should('eql', machineLearningRule.riskScore);
getDescriptionForTitle('Reference URLs').invoke('text').should('eql', expectedUrls);
getDescriptionForTitle('False positive examples')
.invoke('text')
.should('eql', expectedFalsePositives);
getDescriptionForTitle('MITRE ATT&CK').invoke('text').should('eql', expectedMitre);
getDescriptionForTitle('Tags').invoke('text').should('eql', expectedTags);
});

cy.get(DEFINITION_DETAILS).within(() => {
getDescriptionForTitle('Anomaly score')
.invoke('text')
.should('eql', machineLearningRule.anomalyScoreThreshold);
getDescriptionForTitle('Anomaly score')
.invoke('text')
.should('eql', machineLearningRule.anomalyScoreThreshold);
getDescriptionForTitle('Rule type').invoke('text').should('eql', 'Machine Learning');
getDescriptionForTitle('Timeline template').invoke('text').should('eql', 'None');
cy.get(MACHINE_LEARNING_JOB_STATUS).invoke('text').should('eql', 'Stopped');
cy.get(MACHINE_LEARNING_JOB_ID)
.invoke('text')
.should('eql', machineLearningRule.machineLearningJob);
});

cy.get(SCHEDULE_DETAILS).within(() => {
getDescriptionForTitle('Runs every').invoke('text').should('eql', '5m');
getDescriptionForTitle('Additional look-back time').invoke('text').should('eql', '1m');
});
});
});
112 changes: 50 additions & 62 deletions x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules_override.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,29 +16,17 @@ import {
} from '../screens/alerts_detection_rules';
import {
ABOUT_INVESTIGATION_NOTES,
ABOUT_OVERRIDE_FALSE_POSITIVES,
ABOUT_OVERRIDE_MITRE,
ABOUT_OVERRIDE_NAME_OVERRIDE,
ABOUT_OVERRIDE_RISK,
ABOUT_OVERRIDE_RISK_OVERRIDE,
ABOUT_OVERRIDE_SEVERITY_OVERRIDE,
ABOUT_OVERRIDE_TAGS,
ABOUT_OVERRIDE_TIMESTAMP_OVERRIDE,
ABOUT_OVERRIDE_URLS,
ABOUT_RULE_DESCRIPTION,
ABOUT_SEVERITY,
ABOUT_STEP,
DEFINITION_CUSTOM_QUERY,
DEFINITION_INDEX_PATTERNS,
DEFINITION_TIMELINE,
DEFINITION_STEP,
INVESTIGATION_NOTES_MARKDOWN,
INVESTIGATION_NOTES_TOGGLE,
RULE_ABOUT_DETAILS_HEADER_TOGGLE,
RULE_NAME_HEADER,
SCHEDULE_LOOPBACK,
SCHEDULE_RUNS,
SCHEDULE_STEP,
ABOUT_DETAILS,
getDescriptionForTitle,
DEFINITION_DETAILS,
SCHEDULE_DETAILS,
DETAILS_TITLE,
DETAILS_DESCRIPTION,
} from '../screens/rule_details';

import {
Expand Down Expand Up @@ -141,56 +129,56 @@ describe('Detection rules, override', () => {

const expectedOverrideSeverities = ['Low', 'Medium', 'High', 'Critical'];

cy.get(ABOUT_STEP).eq(ABOUT_SEVERITY).invoke('text').should('eql', newOverrideRule.severity);
newOverrideRule.severityOverride.forEach((severity, i) => {
cy.get(ABOUT_STEP)
.eq(ABOUT_OVERRIDE_SEVERITY_OVERRIDE + i)
cy.get(ABOUT_DETAILS).within(() => {
getDescriptionForTitle('Severity').invoke('text').should('eql', newOverrideRule.severity);
getDescriptionForTitle('Risk score').invoke('text').should('eql', newOverrideRule.riskScore);
getDescriptionForTitle('Risk score override')
.invoke('text')
.should(
'eql',
`${severity.sourceField}:${severity.sourceValue}${expectedOverrideSeverities[i]}`
);
.should('eql', `${newOverrideRule.riskOverride}signal.rule.risk_score`);
getDescriptionForTitle('Rule name override')
.invoke('text')
.should('eql', newOverrideRule.nameOverride);
getDescriptionForTitle('Reference URLs').invoke('text').should('eql', expectedUrls);
getDescriptionForTitle('False positive examples')
.invoke('text')
.should('eql', expectedFalsePositives);
getDescriptionForTitle('MITRE ATT&CK').invoke('text').should('eql', expectedMitre);
getDescriptionForTitle('Tags').invoke('text').should('eql', expectedTags);
getDescriptionForTitle('Timestamp override')
.invoke('text')
.should('eql', newOverrideRule.timestampOverride);
cy.contains(DETAILS_TITLE, 'Severity override')
.invoke('index', DETAILS_TITLE) // get index relative to other titles, not all siblings
.then((severityOverrideIndex) => {
newOverrideRule.severityOverride.forEach((severity, i) => {
cy.get(DETAILS_DESCRIPTION)
.eq(severityOverrideIndex + i)
.invoke('text')
.should(
'eql',
`${severity.sourceField}:${severity.sourceValue}${expectedOverrideSeverities[i]}`
);
});
});
});

cy.get(ABOUT_STEP)
.eq(ABOUT_OVERRIDE_RISK)
.invoke('text')
.should('eql', newOverrideRule.riskScore);
cy.get(ABOUT_STEP)
.eq(ABOUT_OVERRIDE_RISK_OVERRIDE)
.invoke('text')
.should('eql', `${newOverrideRule.riskOverride}signal.rule.risk_score`);
cy.get(ABOUT_STEP).eq(ABOUT_OVERRIDE_URLS).invoke('text').should('eql', expectedUrls);
cy.get(ABOUT_STEP)
.eq(ABOUT_OVERRIDE_FALSE_POSITIVES)
.invoke('text')
.should('eql', expectedFalsePositives);
cy.get(ABOUT_STEP)
.eq(ABOUT_OVERRIDE_NAME_OVERRIDE)
.invoke('text')
.should('eql', newOverrideRule.nameOverride);
cy.get(ABOUT_STEP).eq(ABOUT_OVERRIDE_MITRE).invoke('text').should('eql', expectedMitre);
cy.get(ABOUT_STEP)
.eq(ABOUT_OVERRIDE_TIMESTAMP_OVERRIDE)
.invoke('text')
.should('eql', newOverrideRule.timestampOverride);
cy.get(ABOUT_STEP).eq(ABOUT_OVERRIDE_TAGS).invoke('text').should('eql', expectedTags);

cy.get(RULE_ABOUT_DETAILS_HEADER_TOGGLE).eq(INVESTIGATION_NOTES_TOGGLE).click({ force: true });
cy.get(ABOUT_INVESTIGATION_NOTES).invoke('text').should('eql', INVESTIGATION_NOTES_MARKDOWN);

cy.get(DEFINITION_INDEX_PATTERNS).then((patterns) => {
cy.wrap(patterns).each((pattern, index) => {
cy.wrap(pattern).invoke('text').should('eql', expectedIndexPatterns[index]);
});
cy.get(DEFINITION_DETAILS).within(() => {
getDescriptionForTitle('Index patterns')
.invoke('text')
.should('eql', expectedIndexPatterns.join(''));
getDescriptionForTitle('Custom query')
.invoke('text')
.should('eql', `${newOverrideRule.customQuery} `);
getDescriptionForTitle('Rule type').invoke('text').should('eql', 'Query');
getDescriptionForTitle('Timeline template').invoke('text').should('eql', 'None');
});

cy.get(SCHEDULE_DETAILS).within(() => {
getDescriptionForTitle('Runs every').invoke('text').should('eql', '5m');
getDescriptionForTitle('Additional look-back time').invoke('text').should('eql', '1m');
});
cy.get(DEFINITION_STEP)
.eq(DEFINITION_CUSTOM_QUERY)
.invoke('text')
.should('eql', `${newOverrideRule.customQuery} `);
cy.get(DEFINITION_STEP).eq(DEFINITION_TIMELINE).invoke('text').should('eql', 'None');

cy.get(SCHEDULE_STEP).eq(SCHEDULE_RUNS).invoke('text').should('eql', '5m');
cy.get(SCHEDULE_STEP).eq(SCHEDULE_LOOPBACK).invoke('text').should('eql', '1m');
});
});
Loading

0 comments on commit 569980b

Please sign in to comment.