From 0d802cb247ee77d0246824310a6a4e32df8a2454 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Tue, 25 Jun 2024 16:10:25 -0500 Subject: [PATCH] [Security Solution][Detection Engine] ML Cypress Improvements (#186831) ## Summary This PR contains a collection of fixes, mainly cherry-picked from #181926. Since some of these tests are failing sporadically on main, I'm opening this "test fix" PR for expedience while #181926 is still in review. ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [ ] [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was used on any tests changed * [Cypress * 100](https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/6385) * [Serverless Cypress * 100](https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/6386) --- .../rule_creation/machine_learning_rule.cy.ts | 27 +++++++++++------- .../cypress/objects/rule.ts | 4 +-- .../cypress/support/machine_learning.ts | 28 +++++++++++++++++++ .../cypress/tasks/alerts_detection_rules.ts | 2 +- .../cypress/tasks/create_new_rule.ts | 28 +++++++++++++------ 5 files changed, 67 insertions(+), 22 deletions(-) create mode 100644 x-pack/test/security_solution_cypress/cypress/support/machine_learning.ts diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/machine_learning_rule.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/machine_learning_rule.cy.ts index 9121e524ddbb4..0f90e406682f3 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/machine_learning_rule.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/machine_learning_rule.cy.ts @@ -4,13 +4,11 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { isArray } from 'lodash'; import { formatMitreAttackDescription, getHumanizedDuration } from '../../../../helpers/rules'; import { getMachineLearningRule } from '../../../../objects/rule'; import { - CUSTOM_RULES_BTN, RISK_SCORE, RULES_MANAGEMENT_TABLE, RULE_NAME, @@ -53,13 +51,27 @@ import { login } from '../../../../tasks/login'; import { visit } from '../../../../tasks/navigation'; import { openRuleManagementPageViaBreadcrumbs } from '../../../../tasks/rules_management'; import { CREATE_RULE_URL } from '../../../../urls/navigation'; +import { forceStopAndCloseJob } from '../../../../support/machine_learning'; +import { deleteAlertsAndRules } from '../../../../tasks/api_calls/common'; describe('Machine Learning rules', { tags: ['@ess', '@serverless'] }, () => { const expectedUrls = (getMachineLearningRule().references ?? []).join(''); const expectedFalsePositives = (getMachineLearningRule().false_positives ?? []).join(''); const expectedTags = (getMachineLearningRule().tags ?? []).join(''); const expectedMitre = formatMitreAttackDescription(getMachineLearningRule().threat ?? []); - const expectedNumberOfRules = 1; + const expectedJobText = [ + 'Unusual Linux Network Activity', + 'Anomalous Process for a Linux Population', + ].join(''); + + before(() => { + const machineLearningJobIds = ([] as string[]).concat( + getMachineLearningRule().machine_learning_job_id + ); + // ensure no ML jobs are started before the suite + machineLearningJobIds.forEach((jobId) => forceStopAndCloseJob({ jobId })); + deleteAlertsAndRules(); + }); beforeEach(() => { login(); @@ -75,9 +87,7 @@ describe('Machine Learning rules', { tags: ['@ess', '@serverless'] }, () => { createAndEnableRule(); openRuleManagementPageViaBreadcrumbs(); - cy.get(CUSTOM_RULES_BTN).should('have.text', 'Custom rules (1)'); - - expectNumberOfRules(RULES_MANAGEMENT_TABLE, expectedNumberOfRules); + expectNumberOfRules(RULES_MANAGEMENT_TABLE, 1); cy.get(RULE_NAME).should('have.text', mlRule.name); cy.get(RISK_SCORE).should('have.text', mlRule.risk_score); @@ -104,15 +114,12 @@ describe('Machine Learning rules', { tags: ['@ess', '@serverless'] }, () => { getDetails(ANOMALY_SCORE_DETAILS).should('have.text', mlRule.anomaly_threshold); getDetails(RULE_TYPE_DETAILS).should('have.text', 'Machine Learning'); getDetails(TIMELINE_TEMPLATE_DETAILS).should('have.text', 'None'); - const machineLearningJobsArray = isArray(mlRule.machine_learning_job_id) - ? mlRule.machine_learning_job_id - : [mlRule.machine_learning_job_id]; // With the #1912 ML rule improvement changes we enable jobs on rule creation. // Though, in cypress jobs enabling does not work reliably and job can be started or stopped. // Thus, we disable next check till we fix the issue with enabling jobs in cypress. // Relevant ticket: https://github.com/elastic/security-team/issues/5389 // cy.get(MACHINE_LEARNING_JOB_STATUS).should('have.text', 'StoppedStopped'); - cy.get(MACHINE_LEARNING_JOB_ID).should('have.text', machineLearningJobsArray.join('')); + cy.get(MACHINE_LEARNING_JOB_ID).should('have.text', expectedJobText); }); cy.get(SCHEDULE_DETAILS).within(() => { getDetails(RUNS_EVERY_DETAILS) diff --git a/x-pack/test/security_solution_cypress/cypress/objects/rule.ts b/x-pack/test/security_solution_cypress/cypress/objects/rule.ts index 04ba983664952..50e358515d922 100644 --- a/x-pack/test/security_solution_cypress/cypress/objects/rule.ts +++ b/x-pack/test/security_solution_cypress/cypress/objects/rule.ts @@ -361,8 +361,8 @@ export const getMachineLearningRule = ( ): MachineLearningRuleCreateProps => ({ type: 'machine_learning', machine_learning_job_id: [ - 'Unusual Linux Network Activity', - 'Anomalous Process for a Linux Population', + 'v3_linux_anomalous_network_activity', + 'v3_linux_anomalous_process_all_hosts', ], anomaly_threshold: 20, name: 'New ML Rule Test', diff --git a/x-pack/test/security_solution_cypress/cypress/support/machine_learning.ts b/x-pack/test/security_solution_cypress/cypress/support/machine_learning.ts new file mode 100644 index 0000000000000..e562a693865e3 --- /dev/null +++ b/x-pack/test/security_solution_cypress/cypress/support/machine_learning.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { rootRequest } from '../tasks/api_calls/common'; + +/** + * Calls the internal ML Jobs API to force stop the datafeed of, and force close + * the job with the given ID. + * + * @param jobId the ID of the ML job to stop and close + * @returns the response from the force stop and close job request + */ +export const forceStopAndCloseJob = ({ jobId }: { jobId: string }) => + rootRequest({ + headers: { + 'elastic-api-version': 1, + }, + method: 'POST', + url: '/internal/ml/jobs/force_stop_and_close_job', + failOnStatusCode: false, + body: { + jobId, + }, + }); diff --git a/x-pack/test/security_solution_cypress/cypress/tasks/alerts_detection_rules.ts b/x-pack/test/security_solution_cypress/cypress/tasks/alerts_detection_rules.ts index 97d6c34fdd040..7304a23f75e77 100644 --- a/x-pack/test/security_solution_cypress/cypress/tasks/alerts_detection_rules.ts +++ b/x-pack/test/security_solution_cypress/cypress/tasks/alerts_detection_rules.ts @@ -384,7 +384,7 @@ export const expectNumberOfRules = ( expectedNumber: number ) => { cy.log(`Expecting rules table to contain #${expectedNumber} rules`); - cy.get(tableSelector).find(RULES_ROW).should('have.length', expectedNumber); + cy.get(tableSelector).find(RULES_ROW).its('length').should('be.gte', expectedNumber); }; export const expectToContainRule = ( diff --git a/x-pack/test/security_solution_cypress/cypress/tasks/create_new_rule.ts b/x-pack/test/security_solution_cypress/cypress/tasks/create_new_rule.ts index d9f0120ab0199..e8be51d0d3731 100644 --- a/x-pack/test/security_solution_cypress/cypress/tasks/create_new_rule.ts +++ b/x-pack/test/security_solution_cypress/cypress/tasks/create_new_rule.ts @@ -802,21 +802,20 @@ export const continueFromDefineStep = () => { getDefineContinueButton().should('exist').click({ force: true }); }; -export const fillDefineMachineLearningRuleAndContinue = (rule: MachineLearningRuleCreateProps) => { +export const fillDefineMachineLearningRule = (rule: MachineLearningRuleCreateProps) => { const jobsAsArray = isArray(rule.machine_learning_job_id) ? rule.machine_learning_job_id : [rule.machine_learning_job_id]; - const text = jobsAsArray - .map((machineLearningJob) => `${machineLearningJob}{downArrow}{enter}`) - .join(''); cy.get(MACHINE_LEARNING_DROPDOWN_INPUT).click({ force: true }); - cy.get(MACHINE_LEARNING_DROPDOWN_INPUT).type(text); - - cy.get(MACHINE_LEARNING_DROPDOWN_INPUT).type('{esc}'); + cy.get(MACHINE_LEARNING_DROPDOWN_INPUT).type(optionsToComboboxText(jobsAsArray)); cy.get(ANOMALY_THRESHOLD_INPUT).type(`{selectall}${rule.anomaly_threshold}`, { force: true, }); +}; + +export const fillDefineMachineLearningRuleAndContinue = (rule: MachineLearningRuleCreateProps) => { + fillDefineMachineLearningRule(rule); getDefineContinueButton().should('exist').click({ force: true }); }; @@ -909,9 +908,20 @@ export const enablesAndPopulatesThresholdSuppression = ( cy.get(ALERT_SUPPRESSION_DURATION_PER_TIME_INTERVAL).should('be.enabled').should('be.checked'); }; +const optionsToComboboxText = (options: string[]) => { + return options.map((o) => `${o}{downArrow}{enter}{esc}`).join(''); +}; + export const fillAlertSuppressionFields = (fields: string[]) => { - fields.forEach((field) => { - cy.get(ALERT_SUPPRESSION_FIELDS_COMBO_BOX).type(`${field}{enter}`); + cy.get(ALERT_SUPPRESSION_FIELDS_COMBO_BOX).should('not.be.disabled'); + cy.get(ALERT_SUPPRESSION_FIELDS_COMBO_BOX).click(); + cy.get(ALERT_SUPPRESSION_FIELDS_COMBO_BOX).type(optionsToComboboxText(fields)); +}; + +export const clearAlertSuppressionFields = () => { + cy.get(ALERT_SUPPRESSION_FIELDS_COMBO_BOX).should('not.be.disabled'); + cy.get(ALERT_SUPPRESSION_FIELDS).within(() => { + cy.get(COMBO_BOX_CLEAR_BTN).click(); }); };