[7.x] [a11y] add initial accessibility functional tests (#4358… (#50086)

* [a11y] add initial accessibility functional tests

* add accessibility jobs

* fix config path

* remove percy setup from scripts

* disable color-contrast rule

* apply changes from @myasonik

* define aria-controls/owns props even when suggestions aren't visible

* [ftr/a11y] only throw error when there are errors

* adding tests for management page

* add a11y test for management page

* adding ignore rules' to a11y

* accessibility test for kibana home

* 7 passing tests, 0 failures

* jest snapshot update

* support a11y test in pipeline job

* update a11y test script for pipelines

* use oss compatible ci setup

* fix exclude syntax

* add default exclusion syntax

.ci/jobs.yml

@@ -14,6 +14,7 @@ JOB:
   - kibana-ciGroup10
   - kibana-ciGroup11
   - kibana-ciGroup12
+  - kibana-accessibility
   - kibana-visualRegression
 
   # make sure all x-pack-ciGroups are listed in test/scripts/jenkins_xpack_ci_group.sh
@@ -28,6 +29,7 @@ JOB:
   - x-pack-ciGroup8
   - x-pack-ciGroup9
   - x-pack-ciGroup10
+  - x-pack-accessibility
   - x-pack-visualRegression
 
 # `~` is yaml for `null`

.ci/run.sh

@@ -21,6 +21,9 @@ kibana-ciGroup*)
 kibana-visualRegression*)
   ./test/scripts/jenkins_visual_regression.sh
   ;;
+kibana-accessibility*)
+  ./test/scripts/jenkins_accessibility.sh
+  ;;
 kibana-firefoxSmoke*)
   ./test/scripts/jenkins_firefox_smoke.sh
   ;;
@@ -34,6 +37,9 @@ x-pack-ciGroup*)
 x-pack-visualRegression*)
   ./test/scripts/jenkins_xpack_visual_regression.sh
   ;;
+x-pack-accessibility*)
+  ./test/scripts/jenkins_xpack_accessibility.sh
+  ;;
 x-pack-firefoxSmoke*)
   ./test/scripts/jenkins_xpack_firefox_smoke.sh
   ;;

.eslintrc.js

@@ -394,6 +394,7 @@ module.exports = {
         'x-pack/test/functional/apps/**/*.js',
         'x-pack/legacy/plugins/apm/**/*.js',
         'test/*/config.ts',
+        'test/*/{tests,test_suites,apis,apps}/**/*',
         'test/visual_regression/tests/**/*',
         'x-pack/test/*/{tests,test_suites,apis,apps}/**/*',
         'x-pack/test/*/*config.*ts',

Jenkinsfile

@@ -25,6 +25,7 @@ stage("Kibana Pipeline") { // This stage is just here to help the BlueOcean UI a
               'oss-ciGroup11': kibanaPipeline.getOssCiGroupWorker(11),
               'oss-ciGroup12': kibanaPipeline.getOssCiGroupWorker(12),
               'oss-firefoxSmoke': kibanaPipeline.getPostBuildWorker('firefoxSmoke', { runbld('./test/scripts/jenkins_firefox_smoke.sh', 'Execute kibana-firefoxSmoke') }),
+              'oss-accessibility': kibanaPipeline.getPostBuildWorker('accessibility', { runbld('./test/scripts/jenkins_accessibility.sh', 'Execute accessibility tests') }),
               'oss-visualRegression': kibanaPipeline.getPostBuildWorker('visualRegression', { runbld('./test/scripts/jenkins_visual_regression.sh', 'Execute kibana-visualRegression') }),
             ]),
             'kibana-xpack-agent': kibanaPipeline.withWorkers('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [
@@ -39,6 +40,7 @@ stage("Kibana Pipeline") { // This stage is just here to help the BlueOcean UI a
               'xpack-ciGroup9': kibanaPipeline.getXpackCiGroupWorker(9),
               'xpack-ciGroup10': kibanaPipeline.getXpackCiGroupWorker(10),
               'xpack-firefoxSmoke': kibanaPipeline.getPostBuildWorker('xpack-firefoxSmoke', { runbld('./test/scripts/jenkins_xpack_firefox_smoke.sh', 'Execute xpack-firefoxSmoke') }),
+              'xpack-accessibility': kibanaPipeline.getPostBuildWorker('xpack-accessibility', { runbld('./test/scripts/jenkins_xpack_accessibility.sh', 'Execute xpack-accessibility tests') }),
               'xpack-visualRegression': kibanaPipeline.getPostBuildWorker('xpack-visualRegression', { runbld('./test/scripts/jenkins_xpack_visual_regression.sh', 'Execute xpack-visualRegression') }),
             ]),
           ])

package.json

@@ -347,6 +347,7 @@
     "@typescript-eslint/parser": "^2.8.0",
     "angular-mocks": "^1.7.8",
     "archiver": "^3.1.1",
+    "axe-core": "^3.3.2",
     "babel-eslint": "^10.0.3",
     "babel-jest": "^24.9.0",
     "babel-plugin-dynamic-import-node": "^2.2.0",

src/legacy/core_plugins/kibana/public/management/sections/settings/components/field/field.js

@@ -195,7 +195,7 @@ export class Field extends PureComponent {
     this.setState({
       unsavedValue: newUnsavedValue,
       isInvalid,
-      error
+      error,
     });
   };
 
@@ -764,6 +764,7 @@ export class Field extends PureComponent {
               helpText={this.renderHelpText(setting)}
               describedByIds={[`${setting.name}-aria`]}
               className="mgtAdvancedSettings__fieldRow"
+              hasChildLabel={setting.type !== 'boolean'}
             >
               {this.renderField(setting)}
             </EuiFormRow>

src/legacy/ui/public/chrome/directives/kbn_chrome.html

@@ -1,4 +1,4 @@
-<div class="app-wrapper-panel">
+<div class="app-wrapper-panel" data-test-subj="appA11yRoot">
   <div id="globalBannerList"></div>
 
   <div

test/accessibility/apps/discover.ts

@@ -0,0 +1,46 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { FtrProviderContext } from '../ftr_provider_context';
+
+const FROM_TIME = '2015-09-19 06:31:44.000';
+const TO_TIME = '2015-09-23 18:31:44.000';
+
+export default function({ getService, getPageObjects }: FtrProviderContext) {
+  const PageObjects = getPageObjects(['common', 'timePicker']);
+  const a11y = getService('a11y');
+  const esArchiver = getService('esArchiver');
+  const kibanaServer = getService('kibanaServer');
+
+  describe('Discover', () => {
+    before(async () => {
+      await esArchiver.load('discover');
+      await esArchiver.loadIfNeeded('logstash_functional');
+      await kibanaServer.uiSettings.update({
+        defaultIndex: 'logstash-*',
+      });
+      await PageObjects.common.navigateToApp('discover');
+      await PageObjects.timePicker.setAbsoluteRange(FROM_TIME, TO_TIME);
+    });
+
+    it('main view', async () => {
+      await a11y.testAppSnapshot();
+    });
+  });
+}

test/accessibility/apps/home.ts

@@ -0,0 +1,35 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { FtrProviderContext } from '../ftr_provider_context';
+
+export default function({ getService, getPageObjects }: FtrProviderContext) {
+  const PageObjects = getPageObjects(['common']);
+  const a11y = getService('a11y');
+
+  describe('Kibana Home', () => {
+    before(async () => {
+      await PageObjects.common.navigateToApp('home');
+    });
+
+    it('Kibana Home view', async () => {
+      await a11y.testAppSnapshot();
+    });
+  });
+}

test/accessibility/apps/management.ts

@@ -0,0 +1,55 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { FtrProviderContext } from '../ftr_provider_context';
+
+export default function({ getService, getPageObjects }: FtrProviderContext) {
+  const PageObjects = getPageObjects(['common', 'settings']);
+  const a11y = getService('a11y');
+
+  describe('Management', () => {
+    before(async () => {
+      await PageObjects.common.navigateToApp('settings');
+    });
+
+    it('main view', async () => {
+      await a11y.testAppSnapshot();
+    });
+
+    it('index pattern page', async () => {
+      await PageObjects.settings.clickKibanaIndexPatterns();
+      await a11y.testAppSnapshot();
+    });
+
+    it('Single indexpattern view', async () => {
+      await PageObjects.settings.clickIndexPatternLogstash();
+      await a11y.testAppSnapshot();
+    });
+
+    it('Saved objects view', async () => {
+      await PageObjects.settings.clickKibanaSavedObjects();
+      await a11y.testAppSnapshot();
+    });
+
+    it('Advanced settings', async () => {
+      await PageObjects.settings.clickKibanaSettings();
+      await a11y.testAppSnapshot();
+    });
+  });
+}

test/accessibility/config.ts

@@ -0,0 +1,42 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { FtrConfigProviderContext } from '@kbn/test/types/ftr';
+import { services } from './services';
+import { pageObjects } from './page_objects';
+
+export default async function({ readConfigFile }: FtrConfigProviderContext) {
+  const functionalConfig = await readConfigFile(require.resolve('../functional/config'));
+
+  return {
+    ...functionalConfig.getAll(),
+
+    testFiles: [
+      require.resolve('./apps/discover'),
+      require.resolve('./apps/management'),
+      require.resolve('./apps/home'),
+    ],
+    pageObjects,
+    services,
+
+    junit: {
+      reportName: 'Accessibility Tests',
+    },
+  };
+}

test/accessibility/ftr_provider_context.d.ts

@@ -0,0 +1,25 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { GenericFtrProviderContext } from '@kbn/test/types/ftr';
+
+import { pageObjects } from './page_objects';
+import { services } from './services';
+
+export type FtrProviderContext = GenericFtrProviderContext<typeof services, typeof pageObjects>;

test/accessibility/page_objects.ts

@@ -0,0 +1,20 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export { pageObjects } from '../functional/page_objects';