Skip to content
This repository has been archived by the owner on Jan 13, 2025. It is now read-only.

Commit

Permalink
chore(infrastructure): Disable flaky Edge tests (#3334)
Browse files Browse the repository at this point in the history 
### What it does

- Disables 7 `mdc-typography` and `mdc-text-field--textarea` tests in Edge
- Even with **6** retries, Edge was still flaking frequently enough to fail lots of tests:
    - https://travis-ci.com/material-components/material-components-web/jobs/139469358
    - https://storage.googleapis.com/mdc-web-screenshot-tests/travis/2018/08/11/18_50_20_627/report/report.html
- More examples of flaky diffs (just a small subset):
    - [4 diffs](https://storage.googleapis.com/mdc-web-screenshot-tests/advorak/2018/08/08/03_20_16_788/report/report.html)
    - [4 diffs](https://storage.googleapis.com/mdc-web-screenshot-tests/advorak/2018/08/08/03_58_53_044/report/report.html)
    - [4 diffs](https://storage.googleapis.com/mdc-web-screenshot-tests/advorak/2018/08/08/04_01_04_052/report/report.html)
    - [4 diffs](https://storage.googleapis.com/mdc-web-screenshot-tests/advorak/2018/08/08/04_14_34_272/report/report.html)
    - [4 diffs](https://storage.googleapis.com/mdc-web-screenshot-tests/advorak/2018/08/08/04_51_41_853/report/report.html)
    - [2 diffs](https://storage.googleapis.com/mdc-web-screenshot-tests/advorak/2018/08/08/04_16_46_248/report/report.html)
    - [2 diffs](https://storage.googleapis.com/mdc-web-screenshot-tests/advorak/2018/08/08/04_23_40_378/report/report.html)
    - [2 diffs](https://storage.googleapis.com/mdc-web-screenshot-tests/advorak/2018/08/08/04_30_22_914/report/report.html)
    - [1 diff](https://storage.googleapis.com/mdc-web-screenshot-tests/advorak/2018/08/08/03_23_27_258/report/report.html)
    - [1 diff](https://storage.googleapis.com/mdc-web-screenshot-tests/advorak/2018/08/08/04_59_42_533/report/report.html)
    - [1 diff](https://storage.googleapis.com/mdc-web-screenshot-tests/advorak/2018/08/08/05_02_52_940/report/report.html)
    - [1 diff](https://storage.googleapis.com/mdc-web-screenshot-tests/advorak/2018/08/08/05_17_00_128/report/report.html)

### Example output

#### Before:

![image](https://user-images.githubusercontent.com/409245/44006401-e0bbfd22-9e38-11e8-9b8c-0a2a806c5f4d.png)

#### After:

![image](https://user-images.githubusercontent.com/409245/44006409-f2857cfe-9e38-11e8-8cd0-8970e7f54786.png)
  • Loading branch information
@acdvorak
acdvorak authored Aug 12, 2018
1 parent f54d42e commit 1a7f6e7
Show file tree
Hide file tree
Showing 7 changed files with 78 additions and 14 deletions.
19 changes: 17 additions & 2 deletions test/screenshot/diffing.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,18 @@

"config_overrides": [
{
"description": "IE and Edge flake more frequently than other browsers, especially with font rendering",
"browser_regex_patterns": [
"desktop_windows_edge@latest",
"desktop_windows_ie@11"
],
"custom_config": {
"max_retries": 4,
"fonts_loaded_reflow_delay_ms": 300
"max_retries": 3,
"fonts_loaded_reflow_delay_ms": 250
}
},
{
"description": "IE and Edge are extra flaky on Textfield and Typography pages for some reason",
"browser_regex_patterns": [
"desktop_windows_edge@latest",
"desktop_windows_ie@11"
Expand All @@ -39,6 +41,19 @@
"max_retries": 6,
"fonts_loaded_reflow_delay_ms": 500
}
},
{
"description": "Edge flakes so frequently on certain pages that it needs to be disabled entirely",
"browser_regex_patterns": [
"desktop_windows_edge@latest"
],
"url_regex_patterns": [
"mdc-textfield/.*textarea",
"mdc-typography"
],
"custom_config": {
"skip_all": true
}
}
]
}
Expand Down
7 changes: 0 additions & 7 deletions test/screenshot/golden.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -714,7 +714,6 @@
"public_url": "https://storage.googleapis.com/mdc-web-screenshot-tests/williamernest/2018/08/01/15_41_51_957/spec/mdc-textfield/classes/textarea-disabled.html",
"screenshots": {
"desktop_windows_chrome@latest": "https://storage.googleapis.com/mdc-web-screenshot-tests/advorak/2018/08/08/08_35_31_316/spec/mdc-textfield/classes/textarea-disabled.html.windows_chrome_68.png",
"desktop_windows_edge@latest": "https://storage.googleapis.com/mdc-web-screenshot-tests/advorak/2018/08/08/05_21_53_977/spec/mdc-textfield/classes/textarea-disabled.html.windows_edge_17.png",
"desktop_windows_firefox@latest": "https://storage.googleapis.com/mdc-web-screenshot-tests/advorak/2018/08/08/08_35_31_316/spec/mdc-textfield/classes/textarea-disabled.html.windows_firefox_61.png",
"desktop_windows_ie@11": "https://storage.googleapis.com/mdc-web-screenshot-tests/advorak/2018/08/08/08_35_31_316/spec/mdc-textfield/classes/textarea-disabled.html.windows_ie_11.png"
}
Expand All @@ -723,7 +722,6 @@
"public_url": "https://storage.googleapis.com/mdc-web-screenshot-tests/williamernest/2018/08/01/15_41_51_957/spec/mdc-textfield/classes/textarea-focused.html",
"screenshots": {
"desktop_windows_chrome@latest": "https://storage.googleapis.com/mdc-web-screenshot-tests/advorak/2018/08/08/08_35_31_316/spec/mdc-textfield/classes/textarea-focused.html.windows_chrome_68.png",
"desktop_windows_edge@latest": "https://storage.googleapis.com/mdc-web-screenshot-tests/advorak/2018/08/08/05_21_53_977/spec/mdc-textfield/classes/textarea-focused.html.windows_edge_17.png",
"desktop_windows_firefox@latest": "https://storage.googleapis.com/mdc-web-screenshot-tests/advorak/2018/08/08/08_35_31_316/spec/mdc-textfield/classes/textarea-focused.html.windows_firefox_61.png",
"desktop_windows_ie@11": "https://storage.googleapis.com/mdc-web-screenshot-tests/advorak/2018/08/08/08_35_31_316/spec/mdc-textfield/classes/textarea-focused.html.windows_ie_11.png"
}
Expand All @@ -732,7 +730,6 @@
"public_url": "https://storage.googleapis.com/mdc-web-screenshot-tests/williamernest/2018/08/01/15_41_51_957/spec/mdc-textfield/classes/textarea-invalid.html",
"screenshots": {
"desktop_windows_chrome@latest": "https://storage.googleapis.com/mdc-web-screenshot-tests/advorak/2018/08/08/08_35_31_316/spec/mdc-textfield/classes/textarea-invalid.html.windows_chrome_68.png",
"desktop_windows_edge@latest": "https://storage.googleapis.com/mdc-web-screenshot-tests/advorak/2018/08/08/05_21_53_977/spec/mdc-textfield/classes/textarea-invalid.html.windows_edge_17.png",
"desktop_windows_firefox@latest": "https://storage.googleapis.com/mdc-web-screenshot-tests/advorak/2018/08/08/08_35_31_316/spec/mdc-textfield/classes/textarea-invalid.html.windows_firefox_61.png",
"desktop_windows_ie@11": "https://storage.googleapis.com/mdc-web-screenshot-tests/advorak/2018/08/08/08_35_31_316/spec/mdc-textfield/classes/textarea-invalid.html.windows_ie_11.png"
}
Expand All @@ -741,7 +738,6 @@
"public_url": "https://storage.googleapis.com/mdc-web-screenshot-tests/williamernest/2018/08/01/15_41_51_957/spec/mdc-textfield/classes/textarea.html",
"screenshots": {
"desktop_windows_chrome@latest": "https://storage.googleapis.com/mdc-web-screenshot-tests/advorak/2018/08/08/08_35_31_316/spec/mdc-textfield/classes/textarea.html.windows_chrome_68.png",
"desktop_windows_edge@latest": "https://storage.googleapis.com/mdc-web-screenshot-tests/travis/2018/08/08/16_51_37_749/spec/mdc-textfield/classes/textarea.html.windows_edge_17.png",
"desktop_windows_firefox@latest": "https://storage.googleapis.com/mdc-web-screenshot-tests/advorak/2018/08/08/08_35_31_316/spec/mdc-textfield/classes/textarea.html.windows_firefox_61.png",
"desktop_windows_ie@11": "https://storage.googleapis.com/mdc-web-screenshot-tests/advorak/2018/08/08/08_35_31_316/spec/mdc-textfield/classes/textarea.html.windows_ie_11.png"
}
Expand All @@ -750,7 +746,6 @@
"public_url": "https://storage.googleapis.com/mdc-web-screenshot-tests/advorak/2018/08/03/00_19_30_622/spec/mdc-typography/classes/baseline-large.html",
"screenshots": {
"desktop_windows_chrome@latest": "https://storage.googleapis.com/mdc-web-screenshot-tests/advorak/2018/08/08/08_47_39_000/spec/mdc-typography/classes/baseline-large.html.windows_chrome_68.png",
"desktop_windows_edge@latest": "https://storage.googleapis.com/mdc-web-screenshot-tests/advorak/2018/08/08/08_47_39_000/spec/mdc-typography/classes/baseline-large.html.windows_edge_17.png",
"desktop_windows_firefox@latest": "https://storage.googleapis.com/mdc-web-screenshot-tests/advorak/2018/08/08/08_47_39_000/spec/mdc-typography/classes/baseline-large.html.windows_firefox_61.png",
"desktop_windows_ie@11": "https://storage.googleapis.com/mdc-web-screenshot-tests/advorak/2018/08/08/08_47_39_000/spec/mdc-typography/classes/baseline-large.html.windows_ie_11.png"
}
Expand All @@ -759,7 +754,6 @@
"public_url": "https://storage.googleapis.com/mdc-web-screenshot-tests/advorak/2018/08/03/00_19_30_622/spec/mdc-typography/classes/baseline-small.html",
"screenshots": {
"desktop_windows_chrome@latest": "https://storage.googleapis.com/mdc-web-screenshot-tests/advorak/2018/08/08/08_47_39_000/spec/mdc-typography/classes/baseline-small.html.windows_chrome_68.png",
"desktop_windows_edge@latest": "https://storage.googleapis.com/mdc-web-screenshot-tests/advorak/2018/08/08/08_55_59_177/spec/mdc-typography/classes/baseline-small.html.windows_edge_17.png",
"desktop_windows_firefox@latest": "https://storage.googleapis.com/mdc-web-screenshot-tests/advorak/2018/08/08/08_47_39_000/spec/mdc-typography/classes/baseline-small.html.windows_firefox_61.png",
"desktop_windows_ie@11": "https://storage.googleapis.com/mdc-web-screenshot-tests/advorak/2018/08/08/08_47_39_000/spec/mdc-typography/classes/baseline-small.html.windows_ie_11.png"
}
Expand All @@ -768,7 +762,6 @@
"public_url": "https://storage.googleapis.com/mdc-web-screenshot-tests/travis/2018/08/03/13_04_55_027/spec/mdc-typography/mixins/mixins.html",
"screenshots": {
"desktop_windows_chrome@latest": "https://storage.googleapis.com/mdc-web-screenshot-tests/advorak/2018/08/08/08_47_39_000/spec/mdc-typography/mixins/mixins.html.windows_chrome_68.png",
"desktop_windows_edge@latest": "https://storage.googleapis.com/mdc-web-screenshot-tests/advorak/2018/08/08/08_55_59_177/spec/mdc-typography/mixins/mixins.html.windows_edge_17.png",
"desktop_windows_firefox@latest": "https://storage.googleapis.com/mdc-web-screenshot-tests/advorak/2018/08/08/08_47_39_000/spec/mdc-typography/mixins/mixins.html.windows_firefox_61.png",
"desktop_windows_ie@11": "https://storage.googleapis.com/mdc-web-screenshot-tests/advorak/2018/08/08/08_47_39_000/spec/mdc-typography/mixins/mixins.html.windows_ie_11.png"
}
Expand Down
2 changes: 1 addition & 1 deletion test/screenshot/infra/commands/test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -275,7 +275,7 @@ class TestCommand {
const masterScreenshots = masterDiffReportData.screenshots;
const masterGitRev = masterDiffReportData.meta.golden_diff_base.git_revision;

const numTotal = masterScreenshots.actual_screenshot_list.length;
const numTotal = masterScreenshots.runnable_screenshot_list.length;
const numChanged =
masterScreenshots.changed_screenshot_list.length +
masterScreenshots.added_screenshot_list.length +
Expand Down
6 changes: 4 additions & 2 deletions test/screenshot/infra/lib/report-builder.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -676,7 +676,8 @@ class ReportBuilder {

for (const userAgent of allUserAgents) {
const userAgentAlias = userAgent.alias;
const isScreenshotRunnable = isHtmlFileRunnable && userAgent.is_runnable;
const flakeConfig = this.getFlakeConfig_({userAgent, htmlFilePath});
const isScreenshotRunnable = isHtmlFileRunnable && userAgent.is_runnable && !flakeConfig.skip_all;
const expectedScreenshotImageUrl = goldenFile.getScreenshotImageUrl({htmlFilePath, userAgentAlias});

/** @type {?mdc.proto.TestFile} */
Expand All @@ -687,13 +688,14 @@ class ReportBuilder {

allScreenshots.push(Screenshot.create({
is_runnable: isScreenshotRunnable,
is_url_skipped_by_cli: !isHtmlFileRunnable,
user_agent: userAgent,
html_file_path: htmlFilePath,
expected_html_file: expectedHtmlFile,
actual_html_file: actualHtmlFile,
expected_image_file: expectedImageFile,
retry_count: 0,
flake_config: this.getFlakeConfig_({userAgent, htmlFilePath}),
flake_config: flakeConfig,
}));
}
}
Expand Down
12 changes: 10 additions & 2 deletions test/screenshot/infra/lib/report-writer.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,10 +130,18 @@ class ReportWriter {
getLocalChangesMarkup: function(gitStatus) {
return self.getLocalChangesMarkup_(gitStatus);
},
/**
* @param {!mdc.proto.Screenshots} screenshots
*/
getFilteredUrlCountMarkup: function(screenshots) {
const actualScreenshots = screenshots.actual_screenshot_list;
const runnableScreenshotArray = actualScreenshots.filter((screenshot) => !screenshot.is_url_skipped_by_cli);
const skippedScreenshotArray = actualScreenshots.filter((screenshot) => screenshot.is_url_skipped_by_cli);
const runnableUrls = new Set(runnableScreenshotArray.map((screenshot) => screenshot.html_file_path));
const skippedUrls = new Set(skippedScreenshotArray.map((screenshot) => screenshot.html_file_path));
return self.getFilteredCountMarkup_(
screenshots.runnable_test_page_urls.length,
screenshots.skipped_test_page_urls.length
runnableUrls.size,
skippedUrls.size
);
},
getFilteredBrowserIconsMarkup: function(userAgents) {
Expand Down
44 changes: 44 additions & 0 deletions test/screenshot/infra/proto/mdc.pb.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5490,6 +5490,7 @@ $root.mdc = (function() {
* @memberof mdc.proto
* @interface IScreenshot
* @property {boolean|null} [is_runnable] Screenshot is_runnable
* @property {boolean|null} [is_url_skipped_by_cli] Screenshot is_url_skipped_by_cli
* @property {mdc.proto.Screenshot.InclusionType|null} [inclusion_type] Screenshot inclusion_type
* @property {mdc.proto.Screenshot.CaptureState|null} [capture_state] Screenshot capture_state
* @property {mdc.proto.IUserAgent|null} [user_agent] Screenshot user_agent
Expand Down Expand Up @@ -5527,6 +5528,14 @@ $root.mdc = (function() {
*/
Screenshot.prototype.is_runnable = false;

/**
* Screenshot is_url_skipped_by_cli.
* @member {boolean} is_url_skipped_by_cli
* @memberof mdc.proto.Screenshot
* @instance
*/
Screenshot.prototype.is_url_skipped_by_cli = false;

/**
* Screenshot inclusion_type.
* @member {mdc.proto.Screenshot.InclusionType} inclusion_type
Expand Down Expand Up @@ -5673,6 +5682,8 @@ $root.mdc = (function() {
writer.uint32(/* id 12, wireType 0 =*/96).uint32(message.retry_count);
if (message.flake_config != null && message.hasOwnProperty("flake_config"))
$root.mdc.proto.FlakeConfig.encode(message.flake_config, writer.uint32(/* id 13, wireType 2 =*/106).fork()).ldelim();
if (message.is_url_skipped_by_cli != null && message.hasOwnProperty("is_url_skipped_by_cli"))
writer.uint32(/* id 14, wireType 0 =*/112).bool(message.is_url_skipped_by_cli);
return writer;
};

Expand Down Expand Up @@ -5710,6 +5721,9 @@ $root.mdc = (function() {
case 1:
message.is_runnable = reader.bool();
break;
case 14:
message.is_url_skipped_by_cli = reader.bool();
break;
case 2:
message.inclusion_type = reader.int32();
break;
Expand Down Expand Up @@ -5784,6 +5798,9 @@ $root.mdc = (function() {
if (message.is_runnable != null && message.hasOwnProperty("is_runnable"))
if (typeof message.is_runnable !== "boolean")
return "is_runnable: boolean expected";
if (message.is_url_skipped_by_cli != null && message.hasOwnProperty("is_url_skipped_by_cli"))
if (typeof message.is_url_skipped_by_cli !== "boolean")
return "is_url_skipped_by_cli: boolean expected";
if (message.inclusion_type != null && message.hasOwnProperty("inclusion_type"))
switch (message.inclusion_type) {
default:
Expand Down Expand Up @@ -5869,6 +5886,8 @@ $root.mdc = (function() {
var message = new $root.mdc.proto.Screenshot();
if (object.is_runnable != null)
message.is_runnable = Boolean(object.is_runnable);
if (object.is_url_skipped_by_cli != null)
message.is_url_skipped_by_cli = Boolean(object.is_url_skipped_by_cli);
switch (object.inclusion_type) {
case "UNKNOWN":
case 0:
Expand Down Expand Up @@ -5987,6 +6006,7 @@ $root.mdc = (function() {
object.diff_image_result = null;
object.retry_count = 0;
object.flake_config = null;
object.is_url_skipped_by_cli = false;
}
if (message.is_runnable != null && message.hasOwnProperty("is_runnable"))
object.is_runnable = message.is_runnable;
Expand Down Expand Up @@ -6014,6 +6034,8 @@ $root.mdc = (function() {
object.retry_count = message.retry_count;
if (message.flake_config != null && message.hasOwnProperty("flake_config"))
object.flake_config = $root.mdc.proto.FlakeConfig.toObject(message.flake_config, options);
if (message.is_url_skipped_by_cli != null && message.hasOwnProperty("is_url_skipped_by_cli"))
object.is_url_skipped_by_cli = message.is_url_skipped_by_cli;
return object;
};

Expand Down Expand Up @@ -6083,6 +6105,7 @@ $root.mdc = (function() {
* @property {number|null} [max_changed_pixel_fraction_to_retry] FlakeConfig max_changed_pixel_fraction_to_retry
* @property {number|null} [font_face_observer_timeout_ms] FlakeConfig font_face_observer_timeout_ms
* @property {number|null} [fonts_loaded_reflow_delay_ms] FlakeConfig fonts_loaded_reflow_delay_ms
* @property {boolean|null} [skip_all] FlakeConfig skip_all
*/

/**
Expand Down Expand Up @@ -6148,6 +6171,14 @@ $root.mdc = (function() {
*/
FlakeConfig.prototype.fonts_loaded_reflow_delay_ms = 0;

/**
* FlakeConfig skip_all.
* @member {boolean} skip_all
* @memberof mdc.proto.FlakeConfig
* @instance
*/
FlakeConfig.prototype.skip_all = false;

/**
* Creates a new FlakeConfig instance using the specified properties.
* @function create
Expand Down Expand Up @@ -6184,6 +6215,8 @@ $root.mdc = (function() {
writer.uint32(/* id 5, wireType 0 =*/40).uint32(message.font_face_observer_timeout_ms);
if (message.fonts_loaded_reflow_delay_ms != null && message.hasOwnProperty("fonts_loaded_reflow_delay_ms"))
writer.uint32(/* id 6, wireType 0 =*/48).uint32(message.fonts_loaded_reflow_delay_ms);
if (message.skip_all != null && message.hasOwnProperty("skip_all"))
writer.uint32(/* id 7, wireType 0 =*/56).bool(message.skip_all);
return writer;
};

Expand Down Expand Up @@ -6236,6 +6269,9 @@ $root.mdc = (function() {
case 6:
message.fonts_loaded_reflow_delay_ms = reader.uint32();
break;
case 7:
message.skip_all = reader.bool();
break;
default:
reader.skipType(tag & 7);
break;
Expand Down Expand Up @@ -6289,6 +6325,9 @@ $root.mdc = (function() {
if (message.fonts_loaded_reflow_delay_ms != null && message.hasOwnProperty("fonts_loaded_reflow_delay_ms"))
if (!$util.isInteger(message.fonts_loaded_reflow_delay_ms))
return "fonts_loaded_reflow_delay_ms: integer expected";
if (message.skip_all != null && message.hasOwnProperty("skip_all"))
if (typeof message.skip_all !== "boolean")
return "skip_all: boolean expected";
return null;
};

Expand Down Expand Up @@ -6316,6 +6355,8 @@ $root.mdc = (function() {
message.font_face_observer_timeout_ms = object.font_face_observer_timeout_ms >>> 0;
if (object.fonts_loaded_reflow_delay_ms != null)
message.fonts_loaded_reflow_delay_ms = object.fonts_loaded_reflow_delay_ms >>> 0;
if (object.skip_all != null)
message.skip_all = Boolean(object.skip_all);
return message;
};

Expand All @@ -6339,6 +6380,7 @@ $root.mdc = (function() {
object.max_changed_pixel_fraction_to_retry = 0;
object.font_face_observer_timeout_ms = 0;
object.fonts_loaded_reflow_delay_ms = 0;
object.skip_all = false;
}
if (message.max_retries != null && message.hasOwnProperty("max_retries"))
object.max_retries = message.max_retries;
Expand All @@ -6352,6 +6394,8 @@ $root.mdc = (function() {
object.font_face_observer_timeout_ms = message.font_face_observer_timeout_ms;
if (message.fonts_loaded_reflow_delay_ms != null && message.hasOwnProperty("fonts_loaded_reflow_delay_ms"))
object.fonts_loaded_reflow_delay_ms = message.fonts_loaded_reflow_delay_ms;
if (message.skip_all != null && message.hasOwnProperty("skip_all"))
object.skip_all = message.skip_all;
return object;
};

Expand Down
2 changes: 2 additions & 0 deletions test/screenshot/infra/proto/mdc.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -246,6 +246,7 @@ message Screenshot {
}

bool is_runnable = 1;
bool is_url_skipped_by_cli = 14;
InclusionType inclusion_type = 2;
CaptureState capture_state = 3;
UserAgent user_agent = 4;
Expand All @@ -269,6 +270,7 @@ message FlakeConfig {
double max_changed_pixel_fraction_to_retry = 4;
uint32 font_face_observer_timeout_ms = 5;
uint32 fonts_loaded_reflow_delay_ms = 6;
bool skip_all = 7;
}

message Dimensions {
Expand Down

0 comments on commit 1a7f6e7

Please sign in to comment.