From 12c0eb97dec376298324fca0c949fd9229871da4 Mon Sep 17 00:00:00 2001 From: Luca Casonato Date: Mon, 4 Jan 2021 18:37:19 +0100 Subject: [PATCH 01/11] tests: add web platform test runner --- .dprintrc.json | 1 + .gitmodules | 3 + cli/tests/WPT.md | 26 +++++ cli/tests/integration_tests.rs | 106 +++++++++++++++++ cli/tests/wpt | 1 + cli/tests/wpt.json | 12 ++ cli/tests/wpt_testharnessconsolereporter.js | 119 ++++++++++++++++++++ tools/lint.js | 1 + 8 files changed, 269 insertions(+) create mode 100644 cli/tests/WPT.md create mode 160000 cli/tests/wpt create mode 100644 cli/tests/wpt.json create mode 100644 cli/tests/wpt_testharnessconsolereporter.js diff --git a/.dprintrc.json b/.dprintrc.json index 9db9bca1a5d389..7ce7dd3b565a1c 100644 --- a/.dprintrc.json +++ b/.dprintrc.json @@ -21,6 +21,7 @@ "cli/dts/lib.webworker*.d.ts", "cli/dts/typescript.d.ts", "cli/tests/encoding", + "cli/tests/wpt", "cli/tsc/*typescript.js", "gh-pages", "std/**/testdata", diff --git a/.gitmodules b/.gitmodules index cd52dfda310fed..77c8380b719043 100644 --- a/.gitmodules +++ b/.gitmodules @@ -5,3 +5,6 @@ [submodule "std/wasi/testdata"] path = std/wasi/testdata url = https://github.com/khronosproject/wasi-test-suite.git +[submodule "cli/tests/wpt"] + path = cli/tests/wpt + url = https://github.com/web-platform-tests/wpt.git diff --git a/cli/tests/WPT.md b/cli/tests/WPT.md new file mode 100644 index 00000000000000..682bbd974db7c5 --- /dev/null +++ b/cli/tests/WPT.md @@ -0,0 +1,26 @@ +## Web Platform Tests + +The WPT are test suites for Web platform specs, like Fetch, WHATWG Streams, or console. Deno is able to run most `.any.js` and `.window.js` web platform tests. + +This directory contains a `wpt.json` file that is used to configure our WPT test runner. You can use this json file to set which WPT suites to run, and which tests we expect to fail (due to bugs or because they are out of scope for Deno). + +To include a new test file to run, add it to the array of test files for the corresponding suite. For example we want to enable `streams/readable-streams/general`. The file would then look like this: + +```json +{ + "streams": ["readable-streams/general"] +} +``` + +If you need more configurability over which test cases in a test file of a suite to run, you can use the object representation. In the example below, we configure `streams/readable-streams/general` to expect `ReadableStream can't be constructed with an invalid type` to fail. + +```json +{ + "streams": [ + { + "name": "readable-streams/general", + "expectFail": ["ReadableStream can't be constructed with an invalid type"] + } + ] +} +``` diff --git a/cli/tests/integration_tests.rs b/cli/tests/integration_tests.rs index 9400db4add6c8f..b616b3daf8e055 100644 --- a/cli/tests/integration_tests.rs +++ b/cli/tests/integration_tests.rs @@ -6,6 +6,7 @@ use std::io::{BufRead, Write}; use std::process::Command; use tempfile::TempDir; use test_util as util; +use walkdir::WalkDir; macro_rules! itest( ($name:ident {$( $key:ident: $value:expr,)*}) => { @@ -4807,3 +4808,108 @@ fn compile_and_overwrite_file() { .unwrap(); assert!(recompile_output.status.success()); } + + +#[test] +fn web_platform_tests() { + use deno_core::serde::Deserialize; + + #[derive(Deserialize)] + #[serde(untagged)] + enum WptConfig { + Simple(String), + #[serde(rename_all = "camelCase")] + Options { name: String, expect_fail: Vec } + } + + let text = std::fs::read_to_string(util::tests_path().join("wpt.json")).unwrap(); + let config: std::collections::HashMap> = deno_core::serde_json::from_str(&text).unwrap(); + + for (suite_name, includes) in config.into_iter() { + let suite_path = util::tests_path().join("wpt").join(suite_name); + let dir = WalkDir::new(&suite_path) + .into_iter() + .filter_map(Result::ok) + .filter(|e| e.file_type().is_file()) + .filter(|f| { + let filename = f.file_name().to_str().unwrap(); + filename.ends_with(".any.js") || filename.ends_with(".window.js") + }).filter_map(|f| { + let path = f.path().strip_prefix(&suite_path).unwrap().to_str().unwrap(); + for cfg in &includes { + match cfg { + WptConfig::Simple(name) if path.starts_with(name) => return Some((f.path().to_owned(), vec![])), + WptConfig::Options { name, expect_fail } if path.starts_with(name) => return Some((f.path().to_owned(), expect_fail.to_vec())), + _ => {} + } + } + None + }); + let testharness_js = url::Url::from_file_path( + util::tests_path().join("wpt/resources/testharness.js"), + ) + .unwrap() + .to_string(); + let testharnessreporter_js = url::Url::from_file_path( + util::tests_path().join("wpt_testharnessconsolereporter.js"), + ) + .unwrap() + .to_string(); + for (path, expect_fail) in dir { + let txt = std::fs::read_to_string(&path).unwrap(); + let scripts: Vec = txt + .split('\n') + .into_iter() + .filter_map(|t| t.strip_prefix("// META: script=")) + .map(|s| { + let s = if s == "/resources/WebIDLParser.js" { + "/resources/webidl2/lib/webidl2.js" + } else { + s + }; + if s.starts_with("/") { + let path = util::tests_path().join("wpt").join(format!(".{}", s)); + url::Url::from_file_path(&path).unwrap().to_string() + } else { + s.to_string() + } + }) + .collect(); + + let input = format!( + r#" +import "{}"; +import "{}"; +{} +import "{}"; + "#, + testharness_js, + testharnessreporter_js, + scripts + .into_iter() + .map(|path| format!(r#"import "{}";"#, path)) + .collect::>() + .join("\n"), + url::Url::from_file_path(&path).unwrap().to_string() + ); + + println!("Running {}", path.to_str().unwrap()); + let mut child = util::deno_cmd() + .current_dir(path.parent().unwrap()) + .arg("run") + .arg("-A") + .arg("--no-check") + .arg("-") + .arg(deno_core::serde_json::to_string(&expect_fail).unwrap()) + .stdin(std::process::Stdio::piped()) + .spawn() + .unwrap(); + let child_stdin = child.stdin.as_mut().unwrap(); + child_stdin.write_all(input.as_bytes()).unwrap(); + drop(child_stdin); + + let output = child.wait_with_output().unwrap(); + assert!(output.status.success()); + } + } +} diff --git a/cli/tests/wpt b/cli/tests/wpt new file mode 160000 index 00000000000000..77ffd6eff47bed --- /dev/null +++ b/cli/tests/wpt @@ -0,0 +1 @@ +Subproject commit 77ffd6eff47bed46236fbea6e998a18e6f4a7800 diff --git a/cli/tests/wpt.json b/cli/tests/wpt.json new file mode 100644 index 00000000000000..013c8e601794a1 --- /dev/null +++ b/cli/tests/wpt.json @@ -0,0 +1,12 @@ +{ + "streams": [ + { + "name": "readable-streams/general", + "expectFail": [ + "ReadableStream can't be constructed with an invalid type", + "default ReadableStream getReader() should only accept mode:undefined" + ] + }, + "writable-streams/general" + ] +} diff --git a/cli/tests/wpt_testharnessconsolereporter.js b/cli/tests/wpt_testharnessconsolereporter.js new file mode 100644 index 00000000000000..b0cc808632d93f --- /dev/null +++ b/cli/tests/wpt_testharnessconsolereporter.js @@ -0,0 +1,119 @@ +const noColor = globalThis.Deno?.noColor ?? true; +const enabled = !noColor; + +function code(open, close) { + return { + open: `\x1b[${open.join(";")}m`, + close: `\x1b[${close}m`, + regexp: new RegExp(`\\x1b\\[${close}m`, "g"), + }; +} + +function run(str, code) { + return enabled + ? `${code.open}${str.replace(code.regexp, code.open)}${code.close}` + : str; +} + +function red(str) { + return run(str, code([31], 39)); +} + +export function green(str) { + return run(str, code([32], 39)); +} + +export function yellow(str) { + return run(str, code([33], 39)); +} + +const testResults = []; +const testsExpectFail = JSON.parse(Deno.args[0]); + +window.add_result_callback((test) => { + const expectFail = testsExpectFail.includes(test.name); + let simpleMessage = `test ${test.name} ... `; + switch (test.status) { + case 0: + if (expectFail) { + simpleMessage += red("ok (expected fail)"); + } else { + simpleMessage += green("ok"); + } + break; + case 1: + if (expectFail) { + simpleMessage += yellow("failed (expected)"); + } else { + simpleMessage += red("failed"); + } + break; + case 2: + if (expectFail) { + simpleMessage += yellow("failed (expected)"); + } else { + simpleMessage += red("failed (timeout)"); + } + break; + case 3: + if (expectFail) { + simpleMessage += yellow("failed (expected)"); + } else { + simpleMessage += red("failed (incomplete)"); + } + break; + } + + console.log(simpleMessage); + + testResults.push({ + name: test.name, + passed: test.status === 0, + expectFail, + message: test.message, + stack: test.stack, + }); +}); + +window.add_completion_callback((tests, harnessStatus) => { + const failed = testResults.filter((t) => !t.expectFail && !t.passed); + const expectedFailedButPassed = testResults.filter((t) => + t.expectFail && t.passed + ); + const expectedFailedButPassedCount = expectedFailedButPassed.length; + const failedCount = failed.length + expectedFailedButPassedCount; + const expectedFailedAndFailedCount = testResults.filter((t) => + t.expectFail && !t.passed + ).length; + const totalCount = testResults.length; + const passedCount = totalCount - failedCount - expectedFailedAndFailedCount; + + if (failed.length > 0) { + console.log(`\nfailures:`); + } + for (const result of failed) { + console.log( + `\n${result.name}\n${result.message}\n${result.stack}`, + ); + } + + if (failed.length > 0) { + console.log(`\nfailures:\n`); + } + for (const result of failed) { + console.log(" " + result.name); + } + if (expectedFailedButPassedCount > 0) { + console.log(`\nexpected failures that passed:\n`); + } + for (const result of expectedFailedButPassed) { + console.log(" " + result.name); + } + console.log( + `\ntest result: ${ + failedCount > 0 ? red("failed") : green("ok") + }. ${passedCount} passed; ${failedCount} failed; ${expectedFailedAndFailedCount} expected failure; total ${totalCount}\n`, + ); + + Deno.exit(failedCount > 0 ? 1 : 0); +}); diff --git a/tools/lint.js b/tools/lint.js index 18de2aef30ee1e..519ce6679c5484 100755 --- a/tools/lint.js +++ b/tools/lint.js @@ -27,6 +27,7 @@ async function dlint() { ":!:cli/tests/lint/**", ":!:cli/tests/tsc/**", ":!:cli/tsc/*typescript.js", + ":!:cli/tests/wpt/**", ]); if (!sourceFiles.length) { From 09e22f038532c0225ee33cb014cbac28b818511b Mon Sep 17 00:00:00 2001 From: Luca Casonato Date: Mon, 4 Jan 2021 18:38:01 +0100 Subject: [PATCH 02/11] fmt --- cli/tests/WPT.md | 16 ++++++++++++---- cli/tests/integration_tests.rs | 34 +++++++++++++++++++++++++--------- 2 files changed, 37 insertions(+), 13 deletions(-) diff --git a/cli/tests/WPT.md b/cli/tests/WPT.md index 682bbd974db7c5..553fe3263f149d 100644 --- a/cli/tests/WPT.md +++ b/cli/tests/WPT.md @@ -1,10 +1,15 @@ ## Web Platform Tests -The WPT are test suites for Web platform specs, like Fetch, WHATWG Streams, or console. Deno is able to run most `.any.js` and `.window.js` web platform tests. +The WPT are test suites for Web platform specs, like Fetch, WHATWG Streams, or +console. Deno is able to run most `.any.js` and `.window.js` web platform tests. -This directory contains a `wpt.json` file that is used to configure our WPT test runner. You can use this json file to set which WPT suites to run, and which tests we expect to fail (due to bugs or because they are out of scope for Deno). +This directory contains a `wpt.json` file that is used to configure our WPT test +runner. You can use this json file to set which WPT suites to run, and which +tests we expect to fail (due to bugs or because they are out of scope for Deno). -To include a new test file to run, add it to the array of test files for the corresponding suite. For example we want to enable `streams/readable-streams/general`. The file would then look like this: +To include a new test file to run, add it to the array of test files for the +corresponding suite. For example we want to enable +`streams/readable-streams/general`. The file would then look like this: ```json { @@ -12,7 +17,10 @@ To include a new test file to run, add it to the array of test files for the cor } ``` -If you need more configurability over which test cases in a test file of a suite to run, you can use the object representation. In the example below, we configure `streams/readable-streams/general` to expect `ReadableStream can't be constructed with an invalid type` to fail. +If you need more configurability over which test cases in a test file of a suite +to run, you can use the object representation. In the example below, we +configure `streams/readable-streams/general` to expect +`ReadableStream can't be constructed with an invalid type` to fail. ```json { diff --git a/cli/tests/integration_tests.rs b/cli/tests/integration_tests.rs index b616b3daf8e055..547ced6c03abc1 100644 --- a/cli/tests/integration_tests.rs +++ b/cli/tests/integration_tests.rs @@ -4809,7 +4809,6 @@ fn compile_and_overwrite_file() { assert!(recompile_output.status.success()); } - #[test] fn web_platform_tests() { use deno_core::serde::Deserialize; @@ -4819,11 +4818,16 @@ fn web_platform_tests() { enum WptConfig { Simple(String), #[serde(rename_all = "camelCase")] - Options { name: String, expect_fail: Vec } + Options { + name: String, + expect_fail: Vec, + }, } - - let text = std::fs::read_to_string(util::tests_path().join("wpt.json")).unwrap(); - let config: std::collections::HashMap> = deno_core::serde_json::from_str(&text).unwrap(); + + let text = + std::fs::read_to_string(util::tests_path().join("wpt.json")).unwrap(); + let config: std::collections::HashMap> = + deno_core::serde_json::from_str(&text).unwrap(); for (suite_name, includes) in config.into_iter() { let suite_path = util::tests_path().join("wpt").join(suite_name); @@ -4834,12 +4838,24 @@ fn web_platform_tests() { .filter(|f| { let filename = f.file_name().to_str().unwrap(); filename.ends_with(".any.js") || filename.ends_with(".window.js") - }).filter_map(|f| { - let path = f.path().strip_prefix(&suite_path).unwrap().to_str().unwrap(); + }) + .filter_map(|f| { + let path = f + .path() + .strip_prefix(&suite_path) + .unwrap() + .to_str() + .unwrap(); for cfg in &includes { match cfg { - WptConfig::Simple(name) if path.starts_with(name) => return Some((f.path().to_owned(), vec![])), - WptConfig::Options { name, expect_fail } if path.starts_with(name) => return Some((f.path().to_owned(), expect_fail.to_vec())), + WptConfig::Simple(name) if path.starts_with(name) => { + return Some((f.path().to_owned(), vec![])) + } + WptConfig::Options { name, expect_fail } + if path.starts_with(name) => + { + return Some((f.path().to_owned(), expect_fail.to_vec())) + } _ => {} } } From 8ffe8754c5684695abd24ae878a277176f94bf2b Mon Sep 17 00:00:00 2001 From: Luca Casonato Date: Mon, 4 Jan 2021 18:40:13 +0100 Subject: [PATCH 03/11] lint --- cli/tests/integration_tests.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/cli/tests/integration_tests.rs b/cli/tests/integration_tests.rs index 547ced6c03abc1..7adcc03fa38707 100644 --- a/cli/tests/integration_tests.rs +++ b/cli/tests/integration_tests.rs @@ -4883,7 +4883,7 @@ fn web_platform_tests() { } else { s }; - if s.starts_with("/") { + if s.starts_with('/') { let path = util::tests_path().join("wpt").join(format!(".{}", s)); url::Url::from_file_path(&path).unwrap().to_string() } else { @@ -4920,10 +4920,11 @@ import "{}"; .stdin(std::process::Stdio::piped()) .spawn() .unwrap(); - let child_stdin = child.stdin.as_mut().unwrap(); - child_stdin.write_all(input.as_bytes()).unwrap(); - drop(child_stdin); - + { + let child_stdin = child.stdin.as_mut().unwrap(); + child_stdin.write_all(input.as_bytes()).unwrap(); + } + let output = child.wait_with_output().unwrap(); assert!(output.status.success()); } From 28d3d9f9ef5e79d89289b3137b4d07fe047c80a5 Mon Sep 17 00:00:00 2001 From: Luca Casonato Date: Mon, 4 Jan 2021 19:22:43 +0100 Subject: [PATCH 04/11] fmt --- cli/tests/integration_tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/tests/integration_tests.rs b/cli/tests/integration_tests.rs index 7adcc03fa38707..d8806e54a37415 100644 --- a/cli/tests/integration_tests.rs +++ b/cli/tests/integration_tests.rs @@ -4924,7 +4924,7 @@ import "{}"; let child_stdin = child.stdin.as_mut().unwrap(); child_stdin.write_all(input.as_bytes()).unwrap(); } - + let output = child.wait_with_output().unwrap(); assert!(output.status.success()); } From 5384ab837837248199c50f7b62bf0d69fbfc8066 Mon Sep 17 00:00:00 2001 From: Luca Casonato Date: Mon, 4 Jan 2021 20:21:45 +0100 Subject: [PATCH 05/11] wpt inline code --- cli/tests/integration_tests.rs | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/cli/tests/integration_tests.rs b/cli/tests/integration_tests.rs index d8806e54a37415..9f4b811f4916e8 100644 --- a/cli/tests/integration_tests.rs +++ b/cli/tests/integration_tests.rs @@ -4872,8 +4872,8 @@ fn web_platform_tests() { .unwrap() .to_string(); for (path, expect_fail) in dir { - let txt = std::fs::read_to_string(&path).unwrap(); - let scripts: Vec = txt + let test_file_text = std::fs::read_to_string(&path).unwrap(); + let scripts: Vec = test_file_text .split('\n') .into_iter() .filter_map(|t| t.strip_prefix("// META: script=")) @@ -4885,7 +4885,10 @@ fn web_platform_tests() { }; if s.starts_with('/') { let path = util::tests_path().join("wpt").join(format!(".{}", s)); - url::Url::from_file_path(&path).unwrap().to_string() + path.to_str().unwrap().to_string() + } else if s.starts_with('.') { + let path = path.parent().unwrap().join(s); + path.to_str().unwrap().to_string() } else { s.to_string() } @@ -4897,16 +4900,19 @@ fn web_platform_tests() { import "{}"; import "{}"; {} -import "{}"; +{} "#, testharness_js, testharnessreporter_js, scripts .into_iter() - .map(|path| format!(r#"import "{}";"#, path)) + .map(|path| { + println!("{}", path); + std::fs::read_to_string(path).unwrap() + }) .collect::>() - .join("\n"), - url::Url::from_file_path(&path).unwrap().to_string() + .join("\n\n"), + test_file_text ); println!("Running {}", path.to_str().unwrap()); From 88784803006ea1c4412b8b9342814b08c566dc36 Mon Sep 17 00:00:00 2001 From: Luca Casonato Date: Mon, 4 Jan 2021 21:09:47 +0100 Subject: [PATCH 06/11] fix itest hang --- cli/tests/integration_tests.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/cli/tests/integration_tests.rs b/cli/tests/integration_tests.rs index 9f4b811f4916e8..016358d0e4880b 100644 --- a/cli/tests/integration_tests.rs +++ b/cli/tests/integration_tests.rs @@ -2635,11 +2635,12 @@ itest!(bundle { output: "bundle.test.out", }); -itest!(fmt_check_tests_dir { - args: "fmt --check ./", - output: "fmt/expected_fmt_check_tests_dir.out", - exit_code: 1, -}); +// TODO(lucacasonato): too many files in cli/tests because of wpt +// itest!(fmt_check_tests_dir { +// args: "fmt --check ./", +// output: "fmt/expected_fmt_check_tests_dir.out", +// exit_code: 1, +// }); itest!(fmt_quiet_check_fmt_dir { args: "fmt --check --quiet fmt/", From cb3da5f949263e213cf7a2f3a996ae09469b74b9 Mon Sep 17 00:00:00 2001 From: Luca Casonato Date: Mon, 4 Jan 2021 23:48:52 +0100 Subject: [PATCH 07/11] sourcemap for the concat bundle --- cli/tests/integration_tests.rs | 106 ++++++++++++++++++++------------- test_util/src/lib.rs | 1 - 2 files changed, 66 insertions(+), 41 deletions(-) diff --git a/cli/tests/integration_tests.rs b/cli/tests/integration_tests.rs index 016358d0e4880b..6ea7455059184f 100644 --- a/cli/tests/integration_tests.rs +++ b/cli/tests/integration_tests.rs @@ -7,6 +7,7 @@ use std::process::Command; use tempfile::TempDir; use test_util as util; use walkdir::WalkDir; +use std::path::PathBuf; macro_rules! itest( ($name:ident {$( $key:ident: $value:expr,)*}) => { @@ -4810,6 +4811,44 @@ fn compile_and_overwrite_file() { assert!(recompile_output.status.success()); } +fn concat_bundle(files: Vec<(PathBuf, String)>) -> String { + let original_url = { + let (original_file_path, _) = files.last().unwrap(); + url::Url::from_file_path(original_file_path).unwrap().to_string() + }; + + let mut bundle = String::new(); + let mut bundle_line_count = 0; + let mut source_map = sourcemap::SourceMapBuilder::new(Some(&original_url)); + + for (path, text) in files { + let url = url::Url::from_file_path(path).unwrap().to_string(); + let src_id = source_map.add_source(&url); + source_map.set_source_contents(src_id, Some(&text)); + + let mut line_index = 0; + for line in text.lines() { + bundle.push_str(line); + bundle.push('\n'); + source_map.add_raw(bundle_line_count, 0, line_index, 0, Some(src_id), None); + + bundle_line_count += 1; + line_index += 1; + } + bundle.push('\n'); + bundle_line_count += 1; + } + + let mut source_map_buf: Vec = vec![]; + source_map.into_sourcemap().to_writer(&mut source_map_buf).unwrap(); + + bundle.push_str("//# sourceMappingURL=data:application/json;base64,"); + let encoded_map = base64::encode(source_map_buf); + bundle.push_str(&encoded_map); + + bundle +} + #[test] fn web_platform_tests() { use deno_core::serde::Deserialize; @@ -4862,19 +4901,15 @@ fn web_platform_tests() { } None }); - let testharness_js = url::Url::from_file_path( - util::tests_path().join("wpt/resources/testharness.js"), - ) - .unwrap() - .to_string(); - let testharnessreporter_js = url::Url::from_file_path( - util::tests_path().join("wpt_testharnessconsolereporter.js"), - ) - .unwrap() - .to_string(); - for (path, expect_fail) in dir { - let test_file_text = std::fs::read_to_string(&path).unwrap(); - let scripts: Vec = test_file_text + + let testharness_path = util::tests_path().join("wpt/resources/testharness.js"); + let testharness_text = std::fs::read_to_string(&testharness_path).unwrap(); + let testharnessreporter_path = util::tests_path().join("wpt_testharnessconsolereporter.js"); + let testharnessreporter_text = std::fs::read_to_string(&testharnessreporter_path).unwrap(); + + for (test_file_path, expect_fail) in dir { + let test_file_text = std::fs::read_to_string(&test_file_path).unwrap(); + let imports: Vec<(PathBuf, String)> = test_file_text .split('\n') .into_iter() .filter_map(|t| t.strip_prefix("// META: script=")) @@ -4886,39 +4921,30 @@ fn web_platform_tests() { }; if s.starts_with('/') { let path = util::tests_path().join("wpt").join(format!(".{}", s)); - path.to_str().unwrap().to_string() + path } else if s.starts_with('.') { - let path = path.parent().unwrap().join(s); - path.to_str().unwrap().to_string() + let path = test_file_path.parent().unwrap().join(s); + path } else { - s.to_string() + PathBuf::from(s) } }) + .map(|path| { + let text = std::fs::read_to_string(&path).unwrap(); + (path, text) + }) .collect(); - let input = format!( - r#" -import "{}"; -import "{}"; -{} -{} - "#, - testharness_js, - testharnessreporter_js, - scripts - .into_iter() - .map(|path| { - println!("{}", path); - std::fs::read_to_string(path).unwrap() - }) - .collect::>() - .join("\n\n"), - test_file_text - ); - - println!("Running {}", path.to_str().unwrap()); + let mut files = Vec::with_capacity(3 + imports.len()); + files.push((testharness_path.clone(), testharness_text.clone())); + files.push((testharnessreporter_path.clone(), testharnessreporter_text.clone())); + files.extend(imports); + files.push((test_file_path.clone(), test_file_text)); + + let bundle = concat_bundle(files); + let mut child = util::deno_cmd() - .current_dir(path.parent().unwrap()) + .current_dir(test_file_path.parent().unwrap()) .arg("run") .arg("-A") .arg("--no-check") @@ -4929,7 +4955,7 @@ import "{}"; .unwrap(); { let child_stdin = child.stdin.as_mut().unwrap(); - child_stdin.write_all(input.as_bytes()).unwrap(); + child_stdin.write_all(bundle.as_bytes()).unwrap(); } let output = child.wait_with_output().unwrap(); diff --git a/test_util/src/lib.rs b/test_util/src/lib.rs index a91416bb5736e7..214cbfc4e074f1 100644 --- a/test_util/src/lib.rs +++ b/test_util/src/lib.rs @@ -90,7 +90,6 @@ pub fn third_party_path() -> PathBuf { pub fn target_dir() -> PathBuf { let current_exe = std::env::current_exe().unwrap(); let target_dir = current_exe.parent().unwrap().parent().unwrap(); - println!("target_dir {}", target_dir.display()); target_dir.into() } From 115c7843d17b059e846d293389a13393494a8c78 Mon Sep 17 00:00:00 2001 From: Luca Casonato Date: Mon, 4 Jan 2021 23:55:15 +0100 Subject: [PATCH 08/11] Address review comments Co-Authored-By: Kitson Kelly --- cli/tests/wpt_testharnessconsolereporter.js | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/cli/tests/wpt_testharnessconsolereporter.js b/cli/tests/wpt_testharnessconsolereporter.js index b0cc808632d93f..9e34d0689ac8cb 100644 --- a/cli/tests/wpt_testharnessconsolereporter.js +++ b/cli/tests/wpt_testharnessconsolereporter.js @@ -30,10 +30,10 @@ export function yellow(str) { const testResults = []; const testsExpectFail = JSON.parse(Deno.args[0]); -window.add_result_callback((test) => { - const expectFail = testsExpectFail.includes(test.name); - let simpleMessage = `test ${test.name} ... `; - switch (test.status) { +window.add_result_callback(({ message, name, stack, status }) => { + const expectFail = testsExpectFail.includes(name); + let simpleMessage = `test ${name} ... `; + switch (status) { case 0: if (expectFail) { simpleMessage += red("ok (expected fail)"); @@ -67,11 +67,11 @@ window.add_result_callback((test) => { console.log(simpleMessage); testResults.push({ - name: test.name, - passed: test.status === 0, + name, + passed: status === 0, expectFail, - message: test.message, - stack: test.stack, + message, + stack, }); }); @@ -101,13 +101,13 @@ window.add_completion_callback((tests, harnessStatus) => { console.log(`\nfailures:\n`); } for (const result of failed) { - console.log(" " + result.name); + console.log(` ${result.name}`); } if (expectedFailedButPassedCount > 0) { console.log(`\nexpected failures that passed:\n`); } for (const result of expectedFailedButPassed) { - console.log(" " + result.name); + console.log(` ${result.name}`); } console.log( `\ntest result: ${ From 0d99cfce6d8032601e7e95ffff478b9a5eb35f52 Mon Sep 17 00:00:00 2001 From: Luca Casonato Date: Mon, 4 Jan 2021 23:57:49 +0100 Subject: [PATCH 09/11] lint + fmt --- cli/tests/integration_tests.rs | 58 +++++++++++++++++++++------------- 1 file changed, 36 insertions(+), 22 deletions(-) diff --git a/cli/tests/integration_tests.rs b/cli/tests/integration_tests.rs index 6ea7455059184f..af7513f556a160 100644 --- a/cli/tests/integration_tests.rs +++ b/cli/tests/integration_tests.rs @@ -3,11 +3,11 @@ use deno_core::futures; use deno_core::futures::prelude::*; use deno_core::url; use std::io::{BufRead, Write}; +use std::path::PathBuf; use std::process::Command; use tempfile::TempDir; use test_util as util; use walkdir::WalkDir; -use std::path::PathBuf; macro_rules! itest( ($name:ident {$( $key:ident: $value:expr,)*}) => { @@ -4814,34 +4814,44 @@ fn compile_and_overwrite_file() { fn concat_bundle(files: Vec<(PathBuf, String)>) -> String { let original_url = { let (original_file_path, _) = files.last().unwrap(); - url::Url::from_file_path(original_file_path).unwrap().to_string() + url::Url::from_file_path(original_file_path) + .unwrap() + .to_string() }; let mut bundle = String::new(); let mut bundle_line_count = 0; let mut source_map = sourcemap::SourceMapBuilder::new(Some(&original_url)); - + for (path, text) in files { let url = url::Url::from_file_path(path).unwrap().to_string(); let src_id = source_map.add_source(&url); source_map.set_source_contents(src_id, Some(&text)); - - let mut line_index = 0; - for line in text.lines() { + + for (line_index, line) in text.lines().enumerate() { bundle.push_str(line); bundle.push('\n'); - source_map.add_raw(bundle_line_count, 0, line_index, 0, Some(src_id), None); + source_map.add_raw( + bundle_line_count, + 0, + line_index as u32, + 0, + Some(src_id), + None, + ); bundle_line_count += 1; - line_index += 1; } bundle.push('\n'); bundle_line_count += 1; } let mut source_map_buf: Vec = vec![]; - source_map.into_sourcemap().to_writer(&mut source_map_buf).unwrap(); - + source_map + .into_sourcemap() + .to_writer(&mut source_map_buf) + .unwrap(); + bundle.push_str("//# sourceMappingURL=data:application/json;base64,"); let encoded_map = base64::encode(source_map_buf); bundle.push_str(&encoded_map); @@ -4902,10 +4912,13 @@ fn web_platform_tests() { None }); - let testharness_path = util::tests_path().join("wpt/resources/testharness.js"); + let testharness_path = + util::tests_path().join("wpt/resources/testharness.js"); let testharness_text = std::fs::read_to_string(&testharness_path).unwrap(); - let testharnessreporter_path = util::tests_path().join("wpt_testharnessconsolereporter.js"); - let testharnessreporter_text = std::fs::read_to_string(&testharnessreporter_path).unwrap(); + let testharnessreporter_path = + util::tests_path().join("wpt_testharnessconsolereporter.js"); + let testharnessreporter_text = + std::fs::read_to_string(&testharnessreporter_path).unwrap(); for (test_file_path, expect_fail) in dir { let test_file_text = std::fs::read_to_string(&test_file_path).unwrap(); @@ -4920,11 +4933,9 @@ fn web_platform_tests() { s }; if s.starts_with('/') { - let path = util::tests_path().join("wpt").join(format!(".{}", s)); - path + util::tests_path().join("wpt").join(format!(".{}", s)) } else if s.starts_with('.') { - let path = test_file_path.parent().unwrap().join(s); - path + test_file_path.parent().unwrap().join(s) } else { PathBuf::from(s) } @@ -4935,11 +4946,14 @@ fn web_platform_tests() { }) .collect(); - let mut files = Vec::with_capacity(3 + imports.len()); - files.push((testharness_path.clone(), testharness_text.clone())); - files.push((testharnessreporter_path.clone(), testharnessreporter_text.clone())); - files.extend(imports); - files.push((test_file_path.clone(), test_file_text)); + let mut files = Vec::with_capacity(3 + imports.len()); + files.push((testharness_path.clone(), testharness_text.clone())); + files.push(( + testharnessreporter_path.clone(), + testharnessreporter_text.clone(), + )); + files.extend(imports); + files.push((test_file_path.clone(), test_file_text)); let bundle = concat_bundle(files); From a5eb494cf457298a3c9443a5b79ae3480df2f5dd Mon Sep 17 00:00:00 2001 From: Luca Casonato Date: Tue, 5 Jan 2021 01:19:12 +0100 Subject: [PATCH 10/11] use tmp file --- cli/tests/integration_tests.rs | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/cli/tests/integration_tests.rs b/cli/tests/integration_tests.rs index af7513f556a160..e886177559a27c 100644 --- a/cli/tests/integration_tests.rs +++ b/cli/tests/integration_tests.rs @@ -3,6 +3,7 @@ use deno_core::futures; use deno_core::futures::prelude::*; use deno_core::url; use std::io::{BufRead, Write}; +use std::path::Path; use std::path::PathBuf; use std::process::Command; use tempfile::TempDir; @@ -4811,17 +4812,12 @@ fn compile_and_overwrite_file() { assert!(recompile_output.status.success()); } -fn concat_bundle(files: Vec<(PathBuf, String)>) -> String { - let original_url = { - let (original_file_path, _) = files.last().unwrap(); - url::Url::from_file_path(original_file_path) - .unwrap() - .to_string() - }; +fn concat_bundle(files: Vec<(PathBuf, String)>, bundle_path: &Path) -> String { + let bundle_url = url::Url::from_file_path(bundle_path).unwrap().to_string(); let mut bundle = String::new(); let mut bundle_line_count = 0; - let mut source_map = sourcemap::SourceMapBuilder::new(Some(&original_url)); + let mut source_map = sourcemap::SourceMapBuilder::new(Some(&bundle_url)); for (path, text) in files { let url = url::Url::from_file_path(path).unwrap().to_string(); @@ -4955,24 +4951,30 @@ fn web_platform_tests() { files.extend(imports); files.push((test_file_path.clone(), test_file_text)); - let bundle = concat_bundle(files); + let mut file = tempfile::Builder::new() + .prefix("wpt-bundle-") + .suffix(".js") + .rand_bytes(5) + .tempfile() + .unwrap(); + + let bundle = concat_bundle(files, file.path()); + file.write_all(bundle.as_bytes()).unwrap(); - let mut child = util::deno_cmd() + let child = util::deno_cmd() .current_dir(test_file_path.parent().unwrap()) .arg("run") .arg("-A") - .arg("--no-check") - .arg("-") + .arg(file.path()) .arg(deno_core::serde_json::to_string(&expect_fail).unwrap()) .stdin(std::process::Stdio::piped()) .spawn() .unwrap(); - { - let child_stdin = child.stdin.as_mut().unwrap(); - child_stdin.write_all(bundle.as_bytes()).unwrap(); - } let output = child.wait_with_output().unwrap(); + if !output.status.success() { + file.keep().unwrap(); + } assert!(output.status.success()); } } From b53afddba48645e0c4581c67adb942331fd195e7 Mon Sep 17 00:00:00 2001 From: Luca Casonato Date: Tue, 5 Jan 2021 03:38:28 +0100 Subject: [PATCH 11/11] move to test_util/wpt --- .dprintrc.json | 2 +- .gitmodules | 4 ++-- cli/tests/integration_tests.rs | 19 +++++++++---------- cli/tests/wpt | 1 - test_util/src/lib.rs | 4 ++++ test_util/wpt | 1 + tools/lint.js | 2 +- 7 files changed, 18 insertions(+), 15 deletions(-) delete mode 160000 cli/tests/wpt create mode 160000 test_util/wpt diff --git a/.dprintrc.json b/.dprintrc.json index 7ce7dd3b565a1c..9217366411cdd0 100644 --- a/.dprintrc.json +++ b/.dprintrc.json @@ -21,8 +21,8 @@ "cli/dts/lib.webworker*.d.ts", "cli/dts/typescript.d.ts", "cli/tests/encoding", - "cli/tests/wpt", "cli/tsc/*typescript.js", + "test_util/wpt", "gh-pages", "std/**/testdata", "std/**/vendor", diff --git a/.gitmodules b/.gitmodules index 77c8380b719043..22cc5436a9d222 100644 --- a/.gitmodules +++ b/.gitmodules @@ -5,6 +5,6 @@ [submodule "std/wasi/testdata"] path = std/wasi/testdata url = https://github.com/khronosproject/wasi-test-suite.git -[submodule "cli/tests/wpt"] - path = cli/tests/wpt +[submodule "test_util/wpt"] + path = test_util/wpt url = https://github.com/web-platform-tests/wpt.git diff --git a/cli/tests/integration_tests.rs b/cli/tests/integration_tests.rs index e886177559a27c..b6916a6af91efe 100644 --- a/cli/tests/integration_tests.rs +++ b/cli/tests/integration_tests.rs @@ -2637,12 +2637,11 @@ itest!(bundle { output: "bundle.test.out", }); -// TODO(lucacasonato): too many files in cli/tests because of wpt -// itest!(fmt_check_tests_dir { -// args: "fmt --check ./", -// output: "fmt/expected_fmt_check_tests_dir.out", -// exit_code: 1, -// }); +itest!(fmt_check_tests_dir { + args: "fmt --check ./", + output: "fmt/expected_fmt_check_tests_dir.out", + exit_code: 1, +}); itest!(fmt_quiet_check_fmt_dir { args: "fmt --check --quiet fmt/", @@ -4820,6 +4819,7 @@ fn concat_bundle(files: Vec<(PathBuf, String)>, bundle_path: &Path) -> String { let mut source_map = sourcemap::SourceMapBuilder::new(Some(&bundle_url)); for (path, text) in files { + let path = std::fs::canonicalize(path).unwrap(); let url = url::Url::from_file_path(path).unwrap().to_string(); let src_id = source_map.add_source(&url); source_map.set_source_contents(src_id, Some(&text)); @@ -4876,7 +4876,7 @@ fn web_platform_tests() { deno_core::serde_json::from_str(&text).unwrap(); for (suite_name, includes) in config.into_iter() { - let suite_path = util::tests_path().join("wpt").join(suite_name); + let suite_path = util::wpt_path().join(suite_name); let dir = WalkDir::new(&suite_path) .into_iter() .filter_map(Result::ok) @@ -4908,8 +4908,7 @@ fn web_platform_tests() { None }); - let testharness_path = - util::tests_path().join("wpt/resources/testharness.js"); + let testharness_path = util::wpt_path().join("resources/testharness.js"); let testharness_text = std::fs::read_to_string(&testharness_path).unwrap(); let testharnessreporter_path = util::tests_path().join("wpt_testharnessconsolereporter.js"); @@ -4929,7 +4928,7 @@ fn web_platform_tests() { s }; if s.starts_with('/') { - util::tests_path().join("wpt").join(format!(".{}", s)) + util::wpt_path().join(format!(".{}", s)) } else if s.starts_with('.') { test_file_path.parent().unwrap().join(s) } else { diff --git a/cli/tests/wpt b/cli/tests/wpt deleted file mode 160000 index 77ffd6eff47bed..00000000000000 --- a/cli/tests/wpt +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 77ffd6eff47bed46236fbea6e998a18e6f4a7800 diff --git a/test_util/src/lib.rs b/test_util/src/lib.rs index 214cbfc4e074f1..262084af72909d 100644 --- a/test_util/src/lib.rs +++ b/test_util/src/lib.rs @@ -83,6 +83,10 @@ pub fn tests_path() -> PathBuf { root_path().join("cli").join("tests") } +pub fn wpt_path() -> PathBuf { + root_path().join("test_util").join("wpt") +} + pub fn third_party_path() -> PathBuf { root_path().join("third_party") } diff --git a/test_util/wpt b/test_util/wpt new file mode 160000 index 00000000000000..077d53c8da8b47 --- /dev/null +++ b/test_util/wpt @@ -0,0 +1 @@ +Subproject commit 077d53c8da8b47c1d5060893af96a29f27b10008 diff --git a/tools/lint.js b/tools/lint.js index 519ce6679c5484..3f30a7915e050c 100755 --- a/tools/lint.js +++ b/tools/lint.js @@ -27,7 +27,7 @@ async function dlint() { ":!:cli/tests/lint/**", ":!:cli/tests/tsc/**", ":!:cli/tsc/*typescript.js", - ":!:cli/tests/wpt/**", + ":!:test_util/wpt/**", ]); if (!sourceFiles.length) {