Auto merge of rust-lang#85652 - ehuss:linkchecker-perf, r=Mark-Simula…

…crum Optimize linkchecker and add report. This makes three changes to the linkchecker: * Adds a report displayed after it finishes. * Improves the performance by caching all filesystem access. The linkchecker can take over a minute to run on some systems, and this should make it about 2-3 times faster. * Added a few tests.
rust-lang-ci · May 26, 2021 · 86ac0b4 · 86ac0b4
2 parents 9111b8a + b6532eb
commit 86ac0b4
Show file tree

Hide file tree

Showing 18 changed files with 505 additions and 227 deletions.
diff --git a/src/tools/linkchecker/main.rs b/src/tools/linkchecker/main.rs
diff --git a/src/tools/linkchecker/tests/basic_broken/foo.html b/src/tools/linkchecker/tests/basic_broken/foo.html
@@ -0,0 +1,5 @@
+<html>
+<body>
+<a href="bar.html">test</a>
+</body>
+</html>
diff --git a/src/tools/linkchecker/tests/broken_fragment_local/foo.html b/src/tools/linkchecker/tests/broken_fragment_local/foo.html
@@ -0,0 +1,5 @@
+<html>
+<body>
+<a href="#somefrag">test</a>
+</body>
+</html>
diff --git a/src/tools/linkchecker/tests/broken_fragment_remote/bar.html b/src/tools/linkchecker/tests/broken_fragment_remote/bar.html
@@ -0,0 +1,4 @@
+<html>
+<body>
+</body>
+</html>
diff --git a/src/tools/linkchecker/tests/broken_fragment_remote/inner/foo.html b/src/tools/linkchecker/tests/broken_fragment_remote/inner/foo.html
@@ -0,0 +1,5 @@
+<html>
+<body>
+<a href="../bar.html#somefrag">test</a>
+</body>
+</html>
diff --git a/src/tools/linkchecker/tests/broken_redir/foo.html b/src/tools/linkchecker/tests/broken_redir/foo.html
@@ -0,0 +1,5 @@
+<html>
+<body>
+  <a href="redir-bad.html">bad redir</a>
+</body>
+</html>
diff --git a/src/tools/linkchecker/tests/broken_redir/redir-bad.html b/src/tools/linkchecker/tests/broken_redir/redir-bad.html
@@ -0,0 +1,10 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta http-equiv="refresh" content="0;URL=sometarget">
+</head>
+<body>
+    <p>Redirecting to <a href="sometarget">sometarget</a>...</p>
+    <script>location.replace("sometarget" + location.search + location.hash);</script>
+</body>
+</html>
diff --git a/src/tools/linkchecker/tests/checks.rs b/src/tools/linkchecker/tests/checks.rs
@@ -0,0 +1,77 @@
+use std::path::Path;
+use std::process::{Command, ExitStatus};
+
+fn run(dirname: &str) -> (ExitStatus, String, String) {
+    let output = Command::new(env!("CARGO_BIN_EXE_linkchecker"))
+        .current_dir(Path::new(env!("CARGO_MANIFEST_DIR")).join("tests"))
+        .arg(dirname)
+        .output()
+        .unwrap();
+    let stdout = String::from_utf8(output.stdout).unwrap();
+    let stderr = String::from_utf8(output.stderr).unwrap();
+    (output.status, stdout, stderr)
+}
+
+fn broken_test(dirname: &str, expected: &str) {
+    let (status, stdout, stderr) = run(dirname);
+    assert!(!status.success());
+    if !stdout.contains(expected) {
+        panic!(
+            "stdout did not contain expected text: {}\n\
+            --- stdout:\n\
+            {}\n\
+            --- stderr:\n\
+            {}\n",
+            expected, stdout, stderr
+        );
+    }
+}
+
+fn valid_test(dirname: &str) {
+    let (status, stdout, stderr) = run(dirname);
+    if !status.success() {
+        panic!(
+            "test did not succeed as expected\n\
+            --- stdout:\n\
+            {}\n\
+            --- stderr:\n\
+            {}\n",
+            stdout, stderr
+        );
+    }
+}
+
+#[test]
+fn valid() {
+    valid_test("valid/inner");
+}
+
+#[test]
+fn basic_broken() {
+    broken_test("basic_broken", "bar.html");
+}
+
+#[test]
+fn broken_fragment_local() {
+    broken_test("broken_fragment_local", "#somefrag");
+}
+
+#[test]
+fn broken_fragment_remote() {
+    broken_test("broken_fragment_remote/inner", "#somefrag");
+}
+
+#[test]
+fn broken_redir() {
+    broken_test("broken_redir", "sometarget");
+}
+
+#[test]
+fn directory_link() {
+    broken_test("directory_link", "somedir");
+}
+
+#[test]
+fn redirect_loop() {
+    broken_test("redirect_loop", "redir-bad.html");
+}
diff --git a/src/tools/linkchecker/tests/directory_link/foo.html b/src/tools/linkchecker/tests/directory_link/foo.html
@@ -0,0 +1,5 @@
+<html>
+<body>
+  <a href="somedir">dir link</a>
+</body>
+</html>
diff --git a/src/tools/linkchecker/tests/directory_link/somedir/index.html b/src/tools/linkchecker/tests/directory_link/somedir/index.html
@@ -0,0 +1,4 @@
+<html>
+<body>
+</body>
+</html>
diff --git a/src/tools/linkchecker/tests/redirect_loop/foo.html b/src/tools/linkchecker/tests/redirect_loop/foo.html
@@ -0,0 +1,5 @@
+<html>
+<body>
+  <a href="redir-bad.html">loop link</a>
+</body>
+</html>
diff --git a/src/tools/linkchecker/tests/redirect_loop/redir-bad.html b/src/tools/linkchecker/tests/redirect_loop/redir-bad.html
@@ -0,0 +1,10 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta http-equiv="refresh" content="0;URL=redir-bad.html">
+</head>
+<body>
+    <p>Redirecting to <a href="redir-bad.html">redir-bad.html</a>...</p>
+    <script>location.replace("redir-bad.html" + location.search + location.hash);</script>
+</body>
+</html>
diff --git a/src/tools/linkchecker/tests/valid/inner/bar.html b/src/tools/linkchecker/tests/valid/inner/bar.html
@@ -0,0 +1,7 @@
+<html>
+<body>
+
+  <h2 id="barfrag">Bar</h2>
+
+</body>
+</html>
diff --git a/src/tools/linkchecker/tests/valid/inner/foo.html b/src/tools/linkchecker/tests/valid/inner/foo.html
@@ -0,0 +1,14 @@
+<html>
+<body>
+  <a href="#localfrag">test local frag</a>
+  <a href="../outer.html">remote link</a>
+  <a href="../outer.html#somefrag">remote link with fragment</a>
+  <a href="bar.html">this book</a>
+  <a href="bar.html#barfrag">this book with fragment</a>
+  <a href="https://example.com/doesnotexist">external links not validated</a>
+  <a href="redir.html#redirfrag">Redirect</a>
+
+  <h2 id="localfrag">Local</h2>
+
+</body>
+</html>
diff --git a/src/tools/linkchecker/tests/valid/inner/redir-bad.html b/src/tools/linkchecker/tests/valid/inner/redir-bad.html
@@ -0,0 +1,11 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta http-equiv="refresh" content="0;URL=xxx">
+</head>
+<body>
+    <p>Redirecting to <a href="xxx">xxx</a>...</p>
+    <script>location.replace("xxx" + location.search + location.hash);</script>
+    These files are skipped, but probably shouldn't be.
+</body>
+</html>
diff --git a/src/tools/linkchecker/tests/valid/inner/redir-target.html b/src/tools/linkchecker/tests/valid/inner/redir-target.html
@@ -0,0 +1,5 @@
+<html>
+<body>
+  <h2 id="redirfrag">Redir</h2>
+</body>
+</html>
diff --git a/src/tools/linkchecker/tests/valid/inner/redir.html b/src/tools/linkchecker/tests/valid/inner/redir.html
@@ -0,0 +1,10 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta http-equiv="refresh" content="0;URL=redir-target.html">
+</head>
+<body>
+    <p>Redirecting to <a href="redir-target.html">redir-target.html</a>...</p>
+    <script>location.replace("redir-target.html" + location.search + location.hash);</script>
+</body>
+</html>
diff --git a/src/tools/linkchecker/tests/valid/outer.html b/src/tools/linkchecker/tests/valid/outer.html
@@ -0,0 +1,5 @@
+<html>
+<body>
+<a id="somefrag"></a>
+</body>
+</html>
-Original file line number
+Diff line change
@@ -0,0 +1,4 @@
+    <html>
+    <body>
+    </body>
+    </html>