Catch 4xx and 5xx page.goto() responses to mark invalid URLs as failed

crawler.js

@@ -352,6 +352,14 @@ export class Crawler {
     return seed.isIncluded(url, depth, extraHops, logDetails);
   }
 
+  isSeedUrl(url) {
+    const seeds = this.params.scopedSeeds.map(seed => seed.url);
+    if (seeds.indexOf(url) != -1){
+      return true;
+    }
+    return false;
+  }
+
   async setupPage({page, cdp, workerid}) {
     await this.browser.setupPage({page, cdp});
 
@@ -941,6 +949,8 @@ export class Crawler {
 
     const logDetails = data.logDetails;
 
+    const failCrawlOnError = (this.isSeedUrl(url) && this.params.failOnFailedSeed);
+
     let isHTMLPage = await timedRun(
       this.isHTML(url),
       FETCH_TIMEOUT_SECS,
@@ -998,6 +1008,17 @@ export class Crawler {
     try {
       const resp = await page.goto(url, gotoOpts);
 
+      // Handle 4xx or 5xx response as a page load error
+      const statusCode = resp.status();
+      if (statusCode.toString().startsWith("4") || statusCode.toString().startsWith("5")) {
+        if (failCrawlOnError) {
+          logger.fatal("Seed Page Load Error, failing crawl", {statusCode, ...logDetails});
+        } else {
+          logger.error("Page Load Error, skipping page", {statusCode, ...logDetails});
+          throw new Error(`Page ${url} returned status code ${statusCode}`);
+        }
+      }
+
       const contentType = await resp.headerValue("content-type");
 
       isHTMLPage = this.isHTMLContentType(contentType);
@@ -1007,15 +1028,23 @@ export class Crawler {
       if (!msg.startsWith("net::ERR_ABORTED") || !ignoreAbort) {
         if (e.name === "TimeoutError") {
           if (data.loadState !== LoadState.CONTENT_LOADED) {
-            logger.error("Page Load Timeout, skipping page", {msg, ...logDetails});
-            throw e;
+            if (failCrawlOnError) {
+              logger.fatal("Seed Page Load Timeout, failing crawl", {msg, ...logDetails});
+            } else {
+              logger.error("Page Load Timeout, skipping page", {msg, ...logDetails});
+              throw e;
+            }
           } else {
             logger.warn("Page Loading Slowly, skipping behaviors", {msg, ...logDetails});
             data.skipBehaviors = true;
           }
         } else {
-          logger.error("Page Load Error, skipping page", {msg, ...logDetails});
-          throw e;
+          if (failCrawlOnError) {
+            logger.fatal("Seed Page Load Timeout, failing crawl", {msg, ...logDetails});
+          } else {
+            logger.error("Page Load Error, skipping page", {msg, ...logDetails});
+            throw e;
+          }
         }
       }
     }

util/argParser.js

@@ -372,6 +372,12 @@ class ArgParser {
         descripe: "If set, write error messages to redis",
         type: "boolean",
         default: false,
+      },
+
+      "failOnFailedSeed": {
+        describe: "If set, crawler will fail with exit code 1 if initial seed fails",
+        type: "boolean",
+        default: false
       }
     };
   }