crawl stopping / additional states: (#303)

* crawl stopping / additional states:
- adds check for 'isCrawlStopped()' which checks redis key to see if crawl has been stopped externally, and interrupts work
loop and prevents crawl from starting on load
- additional crawl states: 'generate-wacz', 'generate-cdx', 'generate-warc', 'uploading-wacz', and 'pending-wait' to indicate
when crawl is no longer running but crawler performing work
- addresses part of webrecorder/browsertrix#263, webrecorder/browsertrix#637

Author: ikreymer

crawler.js

@@ -44,6 +44,8 @@ const behaviors = fs.readFileSync(new URL("./node_modules/browsertrix-behaviors/
 const FETCH_TIMEOUT_SECS = 30;
 const PAGE_OP_TIMEOUT_SECS = 5;
 
+const POST_CRAWL_STATES = ["generate-wacz", "uploading-wacz", "generate-cdx", "generate-warc"];
+
 
 // ============================================================================
 export class Crawler {
@@ -706,7 +708,12 @@ export class Crawler {
       this.storage = initStorage();
     }
 
-    if (initState === "finalize") {
+    if (POST_CRAWL_STATES.includes(initState)) {
+      logger.info("crawl already finished, running post-crawl tasks", {state: initState});
+      await this.postCrawl();
+      return;
+    } else if (await this.crawlState.isCrawlStopped()) {
+      logger.info("crawl stopped, running post-crawl tasks");
       await this.postCrawl();
       return;
     }
@@ -784,6 +791,7 @@ export class Crawler {
 
     if (this.params.generateCDX) {
       logger.info("Generating CDX");
+      await this.crawlState.setStatus("generate-cdx");
       await this.awaitProcess(child_process.spawn("wb-manager", ["reindex", this.params.collection], {cwd: this.params.cwd}));
     }
 
@@ -824,6 +832,7 @@ export class Crawler {
 
   async generateWACZ() {
     logger.info("Generating WACZ");
+    await this.crawlState.setStatus("generate-wacz");
 
     const archiveDir = path.join(this.collDir, "archive");
 
@@ -901,6 +910,7 @@ export class Crawler {
     }
 */
     if (this.storage) {
+      await this.crawlState.setStatus("uploading-wacz");
       const filename = process.env.STORE_FILENAME || "@[email protected]";
       const targetFilename = interpolateFilename(filename, this.crawlId);
 
@@ -1318,6 +1328,7 @@ export class Crawler {
 
   async awaitPendingClear() {
     logger.info("Waiting to ensure pending data is written to WARCs...");
+    await this.crawlState.setStatus("pending-wait");
 
     const redis = await initRedis("redis://localhost/0");
 
@@ -1353,6 +1364,7 @@ export class Crawler {
 
   async combineWARC() {
     logger.info("Generating Combined WARCs");
+    await this.crawlState.setStatus("generate-warc");
 
     // Get the list of created Warcs
     const warcLists = await fsp.readdir(path.join(this.collDir, "archive"));

util/state.js

@@ -208,6 +208,16 @@ return 0;
     return await this.redis.hset(this.crawlSizeKey, this.uid, size);
   }
 
+  async isCrawlStopped() {
+    return await this.redis.get("crawl-stop") === "1";
+  }
+
+  // note: not currently called in crawler, but could be
+  // crawl may be stopped by setting this elsewhere in shared redis
+  async stopCrawl() {
+    await this.redis.set("crawl-stop", "1");
+  }
+
   async incFailCount() {
     const key = `${this.key}:status:failcount:${this.uid}`;
     const res = await this.redis.incr(key);

util/worker.js

@@ -180,7 +180,7 @@ export class PageWorker
   async runLoop() {
     const crawlState = this.crawler.crawlState;
 
-    while (!this.crawler.interrupted) {
+    while (!this.crawler.interrupted && !await crawlState.isCrawlStopped()) {
       const data = await crawlState.nextFromQueue();
 
       // see if any work data in the queue