Add support for cancelled jobs.

docs/docs/basics/jobs-overview.md

@@ -36,6 +36,7 @@ A `Job` can have any of the following statuses:
 - `JobStatus.COMPLETED`
 - `JobStatus.FAILED`
 - `JobStatus.ABORTED`
+- `JobStatus.CANCELLED`
 
 
 ## Triggering/creating a job

ergate/job.py

@@ -29,12 +29,15 @@ def get_input_value(self) -> Any:
 
         return copy.deepcopy(input_val)
 
-    def mark_running(self, step: WorkflowStep) -> None:
-        self.status = JobStatus.RUNNING
+    def mark_aborted(self, message: str) -> None:
+        self.status = JobStatus.ABORTED
 
     def mark_failed(self, exception: Exception) -> None:
         self.status = JobStatus.FAILED
 
+    def mark_running(self, step: WorkflowStep) -> None:
+        self.status = JobStatus.RUNNING
+
     def mark_step_n_completed(
         self,
         n: int,
@@ -51,12 +54,11 @@ def mark_step_n_completed(
         )
         self.last_return_value = return_value
 
-    def mark_aborted(self, message: str) -> None:
-        self.status = JobStatus.ABORTED
-
+    @property
     def should_be_requeued(self) -> bool:
         return self.status not in (
             JobStatus.ABORTED,
             JobStatus.COMPLETED,
             JobStatus.FAILED,
+            JobStatus.CANCELLED,
         )

ergate/job_runner.py

@@ -109,7 +109,7 @@ def _run_job(self, job: JobType) -> None:
 
         self.state_store.update(job)
 
-        if job.should_be_requeued():
+        if job.should_be_requeued:
             LOG.info("Requeuing job")
             self.queue.put(job)
 
@@ -121,6 +121,9 @@ def run(self) -> None:
             except KeyboardInterrupt:
                 return
 
+            if job.should_be_requeued:
+                return
+
             LOG.info("Job acquired")
             try:
                 with DelayedKeyboardInterrupt():

ergate/job_status.py

@@ -8,3 +8,4 @@ class JobStatus(IntEnum):
     COMPLETED = auto()
     FAILED = auto()
     ABORTED = auto()
+    CANCELLED = auto()

tox.ini

@@ -6,7 +6,7 @@ extras = test
 
 [testenv:check]
 commands =
-    ruff format --check ergate
+    ruff format --diff ergate
     ruff check ergate
     mypy ergate
     flake8 ergate