feat: 🚀 generic task view

mula/scheduler/server/server.py

@@ -105,6 +105,15 @@ def __init__(
             description="List all schedulers",
         )
 
+        self.api.add_api_route(
+            path="/schedulers/by_type/{item_type}",
+            endpoint=self.get_schedulers_by_type,
+            methods=["GET"],
+            response_model=list[models.Scheduler],
+            status_code=status.HTTP_200_OK,
+            description="List all schedulers of a specific type.",
+        )
+
         self.api.add_api_route(
             path="/schedulers/{scheduler_id}",
             endpoint=self.get_scheduler,
@@ -236,6 +245,9 @@ def metrics(self) -> Any:
     def get_schedulers(self) -> Any:
         return [models.Scheduler(**s.dict()) for s in self.schedulers.values()]
 
+    def get_schedulers_by_type(self, item_type: str) -> Any:
+        return [models.Scheduler(**s.dict()) for s in self.schedulers.values() if s.queue.item_type.type == item_type]
+
     def get_scheduler(self, scheduler_id: str) -> Any:
         s = self.schedulers.get(scheduler_id)
         if s is None:

rocky/assets/js/renderNormalizerOutputOOIs.js

@@ -10,11 +10,14 @@ buttons.forEach((button) => {
     .closest("tr")
     .getAttribute("data-task-id")
     .replace(/-/g, "");
+  const organization =
+    organization_code ||
+    button.closest("tr").getAttribute("data-organization-code");
   const json_url =
     "/" +
     language +
     "/" +
-    organization_code +
+    organization +
     "/tasks/normalizers/" +
     encodeURI(task_id);
 
@@ -78,7 +81,7 @@ buttons.forEach((button) => {
             "/" +
             language +
             "/" +
-            escapeHTMLEntities(encodeURIComponent(organization_code));
+            escapeHTMLEntities(encodeURIComponent(organization));
           let object_list = "";
 
           // Build HTML snippet for every yielded object.

rocky/katalogus/templates/partials/single_action_form.html

@@ -2,6 +2,11 @@
     {% csrf_token %}
     <input type="hidden" name="action" value="{{ action }}">
     {% if key %}<input type="hidden" name="{{ key }}" value="{{ value }}">{% endif %}
+    {% if organization_code %}
+        <input type="hidden"
+               name="organization_code"
+               value="{{ organization_code }}">
+    {% endif %}
     <button type="submit"
             class="{{ btn_class }}"
             {% if btn_disabled %}disabled{% endif %}>

rocky/rocky/scheduler.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
+import collections
 import datetime
-import json
 import uuid
 from enum import Enum
 from logging import getLogger
@@ -154,6 +154,7 @@ def __getitem__(self, key) -> list[Task]:
         else:
             raise TypeError("Invalid slice argument type.")
 
+        logger.info("Getting max %s lazy items at offset %s with filter %s", limit, offset, self.kwargs)
         res = self.scheduler_client.list_tasks(
             limit=limit,
             offset=offset,
@@ -196,12 +197,14 @@ def list_tasks(
         **kwargs,
     ) -> PaginatedTasksResponse:
         kwargs = {k: v for k, v in kwargs.items() if v is not None}  # filter Nones from kwargs
+        logger.warning("hello there")
+        logger.warning(kwargs)
         res = self._client.get("/tasks", params=kwargs)
         return PaginatedTasksResponse.model_validate_json(res.content)
 
     def get_lazy_task_list(
         self,
-        scheduler_id: str,
+        scheduler_id: str | None,
         task_type: str | None = None,
         status: str | None = None,
         min_created_at: datetime.datetime | None = None,
@@ -213,7 +216,7 @@ def get_lazy_task_list(
         return LazyTaskList(
             self,
             scheduler_id=scheduler_id,
-            type=task_type,
+            task_type=task_type,
             status=status,
             min_created_at=min_created_at,
             max_created_at=max_created_at,
@@ -262,14 +265,39 @@ def health(self) -> ServiceHealth:
         health_endpoint.raise_for_status()
         return ServiceHealth.model_validate_json(health_endpoint.content)
 
-    def get_task_stats(self, organization_code: str, task_type: str) -> dict:
+    def _get(self, path: str) -> dict:
+        """Helper to do a get request."""
         try:
-            res = self._client.get(f"/tasks/stats/{task_type}-{organization_code}")
+            res = self._client.get(path)
             res.raise_for_status()
         except HTTPError:
-            raise SchedulerError()
-        task_stats = json.loads(res.content)
-        return task_stats
+            raise SchedulerError(path)
+        return res.json()
+
+    def _get_task_stats(self, scheduler_id: str) -> dict:
+        """Return task stats for specific scheduler."""
+        return self._get(f"/tasks/stats/{scheduler_id}")
+
+    def get_task_stats(self, organization_code: str, task_type: str) -> dict:
+        """Return task stats for specific organization and task combination."""
+        return self._get_task_stats(scheduler_id=f"{task_type}-{organization_code}")
+
+    @staticmethod
+    def _merge_stat_dicts(dicts: list) -> dict:
+        """Merge multiple stats dicts."""
+        stat_sum: dict = collections.defaultdict(collections.Counter)
+        for dct in dicts:
+            for timeslot, counts in dct.items():
+                stat_sum[timeslot].update(counts)
+        return dict(stat_sum)
+
+    def get_all_task_stats(self, task_type: str) -> dict:
+        """Return all task stats for specific task type (e.g. 'boefje')."""
+        all_stats = [
+            self._get_task_stats(scheduler_id=scheduler.get("id"))
+            for scheduler in self._get(f"/schedulers/by_type/{task_type}")
+        ]
+        return SchedulerClient._merge_stat_dicts(dicts=all_stats)
 
 
 client = SchedulerClient(settings.SCHEDULER_API)

rocky/rocky/templates/header.html

@@ -32,6 +32,11 @@
                                 <a href="{{ index_url }}"
                                    {% if index_url == request.path|urlencode %}aria-current="page"{% endif %}>{% translate "Crisis room" %}</a>
                             </li>
+                            <li>
+                                {% url "all_task_list" as index_url %}
+                                <a href="{{ index_url }}"
+                                   {% if index_url in request.path|urlencode %}aria-current="page"{% endif %}>{% translate "Tasks" %}</a>
+                            </li>
                         {% else %}
                             <li>
                                 {% url "organization_crisis_room" organization.code as index_url %}

rocky/rocky/templates/tasks/partials/boefje_task_history.html

@@ -8,18 +8,27 @@ <h2>{% translate "Tasks" %}</h2>
         <table rf-selector="table-scan-history">
             <thead>
                 <tr>
+                    {% if not organization.code %}
+                        <th scope="col">{% translate "Organization Code" %}</th>
+                    {% endif %}
                     <th scope="col">{% translate "Boefje" %}</th>
                     <th scope="col">{% translate "Status" %}</th>
                     <th scope="col">{% translate "Created date" %}</th>
+                    <th scope="col">{% translate "Modified date" %}</th>
                     <th scope="col">{% translate "Input Object" %}</th>
                     <th scope="col" class="visually-hidden">{% translate "Details" %}</th>
                 </tr>
             </thead>
             <tbody>
                 {% for task in task_history %}
                     <tr data-task-id="{{ task.id }}">
+                        {% if not organization.code %}
+                            <td>
+                                <a href="{% url "organization_crisis_room" task.p_item.data.organization %}">{{ task.p_item.data.organization }}</a>
+                            </td>
+                        {% endif %}
                         <td>
-                            <a href="{% url "boefje_detail" organization.code task.p_item.data.boefje.id %}">{{ task.p_item.data.boefje.name }}</a>
+                            <a href="{% url "boefje_detail" task.p_item.data.organization task.p_item.data.boefje.id %}">{{ task.p_item.data.boefje.name }}</a>
                         </td>
                         <td class="nowrap">
                             {% if task.status.value == "pending" or task.status.value == "queued" %}
@@ -37,8 +46,9 @@ <h2>{% translate "Tasks" %}</h2>
                             {% endif %}
                         </td>
                         <td>{{ task.created_at }}</td>
+                        <td>{{ task.modified_at }}</td>
                         <td>
-                            <a href="{% ooi_url "ooi_detail" task.p_item.data.input_ooi organization.code observed_at=task.created_at|date:'c' %}">{{ task.p_item.data.input_ooi }}</a>
+                            <a href="{% ooi_url "ooi_detail" task.p_item.data.input_ooi task.p_item.data.organization observed_at=task.created_at|date:'c' %}">{{ task.p_item.data.input_ooi }}</a>
                         </td>
                         {# FIXME: implement detail page according to designs #}
                         <td>
@@ -57,12 +67,12 @@ <h2>{% translate "Tasks" %}</h2>
                                 <div class="button-container">
                                     {% if task.status.value in "completed,failed" %}
                                         <a class="button"
-                                           href="{% url 'bytes_raw' organization_code=organization.code boefje_meta_id=task.id %}"><span class="icon ti-download"></span>{% translate "Download meta and raw data" %}</a>
-                                        {% include "partials/single_action_form.html" with btn_text=_("Reschedule") btn_class="ghost" btn_icon="icon ti-refresh" action="reschedule_task" key="task_id" value=task.id %}
+                                           href="{% url 'bytes_raw' organization_code=task.p_item.data.organization boefje_meta_id=task.id %}"><span class="icon ti-download"></span>{% translate "Download meta and raw data" %}</a>
+                                        {% include "partials/single_action_form.html" with btn_text=_("Reschedule") btn_class="ghost" btn_icon="icon ti-refresh" action="reschedule_task" key="task_id" organization_code=task.p_item.data.organization value=task.id %}
 
                                     {% else %}
                                         <a class="button"
-                                           href="{% url 'download_task_meta' organization_code=organization.code task_id=task.id %}"><span class="icon ti-download"></span>{% translate "Download task data" %}</a>
+                                           href="{% url 'download_task_meta' organization_code=task.p_item.data.organization task_id=task.id %}"><span class="icon ti-download"></span>{% translate "Download task data" %}</a>
                                     {% endif %}
                                 </div>
                             </section>

rocky/rocky/templates/tasks/partials/normalizer_task_history.html

@@ -8,19 +8,29 @@ <h2>{% translate "Tasks" %}</h2>
         <table rf-selector="table-scan-history">
             <thead>
                 <tr>
+                    {% if not organization.code %}
+                        <th scope="col">{% translate "Organization Code" %}</th>
+                    {% endif %}
                     <th scope="col">{% translate "Normalizer" %}</th>
                     <th scope="col">{% translate "Status" %}</th>
                     <th scope="col">{% translate "Created date" %}</th>
+                    <th scope="col">{% translate "Modified date" %}</th>
                     <th scope="col">{% translate "Boefje" %}</th>
                     <th scope="col">{% translate "Boefje input OOI" %}</th>
                     <th scope="col" class="visually-hidden">{% translate "Details" %}</th>
                 </tr>
             </thead>
             <tbody>
                 {% for task in task_history %}
-                    <tr data-task-id="{{ task.id }}">
+                    <tr data-task-id="{{ task.id }}"
+                        {% if not organization %}data-organization-code="{{ task.p_item.data.raw_data.boefje_meta.organization }}"{% endif %}>
+                        {% if not organization %}
+                            <td>
+                                <a href="{% url "organization_crisis_room" task.p_item.data.raw_data.boefje_meta.organization %}">{{ task.p_item.data.raw_data.boefje_meta.organization }}</a>
+                            </td>
+                        {% endif %}
                         <td>
-                            <a href="{% url "normalizer_detail" organization.code task.p_item.data.normalizer.id %}">{{ task.p_item.data.normalizer.id }}</a>
+                            <a href="{% url "normalizer_detail" task.p_item.data.raw_data.boefje_meta.organization task.p_item.data.normalizer.id %}">{{ task.p_item.data.normalizer.id }}</a>
                         </td>
                         <td class="nowrap">
                             {% if task.status.value == "pending" or task.status.value == "queued" %}
@@ -38,11 +48,12 @@ <h2>{% translate "Tasks" %}</h2>
                             {% endif %}
                         </td>
                         <td>{{ task.created_at }}</td>
+                        <td>{{ task.modified_at }}</td>
                         <td>
-                            <a href="{% url "boefje_detail" organization.code task.p_item.data.raw_data.boefje_meta.boefje.id %}">{{ task.p_item.data.raw_data.boefje_meta.boefje.id }}</a>
+                            <a href="{% url "boefje_detail" task.p_item.data.raw_data.boefje_meta.organization task.p_item.data.raw_data.boefje_meta.boefje.id %}">{{ task.p_item.data.raw_data.boefje_meta.boefje.id }}</a>
                         </td>
                         <td>
-                            <a href="{% ooi_url "ooi_detail" task.p_item.data.raw_data.boefje_meta.input_ooi organization.code observed_at=task.created_at|date:'c' %}">{{ task.p_item.data.raw_data.boefje_meta.input_ooi }}</a>
+                            <a href="{% ooi_url "ooi_detail" task.p_item.data.raw_data.boefje_meta.input_ooi task.p_item.data.raw_data.boefje_meta.organization observed_at=task.created_at|date:'c' %}">{{ task.p_item.data.raw_data.boefje_meta.input_ooi }}</a>
                         </td>
                         {# FIXME: implement detail page according to designs #}
                         <td>
@@ -68,12 +79,12 @@ <h2>{% translate "Yielded objects" %}</h2>
                                     <div class="button-container">
                                         {% if task.status.value in "completed,failed" %}
                                             <a class="button"
-                                               href="{% url 'bytes_raw' organization_code=organization.code boefje_meta_id=task.p_item.data.raw_data.boefje_meta.id %}"><span class="icon ti-download"></span>{% translate "Download meta and raw data" %}</a>
-                                            {% include "partials/single_action_form.html" with btn_text=_("Reschedule") btn_class="ghost" btn_icon="icon ti-refresh" action="reschedule_task" key="task_id" value=task.id %}
+                                               href="{% url 'bytes_raw' organization_code=task.p_item.data.raw_data.boefje_meta.organization boefje_meta_id=task.p_item.data.raw_data.boefje_meta.id %}"><span class="icon ti-download"></span>{% translate "Download meta and raw data" %}</a>
+                                            {% include "partials/single_action_form.html" with btn_text=_("Reschedule") btn_class="ghost" btn_icon="icon ti-refresh" action="reschedule_task" key="task_id" organization_code=task.p_item.data.raw_data.boefje_meta.organization value=task.id %}
 
                                         {% else %}
                                             <a class="button"
-                                               href="{% url 'download_task_meta' organization_code=organization.code task_id=task.id %}"><span class="icon ti-download"></span>{% translate "Download task data" %}</a>
+                                               href="{% url 'download_task_meta' organization_code=task.p_item.data.raw_data.boefje_meta.organization task_id=task.id %}"><span class="icon ti-download"></span>{% translate "Download task data" %}</a>
                                         {% endif %}
                                     </div>
                                 </div>

rocky/rocky/templates/tasks/partials/tab_navigation.html

@@ -3,10 +3,18 @@
 <nav class="tabs" aria-label="{% translate "List of tasks" %}">
     <ul>
         <li {% if view == "boefjes_tasks" %}aria-current="page"{% endif %}>
-            <a href="{% url 'boefjes_task_list' organization.code %}">{% translate "Boefjes" %}</a>
+            {% if organization %}
+                <a href="{% url 'boefjes_task_list' organization.code %}">{% translate "Boefjes" %}</a>
+            {% else %}
+                <a href="{% url 'all_boefjes_task_list' %}">{% translate "Boefjes" %}</a>
+            {% endif %}
         </li>
         <li {% if view == "normalizers_tasks" %}aria-current="page"{% endif %}>
-            <a href="{% url 'normalizers_task_list' organization.code %}">{% translate "Normalizers" %}</a>
+            {% if organization %}
+                <a href="{% url 'normalizers_task_list' organization.code %}">{% translate "Normalizers" %}</a>
+            {% else %}
+                <a href="{% url 'all_normalizers_task_list' %}">{% translate "Normalizers" %}</a>
+            {% endif %}
         </li>
     </ul>
 </nav>

rocky/rocky/urls.py

@@ -40,7 +40,13 @@
 from rocky.views.scan_profile import ScanProfileDetailView, ScanProfileResetView
 from rocky.views.scans import ScanListView
 from rocky.views.task_detail import BoefjeTaskDetailView, NormalizerTaskJSONView
-from rocky.views.tasks import BoefjesTaskListView, DownloadTaskDetail, NormalizersTaskListView
+from rocky.views.tasks import (
+    AllBoefjesTaskListView,
+    AllNormalizersTaskListView,
+    BoefjesTaskListView,
+    DownloadTaskDetail,
+    NormalizersTaskListView,
+)
 from rocky.views.upload_csv import UploadCSV
 from rocky.views.upload_raw import UploadRaw
 
@@ -72,6 +78,9 @@
         PrivacyStatementView.as_view(),
         name="privacy_statement",
     ),
+    path("tasks/", AllBoefjesTaskListView.as_view(), name="all_task_list"),
+    path("tasks/boefjes", AllBoefjesTaskListView.as_view(), name="all_boefjes_task_list"),
+    path("tasks/normalizers", AllNormalizersTaskListView.as_view(), name="all_normalizers_task_list"),
     path(
         "<organization_code>/settings/indemnifications/",
         IndemnificationAddView.as_view(),