Gui reprocessing: update variable list from file content

damnit/backend/extract_data.py

@@ -81,8 +81,8 @@ def find_class(self, module, name):
         else:
             return super().find_class(module, name)
 
-def get_context_file(ctx_path: Path, context_python=None):
-    ctx_path = ctx_path.absolute()
+def get_context_file(ctx_path: str | Path, context_python=None):
+    ctx_path = Path(ctx_path).absolute()
     db_dir = ctx_path.parent
 
     if context_python is None:

damnit/ctxsupport/damnit_ctx.py

@@ -50,6 +50,8 @@ def __init__(
     def __call__(self, func):
         self.func = func
         self.name = func.__name__
+        if self.title is None:
+            self.title = self.name
         return self
 
     def check(self):

damnit/gui/main_window.py

@@ -50,6 +50,7 @@
 
     context_dir_changed = QtCore.pyqtSignal(str)
     save_context_finished = QtCore.pyqtSignal(bool)  # True if saved
+    context_saved = QtCore.pyqtSignal()
 
     db = None
     db_id = None
@@ -88,6 +89,8 @@
         self._tab_widget.setEnabled(False)
         self.setCentralWidget(self._tab_widget)
 
+        self.context_saved.connect(self.launch_update_computed_vars)
+
         self.table = None
 
         self.zulip_messenger = None
@@ -262,6 +265,7 @@
         self.launch_update_computed_vars()
 
     def launch_update_computed_vars(self):
+        # Triggered when we open a proposal & when saving the context file
         log.debug("Launching subprocess to read variables from context file")
         proc = QtCore.QProcess(parent=self)
         # Show stdout & stderr with the parent process
@@ -341,6 +345,7 @@
         self.context_dir_changed.connect(lambda _: self.action_export.setEnabled(True))
         self.action_export.triggered.connect(self.export_table)
         self.action_process = QtWidgets.QAction("Reprocess runs", self)
+        self.action_process.setShortcut("Shift+R")
         self.action_process.triggered.connect(self.process_runs)
 
         action_adeqt = QtWidgets.QAction("Python console", self)
@@ -812,6 +817,7 @@
                 self.mark_context_saved()
             self._context_code_to_save = None
             self.save_context_finished.emit(saving)
+            self.context_saved.emit()
 
         if test_result == ContextTestResult.ERROR:
             self.set_error_widget_text(output)
@@ -919,10 +925,7 @@
             prop = self.db.metameta.get("proposal", "")
             sel_runs = []
 
-        var_ids_titles = zip(self.table.computed_columns(),
-                             self.table.computed_columns(by_title=True))
-
-        dlg = ProcessingDialog(str(prop), sel_runs, var_ids_titles, parent=self)
+        dlg = ProcessingDialog(str(prop), sel_runs, parent=self)
         if dlg.exec() == QtWidgets.QDialog.Accepted:
             submitter = ExtractionSubmitter(self.context_dir, self.db)
 

damnit/gui/process.py

@@ -1,23 +1,99 @@
+import ast
 import logging
 import re
+from collections import defaultdict
+from dataclasses import dataclass, field
+from functools import partial
 from pathlib import Path
 
 from PyQt5 import QtWidgets
 from PyQt5.QtCore import Qt
 from PyQt5.QtWidgets import QDialogButtonBox
 
 from extra_data.read_machinery import find_proposal
+from natsort import natsorted
+from superqt.utils import thread_worker
+from superqt import QSearchableListWidget
 
-from ..context import RunData
 from ..backend.extraction_control import ExtractionRequest
+from ..backend.extract_data import get_context_file
+from ..context import RunData
+from .widgets import QtWaitingSpinner
 
 log = logging.getLogger(__name__)
 
 run_range_re = re.compile(r"(\d+)(-\d+)?$")
 
 RUNS_MSG = "Enter run numbers & ranges e.g. '17, 20-32'"
 
-deselected_vars = set()
+
+@dataclass
+class VariableInfo:
+    name: str = None
+    title: str = None
+    selected: bool = True
+
+
+@dataclass
+class ContextFileInfo:
+    file_path: Path = None
+    mtime: float = None
+    error: bool = False
+    variables: defaultdict = field(default_factory=partial(defaultdict, VariableInfo))
+
+    def sorted_vars(self):
+        return [v for v in natsorted(self.variables.values(), key=lambda e: e.title)]
+
+ctx_info = ContextFileInfo()
+
+
+def get_variable_list(ctx_path: Path):
+    context = ctx_path.read_text()
+
+    try:
+        tree = ast.parse(context)
+    except SyntaxError:
+        return
+
+    def _is_variable(node):
+        for decorator in node.decorator_list:
+            func = decorator.func
+            while isinstance(func, ast.Attribute):
+                func = func.value
+            if func.id == 'Variable':
+                return decorator
+
+    def _get_kwarg(decorator, name='title'):
+        for kw in decorator.keywords:
+            if kw.arg == name:
+                return kw.value.value
+
+    variables = []
+    for node in ast.walk(tree):
+        if isinstance(node, ast.FunctionDef):
+            if variable := _is_variable(node):
+                title = _get_kwarg(variable) or node.name
+                variables.append(VariableInfo(node.name, title))
+
+    return variables
+
+
+@thread_worker
+def get_context_file_vars(ctx_path: Path):
+    variables = get_variable_list(ctx_path)
+    if variables is None:
+        ctx_info.variables.clear()
+        ctx_info.error = True
+        return
+
+    for var in set(ctx_info.variables).difference({v.name for v in variables}):
+        ctx_info.variables.pop(var)
+
+    for var in natsorted(variables, key=lambda e: e.title):
+        # try to get the exising var info, to keep the checkbox state
+        viv = ctx_info.variables.setdefault(var.name, var)
+        # update the title in case it has changed
+        viv.title = var.title
 
 
 def parse_run_ranges(ranges: str) -> list[int]:
@@ -76,10 +152,12 @@
     all_vars_selected = False
     no_vars_selected = False
 
-    def __init__(self, proposal: str, runs: list[int], var_ids_titles, parent=None):
+    def __init__(self, proposal: str, runs: list[int], parent=None):
         super().__init__(parent)
 
         self.setWindowTitle("Process runs")
+        self.setSizeGripEnabled(True)
+        self.setMinimumSize(300, 200)
 
         main_vbox = QtWidgets.QVBoxLayout()
         self.setLayout(main_vbox)
@@ -106,7 +184,9 @@
         grid1.addWidget(self.edit_runs, 1, 1)
         grid1.addWidget(self.runs_hint, 2, 0, 1, 2)
 
-        self.vars_list = QtWidgets.QListWidget()
+        self.vars_list = QSearchableListWidget()
+        self.vars_list.filter_widget.setPlaceholderText("Search variable")
+        self.vars_list.layout().setContentsMargins(0, 0, 0, 0)
         vbox2.addWidget(self.vars_list)
 
         self.btn_select_all = QtWidgets.QPushButton("Select all")
@@ -124,19 +204,52 @@
         self.dlg_buttons.rejected.connect(self.reject)
         main_vbox.addWidget(self.dlg_buttons)
 
-        for var_id, title in var_ids_titles:
-            itm = QtWidgets.QListWidgetItem(title)
-            itm.setData(Qt.UserRole, var_id)
-            itm.setCheckState(Qt.Unchecked if var_id in deselected_vars else Qt.Checked)
-            self.vars_list.addItem(itm)
-
-        self.vars_list.itemChanged.connect(self.validate_vars)
+        context_file = Path(self.parent()._context_path)
+        vars_getter = None
+        if ctx_info.file_path != context_file:
+            ctx_info.file_path = context_file
+            ctx_info.mtime = context_file.stat().st_mtime
+            ctx_info.variables.clear()
+            vars_getter = get_context_file_vars(context_file)
+        elif ctx_info.mtime != context_file.stat().st_mtime:
+            vars_getter = get_context_file_vars(context_file)
+
+        self.spinner = QtWaitingSpinner(self.vars_list.list_widget, modality=Qt.ApplicationModal)
+        if vars_getter is not None:
+            vars_getter.returned.connect(self.update_list_items)
+            vars_getter.start()
+            self.spinner.start()
+        else:
+            self.update_list_items()
 
         self.validate_runs()
-        self.validate_vars()
 
         self.edit_runs.setFocus()
 
+    def _stop_spinner(self):
+        if self.spinner.isSpinning():
+            self.spinner.stop()
+            self.spinner.hide()
+
+    def update_list_items(self):
+        self._stop_spinner()
+
+        self.vars_list.clear()
+
+        for var in ctx_info.sorted_vars():
+            item = QtWidgets.QListWidgetItem(var.title)
+            item.setData(Qt.UserRole, var.name)
+            item.setCheckState(Qt.Checked if var.selected else Qt.Unchecked)
+            self.vars_list.addItem(item)
+
+        self.validate_vars()
+
+        if ctx_info.error:
+            self.runs_hint.setText(
+                f'{self.runs_hint.text()}\n\n'
+                '/!\\ context file contains error! Check the editor'
+            )
+
     def validate_runs(self):
         runs = parse_run_ranges(self.edit_runs.text())
         self.selected_runs = find_runs(runs, self.edit_prop.text())
@@ -174,12 +287,8 @@
             itm.setCheckState(Qt.Unchecked)
 
     def save_vars_selection(self):
-        # We save the deselected variables, so new variables are selected
-        global deselected_vars
-        deselected_vars = {
-            itm.data(Qt.UserRole) for itm in self._var_list_items()
-            if itm.checkState() == Qt.Unchecked
-        }
+        for item, var in zip(self._var_list_items(), ctx_info.sorted_vars()):
+            var.selected = item.checkState() == Qt.Checked
 
     def accept(self):
         self.save_vars_selection()

damnit/gui/table.py

@@ -115,7 +115,11 @@ def set_column_visibility(self, name, visible, for_restore=False):
         deselected. The `for_restore` argument lets you specify which behaviour
         you want.
         """
-        column_index = self.damnit_model.find_column(name, by_title=True)
+        try:
+            column_index = self.damnit_model.find_column(name, by_title=True)
+        except KeyError:
+            log.error("Could not find column %r to set visibility", name)
+            return
 
         self.setColumnHidden(column_index, not visible)
 

pyproject.toml

@@ -40,11 +40,13 @@ gui = [
     "adeqt",
     "mplcursors",
     "mpl-pan-zoom",
+    "natsort",
     "openpyxl",  # for spreadsheet export
     "PyQt5",
     "PyQtWebEngine",
     "pyflakes",  # for checking context file in editor
     "QScintilla==2.13",
+    "superqt",
     "tabulate",  # used in pandas to make markdown tables (for Zulip)
 ]
 test = [