feature/devops fixes

.pre-commit-config.yaml

@@ -3,7 +3,7 @@ default_language_version:
   python: python3.11
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.8.3
+    rev: v0.9.1
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
@@ -25,7 +25,7 @@ repos:
       - id: fix-byte-order-marker
         name: byte-order
   - repo: https://github.com/pdm-project/pdm
-    rev: 2.22.0
+    rev: 2.22.2
     hooks:
       - id: pdm-lock-check
         name: pdm

CHANGELOG.md

@@ -11,17 +11,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changes
 
+- set dockerfile base to bullseye
+
 ### Deprecated
 
 ### Removed
 
 ### Fixed
 
+- fix module name for dagster startup in compose.yaml
+
 ### Security
 
 ## [0.24.0] - 2025-01-15
 
 ### Added
+
 - extractor for Open Data
 
 ### Changes

Dockerfile

@@ -1,5 +1,8 @@
 # syntax=docker/dockerfile:1
 
+# using bullseye because microsoft does not play nice with debian 12 signature verification yet
+# https://learn.microsoft.com/en-us/answers/questions/1328834/debian-12-public-key-is-not-available
+# debian 11 bullseye is on a LTS schedule until August 31st, 2026
 FROM python:3.11 AS base
 
 LABEL org.opencontainers.image.authors="[email protected]"
@@ -11,9 +14,10 @@ LABEL org.opencontainers.image.vendor="robert-koch-institut"
 ENV PYTHONUNBUFFERED=1
 ENV PYTHONOPTIMIZE=1
 
-ENV PIP_PROGRESS_BAR=off
-ENV PIP_PREFER_BINARY=on
 ENV PIP_DISABLE_PIP_VERSION_CHECK=on
+ENV PIP_NO_INPUT=on
+ENV PIP_PREFER_BINARY=on
+ENV PIP_PROGRESS_BAR=off
 
 WORKDIR /app
 
@@ -31,4 +35,4 @@ RUN --mount=type=cache,target=/root/.cache/pip pip install -r locked-requirement
 
 USER mex
 
-ENTRYPOINT [ "artificial" ]
+ENTRYPOINT [ "all-extractors" ]

compose.yaml

@@ -6,7 +6,7 @@ services:
     environment:
       - DAGSTER_HOME=/app/work
       - MEX_WORK_DIR=/app/work
-    entrypoint: ["dagster-daemon", "run", "--module-name", "mex"]
+    entrypoint: ["dagster-daemon", "run", "--module-name", "mex.extractors"]
     volumes:
       - work:/app/work:rw
   dagster-webserver:
@@ -17,7 +17,7 @@ services:
     environment:
       - DAGSTER_HOME=/app/work
       - MEX_WORK_DIR=/app/work
-    entrypoint: ["dagster-webserver", "--host", "0.0.0.0", "--module-name", "mex"]
+    entrypoint: ["dagster-webserver", "--host", "0.0.0.0", "--module-name", "mex.extractors"]
     expose:
       - 3000
     volumes:

mex/extractors/datscha_web/parse_html.py

@@ -99,8 +99,7 @@ def parse_unit_loz(bs4_object: BeautifulSoup) -> tuple[str, list[str]]:
     table = bs4_object.find("table", {"id": "loz"})
     if not isinstance(table, Tag):
         raise MExError(
-            f"Missing table with ID 'loz' in single item html.\n"
-            f"{bs4_object.prettify()}"
+            f"Missing table with ID 'loz' in single item html.\n{bs4_object.prettify()}"
         )
     column_headers = table.find_all("th")
     key = str(column_headers[0].string)

mex/extractors/ff_projects/filter.py

@@ -124,7 +124,7 @@ def filter_and_log_ff_projects_source(
         log_filter(
             identifier_in_primary_source,
             primary_source_id,
-            f'RKI- OE [{source.rki_oe.replace("/", ",")}] are all not valid units',
+            f"RKI- OE [{source.rki_oe.replace('/', ',')}] are all not valid units",
         )
         return False
 

mex/extractors/international_projects/extract.py

@@ -25,9 +25,9 @@
 
 
 @watch
-def extract_international_projects_sources() -> (
-    Generator[InternationalProjectsSource, None, None]
-):
+def extract_international_projects_sources() -> Generator[
+    InternationalProjectsSource, None, None
+]:
     """Extract international projects sources by loading data from MS-Excel file.
 
     Returns:

mex/extractors/sumo/transform.py

@@ -372,7 +372,7 @@ def transform_nokeda_model_variable_to_mex_variable(
     value_sets = list(extracted_cc1_data_valuesets)
     for variable in extracted_cc1_data_model_nokeda:
         value_set = [
-            f"{v.category_label_de}," f"{v.category_label_en or ''}"
+            f"{v.category_label_de},{v.category_label_en or ''}"
             for v in value_sets
             if v.sheet_name == variable.variable_name
         ]