From c092b22238c9e603c5cba67c78193d8378cf39fd Mon Sep 17 00:00:00 2001 From: Nicolas Drebenstedt Date: Tue, 21 Jan 2025 14:37:50 +0100 Subject: [PATCH 1/5] update versions --- .pre-commit-config.yaml | 4 +-- pdm.lock | 68 +++++++++++++++++------------------ pyproject.toml | 78 ++++++++++++++++++++--------------------- requirements.txt | 1 - 4 files changed, 75 insertions(+), 76 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 26f77074..d328f57f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ default_language_version: python: python3.11 repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.8.3 + rev: v0.9.1 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] @@ -25,7 +25,7 @@ repos: - id: fix-byte-order-marker name: byte-order - repo: https://github.com/pdm-project/pdm - rev: 2.22.0 + rev: 2.22.2 hooks: - id: pdm-lock-check name: pdm diff --git a/pdm.lock b/pdm.lock index 315d6ce5..2df0c889 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "dev"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:0765136f9110dff25280313486160aa36cf831398a49443f2dbc2bd08e653e81" +content_hash = "sha256:a5a55c7757a34734127b044098d93d07a11a8afe97a622ef26f5c142ad4db4eb" [[metadata.targets]] requires_python = "==3.11.*" @@ -1202,8 +1202,8 @@ files = [ [[package]] name = "prompt-toolkit" -version = "3.0.48" -requires_python = ">=3.7.0" +version = "3.0.50" +requires_python = ">=3.8.0" summary = "Library for building powerful interactive command lines in Python" groups = ["dev"] marker = "python_version == \"3.11\"" @@ -1211,8 +1211,8 @@ dependencies = [ "wcwidth", ] files = [ - {file = "prompt_toolkit-3.0.48-py3-none-any.whl", hash = "sha256:f49a827f90062e411f1ce1f854f2aedb3c23353244f8108b89283587397ac10e"}, - {file = "prompt_toolkit-3.0.48.tar.gz", hash = "sha256:d6623ab0477a80df74e646bdbc93621143f5caf104206aa29294d53de1a03d90"}, + {file = "prompt_toolkit-3.0.50-py3-none-any.whl", hash = "sha256:9b6427eb19e479d98acff65196a307c555eb567989e6d88ebbb1b509d9779198"}, + {file = "prompt_toolkit-3.0.50.tar.gz", hash = "sha256:544748f3860a2623ca5cd6d2795e7a14f3d0e1c3c9728359013f79877fc89bab"}, ] [[package]] @@ -2237,7 +2237,7 @@ files = [ [[package]] name = "watchfiles" -version = "1.0.0" +version = "1.0.4" requires_python = ">=3.9" summary = "Simple, modern and high performance file watching and code reload in python." groups = ["default"] @@ -2246,20 +2246,20 @@ dependencies = [ "anyio>=3.0.0", ] files = [ - {file = "watchfiles-1.0.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:fbd0ab7a9943bbddb87cbc2bf2f09317e74c77dc55b1f5657f81d04666c25269"}, - {file = "watchfiles-1.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:774ef36b16b7198669ce655d4f75b4c3d370e7f1cbdfb997fb10ee98717e2058"}, - {file = "watchfiles-1.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9b4fb98100267e6a5ebaff6aaa5d20aea20240584647470be39fe4823012ac96"}, - {file = "watchfiles-1.0.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0fc3bf0effa2d8075b70badfdd7fb839d7aa9cea650d17886982840d71fdeabf"}, - {file = "watchfiles-1.0.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:648e2b6db53eca6ef31245805cd528a16f56fa4cc15aeec97795eaf713c11435"}, - {file = "watchfiles-1.0.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fa13d604fcb9417ae5f2e3de676e66aa97427d888e83662ad205bed35a313176"}, - {file = "watchfiles-1.0.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:936f362e7ff28311b16f0b97ec51e8f2cc451763a3264640c6ed40fb252d1ee4"}, - {file = "watchfiles-1.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:245fab124b9faf58430da547512d91734858df13f2ddd48ecfa5e493455ffccb"}, - {file = "watchfiles-1.0.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4ff9c7e84e8b644a8f985c42bcc81457240316f900fc72769aaedec9d088055a"}, - {file = "watchfiles-1.0.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9c9a8d8fd97defe935ef8dd53d562e68942ad65067cd1c54d6ed8a088b1d931d"}, - {file = "watchfiles-1.0.0-cp311-none-win32.whl", hash = "sha256:a0abf173975eb9dd17bb14c191ee79999e650997cc644562f91df06060610e62"}, - {file = "watchfiles-1.0.0-cp311-none-win_amd64.whl", hash = "sha256:2a825ba4b32c214e3855b536eb1a1f7b006511d8e64b8215aac06eb680642d84"}, - {file = "watchfiles-1.0.0-cp311-none-win_arm64.whl", hash = "sha256:a5a7a06cfc65e34fd0a765a7623c5ba14707a0870703888e51d3d67107589817"}, - {file = "watchfiles-1.0.0.tar.gz", hash = "sha256:37566c844c9ce3b5deb964fe1a23378e575e74b114618d211fbda8f59d7b5dab"}, + {file = "watchfiles-1.0.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:2a9f93f8439639dc244c4d2902abe35b0279102bca7bbcf119af964f51d53c19"}, + {file = "watchfiles-1.0.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9eea33ad8c418847dd296e61eb683cae1c63329b6d854aefcd412e12d94ee235"}, + {file = "watchfiles-1.0.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:31f1a379c9dcbb3f09cf6be1b7e83b67c0e9faabed0471556d9438a4a4e14202"}, + {file = "watchfiles-1.0.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ab594e75644421ae0a2484554832ca5895f8cab5ab62de30a1a57db460ce06c6"}, + {file = "watchfiles-1.0.4-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fc2eb5d14a8e0d5df7b36288979176fbb39672d45184fc4b1c004d7c3ce29317"}, + {file = "watchfiles-1.0.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3f68d8e9d5a321163ddacebe97091000955a1b74cd43724e346056030b0bacee"}, + {file = "watchfiles-1.0.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f9ce064e81fe79faa925ff03b9f4c1a98b0bbb4a1b8c1b015afa93030cb21a49"}, + {file = "watchfiles-1.0.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b77d5622ac5cc91d21ae9c2b284b5d5c51085a0bdb7b518dba263d0af006132c"}, + {file = "watchfiles-1.0.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:1941b4e39de9b38b868a69b911df5e89dc43767feeda667b40ae032522b9b5f1"}, + {file = "watchfiles-1.0.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4f8c4998506241dedf59613082d1c18b836e26ef2a4caecad0ec41e2a15e4226"}, + {file = "watchfiles-1.0.4-cp311-cp311-win32.whl", hash = "sha256:4ebbeca9360c830766b9f0df3640b791be569d988f4be6c06d6fae41f187f105"}, + {file = "watchfiles-1.0.4-cp311-cp311-win_amd64.whl", hash = "sha256:05d341c71f3d7098920f8551d4df47f7b57ac5b8dad56558064c3431bdfc0b74"}, + {file = "watchfiles-1.0.4-cp311-cp311-win_arm64.whl", hash = "sha256:32b026a6ab64245b584acf4931fe21842374da82372d5c039cba6bf99ef722f3"}, + {file = "watchfiles-1.0.4.tar.gz", hash = "sha256:6ba473efd11062d73e4f00c2b730255f9c1bdd73cd5f9fe5b5da8dbd4a717205"}, ] [[package]] @@ -2278,25 +2278,25 @@ files = [ [[package]] name = "websockets" -version = "14.1" +version = "14.2" requires_python = ">=3.9" summary = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)" groups = ["default"] marker = "python_version == \"3.11\"" files = [ - {file = "websockets-14.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:449d77d636f8d9c17952628cc7e3b8faf6e92a17ec581ec0c0256300717e1512"}, - {file = "websockets-14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a35f704be14768cea9790d921c2c1cc4fc52700410b1c10948511039be824aac"}, - {file = "websockets-14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b1f3628a0510bd58968c0f60447e7a692933589b791a6b572fcef374053ca280"}, - {file = "websockets-14.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c3deac3748ec73ef24fc7be0b68220d14d47d6647d2f85b2771cb35ea847aa1"}, - {file = "websockets-14.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7048eb4415d46368ef29d32133134c513f507fff7d953c18c91104738a68c3b3"}, - {file = "websockets-14.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6cf0ad281c979306a6a34242b371e90e891bce504509fb6bb5246bbbf31e7b6"}, - {file = "websockets-14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:cc1fc87428c1d18b643479caa7b15db7d544652e5bf610513d4a3478dbe823d0"}, - {file = "websockets-14.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:f95ba34d71e2fa0c5d225bde3b3bdb152e957150100e75c86bc7f3964c450d89"}, - {file = "websockets-14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9481a6de29105d73cf4515f2bef8eb71e17ac184c19d0b9918a3701c6c9c4f23"}, - {file = "websockets-14.1-cp311-cp311-win32.whl", hash = "sha256:368a05465f49c5949e27afd6fbe0a77ce53082185bbb2ac096a3a8afaf4de52e"}, - {file = "websockets-14.1-cp311-cp311-win_amd64.whl", hash = "sha256:6d24fc337fc055c9e83414c94e1ee0dee902a486d19d2a7f0929e49d7d604b09"}, - {file = "websockets-14.1-py3-none-any.whl", hash = "sha256:4d4fc827a20abe6d544a119896f6b78ee13fe81cbfef416f3f2ddf09a03f0e2e"}, - {file = "websockets-14.1.tar.gz", hash = "sha256:398b10c77d471c0aab20a845e7a60076b6390bfdaac7a6d2edb0d2c59d75e8d8"}, + {file = "websockets-14.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3bdc8c692c866ce5fefcaf07d2b55c91d6922ac397e031ef9b774e5b9ea42166"}, + {file = "websockets-14.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c93215fac5dadc63e51bcc6dceca72e72267c11def401d6668622b47675b097f"}, + {file = "websockets-14.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1c9b6535c0e2cf8a6bf938064fb754aaceb1e6a4a51a80d884cd5db569886910"}, + {file = "websockets-14.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a52a6d7cf6938e04e9dceb949d35fbdf58ac14deea26e685ab6368e73744e4c"}, + {file = "websockets-14.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9f05702e93203a6ff5226e21d9b40c037761b2cfb637187c9802c10f58e40473"}, + {file = "websockets-14.2-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22441c81a6748a53bfcb98951d58d1af0661ab47a536af08920d129b4d1c3473"}, + {file = "websockets-14.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:efd9b868d78b194790e6236d9cbc46d68aba4b75b22497eb4ab64fa640c3af56"}, + {file = "websockets-14.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1a5a20d5843886d34ff8c57424cc65a1deda4375729cbca4cb6b3353f3ce4142"}, + {file = "websockets-14.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:34277a29f5303d54ec6468fb525d99c99938607bc96b8d72d675dee2b9f5bf1d"}, + {file = "websockets-14.2-cp311-cp311-win32.whl", hash = "sha256:02687db35dbc7d25fd541a602b5f8e451a238ffa033030b172ff86a93cb5dc2a"}, + {file = "websockets-14.2-cp311-cp311-win_amd64.whl", hash = "sha256:862e9967b46c07d4dcd2532e9e8e3c2825e004ffbf91a5ef9dde519ee2effb0b"}, + {file = "websockets-14.2-py3-none-any.whl", hash = "sha256:7a6ceec4ea84469f15cf15807a747e9efe57e369c384fa86e022b3bea679b79b"}, + {file = "websockets-14.2.tar.gz", hash = "sha256:5059ed9c54945efb321f097084b4c7e52c246f2c869815876a69d1efc4ad6eb5"}, ] [[package]] diff --git a/pyproject.toml b/pyproject.toml index 97652467..de43bc9c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ optional-dependencies.dev = [ "pytest-random-order>=1,<2", "pytest-xdist>=3,<4", "pytest>=8,<9", - "ruff>=0.7,<1", + "ruff>=0.9,<1", "sphinx>=8,<9", "types-beautifulsoup4>=4,<5", "types-pytz>=2024,<2025", @@ -126,53 +126,53 @@ docstring-code-format = true [tool.ruff.lint] ignore = [ - "AIR", # Disable airflow specific rules (we are not using airflow) - "ANN", # Disable all annotations checks (handled by mypy) - "COM", # Disable flake8-commas checks (let ruff format handle that) - "CPY", # Disable copyright notice checks (we have LICENSE files) - "D100", # Allow missing module docstring (for brevity and speed) - "D104", # Allow missing package docstring (for brevity and speed) - "D203", # Disallow blank line before class docstring (inverse of D211) - "D213", # Disallow multi-line docstring starting at second line (inverse of D212) - "D406", # Allow section name ending with newline (google style compat) - "D407", # Allow missing dashed underline after section (google style compat) - "D413", # Allow missing blank line after last section (google style compat) - "DJ", # Disable django specific checks (we are not using django) - "FBT", # Disable boolean type hint checks (for more flexibility) - "FIX", # Allow committing with open TODOs (don't punish committers) - "ISC001", # Disable checks for implicitly concatenated strings (formatter compat) - "N805", # Allow first argument of a method to be non-self (pydantic compat) - "N815", # Allow mixedCase variables in class scope (model compat) - "PTH123", # Allow using builtin open method (simpler than pathlib) - "RUF012", # Allow mutable class attributes (pydantic compat) - "SIM108", # Allow explicit if-else instead of ternary (easier to read) - "TD003", # Allow TODOs without ticket link (don't punish TODO writers) - "TRY003", # Allow long exception message at the raise site (for pydantic) + "AIR", # Disable airflow specific rules (we are not using airflow) + "ANN", # Disable all annotations checks (handled by mypy) + "COM", # Disable flake8-commas checks (let ruff format handle that) + "CPY", # Disable copyright notice checks (we have LICENSE files) + "D100", # Allow missing module docstring (for brevity and speed) + "D104", # Allow missing package docstring (for brevity and speed) + "D203", # Disallow blank line before class docstring (inverse of D211) + "D213", # Disallow multi-line docstring starting at second line (inverse of D212) + "D406", # Allow section name ending with newline (google style compat) + "D407", # Allow missing dashed underline after section (google style compat) + "D413", # Allow missing blank line after last section (google style compat) + "DJ", # Disable django specific checks (we are not using django) + "FBT", # Disable boolean type hint checks (for more flexibility) + "FIX", # Allow committing with open TODOs (don't punish committers) + "ISC001", # Disable checks for implicitly concatenated strings (formatter compat) + "N805", # Allow first argument of a method to be non-self (pydantic compat) + "N815", # Allow mixedCase variables in class scope (model compat) + "PTH123", # Allow using builtin open method (simpler than pathlib) + "RUF012", # Allow mutable class attributes (pydantic compat) + "SIM108", # Allow explicit if-else instead of ternary (easier to read) + "TD003", # Allow TODOs without ticket link (don't punish TODO writers) + "TRY003", # Allow long exception message at the raise site (for pydantic) ] select = ["ALL"] [tool.ruff.lint.per-file-ignores] "docs/**" = [ - "INP001", # Docs folder does not need to be a package + "INP001", # Docs folder does not need to be a package ] "scripts/**" = [ - "INP001", # Scripts folder does not need to be a package + "INP001", # Scripts folder does not need to be a package ] "tests/**" = [ - "ARG005", # Allow unused lambda arguments for mocking - "D101", # Allow missing docstring in public class - "D102", # Allow missing docstring in public method - "D103", # Allow missing docstring in public function - "D107", # Allow missing docstring in `__init__` - "E501", # Allow longer lines with test data - "ISC", # Allow implicitly concatenated strings - "N807", # Allow mocking `__init__` - "PLR0915", # Allow functions with many statements - "PLR2004", # Allow comparing with static values - "PT004", # Allow public fixtures without returns - "PT013", # Allow more flexible pytest imports - "S101", # Allow use of `assert` in tests - "SLF", # Allow private member access + "ARG005", # Allow unused lambda arguments for mocking + "D101", # Allow missing docstring in public class + "D102", # Allow missing docstring in public method + "D103", # Allow missing docstring in public function + "D107", # Allow missing docstring in `__init__` + "E501", # Allow longer lines with test data + "ISC", # Allow implicitly concatenated strings + "N807", # Allow mocking `__init__` + "PLR0915", # Allow functions with many statements + "PLR2004", # Allow comparing with static values + "PT004", # Allow public fixtures without returns + "PT013", # Allow more flexible pytest imports + "S101", # Allow use of `assert` in tests + "SLF", # Allow private member access ] "mex/extractors/biospecimen/**" = ["PD003", "PLR0913", "PTH207"] "mex/extractors/blueant/**" = ["PLR0913", "PLR2004"] diff --git a/requirements.txt b/requirements.txt index ce8711ab..3eac021a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,3 @@ cruft==2.16.0 mex-release==0.3.0 pdm==2.22.2 pre-commit==4.0.1 -wheel==0.45.1 From 923ce53fa47ebff9833a085a0be9d5bc05610047 Mon Sep 17 00:00:00 2001 From: Nicolas Drebenstedt Date: Tue, 21 Jan 2025 14:39:11 +0100 Subject: [PATCH 2/5] update dockerfile --- CHANGELOG.md | 3 +++ Dockerfile | 10 +++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce17adaf..ba413067 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changes +- set dockerfile base to bullseye + ### Deprecated ### Removed @@ -22,6 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [0.24.0] - 2025-01-15 ### Added + - extractor for Open Data ### Changes diff --git a/Dockerfile b/Dockerfile index e96ef38c..45231869 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,8 @@ # syntax=docker/dockerfile:1 +# using bullseye because microsoft does not play nice with debian 12 signature verification yet +# https://learn.microsoft.com/en-us/answers/questions/1328834/debian-12-public-key-is-not-available +# debian 11 bullseye is on a LTS schedule until August 31st, 2026 FROM python:3.11 AS base LABEL org.opencontainers.image.authors="mex@rki.de" @@ -11,9 +14,10 @@ LABEL org.opencontainers.image.vendor="robert-koch-institut" ENV PYTHONUNBUFFERED=1 ENV PYTHONOPTIMIZE=1 -ENV PIP_PROGRESS_BAR=off -ENV PIP_PREFER_BINARY=on ENV PIP_DISABLE_PIP_VERSION_CHECK=on +ENV PIP_NO_INPUT=on +ENV PIP_PREFER_BINARY=on +ENV PIP_PROGRESS_BAR=off WORKDIR /app @@ -31,4 +35,4 @@ RUN --mount=type=cache,target=/root/.cache/pip pip install -r locked-requirement USER mex -ENTRYPOINT [ "artificial" ] +ENTRYPOINT [ "all-extractors" ] From bff41fa2d31bf0b5fbe44d7df03dc435967fc917 Mon Sep 17 00:00:00 2001 From: Nicolas Drebenstedt Date: Tue, 21 Jan 2025 14:39:51 +0100 Subject: [PATCH 3/5] fix module name for dagster startup in compose.yaml --- CHANGELOG.md | 2 ++ compose.yaml | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ba413067..867b4819 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- fix module name for dagster startup in compose.yaml + ### Security ## [0.24.0] - 2025-01-15 diff --git a/compose.yaml b/compose.yaml index cb6c2290..862e874f 100644 --- a/compose.yaml +++ b/compose.yaml @@ -6,7 +6,7 @@ services: environment: - DAGSTER_HOME=/app/work - MEX_WORK_DIR=/app/work - entrypoint: ["dagster-daemon", "run", "--module-name", "mex"] + entrypoint: ["dagster-daemon", "run", "--module-name", "mex.extractors"] volumes: - work:/app/work:rw dagster-webserver: @@ -17,7 +17,7 @@ services: environment: - DAGSTER_HOME=/app/work - MEX_WORK_DIR=/app/work - entrypoint: ["dagster-webserver", "--host", "0.0.0.0", "--module-name", "mex"] + entrypoint: ["dagster-webserver", "--host", "0.0.0.0", "--module-name", "mex.extractors"] expose: - 3000 volumes: From 33d4a902192502e17623da021cc1be2af7f8b52e Mon Sep 17 00:00:00 2001 From: Nicolas Drebenstedt Date: Tue, 21 Jan 2025 14:45:35 +0100 Subject: [PATCH 4/5] bump mex-common --- pdm.lock | 8 ++++---- pyproject.toml | 11 ++++++++++- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/pdm.lock b/pdm.lock index 2df0c889..1a864d61 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "dev"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:a5a55c7757a34734127b044098d93d07a11a8afe97a622ef26f5c142ad4db4eb" +content_hash = "sha256:08032c6432c97a1723930f74b5f339aea8db04ca54b7019f0e4b3bc4d09dbc7b" [[metadata.targets]] requires_python = "==3.11.*" @@ -973,11 +973,11 @@ files = [ [[package]] name = "mex-common" -version = "0.45.0" +version = "0.46.0" requires_python = ">=3.11,<3.13" git = "https://github.com/robert-koch-institut/mex-common.git" -ref = "0.45.0" -revision = "f5ff330763011e1380d1321bf20bd7b346e1b484" +ref = "0.46.0" +revision = "ca1f62afdf09587d68fbb86c89bdbb2a1176377c" summary = "Common library for MEx python projects." groups = ["default"] marker = "python_version == \"3.11\"" diff --git a/pyproject.toml b/pyproject.toml index de43bc9c..104802f6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ dependencies = [ "dagster-webserver>=1,<2", "dagster>=1,<2", "faker>=33,<34", - "mex-common @ git+https://github.com/robert-koch-institut/mex-common.git@0.45.0", + "mex-common @ git+https://github.com/robert-koch-institut/mex-common.git@0.46.0", "numpy>=2,<3", "openpyxl>=3,<4", "pandas>=2,<3", @@ -152,6 +152,15 @@ ignore = [ select = ["ALL"] [tool.ruff.lint.per-file-ignores] +"mex/extractors/logging.py" = [ + "A005", # Allow custom logging module to shadow standard-library +] +"mex/extractors/types.py" = [ + "A005", # Allow custom types module to shadow standard-library +] +"mex/extractors/mapping/types.py" = [ + "A005", # Allow custom mapping types module to shadow standard-library +] "docs/**" = [ "INP001", # Docs folder does not need to be a package ] From 3d74e098110b0bcda103d3991f42d6c00a2eee1b Mon Sep 17 00:00:00 2001 From: Nicolas Drebenstedt Date: Tue, 21 Jan 2025 14:51:29 +0100 Subject: [PATCH 5/5] lint --- mex/extractors/datscha_web/parse_html.py | 3 +-- mex/extractors/ff_projects/filter.py | 2 +- .../international_projects/extract.py | 6 +++--- mex/extractors/sumo/transform.py | 2 +- tests/datscha_web/test_transform.py | 6 +++--- tests/grippeweb/mocked_grippeweb.py | 18 +++++++++--------- tests/international_projects/conftest.py | 3 +-- tests/odk/conftest.py | 12 ++++++------ tests/seq_repo/conftest.py | 12 ++++++------ tests/synopse/conftest.py | 12 ++++++------ tests/voxco/conftest.py | 12 ++++++------ 11 files changed, 43 insertions(+), 45 deletions(-) diff --git a/mex/extractors/datscha_web/parse_html.py b/mex/extractors/datscha_web/parse_html.py index 62f10d7d..a3ed29f8 100644 --- a/mex/extractors/datscha_web/parse_html.py +++ b/mex/extractors/datscha_web/parse_html.py @@ -99,8 +99,7 @@ def parse_unit_loz(bs4_object: BeautifulSoup) -> tuple[str, list[str]]: table = bs4_object.find("table", {"id": "loz"}) if not isinstance(table, Tag): raise MExError( - f"Missing table with ID 'loz' in single item html.\n" - f"{bs4_object.prettify()}" + f"Missing table with ID 'loz' in single item html.\n{bs4_object.prettify()}" ) column_headers = table.find_all("th") key = str(column_headers[0].string) diff --git a/mex/extractors/ff_projects/filter.py b/mex/extractors/ff_projects/filter.py index a5bf3660..55d46acd 100644 --- a/mex/extractors/ff_projects/filter.py +++ b/mex/extractors/ff_projects/filter.py @@ -124,7 +124,7 @@ def filter_and_log_ff_projects_source( log_filter( identifier_in_primary_source, primary_source_id, - f'RKI- OE [{source.rki_oe.replace("/", ",")}] are all not valid units', + f"RKI- OE [{source.rki_oe.replace('/', ',')}] are all not valid units", ) return False diff --git a/mex/extractors/international_projects/extract.py b/mex/extractors/international_projects/extract.py index 8df1f355..2184106d 100644 --- a/mex/extractors/international_projects/extract.py +++ b/mex/extractors/international_projects/extract.py @@ -25,9 +25,9 @@ @watch -def extract_international_projects_sources() -> ( - Generator[InternationalProjectsSource, None, None] -): +def extract_international_projects_sources() -> Generator[ + InternationalProjectsSource, None, None +]: """Extract international projects sources by loading data from MS-Excel file. Returns: diff --git a/mex/extractors/sumo/transform.py b/mex/extractors/sumo/transform.py index 6d2036a9..ff6d8bb2 100644 --- a/mex/extractors/sumo/transform.py +++ b/mex/extractors/sumo/transform.py @@ -372,7 +372,7 @@ def transform_nokeda_model_variable_to_mex_variable( value_sets = list(extracted_cc1_data_valuesets) for variable in extracted_cc1_data_model_nokeda: value_set = [ - f"{v.category_label_de}," f"{v.category_label_en or ''}" + f"{v.category_label_de},{v.category_label_en or ''}" for v in value_sets if v.sheet_name == variable.variable_name ] diff --git a/tests/datscha_web/test_transform.py b/tests/datscha_web/test_transform.py index 8f1742a3..57c475dd 100644 --- a/tests/datscha_web/test_transform.py +++ b/tests/datscha_web/test_transform.py @@ -32,9 +32,9 @@ def unit_stable_target_ids_by_synonym() -> dict[str, Identifier]: @pytest.fixture -def organizations_stable_target_ids_by_query_string() -> ( - dict[str, MergedOrganizationIdentifier] -): +def organizations_stable_target_ids_by_query_string() -> dict[ + str, MergedOrganizationIdentifier +]: return { "Fancy Fake Firm & CoKG": Identifier("ID000000000077"), "FG99": Identifier("ID000000000884"), diff --git a/tests/grippeweb/mocked_grippeweb.py b/tests/grippeweb/mocked_grippeweb.py index bfbd69f6..e11e3b7d 100644 --- a/tests/grippeweb/mocked_grippeweb.py +++ b/tests/grippeweb/mocked_grippeweb.py @@ -34,16 +34,16 @@ @pytest.fixture -def extracted_mex_functional_units_grippeweb() -> ( - dict[str, MergedContactPointIdentifier] -): +def extracted_mex_functional_units_grippeweb() -> dict[ + str, MergedContactPointIdentifier +]: return {"contactc@rki.de": MergedContactPointIdentifier.generate(42)} @pytest.fixture -def unit_stable_target_ids_by_synonym() -> ( - dict[str, MergedOrganizationalUnitIdentifier] -): +def unit_stable_target_ids_by_synonym() -> dict[ + str, MergedOrganizationalUnitIdentifier +]: """Mock unit stable target ids.""" return {"C1": MergedOrganizationalUnitIdentifier.generate(seed=44)} @@ -64,9 +64,9 @@ def extracted_mex_persons_grippeweb() -> list[ExtractedPerson]: @pytest.fixture -def grippeweb_organization_ids_by_query_string() -> ( - dict[str, MergedOrganizationIdentifier] -): +def grippeweb_organization_ids_by_query_string() -> dict[ + str, MergedOrganizationIdentifier +]: return {"Robert Koch-Institut": MergedOrganizationIdentifier.generate(42)} diff --git a/tests/international_projects/conftest.py b/tests/international_projects/conftest.py index 9620d37d..f427528d 100644 --- a/tests/international_projects/conftest.py +++ b/tests/international_projects/conftest.py @@ -90,8 +90,7 @@ def international_projects_mapping_activity() -> AnyMappingModel: "forValues": [ "Crisis management", "Capacity building including trainings", - "Supporting global governance structures and " - "processes", + "Supporting global governance structures and processes", "Conducting research", ], "rule": "If field is either empty or has another value, " diff --git a/tests/odk/conftest.py b/tests/odk/conftest.py index d53decd4..db3d17f6 100644 --- a/tests/odk/conftest.py +++ b/tests/odk/conftest.py @@ -86,9 +86,9 @@ def extracted_international_projects_activities() -> list[ExtractedActivity]: @pytest.fixture -def unit_stable_target_ids_by_synonym() -> ( - dict[str, MergedOrganizationalUnitIdentifier] -): +def unit_stable_target_ids_by_synonym() -> dict[ + str, MergedOrganizationalUnitIdentifier +]: """Mock unit stable target ids.""" return {"C1": MergedOrganizationalUnitIdentifier.generate(seed=44)} @@ -566,9 +566,9 @@ def odk_resource_mappings() -> list[AnyMappingModel]: @pytest.fixture -def external_partner_and_publisher_by_label() -> ( - dict[str, MergedOrganizationIdentifier] -): +def external_partner_and_publisher_by_label() -> dict[ + str, MergedOrganizationIdentifier +]: """Mocked external partner and publisher dict for OrganizationIDs.""" return { "invidunt": MergedOrganizationIdentifier.generate(42), diff --git a/tests/seq_repo/conftest.py b/tests/seq_repo/conftest.py index d24281f6..8e1c89b9 100644 --- a/tests/seq_repo/conftest.py +++ b/tests/seq_repo/conftest.py @@ -463,9 +463,9 @@ def seq_repo_source_resolved_project_coordinators() -> list[LDAPPersonWithQuery] @pytest.fixture -def project_coordinators_merged_ids_by_query_string() -> ( - dict[str, list[MergedPersonIdentifier]] -): +def project_coordinators_merged_ids_by_query_string() -> dict[ + str, list[MergedPersonIdentifier] +]: """Get project coordinators merged ids.""" return { "mustermann": [MergedPersonIdentifier("e0Rxxm9WvnMqPLZ44UduNx")], @@ -476,9 +476,9 @@ def project_coordinators_merged_ids_by_query_string() -> ( @pytest.fixture -def unit_stable_target_ids_by_synonym() -> ( - dict[str, MergedOrganizationalUnitIdentifier] -): +def unit_stable_target_ids_by_synonym() -> dict[ + str, MergedOrganizationalUnitIdentifier +]: """Extract the dummy units and return them grouped by synonyms.""" return { "child-unit": MergedOrganizationalUnitIdentifier("g2AinFG4E6n8H1ZMuaBW6o"), diff --git a/tests/synopse/conftest.py b/tests/synopse/conftest.py index fcf94319..9afaf16e 100644 --- a/tests/synopse/conftest.py +++ b/tests/synopse/conftest.py @@ -208,9 +208,9 @@ def synopse_variables_by_thema( @pytest.fixture -def synopse_variables_extended_data_use_raw() -> ( - list[dict[str, str | int | float | None]] -): +def synopse_variables_extended_data_use_raw() -> list[ + dict[str, str | int | float | None] +]: """Return a list of dicts in the required format for Synopse Variables.""" return [ { # variable not in synopse_overviews @@ -363,9 +363,9 @@ def synopse_study(synopse_studies: list[SynopseStudy]) -> SynopseStudy: @pytest.fixture -def synopse_organization_ids_by_query_string() -> ( - dict[str, MergedOrganizationIdentifier] -): +def synopse_organization_ids_by_query_string() -> dict[ + str, MergedOrganizationIdentifier +]: """Return merged organizations id by org name.""" organization_id = MergedOrganizationIdentifier.generate(seed=44) return {"Test-Institute": organization_id} diff --git a/tests/voxco/conftest.py b/tests/voxco/conftest.py index 3cab9b17..4014b6a3 100644 --- a/tests/voxco/conftest.py +++ b/tests/voxco/conftest.py @@ -24,9 +24,9 @@ def extracted_mex_functional_units_voxco() -> dict[str, MergedContactPointIdenti @pytest.fixture -def unit_stable_target_ids_by_synonym() -> ( - dict[str, MergedOrganizationalUnitIdentifier] -): +def unit_stable_target_ids_by_synonym() -> dict[ + str, MergedOrganizationalUnitIdentifier +]: """Mock unit stable target ids.""" return {"C1": MergedOrganizationalUnitIdentifier.generate(seed=44)} @@ -47,9 +47,9 @@ def extracted_mex_persons_voxco() -> list[ExtractedPerson]: @pytest.fixture -def organization_stable_target_id_by_query_voxco() -> ( - dict[str, MergedOrganizationIdentifier] -): +def organization_stable_target_id_by_query_voxco() -> dict[ + str, MergedOrganizationIdentifier +]: return {"Robert Koch-Institut": MergedOrganizationIdentifier.generate(42)}