feature/mx-1381 prepare database model rework

.cruft.json

@@ -1,6 +1,6 @@
 {
   "checkout": null,
-  "commit": "6067fc53d1335a9bda900c5eff8dbf1c42bfe4ca",
+  "commit": "77ce51f46b0897c6fbd29782c32923075e296c2a",
   "context": {
     "cookiecutter": {
       "project_name": "common",

.github/dependabot.yml

@@ -3,14 +3,22 @@ updates:
   - package-ecosystem: "github-actions"
     allow:
       - dependency-type: "all"
+    assignees:
+      - ${{secrets.MEX_BOT_USER}}
     directory: "/"
+    labels:
+      - "dependabot"
     open-pull-requests-limit: 1
     schedule:
-      interval: "monthly"
+      interval: "weekly"
   - package-ecosystem: "pip"
     allow:
       - dependency-type: "all"
+    assignees:
+      - ${{secrets.MEX_BOT_USER}}
     directory: "/"
+    labels:
+      - "dependabot"
     open-pull-requests-limit: 1
     schedule:
       interval: "daily"

.github/workflows/cookiecutter.yml

@@ -2,7 +2,8 @@ name: Cookiecutter
 
 on:
   push:
-  pull_request:
+    branches: ["main"]
+    tags: ["**"]
   schedule:
     - cron: '0 0 * * *'
   workflow_dispatch:
@@ -18,12 +19,13 @@ jobs:
     timeout-minutes: 10
     steps:
       - name: Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
-          fetch-depth: 1
+          fetch-depth: 0
+          token: ${{ secrets.WORKFLOW_TOKEN }}
 
       - name: Cache requirements
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         env:
           cache-name: cache-requirements
         with:
@@ -33,12 +35,57 @@ jobs:
             ${{ env.cache-name }}-
 
       - name: Setup python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.11
 
       - name: Install requirements
         run: make setup
 
-      - name: Check template
-        run: cruft check
+      - name: Configure git
+        env:
+          SIGNING_KEY: ${{ secrets.SIGNING_KEY }}
+          SIGNING_PUB: ${{ secrets.SIGNING_PUB }}
+        run: |
+          eval "$(ssh-agent -s)"
+          install --directory ~/.ssh --mode 700
+          base64 -d <<< '${{ secrets.SIGNING_KEY }}' > ~/.ssh/mex
+          base64 -d <<< '${{ secrets.SIGNING_PUB }}' > ~/.ssh/mex.pub
+          chmod 600 ~/.ssh/*
+          ssh-add ~/.ssh/mex
+          git config --local user.email ${{ vars.MEX_BOT_EMAIL }}
+          git config --local user.name ${{ vars.MEX_BOT_USER }}
+          git config --local gpg.format ssh
+          git config --local user.signingkey ~/.ssh/mex.pub
+          git config --local commit.gpgsign true
+
+      - name: Update template
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          if cruft check; then
+            echo template is up to date
+            exit 0
+          fi
+          if [[ $(gh pr list --label cruft | wc -c) -ne 0 ]]; then
+            echo already seeing pull request
+            exit 0
+          fi
+          template_url=$(python -c "print(__import__('json').load(open('.cruft.json'))['template'])")
+          template_ref=$(git ls-remote ${template_url} --heads main --exit-code | cut -c -6)
+          git checkout main
+          git checkout -b cruft/cookiecutter-template-${template_ref}
+          cruft update --skip-apply-ask
+          printf '# Changes\n\n- bumped cookiecutter template to %s/commit/%s\n' "$template_url" "$template_ref" > .cruft-pr-body
+          if [[ $(git status --porcelain | grep .rej | wc -c) -ne 0 ]]; then
+            printf '\n# Conflicts\n' > .cruft-pr-body
+          fi
+          git status --porcelain | grep .rej | awk '{print $2;}' | while read -r line ; do
+            printf '\n```' >> .cruft-pr-body
+            cat ${line} >> .cruft-pr-body
+            printf '```\n' >> .cruft-pr-body
+          done
+          git add --all --verbose
+          git commit --message "Bump cookiecutter template to $template_ref" --verbose
+          git push --set-upstream origin cruft/cookiecutter-template-${template_ref} --force --verbose
+          gh pr create --title "Bump cookiecutter template to $template_ref" --body-file .cruft-pr-body --label cruft --assignee ${{ vars.MEX_BOT_USER }}

.github/workflows/cve-scan.yml

@@ -8,6 +8,10 @@ on:
     branches-ignore:
       - 'dependabot/**'
   pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
   schedule:
     - cron: '0 0 * * *'
   workflow_dispatch:
@@ -18,12 +22,12 @@ jobs:
     timeout-minutes: 10
     steps:
       - name: Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 1
 
       - name: Cache trivy
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         env:
           cache-name: cache-trivy
         with:

.github/workflows/documentation.yml

@@ -25,12 +25,12 @@ jobs:
     timeout-minutes: 10
     steps:
       - name: Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 1
 
       - name: Cache requirements
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         env:
           cache-name: cache-requirements
         with:
@@ -40,7 +40,7 @@ jobs:
             ${{ env.cache-name }}-
 
       - name: Cache poetry
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         env:
           cache-name: cache-poetry
         with:
@@ -50,12 +50,12 @@ jobs:
             ${{ env.cache-name }}-
 
       - name: Setup python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.11
 
       - name: Setup pages
-        uses: actions/configure-pages@v3
+        uses: actions/configure-pages@v4
 
       - name: Install requirements
         run: make install
@@ -64,7 +64,7 @@ jobs:
         run: make docs
 
       - name: Upload artifact
-        uses: actions/upload-pages-artifact@v2
+        uses: actions/upload-pages-artifact@v3
         with:
           path: ./docs/dist
 
@@ -77,4 +77,4 @@ jobs:
     steps:
       - name: Deploy to GitHub Pages
         id: deployment
-        uses: actions/deploy-pages@v2
+        uses: actions/deploy-pages@v4

.github/workflows/linting.yml

@@ -3,6 +3,10 @@ name: Linting
 on:
   push:
   pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
   workflow_dispatch:
 
 env:
@@ -16,12 +20,12 @@ jobs:
     timeout-minutes: 10
     steps:
       - name: Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 1
 
       - name: Cache requirements
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         env:
           cache-name: cache-requirements
         with:
@@ -31,7 +35,7 @@ jobs:
             ${{ env.cache-name }}-
 
       - name: Cache poetry
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         env:
           cache-name: cache-poetry
         with:
@@ -41,7 +45,7 @@ jobs:
             ${{ env.cache-name }}-
 
       - name: Setup python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.11
 

.github/workflows/open-code.yml

@@ -12,7 +12,7 @@ jobs:
     timeout-minutes: 10
     steps:
       - name: Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: 'main'
           fetch-depth: 0

.github/workflows/reviewing.yml

@@ -0,0 +1,26 @@
+name: Review
+
+on:
+  pull_request:
+    types:
+      - opened
+  workflow_dispatch:
+
+jobs:
+  assignee:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    if: >-
+      github.event.pull_request.user.login != ${{ vars.MEX_BOT_USER }} &&
+      github.event.pull_request.user.login != 'dependabot[bot]'
+    steps:
+      - name: Add assignee
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          if [[ -z "${{ github.event.pull_request.assignee.login }}" ]]; then
+            echo assigning to ${{ github.event.pull_request.user.login }}
+            gh pr edit ${{ github.event.pull_request.html_url }} --add-assignee ${{ github.event.pull_request.user.login }}
+          else
+            echo already assigned
+          fi

.github/workflows/testing.yml

@@ -3,6 +3,10 @@ name: Testing
 on:
   push:
   pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
   workflow_dispatch:
 
 env:
@@ -16,12 +20,12 @@ jobs:
     timeout-minutes: 10
     steps:
       - name: Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 1
 
       - name: Cache requirements
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         env:
           cache-name: cache-requirements
         with:
@@ -31,7 +35,7 @@ jobs:
             ${{ env.cache-name }}-
 
       - name: Cache poetry
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         env:
           cache-name: cache-poetry
         with:
@@ -41,7 +45,7 @@ jobs:
             ${{ env.cache-name }}-
 
       - name: Setup python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.11
 

.gitignore

@@ -100,6 +100,10 @@ dmypy.json
 # ruff
 .ruff_cache
 
+# cruft
+.cruft-pr-body
+*.rej
+
 # PyCharm
 .idea/
 .idea_modules/

.pre-commit-config.yaml

@@ -3,7 +3,7 @@ default_language_version:
   python: python3.11
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.2.1
+    rev: v0.2.2
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
@@ -27,7 +27,7 @@ repos:
       - id: fix-byte-order-marker
         name: byte-order
   - repo: https://github.com/python-poetry/poetry
-    rev: 1.7.1
+    rev: 1.8.1
     hooks:
       - id: poetry-check
         name: poetry

CHANGELOG.md

@@ -19,6 +19,31 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Security
 
+## [0.21.0] - 2024-03-04
+
+### Added
+
+- add `entityType` type hint to `MExModel` (now `BaseEntity`)
+- add types for `AnyBaseModel`, `AnyExtractedModel` and `AnyMergedModel`
+- create more specific subclasses of `Identifier` (for extracted and merged)
+- expose unions, lists and lookups for `Identifier` subclasses in `mex.common.types`
+
+### Changes
+
+- swap `contextvars.ContextVar` for `mex.common.context.ContextStore`
+- move `stableTargetId` property from base models to extracted models
+- update typing of identifiers to specific subclasses
+- use `Annotated[..., Field(...)]` notation for pydantic field configs
+- split up `mex.common.models.base` and move out `MExModel` and `JsonSchemaGenerator`
+- rename `MExModel` to `BaseEntity` with only type hints an model config
+- declare `hadPrimarySource`, `identifier` and `identifierInPrimarySource` as frozen
+
+### Removed
+
+- absorb unused `BaseExtractedData` into `ExtractedData`
+- remove `stableTargetId` property from merged models
+- drop support for sinks to accept merged items (now only for extracted data)
+
 ## [0.20.0] - 2024-02-22
 
 ### Changes