(Dependencies) Upgrade SQLAlchemy to 1.4 and other requirements (flask 3.0), remove Debian 10 and Python 3.7 support (PnX-SI#2751)

* Drop support for Debian 10 and python <3.9 @jacquesfize @Pierre-Narcisi @bouttier 
* Update to Flask 3.0 
* Update to SQLAlchemy 1.4 (query to 2.0 style, fix warnings and tests)
* new requirements-dev.txt
* Increase test coverage
  - gn_meta/repositories 
  - gn_meta/mtd
  - occtax
  - occhab
  - utilstoml
  - install-gn-module commands
* (temporary) use of CustomSelect instead of Query --> Station (deleted later)
* Change fixtures: datasets + stations + user
* Remove deprecated and unused modules (utilsgeometry.py, utilssqlalchemy.py,config_manager.py)
---------

Co-authored-by: TheoLechemia <[email protected]>
Co-authored-by: Élie Bouttier <[email protected]>
Co-authored-by: Jacques Fize <[email protected]>
Co-authored-by: Pierre Narcisi <[email protected]>

Author: 4 people

.github/workflows/pytest.yml

@@ -19,20 +19,16 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        debian-version: [ '10', '11', '12' ]
+        debian-version: ["11", "12"]
         include:
-          - debian-version: '10'
-            python-version: '3.7'
-            postgres-version: '11'
-            postgis-version: '2.5'
-          - debian-version: '11'
-            python-version: '3.9'
-            postgres-version: '13'
-            postgis-version: '3.2'
-          - debian-version: '12'
-            python-version: '3.11'
-            postgres-version: '15'
-            postgis-version: '3.3'
+          - debian-version: "11"
+            python-version: "3.9"
+            postgres-version: "13"
+            postgis-version: "3.2"
+          - debian-version: "12"
+            python-version: "3.11"
+            postgres-version: "15"
+            postgis-version: "3.3"
 
     name: Debian ${{ matrix.debian-version }}
 

backend/dependencies/Habref-api-module

@@ -17,7 +17,7 @@
 from flask_mail import Message
 from flask_cors import CORS
 from flask_login import current_user
-from flask_sqlalchemy import before_models_committed
+from flask_sqlalchemy.track_modifications import before_models_committed
 from werkzeug.middleware.proxy_fix import ProxyFix
 from werkzeug.middleware.shared_data import SharedDataMiddleware
 from werkzeug.middleware.dispatcher import DispatcherMiddleware
@@ -86,7 +86,7 @@ class MyJSONProvider(DefaultJSONProvider):
     @staticmethod
     def default(o):
         if isinstance(o, Row):
-            return dict(o)
+            return o._asdict()
         return DefaultJSONProvider.default(o)
 
 

backend/dependencies/Nomenclature-api-module

@@ -1,24 +1,23 @@
+import importlib
 import os
-import sys
-import subprocess
 import site
-import importlib
+import subprocess
+import sys
 from pathlib import Path
 
+import pathlib  # For testing purposes
 import click
+import geonature.utils.config
 from click import ClickException
-
-from geonature.utils.env import ROOT_DIR
-from geonature.utils.module import iter_modules_dist, get_dist_from_code, module_db_upgrade
-
 from geonature.core.command.main import main
-import geonature.utils.config
-from geonature.utils.config import config
 from geonature.utils.command import (
-    install_frontend_dependencies,
-    create_frontend_module_config,
     build_frontend,
+    create_frontend_module_config,
+    install_frontend_dependencies,
 )
+from geonature.utils.config import config
+from geonature.utils.env import ROOT_DIR
+from geonature.utils.module import get_dist_from_code, iter_modules_dist, module_db_upgrade
 
 
 @main.command()
@@ -30,6 +29,29 @@
 @click.option("--build", type=bool, required=False, default=True)
 @click.option("--upgrade-db", type=bool, required=False, default=True)
 def install_gn_module(x_arg, module_path, module_code, build, upgrade_db):
+    """
+    Command definition to install a GeoNature module
+
+    Parameters
+    ----------
+    x_arg : list
+        additional arguments
+    module_path : str
+        path of the module directory
+    module_code : str
+        code of the module, deprecated in future release
+    build : boolean
+        is the frontend rebuild
+    upgrade_db : boolean
+        migrate the revision associated with the module
+
+    Raises
+    ------
+    ClickException
+        No module found with the given module code
+    ClickException
+        No module code was detected in the code
+    """
     click.echo("Installation du backend…")
     subprocess.run(f"pip install -e '{module_path}'", shell=True, check=True)
 
@@ -40,7 +62,7 @@ def install_gn_module(x_arg, module_path, module_code, build, upgrade_db):
     if module_code:
         # load python package
         module_dist = get_dist_from_code(module_code)
-        if not module_dist:
+        if not module_dist:  # FIXME : technically can't go there...
             raise ClickException(f"Aucun module ayant pour code {module_code} n’a été trouvé")
     else:
         for module_dist in iter_modules_dist():
@@ -56,7 +78,6 @@ def install_gn_module(x_arg, module_path, module_code, build, upgrade_db):
             raise ClickException(
                 f"Impossible de détecter le code du module, essayez de le spécifier."
             )
-
     # symlink module in exernal module directory
     module_frontend_path = (module_path / "frontend").resolve()
     module_symlink = ROOT_DIR / "frontend" / "external_modules" / module_code.lower()
@@ -68,7 +89,6 @@ def install_gn_module(x_arg, module_path, module_code, build, upgrade_db):
     else:
         click.echo(f"Création du lien symbolique {module_symlink} → {module_frontend_path}")
         os.symlink(module_frontend_path, module_symlink)
-
     if (Path(module_path) / "frontend" / "package-lock.json").is_file():
         click.echo("Installation des dépendances frontend…")
         install_frontend_dependencies(module_frontend_path)
@@ -80,7 +100,6 @@ def install_gn_module(x_arg, module_path, module_code, build, upgrade_db):
         click.echo("Rebuild du frontend …")
         build_frontend()
         click.secho("Rebuild du frontend terminé.", fg="green")
-
     if upgrade_db:
         click.echo("Installation / mise à jour de la base de données…")
         if not module_db_upgrade(module_dist, x_arg=x_arg):

backend/dependencies/RefGeo

@@ -4,7 +4,7 @@
 
 from flask import current_app, request, json, redirect
 from werkzeug.exceptions import Unauthorized, InternalServerError, HTTPException, BadRequest
-from werkzeug.urls import url_encode
+from urllib.parse import urlencode
 from marshmallow.exceptions import ValidationError
 
 
@@ -32,7 +32,7 @@ def handle_unauthenticated_request(e):
             next_url = request.full_path
         else:
             next_url = request.url
-        query_string = url_encode({"next": next_url})
+        query_string = urlencode({"next": next_url})
         return redirect(f"{base_url}{login_path}?{query_string}")
 
 

backend/dependencies/TaxHub

@@ -79,16 +79,20 @@ class BibFieldAdmin(CruvedProtectedMixin, ModelView):
         "field_order": {"label": "Ordre"},
         "additional_attributes": {"label": "Attribut additionnels"},
         "modules": {
-            "query_factory": lambda: DB.session.query(TModules).filter(
-                TModules.module_code.in_(
-                    current_app.config["ADDITIONAL_FIELDS"]["IMPLEMENTED_MODULES"]
+            "query_factory": lambda: DB.session.scalars(
+                DB.select(TModules).where(
+                    TModules.module_code.in_(
+                        current_app.config["ADDITIONAL_FIELDS"]["IMPLEMENTED_MODULES"]
+                    )
                 )
             )
         },
         "objects": {
-            "query_factory": lambda: DB.session.query(PermObject).filter(
-                PermObject.code_object.in_(
-                    current_app.config["ADDITIONAL_FIELDS"]["IMPLEMENTED_OBJECTS"]
+            "query_factory": lambda: DB.session.scalars(
+                DB.select(PermObject).where(
+                    PermObject.code_object.in_(
+                        current_app.config["ADDITIONAL_FIELDS"]["IMPLEMENTED_OBJECTS"]
+                    )
                 )
             )
         },

backend/dependencies/UsersHub-authentification-module

@@ -2,22 +2,15 @@
     Route permettant de manipuler les fichiers
     contenus dans gn_media
 """
-import json
-
-from flask import Blueprint, request, current_app, redirect, jsonify
+from flask import request, redirect, jsonify
 from werkzeug.exceptions import NotFound
 
-from geonature.core.gn_commons.repositories import TMediaRepository, TMediumRepository
+from geonature.core.gn_commons.repositories import TMediaRepository
 from geonature.core.gn_commons.models import TMedias
 from geonature.utils.env import DB
 from utils_flask_sqla.response import json_resp, json_resp_accept_empty_list
 
 
-from geonature.utils.errors import (
-    GeoNatureError,
-    GeonatureApiError,
-)
-
 from ..routes import routes
 
 
@@ -29,8 +22,9 @@ def get_medias(uuid_attached_row):
     .. :quickref: Commons;
     """
 
-    res = DB.session.query(TMedias).filter(TMedias.uuid_attached_row == uuid_attached_row).all()
-
+    res = DB.session.scalars(
+        DB.select(TMedias).filter(TMedias.uuid_attached_row == uuid_attached_row)
+    ).all()
     return [r.as_dict() for r in (res or [])]
 
 
@@ -41,10 +35,10 @@ def get_media(id_media):
     .. :quickref: Commons;
     """
 
-    m = TMediaRepository(id_media=id_media).media
-    if not m:
+    media = TMediaRepository(id_media=id_media).media
+    if not media:
         raise NotFound
-    return jsonify(m.as_dict())
+    return jsonify(media.as_dict())
 
 
 @routes.route("/media", methods=["POST", "PUT"])
@@ -59,14 +53,14 @@ def insert_or_update_media(id_media=None):
     """
 
     # gestion des parametres de route
-
+    # @TODO utilisé quelque part ?
     if request.files:
         file = request.files["file"]
     else:
         file = None
 
     data = {}
-    # Useful ?
+    # Useful ? @jacquesfize YES ! -> used when add media when adding a taxon occurrence
     if request.form:
         formData = dict(request.form)
         for key in formData:

backend/dependencies/Utils-Flask-SQLAlchemy

@@ -37,10 +37,7 @@ class TAdditionalFields(DB.Model):
     exportable = DB.Column(DB.Boolean, default=True)
     field_order = DB.Column(DB.Integer)
     type_widget = DB.relationship("BibWidgets")
-    bib_nomenclature_type = DB.relationship(
-        "BibNomenclaturesTypes",
-        primaryjoin="BibNomenclaturesTypes.mnemonique == TAdditionalFields.code_nomenclature_type",
-    )
+    bib_nomenclature_type = DB.relationship("BibNomenclaturesTypes")
     additional_attributes = DB.Column(JSONB)
     multiselect = DB.Column(DB.Boolean)
     api = DB.Column(DB.String)
@@ -50,7 +47,9 @@ class TAdditionalFields(DB.Model):
         secondary=cor_field_module,
     )
     objects = DB.relationship(PermObject, secondary=cor_field_object)
-    datasets = DB.relationship(TDatasets, secondary=cor_field_dataset)
+    datasets = DB.relationship(
+        TDatasets, secondary=cor_field_dataset, back_populates="additional_fields"
+    )
 
     def __str__(self):
         return f"{self.field_label} ({self.description})"

backend/dependencies/Utils-Flask-SQLAlchemy-Geo

@@ -121,7 +121,7 @@ class TMedias(DB.Model):
     id_table_location = DB.Column(
         DB.Integer, ForeignKey("gn_commons.bib_tables_location.id_table_location")
     )
-    unique_id_media = DB.Column(UUID(as_uuid=True), default=select([func.uuid_generate_v4()]))
+    unique_id_media = DB.Column(UUID(as_uuid=True), default=select(func.uuid_generate_v4()))
     uuid_attached_row = DB.Column(UUID(as_uuid=True))
     title_fr = DB.Column(DB.Unicode)
     title_en = DB.Column(DB.Unicode)
@@ -206,19 +206,24 @@ class TValidations(DB.Model):
     nomenclature_valid_status = relationship(
         TNomenclatures,
         foreign_keys=[id_nomenclature_valid_status],
-        lazy="joined",
+        lazy="joined",  # FIXME: remove and manually join when needed
     )
     id_validator = DB.Column(DB.Integer, ForeignKey(User.id_role))
     validator_role = DB.relationship(User)
     validation_auto = DB.Column(DB.Boolean)
     validation_comment = DB.Column(DB.Unicode)
     validation_date = DB.Column(DB.TIMESTAMP)
     validation_auto = DB.Column(DB.Boolean)
-    validation_label = DB.relationship(TNomenclatures)
+    # FIXME: remove and use nomenclature_valid_status
+    validation_label = DB.relationship(
+        TNomenclatures,
+        foreign_keys=[id_nomenclature_valid_status],
+        overlaps="nomenclature_valid_status",  # overlaps expected
+    )
 
 
 last_validation_query = (
-    select([TValidations])
+    select(TValidations)
     .order_by(TValidations.validation_date.desc())
     .limit(1)
     .alias("last_validation")