Test both SQLAlchemy 1 and 2

meltano · Jun 2, 2023 · 4bcc9ef · 4bcc9ef
1 parent 5473c30
commit 4bcc9ef
Show file tree

Hide file tree

Showing 3 changed files with 39 additions and 29 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -20,26 +20,28 @@ jobs:
     name: Test on ${{ matrix.python-version }} (${{ matrix.session }}) / ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     env:
-      NOXSESSION: ${{ matrix.session }}
+      NOXSESSION: ${{ matrix.session }}-${{ matrix.python-version }}
     strategy:
       fail-fast: false
       matrix:
         include:
-        - { session: tests,   python-version: "3.7",  os: "ubuntu-latest" }
-        - { session: tests,   python-version: "3.8",  os: "ubuntu-latest" }
-        - { session: tests,   python-version: "3.9",  os: "ubuntu-latest" }
-        - { session: tests,   python-version: "3.10", os: "ubuntu-latest" }
-        - { session: tests,   python-version: "3.11", os: "ubuntu-latest" }
-        - { session: tests,   python-version: "3.7",  os: "macos-latest" }
-        - { session: tests,   python-version: "3.8",  os: "macos-latest" }
-        - { session: tests,   python-version: "3.9",  os: "macos-latest" }
-        - { session: tests,   python-version: "3.10", os: "macos-latest" }
-        - { session: tests,   python-version: "3.11", os: "macos-latest" }
-        - { session: tests,   python-version: "3.7",  os: "windows-latest" }
-        - { session: tests,   python-version: "3.8",  os: "windows-latest" }
-        - { session: tests,   python-version: "3.9",  os: "windows-latest" }
-        - { session: tests,   python-version: "3.10", os: "windows-latest" }
-        - { session: tests,   python-version: "3.11", os: "windows-latest" }
+        - { session: tests,   python-version: "3.7",  os: "ubuntu-latest",  keywords: sqlalchemy2 }
+        - { session: tests,   python-version: "3.8",  os: "ubuntu-latest",  keywords: sqlalchemy2 }
+        - { session: tests,   python-version: "3.9",  os: "ubuntu-latest",  keywords: sqlalchemy2 }
+        - { session: tests,   python-version: "3.10", os: "ubuntu-latest",  keywords: sqlalchemy2 }
+        - { session: tests,   python-version: "3.11", os: "ubuntu-latest",  keywords: sqlalchemy2 }
+        - { session: tests,   python-version: "3.7",  os: "macos-latest",   keywords: sqlalchemy2 }
+        - { session: tests,   python-version: "3.8",  os: "macos-latest",   keywords: sqlalchemy2 }
+        - { session: tests,   python-version: "3.9",  os: "macos-latest",   keywords: sqlalchemy2 }
+        - { session: tests,   python-version: "3.10", os: "macos-latest",   keywords: sqlalchemy2 }
+        - { session: tests,   python-version: "3.11", os: "macos-latest",   keywords: sqlalchemy2 }
+        - { session: tests,   python-version: "3.7",  os: "windows-latest", keywords: sqlalchemy2 }
+        - { session: tests,   python-version: "3.8",  os: "windows-latest", keywords: sqlalchemy2 }
+        - { session: tests,   python-version: "3.9",  os: "windows-latest", keywords: sqlalchemy2 }
+        - { session: tests,   python-version: "3.10", os: "windows-latest", keywords: sqlalchemy2 }
+        - { session: tests,   python-version: "3.11", os: "windows-latest", keywords: sqlalchemy2 }
+        - { session: tests,   python-version: "3.10", os: "ubuntu-latest",  keywords: sqlalchemy1 }
+        - { session: tests,   python-version: "3.11", os: "ubuntu-latest",  keywords: sqlalchemy1 }
         - { session: doctest, python-version: "3.10", os: "ubuntu-latest" }
         - { session: mypy,    python-version: "3.8",  os: "ubuntu-latest" }
 
@@ -79,7 +81,7 @@ jobs:
 
     - name: Run Nox
       run: |
-        nox --python=${{ matrix.python-version }}
+        nox
 
     - name: Upload coverage data
       if: always() && (matrix.session == 'tests')
@@ -93,6 +95,7 @@ jobs:
     runs-on: ubuntu-latest
     if: ${{ !github.event.pull_request.head.repo.fork }}
     env:
+      NOXSESSION: tests-${{ matrix.python-version }}
       SAMPLE_TAP_GITLAB_AUTH_TOKEN: ${{ secrets.SAMPLE_TAP_GITLAB_AUTH_TOKEN }}
       SAMPLE_TAP_GITLAB_GROUP_IDS: ${{ secrets.SAMPLE_TAP_GITLAB_GROUP_IDS }}
       SAMPLE_TAP_GITLAB_PROJECT_IDS: ${{ secrets.SAMPLE_TAP_GITLAB_PROJECT_IDS }}
@@ -137,12 +140,14 @@ jobs:
 
     - name: Run Nox
       run: |
-        nox -s tests -p 3.10 -- -m "external"
+        nox -- -m "external"
 
   coverage:
     name: Coverage
     runs-on: ubuntu-latest
     needs: tests
+    env:
+      NOXSESSION: coverage
     steps:
     - name: Check out the repository
       uses: actions/[email protected]
@@ -179,11 +184,11 @@ jobs:
 
     - name: Combine coverage data and display human readable report
       run: |
-        nox --session=coverage
+        nox
 
     - name: Create coverage report
       run: |
-        nox --session=coverage -- xml
+        nox -- xml
 
     - name: Upload coverage report
       uses: codecov/[email protected]
diff --git a/noxfile.py b/noxfile.py
@@ -71,11 +71,19 @@ def mypy(session: Session) -> None:
 
 
 @session(python=python_versions)
-def tests(session: Session) -> None:
+@nox.parametrize(
+    "sqlalchemy",
+    [">=2.0", ">=1.0,<2.0"],
+    ids=["sqlalchemy2", "sqlalchemy1"],
+)
+def tests(session: Session, sqlalchemy: str) -> None:
     """Execute pytest tests and compute coverage."""
     session.install(".[s3]")
     session.install(*test_dependencies)
 
+    # Install SQLAlchemy bypassing constraints
+    session.poetry.session.install(f"sqlalchemy{sqlalchemy}")
+
     try:
         session.run(
             "coverage",

diff --git a/singer_sdk/sinks/sql.py b/singer_sdk/sinks/sql.py
@@ -6,7 +6,7 @@
 from collections import defaultdict
 from copy import copy
 from textwrap import dedent
-from typing import Any, Iterable
+from typing import Any, Sequence
 
 import sqlalchemy
 from pendulum import now
@@ -212,7 +212,7 @@ def conform_schema(self, schema: dict) -> dict:
         }
         return conformed_schema
 
-    def conform_record(self, record: dict) -> dict:
+    def conform_record(self, record: dict[str, Any]) -> dict[str, Any]:
         """Return record dictionary with property names conformed.
 
         Args:
@@ -294,7 +294,7 @@ def bulk_insert_records(
         self,
         full_table_name: str,
         schema: dict,
-        records: Iterable[dict[str, Any]],
+        records: Sequence[dict[str, Any]],
     ) -> int | None:
         """Bulk insert records to an existing destination table.
 
@@ -318,11 +318,8 @@ def bulk_insert_records(
         if isinstance(insert_sql, str):
             insert_sql = sqlalchemy.text(insert_sql)
 
-        conformed_records = (
-            [self.conform_record(record) for record in records]
-            if isinstance(records, list)
-            else (self.conform_record(record) for record in records)
-        )
+        conformed_records = [self.conform_record(record) for record in records]
+
         self.logger.info("Inserting with SQL: %s", insert_sql)
         with self.connector._connect() as conn, conn.begin():
             conn.execute(insert_sql, conformed_records)