Merge branch 'main' into 333-consider-making-logging-more-granular

.flake8

@@ -1,6 +1,7 @@
 [flake8]
 max-line-length = 88
 exclude = cookiecutter
+ignore = E, W
 per-file-ignores =
     # Don't require docstrings conventions in private modules
     singer_sdk/helpers/_*.py:DAR

.github/ISSUE_TEMPLATE/bug.yml

@@ -15,7 +15,7 @@ body:
     attributes:
       label: Singer SDK Version
       description: Version of the library you are using
-      placeholder: "0.32.0"
+      placeholder: "0.33.1"
     validations:
       required: true
   - type: checkboxes

.github/workflows/codeql-analysis.yml

@@ -9,7 +9,13 @@ name: "CodeQL"
 on:
   push:
     branches: [ "main" ]
-  pull_request: {}
+    paths:
+      - .github/workflows/codeql-analysis.yml
+      - '**.py'  # Any Python file
+  pull_request:
+    paths:
+      - .github/workflows/codeql-analysis.yml
+      - '**.py'  # Any Python file
   schedule:
     - cron: '37 10 * * 5'
 
@@ -31,7 +37,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/[email protected].0
+      uses: actions/[email protected].1
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

.github/workflows/codspeed.yml

@@ -0,0 +1,44 @@
+name: codspeed
+
+on:
+  push:
+    branches:
+      - "main"
+  pull_request:
+  # `workflow_dispatch` allows CodSpeed to trigger backtest
+  # performance analysis in order to generate initial data.
+  workflow_dispatch:
+
+jobs:
+  benchmarks:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out the repository
+      uses: actions/[email protected]
+
+    - name: Setup Python 3.11
+      uses: actions/[email protected]
+      with:
+        python-version: 3.11
+        architecture: x64
+
+    - name: Install poetry
+      run: |
+        curl -fsS https://install.python-poetry.org | python - -y
+
+    - name: Configure poetry
+      run: poetry config virtualenvs.create false
+
+    - name: Install project
+      run: >
+        poetry install
+        -vvv
+        --with dev
+        --with benchmark
+        --all-extras
+
+    - name: Run benchmarks
+      uses: CodSpeedHQ/action@v1
+      with:
+        token: ${{ secrets.CODSPEED_TOKEN }}
+        run: pytest tests/ --codspeed

.github/workflows/constraints.txt

@@ -1,5 +1,5 @@
-pip==23.2.1
-poetry==1.6.1
-pre-commit==3.4.0
+pip==23.3.1
+poetry==1.7.0
+pre-commit==3.5.0
 nox==2023.4.22
 nox-poetry==1.0.3

.github/workflows/cookiecutter-e2e.yml

@@ -28,7 +28,7 @@ jobs:
 
     steps:
     - name: Check out the repository
-      uses: actions/[email protected].0
+      uses: actions/[email protected].1
 
     - name: Upgrade pip
       env:
@@ -43,7 +43,7 @@ jobs:
         poetry --version
 
     - name: Setup Python ${{ matrix.python-version }}
-      uses: actions/[email protected].0
+      uses: actions/[email protected].1
       with:
         python-version: ${{ matrix.python-version }}
         architecture: x64

.github/workflows/dependency-review.yml

@@ -5,9 +5,6 @@ on:
   workflow_dispatch:
     inputs: {}
 
-env:
-  FOSSA_CLI_INSTALLER_VERSION: '3.3.10'
-
 permissions:
   contents: read
 
@@ -16,26 +13,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout the repository
-        uses: actions/[email protected].0
+        uses: actions/[email protected].1
 
       - name: GitHub dependency vulnerability check
         if: ${{ github.event_name == 'pull_request_target' }}
-        uses: actions/[email protected].0
+        uses: actions/[email protected].2
         with:
           fail-on-severity: high
-
-      - name: FOSSA dependency license check
-        run: |
-          # `$FOSSA_CLI_INSTALLER_VERSION` only controls the version of the installer used - the latest version of `fossa-cli` will always be used.
-          curl --no-progress-meter -H 'Cache-Control: no-cache' https://raw.githubusercontent.com/fossas/fossa-cli/v${FOSSA_CLI_INSTALLER_VERSION}/install-latest.sh | bash
-
-          echo '## FOSSA dependency license check' >> $GITHUB_STEP_SUMMARY
-          echo '' >> $GITHUB_STEP_SUMMARY
-
-          fossa analyze --fossa-api-key ${{ secrets.MELTYBOT_FOSSA_API_KEY }} --revision ${{ github.sha }} |& tee fossa_analyze.log
-          fossa test --fossa-api-key ${{ secrets.MELTYBOT_FOSSA_API_KEY }} --revision ${{ github.sha }}
-
-          TEST_FAILED=$?
-          FOSSA_REPORT_LINK="$(grep -A 1 '[ INFO] View FOSSA Report:' fossa_analyze.log | tail -n 1 | sed -e 's/^\[ INFO\]\s*//')"
-          echo "[FOSSA detected $([ $TEST_FAILED -ne 0 ] && echo -n '' || echo 'no ')issues](${FOSSA_REPORT_LINK})" >> $GITHUB_STEP_SUMMARY
-          exit $TEST_FAILED

.github/workflows/release.yml

@@ -17,10 +17,10 @@ jobs:
 
     steps:
     - name: Checkout code
-      uses: actions/[email protected].0
+      uses: actions/[email protected].1
 
     - name: Set up Python
-      uses: actions/[email protected].0
+      uses: actions/[email protected].1
       with:
         python-version: "3.10"
 

.github/workflows/test.yml

@@ -55,7 +55,7 @@ jobs:
 
     steps:
     - name: Check out the repository
-      uses: actions/[email protected].0
+      uses: actions/[email protected].1
 
     - name: Install Poetry
       env:
@@ -65,7 +65,7 @@ jobs:
         poetry --version
 
     - name: Setup Python ${{ matrix.python-version }}
-      uses: actions/[email protected].0
+      uses: actions/[email protected].1
       with:
         python-version: ${{ matrix.python-version }}
         architecture: x64
@@ -115,7 +115,7 @@ jobs:
 
     steps:
     - name: Check out the repository
-      uses: actions/[email protected].0
+      uses: actions/[email protected].1
 
     - name: Install Poetry
       env:
@@ -125,7 +125,7 @@ jobs:
         poetry --version
 
     - name: Setup Python 3.10
-      uses: actions/[email protected].0
+      uses: actions/[email protected].1
       with:
         python-version: '3.10'
         architecture: x64
@@ -157,15 +157,15 @@ jobs:
     needs: tests
     steps:
     - name: Check out the repository
-      uses: actions/[email protected].0
+      uses: actions/[email protected].1
 
     - name: Install Poetry
       run: |
         pipx install --pip-args=--constraint=.github/workflows/constraints.txt poetry
         poetry --version
 
     - name: Set up Python
-      uses: actions/[email protected].0
+      uses: actions/[email protected].1
       with:
         python-version: '3.10'
         cache: 'pip'

.github/workflows/version_bump.yml

@@ -35,12 +35,12 @@ jobs:
       pull-requests: write  # to create and update PRs
 
     steps:
-    - uses: actions/[email protected].0
+    - uses: actions/[email protected].1
       with:
         fetch-depth: 0
 
     - name: Set up Python
-      uses: actions/[email protected].0
+      uses: actions/[email protected].1
       with:
         python-version: "3.10"
         architecture: x64

.pre-commit-config.yaml

@@ -5,7 +5,7 @@ ci:
 
 repos:
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.4.0
+  rev: v4.5.0
   hooks:
   - id: check-json
   - id: check-toml
@@ -36,32 +36,26 @@ repos:
         )$
 
 - repo: https://github.com/python-jsonschema/check-jsonschema
-  rev: 0.26.3
+  rev: 0.27.1
   hooks:
   - id: check-dependabot
   - id: check-github-workflows
   - id: check-readthedocs
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.0.291
+  rev: v0.1.5
   hooks:
   - id: ruff
     args: [--fix, --exit-non-zero-on-fix, --show-fixes]
     exclude: |
       (?x)^(
         cookiecutter/.*
       )$
-
-- repo: https://github.com/psf/black
-  rev: 23.9.1
-  hooks:
-  - id: black
+  - id: ruff-format
     exclude: |
-        (?x)^(
-            cookiecutter/.*|
-            singer_sdk/helpers/_simpleeval.py|
-            tests/core/test_simpleeval.py
-        )$
+      (?x)^(
+        cookiecutter/.*
+      )$
 
 - repo: https://github.com/pycqa/flake8
   rev: 6.1.0
@@ -76,6 +70,6 @@ repos:
         )$
 
 - repo: https://github.com/python-poetry/poetry
-  rev: 1.6.0
+  rev: 1.7.0
   hooks:
   - id: poetry-check

CHANGELOG.md

@@ -5,22 +5,34 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## v0.32.0 (2023-09-22)
-
-## v0.32.0b3 (2023-09-22)
+## v0.33.1 (2023-11-08)
 
 ### 🐛 Fixes
 
-- [#1977](https://github.com/meltano/sdk/issues/1977) Fix hanging downstream tests in tap-postgres
-- [#1970](https://github.com/meltano/sdk/issues/1970) Warn instead of crashing when schema helpers cannot append `null` to types
+- [#2035](https://github.com/meltano/sdk/issues/2035) Retry all 5xx status codes -- _**Thanks @asamasoma!**_
 
-## v0.32.0b2 (2023-09-15)
+## v0.33.0 (2023-10-12)
 
-### ⚡ Performance Improvements
+### ✨ New
 
-- [#1962](https://github.com/meltano/sdk/issues/1962) Ensure `raw_schema` in stream mapper is immutable
+- [#1999](https://github.com/meltano/sdk/issues/1999) Log JSONPath match count at the INFO level
+- [#1779](https://github.com/meltano/sdk/issues/1779) Cache SQL columns and schemas
+- [#2003](https://github.com/meltano/sdk/issues/2003) Add ability to do list comprehensions in stream map expressions -- _**Thanks @haleemur!**_
+- [#2018](https://github.com/meltano/sdk/issues/2018) Drop Python 3.7 support in cookiecutter templates -- _**Thanks @visch!**_
+
+### 🐛 Fixes
+
+- [#2006](https://github.com/meltano/sdk/issues/2006) Parse record `time_extracted` into `datetime.datetime` instance
+- [#1996](https://github.com/meltano/sdk/issues/1996) Respect nullability of leaf properties when flattening schema
+- [#1844](https://github.com/meltano/sdk/issues/1844) Safely skip parsing record field as date-time if it is missing in schema
+- [#1885](https://github.com/meltano/sdk/issues/1885) Map `record` field to a JSON `object` type
+- [#2015](https://github.com/meltano/sdk/issues/2015) Ensure `default` property is passed to SCHEMA messages -- _**Thanks @prakharcode!**_
 
-## v0.32.0b1 (2023-09-13)
+### 📚 Documentation Improvements
+
+- [#2017](https://github.com/meltano/sdk/issues/2017) Document support for comprehensions in stream maps
+
+## v0.32.0 (2023-09-22)
 
 ### ✨ New
 
@@ -34,11 +46,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - [#1949](https://github.com/meltano/sdk/issues/1949) Retry SQLAlchemy engine creation for adapters without JSON SerDe support
 - [#1939](https://github.com/meltano/sdk/issues/1939) Handle `decimal.Decimal` instances in flattening
 - [#1927](https://github.com/meltano/sdk/issues/1927) Handle replication key not found in stream schema -- _**Thanks @mjsqu!**_
+- [#1977](https://github.com/meltano/sdk/issues/1977) Fix hanging downstream tests in tap-postgres
+- [#1970](https://github.com/meltano/sdk/issues/1970) Warn instead of crashing when schema helpers cannot append `null` to types
 
-### 📚 Documentation Improvements
+### ⚡ Performance Improvements
 
 - [#1925](https://github.com/meltano/sdk/issues/1925) Add viztracer command for testing targets -- _**Thanks @mjsqu!**_
 
+- [#1962](https://github.com/meltano/sdk/issues/1962) Ensure `raw_schema` in stream mapper is immutable
+
 ## v0.31.1 (2023-08-17)
 
 ### ✨ New

cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml

@@ -21,13 +21,13 @@ packages = [
 {%- endif %}
 
 [tool.poetry.dependencies]
-python = ">=3.7.1,<4"
-singer-sdk = { version="~=0.32.0" }
+python = ">=3.8,<4"
+singer-sdk = { version="~=0.33.1" }
 fs-s3fs = { version = "~=1.1.1", optional = true }
 
 [tool.poetry.group.dev.dependencies]
 pytest = ">=7.4.0"
-singer-sdk = { version="~=0.32.0", extras = ["testing"] }
+singer-sdk = { version="~=0.33.1", extras = ["testing"] }
 
 [tool.poetry.extras]
 s3 = ["fs-s3fs"]

cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml

@@ -20,8 +20,8 @@ packages = [
 {%- endif %}
 
 [tool.poetry.dependencies]
-python = ">=3.7.1,<4"
-singer-sdk = { version="~=0.32.0" }
+python = ">=3.8,<4"
+singer-sdk = { version="~=0.33.1" }
 fs-s3fs = { version = "~=1.1.1", optional = true }
 {%- if cookiecutter.stream_type in ["REST", "GraphQL"] %}
 requests = "~=2.31.0"
@@ -32,7 +32,7 @@ cached-property = "~=1" # Remove after Python 3.7 support is dropped
 
 [tool.poetry.group.dev.dependencies]
 pytest = ">=7.4.0"
-singer-sdk = { version="~=0.32.0", extras = ["testing"] }
+singer-sdk = { version="~=0.33.1", extras = ["testing"] }
 
 [tool.poetry.extras]
 s3 = ["fs-s3fs"]