Merge remote-tracking branch 'origin/master' into artem1205/precommit…

…-fix-unused-imports

.github/CODEOWNERS

@@ -7,15 +7,15 @@
 /airbyte-integrations/connectors/destination-snowflake-cortex @airbytehq/ai-language-models
 
 # CI/CD
-/.github/ @airbytehq/dev-tooling
-/airbyte-ci/ @airbytehq/dev-tooling
+/.github/ @airbytehq/dev-extensibility
+/airbyte-ci/ @airbytehq/dev-extensibility
 
 # Python CDK and Connector Acceptance Tests
-/airbyte-integrations/connector-templates/ @airbytehq/dev-marketplace-contributions
-/airbyte-integrations/bases/connector-acceptance-test/ @airbytehq/dev-tooling
+/airbyte-integrations/connector-templates/ @airbytehq/dev-extensibility
+/airbyte-integrations/bases/connector-acceptance-test/ @airbytehq/dev-extensibility
 
 # Build customization file change
-/airbyte-integrations/connectors/**/build_customization.py @airbytehq/dev-tooling
+/airbyte-integrations/connectors/**/build_customization.py @airbytehq/dev-extensibility
 
 # Protocol related items
 /docs/understanding-airbyte/airbyte-protocol.md @airbytehq/protocol-reviewers
@@ -59,8 +59,8 @@
 /airbyte-integrations/connectors/destination-redshift/ @airbytehq/move-destinations
 
 # Python critical connectors
-/airbyte-integrations/connectors/source-facebook-marketing/ @airbytehq/python-team
-/airbyte-integrations/connectors/source-hubspot/ @airbytehq/python-team
-/airbyte-integrations/connectors/source-salesforce/ @airbytehq/python-team
-/airbyte-integrations/connectors/source-shopify/ @airbytehq/python-team
-/airbyte-integrations/connectors/source-stripe/ @airbytehq/python-team
+/airbyte-integrations/connectors/source-facebook-marketing/ @airbytehq/dev-extensibility
+/airbyte-integrations/connectors/source-hubspot/ @airbytehq/dev-extensibility
+/airbyte-integrations/connectors/source-salesforce/ @airbytehq/dev-extensibility
+/airbyte-integrations/connectors/source-shopify/ @airbytehq/dev-extensibility
+/airbyte-integrations/connectors/source-stripe/ @airbytehq/dev-extensibility

.github/actions/install-airbyte-ci/action.yml

@@ -56,13 +56,16 @@ runs:
         sudo mv airbyte-ci-bin /usr/local/bin/airbyte-ci
         sudo chmod +x /usr/local/bin/airbyte-ci
 
-    - name: Install Python 3.10
-      id: install-python-3-10
+    - name: Install Python 3.11
+      id: install-python-3-11
       uses: actions/setup-python@v4
       # if: steps.determine-install-mode.outputs.install-mode == 'source'
       with:
-        python-version: "3.10"
+        python-version: "3.11"
         token: ${{ inputs.github_token }}
+        cache: "pip"
+        check-latest: true
+        update-environment: true
 
     - name: Install Airbyte CI from source
       id: install-airbyte-ci-source

.github/workflows/airbyte-ci-release.yml

@@ -38,7 +38,9 @@ jobs:
         id: install_python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: "3.11"
+          check-latest: true
+          update-environment: true
 
       - name: Install Poetry
         id: install_poetry

.github/workflows/airbyte-ci-tests.yml

@@ -34,7 +34,6 @@ jobs:
             internal_poetry_packages:
             - airbyte-ci/connectors/pipelines/**
             - airbyte-ci/connectors/base_images/**
-            - airbyte-ci/connectors/common_utils/**
             - airbyte-ci/connectors/connectors_insights/**
             - airbyte-ci/connectors/connector_ops/**
             - airbyte-ci/connectors/connectors_qa/**

.github/workflows/auto_merge.yml

@@ -15,7 +15,9 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: "3.11"
+          check-latest: true
+          update-environment: true
       - name: Install and configure Poetry
         uses: snok/install-poetry@v1
         with:

.github/workflows/bump-version-command.yml

@@ -95,6 +95,7 @@ jobs:
           gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
           sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
           github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
+          git_repo_url: https://github.com/${{ steps.job-vars.outputs.repo }}.git
           subcommand: |
             connectors --modified bump-version \
               ${{ github.event.inputs.type }} \

.github/workflows/connector-performance-command.yml

@@ -168,7 +168,9 @@ jobs:
       - name: Install Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: "3.11"
+          check-latest: true
+          update-environment: true
       - name: Install CI scripts
         run: |
           pip install pipx

.github/workflows/connectors_version_increment_check.yml → .github/workflows/connectors_pre_release_checks.yml

@@ -1,4 +1,4 @@
-name: Connectors Version Increment Check
+name: Connectors Pre-Release Checks
 
 concurrency:
   # This is the name of the concurrency group. It is used to prevent concurrent runs of the same workflow.
@@ -20,9 +20,11 @@ on:
       - "airbyte-integrations/connectors/**/*"
 jobs:
   connectors_ci:
-    name: Connectors Version Increment Check
+    name: Connector Pre-Release Checks
     runs-on: linux-20.04-large # Custom runner, defined in GitHub org settings
-    if: github.event.pull_request.head.repo.fork != true
+    if: >
+      github.event.pull_request.head.repo.fork != true &&
+      github.event.pull_request.draft == false
     timeout-minutes: 22
     steps:
       - name: Checkout Airbyte
@@ -51,4 +53,4 @@ jobs:
           github_token: ${{ env.PAT }}
           s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
           s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
-          subcommand: "connectors --modified test --only-step=version_inc_check --global-status-check-context='Version increment check for Java connectors' --global-status-check-description='Checking if java connectors modified in this PR got their version bumped'"
+          subcommand: "connectors --modified test --only-step=version_inc_check --only-step=qa_checks --global-status-check-context='Connectors Pre-Release Check' --global-status-check-description='Checking if connectors modified in this PR are ready for release'"

.github/workflows/connectors_tests.yml

@@ -133,4 +133,4 @@ jobs:
           s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
           s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
           # A connector test can't take more than 5 hours to run (5 * 60 * 60 = 18000 seconds)
-          subcommand: "connectors --execute-timeout=18000 --modified  test"
+          subcommand: "connectors --execute-timeout=18000 --modified test --skip-step=qa_checks --skip-step=version_inc_check"

.github/workflows/format-fix-command.yml

@@ -76,8 +76,10 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: "3.11"
           cache: "pip"
+          check-latest: true
+          update-environment: true
       - name: Run pre-commit
         uses: pre-commit/[email protected]
         continue-on-error: true

.github/workflows/format_check.yml

@@ -22,8 +22,10 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: "3.11"
           cache: "pip"
+          check-latest: true
+          update-environment: true
       - name: Run pre-commit
         uses: pre-commit/[email protected]
         id: format-check

.github/workflows/live_tests.yml

@@ -33,6 +33,10 @@ on:
         description: Use the local CDK when building the target connector
         default: "false"
         type: boolean
+      disable_proxy:
+        description: Disable proxy for requests
+        default: "false"
+        type: boolean
       connection_subset:
         description: The subset of connections to select from.
         required: true
@@ -108,6 +112,15 @@ jobs:
             echo "READ_WITH_STATE_FLAG=" >> $GITHUB_ENV
           fi
 
+      - name: Setup Proxy Flag
+        if: github.event_name == 'workflow_dispatch'
+        run: |
+          if ${{ github.event.inputs.disable_proxy }}; then
+            echo "DISABLE_PROXY_FLAG=--connector_live_tests.disable-proxy" >> $GITHUB_ENV
+          else
+            echo "DISABLE_PROXY_FLAG=" >> $GITHUB_ENV
+          fi
+
       - name: Setup Connection Subset Option
         if: github.event_name == 'workflow_dispatch'
         run: |
@@ -129,4 +142,4 @@ jobs:
           github_token: ${{ secrets.GH_PAT_MAINTENANCE_OSS }}
           s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
           s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
-          subcommand: connectors ${{ env.USE_LOCAL_CDK_FLAG }} --name ${{ github.event.inputs.connector_name }} test --only-step connector_live_tests --connector_live_tests.test-suite=live --connector_live_tests.connection-id=${{ github.event.inputs.connection_id }} --connector_live_tests.pr-url=${{ github.event.inputs.pr_url }} ${{ env.READ_WITH_STATE_FLAG }} ${{ env.STREAM_PARAMS }} ${{ env.CONNECTION_SUBSET }}
+          subcommand: connectors ${{ env.USE_LOCAL_CDK_FLAG }} --name ${{ github.event.inputs.connector_name }} test --only-step connector_live_tests --connector_live_tests.test-suite=live --connector_live_tests.connection-id=${{ github.event.inputs.connection_id }} --connector_live_tests.pr-url=${{ github.event.inputs.pr_url }} ${{ env.READ_WITH_STATE_FLAG }} ${{ env.DISABLE_PROXY_FLAG }} ${{ env.STREAM_PARAMS }} ${{ env.CONNECTION_SUBSET }}

.github/workflows/regression_tests.yml

@@ -33,6 +33,10 @@ on:
         description: Use the local CDK when building the target connector
         default: "false"
         type: boolean
+      disable_proxy:
+        description: Disable proxy for requests
+        default: "false"
+        type: boolean
       connection_subset:
         description: The subset of connections to select from.
         required: true
@@ -51,7 +55,9 @@ jobs:
         id: install_python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: "3.11"
+          check-latest: true
+          update-environment: true
 
       - name: Checkout Airbyte
         uses: actions/checkout@v4
@@ -114,6 +120,15 @@ jobs:
             echo "READ_WITH_STATE_FLAG=" >> $GITHUB_ENV
           fi
 
+      - name: Setup Proxy Flag
+        if: github.event_name == 'workflow_dispatch'
+        run: |
+          if ${{ github.event.inputs.disable_proxy }}; then
+            echo "DISABLE_PROXY_FLAG=--connector_live_tests.disable-proxy" >> $GITHUB_ENV
+          else
+            echo "DISABLE_PROXY_FLAG=" >> $GITHUB_ENV
+          fi
+
       - name: Setup Connection Subset Option
         if: github.event_name == 'workflow_dispatch'
         run: |
@@ -135,4 +150,4 @@ jobs:
           github_token: ${{ secrets.GH_PAT_MAINTENANCE_OSS }}
           s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
           s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
-          subcommand: connectors ${{ env.USE_LOCAL_CDK_FLAG }} --name ${{ github.event.inputs.connector_name }} test --only-step connector_live_tests --connector_live_tests.test-suite=regression --connector_live_tests.connection-id=${{ github.event.inputs.connection_id }} --connector_live_tests.pr-url=${{ github.event.inputs.pr_url }} ${{ env.READ_WITH_STATE_FLAG }} ${{ env.STREAM_PARAMS }} ${{ env.CONNECTION_SUBSET }} --global-status-check-context="Regression Tests" --global-status-check-description='Running regression tests'
+          subcommand: connectors ${{ env.USE_LOCAL_CDK_FLAG }} --name ${{ github.event.inputs.connector_name }} test --only-step connector_live_tests --connector_live_tests.test-suite=regression --connector_live_tests.connection-id=${{ github.event.inputs.connection_id }} --connector_live_tests.pr-url=${{ github.event.inputs.pr_url }} ${{ env.READ_WITH_STATE_FLAG }} ${{ env.DISABLE_PROXY_FLAG }} ${{ env.STREAM_PARAMS }} ${{ env.CONNECTION_SUBSET }} --global-status-check-context="Regression Tests" --global-status-check-description='Running regression tests'

.github/workflows/release-airbyte-os.yml

@@ -115,7 +115,7 @@ jobs:
     steps:
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: "3.11"
 
       - name: Checkout Airbyte
         uses: actions/checkout@v3
@@ -188,7 +188,7 @@ jobs:
 
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: "3.11"
 
       - name: Release Octavia
         id: release_octavia

.github/workflows/test-performance-command.yml

@@ -92,7 +92,9 @@ jobs:
       - name: Install Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: "3.11"
+          check-latest: true
+          update-environment: true
       - name: Install CI scripts
         run: |
           pip install pipx

.github/workflows/upload-metadata-files.yml

@@ -19,7 +19,9 @@ jobs:
         if: steps.changed-files.outputs.any_changed == 'true'
         uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: "3.11"
+          check-latest: true
+          update-environment: true
       - name: Install metadata_service
         if: steps.changed-files.outputs.any_changed == 'true'
         run: pip install airbyte-ci/connectors/metadata_service/lib

.pre-commit-config.yaml

@@ -53,7 +53,7 @@ repos:
 
       - id: spotless
         name: Format Java files with Spotless
-        entry: bash -c 'command -v mvn >/dev/null 2>&1 || { if [ -z "$CI" ]; then echo "Maven not installed, skipping spotless" >&2; exit 0; fi }; mvn -f spotless-maven-pom.xml spotless:apply'
+        entry: bash -c 'command -v mvn >/dev/null 2>&1 || { echo "Maven not installed. Install with brew install maven" >&2; exit 1; }; mvn -f spotless-maven-pom.xml spotless:apply'
         language: system
         files: \.(java|kt|gradle)$
         pass_filenames: false

.python-version

@@ -1 +1 @@
-3.10
+3.11

Makefile

@@ -36,7 +36,7 @@ tools.pre-commit.install.Linux:
 
 tools.pre-commit.install.Darwin:
 	@echo "Installing pre-commit with brew..."
-	@brew install pre-commit
+	@brew install pre-commit maven
 	@echo "Pre-commit installation complete"
 
 tools.git-hooks.install: tools.airbyte-ci.install tools.pre-commit.install.$(OS) tools.git-hooks.clean ## Setup pre-commit hooks

airbyte-cdk/bulk/core/extract/src/main/kotlin/io/airbyte/cdk/command/SourceConfiguration.kt

@@ -20,6 +20,11 @@ interface SourceConfiguration : Configuration, SshTunnelConfiguration {
     val maxConcurrency: Int
     val resourceAcquisitionHeartbeat: Duration
 
+    /** Whether it's a CDC configuration. Default to global state */
+    fun isCdc(): Boolean {
+        return global
+    }
+
     /**
      * Micronaut factory which glues [ConfigurationSpecificationSupplier] and
      * [SourceConfigurationFactory] together to produce a [SourceConfiguration] singleton.

airbyte-cdk/bulk/core/extract/src/main/kotlin/io/airbyte/cdk/discover/AirbyteStreamFactory.kt

@@ -1,17 +1,15 @@
 /* Copyright (c) 2024 Airbyte, Inc., all rights reserved. */
 package io.airbyte.cdk.discover
 
+import io.airbyte.cdk.command.SourceConfiguration
 import io.airbyte.protocol.models.Field as AirbyteField
 import io.airbyte.protocol.models.v0.AirbyteStream
 import io.airbyte.protocol.models.v0.CatalogHelpers
 
 /** Stateless object for building an [AirbyteStream] during DISCOVER. */
 interface AirbyteStreamFactory {
-    /** Connector-specific [AirbyteStream] creation logic for GLOBAL-state streams. */
-    fun createGlobal(discoveredStream: DiscoveredStream): AirbyteStream
-
-    /** Connector-specific [AirbyteStream] creation logic for STREAM-state streams. */
-    fun createNonGlobal(discoveredStream: DiscoveredStream): AirbyteStream
+    /** Connector-specific [AirbyteStream] creation logic. */
+    fun create(config: SourceConfiguration, discoveredStream: DiscoveredStream): AirbyteStream
 
     companion object {
 

airbyte-cdk/bulk/core/extract/src/main/kotlin/io/airbyte/cdk/discover/DiscoverOperation.kt

@@ -38,11 +38,7 @@ class DiscoverOperation(
                     val primaryKey: List<List<String>> = metadataQuerier.primaryKey(streamID)
                     val discoveredStream = DiscoveredStream(streamID, fields, primaryKey)
                     val airbyteStream: AirbyteStream =
-                        if (config.global) {
-                            airbyteStreamFactory.createGlobal(discoveredStream)
-                        } else {
-                            airbyteStreamFactory.createNonGlobal(discoveredStream)
-                        }
+                        airbyteStreamFactory.create(config, discoveredStream)
                     airbyteStreams.add(airbyteStream)
                 }
             }