🚀 Launch Java CDK v0.1.0

.devcontainer/java-connectors-generic/devcontainer.json

@@ -0,0 +1,58 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+{
+  "name": "Connector Development DevContainer (Generic)",
+
+  "image": "mcr.microsoft.com/devcontainers/java:0-17",
+  "features": {
+    "ghcr.io/devcontainers/features/docker-in-docker": {},
+    "ghcr.io/devcontainers/features/java:1": {
+      "installGradle": true,
+      "version": "latest",
+      "jdkDistro": "open",
+      "gradleVersion": "7.5.1"
+    },
+    // Python needed for `airbyte-ci` CLI
+    "ghcr.io/devcontainers/features/python:1": {
+      "installGradle": true,
+      "version": "3.10",
+      "installTools": true
+    },
+    "ghcr.io/devcontainers-contrib/features/poetry:2": {}
+  },
+
+  // Deterministic order reduces cache busting
+  "overrideFeatureInstallOrder": [
+    "ghcr.io/devcontainers/features/docker-in-docker",
+    "ghcr.io/devcontainers/features/java",
+    "ghcr.io/devcontainers/features/python",
+    "ghcr.io/devcontainers-contrib/features/poetry"
+  ],
+
+  // Configure tool-specific properties.
+  "customizations": {
+    "vscode": {
+      "extensions": ["vscjava.vscode-gradle", "tamasfe.even-better-toml"],
+      "settings": {
+        "extensions.ignoreRecommendations": true,
+        "git.openRepositoryInParentFolders": "always"
+      }
+    }
+  },
+
+  // Mark the root directory as 'safe' for git.
+  "initializeCommand": "git config --add safe.directory /workspaces/airbyte",
+
+  // Install Gradle, `airbyte-ci` CLI, and Dagger (installed via airbyte-ci --help)
+  "postCreateCommand": "./gradlew --version && pipx install --editable ./airbyte-ci/connectors/pipelines/ || airbyte-ci --help || true",
+
+  "containerEnv": {
+    // Deterministic Poetry virtual env location: `./.venv`
+    "POETRY_VIRTUALENVS_IN_PROJECT": "true"
+  }
+
+  // Override to change the directory that the IDE opens by default:
+  // "workspaceFolder": "/workspaces/airbyte"
+
+  // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
+  // "remoteUser": "root"
+}

.github/workflows/gradle.yml

@@ -245,7 +245,10 @@ jobs:
       - name: Build
         run: ./gradlew :airbyte-cdk:java:airbyte-cdk:build --scan
 
-  # Connectors Base
+      - name: Run Integration Tests
+        run: ./gradlew :airbyte-cdk:java:airbyte-cdk:integrationTest --scan
+
+      # Connectors Base
   # In case of self-hosted EC2 errors, remove this block.
   start-connectors-base-build-runner:
     name: "Connectors Base: Start Build EC2 Runner"

.github/workflows/publish-java-cdk-command.yml

@@ -54,9 +54,7 @@ env:
   GITREF: ${{ github.event.inputs.gitref || github.ref }}
   FORCE: "${{ github.event.inputs.force == null && 'false' || github.event.inputs.force }}"
   DRY_RUN: "${{ github.event.inputs.dry-run == null && 'true' || github.event.inputs.dry-run }}"
-  CDK_VERSION_FILE_PATH: "./airbyte-cdk/java/airbyte-cdk/src/main/resources/version.properties"
-  # Uncomment and replace above when CDK 0.1.0 is released:
-  # CDK_VERSION_FILE_PATH: "./airbyte-cdk/java/airbyte-cdk/src/main/resources/version.properties"
+  CDK_VERSION_FILE_PATH: "./airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties"
 
 jobs:
   # We are using these runners because they are the same as the one for `publish-command.yml`
@@ -114,27 +112,14 @@ jobs:
             exit 1
           fi
           echo "CDK_VERSION=${cdk_version}" >> $GITHUB_ENV
-      - name: Check for already-published version (${{ env.CDK_VERSION }}, FORCE=${{ env.FORCE }})
-        if: ${{ !(env.FORCE == 'true') }}
-        run: |
-          GROUP_ID="io.airbyte"
-          ARTIFACT_ID="airbyte-cdk"
-          VERSION=${CDK_VERSION}
-          REPO_URL="https://airbyte.mycloudrepo.io/repositories/airbyte-public-jars"
-          GROUP_ID_URL=$(echo $GROUP_ID | tr '.' '/')
-          URL="${REPO_URL}/${GROUP_ID_URL}/${ARTIFACT_ID}/${VERSION}/${ARTIFACT_ID}-${VERSION}.pom"
-          echo "Checking if URL exists: ${URL}"
-          if curl --output /dev/null --silent --head --fail "$URL"; then
-            echo "Version ${VERSION} of ${ARTIFACT_ID} has already been published. Aborting."
-              exit 1
-          else
-              echo "Version ${VERSION} of ${ARTIFACT_ID} has not been published. Proceeding..."
-          fi
       - name: Setup Java
         uses: actions/setup-java@v3
         with:
           distribution: "zulu"
           java-version: "17"
+      - name: Check for already-published version (${{ env.CDK_VERSION }}, FORCE=${{ env.FORCE }})
+        if: ${{ !(env.FORCE == 'true') }}
+        run: ./gradlew :airbyte-cdk:java:airbyte-cdk:assertCdkVersionNotPublished
       - name: Build Java CDK
         run: ./gradlew --no-daemon :airbyte-cdk:java:airbyte-cdk:build
       - name: Publish Java Modules to MavenLocal (Dry-Run)

airbyte-cdk/java/airbyte-cdk/README.md

@@ -2,91 +2,149 @@
 
 This page will walk through the process of developing with the Java CDK.
 
-## Building the CDK
+- [Developing with the Java CDK](#developing-with-the-java-cdk)
+  - [Intro to the Java CDK](#intro-to-the-java-cdk)
+    - [What is included in the Java CDK?](#what-is-included-in-the-java-cdk)
+    - [How is the CDK published?](#how-is-the-cdk-published)
+  - [Using the Java CDK](#using-the-java-cdk)
+    - [Building the CDK](#building-the-cdk)
+    - [Bumping the CDK version](#bumping-the-cdk-version)
+    - [Publishing the CDK](#publishing-the-cdk)
+  - [Developing Connectors with the Java CDK](#developing-connectors-with-the-java-cdk)
+    - [Referencing the CDK from Java connectors](#referencing-the-cdk-from-java-connectors)
+    - [Developing a connector alongside the CDK](#developing-a-connector-alongside-the-cdk)
+    - [Developing a connector against a pinned CDK version](#developing-a-connector-against-a-pinned-cdk-version)
+  - [Common Debugging Tips](#common-debugging-tips)
+  - [Changelog](#changelog)
+    - [Java CDK](#java-cdk)
+
+## Intro to the Java CDK
+
+### What is included in the Java CDK?
+
+The java CDK is comprised of separate modules:
+
+- `core` - Shared classes for building connectors of all types.
+- `db-sources` - Shared classes for building DB sources.
+- `db-destinations` - Shared classes for building DB destinations.
+
+Each CDK submodule may contain these elements:
+
+- `src/main` - (Required.) The classes that will ship with the connector, providing capabilities to the connectors.
+- `src/test` - (Required.) These are unit tests that run as part of every build of the CDK. They help ensure that CDK `main` code is in a healthy state.
+- `src/test-integration` - (Optional.) Integration tests which provide a more extensive test of the code in `src/main`. These are not by the `build` command but are executed as part of the `integrationTest` or `integrationTestJava` Gradle tasks.
+- `src/testFixtures` - (Optional.) These shared classes are exported for connectors for use in the connectors' own test implementations. Connectors will have access to these classes within their unit and integration tests, but the classes will not be shipped with connectors when they are published.
+
+### How is the CDK published?
+
+The CDK is published as a set of jar files sharing a version number. Every submodule generates one runtime jar for the main classes. If the submodule contains test fixtures, a second jar will be published with the test fixtures classes.
+
+Note: Connectors do not have to manage which jars they should depend on, as this is handled automatically by the `airbyte-java-connector` plugin. See example below.
+
+## Using the Java CDK
+
+### Building the CDK
 
 To build and test the Java CDK, execute the following:
 
 ```sh
 ./gradlew :airbyte-cdk:java:airbyte-cdk:build
 ```
 
-## Bumping the declared CDK version
+### Bumping the CDK version
 
 You will need to bump this version manually whenever you are making changes to code inside the CDK.
 
-While under development, the next version number for the CDK is tracked in the file: `airbyte-cdk/java/airbyte-cdk/src/main/resources/version.properties`.
+While under development, the next version number for the CDK is tracked in the file: `airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties`.
 
 If the CDK is not being modified, this file will contain the most recently published version number.
 
-## Publishing the CDK to Local Maven
+### Publishing the CDK
+
+_⚠️ These steps should only be performed after all testing and approvals are in place on the PR. ⚠️_
+
+The CDK can be published with a GitHub Workflow and a slash command which can be run by Airbyte personnel.
+
+To invoke via slash command (recommended), use the following syntax in a comment on the PR that contains your changes:
+
+```bash
+/publish-java-cdk                # Run with the defaults (dry-run=false, force=false)
+/publish-java-cdk dry-run=true   # Run in dry-run mode (no-op)
+/publish-java-cdk force=true     # Force-publish if needing to replace an already published version
+```
+
+Note:
 
-If your connector pins to a work-in-progress `-SNAPSHOT` version of the CDK (e.g. `0.0.1-SNAPSHOT` or `0.2.0-SNAPSHOT`), Gradle will notice this and automatically run the task to build and publish it to your MavenLocal repository before running the connector's own build and test tasks.
+- Remember to **document your changes** in the Changelog section below.
+- After you publish the CDK, remember to toggle `useLocalCdk` back to `false` in all connectors.
+- Unless you specify `force=true`, the pipeline should fail if the version you are trying to publish already exists.
+- By running the publish with `dry-run=true`, you can confirm the process is working as expected, without actually publishing the changes.
+- In dry-run mode, you can also view and download the jars that are generated. To do so, navigate to the job status in GitHub Actions and navigate to the 'artifacts' section.
+- You can also invoke manually in the GitHub Web UI. To do so: go to `Actions` tab, select the `Publish Java CDK` workflow, and click `Run workflow`.
+- You can view and administer published CDK versions here: https://admin.cloudrepo.io/repository/airbyte-public-jars/io/airbyte/airbyte-cdk
+- The public endpoint for published CDK versions is here: https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars/io/airbyte/airbyte-cdk/
 
-## Referencing the CDK from Java connectors
+## Developing Connectors with the Java CDK
+
+### Referencing the CDK from Java connectors
 
 You can reference the CDK in your connector's `build.gradle` file:
 
 ```groovy
-dependencies {
-    implementation 'io.airbyte:airbyte-cdk:0.0.1-SNAPSHOT'
+plugins {
+    id 'application'
+    id 'airbyte-java-connector'
 }
+
+airbyteJavaConnector {
+    cdkVersionRequired = '0.1.0'   // The CDK version to pin to.
+    features = ['db-destinations'] // An array of CDK features to depend on.
+    useLocalCdk = false            // Use 'true' to use a live reference to the 
+                                   // local cdk project.
+}
+
+airbyteJavaConnector.addCdkDependencies()
 ```
 
-Replace `0.0.1-SNAPSHOT` with the version you are working with. If you're actively developing the CDK and want to use the latest version locally, use the `-SNAPSHOT` suffix to reference a bumped version number. (See below for version bump instructions.)
+Replace `0.1.0` with the CDK version you are working with. If you're actively developing the CDK and want to use the latest version locally, use the `useLocalCdk` flag to use the live CDK code during builds and tests.
 
-## Developing a connector alongside the CDK
+### Developing a connector alongside the CDK
 
 You can iterate on changes in the CDK local and test them in the connector without needing to publish the CDK changes publicly.
 
 When modifying the CDK and a connector in the same PR or branch, please use the following steps:
 
-1. Set the version of the CDK in `version.properties` to the next appropriate version number, along with a `-SNAPSHOT` suffix, as explained above.
-1. In your connector project, modify the `build.gradle` to use the _new_ local CDK version with the `-SNAPSHOT` suffix, as explained above.
-1. Build and test your connector as usual. Gradle will automatically build the snapshot version of the CDK, and it will use this version when building and testing your connector.
-1. As you make additional changes to the CDK, Gradle will automatically rebuild and republish the CDK locally in order to incorporate the latest changes.
+1. Set the version of the CDK in `version.properties` to the next appropriate version number and add a description in the `Changelog` at the bottom of this readme file.
+2. Modify your connector's build.gradle file as follows:
+   1. Set `useLocalCdk` to `true` in the connector you are working on. This will ensure the connector always uses the local CDK definitions instead of the published version.
+   2. Set `cdkVersionRequired` to use the new _to-be-published_ CDK version.
 
-## Developing a connector against a pinned CDK version
+After the above, you can build and test your connector as usual. Gradle will automatically use the local CDK code files while you are working on the connector.
 
-You can always pin your connector to a prior stable version of the CDK, which may not match what is the latest version in the `airbyte` repo. For instance, your connector can be pinned to `0.1.1` while the latest version may be `0.2.0`.
+Once you are done developing and testing your CDK changes:
 
-Maven and Gradle will automatically reference the correct (pinned) version of the CDK for your connector, and you can use your local IDE to browse the prior version of the codebase that corresponds to that version.
+1. Publish the CDK using the instructions here in this readme.
+2. After publishing the CDK, update the `useLocalCdk` setting by running `./gradlew :airbyte-integrations:connectors:<connector-name>:disableLocalCdkRefs`. to automatically revert `useLocalCdk` to `false`.
+3. You can optionally run `./gradlew :airbyte-integrations:connectors:<connector-name>:assertNotUsingLocalCdk` to ensure that the project is not using a local CDK reference.
 
-<!--
-TODO: More detailed instructions needed.
+_Note: You can also use  `./gradlew assertNotUsingLocalCdk` or `./gradlew disableLocalCdkRefs` to run these tasks on **all** connectors simultaneously._
 
-Add screenshots and additional details for IntelliJ IDEA and/or VS Code.
--->
+### Developing a connector against a pinned CDK version
 
-## Publish and release
-
-_⚠️ These steps should only be performed after all testing and approvals are in place on the PR. ⚠️_
-
-1. Remove `-SNAPSHOT` suffix from CDK version.
-   - e.g. by running `nano airbyte-cdk/java/airbyte-cdk/src/main/resources/
-version.properties`.
-2. Publish the CDK to Maven ([mycloudrepo](https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars/io/airbyte/airbyte-cdk/))
-   - `./gradlew :airbyte-cdk:java:airbyte-cdk:publish`
-   - Note: you will need to export the env vars `CLOUDREPO_USER` and `CLOUDREPO_PASSWORD` before publishing.
-3. Remove the `-SNAPSHOT` suffix from any connector(s) using the latest version.
-   - E.g. If modifying `source-mysql`, then remove `-SNAPSHOT` from the CDK `implements` declaration in `airbyte-integrations/connectors/source-mysql/build.gradle`.
-4. As per the normal process, modified connector(s) will be automatically published after they are merged to the main branch.
-
-Note:
+You can always pin your connector to a prior stable version of the CDK, which may not match what is the latest version in the `airbyte` repo. For instance, your connector can be pinned to `0.1.1` while the latest version may be `0.2.0`.
 
-- This is documented as a manual process, but we will automate it into a CI workflow.
-- You can view and administer published CDK versions here: https://admin.cloudrepo.io/repository/airbyte-public-jars/io/airbyte/airbyte-cdk
-- The corresponding public endpoint for published CDK versions is here: https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars/io/airbyte/airbyte-cdk/
+Maven and Gradle will automatically reference the correct (pinned) version of the CDK for your connector, and you can use your local IDE to browse the prior version of the codebase that corresponds to that version.
 
-## Debugging
+## Common Debugging Tips
 
 MavenLocal debugging steps:
 
 1. Confirm local publish status by running:
-   `ls -la ~/.m2/repository/io/airbyte/airbyte-cdk\*`
+   `ls -la ~/.m2/repository/io/airbyte/airbyte-cdk/*`
 2. Confirm jar contents by running:
    `jar tf ~/.m2/repository/io/airbyte/airbyte-cdk/0.0.2-SNAPSHOT/airbyte-cdk-0.0.2-SNAPSHOT.jar`
 3. Remove CDK artifacts from MavenLocal by running:
-   `rm -rf ~/.m2/repository/io/airbyte/airbyte-cdk\*`
+   `rm -rf ~/.m2/repository/io/airbyte/airbyte-cdk/*`
 4. Rebuid CDK artifacts by running:
    `./gradlew :airbyte-cdk:java:airbyte-cdk:build`
    or
@@ -98,5 +156,6 @@ MavenLocal debugging steps:
 
 | Version | Date       | Pull Request                                               | Subject                               |
 | :------ | :--------- | :--------------------------------------------------------- | :------------------------------------ |
+| 0.1.0   | 2023-09-27 | [\#30445](https://github.com/airbytehq/airbyte/pull/30445) | First launch, including shared classes for all connectors. |
 | 0.0.2   | 2023-08-21 | [\#28687](https://github.com/airbytehq/airbyte/pull/28687) | Version bump only (no other changes). |
 | 0.0.1   | 2023-08-08 | [\#28687](https://github.com/airbytehq/airbyte/pull/28687) | Initial release for testing.          |

airbyte-cdk/java/airbyte-cdk/_rebuild_full_fileset_branch.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# This script is used to rebuild the java-cdk-010-d2 branch from scratch.
+# TODO: Delete this script once the migration is complete.
+
+# Check with the use to confirm the action
+read -p "This script will delete and recreate the java-cdk-010-d2 branch. You should merge in the latest from from master before running this. Are you sure? (y/n) " -n 1 -r
+
+# Switch to the base branch
+git checkout java-cdk-010-d
+# Make sure the base branch is up-to-date
+# git merge origin/master
+# Delete and recreate the old branch locally
+git branch -D java-cdk-010-d2
+git checkout -b java-cdk-010-d2
+# Run the migration script
+python ./airbyte-cdk/java/airbyte-cdk/_temp_migration_script.py 
+# Stage the changes and commit
+git add .
+git commit -m "commit the migration script result"
+# Force push to the origin (this PR)
+git push -uf origin java-cdk-010-d2
+
+# git cherry-pick c125f14cdce8be61da85ebedffd3c4a576dc1fc5  # delete dead code (incorrectly annotated as override)
+# git cherry-pick 89d81ec62fcfa44fff36196e2d34571b46593818  # disable GlobalStateManagerTest.testToState