airbyte-ci: Add pypi publishing logic (#34111)

.github/actions/run-dagger-pipeline/action.yml

@@ -83,6 +83,9 @@ inputs:
     description: "URL to airbyte-ci binary"
     required: false
     default: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci
+  python_registry_token:
+    description: "Python registry API token to publish python package"
+    required: false
 
 runs:
   using: "composite"
@@ -182,3 +185,4 @@ runs:
         CI: "True"
         TAILSCALE_AUTH_KEY: ${{ inputs.tailscale_auth_key }}
         DOCKER_REGISTRY_MIRROR_URL: ${{ inputs.docker_registry_mirror_url }}
+        PYTHON_REGISTRY_TOKEN: ${{ inputs.python_registry_token }}

.github/workflows/publish_connectors.yml

@@ -63,6 +63,7 @@ jobs:
           s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
           tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }}
           subcommand: "connectors --concurrency=1 --execute-timeout=3600 --metadata-changes-only publish --main-release"
+          python_registry_token: ${{ secrets.PYPI_TOKEN }}
 
       - name: Publish connectors [manual]
         id: publish-connectors
@@ -84,6 +85,7 @@ jobs:
           s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
           tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }}
           subcommand: "connectors ${{ github.event.inputs.connectors-options }} publish ${{ github.event.inputs.publish-options }}"
+          python_registry_token: ${{ secrets.PYPI_TOKEN }}
 
   set-instatus-incident-on-failure:
     name: Create Instatus Incident on Failure

airbyte-ci/connectors/pipelines/README.md

@@ -491,6 +491,37 @@ This command runs formatting checks and reformats any code that would be reforma
 
 Running `airbyte-ci format fix all` will format all of the different types of code. Run `airbyte-ci format fix --help` for subcommands to format only certain types of files.
 
+### <a id="poetry-subgroup"></a>`poetry` command subgroup
+
+Available commands:
+
+- `airbyte-ci poetry publish`
+
+### Options
+
+| Option              | Required | Default | Mapped environment variable | Description                                                                                 |
+| ------------------- | -------- | ------- | --------------------------- | ------------------------------------------------------------------------------------------- |
+| `--package-path`    | True     |         |                             | The path to the python package to execute a poetry command on.                              |
+
+### Examples
+
+- Publish a python package: `airbyte-ci poetry --package-path=path/to/package publish --publish-name=my-package --publish-version="1.2.3" --python-registry-token="..." --registry-url="http://host.docker.internal:8012/"`
+
+### <a id="format-check-command"></a>`publish` command
+
+This command publishes poetry packages (using `pyproject.toml`) or python packages (using `setup.py`) to a python registry.
+
+For poetry packages, the package name and version can be taken from the `pyproject.toml` file or be specified as options.
+
+#### Options
+
+| Option                    | Required | Default                 | Mapped environment variable | Description                                                                                                     |
+| ------------------------- | -------- | ----------------------- | --------------------------- | --------------------------------------------------------------------------------------------------------------- |
+| `--publish-name`          | False    |                         |                             | The name of the package. Not required for poetry packages that define it in the `pyproject.toml` file           |
+| `--publish-version`       | False    |                         |                             | The version of the package. Not required for poetry packages that define it in the `pyproject.toml` file        |
+| `--python-registry-token` | True     |                         | PYTHON_REGISTRY_TOKEN       | The API token to authenticate with the registry. For pypi, the `pypi-` prefix needs to be specified             |
+| `--registry-url`          | False    | https://pypi.org/simple |                             | The python registry to publish to. Defaults to main pypi |
+
 ### <a id="metadata-validate-command-subgroup"></a>`metadata` command subgroup
 
 Available commands:
@@ -547,7 +578,8 @@ E.G.: running `pytest` on a specific test folder:
 
 | Version | PR                                                         | Description                                                                                                       |
 | ------- | ---------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------- |
-| 3.5.3   | [#34339](https://github.com/airbytehq/airbyte/pull/34339)  | only do minimal changes on a connector version_bump                                                           |
+| 3.6.0   | [#34111](https://github.com/airbytehq/airbyte/pull/34111)  | Add python registry publishing                                                                                    |
+| 3.5.3   | [#34339](https://github.com/airbytehq/airbyte/pull/34339)  | only do minimal changes on a connector version_bump                                                               |
 | 3.5.2   | [#34381](https://github.com/airbytehq/airbyte/pull/34381)  | Bind a sidecar docker host for `airbyte-ci test`                                                                  |
 | 3.5.1   | [#34321](https://github.com/airbytehq/airbyte/pull/34321)  | Upgrade to Dagger 0.9.6 .                                                                                         |
 | 3.5.0   | [#33313](https://github.com/airbytehq/airbyte/pull/33313)  | Pass extra params after Gradle tasks.                                                                             |

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py

@@ -89,12 +89,16 @@ def metadata_service_gcs_credentials_secret(self) -> Secret:
     def spec_cache_gcs_credentials_secret(self) -> Secret:
         return self.dagger_client.set_secret("spec_cache_gcs_credentials", self.spec_cache_gcs_credentials)
 
+    @property
+    def pre_release_suffix(self) -> str:
+        return self.git_revision[:10]
+
     @property
     def docker_image_tag(self) -> str:
         # get the docker image tag from the parent class
         metadata_tag = super().docker_image_tag
         if self.pre_release:
-            return f"{metadata_tag}-dev.{self.git_revision[:10]}"
+            return f"{metadata_tag}-dev.{self.pre_release_suffix}"
         else:
             return metadata_tag
 

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py

@@ -14,8 +14,10 @@
 from pipelines.airbyte_ci.connectors.publish.context import PublishConnectorContext
 from pipelines.airbyte_ci.connectors.reports import ConnectorReport
 from pipelines.airbyte_ci.metadata.pipeline import MetadataUpload, MetadataValidation
+from pipelines.airbyte_ci.steps.python_registry import PublishToPythonRegistry, PythonRegistryPublishContext
 from pipelines.dagger.actions.remote_storage import upload_to_gcs
 from pipelines.dagger.actions.system import docker
+from pipelines.helpers.pip import is_package_published
 from pipelines.models.steps import Step, StepResult, StepStatus
 from pydantic import ValidationError
 
@@ -52,6 +54,28 @@ async def _run(self) -> StepResult:
             return StepResult(self, status=StepStatus.SUCCESS, stdout=f"No manifest found for {self.context.docker_image}.")
 
 
+class CheckPythonRegistryPackageDoesNotExist(Step):
+    context: PythonRegistryPublishContext
+    title = "Check if the connector is published on python registry"
+
+    async def _run(self) -> StepResult:
+        is_published = is_package_published(
+            self.context.package_metadata.name, self.context.package_metadata.version, self.context.registry
+        )
+        if is_published:
+            return StepResult(
+                self,
+                status=StepStatus.SKIPPED,
+                stderr=f"{self.context.package_metadata.name} already exists in version {self.context.package_metadata.version}.",
+            )
+        else:
+            return StepResult(
+                self,
+                status=StepStatus.SUCCESS,
+                stdout=f"{self.context.package_metadata.name} does not exist in version {self.context.package_metadata.version}.",
+            )
+
+
 class PushConnectorImageToRegistry(Step):
     context: PublishConnectorContext
     title = "Push connector image to registry"
@@ -259,6 +283,11 @@ def create_connector_report(results: List[StepResult]) -> ConnectorReport:
             check_connector_image_results = await CheckConnectorImageDoesNotExist(context).run()
             results.append(check_connector_image_results)
 
+            python_registry_steps, terminate_early = await _run_python_registry_publish_pipeline(context)
+            results.extend(python_registry_steps)
+            if terminate_early:
+                return create_connector_report(results)
+
             # If the connector image already exists, we don't need to build it, but we still need to upload the metadata file.
             # We also need to upload the spec to the spec cache bucket.
             if check_connector_image_results.status is StepStatus.SKIPPED:
@@ -312,6 +341,33 @@ def create_connector_report(results: List[StepResult]) -> ConnectorReport:
     return connector_report
 
 
+async def _run_python_registry_publish_pipeline(context: PublishConnectorContext) -> Tuple[List[StepResult], bool]:
+    """
+    Run the python registry publish pipeline for a single connector.
+    Return the results of the steps and a boolean indicating whether there was an error and the pipeline should be stopped.
+    """
+    results: List[StepResult] = []
+    # Try to convert the context to a PythonRegistryPublishContext. If it returns None, it means we don't need to publish to a python registry.
+    python_registry_context = await PythonRegistryPublishContext.from_publish_connector_context(context)
+    if not python_registry_context:
+        return results, False
+
+    check_python_registry_package_exists_results = await CheckPythonRegistryPackageDoesNotExist(python_registry_context).run()
+    results.append(check_python_registry_package_exists_results)
+    if check_python_registry_package_exists_results.status is StepStatus.SKIPPED:
+        context.logger.info("The connector version is already published on python registry.")
+    elif check_python_registry_package_exists_results.status is StepStatus.SUCCESS:
+        context.logger.info("The connector version is not published on python registry. Let's build and publish it.")
+        publish_to_python_registry_results = await PublishToPythonRegistry(python_registry_context).run()
+        results.append(publish_to_python_registry_results)
+        if publish_to_python_registry_results.status is StepStatus.FAILURE:
+            return results, True
+    elif check_python_registry_package_exists_results.status is StepStatus.FAILURE:
+        return results, True
+
+    return results, False
+
+
 def reorder_contexts(contexts: List[PublishConnectorContext]) -> List[PublishConnectorContext]:
     """Reorder contexts so that the ones that are for strict-encrypt/secure connectors come first.
     The metadata upload on publish checks if the the connectors referenced in the metadata file are already published to DockerHub.

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/__init__.py

@@ -0,0 +1,3 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/commands.py

@@ -0,0 +1,34 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+"""
+Module exposing the format commands.
+"""
+from __future__ import annotations
+
+import asyncclick as click
+from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj
+from pipelines.cli.lazy_group import LazyGroup
+from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context
+
+
+@click.group(
+    name="poetry",
+    help="Commands related to running poetry commands.",
+    cls=LazyGroup,
+    lazy_subcommands={
+        "publish": "pipelines.airbyte_ci.poetry.publish.commands.publish",
+    },
+)
+@click.option(
+    "--package-path",
+    help="The path to publish",
+    type=click.STRING,
+    required=True,
+)
+@click_merge_args_into_context_obj
+@pass_pipeline_context
+@click_ignore_unused_kwargs
+async def poetry(pipeline_context: ClickPipelineContext) -> None:
+    pass

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/__init__.py

@@ -0,0 +1,3 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/commands.py

@@ -0,0 +1,105 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+"""
+Module exposing the format commands.
+"""
+from __future__ import annotations
+
+from typing import Optional
+
+import asyncclick as click
+from packaging import version
+from pipelines.airbyte_ci.steps.python_registry import PublishToPythonRegistry
+from pipelines.cli.confirm_prompt import confirm
+from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand
+from pipelines.consts import DEFAULT_PYTHON_PACKAGE_REGISTRY_URL
+from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context
+from pipelines.models.contexts.python_registry_publish import PythonRegistryPublishContext
+from pipelines.models.steps import StepStatus
+
+
+async def _has_metadata_yaml(context: PythonRegistryPublishContext) -> bool:
+    dir_to_publish = context.get_repo_dir(context.package_path)
+    return "metadata.yaml" in await dir_to_publish.entries()
+
+
+def _validate_python_version(_ctx: dict, _param: dict, value: Optional[str]) -> Optional[str]:
+    """
+    Check if an given version is valid.
+    """
+    if value is None:
+        return value
+    try:
+        version.Version(value)
+        return value
+    except version.InvalidVersion:
+        raise click.BadParameter(f"Version {value} is not a valid version.")
+
+
+@click.command(cls=DaggerPipelineCommand, name="publish", help="Publish a Python package to a registry.")
+@click.option(
+    "--python-registry-token",
+    help="Access token",
+    type=click.STRING,
+    required=True,
+    envvar="PYTHON_REGISTRY_TOKEN",
+)
+@click.option(
+    "--registry-url",
+    help="Which registry to publish to. If not set, the default pypi is used. For test pypi, use https://test.pypi.org/legacy/",
+    type=click.STRING,
+    default=DEFAULT_PYTHON_PACKAGE_REGISTRY_URL,
+)
+@click.option(
+    "--publish-name",
+    help="The name of the package to publish. If not set, the name will be inferred from the pyproject.toml file of the package.",
+    type=click.STRING,
+)
+@click.option(
+    "--publish-version",
+    help="The version of the package to publish. If not set, the version will be inferred from the pyproject.toml file of the package.",
+    type=click.STRING,
+    callback=_validate_python_version,
+)
+@pass_pipeline_context
+@click.pass_context
+async def publish(
+    ctx: click.Context,
+    click_pipeline_context: ClickPipelineContext,
+    python_registry_token: str,
+    registry_url: str,
+    publish_name: Optional[str],
+    publish_version: Optional[str],
+) -> bool:
+    context = PythonRegistryPublishContext(
+        is_local=ctx.obj["is_local"],
+        git_branch=ctx.obj["git_branch"],
+        git_revision=ctx.obj["git_revision"],
+        ci_report_bucket=ctx.obj["ci_report_bucket_name"],
+        report_output_prefix=ctx.obj["report_output_prefix"],
+        gha_workflow_run_url=ctx.obj.get("gha_workflow_run_url"),
+        dagger_logs_url=ctx.obj.get("dagger_logs_url"),
+        pipeline_start_timestamp=ctx.obj.get("pipeline_start_timestamp"),
+        ci_context=ctx.obj.get("ci_context"),
+        ci_gcs_credentials=ctx.obj["ci_gcs_credentials"],
+        python_registry_token=python_registry_token,
+        registry=registry_url,
+        package_path=ctx.obj["package_path"],
+        package_name=publish_name,
+        version=publish_version,
+    )
+
+    dagger_client = await click_pipeline_context.get_dagger_client(pipeline_name=f"Publish {ctx.obj['package_path']} to python registry")
+    context.dagger_client = dagger_client
+
+    if await _has_metadata_yaml(context):
+        confirm(
+            "It looks like you are trying to publish a connector. In most cases, the `connectors` command group should be used instead. Do you want to continue?",
+            abort=True,
+        )
+
+    publish_result = await PublishToPythonRegistry(context).run()
+
+    return publish_result.status is StepStatus.SUCCESS