diff --git a/airbyte-integrations/connectors/source-airbyte/README.md b/airbyte-integrations/connectors/source-airbyte/README.md new file mode 100644 index 0000000000000..6ad9c88d481b5 --- /dev/null +++ b/airbyte-integrations/connectors/source-airbyte/README.md @@ -0,0 +1,33 @@ +# Airbyte +This directory contains the manifest-only connector for `source-airbyte`. + + + +## Usage +There are multiple ways to use this connector: +- You can use this connector as any other connector in Airbyte Marketplace. +- You can load this connector in `pyairbyte` using `get_source`! +- You can open this connector in Connector Builder, edit it, and publish to your workspaces. + +Please refer to the manifest-only connector documentation for more details. + +## Local Development +We recommend you use the Connector Builder to edit this connector. + +But, if you want to develop this connector locally, you can use the following steps. + +### Environment Setup +You will need `airbyte-ci` installed. You can find the documentation [here](airbyte-ci). + +### Build +This will create a dev image (`source-airbyte:dev`) that you can use to test the connector locally. +```bash +airbyte-ci connectors --name=source-airbyte build +``` + +### Test +This will run the acceptance tests for the connector. +```bash +airbyte-ci connectors --name=source-airbyte test +``` + diff --git a/airbyte-integrations/connectors/source-airbyte/acceptance-test-config.yml b/airbyte-integrations/connectors/source-airbyte/acceptance-test-config.yml new file mode 100644 index 0000000000000..4a6d901968c4d --- /dev/null +++ b/airbyte-integrations/connectors/source-airbyte/acceptance-test-config.yml @@ -0,0 +1,17 @@ +# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-airbyte:dev +acceptance_tests: + spec: + tests: + - spec_path: "manifest.yaml" + connection: + bypass_reason: "This is a builder contribution, and we do not have secrets at this time" + discovery: + bypass_reason: "This is a builder contribution, and we do not have secrets at this time" + basic_read: + bypass_reason: "This is a builder contribution, and we do not have secrets at this time" + incremental: + bypass_reason: "This is a builder contribution, and we do not have secrets at this time" + full_refresh: + bypass_reason: "This is a builder contribution, and we do not have secrets at this time" diff --git a/airbyte-integrations/connectors/source-airbyte/icon.svg b/airbyte-integrations/connectors/source-airbyte/icon.svg new file mode 100644 index 0000000000000..ca5ebd9339499 --- /dev/null +++ b/airbyte-integrations/connectors/source-airbyte/icon.svg @@ -0,0 +1,93 @@ + + + + + + \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-airbyte/manifest.yaml b/airbyte-integrations/connectors/source-airbyte/manifest.yaml new file mode 100644 index 0000000000000..0bb5a2a83899b --- /dev/null +++ b/airbyte-integrations/connectors/source-airbyte/manifest.yaml @@ -0,0 +1,383 @@ +version: 4.5.4 + +type: DeclarativeSource + +description: "" + +check: + type: CheckStream + stream_names: + - Jobs + +definitions: + streams: + Jobs: + type: DeclarativeStream + name: Jobs + retriever: + type: SimpleRetriever + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: '{{ response.get("next", {}) }}' + stop_condition: '{{ not response.get("next", {}) }}' + requester: + $ref: "#/definitions/base_requester" + path: https://api.airbyte.com/v1/jobs + http_method: GET + request_parameters: + limit: "100" + orderBy: updatedAt|DESC + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + stream: + $ref: "#/definitions/streams/Connections" + parent_key: connectionId + request_option: + type: RequestOption + field_name: connectionId + inject_into: request_parameter + partition_field: connectionId + primary_key: + - jobId + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/Jobs" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: lastUpdatedAt + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_datetime: + type: MinMaxDatetime + datetime: '{{ config["start_date"] }}' + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_time_option: + type: RequestOption + field_name: updatedAtStart + inject_into: request_parameter + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%SZ" + - "%Y-%m-%dT%H:%MZ" + Workspaces: + type: DeclarativeStream + name: Workspaces + retriever: + type: SimpleRetriever + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: '{{ response.get("next", {}) }}' + stop_condition: '{{ not response.get("next", {}) }}' + requester: + $ref: "#/definitions/base_requester" + path: https://api.airbyte.com/v1/workspaces + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + primary_key: + - workspaceId + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/Workspaces" + Connections: + type: DeclarativeStream + name: Connections + retriever: + type: SimpleRetriever + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: '{{ response.get("next", {}) }}' + stop_condition: '{{ not response.get("next", {}) }}' + requester: + $ref: "#/definitions/base_requester" + path: https://api.airbyte.com/v1/connections + http_method: GET + request_parameters: + limit: "100" + includeDeleted: "false" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + primary_key: + - connectionId + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/Connections" + base_requester: + type: HttpRequester + url_base: https://api.airbyte.com/v1/ + authenticator: + type: SessionTokenAuthenticator + login_requester: + type: HttpRequester + path: token + url_base: https://api.airbyte.com/v1/applications + http_method: POST + authenticator: + type: NoAuth + request_headers: {} + request_body_json: + client_id: "{{ config['client_id'] }}" + grant-type: client_credentials + client_secret: "{{ config['client_secret'] }}" + request_parameters: {} + session_token_path: + - access_token + request_authentication: + type: Bearer + +streams: + - $ref: "#/definitions/streams/Jobs" + - $ref: "#/definitions/streams/Connections" + - $ref: "#/definitions/streams/Workspaces" + +spec: + type: Spec + connection_specification: + type: object + $schema: http://json-schema.org/draft-07/schema# + required: + - start_date + - client_id + - client_secret + properties: + client_id: + type: string + order: 1 + title: client_id + start_date: + type: string + order: 0 + title: Start date + format: date-time + pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ + client_secret: + type: string + order: 2 + title: client_secret + airbyte_secret: true + additionalProperties: true + +metadata: + assist: {} + testedStreams: + Jobs: + hasRecords: true + streamHash: ffbb1a10bf9fdc313adb2d1d6f20e3c098e8f860 + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + Workspaces: + hasRecords: true + streamHash: 2429308a22a9631fb84976c3d8304e50a6fb8fd9 + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + Connections: + hasRecords: true + streamHash: d79005bbf9ee139a7fea9bcbb3e469d5faaf29ba + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + autoImportSchema: + Jobs: true + Workspaces: true + Connections: true + +schemas: + Jobs: + type: object + $schema: http://json-schema.org/schema# + required: + - jobId + - lastUpdatedAt + properties: + jobId: + type: number + status: + type: + - string + - "null" + jobType: + type: + - string + - "null" + duration: + type: + - string + - "null" + startTime: + type: + - string + - "null" + rowsSynced: + type: + - number + - "null" + bytesSynced: + type: + - number + - "null" + connectionId: + type: + - string + - "null" + lastUpdatedAt: + type: string + additionalProperties: true + Workspaces: + type: object + $schema: http://json-schema.org/schema# + required: + - workspaceId + properties: + name: + type: + - string + - "null" + workspaceId: + type: string + dataResidency: + type: + - string + - "null" + additionalProperties: true + Connections: + type: object + $schema: http://json-schema.org/schema# + required: + - connectionId + properties: + name: + type: + - string + - "null" + prefix: + type: + - string + - "null" + status: + type: + - string + - "null" + schedule: + type: + - object + - "null" + properties: + basicTiming: + type: + - string + - "null" + scheduleType: + type: + - string + - "null" + sourceId: + type: + - string + - "null" + workspaceId: + type: + - string + - "null" + connectionId: + type: string + dataResidency: + type: + - string + - "null" + destinationId: + type: + - string + - "null" + configurations: + type: + - object + - "null" + properties: + streams: + type: + - array + - "null" + items: + type: + - object + - "null" + properties: + name: + type: + - string + - "null" + syncMode: + type: + - string + - "null" + primaryKey: + type: + - array + - "null" + items: + type: + - array + - "null" + items: + type: + - string + - "null" + cursorField: + type: + - array + - "null" + items: + type: + - string + - "null" + namespaceFormat: + type: + - string + - "null" + namespaceDefinition: + type: + - string + - "null" + nonBreakingSchemaUpdatesBehavior: + type: + - string + - "null" + additionalProperties: true diff --git a/airbyte-integrations/connectors/source-airbyte/metadata.yaml b/airbyte-integrations/connectors/source-airbyte/metadata.yaml new file mode 100644 index 0000000000000..521e8f35f2096 --- /dev/null +++ b/airbyte-integrations/connectors/source-airbyte/metadata.yaml @@ -0,0 +1,35 @@ +metadataSpecVersion: "1.0" +data: + allowedHosts: + hosts: + - api.airbyte.com + registryOverrides: + oss: + enabled: true + cloud: + enabled: true + remoteRegistries: + pypi: + enabled: false + packageName: airbyte-source-airbyte + connectorBuildOptions: + baseImage: docker.io/airbyte/source-declarative-manifest:4.5.4@sha256:b07a521add11f987c63c0db68c1b57e90bec0c985f1cb6f3c5a1940cde628a70 + connectorSubtype: api + connectorType: source + definitionId: 284f6466-3004-4d83-a9b2-e4b36cbbbd41 + dockerImageTag: 0.0.1 + dockerRepository: airbyte/source-airbyte + githubIssueLabel: source-airbyte + icon: icon.svg + license: MIT + name: Airbyte + releaseDate: 2024-08-27 + releaseStage: alpha + supportLevel: community + documentationUrl: https://docs.airbyte.com/integrations/sources/airbyte + tags: + - language:manifest-only + - cdk:low-code + ab_internal: + ql: 100 + sl: 100 diff --git a/docs/integrations/sources/airbyte.md b/docs/integrations/sources/airbyte.md new file mode 100644 index 0000000000000..de7e8d5eb3a6f --- /dev/null +++ b/docs/integrations/sources/airbyte.md @@ -0,0 +1,28 @@ +# Airbyte + +This source allows you to sync up data about your Airbyte Cloud workspaces. [Take a look at this guide](https://docs.airbyte.com/using-airbyte/configuring-api-access) to setup API access tokens. +## Configuration + +| Input | Type | Description | Default Value | +|-------|------|-------------|---------------| +| `client_id` | `string` | client_id. | | +| `start_date` | `string` | Start date. | | +| `client_secret` | `string` | client_secret. | | + +## Streams +| Stream Name | Primary Key | Pagination | Supports Full Sync | Supports Incremental | +|-------------|-------------|------------|---------------------|----------------------| +| Jobs | jobId | DefaultPaginator | ✅ | ✅ | +| Connections | connectionId | DefaultPaginator | ✅ | ❌ | +| Workspaces | workspaceId | DefaultPaginator | ✅ | ❌ | + +## Changelog + +
+ Expand to review + +| Version | Date | Subject | +|------------------|------------|----------------| +| 0.0.1 | 2024-08-27 | Initial release by [@johnwasserman](https://github.com/johnwasserman) via Connector Builder| + +
\ No newline at end of file