Merge remote-tracking branch 'origin/master' into flash1293/refactor-…

…connectorview
airbytehq · Jan 20, 2023 · d63b056 · d63b056
2 parents dbbfc23 + 1ad0e96
commit d63b056
Show file tree

Hide file tree

Showing 53 changed files with 1,633 additions and 2,823 deletions.
diff --git a/.github/workflows/connector_integration_tests.yml b/.github/workflows/connector_integration_tests.yml
@@ -29,3 +29,22 @@ jobs:
         run: python ./tools/bin/ci_integration_workflow_launcher.py base-normalization source-acceptance-test source:beta source:GA destination:beta destination:GA
         env:
           GITHUB_TOKEN: ${{ secrets.SLASH_COMMAND_PAT }}
+  launch_integration_tests_alpha_only:
+    runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/master'
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v3
+      - name: Install Java
+        uses: actions/setup-java@v3
+        with:
+          distribution: "zulu"
+          java-version: "17"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install PyYAML requests
+      - name: Launch Integration Tests (Alpha connectors)
+        run: python ./tools/bin/ci_integration_workflow_launcher.py source:alpha destination:alpha
+        env:
+          GITHUB_TOKEN: ${{ secrets.SLASH_COMMAND_PAT }}
diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml
@@ -285,7 +285,7 @@ jobs:
         if: always()
         with:
           junit_files: "/actions-runner/_work/airbyte/airbyte/*/build/test-results/*/*.xml\n/actions-runner/_work/airbyte/airbyte/*/*/build/test-results/*/*.xml"
-          comment_mode: off
+          comment_mode: failures
           json_file: connectors_base_results.json
           json_test_case_results: true
           check_name: "Connectors Base Test Results"
@@ -318,6 +318,7 @@ jobs:
           # Specify top-level and second-level modules. Note there cannot be a space between the comma.
           path: "/actions-runner/_work/airbyte/airbyte/*/build/test-results/*/*.xml,/actions-runner/_work/airbyte/airbyte/*/*/build/test-results/*/*.xml"
           reporter: java-junit
+          fail-on-error: 'false'
 
   # In case of self-hosted EC2 errors, remove this block.
   stop-connectors-base-build-runner:
@@ -806,7 +807,7 @@ jobs:
         if: always()
         with:
           junit_files: "/actions-runner/_work/airbyte/airbyte/*/build/test-results/*/*.xml\n/actions-runner/_work/airbyte/airbyte/*/*/build/test-results/*/*.xml"
-          comment_mode: off
+          comment_mode: failures
           json_file: platform_results.json
           json_test_case_results: true
           check_name: "Platform Test Results"
@@ -839,6 +840,7 @@ jobs:
           # Specify top-level and second-level modules. Note there cannot be a space between the comma.
           path: "/actions-runner/_work/airbyte/airbyte/*/build/test-results/*/*.xml,/actions-runner/_work/airbyte/airbyte/*/*/build/test-results/*/*.xml"
           reporter: java-junit
+          fail-on-error: 'false'
 
       - name: Upload test results to BuildPulse for flaky test detection
         if: "!cancelled()" # Run this step even when the tests fail. Skip if the workflow is cancelled.
@@ -1026,7 +1028,7 @@ jobs:
         if: always()
         with:
           junit_files: "/actions-runner/_work/airbyte/airbyte/*/build/test-results/*/*.xml\n/actions-runner/_work/airbyte/airbyte/*/*/build/test-results/*/*.xml"
-          comment_mode: off
+          comment_mode: failures
           json_file: kube_results.json
           json_test_case_results: true
           check_name: "Kube Test Results"
@@ -1058,6 +1060,7 @@ jobs:
           name: Platform Kubernetes E2E Test Report
           path: "/actions-runner/_work/airbyte/airbyte/*/build/test-results/*/*.xml"
           reporter: java-junit
+          fail-on-error: 'false'
 
       - name: Upload test results to BuildPulse for flaky test detection
         if: "!cancelled()" # Run this step even when the tests fail. Skip if the workflow is cancelled.
@@ -1285,6 +1288,7 @@ jobs:
           name: Platform Helm E2E Test Report
           path: "./*/build/test-results/*/*.xml"
           reporter: java-junit
+          fail-on-error: 'false'
 
       - uses: actions/upload-artifact@v2
         if: failure()

diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml
@@ -85,7 +85,7 @@
 - name: Clickhouse
   destinationDefinitionId: ce0d828e-1dc4-496c-b122-2da42e637e48
   dockerRepository: airbyte/destination-clickhouse
-  dockerImageTag: 0.2.1
+  dockerImageTag: 0.2.2
   documentationUrl: https://docs.airbyte.com/integrations/destinations/clickhouse
   icon: clickhouse.svg
   releaseStage: alpha

diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml
@@ -1133,7 +1133,7 @@
     supported_destination_sync_modes:
     - "overwrite"
     - "append"
-- dockerImage: "airbyte/destination-clickhouse:0.2.1"
+- dockerImage: "airbyte/destination-clickhouse:0.2.2"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/destinations/clickhouse"
     connectionSpecification:

diff --git a/airbyte-featureflag/src/test/kotlin/ClientTest.kt b/airbyte-featureflag/src/test/kotlin/ClientTest.kt
@@ -25,6 +25,7 @@ import java.nio.file.Path
 import java.util.concurrent.TimeUnit
 import kotlin.io.path.createTempFile
 import kotlin.io.path.writeText
+import kotlin.test.Ignore
 import kotlin.test.assertFailsWith
 import kotlin.test.assertFalse
 import kotlin.test.assertTrue
@@ -48,7 +49,16 @@ class ConfigFileClient {
         }
     }
 
+    /**
+     * Ignore this test for now as it is unreliable in a unit-test scenario due to the
+     * unpredictable nature of knowing when the WatchService (inside the ConfigFileClient) will
+     * actually see the changed file.  Currently, this test sleeps for a few seconds, which works 90%
+     * of the time, however there has been instances where it has taken over 20 seconds.
+     *
+     * TODO: move this to a different test suite
+     */
     @Test
+    @Ignore
     fun `verify platform reload capabilities`() {
         val contents0 = """flags:
             |  - name: reload-test-true

diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile
@@ -16,5 +16,5 @@ ENV APPLICATION destination-clickhouse-strict-encrypt
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.2.1
+LABEL io.airbyte.version=0.2.2
 LABEL io.airbyte.name=airbyte/destination-clickhouse-strict-encrypt
diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle
@@ -5,7 +5,7 @@ plugins {
 }
 
 application {
-    mainClass = 'io.airbyte.integrations.destination.clickhouse.ClickhouseDestination'
+    mainClass = 'io.airbyte.integrations.destination.clickhouse.ClickhouseDestinationStrictEncrypt'
     applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0']
 }
 

diff --git a/airbyte-integrations/connectors/destination-clickhouse/Dockerfile b/airbyte-integrations/connectors/destination-clickhouse/Dockerfile
@@ -16,5 +16,5 @@ ENV APPLICATION destination-clickhouse
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.2.1
+LABEL io.airbyte.version=0.2.2
 LABEL io.airbyte.name=airbyte/destination-clickhouse
diff --git a/airbyte-integrations/connectors/source-freshdesk/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-freshdesk/integration_tests/expected_records.jsonl
@@ -1,5 +1,5 @@
 {"stream": "agents", "data": {"available": false, "occasional": true, "id": 67013469930, "ticket_scope": 1, "created_at": "2020-10-22T02:37:15Z", "updated_at": "2020-10-22T02:37:15Z", "last_active_at": null, "available_since": null, "type": "support_agent", "contact": {"active": false, "email": "[email protected]", "job_title": null, "language": "en", "last_login_at": null, "mobile": null, "name": "Customer Service", "phone": null, "time_zone": "Magadan", "created_at": "2020-10-22T02:37:15Z", "updated_at": "2020-10-22T02:37:15Z"}, "signature": null}, "emitted_at": 1672876286668}
-{"stream": "agents", "data": {"available": false, "occasional": false, "id": 67013469919, "ticket_scope": 1, "created_at": "2020-10-22T02:37:13Z", "updated_at": "2022-12-29T13:41:44Z", "last_active_at": "2022-12-29T13:41:44Z", "available_since": null, "type": "support_agent", "contact": {"active": true, "email": "[email protected]", "job_title": null, "language": "en", "last_login_at": "2022-12-12T22:59:08Z", "mobile": null, "name": "Team Airbyte", "phone": null, "time_zone": "Magadan", "created_at": "2020-10-22T02:37:13Z", "updated_at": "2020-10-22T03:30:13Z"}, "signature": null}, "emitted_at": 1672876286669}
+{"stream": "agents", "data": {"available": false, "occasional": false, "id": 67013469919, "ticket_scope": 1, "created_at": "2020-10-22T02:37:13Z", "updated_at": "2023-01-16T15:57:48Z", "last_active_at": "2023-01-16T15:57:48Z", "available_since": null, "type": "support_agent", "contact": {"active": true, "email": "[email protected]", "job_title": null, "language": "en", "last_login_at": "2022-12-29T13:41:44Z", "mobile": null, "name": "Team Airbyte", "phone": null, "time_zone": "Magadan", "created_at": "2020-10-22T02:37:13Z", "updated_at": "2020-10-22T03:30:13Z"}, "signature": null}, "emitted_at": 1672876286669}
 {"stream": "agents", "data": {"available": false, "occasional": true, "id": 67021120644, "ticket_scope": 1, "created_at": "2021-02-11T22:20:50Z", "updated_at": "2021-12-01T00:09:07Z", "last_active_at": null, "available_since": null, "type": "support_agent", "contact": {"active": false, "email": "[email protected]", "job_title": null, "language": "en", "last_login_at": null, "mobile": null, "name": "Test Agent 1", "phone": null, "time_zone": "Magadan", "created_at": "2021-02-11T22:20:50Z", "updated_at": "2021-02-11T22:20:50Z"}, "signature": "<div dir=\"ltr\"><p><br></p>\n</div>"}, "emitted_at": 1672876286670}
 {"stream": "agents", "data": {"available": false, "occasional": true, "id": 67021120892, "ticket_scope": 1, "created_at": "2021-02-11T22:21:44Z", "updated_at": "2021-12-01T00:09:07Z", "last_active_at": null, "available_since": null, "type": "support_agent", "contact": {"active": false, "email": "[email protected]", "job_title": null, "language": "en", "last_login_at": null, "mobile": null, "name": "Test Agent 10", "phone": null, "time_zone": "Magadan", "created_at": "2021-02-11T22:21:44Z", "updated_at": "2021-02-11T22:21:44Z"}, "signature": "<div dir=\"ltr\"><p><br></p>\n</div>"}, "emitted_at": 1672876286671}
 {"stream": "agents", "data": {"available": false, "occasional": true, "id": 67021120894, "ticket_scope": 1, "created_at": "2021-02-11T22:21:45Z", "updated_at": "2021-12-01T00:09:07Z", "last_active_at": null, "available_since": null, "type": "support_agent", "contact": {"active": false, "email": "[email protected]", "job_title": null, "language": "en", "last_login_at": null, "mobile": null, "name": "Test Agent 11", "phone": null, "time_zone": "Magadan", "created_at": "2021-02-11T22:21:45Z", "updated_at": "2021-02-11T22:21:45Z"}, "signature": "<div dir=\"ltr\"><p><br></p>\n</div>"}, "emitted_at": 1672876286671}

diff --git a/airbyte-integrations/connectors/source-github/Dockerfile b/airbyte-integrations/connectors/source-github/Dockerfile
@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.3.11
+LABEL io.airbyte.version=0.3.12
 LABEL io.airbyte.name=airbyte/source-github
diff --git a/airbyte-integrations/connectors/source-github/source_github/streams.py b/airbyte-integrations/connectors/source-github/source_github/streams.py
@@ -171,6 +171,8 @@ def read_records(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> Iter
                 )
             elif e.response.status_code == requests.codes.SERVER_ERROR and isinstance(self, WorkflowRuns):
                 error_msg = f"Syncing `{self.name}` stream isn't available for repository `{stream_slice['repository']}`."
+            elif e.response.status_code == requests.codes.BAD_GATEWAY:
+                error_msg = f"Stream {self.name} temporary failed. Try to re-run sync later"
             else:
                 # most probably here we're facing a 500 server error and a risk to get a non-json response, so lets output response.text
                 self.logger.error(f"Undefined error while reading records: {e.response.text}")

diff --git a/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py b/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py
@@ -189,6 +189,26 @@ def test_stream_teams_404():
     assert responses.calls[0].request.url == "https://api.github.com/orgs/org_name/teams?per_page=100"
 
 
+@responses.activate
+@patch("time.sleep")
+def test_stream_teams_502(sleep_mock):
+    organization_args = {"organizations": ["org_name"]}
+    stream = Teams(**organization_args)
+
+    url = "https://api.github.com/orgs/org_name/teams"
+    responses.add(
+        method="GET",
+        url=url,
+        status=requests.codes.BAD_GATEWAY,
+        json={"message": "Server Error"},
+    )
+
+    assert list(read_full_refresh(stream)) == []
+    assert len(responses.calls) == 6
+    # Check whether url is the same for all response.calls
+    assert set(call.request.url for call in responses.calls).symmetric_difference({f"{url}?per_page=100"}) == set()
+
+
 @responses.activate
 def test_stream_organizations_read():
     organization_args = {"organizations": ["org1", "org2"]}