🎉 Source greenhouse: added identification of accessible streams for API keys with limited permissions

.github/workflows/publish-command.yml

@@ -112,6 +112,7 @@ jobs:
           GOOGLE_SHEETS_TESTS_CREDS: ${{ secrets.GOOGLE_SHEETS_TESTS_CREDS }}
           GOOGLE_WORKSPACE_ADMIN_REPORTS_TEST_CREDS: ${{ secrets.GOOGLE_WORKSPACE_ADMIN_REPORTS_TEST_CREDS }}
           GREENHOUSE_TEST_CREDS: ${{ secrets.GREENHOUSE_TEST_CREDS }}
+          GREENHOUSE_TEST_CREDS_LIMITED: ${{ secrets.GREENHOUSE_TEST_CREDS_LIMITED }}
           HARVEST_INTEGRATION_TESTS_CREDS: ${{ secrets.HARVEST_INTEGRATION_TESTS_CREDS }}
           HUBSPOT_INTEGRATION_TESTS_CREDS: ${{ secrets.HUBSPOT_INTEGRATION_TESTS_CREDS }}
           INSTAGRAM_INTEGRATION_TESTS_CREDS: ${{ secrets.INSTAGRAM_INTEGRATION_TESTS_CREDS }}

.github/workflows/test-command.yml

@@ -112,6 +112,7 @@ jobs:
           GOOGLE_SHEETS_TESTS_CREDS: ${{ secrets.GOOGLE_SHEETS_TESTS_CREDS }}
           GOOGLE_WORKSPACE_ADMIN_REPORTS_TEST_CREDS: ${{ secrets.GOOGLE_WORKSPACE_ADMIN_REPORTS_TEST_CREDS }}
           GREENHOUSE_TEST_CREDS: ${{ secrets.GREENHOUSE_TEST_CREDS }}
+          GREENHOUSE_TEST_CREDS_LIMITED: ${{ secrets.GREENHOUSE_TEST_CREDS_LIMITED }}
           HARVEST_INTEGRATION_TESTS_CREDS: ${{ secrets.HARVEST_INTEGRATION_TESTS_CREDS }}
           HUBSPOT_INTEGRATION_TESTS_CREDS: ${{ secrets.HUBSPOT_INTEGRATION_TESTS_CREDS }}
           INSTAGRAM_INTEGRATION_TESTS_CREDS: ${{ secrets.INSTAGRAM_INTEGRATION_TESTS_CREDS }}

airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/59f1e50a-331f-4f09-b3e8-2e8d4d355f44.json

@@ -2,7 +2,7 @@
   "sourceDefinitionId": "59f1e50a-331f-4f09-b3e8-2e8d4d355f44",
   "name": "Greenhouse",
   "dockerRepository": "airbyte/source-greenhouse",
-  "dockerImageTag": "0.2.3",
+  "dockerImageTag": "0.2.4",
   "documentationUrl": "https://docs.airbyte.io/integrations/sources/greenhouse",
   "icon": "greenhouse.svg"
 }

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -182,7 +182,7 @@
 - sourceDefinitionId: 59f1e50a-331f-4f09-b3e8-2e8d4d355f44
   name: Greenhouse
   dockerRepository: airbyte/source-greenhouse
-  dockerImageTag: 0.2.3
+  dockerImageTag: 0.2.4
   documentationUrl: https://docs.airbyte.io/integrations/sources/greenhouse
   icon: greenhouse.svg
 - sourceDefinitionId: 40d24d0f-b8f9-4fe0-9e6c-b06c0f3f45e4

airbyte-integrations/connectors/source-greenhouse/Dockerfile

@@ -14,5 +14,5 @@ RUN pip install .
 
 ENV AIRBYTE_ENTRYPOINT "/airbyte/base.sh"
 
-LABEL io.airbyte.version=0.2.3
+LABEL io.airbyte.version=0.2.4
 LABEL io.airbyte.name=airbyte/source-greenhouse

airbyte-integrations/connectors/source-greenhouse/acceptance-test-config.yml

@@ -0,0 +1,26 @@
+# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference)
+# for more information about how to configure these tests
+connector_image: airbyte/source-greenhouse:dev
+tests:
+  spec:
+    - spec_path: "source_greenhouse/spec.json"
+  connection:
+    - config_path: "secrets/config.json"
+      status: "succeed"
+    - config_path: "secrets/config_users_only.json"
+      status: "succeed"
+    - config_path: "integration_tests/config_invalid.json"
+      status: "failed"
+  discovery:
+    - config_path: "secrets/config.json"
+    - config_path: "secrets/config_users_only.json"
+  basic_read:
+    - config_path: "secrets/config.json"
+      configured_catalog_path: "integration_tests/configured_catalog.json"
+    - config_path: "secrets/config.json"
+      configured_catalog_path: "integration_tests/configured_catalog_users_only.json"
+  full_refresh:
+    - config_path: "secrets/config.json"
+      configured_catalog_path: "integration_tests/configured_catalog.json"
+    - config_path: "secrets/config_users_only.json"
+      configured_catalog_path: "integration_tests/configured_catalog_users_only.json"

airbyte-integrations/connectors/source-greenhouse/acceptance-test-docker.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env sh
+
+# Build latest connector image
+docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2)
+
+# Pull latest acctest image
+docker pull airbyte/source-acceptance-test:latest
+
+# Run
+docker run --rm -it \
+    -v /var/run/docker.sock:/var/run/docker.sock \
+    -v /tmp:/tmp \
+    -v $(pwd):/test_input \
+    airbyte/source-acceptance-test \
+    --acceptance-test-config /test_input
+

airbyte-integrations/connectors/source-greenhouse/unit_tests/test_client.py → airbyte-integrations/connectors/source-greenhouse/integration_tests/acceptance.py

@@ -24,25 +24,11 @@
 
 
 import pytest
-from grnhse.exceptions import EndpointNotFound, HTTPError
-from source_greenhouse.client import Client
 
+pytest_plugins = ("source_acceptance_test.plugin",)
 
-def test__heal_check_with_wrong_api_key():
-    client = Client(api_key="wrong_key")
-    alive, error = client.health_check()
 
-    assert not alive
-    assert error == '401 {"message":"Invalid Basic Auth credentials"}'
-
-
-def test__custom_fields_with_wrong_api_key():
-    client = Client(api_key="wrong_key")
-    with pytest.raises(HTTPError, match='401 {"message":"Invalid Basic Auth credentials"}'):
-        list(client.list("custom_fields"))
-
-
-def test_client_wrong_endpoint():
-    client = Client(api_key="wrong_key")
-    with pytest.raises(EndpointNotFound, match="unknown_endpoint"):
-        next(client.list("unknown_endpoint"))
+@pytest.fixture(scope="session", autouse=True)
+def connector_setup():
+    """ This fixture is a placeholder for external resources that acceptance test might require."""
+    yield

airbyte-integrations/connectors/source-greenhouse/integration_tests/config_invalid.json

@@ -0,0 +1,3 @@
+{
+  "api_key": "bla"
+}

airbyte-integrations/connectors/source-greenhouse/integration_tests/configured_catalog.json

@@ -0,0 +1,114 @@
+{
+  "streams": [
+    {
+      "stream": {
+        "name": "applications",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_cursor": false
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "candidates",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_cursor": false
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "close_reasons",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_cursor": false
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "degrees",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_cursor": false
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "departments",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_cursor": false
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "job_posts",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_cursor": false
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "jobs",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_cursor": false
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "offers",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_cursor": false
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "scorecards",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_cursor": false
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "users",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_cursor": false
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "custom_fields",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_cursor": false
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    }
+  ]
+}

airbyte-integrations/connectors/source-greenhouse/integration_tests/configured_catalog_users_only.json

@@ -0,0 +1,14 @@
+{
+  "streams": [
+    {
+      "stream": {
+        "name": "users",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_cursor": false
+       },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    }
+  ]
+}

airbyte-integrations/connectors/source-greenhouse/requirements.txt

@@ -1,4 +1,5 @@
 # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies.
 -e ../../bases/airbyte-protocol
 -e ../../bases/base-python
+-e ../../bases/source-acceptance-test
 -e .

airbyte-integrations/connectors/source-greenhouse/setup.py

@@ -25,12 +25,19 @@
 
 from setuptools import find_packages, setup
 
+TEST_REQUIREMENTS = [
+    "pytest~=6.1",
+    "source-acceptance-test",
+]
 setup(
     name="source_greenhouse",
     description="Source implementation for Greenhouse.",
     author="Airbyte",
     author_email="[email protected]",
     packages=find_packages(),
-    install_requires=["airbyte-protocol", "base-python", "six==1.15.0", "grnhse-api==0.1.1", "pytest==6.1.2"],
+    install_requires=["airbyte-protocol", "base-python", "six==1.15.0", "grnhse-api==0.1.1"],
     package_data={"": ["*.json", "schemas/*.json"]},
+    extras_require={
+        "tests": TEST_REQUIREMENTS,
+    },
 )

airbyte-integrations/connectors/source-greenhouse/source_greenhouse/client.py

@@ -24,9 +24,10 @@
 
 
 from functools import partial
-from typing import Mapping, Tuple
+from typing import Generator, List, Mapping, Tuple
 
-from base_python import BaseClient
+from airbyte_protocol import AirbyteStream
+from base_python import AirbyteLogger, BaseClient
 from grnhse import Harvest
 from grnhse.exceptions import HTTPError
 
@@ -67,16 +68,42 @@ def list(self, name, **kwargs):
     def _enumerate_methods(self) -> Mapping[str, callable]:
         return {entity: partial(self.list, name=entity) for entity in self.ENTITIES}
 
+    def get_accessible_endpoints(self) -> List[str]:
+        """Try to read each supported endpoint and return accessible stream names"""
+        logger = AirbyteLogger()
+        accessible_endpoints = []
+        for entity in self.ENTITIES:
+            try:
+                getattr(self._client, entity).get()
+                accessible_endpoints.append(entity)
+            except HTTPError as error:
+                logger.warn(f"Endpoint '{entity}' error: {str(error)}")
+                if "This API Key does not have permission for this endpoint" not in str(error):
+                    raise error
+        logger.info(f"API key has access to {len(accessible_endpoints)} endpoints: {accessible_endpoints}")
+        return accessible_endpoints
+
     def health_check(self) -> Tuple[bool, str]:
         alive = True
         error_msg = None
-
         try:
-            # because there is no good candidate to try our connection
-            # we use users endpoint as potentially smallest dataset
-            self._client.users.get()
+            accessible_endpoints = self.get_accessible_endpoints()
+            if not accessible_endpoints:
+                alive = False
+                error_msg = (
+                    "Your API Key does not have permission for any existing endpoints. Please grant read permissions for required streams/endpoints"
+                )
+
         except HTTPError as error:
             alive = False
             error_msg = str(error)
 
         return alive, error_msg
+
+    @property
+    def streams(self) -> Generator[AirbyteStream, None, None]:
+        """Process accessible streams only"""
+        accessible_endpoints = self.get_accessible_endpoints()
+        for stream in super().streams:
+            if stream.name in accessible_endpoints:
+                yield stream

airbyte-integrations/connectors/source-greenhouse/source_greenhouse/schemas/applications.json

@@ -41,7 +41,7 @@
           "type": ["null", "string"]
         },
         "prospect_owner": {
-          "type": "object",
+          "type": ["null", "object"],
           "properties": {
             "name": {
               "type": "string"
@@ -69,7 +69,15 @@
       "type": "integer"
     },
     "current_stage": {
-      "type": ["null", "string"]
+      "type": ["null", "object"],
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "id": {
+          "type": "integer"
+        }
+      }
     },
     "credited_to": {
       "type": "object",

airbyte-integrations/connectors/source-greenhouse/source_greenhouse/schemas/candidates.json

@@ -109,7 +109,7 @@
                 "type": ["null", "string"]
               },
               "prospect_owner": {
-                "type": "object",
+                "type": ["null", "object"],
                 "properties": {
                   "name": {
                     "type": "string"
@@ -137,7 +137,15 @@
             "type": "integer"
           },
           "current_stage": {
-            "type": ["null", "string"]
+            "type": ["null", "object"],
+            "properties": {
+              "name": {
+                "type": "string"
+              },
+              "id": {
+                "type": "integer"
+              }
+            }
           },
           "credited_to": {
             "type": "object",