TDL-5961 Support of custom domain

tap_github/client.py

@@ -7,6 +7,7 @@
 
 LOGGER = singer.get_logger()
 DEFAULT_SLEEP_SECONDS = 600
+DEFAULT_DOMAIN = "https://api.github.com"
 
 # Set default timeout of 300 seconds
 REQUEST_TIMEOUT = 300
@@ -137,7 +138,7 @@ class GithubClient:
     def __init__(self, config):
         self.config = config
         self.session = requests.Session()
-        self.base_url = "https://api.github.com"
+        self.base_url = config.get('api_endpoint', DEFAULT_DOMAIN)
         self.max_sleep_seconds = self.config.get('max_sleep_seconds', DEFAULT_SLEEP_SECONDS)
 
     # Return the 'timeout'
@@ -261,7 +262,7 @@ def get_all_repos(self, organizations: list):
                     repo_full_name = repo.get('full_name')
 
                     self.verify_repo_access(
-                        'https://api.github.com/repos/{}/commits'.format(repo_full_name),
+                        '{}/repos/{}/commits'.format(self.base_url, repo_full_name),
                         repo
                     )
 

tap_github/streams.py

@@ -23,28 +23,28 @@ def get_schema(catalog, stream_id):
     stream_catalog = [cat for cat in catalog if cat['tap_stream_id'] == stream_id ][0]
     return stream_catalog
 
-def get_child_full_url(child_object, repo_path, parent_id, grand_parent_id):
+def get_child_full_url(domain, child_object, repo_path, parent_id, grand_parent_id):
     """
     Build the child stream's URL based on the parent and the grandparent's ids.
     """
 
     if child_object.is_repository:
         # The `is_repository` represents that the url contains /repos and the repository name.
         child_full_url = '{}/repos/{}/{}'.format(
-            child_object.url,
+            domain,
             repo_path,
             child_object.path).format(*parent_id)
 
     elif child_object.is_organization:
         # The `is_organization` represents that the url contains the organization name.
         org = repo_path.split('/')[0]
         child_full_url = '{}/{}'.format(
-            child_object.url,
+            domain,
             child_object.path).format(org, *parent_id, *grand_parent_id)
 
     else:
         child_full_url = '{}/{}'.format(
-            child_object.url,
+            domain,
             child_object.path).format(*grand_parent_id)
     LOGGER.info(child_full_url)
 
@@ -67,12 +67,11 @@ class Stream:
     is_repository = False
     headers = None
     parent = None
-    url = "https://api.github.com"
 
     def add_fields_at_1st_level(self, rec, parent_record):
         pass
 
-    def build_url(self, repo_path, bookmark):
+    def build_url(self, base_url, repo_path, bookmark):
         """
         Build the full url with parameters and attributes.
         """
@@ -85,11 +84,11 @@ def build_url(self, repo_path, bookmark):
         if self.is_organization:
             org = repo_path.split('/')[0]
             full_url = '{}/{}'.format(
-                self.url,
+                base_url,
                 self.path).format(org)
         else:
             full_url = '{}/repos/{}/{}{}'.format(
-                self.url,
+                base_url,
                 repo_path,
                 self.path,
                 query_string)
@@ -150,7 +149,7 @@ def get_child_records(self,
         if not parent_id:
             parent_id = grand_parent_id
 
-        child_full_url = get_child_full_url(child_object, repo_path, parent_id, grand_parent_id)
+        child_full_url = get_child_full_url(client.base_url, child_object, repo_path, parent_id, grand_parent_id)
         stream_catalog = get_schema(catalog, child_object.tap_stream_id)
 
         with metrics.record_counter(child_object.tap_stream_id) as counter:
@@ -207,7 +206,7 @@ def sync_endpoint(self,
         """
 
         # build full url
-        full_url = self.build_url(repo_path, None)
+        full_url = self.build_url(client.base_url, repo_path, None)
 
         headers = {}
         if self.headers:
@@ -281,7 +280,7 @@ def sync_endpoint(self,
         max_bookmark_value = min_bookmark_value
 
         # build full url
-        full_url = self.build_url(repo_path, min_bookmark_value)
+        full_url = self.build_url(client.base_url, repo_path, min_bookmark_value)
 
         headers = {}
         if self.headers:
@@ -370,7 +369,7 @@ def sync_endpoint(self,
         bookmark_time = singer.utils.strptime_to_utc(min_bookmark_value)
 
         # Build full url
-        full_url = self.build_url(repo_path, bookmark_value)
+        full_url = self.build_url(client.base_url, repo_path, bookmark_value)
         synced_all_records = False
         stream_catalog = get_schema(catalog, self.tap_stream_id)
 

tests/unittests/test_custom_domain.py

@@ -0,0 +1,29 @@
+import unittest
+from unittest import mock
+from tap_github.client import GithubClient, DEFAULT_DOMAIN
+
+@mock.patch('tap_github.GithubClient.verify_access_for_repo', return_value = None)
+class TestCustomDomain(unittest.TestCase):
+    """
+    Test custom domain is supported in client
+    """
+
+    def test_config_without_domain(self, mock_verify_access):
+        """
+        Test if the domain is not given in the config
+        """
+        mock_config = {'repository': 'singer-io/test-repo', "access_token": ""}
+        test_client = GithubClient(mock_config)
+
+        # Verify domain in client is default
+        self.assertEqual(test_client.base_url, DEFAULT_DOMAIN)
+
+    def test_config_with_domain(self, mock_verify_access):
+        """
+        Test if the domain is given in the config
+        """
+        mock_config = {'repository': 'singer-io/test-repo', "api_endpoint": "http://CUSTOM-git.com", "access_token": ""}
+        test_client = GithubClient(mock_config)
+
+        # Verify domain in client is from config
+        self.assertEqual(test_client.base_url, mock_config["api_endpoint"])

tests/unittests/test_stream.py

@@ -16,7 +16,7 @@ def test_get_schema(self):
             {"tap_stream_id": "events"},
         ]
         expected_schema = {"tap_stream_id": "comments"}
-        
+
         # Verify returned schema is same as exected schema 
         self.assertEqual(get_schema(catalog, "comments"), expected_schema)
 
@@ -27,7 +27,7 @@ class TestGetBookmark(unittest.TestCase):
     """
 
     test_stream = Comments()
-    
+
     def test_with_out_repo_path(self):
         """
         Test if state does not contains repo path
@@ -39,7 +39,7 @@ def test_with_out_repo_path(self):
         }
         returned_bookmark = get_bookmark(state, "org/test-repo", "projects", "since", "2021-01-01T00:00:00Z")
         self.assertEqual(returned_bookmark, "2021-01-01T00:00:00Z")
-        
+
     def test_with_repo_path(self):
         """
         Test if state does contains repo path
@@ -59,14 +59,15 @@ class TestBuildUrl(unittest.TestCase):
     """
     Test `build_url` method of stream class
     """
-
+    base_url = "https://api.github.com"
+
     def test_stream_with_filter_params(self):
         """
         Test for stream with filter param
         """
         test_streams = Comments()
         expected_url = "https://api.github.com/repos/org/test-repo/issues/comments?sort=updated&direction=desc?since=2022-01-01T00:00:00Z"
-        full_url = test_streams.build_url("org/test-repo", "2022-01-01T00:00:00Z")
+        full_url = test_streams.build_url(self.base_url, "org/test-repo", "2022-01-01T00:00:00Z")
 
         # verify returned url is expected
         self.assertEqual(expected_url, full_url)
@@ -77,15 +78,15 @@ def test_stream_with_organization(self):
         """
         test_streams = Teams()
         expected_url = "https://api.github.com/orgs/org/teams"
-        full_url = test_streams.build_url("org/test-repo", "2022-01-01T00:00:00Z")
+        full_url = test_streams.build_url(self.base_url, "org/test-repo", "2022-01-01T00:00:00Z")
 
         # verify returned url is expected
         self.assertEqual(expected_url, full_url)
 
 
 class GetMinBookmark(unittest.TestCase):
     """
-    Test `get_min_bookmark` method of stream class
+    Test `get_min_bookmark` method of the stream class
     """
 
     state = {
@@ -112,7 +113,7 @@ def test_multiple_children(self):
 
         # Verify returned bookmark is expected
         self.assertEqual(bookmark, "2022-02-01T00:00:00Z")
-    
+
     def test_children_with_only_parent_selected(self):
         """
         Test for stream with multiple children and only parent is selected
@@ -123,7 +124,7 @@ def test_children_with_only_parent_selected(self):
 
         # Verify returned bookmark is expected
         self.assertEqual(bookmark, "2022-04-01T00:00:00Z")
-    
+
     def test_for_mid_child_in_stream(self):
         """
         Test for stream with multiple children and mid_child is selected
@@ -134,7 +135,7 @@ def test_for_mid_child_in_stream(self):
 
         # Verify returned bookmark is expected
         self.assertEqual(bookmark, "2022-03-01T00:00:00Z")
-    
+
     def test_nested_child_bookmark(self):
         """
         Test for stream with multiple children and nested child is selected
@@ -206,14 +207,15 @@ class TestGetChildUrl(unittest.TestCase):
     """
     Test `get_child_full_url` method of stream class
     """
+    domain = 'https://api.github.com'
 
     def test_child_stream(self):
         """
         Test for stream with one child
         """
         child_stream = ProjectColumns()
         expected_url = "https://api.github.com/projects/1309875/columns"
-        full_url = get_child_full_url(child_stream, "org1/test-repo",
+        full_url = get_child_full_url(self.domain, child_stream, "org1/test-repo",
                                                        None, (1309875,))
         self.assertEqual(expected_url, full_url)
 
@@ -223,7 +225,7 @@ def test_child_is_repository(self):
         """
         child_stream = Reviews()
         expected_url = "https://api.github.com/repos/org1/test-repo/pulls/11/reviews"
-        full_url = get_child_full_url(child_stream, "org1/test-repo",
+        full_url = get_child_full_url(self.domain, child_stream, "org1/test-repo",
                                                        (11,), None)
         self.assertEqual(expected_url, full_url)
 
@@ -233,6 +235,6 @@ def test_child_is_organization(self):
         """
         child_stream = TeamMemberships()
         expected_url = "https://api.github.com/orgs/org1/teams/dev-team/memberships/demo-user-1"
-        full_url = get_child_full_url(child_stream, "org1/test-repo",
+        full_url = get_child_full_url(self.domain, child_stream, "org1/test-repo",
                                                        ("dev-team",), ("demo-user-1",))
         self.assertEqual(expected_url, full_url)