Skip to content

Commit

Permalink
🐛Source GitLab: Fix syncing branches for the project with Repository …
Browse files Browse the repository at this point in the history 
…feature disabled (airbytehq#26422)

Add handling blocked repository for branches stream
  • Loading branch information
@tolik0 @marcosmarxm
tolik0 authored and marcosmarxm committed Jun 8, 2023
1 parent 2f2b258 commit 5894fbd
Show file tree
Hide file tree
Showing 6 changed files with 20 additions and 8 deletions.
2 changes: 1 addition & 1 deletion airbyte-integrations/connectors/source-gitlab/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,5 +13,5 @@ COPY main.py ./

ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]

LABEL io.airbyte.version=1.1.0
LABEL io.airbyte.version=1.1.1
LABEL io.airbyte.name=airbyte/source-gitlab
2 changes: 1 addition & 1 deletion ...e-integrations/connectors/source-gitlab/integration_tests/expected_records_with_ids.jsonl
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{"stream": "groups", "data": {"id": 11329647, "web_url": "https://gitlab.com/groups/new-group-airbute", "name": "New Group Airbute", "path": "new-group-airbute", "description": "", "visibility": "public", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": null, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute", "full_path": "new-group-airbute", "created_at": "2021-03-15T15:55:53.613Z", "parent_id": null, "ldap_cn": null, "ldap_access": null, "wiki_access_level": "enabled", "shared_with_groups": [], "runners_token": "GR1348941-PhosPap-Sf1UxL1g6m4", "prevent_sharing_groups_outside_hierarchy": false, "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": [{"id": 25157276, "path_with_namespace": "new-group-airbute/new-ci-test-project"}]}, "emitted_at": 1683114590269}
{"stream": "groups", "data": {"id": 61014882, "web_url": "https://gitlab.com/groups/new-group-airbute/test-subgroup-airbyte/test-private-sg", "name": "Test Private SG", "path": "test-private-sg", "description": "", "visibility": "private", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": null, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute / Test Subgroup Airbyte / Test Private SG", "full_path": "new-group-airbute/test-subgroup-airbyte/test-private-sg", "created_at": "2022-12-02T08:46:22.648Z", "parent_id": 61014863, "ldap_cn": null, "ldap_access": null, "wiki_access_level": "enabled", "shared_with_groups": [], "runners_token": "GR1348941bjUaJQy2zzar-JmNBjfq", "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": []}, "emitted_at": 1683114590768}
{"stream": "groups", "data": {"id": 61015181, "web_url": "https://gitlab.com/groups/new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1", "name": "Test Private SubSubG 1", "path": "test-private-subsubg-1", "description": "", "visibility": "private", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": null, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute / Test Public SG / Test SG Public 2 / Test Private SubSubG 1", "full_path": "new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1", "created_at": "2022-12-02T08:54:42.252Z", "parent_id": 61014943, "ldap_cn": null, "ldap_access": null, "wiki_access_level": "enabled", "shared_with_groups": [], "runners_token": "GR1348941x8xQf6K-UvnnyJ-bcut4", "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": [{"id": 41551658, "path_with_namespace": "new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1/test_project_in_nested_subgroup"}]}, "emitted_at": 1683114591226}
{"stream": "projects", "data": {"id": 25157276, "description": "", "name": "New CI Test Project ", "name_with_namespace": "New Group Airbute / New CI Test Project ", "path": "new-ci-test-project", "path_with_namespace": "new-group-airbute/new-ci-test-project", "created_at": "2021-03-15T15:08:36.498Z", "default_branch": "master", "tag_list": [], "topics": [], "ssh_url_to_repo": "[email protected]:new-group-airbute/new-ci-test-project.git", "http_url_to_repo": "https://gitlab.com/new-group-airbute/new-ci-test-project.git", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project", "readme_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/blob/master/README.md", "forks_count": 0, "avatar_url": null, "star_count": 0, "last_activity_at": "2022-12-13T09:39:47.235Z", "namespace": {"id": 11329647, "name": "New Group Airbute", "path": "new-group-airbute", "kind": "group", "full_path": "new-group-airbute", "parent_id": null, "avatar_url": null, "web_url": "https://gitlab.com/groups/new-group-airbute"}, "container_registry_image_prefix": "registry.gitlab.com/new-group-airbute/new-ci-test-project", "_links": {"self": "https://gitlab.com/api/v4/projects/25157276", "issues": "https://gitlab.com/api/v4/projects/25157276/issues", "merge_requests": "https://gitlab.com/api/v4/projects/25157276/merge_requests", "repo_branches": "https://gitlab.com/api/v4/projects/25157276/repository/branches", "labels": "https://gitlab.com/api/v4/projects/25157276/labels", "events": "https://gitlab.com/api/v4/projects/25157276/events", "members": "https://gitlab.com/api/v4/projects/25157276/members", "cluster_agents": "https://gitlab.com/api/v4/projects/25157276/cluster_agents"}, "packages_enabled": true, "empty_repo": false, "archived": false, "visibility": "private", "resolve_outdated_diff_discussions": false, "container_expiration_policy": {"cadence": "1d", "enabled": false, "keep_n": 10, "older_than": "90d", "name_regex": ".*", "name_regex_keep": null, "next_run_at": "2021-03-16T15:08:36.518Z"}, "issues_enabled": true, "merge_requests_enabled": true, "wiki_enabled": true, "jobs_enabled": true, "snippets_enabled": true, "container_registry_enabled": true, "service_desk_enabled": true, "service_desk_address": "contact-project+new-group-airbute-new-ci-test-project-25157276-issue-@incoming.gitlab.com", "can_create_merge_request_in": true, "issues_access_level": "enabled", "repository_access_level": "enabled", "merge_requests_access_level": "enabled", "forking_access_level": "enabled", "wiki_access_level": "enabled", "builds_access_level": "enabled", "snippets_access_level": "enabled", "pages_access_level": "private", "analytics_access_level": "enabled", "container_registry_access_level": "enabled", "security_and_compliance_access_level": "private", "releases_access_level": "enabled", "environments_access_level": "enabled", "feature_flags_access_level": "enabled", "infrastructure_access_level": "enabled", "monitor_access_level": "enabled", "emails_disabled": null, "shared_runners_enabled": true, "lfs_enabled": true, "creator_id": 8375961, "import_url": null, "import_type": null, "import_status": "none", "import_error": null, "open_issues_count": 31, "description_html": "", "updated_at": "2022-12-13T09:39:47.235Z", "ci_default_git_depth": 50, "ci_forward_deployment_enabled": true, "ci_job_token_scope_enabled": false, "ci_separated_caches": true, "ci_allow_fork_pipelines_to_run_in_parent_project": true, "build_git_strategy": "fetch", "keep_latest_artifact": true, "restrict_user_defined_variables": false, "runners_token": "GR1348941eMJgWDU69xyyshaNsaTZ", "runner_token_expiration_interval": null, "group_runners_enabled": true, "auto_cancel_pending_pipelines": "enabled", "build_timeout": 3600, "auto_devops_enabled": false, "auto_devops_deploy_strategy": "continuous", "ci_config_path": "", "public_jobs": true, "shared_with_groups": [], "only_allow_merge_if_pipeline_succeeds": false, "allow_merge_on_skipped_pipeline": null, "request_access_enabled": true, "only_allow_merge_if_all_discussions_are_resolved": false, "remove_source_branch_after_merge": true, "printing_merge_request_link_enabled": true, "merge_method": "merge", "squash_option": "default_off", "enforce_auth_checks_on_uploads": true, "suggestion_commit_message": null, "merge_commit_template": null, "squash_commit_template": null, "issue_branch_template": null, "statistics": {"commit_count": 3, "storage_size": 291925, "repository_size": 283115, "wiki_size": 0, "lfs_objects_size": 0, "job_artifacts_size": 8810, "pipeline_artifacts_size": 0, "packages_size": 0, "snippets_size": 0, "uploads_size": 0}, "autoclose_referenced_issues": true, "external_authorization_classification_label": "", "requirements_enabled": false, "requirements_access_level": "enabled", "security_and_compliance_enabled": true, "compliance_frameworks": [], "permissions": {"project_access": {"access_level": 40, "notification_level": 3}, "group_access": {"access_level": 50, "notification_level": 3}}}, "emitted_at": 1684137553535}
{"stream": "projects", "data": {"id": 25157276, "description": "", "name": "New CI Test Project ", "name_with_namespace": "New Group Airbute / New CI Test Project ", "path": "new-ci-test-project", "path_with_namespace": "new-group-airbute/new-ci-test-project", "created_at": "2021-03-15T15:08:36.498Z", "default_branch": "master", "tag_list": [], "topics": [], "ssh_url_to_repo": "[email protected]:new-group-airbute/new-ci-test-project.git", "http_url_to_repo": "https://gitlab.com/new-group-airbute/new-ci-test-project.git", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project", "readme_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/blob/master/README.md", "forks_count": 0, "avatar_url": null, "star_count": 0, "last_activity_at": "2022-12-13T09:39:47.235Z", "namespace": {"id": 11329647, "name": "New Group Airbute", "path": "new-group-airbute", "kind": "group", "full_path": "new-group-airbute", "parent_id": null, "avatar_url": null, "web_url": "https://gitlab.com/groups/new-group-airbute"}, "container_registry_image_prefix": "registry.gitlab.com/new-group-airbute/new-ci-test-project", "_links": {"self": "https://gitlab.com/api/v4/projects/25157276", "issues": "https://gitlab.com/api/v4/projects/25157276/issues", "merge_requests": "https://gitlab.com/api/v4/projects/25157276/merge_requests", "repo_branches": "https://gitlab.com/api/v4/projects/25157276/repository/branches", "labels": "https://gitlab.com/api/v4/projects/25157276/labels", "events": "https://gitlab.com/api/v4/projects/25157276/events", "members": "https://gitlab.com/api/v4/projects/25157276/members", "cluster_agents": "https://gitlab.com/api/v4/projects/25157276/cluster_agents"}, "packages_enabled": true, "empty_repo": false, "archived": false, "visibility": "private", "resolve_outdated_diff_discussions": false, "container_expiration_policy": {"cadence": "1d", "enabled": false, "keep_n": 10, "older_than": "90d", "name_regex": ".*", "name_regex_keep": null, "next_run_at": "2021-03-16T15:08:36.518Z"}, "issues_enabled": true, "merge_requests_enabled": true, "wiki_enabled": true, "jobs_enabled": true, "snippets_enabled": true, "container_registry_enabled": true, "service_desk_enabled": true, "service_desk_address": "contact-project+new-group-airbute-new-ci-test-project-25157276-issue-@incoming.gitlab.com", "can_create_merge_request_in": true, "issues_access_level": "enabled", "repository_access_level": "private", "merge_requests_access_level": "private", "forking_access_level": "enabled", "wiki_access_level": "enabled", "builds_access_level": "private", "snippets_access_level": "enabled", "pages_access_level": "private", "analytics_access_level": "enabled", "container_registry_access_level": "enabled", "security_and_compliance_access_level": "private", "releases_access_level": "enabled", "environments_access_level": "enabled", "feature_flags_access_level": "enabled", "infrastructure_access_level": "enabled", "monitor_access_level": "enabled", "emails_disabled": false, "shared_runners_enabled": true, "lfs_enabled": true, "creator_id": 8375961, "import_url": null, "import_type": null, "import_status": "none", "import_error": null, "open_issues_count": 31, "description_html": "", "updated_at": "2023-05-23T12:12:18.623Z", "ci_default_git_depth": 50, "ci_forward_deployment_enabled": true, "ci_job_token_scope_enabled": false, "ci_separated_caches": true, "ci_allow_fork_pipelines_to_run_in_parent_project": true, "build_git_strategy": "fetch", "keep_latest_artifact": true, "restrict_user_defined_variables": false, "runners_token": "GR1348941eMJgWDU69xyyshaNsaTZ", "runner_token_expiration_interval": null, "group_runners_enabled": true, "auto_cancel_pending_pipelines": "enabled", "build_timeout": 3600, "auto_devops_enabled": false, "auto_devops_deploy_strategy": "continuous", "ci_config_path": "", "public_jobs": true, "shared_with_groups": [], "only_allow_merge_if_pipeline_succeeds": false, "allow_merge_on_skipped_pipeline": null, "request_access_enabled": true, "only_allow_merge_if_all_discussions_are_resolved": false, "remove_source_branch_after_merge": true, "printing_merge_request_link_enabled": true, "merge_method": "merge", "squash_option": "default_off", "enforce_auth_checks_on_uploads": true, "suggestion_commit_message": null, "merge_commit_template": null, "squash_commit_template": null, "issue_branch_template": null, "statistics": {"commit_count": 3, "storage_size": 291925, "repository_size": 283115, "wiki_size": 0, "lfs_objects_size": 0, "job_artifacts_size": 8810, "pipeline_artifacts_size": 0, "packages_size": 0, "snippets_size": 0, "uploads_size": 0}, "autoclose_referenced_issues": true, "external_authorization_classification_label": "", "requirements_enabled": false, "requirements_access_level": "enabled", "security_and_compliance_enabled": true, "compliance_frameworks": [], "permissions": {"project_access": {"access_level": 40, "notification_level": 3}, "group_access": {"access_level": 50, "notification_level": 3}}}, "emitted_at": 1684137553535}
{"stream": "branches", "data": {"name": "31-fake-issue-30", "merged": true, "protected": false, "developers_can_push": false, "developers_can_merge": false, "can_push": true, "default": false, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/tree/31-fake-issue-30", "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1683114594890}
{"stream": "branches", "data": {"name": "master", "merged": false, "protected": true, "developers_can_push": false, "developers_can_merge": false, "can_push": true, "default": true, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/tree/master", "commit_id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "project_id": 25157276}, "emitted_at": 1683114594892}
{"stream": "branches", "data": {"name": "new-test-branch", "merged": true, "protected": false, "developers_can_push": false, "developers_can_merge": false, "can_push": true, "default": false, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/tree/new-test-branch", "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1683114594892}
Expand Down
2 changes: 1 addition & 1 deletion airbyte-integrations/connectors/source-gitlab/metadata.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ data:
connectorSubtype: api
connectorType: source
definitionId: 5e6175e5-68e1-4c17-bff9-56103bbb0d80
dockerImageTag: 1.1.0
dockerImageTag: 1.1.1
dockerRepository: airbyte/source-gitlab
githubIssueLabel: source-gitlab
icon: gitlab.svg
Expand Down
10 changes: 6 additions & 4 deletions airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ class GitlabStream(HttpStream, ABC):
flatten_id_keys = []
flatten_list_keys = []
per_page = 50
non_retriable_codes: List[int] = (403,)

def __init__(self, api_url: str, **kwargs):
super().__init__(**kwargs)
Expand Down Expand Up @@ -66,17 +67,17 @@ def availability_strategy(self) -> Optional["AvailabilityStrategy"]:

def should_retry(self, response: requests.Response) -> bool:
# Gitlab API returns a 403 response in case a feature is disabled in a project (pipelines/jobs for instance).
if response.status_code == 403:
if response.status_code in self.non_retriable_codes:
setattr(self, "raise_on_http_errors", False)
self.logger.warning(
f"Got 403 error when accessing URL {response.request.url}."
f"Got {response.status_code} error when accessing URL {response.request.url}."
f" Very likely the feature is disabled for this project and/or group. Please double check it, or report a bug otherwise."
)
return False
return super().should_retry(response)

def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]:
if response.status_code != 200:
if response.status_code in self.non_retriable_codes:
return
response_data = response.json()
if isinstance(response_data, dict):
Expand All @@ -86,7 +87,7 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str,
return {"page": self.page}

def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
if response.status_code == 403:
if response.status_code in self.non_retriable_codes:
return []
response_data = response.json()
if isinstance(response_data, list):
Expand Down Expand Up @@ -310,6 +311,7 @@ class Branches(GitlabChildStream):
primary_key = "name"
flatten_id_keys = ["commit"]
flatten_parent_id = True
non_retriable_codes = (403, 404)


class Commits(IncrementalGitlabChildStream):
Expand Down
11 changes: 10 additions & 1 deletion airbyte-integrations/connectors/source-gitlab/unit_tests/test_streams.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

import pytest
from airbyte_cdk.sources.streams.http.auth import NoAuth
from source_gitlab.streams import Commits, Jobs, MergeRequestCommits, MergeRequests, Pipelines, Projects, Releases, Tags
from source_gitlab.streams import Branches, Commits, Jobs, MergeRequestCommits, MergeRequests, Pipelines, Projects, Releases, Tags

auth_params = {"authenticator": NoAuth(), "api_url": "gitlab.com"}

Expand All @@ -19,6 +19,7 @@
releases = Releases(parent_stream=projects, **auth_params)
jobs = Jobs(parent_stream=pipelines, **auth_params)
merge_request_commits = MergeRequestCommits(parent_stream=merge_requests, **auth_params)
branches = Branches(parent_stream=projects, **auth_params)
commits = Commits(parent_stream=projects, repository_part=True, start_date=str(start_date), **auth_params)


Expand Down Expand Up @@ -136,3 +137,11 @@ def test_transform(requests_mock, stream, response_mocks, expected_records):
)
def test_updated_state(stream, current_state, latest_record, new_state):
assert stream.get_updated_state(current_state, latest_record) == new_state


def test_blocked_branches(requests_mock):
requests_mock.get("/api/v4/projects/p_1/branches", status_code=404)
for stream_slice in branches.stream_slices(sync_mode="full_refresh"):
records = list(branches.read_records(sync_mode="full_refresh", stream_slice=stream_slice))
assert records == []
assert requests_mock.call_count == 1
1 change: 1 addition & 0 deletions docs/integrations/sources/gitlab.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,7 @@ Gitlab has the [rate limits](https://docs.gitlab.com/ee/user/gitlab_com/index.ht

| Version | Date | Pull Request | Subject |
|:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------------------|
| 1.1.1 | 2023-05-23 | [26422](https://github.com/airbytehq/airbyte/pull/26422) | Fix error `404 Repository Not Found` when syncing project with Repository feature disabled |
| 1.1.0 | 2023-05-10 | [25948](https://github.com/airbytehq/airbyte/pull/25948) | Introduce two new fields in the `Projects` stream schema |
| 1.0.4 | 2023-04-20 | [21373](https://github.com/airbytehq/airbyte/pull/21373) | Accept api_url with or without scheme |
| 1.0.3 | 2023-02-14 | [22992](https://github.com/airbytehq/airbyte/pull/22992) | Specified date formatting in specification |
Expand Down

0 comments on commit 5894fbd

Please sign in to comment.