From 60db1f37e43190cc35a4f78e86d6a492a8284de8 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Thu, 17 Oct 2024 09:27:03 +0200 Subject: [PATCH] feat!: support data connectors (#1991) Add support for mounting data connectors (instead of `storages_v2`) with Renku 2.0 sessions. --- renku_notebooks/api/classes/data_service.py | 66 +++++++++++++++------ renku_notebooks/api/notebooks.py | 2 +- 2 files changed, 50 insertions(+), 18 deletions(-) diff --git a/renku_notebooks/api/classes/data_service.py b/renku_notebooks/api/classes/data_service.py index 48736d5c..5ac1bd8b 100644 --- a/renku_notebooks/api/classes/data_service.py +++ b/renku_notebooks/api/classes/data_service.py @@ -44,6 +44,8 @@ def get_storage_by_id(self, user: User, endpoint: str, storage_id: str) -> Cloud } endpoint = endpoint.strip("/") request_url = self.data_service_url + f"/{endpoint}/{storage_id}" + if endpoint == "data_connectors": + return self._get_data_connector_by_id(user, storage_id, request_url, headers) current_app.logger.info(f"getting storage info by id: {request_url}") res = requests.get(request_url, headers=headers) if res.status_code == 404: @@ -66,6 +68,48 @@ def get_storage_by_id(self, user: User, endpoint: str, storage_id: str) -> Cloud secrets=secrets, ) + def _get_data_connector_by_id( + self, user: User, data_connector_id: str, request_url: str, headers: dict[str, Any] | None + ) -> CloudStorageConfig: + """Returns the storage configuration for a data connector.""" + current_app.logger.info(f"getting data connector info by id: {request_url}") + res = requests.get(request_url, headers=headers) + if res.status_code == 404: + raise MissingResourceError(message=f"Couldn't find data connector with id {data_connector_id}") + if res.status_code == 401 or res.status_code == 403: + raise AuthenticationError("User is not authorized to access this data connector.") + if res.status_code != 200: + raise IntermittentError( + message="The data service sent an unexpected response, please try again later", + ) + data_connector = res.json() + storage = data_connector["storage"] + secrets = {} + # Get secrets only for authenticated users + if user is not None and headers is not None: + request_url_secrets = request_url + "/secrets" + res = requests.get(request_url_secrets, headers=headers) + if res.status_code == 404: + raise MissingResourceError( + message=f"Couldn't find secrets for data connector with id {data_connector_id}" + ) + if res.status_code == 401 or res.status_code == 403: + raise AuthenticationError("User is not authorized to access this data connector.") + if res.status_code != 200: + raise IntermittentError( + message="The data service sent an unexpected response, please try again later", + ) + response = res.json() + secrets = {s["secret_id"]: s["name"] for s in response} + return CloudStorageConfig( + config=storage["configuration"], + source_path=storage["source_path"], + target_path=storage["target_path"], + readonly=storage.get("readonly", True), + name=data_connector["name"], + secrets=secrets, + ) + def validate_storage_configuration(self, configuration: dict[str, Any], source_path: str) -> None: res = requests.post(self.data_service_url + "/storage_schema/validate", json=configuration) if res.status_code == 422: @@ -269,9 +313,7 @@ def get_providers(self, user: User) -> list[GitProvider]: providers_list = list(providers.values()) # Insert the internal GitLab as the first provider - internal_gitlab_access_token_url = urljoin( - self.renku_url, "/api/auth/gitlab/exchange" - ) + internal_gitlab_access_token_url = urljoin(self.renku_url, "/api/auth/gitlab/exchange") providers_list.insert( 0, GitProvider( @@ -283,33 +325,23 @@ def get_providers(self, user: User) -> list[GitProvider]: ) return providers_list - def get_oauth2_connections( - self, user: User | None = None - ) -> list[OAuth2Connection]: + def get_oauth2_connections(self, user: User | None = None) -> list[OAuth2Connection]: if user is None or user.access_token is None: return [] request_url = f"{self.service_url}/oauth2/connections" headers = {"Authorization": f"bearer {user.access_token}"} res = requests.get(request_url, headers=headers) if res.status_code != 200: - raise IntermittentError( - message="The data service sent an unexpected response, please try again later" - ) + raise IntermittentError(message="The data service sent an unexpected response, please try again later") connections = res.json() - connections = [ - OAuth2Connection.from_dict(c) - for c in connections - if c["status"] == "connected" - ] + connections = [OAuth2Connection.from_dict(c) for c in connections if c["status"] == "connected"] return connections def get_oauth2_provider(self, provider_id: str) -> OAuth2Provider: request_url = f"{self.service_url}/oauth2/providers/{provider_id}" res = requests.get(request_url) if res.status_code != 200: - raise IntermittentError( - message="The data service sent an unexpected response, please try again later" - ) + raise IntermittentError(message="The data service sent an unexpected response, please try again later") provider = res.json() return OAuth2Provider.from_dict(provider) diff --git a/renku_notebooks/api/notebooks.py b/renku_notebooks/api/notebooks.py index e53c08f2..b3ef6e8b 100644 --- a/renku_notebooks/api/notebooks.py +++ b/renku_notebooks/api/notebooks.py @@ -270,7 +270,7 @@ def renku_2_launch_notebook_helper( default_url=default_url, lfs_auto_fetch=lfs_auto_fetch, cloudstorage=cloudstorage, - cloudstorage_endpoint="storages_v2", + cloudstorage_endpoint="data_connectors", server_options=server_options, project_id=project_id, launcher_id=launcher_id,