feat(sidecar): symlink cloudstorage into project (#1230)

.github/workflows/acceptance-tests.yml

@@ -28,6 +28,7 @@ jobs:
       renku-ui: ${{ steps.deploy-comment.outputs.renku-ui}}
       test-enabled: ${{ steps.deploy-comment.outputs.test-enabled}}
       persist: ${{ steps.deploy-comment.outputs.persist}}
+      extra-values: ${{ steps.deploy-comment.outputs.extra-values}}
     steps:
       - id: deploy-comment
         uses: SwissDataScienceCenter/renku-actions/[email protected]
@@ -96,7 +97,7 @@ jobs:
           persist: "${{ needs.check-deploy.outputs.persist }}"
           s3-results-access-key: ${{ secrets.ACCEPTANCE_TESTS_BUCKET_ACCESS_KEY }}
           s3-results-secret-key: ${{ secrets.ACCEPTANCE_TESTS_BUCKET_SECRET_KEY }}
-          test-timeout-mins: "80"
+          test-timeout-mins: "120"
 
   cleanup:
     if: github.event.action == 'closed'

git_services/git_services/init/clone.py

@@ -15,7 +15,9 @@
         config.git_url,
         config.repository_url,
         config.user,
-        config.lfs_auto_fetch == "1",
+        config.lfs_auto_fetch,
         config.mount_path,
     )
-    git_cloner.run(config.git_autosave == "1", config.branch, config.commit_sha)
+    git_cloner.run(
+        config.git_autosave, config.branch, config.commit_sha, config.s3_mount
+    )

git_services/git_services/init/cloner.py

@@ -1,4 +1,5 @@
 from contextlib import contextmanager
+import os
 import requests
 from datetime import datetime, timedelta
 import logging
@@ -29,11 +30,11 @@ def __init__(
         logging.basicConfig(level=logging.INFO)
         self.git_url = git_url
         self.repo_url = repo_url
-        repo_directory = Path(repo_directory)
-        if not repo_directory.exists():
-            logging.info(f"{repo_directory} does not exist, creating it.")
-            repo_directory.mkdir(parents=True, exist_ok=True)
-        self.cli = GitCLI(repo_directory)
+        self.repo_directory = Path(repo_directory)
+        if not self.repo_directory.exists():
+            logging.info(f"{self.repo_directory} does not exist, creating it.")
+            self.repo_directory.mkdir(parents=True, exist_ok=True)
+        self.cli = GitCLI(self.repo_directory)
         self.user = user
         self.git_host = urlparse(git_url).netloc
         self.lfs_auto_fetch = lfs_auto_fetch
@@ -73,6 +74,20 @@ def _setup_proxy(self):
         self.cli.git_config(f"http.proxy {self.proxy_url}")
         self.cli.git_config("http.sslVerify false")
 
+    def _setup_cloudstorage_symlink(self, mount_folder):
+        """Setup a symlink to cloudstorage directory."""
+        logging.info("Setting up cloudstorage symlink")
+        link_path = self.repo_directory / "cloudstorage"
+        if link_path.exists():
+            logging.warn(f"Cloud storage path in repo already exists: {link_path}")
+            return
+        os.symlink(mount_folder, link_path, target_is_directory=True)
+
+        with open(
+            self.repo_directory / ".git" / "info" / "exclude", "a"
+        ) as exclude_file:
+            exclude_file.write("\n/cloudstorage\n")
+
     @contextmanager
     def _temp_plaintext_credentials(self):
         try:
@@ -166,7 +181,7 @@ def _repo_exists(self):
             return False
         return res.lower().strip() == "true"
 
-    def run(self, recover_autosave, session_branch, root_commit_sha):
+    def run(self, recover_autosave, session_branch, root_commit_sha, s3_mount):
         logging.info("Checking if the repo already exists.")
         if self._repo_exists():
             logging.info("The repo already exists - exiting.")
@@ -183,3 +198,5 @@ def run(self, recover_autosave, session_branch, root_commit_sha):
                 else:
                     self._recover_autosave(autosave_branch)
         self._setup_proxy()
+        if s3_mount:
+            self._setup_cloudstorage_symlink(s3_mount)

git_services/git_services/init/config.py

@@ -26,15 +26,14 @@ class Config:
     git_autosave: str = "0"
     lfs_auto_fetch: str = "0"
     mount_path: str = "/work"
+    s3_mount: str = ""
 
     def __post_init__(self):
         allowed_string_flags = ["0", "1"]
         if self.git_autosave not in allowed_string_flags:
             raise ValueError("git_autosave can only be a string with values '0' or '1'")
         if self.lfs_auto_fetch not in allowed_string_flags:
-            raise ValueError(
-                "lfs_auto_fetch can only be a string with values '0' or '1'"
-            )
+            raise ValueError("lfs_auto_fetch can only be a string with values '0' or '1'")
 
 
 def config_from_env() -> Config:

renku_notebooks/api/amalthea_patches/init_containers.py

@@ -70,6 +70,12 @@ def git_clone(server: "UserServer"):
                 Path(etc_cert_volume_mount[0]["mountPath"]) / "ca-certificates.crt"
             ),
         },
+        {
+            "name": "GIT_CLONE_S3_MOUNT",
+            "value": server.cloudstorage[0].mount_folder
+            if config.s3_mounts_enabled and server.cloudstorage
+            else "",
+        },
     ]
     if type(server._user) is RegisteredUser:
         env += [

renku_notebooks/api/classes/server.py

@@ -54,7 +54,7 @@ def __init__(
         image: Optional[str],
         server_options: Dict[str, Any],
         environment_variables: Dict[str, str],
-        cloudstorage: List[Dict[str, str]],
+        cloudstorage: List[S3mount],
     ):
         self._check_flask_config()
         self._user = user
@@ -74,7 +74,7 @@ def __init__(
         self.verified_image: Optional[str] = None
         self.is_image_private: Optional[bool] = None
         self.image_workdir: Optional[str] = None
-        self.cloudstorage: Optional[Dict[str, str]] = cloudstorage
+        self.cloudstorage: Optional[List[S3mount]] = cloudstorage
         self.gl_project_name = f"{self.namespace}/{self.project}"
         self.js: Optional[Dict[str, Any]] = None