feat: only clone init templates when required

samcli/commands/init/init_templates.py

@@ -8,6 +8,7 @@
 import re
 from enum import Enum
 from pathlib import Path
+from subprocess import STDOUT, CalledProcessError, check_output
 from typing import Dict, Optional
 
 import requests
@@ -87,6 +88,9 @@ def clone_templates_repo(self):
             os_name = system().lower()
             cloned_folder_name = APP_TEMPLATES_REPO_NAME_WINDOWS if os_name == "windows" else APP_TEMPLATES_REPO_NAME
 
+            if not self._check_upsert_templates(shared_dir, cloned_folder_name):
+                return
+
             try:
                 self._git_repo.clone(
                     clone_dir=shared_dir,
@@ -102,6 +106,38 @@ def clone_templates_repo(self):
                 if expected_previous_clone_local_path.exists():
                     self._git_repo.local_path = expected_previous_clone_local_path
 
+    def _check_upsert_templates(self, shared_dir: Path, cloned_folder_name: str) -> bool:
+        """
+        Check if the app templates repository should be cloned, or if cloning should be skipped.
+
+        Parameters
+        ----------
+        shared_dir: Path
+            Folder containing the aws-sam-cli shared data
+
+        cloned_folder_name: str
+            Name of the directory into which the app templates will be copied
+
+        Returns
+        -------
+        bool
+            True if the cache should be updated, False otherwise
+
+        """
+        cache_dir = Path(shared_dir, cloned_folder_name)
+        git_executable = self._git_repo.git_executable()
+        command = [git_executable, "rev-parse", "--verify", "HEAD"]
+        try:
+            existing_hash = check_output(command, cwd=cache_dir, stderr=STDOUT).decode("utf-8").strip()
+        except CalledProcessError as ex:
+            LOG.debug(f"Unable to check existing cache hash\n{ex.output.decode('utf-8')}")
+            return True
+        except (FileNotFoundError, NotADirectoryError):
+            LOG.debug("Cache directory does not yet exist, creating one.")
+            return True
+        self._git_repo.local_path = cache_dir
+        return not existing_hash == APP_TEMPLATES_REPO_COMMIT
+
     def _init_options_from_manifest(self, package_type, runtime, base_image, dependency_manager):
         manifest_path = self.get_manifest_path()
         with open(str(manifest_path)) as fp:

samcli/lib/utils/git_repo.py

@@ -71,7 +71,7 @@ def _ensure_clone_directory_exists(clone_dir: Path) -> None:
             raise
 
     @staticmethod
-    def _git_executable() -> str:
+    def git_executable() -> str:
         if platform.system().lower() == "windows":
             executables = ["git", "git.cmd", "git.exe", "git.bat"]
         else:
@@ -127,7 +127,7 @@ def clone(self, clone_dir: Path, clone_name: str, replace_existing: bool = False
         with osutils.mkdir_temp(ignore_errors=True) as tempdir:
             try:
                 temp_path = os.path.normpath(os.path.join(tempdir, clone_name))
-                git_executable: str = GitRepo._git_executable()
+                git_executable: str = GitRepo.git_executable()
                 LOG.info("\nCloning from %s (process may take a moment)", self.url)
                 command = [git_executable, "clone", self.url, clone_name]
                 if platform.system().lower() == "windows":
@@ -172,7 +172,7 @@ def _persist_local_repo(temp_path: str, dest_dir: Path, dest_name: str, replace_
 
             LOG.debug("Copying from %s to %s", temp_path, dest_path)
             # Todo consider not removing the .git files/directories
-            shutil.copytree(temp_path, dest_path, ignore=shutil.ignore_patterns("*.git"))
+            shutil.copytree(temp_path, dest_path)
             return Path(dest_path)
         except (OSError, shutil.Error) as ex:
             # UNSTABLE STATE
@@ -198,7 +198,7 @@ def _persist_local_repo(temp_path: str, dest_dir: Path, dest_name: str, replace_
     def _checkout_commit(repo_dir: str, commit: str):
         try:
             # if the checkout commit failed, it will use the latest commit instead
-            git_executable = GitRepo._git_executable()
+            git_executable = GitRepo.git_executable()
             check_output(
                 [git_executable, "checkout", commit],
                 cwd=repo_dir,

tests/integration/init/test_init_command.py

@@ -4,6 +4,7 @@
 
 from click.testing import CliRunner
 
+from samcli.cli.global_config import GlobalConfig
 from samcli.commands.init import cli as init_cmd
 from unittest import TestCase
 
@@ -838,6 +839,62 @@ def test_interactive_init_default_runtime(self):
             self.assertTrue(Path(expected_output_folder, "hello_world", "app.py").is_file())
 
 
+class TestSubsequentInitCaching(TestCase):
+    def test_subsequent_init_skips_cloning(self):
+        with tempfile.TemporaryDirectory() as temp:
+            project_directory = Path(temp, "sam-app")
+            cache_dir = GlobalConfig().config_dir / "aws-sam-cli-app-templates"
+
+            # Run the first time, get cache last modification time
+            self._run_init(temp)
+            first_modification_time = self._last_modification_time(cache_dir)
+            self.assertTrue(project_directory.is_dir())
+            shutil.rmtree(project_directory)
+
+            # Run init again, get the second modification time
+            self._run_init(temp)
+            second_modification_time = self._last_modification_time(cache_dir)
+            self.assertTrue(project_directory.is_dir())
+
+            self.assertEqual(first_modification_time, second_modification_time)
+
+    def _run_init(self, cwd):
+        process = Popen(
+            [
+                get_sam_command(),
+                "init",
+                "--runtime",
+                "nodejs14.x",
+                "--dependency-manager",
+                "npm",
+                "--architecture",
+                "arm64",
+                "--app-template",
+                "hello-world",
+                "--name",
+                "sam-app",
+                "--no-interactive",
+                "-o",
+                cwd,
+            ],
+            stdout=PIPE,
+            stderr=PIPE,
+        )
+        try:
+            stdout_data, stderr_data = process.communicate(timeout=TIMEOUT)
+            stderr = stderr_data.decode("utf-8")
+        except TimeoutExpired:
+            process.kill()
+            raise
+
+        self.assertEqual(process.returncode, 0)
+        self.assertNotIn(COMMIT_ERROR, stderr)
+
+    @staticmethod
+    def _last_modification_time(file):
+        return os.path.getmtime(file)
+
+
 class TestInitProducesSamconfigFile(TestCase):
     def test_zip_template_config(self):
         with tempfile.TemporaryDirectory() as temp:

tests/unit/commands/init/test_templates.py

@@ -1,20 +1,24 @@
+from subprocess import STDOUT
+
 import json
+from parameterized import parameterized
 from pathlib import Path
 from re import search
 from unittest import TestCase
-from unittest.mock import mock_open, patch, PropertyMock
+from unittest.mock import mock_open, patch, PropertyMock, Mock
 
 from samcli.commands.init.init_templates import InitTemplates
 from samcli.lib.utils.packagetype import IMAGE, ZIP
 
 
 class TestTemplates(TestCase):
     @patch("samcli.lib.utils.git_repo.check_output")
-    @patch("samcli.lib.utils.git_repo.GitRepo._git_executable")
+    @patch("samcli.lib.utils.git_repo.GitRepo.git_executable")
     @patch("samcli.lib.utils.git_repo.GitRepo._ensure_clone_directory_exists")
     @patch("shutil.copytree")
     def test_location_from_app_template_zip(self, subprocess_mock, git_exec_mock, cd_mock, copy_mock):
         it = InitTemplates()
+        it._check_upsert_templates = Mock()
 
         manifest = {
             "ruby2.7": [
@@ -37,11 +41,12 @@ def test_location_from_app_template_zip(self, subprocess_mock, git_exec_mock, cd
                 self.assertTrue(search("mock-ruby-template", location))
 
     @patch("samcli.lib.utils.git_repo.check_output")
-    @patch("samcli.lib.utils.git_repo.GitRepo._git_executable")
+    @patch("samcli.lib.utils.git_repo.GitRepo.git_executable")
     @patch("samcli.lib.utils.git_repo.GitRepo._ensure_clone_directory_exists")
     @patch("shutil.copytree")
     def test_location_from_app_template_image(self, subprocess_mock, git_exec_mock, cd_mock, copy_mock):
         it = InitTemplates()
+        it._check_upsert_templates = Mock()
 
         manifest = {
             "ruby2.7-image": [
@@ -64,3 +69,17 @@ def test_location_from_app_template_image(self, subprocess_mock, git_exec_mock,
                     IMAGE, None, "ruby2.7-image", "bundler", "hello-world-lambda-image"
                 )
                 self.assertTrue(search("mock-ruby-image-template", location))
+
+    @parameterized.expand([("hash_a", "hash_a", False), ("hash_a", "hash_b", True)])
+    @patch("samcli.lib.utils.git_repo.GitRepo.git_executable")
+    @patch("samcli.commands.init.init_templates.check_output")
+    def test_check_upsert_templates(self, first_hash, second_hash, expected_value, check_output_mock, git_exec_mock):
+        it = InitTemplates()
+        git_exec_mock.return_value = "git"
+        check_output_mock.return_value = second_hash.encode("utf-8")
+        with patch("samcli.commands.init.init_templates.APP_TEMPLATES_REPO_COMMIT", first_hash):
+            return_value = it._check_upsert_templates(Path("shared_dir"), Path("cloned_folder_dir"))
+        check_output_mock.assert_called_once_with(
+            ["git", "rev-parse", "--verify", "HEAD"], cwd=Path("shared_dir/cloned_folder_dir"), stderr=STDOUT
+        )
+        self.assertEqual(return_value, expected_value)

tests/unit/lib/utils/test_git_repo.py

@@ -31,21 +31,21 @@ def test_ensure_clone_directory_exists_fail(self):
     @patch("samcli.lib.utils.git_repo.platform.system")
     def test_git_executable_not_windows(self, mock_platform, mock_popen):
         mock_platform.return_value = "Not Windows"
-        executable = self.repo._git_executable()
+        executable = self.repo.git_executable()
         self.assertEqual(executable, "git")
 
     @patch("samcli.lib.utils.git_repo.subprocess.Popen")
     @patch("samcli.lib.utils.git_repo.platform.system")
     def test_git_executable_windows(self, mock_platform, mock_popen):
         mock_platform.return_value = "Windows"
-        executable = self.repo._git_executable()
+        executable = self.repo.git_executable()
         self.assertEqual(executable, "git")
 
     @patch("samcli.lib.utils.git_repo.subprocess.Popen")
     def test_git_executable_fails(self, mock_popen):
         mock_popen.side_effect = OSError("fail")
         with self.assertRaises(OSError):
-            self.repo._git_executable()
+            self.repo.git_executable()
 
     @patch("samcli.lib.utils.git_repo.Path.exists")
     @patch("samcli.lib.utils.git_repo.shutil")
@@ -62,8 +62,7 @@ def test_clone_happy_case(self, platform_mock, popen_mock, check_output_mock, sh
             [call(["git", "clone", self.repo.url, REPO_NAME], cwd=ANY, stderr=subprocess.STDOUT)]
         )
         shutil_mock.rmtree.assert_not_called()
-        shutil_mock.copytree.assert_called_with(ANY, EXPECTED_DEFAULT_CLONE_PATH, ignore=ANY)
-        shutil_mock.ignore_patterns.assert_called_with("*.git")
+        shutil_mock.copytree.assert_called_with(ANY, EXPECTED_DEFAULT_CLONE_PATH)
 
     @patch("samcli.lib.utils.git_repo.Path.exists")
     @patch("samcli.lib.utils.git_repo.shutil")
@@ -76,8 +75,7 @@ def test_clone_create_new_local_repo(
         path_exist_mock.return_value = False
         self.repo.clone(clone_dir=self.local_clone_dir, clone_name=REPO_NAME)
         shutil_mock.rmtree.assert_not_called()
-        shutil_mock.copytree.assert_called_with(ANY, EXPECTED_DEFAULT_CLONE_PATH, ignore=ANY)
-        shutil_mock.ignore_patterns.assert_called_with("*.git")
+        shutil_mock.copytree.assert_called_with(ANY, EXPECTED_DEFAULT_CLONE_PATH)
 
     @patch("samcli.lib.utils.git_repo.Path.exists")
     @patch("samcli.lib.utils.git_repo.shutil")
@@ -91,8 +89,7 @@ def test_clone_replace_current_local_repo_if_replace_existing_flag_is_set(
         self.repo.clone(clone_dir=self.local_clone_dir, clone_name=REPO_NAME, replace_existing=True)
         self.local_clone_dir.mkdir.assert_called_once_with(mode=0o700, parents=True, exist_ok=True)
         shutil_mock.rmtree.assert_called_with(EXPECTED_DEFAULT_CLONE_PATH, onerror=rmtree_callback)
-        shutil_mock.copytree.assert_called_with(ANY, EXPECTED_DEFAULT_CLONE_PATH, ignore=ANY)
-        shutil_mock.ignore_patterns.assert_called_with("*.git")
+        shutil_mock.copytree.assert_called_with(ANY, EXPECTED_DEFAULT_CLONE_PATH)
 
     @patch("samcli.lib.utils.git_repo.Path.exists")
     @patch("samcli.lib.utils.git_repo.check_output")
@@ -186,8 +183,7 @@ def test_clone_when_failed_to_move_cloned_repo_from_temp_to_final_destination(
         except Exception:
             pass
         shutil_mock.rmtree.assert_called_once_with(EXPECTED_DEFAULT_CLONE_PATH, onerror=rmtree_callback)
-        shutil_mock.copytree.assert_called_once_with(ANY, EXPECTED_DEFAULT_CLONE_PATH, ignore=ANY)
-        shutil_mock.ignore_patterns.assert_called_once_with("*.git")
+        shutil_mock.copytree.assert_called_once_with(ANY, EXPECTED_DEFAULT_CLONE_PATH)
 
     @patch("samcli.lib.utils.git_repo.LOG")
     @patch("samcli.lib.utils.git_repo.check_output")
@@ -221,8 +217,7 @@ def test_clone_with_commit(self, platform_mock, popen_mock, check_output_mock, s
             [call(["git", "checkout", COMMIT], cwd=ANY, stderr=subprocess.STDOUT)],
         )
         shutil_mock.rmtree.assert_not_called()
-        shutil_mock.copytree.assert_called_with(ANY, EXPECTED_DEFAULT_CLONE_PATH, ignore=ANY)
-        shutil_mock.ignore_patterns.assert_called_with("*.git")
+        shutil_mock.copytree.assert_called_with(ANY, EXPECTED_DEFAULT_CLONE_PATH)
 
     @patch("samcli.lib.utils.git_repo.Path.exists")
     @patch("samcli.lib.utils.git_repo.shutil")
@@ -247,8 +242,7 @@ def test_clone_with_longpaths_configured_in_windows(
             ]
         )
         shutil_mock.rmtree.assert_not_called()
-        shutil_mock.copytree.assert_called_with(ANY, EXPECTED_DEFAULT_CLONE_PATH, ignore=ANY)
-        shutil_mock.ignore_patterns.assert_called_with("*.git")
+        shutil_mock.copytree.assert_called_with(ANY, EXPECTED_DEFAULT_CLONE_PATH)
 
     @patch("samcli.lib.utils.git_repo.Path")
     @patch("samcli.lib.utils.git_repo.platform.system")