From c2f95efeced24b004b3ac130d8982d45abb842e4 Mon Sep 17 00:00:00 2001 From: gregorysprenger <42686628+gregorysprenger@users.noreply.github.com> Date: Tue, 6 Feb 2024 13:33:34 -0500 Subject: [PATCH 1/7] Place checkouts in try block and force if needed --- nf_core/synced_repo.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/nf_core/synced_repo.py b/nf_core/synced_repo.py index ac0f467e66..44ac540503 100644 --- a/nf_core/synced_repo.py +++ b/nf_core/synced_repo.py @@ -208,7 +208,10 @@ def checkout_branch(self): """ Checks out the specified branch of the repository """ - self.repo.git.checkout(self.branch) + try: + self.repo.git.checkout(self.branch) + except GitCommandError: + self.repo.git.checkout(self.branch, force=True) def checkout(self, commit): """ @@ -217,7 +220,10 @@ def checkout(self, commit): Args: commit (str): Git SHA of the commit """ - self.repo.git.checkout(commit) + try: + self.repo.git.checkout(commit) + except GitCommandError: + self.repo.git.checkout(commit, force=True) def component_exists(self, component_name, component_type, checkout=True, commit=None): """ From ebdfff0994ed6230a3ef4f972792dfc15323e3d2 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 6 Feb 2024 18:42:49 +0000 Subject: [PATCH 2/7] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2de939183c..36d6149cca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ - update ruff to 0.2.0 and add it to pre-commit step ([#2725](https://github.com/nf-core/tools/pull/2725)) - Update codecov/codecov-action digest to e0b68c6 ([#2728](https://github.com/nf-core/tools/pull/2728)) +- Force checkout of commits and branches if needed ([#2734](https://github.com/nf-core/tools/pull/2734)) ## [v2.12.1 - Aluminium Wolf - Patch](https://github.com/nf-core/tools/releases/tag/2.12.1) - [2024-02-01] From 47713a292e1810c074c513a02a85c2c7ea30e5ed Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 7 Feb 2024 09:32:54 +0100 Subject: [PATCH 3/7] only use force on specific git errors --- nf_core/synced_repo.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nf_core/synced_repo.py b/nf_core/synced_repo.py index 44ac540503..eaef930418 100644 --- a/nf_core/synced_repo.py +++ b/nf_core/synced_repo.py @@ -210,8 +210,10 @@ def checkout_branch(self): """ try: self.repo.git.checkout(self.branch) - except GitCommandError: - self.repo.git.checkout(self.branch, force=True) + except GitCommandError as e: + if "Your local changes to the following files would be overwritten by checkout" in str(e): + log.debug(f"Overwriting local changes in '{self.local_repo_dir}'") + self.repo.git.checkout(self.branch, force=True) def checkout(self, commit): """ From 5163e4a63e1104786ddb4c8723b8f9e763c5a32f Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 7 Feb 2024 09:34:56 +0100 Subject: [PATCH 4/7] add logic to second occurrence of commands --- nf_core/synced_repo.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nf_core/synced_repo.py b/nf_core/synced_repo.py index eaef930418..36dd22bc80 100644 --- a/nf_core/synced_repo.py +++ b/nf_core/synced_repo.py @@ -224,8 +224,10 @@ def checkout(self, commit): """ try: self.repo.git.checkout(commit) - except GitCommandError: - self.repo.git.checkout(commit, force=True) + except GitCommandError as e: + if "Your local changes to the following files would be overwritten by checkout" in str(e): + log.debug(f"Overwriting local changes in '{self.local_repo_dir}'") + self.repo.git.checkout(self.branch, force=True) def component_exists(self, component_name, component_type, checkout=True, commit=None): """ From 372a9f2faad0eaf8ab0c91f22b2dbc3453df0c33 Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 7 Feb 2024 10:37:22 +0100 Subject: [PATCH 5/7] fix tests --- nf_core/synced_repo.py | 4 ++++ tests/modules/modules_json.py | 25 ++++++++++++------------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/nf_core/synced_repo.py b/nf_core/synced_repo.py index 36dd22bc80..c668be28d2 100644 --- a/nf_core/synced_repo.py +++ b/nf_core/synced_repo.py @@ -214,6 +214,8 @@ def checkout_branch(self): if "Your local changes to the following files would be overwritten by checkout" in str(e): log.debug(f"Overwriting local changes in '{self.local_repo_dir}'") self.repo.git.checkout(self.branch, force=True) + else: + raise e def checkout(self, commit): """ @@ -228,6 +230,8 @@ def checkout(self, commit): if "Your local changes to the following files would be overwritten by checkout" in str(e): log.debug(f"Overwriting local changes in '{self.local_repo_dir}'") self.repo.git.checkout(self.branch, force=True) + else: + raise e def component_exists(self, component_name, component_type, checkout=True, commit=None): """ diff --git a/tests/modules/modules_json.py b/tests/modules/modules_json.py index a054b6b131..e0100adfb7 100644 --- a/tests/modules/modules_json.py +++ b/tests/modules/modules_json.py @@ -1,6 +1,5 @@ import copy import json -import os import shutil from pathlib import Path @@ -16,7 +15,7 @@ def test_get_modules_json(self): """Checks that the get_modules_json function returns the correct result""" - mod_json_path = os.path.join(self.pipeline_dir, "modules.json") + mod_json_path = Path(self.pipeline_dir, "modules.json") with open(mod_json_path) as fh: try: mod_json_sb = json.load(fh) @@ -49,16 +48,16 @@ def test_mod_json_update(self): def test_mod_json_create(self): """Test creating a modules.json file from scratch""" - mod_json_path = os.path.join(self.pipeline_dir, "modules.json") + mod_json_path = Path(self.pipeline_dir, "modules.json") # Remove the existing modules.json file - os.remove(mod_json_path) + mod_json_path.unlink() # Create the new modules.json file # (There are no prompts as long as there are only nf-core modules) ModulesJson(self.pipeline_dir).create() # Check that the file exists - assert os.path.exists(mod_json_path) + assert (mod_json_path).exists() # Get the contents of the file mod_json_obj = ModulesJson(self.pipeline_dir) @@ -94,7 +93,7 @@ def test_mod_json_create_with_patch(self): patch_obj.patch("fastqc") # Remove the existing modules.json file - os.remove(mod_json_path) + mod_json_path.unlink() # Create the new modules.json file ModulesJson(self.pipeline_dir).create() @@ -137,7 +136,7 @@ def test_mod_json_up_to_date_module_removed(self): but is missing in the pipeline """ # Remove the fastqc module - fastqc_path = os.path.join(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME, "fastqc") + fastqc_path = Path(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME, "fastqc") shutil.rmtree(fastqc_path) # Check that the modules.json file is up to date, and reinstall the module @@ -146,9 +145,9 @@ def test_mod_json_up_to_date_module_removed(self): # Check that the module has been reinstalled files = ["main.nf", "meta.yml"] - assert os.path.exists(fastqc_path) + assert fastqc_path.exists() for f in files: - assert os.path.exists(os.path.join(fastqc_path, f)) + assert Path(fastqc_path, f).exists() def test_mod_json_up_to_date_reinstall_fails(self): @@ -161,7 +160,7 @@ def test_mod_json_up_to_date_reinstall_fails(self): mod_json_obj.update("modules", ModulesRepo(), "fastqc", "INVALID_GIT_SHA", "modules", write_file=True) # Remove the fastqc module - fastqc_path = os.path.join(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME, "fastqc") + fastqc_path = Path(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME, "fastqc") shutil.rmtree(fastqc_path) # Check that the modules.json file is up to date, and remove the fastqc module entry @@ -206,12 +205,12 @@ def test_mod_json_dump(self): mod_json_obj = ModulesJson(self.pipeline_dir) mod_json = mod_json_obj.get_modules_json() # Remove the modules.json file - mod_json_path = os.path.join(self.pipeline_dir, "modules.json") - os.remove(mod_json_path) + mod_json_path = Path(self.pipeline_dir, "modules.json") + mod_json_path.unlink() # Check that the dump function creates the file mod_json_obj.dump() - assert os.path.exists(mod_json_path) + assert mod_json_path.exists() # Check that the dump function writes the correct content with open(mod_json_path) as f: From 3e95d02e402ac74b636008f2756cf3af104d41cf Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Wed, 7 Feb 2024 09:49:35 +0000 Subject: [PATCH 6/7] [automated] Update CHANGELOG.md --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e040a3dbf..1a6ec96a1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,13 +11,14 @@ ### Modules +- Handle dirty local module repos by force checkout of commits and branches if needed ([#2734](https://github.com/nf-core/tools/pull/2734)) + ### General - fix ignoring changes in partially templated files (e.g. `.gitignore`) ([#2722](https://github.com/nf-core/tools/pull/2722)) - update ruff to 0.2.0 and add it to pre-commit step ([#2725](https://github.com/nf-core/tools/pull/2725)) - Update codecov/codecov-action digest to e0b68c6 ([#2728](https://github.com/nf-core/tools/pull/2728)) - Update pre-commit hook astral-sh/ruff-pre-commit to v0.2.1 ([#2730](https://github.com/nf-core/tools/pull/2730)) -- Force checkout of commits and branches if needed ([#2734](https://github.com/nf-core/tools/pull/2734)) ## [v2.12.1 - Aluminium Wolf - Patch](https://github.com/nf-core/tools/releases/tag/2.12.1) - [2024-02-01] From 77aa1cbf9fb65feed9a4665f18ef57f01626853a Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 13 Feb 2024 10:37:54 +0100 Subject: [PATCH 7/7] check if we are in a modules repo before we force checkout --- nf_core/synced_repo.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/nf_core/synced_repo.py b/nf_core/synced_repo.py index c668be28d2..5c31e96911 100644 --- a/nf_core/synced_repo.py +++ b/nf_core/synced_repo.py @@ -211,7 +211,11 @@ def checkout_branch(self): try: self.repo.git.checkout(self.branch) except GitCommandError as e: - if "Your local changes to the following files would be overwritten by checkout" in str(e): + if ( + self.fullname + and "modules" in self.fullname + and "Your local changes to the following files would be overwritten by checkout" in str(e) + ): log.debug(f"Overwriting local changes in '{self.local_repo_dir}'") self.repo.git.checkout(self.branch, force=True) else: @@ -227,7 +231,11 @@ def checkout(self, commit): try: self.repo.git.checkout(commit) except GitCommandError as e: - if "Your local changes to the following files would be overwritten by checkout" in str(e): + if ( + self.fullname + and "modules" in self.fullname + and "Your local changes to the following files would be overwritten by checkout" in str(e) + ): log.debug(f"Overwriting local changes in '{self.local_repo_dir}'") self.repo.git.checkout(self.branch, force=True) else: