From 3d6ded489d6c03322ba62dca3451686162159a62 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 4 Nov 2022 16:08:49 +0100 Subject: [PATCH 01/11] move pathc path function to components command and remove check_components_structure --- nf_core/components/components_command.py | 47 +++++++++++--- nf_core/modules/modules_command.py | 66 -------------------- nf_core/subworkflows/subworkflows_command.py | 3 - 3 files changed, 39 insertions(+), 77 deletions(-) diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index d89fb4652e..d671131c16 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -158,22 +158,22 @@ def load_lint_config(self): except FileNotFoundError: log.debug(f"No lint config file found: {config_fn}") - def check_component_structure(self, component_name): + def check_modules_structure(self): """ - Check that the structure of the modules/subworkflow directory in a pipeline is the correct one: - modules/nf-core/TOOL/SUBTOOL | subworkflows/nf-core/SUBWORKFLOW + Check that the structure of the modules directory in a pipeline is the correct one: + modules/nf-core/TOOL/SUBTOOL Prior to nf-core/tools release 2.6 the directory structure had an additional level of nesting: modules/nf-core/modules/TOOL/SUBTOOL """ if self.repo_type == "pipeline": wrong_location_modules = [] - for directory, _, files in os.walk(Path(self.dir, component_name)): + for directory, _, files in os.walk(Path(self.dir, "modules")): if "main.nf" in files: - module_path = Path(directory).relative_to(Path(self.dir, component_name)) + module_path = Path(directory).relative_to(Path(self.dir, "modules")) parts = module_path.parts # Check that there are modules installed directly under the 'modules' directory - if parts[1] == component_name: + if parts[1] == "modules": wrong_location_modules.append(module_path) # If there are modules installed in the wrong location if len(wrong_location_modules) > 0: @@ -185,12 +185,43 @@ def check_component_structure(self, component_name): ) # Move wrong modules to the right directory for module in wrong_location_modules: - modules_dir = Path(component_name).resolve() + modules_dir = Path("modules").resolve() correct_dir = Path(modules_dir, self.modules_repo.repo_path, Path(*module.parts[2:])) wrong_dir = Path(modules_dir, module) shutil.move(wrong_dir, correct_dir) log.info(f"Moved {wrong_dir} to {correct_dir}.") - shutil.rmtree(Path(self.dir, component_name, self.modules_repo.repo_path, component_name)) + shutil.rmtree(Path(self.dir, "modules", self.modules_repo.repo_path, "modules")) # Regenerate modules.json file modules_json = ModulesJson(self.dir) modules_json.check_up_to_date() + + def check_patch_paths(self, patch_path, module_name): + """ + Check that paths in patch files are updated to the new modules path + """ + if patch_path.exists(): + log.info(f"Modules {module_name} contains a patch file.") + rewrite = False + with open(patch_path, "r") as fh: + lines = fh.readlines() + for index, line in enumerate(lines): + # Check if there are old paths in the patch file and replace + if f"modules/{self.modules_repo.repo_path}/modules/{module_name}/" in line: + rewrite = True + lines[index] = line.replace( + f"modules/{self.modules_repo.repo_path}/modules/{module_name}/", + f"modules/{self.modules_repo.repo_path}/{module_name}/", + ) + if rewrite: + log.info(f"Updating paths in {patch_path}") + with open(patch_path, "w") as fh: + for line in lines: + fh.write(line) + # Update path in modules.json if the file is in the correct format + modules_json = ModulesJson(self.dir) + modules_json.load() + if modules_json.has_git_url_and_modules(): + modules_json.modules_json["repos"][self.modules_repo.remote_url]["modules"][ + self.modules_repo.repo_path + ][module_name]["patch"] = str(patch_path.relative_to(Path(self.dir).resolve())) + modules_json.dump() diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py index ef075a20b1..566af4f694 100644 --- a/nf_core/modules/modules_command.py +++ b/nf_core/modules/modules_command.py @@ -17,70 +17,4 @@ class ModuleCommand(ComponentCommand): def __init__(self, dir, remote_url=None, branch=None, no_pull=False, hide_progress=False): super().__init__("modules", dir, remote_url, branch, no_pull, hide_progress) - def check_patch_paths(self, patch_path, module_name): - """ - Check that paths in patch files are updated to the new modules path - """ - if patch_path.exists(): - log.info(f"Modules {module_name} contains a patch file.") - rewrite = False - with open(patch_path, "r") as fh: - lines = fh.readlines() - for index, line in enumerate(lines): - # Check if there are old paths in the patch file and replace - if f"modules/{self.modules_repo.repo_path}/modules/{module_name}/" in line: - rewrite = True - lines[index] = line.replace( - f"modules/{self.modules_repo.repo_path}/modules/{module_name}/", - f"modules/{self.modules_repo.repo_path}/{module_name}/", - ) - if rewrite: - log.info(f"Updating paths in {patch_path}") - with open(patch_path, "w") as fh: - for line in lines: - fh.write(line) - # Update path in modules.json if the file is in the correct format - modules_json = ModulesJson(self.dir) - modules_json.load() - if modules_json.has_git_url_and_modules(): - modules_json.modules_json["repos"][self.modules_repo.remote_url]["modules"][ - self.modules_repo.repo_path - ][module_name]["patch"] = str(patch_path.relative_to(Path(self.dir).resolve())) - modules_json.dump() - def check_modules_structure(self): - """ - Check that the structure of the modules directory in a pipeline is the correct one: - modules/nf-core/TOOL/SUBTOOL - - Prior to nf-core/tools release 2.6 the directory structure had an additional level of nesting: - modules/nf-core/modules/TOOL/SUBTOOL - """ - if self.repo_type == "pipeline": - wrong_location_modules = [] - for directory, _, files in os.walk(Path(self.dir, "modules")): - if "main.nf" in files: - module_path = Path(directory).relative_to(Path(self.dir, "modules")) - parts = module_path.parts - # Check that there are modules installed directly under the 'modules' directory - if parts[1] == "modules": - wrong_location_modules.append(module_path) - # If there are modules installed in the wrong location - if len(wrong_location_modules) > 0: - log.info("The modules folder structure is outdated. Reinstalling modules.") - # Remove the local copy of the modules repository - log.info(f"Updating '{self.modules_repo.local_repo_dir}'") - self.modules_repo.setup_local_repo( - self.modules_repo.remote_url, self.modules_repo.branch, self.hide_progress - ) - # Move wrong modules to the right directory - for module in wrong_location_modules: - modules_dir = Path("modules").resolve() - correct_dir = Path(modules_dir, self.modules_repo.repo_path, Path(*module.parts[2:])) - wrong_dir = Path(modules_dir, module) - shutil.move(wrong_dir, correct_dir) - log.info(f"Moved {wrong_dir} to {correct_dir}.") - shutil.rmtree(Path(self.dir, "modules", self.modules_repo.repo_path, "modules")) - # Regenerate modules.json file - modules_json = ModulesJson(self.dir) - modules_json.check_up_to_date() diff --git a/nf_core/subworkflows/subworkflows_command.py b/nf_core/subworkflows/subworkflows_command.py index e6d48d2a3f..2203707c09 100644 --- a/nf_core/subworkflows/subworkflows_command.py +++ b/nf_core/subworkflows/subworkflows_command.py @@ -1,7 +1,4 @@ import logging -import os -import shutil -from pathlib import Path from nf_core.components.components_command import ComponentCommand From 30021cb0eba480336dc3eb5bac39fd6128eaac49 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 4 Nov 2022 16:20:51 +0100 Subject: [PATCH 02/11] initialise componentcommand with component_type --- nf_core/components/components_test.py | 2 +- nf_core/modules/bump_versions.py | 9 ++++----- nf_core/modules/create.py | 7 +++---- nf_core/modules/info.py | 4 ++-- nf_core/modules/install.py | 6 +++--- nf_core/modules/lint/__init__.py | 6 +++--- nf_core/modules/modules_command.py | 20 -------------------- nf_core/modules/modules_test.py | 6 +++--- nf_core/modules/patch.py | 6 +++--- nf_core/modules/remove.py | 7 +++++-- nf_core/modules/test_yml_builder.py | 6 +++--- nf_core/modules/update.py | 6 +++--- nf_core/subworkflows/create.py | 7 +++---- nf_core/subworkflows/install.py | 7 +++---- nf_core/subworkflows/subworkflows_command.py | 14 -------------- nf_core/subworkflows/subworkflows_test.py | 7 +++---- nf_core/subworkflows/test_yml_builder.py | 6 +++--- tests/modules/patch.py | 6 +++--- 18 files changed, 48 insertions(+), 84 deletions(-) delete mode 100644 nf_core/modules/modules_command.py delete mode 100644 nf_core/subworkflows/subworkflows_command.py diff --git a/nf_core/components/components_test.py b/nf_core/components/components_test.py index e8a600f785..a1f3481305 100644 --- a/nf_core/components/components_test.py +++ b/nf_core/components/components_test.py @@ -38,7 +38,7 @@ def __init__( self.no_prompts = no_prompts self.pytest_args = pytest_args - super().__init__(".", remote_url, branch, no_pull) + super().__init__("component_type", ".", remote_url, branch, no_pull) def run(self): """Run test steps""" diff --git a/nf_core/modules/bump_versions.py b/nf_core/modules/bump_versions.py index 5d064cf2ef..552b9106c8 100644 --- a/nf_core/modules/bump_versions.py +++ b/nf_core/modules/bump_versions.py @@ -18,17 +18,16 @@ import nf_core.modules.modules_utils import nf_core.utils +from nf_core.components.components_command import ComponentCommand from nf_core.utils import plural_s as _s from nf_core.utils import rich_force_colors -from .modules_command import ModuleCommand - log = logging.getLogger(__name__) -class ModuleVersionBumper(ModuleCommand): - def __init__(self, pipeline_dir, remote_url=None, branch=None, no_pull=False): - super().__init__(pipeline_dir, remote_url, branch, no_pull) +class ModuleVersionBumper(ComponentCommand): + def __init__(self, component_type, pipeline_dir, remote_url=None, branch=None, no_pull=False): + super().__init__(component_type, pipeline_dir, remote_url, branch, no_pull) self.up_to_date = None self.updated = None diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py index 1b1eeba5ea..24e2aaeed0 100644 --- a/nf_core/modules/create.py +++ b/nf_core/modules/create.py @@ -17,13 +17,12 @@ import nf_core.components.components_create import nf_core.modules.modules_utils import nf_core.utils - -from .modules_command import ModuleCommand +from nf_core.components.components_command import ComponentCommand log = logging.getLogger(__name__) -class ModuleCreate(ModuleCommand): +class ModuleCreate(ComponentCommand): def __init__( self, directory=".", @@ -36,7 +35,7 @@ def __init__( conda_version=None, repo_type=None, ): - super().__init__(directory) + super().__init__("modules", directory) self.directory = directory self.tool = tool self.author = author diff --git a/nf_core/modules/info.py b/nf_core/modules/info.py index 8fe5c75ead..dec64fa772 100644 --- a/nf_core/modules/info.py +++ b/nf_core/modules/info.py @@ -13,14 +13,14 @@ import nf_core.utils from nf_core.modules.modules_json import ModulesJson -from .modules_command import ModuleCommand +from nf_core.components.components_command import ComponentCommand from .modules_repo import NF_CORE_MODULES_REMOTE from .modules_utils import get_repo_type log = logging.getLogger(__name__) -class ModuleInfo(ModuleCommand): +class ModuleInfo(ComponentCommand): """ Class to print information of a module. diff --git a/nf_core/modules/install.py b/nf_core/modules/install.py index e0919e07dd..65cd9773bc 100644 --- a/nf_core/modules/install.py +++ b/nf_core/modules/install.py @@ -6,12 +6,12 @@ import nf_core.utils from nf_core.modules.modules_json import ModulesJson -from .modules_command import ModuleCommand +from nf_core.components.components_command import ComponentCommand log = logging.getLogger(__name__) -class ModuleInstall(ModuleCommand): +class ModuleInstall(ComponentCommand): def __init__( self, pipeline_dir, @@ -23,7 +23,7 @@ def __init__( no_pull=False, installed_by=False, ): - super().__init__(pipeline_dir, remote_url, branch, no_pull) + super().__init__("modules", pipeline_dir, remote_url, branch, no_pull) self.force = force self.prompt = prompt self.sha = sha diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index f4ab5583ca..d48f21ddca 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -22,7 +22,7 @@ import nf_core.modules.modules_utils import nf_core.utils from nf_core.lint_utils import console -from nf_core.modules.modules_command import ModuleCommand +from nf_core.modules.modules_command import ComponentCommand from nf_core.modules.modules_json import ModulesJson from nf_core.modules.modules_repo import ModulesRepo from nf_core.modules.nfcore_module import NFCoreModule @@ -48,7 +48,7 @@ def __init__(self, mod, lint_test, message, file_path): self.module_name = mod.module_name -class ModuleLint(ModuleCommand): +class ModuleLint(ComponentCommand): """ An object for linting modules either in a clone of the 'nf-core/modules' repository or in any nf-core pipeline directory @@ -73,7 +73,7 @@ def __init__( no_pull=False, hide_progress=False, ): - super().__init__(dir=dir, remote_url=remote_url, branch=branch, no_pull=no_pull, hide_progress=False) + super().__init__("modules", dir=dir, remote_url=remote_url, branch=branch, no_pull=no_pull, hide_progress=False) self.fail_warned = fail_warned self.passed = [] diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py deleted file mode 100644 index 566af4f694..0000000000 --- a/nf_core/modules/modules_command.py +++ /dev/null @@ -1,20 +0,0 @@ -import logging -import os -import shutil -from pathlib import Path - -from nf_core.components.components_command import ComponentCommand -from nf_core.modules.modules_json import ModulesJson - -log = logging.getLogger(__name__) - - -class ModuleCommand(ComponentCommand): - """ - Base class for the 'nf-core modules' commands - """ - - def __init__(self, dir, remote_url=None, branch=None, no_pull=False, hide_progress=False): - super().__init__("modules", dir, remote_url, branch, no_pull, hide_progress) - - diff --git a/nf_core/modules/modules_test.py b/nf_core/modules/modules_test.py index 92227b992b..1bcb4f5b74 100644 --- a/nf_core/modules/modules_test.py +++ b/nf_core/modules/modules_test.py @@ -15,13 +15,13 @@ import nf_core.modules.modules_utils import nf_core.utils -from nf_core.modules.modules_command import ModuleCommand +from nf_core.modules.modules_command import ComponentCommand from nf_core.modules.modules_json import ModulesJson log = logging.getLogger(__name__) -class ModulesTest(ModuleCommand): +class ModulesTest(ComponentCommand): """ Class to run module pytests. @@ -61,7 +61,7 @@ def __init__( self.no_prompts = no_prompts self.pytest_args = pytest_args - super().__init__(".", remote_url, branch, no_pull) + super().__init__("modules", ".", remote_url, branch, no_pull) def run(self): """Run test steps""" diff --git a/nf_core/modules/patch.py b/nf_core/modules/patch.py index 84e839a033..db2243dc4f 100644 --- a/nf_core/modules/patch.py +++ b/nf_core/modules/patch.py @@ -7,17 +7,17 @@ import questionary import nf_core.utils +from nf_core.components.components_command import ComponentCommand -from .modules_command import ModuleCommand from .modules_differ import ModulesDiffer from .modules_json import ModulesJson log = logging.getLogger(__name__) -class ModulePatch(ModuleCommand): +class ModulePatch(ComponentCommand): def __init__(self, dir, remote_url=None, branch=None, no_pull=False): - super().__init__(dir, remote_url, branch, no_pull) + super().__init__("modules", dir, remote_url, branch, no_pull) self.modules_json = ModulesJson(dir) diff --git a/nf_core/modules/remove.py b/nf_core/modules/remove.py index 8afe634257..afe16ca2d5 100644 --- a/nf_core/modules/remove.py +++ b/nf_core/modules/remove.py @@ -4,14 +4,17 @@ import questionary import nf_core.utils +from nf_core.components.components_command import ComponentCommand -from .modules_command import ModuleCommand from .modules_json import ModulesJson log = logging.getLogger(__name__) -class ModuleRemove(ModuleCommand): +class ModuleRemove(ComponentCommand): + def __init__(self): + super().__init__("modules") + def remove(self, module): """ Remove an already installed module diff --git a/nf_core/modules/test_yml_builder.py b/nf_core/modules/test_yml_builder.py index ce5bb9e7b0..13d415a2b0 100644 --- a/nf_core/modules/test_yml_builder.py +++ b/nf_core/modules/test_yml_builder.py @@ -24,14 +24,14 @@ from rich.syntax import Syntax import nf_core.utils +from nf_core.components.components_command import ComponentCommand -from .modules_command import ModuleCommand from .modules_repo import ModulesRepo log = logging.getLogger(__name__) -class ModulesTestYmlBuilder(ModuleCommand): +class ModulesTestYmlBuilder(ComponentCommand): def __init__( self, module_name=None, @@ -41,7 +41,7 @@ def __init__( force_overwrite=False, no_prompts=False, ): - super().__init__(directory) + super().__init__("modules", directory) self.module_name = module_name self.run_tests = run_tests self.test_yml_output_path = test_yml_output_path diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index 223637e73b..297e893401 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -8,10 +8,10 @@ import nf_core.modules.modules_utils import nf_core.utils +from nf_core.components.components_command import ComponentCommand from nf_core.components.components_utils import prompt_component_version_sha from nf_core.utils import plural_es, plural_s, plural_y -from .modules_command import ModuleCommand from .modules_differ import ModulesDiffer from .modules_json import ModulesJson from .modules_repo import ModulesRepo @@ -19,7 +19,7 @@ log = logging.getLogger(__name__) -class ModuleUpdate(ModuleCommand): +class ModuleUpdate(ComponentCommand): def __init__( self, pipeline_dir, @@ -33,7 +33,7 @@ def __init__( branch=None, no_pull=False, ): - super().__init__(pipeline_dir, remote_url, branch, no_pull) + super().__init__("modules", pipeline_dir, remote_url, branch, no_pull) self.force = force self.prompt = prompt self.sha = sha diff --git a/nf_core/subworkflows/create.py b/nf_core/subworkflows/create.py index 481bd1ea94..2440b2ad8d 100644 --- a/nf_core/subworkflows/create.py +++ b/nf_core/subworkflows/create.py @@ -13,14 +13,13 @@ import nf_core import nf_core.components.components_create import nf_core.utils +from nf_core.components.components_command import ComponentCommand from nf_core.modules.modules_utils import get_repo_type -from .subworkflows_command import SubworkflowCommand - log = logging.getLogger(__name__) -class SubworkflowCreate(SubworkflowCommand): +class SubworkflowCreate(ComponentCommand): def __init__( self, directory=".", @@ -29,7 +28,7 @@ def __init__( force=False, repo_type=None, ): - super().__init__(directory) + super().__init__("subworkflows", directory) self.directory = directory self.subworkflow = subworkflow self.author = author diff --git a/nf_core/subworkflows/install.py b/nf_core/subworkflows/install.py index 750007f294..2df0c48970 100644 --- a/nf_core/subworkflows/install.py +++ b/nf_core/subworkflows/install.py @@ -6,15 +6,14 @@ import nf_core.components.components_install import nf_core.modules.modules_utils import nf_core.utils +from nf_core.components.components_command import ComponentCommand from nf_core.modules.install import ModuleInstall from nf_core.modules.modules_json import ModulesJson -from .subworkflows_command import SubworkflowCommand - log = logging.getLogger(__name__) -class SubworkflowInstall(SubworkflowCommand): +class SubworkflowInstall(ComponentCommand): def __init__( self, pipeline_dir, @@ -26,7 +25,7 @@ def __init__( no_pull=False, installed_by=False, ): - super().__init__(pipeline_dir, remote_url, branch, no_pull) + super().__init__("subworkflows", pipeline_dir, remote_url, branch, no_pull) self.force = force self.prompt = prompt self.sha = sha diff --git a/nf_core/subworkflows/subworkflows_command.py b/nf_core/subworkflows/subworkflows_command.py deleted file mode 100644 index 2203707c09..0000000000 --- a/nf_core/subworkflows/subworkflows_command.py +++ /dev/null @@ -1,14 +0,0 @@ -import logging - -from nf_core.components.components_command import ComponentCommand - -log = logging.getLogger(__name__) - - -class SubworkflowCommand(ComponentCommand): - """ - Base class for the 'nf-core subworkflows' commands - """ - - def __init__(self, dir, remote_url=None, branch=None, no_pull=False, hide_progress=False): - super().__init__("subworkflows", dir, remote_url, branch, no_pull, hide_progress) diff --git a/nf_core/subworkflows/subworkflows_test.py b/nf_core/subworkflows/subworkflows_test.py index 41ddd5bdcc..6ecf3aebf7 100644 --- a/nf_core/subworkflows/subworkflows_test.py +++ b/nf_core/subworkflows/subworkflows_test.py @@ -15,14 +15,13 @@ import nf_core.modules.modules_utils import nf_core.utils +from nf_core.components.components_command import ComponentCommand from nf_core.modules.modules_json import ModulesJson -from .subworkflows_command import SubworkflowCommand - log = logging.getLogger(__name__) -class SubworkflowsTest(SubworkflowCommand): +class SubworkflowsTest(ComponentCommand): """ Class to run module pytests. @@ -62,7 +61,7 @@ def __init__( self.no_prompts = no_prompts self.pytest_args = pytest_args - super().__init__(".", remote_url, branch, no_pull) + super().__init__("subworkflows", ".", remote_url, branch, no_pull) def run(self): """Run test steps""" diff --git a/nf_core/subworkflows/test_yml_builder.py b/nf_core/subworkflows/test_yml_builder.py index 59729ba41d..e7752ab5a2 100644 --- a/nf_core/subworkflows/test_yml_builder.py +++ b/nf_core/subworkflows/test_yml_builder.py @@ -27,12 +27,12 @@ import nf_core.utils from nf_core.modules.modules_json import ModulesJson from nf_core.modules.modules_repo import ModulesRepo -from nf_core.subworkflows.subworkflows_command import SubworkflowCommand +from nf_core.subworkflows.subworkflows_command import ComponentCommand log = logging.getLogger(__name__) -class SubworkflowTestYmlBuilder(SubworkflowCommand): +class SubworkflowTestYmlBuilder(ComponentCommand): def __init__( self, subworkflow=None, @@ -42,7 +42,7 @@ def __init__( force_overwrite=False, no_prompts=False, ): - super().__init__(directory) + super().__init__("subworkflows", directory) self.dir = directory self.subworkflow = subworkflow self.run_tests = run_tests diff --git a/tests/modules/patch.py b/tests/modules/patch.py index 57af1c499e..4cd518c667 100644 --- a/tests/modules/patch.py +++ b/tests/modules/patch.py @@ -300,9 +300,9 @@ def test_create_patch_update_fail(self): # Check that the installed files have not been affected by the attempted patch temp_dir = Path(tempfile.mkdtemp()) - nf_core.modules.modules_command.ModuleCommand(self.pipeline_dir, GITLAB_URL, PATCH_BRANCH).install_component_files( - BISMARK_ALIGN, FAIL_SHA, update_obj.modules_repo, temp_dir - ) + nf_core.modules.modules_command.ComponentCommand( + "modules", self.pipeline_dir, GITLAB_URL, PATCH_BRANCH + ).install_component_files(BISMARK_ALIGN, FAIL_SHA, update_obj.modules_repo, temp_dir) temp_module_dir = temp_dir / BISMARK_ALIGN for file in os.listdir(temp_module_dir): From c1643e7c37113a40a3541fd65349875b078a907f Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 4 Nov 2022 16:23:31 +0100 Subject: [PATCH 03/11] fix linting issues --- nf_core/components/components_test.py | 3 +++ nf_core/modules/bump_versions.py | 4 ++-- nf_core/modules/create.py | 2 +- nf_core/modules/info.py | 5 ++--- nf_core/modules/lint/__init__.py | 1 - 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/nf_core/components/components_test.py b/nf_core/components/components_test.py index a1f3481305..7c6a9be649 100644 --- a/nf_core/components/components_test.py +++ b/nf_core/components/components_test.py @@ -1,3 +1,6 @@ +from nf_core.components.components_command import ComponentCommand + + class ComponentsTest(ComponentCommand): """ Class to run module pytests. diff --git a/nf_core/modules/bump_versions.py b/nf_core/modules/bump_versions.py index 552b9106c8..aa09955f2d 100644 --- a/nf_core/modules/bump_versions.py +++ b/nf_core/modules/bump_versions.py @@ -26,8 +26,8 @@ class ModuleVersionBumper(ComponentCommand): - def __init__(self, component_type, pipeline_dir, remote_url=None, branch=None, no_pull=False): - super().__init__(component_type, pipeline_dir, remote_url, branch, no_pull) + def __init__(self, pipeline_dir, remote_url=None, branch=None, no_pull=False): + super().__init__("modules", pipeline_dir, remote_url, branch, no_pull) self.up_to_date = None self.updated = None diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py index 24e2aaeed0..a0e0fa66e4 100644 --- a/nf_core/modules/create.py +++ b/nf_core/modules/create.py @@ -156,7 +156,7 @@ def create(self): pytest_modules_yml = dict(sorted(pytest_modules_yml.items())) with open(os.path.join(self.directory, "tests", "config", "pytest_modules.yml"), "w") as fh: yaml.dump(pytest_modules_yml, fh, sort_keys=True, Dumper=nf_core.utils.custom_yaml_dumper()) - except FileNotFoundError as e: + except FileNotFoundError: raise UserWarning("Could not open 'tests/config/pytest_modules.yml' file!") new_files = list(self.file_paths.values()) diff --git a/nf_core/modules/info.py b/nf_core/modules/info.py index dec64fa772..2db3891ce0 100644 --- a/nf_core/modules/info.py +++ b/nf_core/modules/info.py @@ -11,9 +11,9 @@ from rich.text import Text import nf_core.utils +from nf_core.components.components_command import ComponentCommand from nf_core.modules.modules_json import ModulesJson -from nf_core.components.components_command import ComponentCommand from .modules_repo import NF_CORE_MODULES_REMOTE from .modules_utils import get_repo_type @@ -56,7 +56,7 @@ class ModuleInfo(ComponentCommand): """ def __init__(self, pipeline_dir, tool, remote_url, branch, no_pull): - super().__init__(pipeline_dir, remote_url, branch, no_pull) + super().__init__("modules", pipeline_dir, remote_url, branch, no_pull) self.meta = None self.local_path = None self.remote_location = None @@ -139,7 +139,6 @@ def get_local_yaml(self): if self.repo_type == "pipeline": # Try to find and load the meta.yml file - repo_name = self.modules_repo.repo_path module_base_path = os.path.join(self.dir, "modules") # Check that we have any modules installed from this repo modules = self.modules_json.get_all_modules().get(self.modules_repo.remote_url) diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index d48f21ddca..271c09e66b 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -24,7 +24,6 @@ from nf_core.lint_utils import console from nf_core.modules.modules_command import ComponentCommand from nf_core.modules.modules_json import ModulesJson -from nf_core.modules.modules_repo import ModulesRepo from nf_core.modules.nfcore_module import NFCoreModule from nf_core.utils import plural_s as _s From 8eb8d30ec61f92225d69547874f49baedc127115 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 4 Nov 2022 16:25:27 +0100 Subject: [PATCH 04/11] import ComponentsCommand correctly --- nf_core/components/list.py | 2 -- nf_core/modules/lint/__init__.py | 2 +- nf_core/modules/modules_test.py | 2 +- tests/modules/patch.py | 4 ++-- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/nf_core/components/list.py b/nf_core/components/list.py index eeacb9c39b..b51f8c4aa2 100644 --- a/nf_core/components/list.py +++ b/nf_core/components/list.py @@ -6,8 +6,6 @@ import nf_core.modules.modules_utils from nf_core.components.components_command import ComponentCommand from nf_core.modules.modules_json import ModulesJson - -# from .modules_command import ModulesRepo from nf_core.modules.modules_repo import ModulesRepo log = logging.getLogger(__name__) diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 271c09e66b..dff57f9da5 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -21,8 +21,8 @@ import nf_core.modules.modules_utils import nf_core.utils +from nf_core.components.components_command import ComponentCommand from nf_core.lint_utils import console -from nf_core.modules.modules_command import ComponentCommand from nf_core.modules.modules_json import ModulesJson from nf_core.modules.nfcore_module import NFCoreModule from nf_core.utils import plural_s as _s diff --git a/nf_core/modules/modules_test.py b/nf_core/modules/modules_test.py index 1bcb4f5b74..97ef1b00f2 100644 --- a/nf_core/modules/modules_test.py +++ b/nf_core/modules/modules_test.py @@ -15,7 +15,7 @@ import nf_core.modules.modules_utils import nf_core.utils -from nf_core.modules.modules_command import ComponentCommand +from nf_core.components.components_command import ComponentCommand from nf_core.modules.modules_json import ModulesJson log = logging.getLogger(__name__) diff --git a/tests/modules/patch.py b/tests/modules/patch.py index 4cd518c667..f1597cd776 100644 --- a/tests/modules/patch.py +++ b/tests/modules/patch.py @@ -4,8 +4,8 @@ import pytest +import nf_core.components.components_command import nf_core.modules -import nf_core.modules.modules_command from ..utils import GITLAB_URL @@ -300,7 +300,7 @@ def test_create_patch_update_fail(self): # Check that the installed files have not been affected by the attempted patch temp_dir = Path(tempfile.mkdtemp()) - nf_core.modules.modules_command.ComponentCommand( + nf_core.components.components_command.ComponentCommand( "modules", self.pipeline_dir, GITLAB_URL, PATCH_BRANCH ).install_component_files(BISMARK_ALIGN, FAIL_SHA, update_obj.modules_repo, temp_dir) From d2e5a7a935ea08f2812a591bdd20d3ccbf476f18 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 4 Nov 2022 16:30:04 +0100 Subject: [PATCH 05/11] import ComponentsCommand correctly --- nf_core/subworkflows/test_yml_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/subworkflows/test_yml_builder.py b/nf_core/subworkflows/test_yml_builder.py index e7752ab5a2..271dd7edd5 100644 --- a/nf_core/subworkflows/test_yml_builder.py +++ b/nf_core/subworkflows/test_yml_builder.py @@ -25,9 +25,9 @@ from rich.syntax import Syntax import nf_core.utils +from nf_core.components.components_command import ComponentCommand from nf_core.modules.modules_json import ModulesJson from nf_core.modules.modules_repo import ModulesRepo -from nf_core.subworkflows.subworkflows_command import ComponentCommand log = logging.getLogger(__name__) From 56ff217353003294ba4546384017e6e81ba35026 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 4 Nov 2022 16:46:35 +0100 Subject: [PATCH 06/11] fix bug in remove --- nf_core/modules/remove.py | 4 ++-- tests/test_modules.py | 12 ++++++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/nf_core/modules/remove.py b/nf_core/modules/remove.py index afe16ca2d5..1930ecb879 100644 --- a/nf_core/modules/remove.py +++ b/nf_core/modules/remove.py @@ -12,8 +12,8 @@ class ModuleRemove(ComponentCommand): - def __init__(self): - super().__init__("modules") + def __init__(self, pipeline_dir): + super().__init__("modules", pipeline_dir) def remove(self, module): """ diff --git a/tests/test_modules.py b/tests/test_modules.py index 85e1047bf7..f14d256dc4 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -71,10 +71,18 @@ def setUp(self): branch=OLD_TRIMGALORE_BRANCH, ) self.mods_install_trimgalore = nf_core.modules.ModuleInstall( - self.pipeline_dir, prompt=False, force=True, remote_url=GITLAB_URL, branch=OLD_TRIMGALORE_BRANCH + self.pipeline_dir, + prompt=False, + force=True, + remote_url=GITLAB_URL, + branch=OLD_TRIMGALORE_BRANCH, ) self.mods_install_gitlab = nf_core.modules.ModuleInstall( - self.pipeline_dir, prompt=False, force=True, remote_url=GITLAB_URL, branch=GITLAB_DEFAULT_BRANCH + self.pipeline_dir, + prompt=False, + force=True, + remote_url=GITLAB_URL, + branch=GITLAB_DEFAULT_BRANCH, ) self.mods_install_gitlab_old = nf_core.modules.ModuleInstall( self.pipeline_dir, From bea0218f5442d92d4d20ddda1d5ff20b7c4a4aef Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 7 Nov 2022 10:24:23 +0100 Subject: [PATCH 07/11] run isort --- nf_core/modules/install.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nf_core/modules/install.py b/nf_core/modules/install.py index 65cd9773bc..2a38f16c4c 100644 --- a/nf_core/modules/install.py +++ b/nf_core/modules/install.py @@ -4,9 +4,8 @@ import nf_core.components.components_install import nf_core.modules.modules_utils import nf_core.utils -from nf_core.modules.modules_json import ModulesJson - from nf_core.components.components_command import ComponentCommand +from nf_core.modules.modules_json import ModulesJson log = logging.getLogger(__name__) From 7c93bd59f63954678142a0d5eb9f2e658fc71bb9 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 7 Nov 2022 10:51:12 +0100 Subject: [PATCH 08/11] add component_type to ComponentsTest --- nf_core/components/components_test.py | 21 +++- nf_core/modules/modules_test.py | 172 ++------------------------ 2 files changed, 29 insertions(+), 164 deletions(-) diff --git a/nf_core/components/components_test.py b/nf_core/components/components_test.py index 7c6a9be649..69218e5e5d 100644 --- a/nf_core/components/components_test.py +++ b/nf_core/components/components_test.py @@ -1,4 +1,19 @@ +import logging +import os +import sys +from pathlib import Path +from shutil import which + +import pytest +import questionary +import rich + +import nf_core.modules.modules_utils +import nf_core.utils from nf_core.components.components_command import ComponentCommand +from nf_core.modules.modules_json import ModulesJson + +log = logging.getLogger(__name__) class ComponentsTest(ComponentCommand): @@ -30,6 +45,7 @@ class ComponentsTest(ComponentCommand): def __init__( self, + component_type, module_name=None, no_prompts=False, pytest_args="", @@ -37,12 +53,11 @@ def __init__( branch=None, no_pull=False, ): + super().__init__(component_type=component_type, dir=".", remote_url=remote_url, branch=branch, no_pull=no_pull) self.module_name = module_name self.no_prompts = no_prompts self.pytest_args = pytest_args - super().__init__("component_type", ".", remote_url, branch, no_pull) - def run(self): """Run test steps""" if not self.no_prompts: @@ -61,7 +76,7 @@ def _check_inputs(self): # Retrieving installed modules if self.repo_type == "modules": - installed_modules = self.get_modules_clone_modules() + installed_modules = self.get_components_clone_modules() else: modules_json = ModulesJson(self.dir) modules_json.check_up_to_date() diff --git a/nf_core/modules/modules_test.py b/nf_core/modules/modules_test.py index 97ef1b00f2..f02ea11cdb 100644 --- a/nf_core/modules/modules_test.py +++ b/nf_core/modules/modules_test.py @@ -3,49 +3,12 @@ The ModulesTest class runs the tests locally """ -import logging -import os -import sys -from pathlib import Path -from shutil import which +from nf_core.components.components_test import ComponentsTest -import pytest -import questionary -import rich -import nf_core.modules.modules_utils -import nf_core.utils -from nf_core.components.components_command import ComponentCommand -from nf_core.modules.modules_json import ModulesJson - -log = logging.getLogger(__name__) - - -class ModulesTest(ComponentCommand): +class ModulesTest(ComponentsTest): """ Class to run module pytests. - - ... - - Attributes - ---------- - module_name : str - name of the tool to run tests for - no_prompts : bool - flat indicating if prompts are used - pytest_args : tuple - additional arguments passed to pytest command - - Methods - ------- - run(): - Run test steps - _check_inputs(): - Check inputs. Ask for module_name if not provided and check that the directory exists - _set_profile(): - Set software profile - _run_pytests(self): - Run pytest """ def __init__( @@ -57,125 +20,12 @@ def __init__( branch=None, no_pull=False, ): - self.module_name = module_name - self.no_prompts = no_prompts - self.pytest_args = pytest_args - - super().__init__("modules", ".", remote_url, branch, no_pull) - - def run(self): - """Run test steps""" - if not self.no_prompts: - log.info( - "[yellow]Press enter to use default values [cyan bold](shown in brackets) [yellow]or type your own responses" - ) - self._check_inputs() - self._set_profile() - self._check_profile() - self._run_pytests() - - def _check_inputs(self): - """Do more complex checks about supplied flags.""" - # Check modules directory structure - self.check_modules_structure() - - # Retrieving installed modules - if self.repo_type == "modules": - installed_modules = self.get_components_clone_modules() - else: - modules_json = ModulesJson(self.dir) - modules_json.check_up_to_date() - installed_modules = modules_json.get_all_modules().get(self.modules_repo.remote_url) - - # Get the tool name if not specified - if self.module_name is None: - if self.no_prompts: - raise UserWarning( - "Tool name not provided and prompts deactivated. Please provide the tool name as TOOL/SUBTOOL or TOOL." - ) - if not installed_modules: - raise UserWarning( - f"No installed modules were found from '{self.modules_repo.remote_url}'.\n" - f"Are you running the tests inside the nf-core/modules main directory?\n" - f"Otherwise, make sure that the directory structure is modules/TOOL/SUBTOOL/ and tests/modules/TOOLS/SUBTOOL/" - ) - self.module_name = questionary.autocomplete( - "Tool name:", - choices=installed_modules, - style=nf_core.utils.nfcore_question_style, - ).unsafe_ask() - - # Sanity check that the module directory exists - self._validate_folder_structure() - - def _validate_folder_structure(self): - """Validate that the modules follow the correct folder structure to run the tests: - - modules/nf-core/TOOL/SUBTOOL/ - - tests/modules/nf-core/TOOL/SUBTOOL/ - - """ - module_path = Path(self.default_modules_path) / self.module_name - test_path = Path(self.default_tests_path) / self.module_name - - if not (self.dir / module_path).is_dir(): - raise UserWarning( - f"Cannot find directory '{module_path}'. Should be TOOL/SUBTOOL or TOOL. Are you running the tests inside the nf-core/modules main directory?" - ) - if not (self.dir / test_path).is_dir(): - raise UserWarning( - f"Cannot find directory '{test_path}'. Should be TOOL/SUBTOOL or TOOL. " - "Are you running the tests inside the nf-core/modules main directory? " - "Do you have tests for the specified module?" - ) - - def _set_profile(self): - """Set $PROFILE env variable. - The config expects $PROFILE and Nextflow fails if it's not set. - """ - if os.environ.get("PROFILE") is None: - os.environ["PROFILE"] = "" - if self.no_prompts: - log.info( - "Setting environment variable '$PROFILE' to an empty string as not set.\n" - "Tests will run with Docker by default. " - "To use Singularity set 'export PROFILE=singularity' in your shell before running this command." - ) - else: - question = { - "type": "list", - "name": "profile", - "message": "Choose software profile", - "choices": ["Docker", "Singularity", "Conda"], - } - answer = questionary.unsafe_prompt([question], style=nf_core.utils.nfcore_question_style) - profile = answer["profile"].lower() - os.environ["PROFILE"] = profile - log.info(f"Setting environment variable '$PROFILE' to '{profile}'") - - def _check_profile(self): - """Check if profile is available""" - profile = os.environ.get("PROFILE") - # Make sure the profile read from the environment is a valid Nextflow profile. - valid_nextflow_profiles = ["docker", "singularity", "conda"] - if profile in valid_nextflow_profiles: - if not which(profile): - raise UserWarning(f"Command '{profile}' not found - is it installed?") - else: - raise UserWarning( - f"The PROFILE '{profile}' set in the shell environment is not valid.\n" - f"Valid Nextflow profiles are '{', '.join(valid_nextflow_profiles)}'." - ) - - def _run_pytests(self): - """Given a module name, run tests.""" - # Print nice divider line - console = rich.console.Console() - console.rule(self.module_name, style="black") - - # Set pytest arguments - command_args = ["--tag", f"{self.module_name}", "--symlink", "--keep-workflow-wd", "--git-aware"] - command_args += self.pytest_args - - # Run pytest - log.info(f"Running pytest for module '{self.module_name}'") - sys.exit(pytest.main(command_args)) + super().__init__( + component_type="modules", + module_name=module_name, + no_prompts=no_prompts, + pytest_args=pytest_args, + remote_url=remote_url, + branch=branch, + no_pull=no_pull, + ) From 9fd155631242001bb27af6cf85c20f42a56f86e6 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 8 Nov 2022 13:29:17 +0100 Subject: [PATCH 09/11] SubworkflowsTest inherits ComponentsTest (not functional) --- nf_core/subworkflows/subworkflows_test.py | 169 ++-------------------- 1 file changed, 11 insertions(+), 158 deletions(-) diff --git a/nf_core/subworkflows/subworkflows_test.py b/nf_core/subworkflows/subworkflows_test.py index 6ecf3aebf7..803e808bde 100644 --- a/nf_core/subworkflows/subworkflows_test.py +++ b/nf_core/subworkflows/subworkflows_test.py @@ -3,49 +3,12 @@ The SubworkflowsTest class runs the tests locally """ -import logging -import os -import sys -from pathlib import Path -from shutil import which +from nf_core.components.components_test import ComponentsTest -import pytest -import questionary -import rich -import nf_core.modules.modules_utils -import nf_core.utils -from nf_core.components.components_command import ComponentCommand -from nf_core.modules.modules_json import ModulesJson - -log = logging.getLogger(__name__) - - -class SubworkflowsTest(ComponentCommand): +class SubworkflowsTest(ComponentsTest): """ Class to run module pytests. - - ... - - Attributes - ---------- - subworkflow_name : str - name of the subworkflow to run tests for - no_prompts : bool - flat indicating if prompts are used - pytest_args : tuple - additional arguments passed to pytest command - - Methods - ------- - run(): - Run test steps - _check_inputs(): - Check inputs. Ask for subworkflow_name if not provided and check that the directory exists - _set_profile(): - Set software profile - _run_pytests(self): - Run pytest """ def __init__( @@ -57,122 +20,12 @@ def __init__( branch=None, no_pull=False, ): - self.subworkflow_name = subworkflow_name - self.no_prompts = no_prompts - self.pytest_args = pytest_args - - super().__init__("subworkflows", ".", remote_url, branch, no_pull) - - def run(self): - """Run test steps""" - if not self.no_prompts: - log.info( - "[yellow]Press enter to use default values [cyan bold](shown in brackets) [yellow]or type your own responses" - ) - self._check_inputs() - self._set_profile() - self._check_profile() - self._run_pytests() - - def _check_inputs(self): - """Do more complex checks about supplied flags.""" - # Retrieving installed subworkflows - if self.repo_type == "modules": - installed_subwf = self.get_components_clone_modules() - else: - modules_json = ModulesJson(self.dir) - modules_json.check_up_to_date() - installed_subwf = modules_json.get_installed_subworkflows().get(self.modules_repo.remote_url) - - # Get the subworkflow name if not specified - if self.subworkflow_name is None: - if self.no_prompts: - raise UserWarning( - "Subworkflow name not provided and prompts deactivated. Please provide the Subworkflow name SUBWORKFLOW." - ) - if not installed_subwf: - raise UserWarning( - f"No installed subworkflows were found from '{self.modules_repo.remote_url}'.\n" - f"Are you running the tests inside the nf-core/modules main directory?\n" - f"Otherwise, make sure that the directory structure is subworkflows/SUBWORKFLOW/ and tests/subworkflows/SUBWORKFLOW/" - ) - self.subworkflow_name = questionary.autocomplete( - "Subworkflow name:", - choices=installed_subwf, - style=nf_core.utils.nfcore_question_style, - ).unsafe_ask() - - # Sanity check that the module directory exists - self._validate_folder_structure() - - def _validate_folder_structure(self): - """Validate that the modules follow the correct folder structure to run the tests: - - subworkflows/nf-core/SUBWORKFLOW/ - - tests/subworkflows/nf-core/SUBWORKFLOW/ - - """ - subworkflow_path = Path(self.default_subworkflows_path) / self.subworkflow_name - test_path = Path(self.default_subworkflows_tests_path) / self.subworkflow_name - - if not (self.dir / subworkflow_path).is_dir(): - raise UserWarning( - f"Cannot find directory '{subworkflow_path}'. Should be SUBWORKFLOW. Are you running the tests inside the nf-core/modules main directory?" - ) - if not (self.dir / test_path).is_dir(): - raise UserWarning( - f"Cannot find directory '{test_path}'. Should be SUBWORKFLOW. " - "Are you running the tests inside the nf-core/modules main directory? " - "Do you have tests for the specified module?" - ) - - def _set_profile(self): - """Set $PROFILE env variable. - The config expects $PROFILE and Nextflow fails if it's not set. - """ - if os.environ.get("PROFILE") is None: - os.environ["PROFILE"] = "" - if self.no_prompts: - log.info( - "Setting environment variable '$PROFILE' to an empty string as not set.\n" - "Tests will run with Docker by default. " - "To use Singularity set 'export PROFILE=singularity' in your shell before running this command." - ) - else: - question = { - "type": "list", - "name": "profile", - "message": "Choose software profile", - "choices": ["Docker", "Singularity", "Conda"], - } - answer = questionary.unsafe_prompt([question], style=nf_core.utils.nfcore_question_style) - profile = answer["profile"].lower() - os.environ["PROFILE"] = profile - log.info(f"Setting environment variable '$PROFILE' to '{profile}'") - - def _check_profile(self): - """Check if profile is available""" - profile = os.environ.get("PROFILE") - # Make sure the profile read from the environment is a valid Nextflow profile. - valid_nextflow_profiles = ["docker", "singularity", "conda"] - if profile in valid_nextflow_profiles: - if not which(profile): - raise UserWarning(f"Command '{profile}' not found - is it installed?") - else: - raise UserWarning( - f"The PROFILE '{profile}' set in the shell environment is not valid.\n" - f"Valid Nextflow profiles are '{', '.join(valid_nextflow_profiles)}'." - ) - - def _run_pytests(self): - """Given a subworkflow name, run tests.""" - # Print nice divider line - console = rich.console.Console() - console.rule(self.subworkflow_name, style="black") - - # Set pytest arguments - command_args = ["--tag", f"{self.subworkflow_name}", "--symlink", "--keep-workflow-wd", "--git-aware"] - command_args += self.pytest_args - - # Run pytest - log.info(f"Running pytest for module '{self.subworkflow_name}'") - sys.exit(pytest.main(command_args)) + super().__init__( + component_type="subworkflows", + module_name=subworkflow_name, # TODO: modify when components_test.py is refactored to work with subworkflows + no_prompts=no_prompts, + pytest_args=pytest_args, + remote_url=remote_url, + branch=branch, + no_pull=no_pull, + ) From 6cb550fe494a43657a54734b8c4c926b6e8eb403 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 8 Nov 2022 14:40:49 +0100 Subject: [PATCH 10/11] adapt components test for modules and subworkflows --- nf_core/components/components_test.py | 69 ++++++++++++++--------- nf_core/subworkflows/subworkflows_test.py | 4 +- 2 files changed, 43 insertions(+), 30 deletions(-) diff --git a/nf_core/components/components_test.py b/nf_core/components/components_test.py index 69218e5e5d..048c1a8f83 100644 --- a/nf_core/components/components_test.py +++ b/nf_core/components/components_test.py @@ -18,13 +18,13 @@ class ComponentsTest(ComponentCommand): """ - Class to run module pytests. + Class to run module and subworkflow pytests. ... Attributes ---------- - module_name : str + component_name : str name of the tool to run tests for no_prompts : bool flat indicating if prompts are used @@ -36,7 +36,7 @@ class ComponentsTest(ComponentCommand): run(): Run test steps _check_inputs(): - Check inputs. Ask for module_name if not provided and check that the directory exists + Check inputs. Ask for component_name if not provided and check that the directory exists _set_profile(): Set software profile _run_pytests(self): @@ -46,7 +46,7 @@ class ComponentsTest(ComponentCommand): def __init__( self, component_type, - module_name=None, + component_name=None, no_prompts=False, pytest_args="", remote_url=None, @@ -54,7 +54,7 @@ def __init__( no_pull=False, ): super().__init__(component_type=component_type, dir=".", remote_url=remote_url, branch=branch, no_pull=no_pull) - self.module_name = module_name + self.component_name = component_name self.no_prompts = no_prompts self.pytest_args = pytest_args @@ -76,27 +76,34 @@ def _check_inputs(self): # Retrieving installed modules if self.repo_type == "modules": - installed_modules = self.get_components_clone_modules() + installed_components = self.get_components_clone_modules() else: modules_json = ModulesJson(self.dir) modules_json.check_up_to_date() - installed_modules = modules_json.get_all_modules().get(self.modules_repo.remote_url) + if self.component_type == "modules": + installed_components = modules_json.get_all_modules().get(self.modules_repo.remote_url) + elif self.component_type == "subworkflows": + modules_json.get_installed_subworkflows().get(self.modules_repo.remote_url) - # Get the tool name if not specified - if self.module_name is None: + # Get the component name if not specified + if self.component_name is None: if self.no_prompts: raise UserWarning( - "Tool name not provided and prompts deactivated. Please provide the tool name as TOOL/SUBTOOL or TOOL." + f"{self.component_type[:-1].title()} name not provided and prompts deactivated. Please provide the {self.component_type[:-1]} name{' as TOOL/SUBTOOL or TOOL' if self.component_type == 'modules' else ''}." ) - if not installed_modules: + if not installed_components: + if self.component_type == "modules": + dir_structure_message = f"modules/{self.modules_repo.repo_path}/TOOL/SUBTOOL/ and tests/modules/{self.modules_repo.repo_path}/TOOLS/SUBTOOL/" + elif self.component_type == "subworkflows": + dir_structure_message = f"subworkflows/{self.modules_repo.repo_path}/SUBWORKFLOW/ and tests/subworkflows/{self.modules_repo.repo_path}/SUBWORKFLOW/" raise UserWarning( - f"No installed modules were found from '{self.modules_repo.remote_url}'.\n" - f"Are you running the tests inside the nf-core/modules main directory?\n" - f"Otherwise, make sure that the directory structure is modules/TOOL/SUBTOOL/ and tests/modules/TOOLS/SUBTOOL/" + f"No installed {self.component_type} were found from '{self.modules_repo.remote_url}'.\n" + f"Are you running the tests inside the repository root directory?\n" + f"Make sure that the directory structure is {dir_structure_message}" ) - self.module_name = questionary.autocomplete( + self.component_name = questionary.autocomplete( "Tool name:", - choices=installed_modules, + choices=installed_components, style=nf_core.utils.nfcore_question_style, ).unsafe_ask() @@ -107,19 +114,25 @@ def _validate_folder_structure(self): """Validate that the modules follow the correct folder structure to run the tests: - modules/nf-core/TOOL/SUBTOOL/ - tests/modules/nf-core/TOOL/SUBTOOL/ - + or + - subworkflows/nf-core/SUBWORKFLOW/ + - tests/subworkflows/nf-core/SUBWORKFLOW/ """ - module_path = Path(self.default_modules_path) / self.module_name - test_path = Path(self.default_tests_path) / self.module_name - - if not (self.dir / module_path).is_dir(): + if self.component_type == "modules": + component_path = Path(self.default_modules_path) / self.component_name + test_path = Path(self.default_tests_path) / self.component_name + elif self.component_type == "subworkflows": + component_path = Path(self.default_subworkflows_path) / self.component_name + test_path = Path(self.default_subworkflows_tests_path) / self.component_name + + if not (self.dir / component_path).is_dir(): raise UserWarning( - f"Cannot find directory '{module_path}'. Should be TOOL/SUBTOOL or TOOL. Are you running the tests inside the nf-core/modules main directory?" + f"Cannot find directory '{component_path}'. Should be {'TOOL/SUBTOOL or TOOL' if self.component_type == 'modules' else 'SUBWORKFLOW'}. Are you running the tests inside the modules repository root directory?" ) if not (self.dir / test_path).is_dir(): raise UserWarning( - f"Cannot find directory '{test_path}'. Should be TOOL/SUBTOOL or TOOL. " - "Are you running the tests inside the nf-core/modules main directory? " + f"Cannot find directory '{test_path}'. Should be {'TOOL/SUBTOOL or TOOL' if self.component_type == 'modules' else 'SUBWORKFLOW'}. " + "Are you running the tests inside the modules repository root directory? " "Do you have tests for the specified module?" ) @@ -162,15 +175,15 @@ def _check_profile(self): ) def _run_pytests(self): - """Given a module name, run tests.""" + """Given a module/subworkflow name, run tests.""" # Print nice divider line console = rich.console.Console() - console.rule(self.module_name, style="black") + console.rule(self.component_name, style="black") # Set pytest arguments - command_args = ["--tag", f"{self.module_name}", "--symlink", "--keep-workflow-wd", "--git-aware"] + command_args = ["--tag", f"{self.component_name}", "--symlink", "--keep-workflow-wd", "--git-aware"] command_args += self.pytest_args # Run pytest - log.info(f"Running pytest for module '{self.module_name}'") + log.info(f"Running pytest for {self.component_type[:-1]} '{self.component_name}'") sys.exit(pytest.main(command_args)) diff --git a/nf_core/subworkflows/subworkflows_test.py b/nf_core/subworkflows/subworkflows_test.py index 803e808bde..1f29418f49 100644 --- a/nf_core/subworkflows/subworkflows_test.py +++ b/nf_core/subworkflows/subworkflows_test.py @@ -13,7 +13,7 @@ class SubworkflowsTest(ComponentsTest): def __init__( self, - subworkflow_name=None, + component_name=None, no_prompts=False, pytest_args="", remote_url=None, @@ -22,7 +22,7 @@ def __init__( ): super().__init__( component_type="subworkflows", - module_name=subworkflow_name, # TODO: modify when components_test.py is refactored to work with subworkflows + module_name=component_name, no_prompts=no_prompts, pytest_args=pytest_args, remote_url=remote_url, From 5d598f202f911e3cc96fd2447b24e56995328688 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 8 Nov 2022 14:57:01 +0100 Subject: [PATCH 11/11] add component_name argument and fix text in tests --- nf_core/modules/modules_test.py | 2 +- nf_core/subworkflows/subworkflows_test.py | 4 ++-- tests/modules/modules_test.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nf_core/modules/modules_test.py b/nf_core/modules/modules_test.py index f02ea11cdb..94dfd344c6 100644 --- a/nf_core/modules/modules_test.py +++ b/nf_core/modules/modules_test.py @@ -22,7 +22,7 @@ def __init__( ): super().__init__( component_type="modules", - module_name=module_name, + component_name=module_name, no_prompts=no_prompts, pytest_args=pytest_args, remote_url=remote_url, diff --git a/nf_core/subworkflows/subworkflows_test.py b/nf_core/subworkflows/subworkflows_test.py index 1f29418f49..d072ff678a 100644 --- a/nf_core/subworkflows/subworkflows_test.py +++ b/nf_core/subworkflows/subworkflows_test.py @@ -13,7 +13,7 @@ class SubworkflowsTest(ComponentsTest): def __init__( self, - component_name=None, + subworkflow_name=None, no_prompts=False, pytest_args="", remote_url=None, @@ -22,7 +22,7 @@ def __init__( ): super().__init__( component_type="subworkflows", - module_name=component_name, + component_name=subworkflow_name, no_prompts=no_prompts, pytest_args=pytest_args, remote_url=remote_url, diff --git a/tests/modules/modules_test.py b/tests/modules/modules_test.py index ce57312bf1..eb207fa28b 100644 --- a/tests/modules/modules_test.py +++ b/tests/modules/modules_test.py @@ -25,7 +25,7 @@ def test_modules_test_no_name_no_prompts(self): meta_builder = nf_core.modules.ModulesTest(None, True, "") with pytest.raises(UserWarning) as excinfo: meta_builder._check_inputs() - assert "Tool name not provided and prompts deactivated." in str(excinfo.value) + assert "Module name not provided and prompts deactivated." in str(excinfo.value) def test_modules_test_no_installed_modules(self):