Merge pull request #2239 from mirpedrol/modified-gitlab-org

Fix tests after modifying the GitLab org-path to nf-core-test

.github/workflows/create-lint-wf.yml

@@ -107,25 +107,15 @@ jobs:
       - name: nf-core modules install
         run: nf-core --log-file log.txt modules install fastqc --dir nf-core-testpipeline/ --force
 
+      - name: nf-core modules install gitlab
+        run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git --branch branch-tester install fastp --dir nf-core-testpipeline/
+
       - name: nf-core modules list local
         run: nf-core --log-file log.txt modules list local --dir nf-core-testpipeline/
 
       - name: nf-core modules list remote
         run: nf-core --log-file log.txt modules list remote
 
-      - name: nf-core modules remove
-        run: |
-          find nf-core-testpipeline/workflows/ -type f -exec sed -i '/^include /d' {} \;
-          nf-core --log-file log.txt modules remove fastqc --dir nf-core-testpipeline/
-          nf-core --log-file log.txt modules remove multiqc --dir nf-core-testpipeline/
-          nf-core --log-file log.txt modules remove custom/dumpsoftwareversions --dir nf-core-testpipeline/
-
-      - name: nf-core modules install gitlab
-        run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git --branch main install fastqc --force --dir nf-core-testpipeline/
-
-      - name: nf-core modules list local gitlab
-        run: nf-core --log-file log.txt modules list local --dir nf-core-testpipeline/
-
       - name: nf-core modules list remote gitlab
         run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git list remote
 

CHANGELOG.md

@@ -39,7 +39,7 @@
 - Add the Nextflow version to Gitpod container matching the minimal Nextflow version for nf-core (according to `nextflow.config`) ([#2196](https://github.com/nf-core/tools/pull/2196))
 - Use `nfcore/gitpod:dev` container in the dev branch ([#2196](https://github.com/nf-core/tools/pull/2196))
 - Replace requests_mock with responses in test mocks ([#2165](https://github.com/nf-core/tools/pull/2165)).
-- Add warning when installing a module from an `org_path` that exists in multiple remotes in `modules.json` ([#2228](https://github.com/nf-core/tools/pull/2228)).
+- Add warning when installing a module from an `org_path` that exists in multiple remotes in `modules.json` ([#2228](https://github.com/nf-core/tools/pull/2228) [#2239](https://github.com/nf-core/tools/pull/2239)).
 
 ## [v2.7.2 - Mercury Eagle Patch](https://github.com/nf-core/tools/releases/tag/2.7.2) - [2022-12-19]
 

nf_core/modules/modules_json.py

@@ -427,8 +427,8 @@ def parse_dirs(self, dirs, missing_installation, component_type):
             git_url = None
             for repo in missing_installation:
                 if component_type in missing_installation[repo]:
-                    for dir_name in missing_installation[repo][component_type]:
-                        if component in missing_installation[repo][component_type][dir_name]:
+                    if install_dir in missing_installation[repo][component_type]:
+                        if component in missing_installation[repo][component_type][install_dir]:
                             component_in_file = True
                             git_url = repo
                             break

tests/modules/install.py

@@ -6,10 +6,10 @@
 from nf_core.modules.modules_json import ModulesJson
 
 from ..utils import (
+    GITLAB_BRANCH_ORG_PATH_BRANCH,
     GITLAB_BRANCH_TEST_BRANCH,
     GITLAB_REPO,
     GITLAB_URL,
-    remove_template_modules,
     with_temporary_folder,
 )
 
@@ -50,7 +50,6 @@ def test_modules_install_trimgalore_twice(self):
 
 def test_modules_install_from_gitlab(self):
     """Test installing a module from GitLab"""
-    remove_template_modules(self)
     assert self.mods_install_gitlab.install("fastqc") is True
 
 
@@ -63,7 +62,6 @@ def test_modules_install_different_branch_fail(self):
 
 def test_modules_install_different_branch_succeed(self):
     """Test installing a module from a different branch"""
-    remove_template_modules(self)
     install_obj = ModuleInstall(self.pipeline_dir, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_TEST_BRANCH)
     # The fastp module does exists in the branch-test branch
     assert install_obj.install("fastp") is True
@@ -90,6 +88,8 @@ def test_modules_install_tracking(self):
 
 def test_modules_install_alternate_remote(self):
     """Test installing a module from a different remote with the same organization path"""
-    install_obj = ModuleInstall(self.pipeline_dir, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_TEST_BRANCH)
-    # The fastp module does exists in the branch-test branch
-    assert install_obj.install("fastp") is False
+    install_obj = ModuleInstall(self.pipeline_dir, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_ORG_PATH_BRANCH)
+    # Install fastqc from GitLab which is also installed from GitHub with the same org_path
+    with pytest.raises(Exception) as excinfo:
+        install_obj.install("fastqc")
+        assert "Could not find a 'main.nf' or 'nextflow.config' file" in str(excinfo.value)

tests/modules/lint.py

@@ -5,7 +5,7 @@
 
 import nf_core.modules
 
-from ..utils import GITLAB_URL, remove_template_modules, set_wd
+from ..utils import GITLAB_URL, set_wd
 from .patch import BISMARK_ALIGN, CORRECT_SHA, PATCH_BRANCH, REPO_NAME, modify_main_nf
 
 
@@ -62,7 +62,6 @@ def test_modules_lint_no_gitlab(self):
 
 def test_modules_lint_gitlab_modules(self):
     """Lint modules from a different remote"""
-    remove_template_modules(self)
     self.mods_install_gitlab.install("fastqc")
     self.mods_install_gitlab.install("multiqc")
     module_lint = nf_core.modules.ModuleLint(dir=self.pipeline_dir, remote_url=GITLAB_URL)
@@ -74,7 +73,6 @@ def test_modules_lint_gitlab_modules(self):
 
 def test_modules_lint_multiple_remotes(self):
     """Lint modules from a different remote"""
-    remove_template_modules(self)
     self.mods_install_gitlab.install("multiqc")
     module_lint = nf_core.modules.ModuleLint(dir=self.pipeline_dir, remote_url=GITLAB_URL)
     module_lint.lint(print_results=False, all_modules=True)
@@ -87,7 +85,6 @@ def test_modules_lint_patched_modules(self):
     """
     Test creating a patch file and applying it to a new version of the the files
     """
-    remove_template_modules(self)
     setup_patch(self.pipeline_dir, True)
 
     # Create a patch file

tests/modules/patch.py

@@ -7,7 +7,7 @@
 import nf_core.components.components_command
 import nf_core.modules
 
-from ..utils import GITLAB_URL, remove_template_modules
+from ..utils import GITLAB_URL
 
 """
 Test the 'nf-core modules patch' command
@@ -16,17 +16,17 @@
 testing if the update commands works correctly with patch files
 """
 
-ORG_SHA = "775fcd090fb776a0be695044f8ab1af8896c8452"
-CORRECT_SHA = "335cd32405568ca3b6d4c05ab1e8a98c21e18a4d"
-SUCCEED_SHA = "f1566140c752e9c68fffc189fbe8cb9ee942b3ca"
-FAIL_SHA = "1fc8b0f953d915d66ee40d28bc337ff0998d05bd"
+ORG_SHA = "002623ccc88a3b0cb302c7d8f13792a95354d9f2"
+CORRECT_SHA = "63fd3cdb1be733041db74c15542a7b5b8f4095ed"
+SUCCEED_SHA = "ba15c20c032c549d77c5773659f19c2927daf48e"
+FAIL_SHA = "67b642d4471c4005220a342cad3818d5ba2b5a73"
 BISMARK_ALIGN = "bismark/align"
-REPO_NAME = "nf-core"
+REPO_NAME = "nf-core-test"
 PATCH_BRANCH = "patch-tester"
 REPO_URL = "https://gitlab.com/nf-core/modules-test.git"
 
 
-def setup_patch(pipeline_dir, modify_module, pipeline_name):
+def setup_patch(pipeline_dir, modify_module):
     install_obj = nf_core.modules.ModuleInstall(
         pipeline_dir, prompt=False, force=False, remote_url=GITLAB_URL, branch=PATCH_BRANCH, sha=ORG_SHA
     )
@@ -40,15 +40,6 @@ def setup_patch(pipeline_dir, modify_module, pipeline_name):
         modify_main_nf(module_path / "main.nf")
 
 
-def modify_workflow_nf(path):
-    with open(path, "r") as fh:
-        lines = fh.readlines()
-    with open(path, "w") as fh:
-        for line in lines:
-            if not line.startswith("include {"):
-                fh.write(line)
-
-
 def modify_main_nf(path):
     """Modify a file to test patch creation"""
     with open(path, "r") as fh:
@@ -69,10 +60,7 @@ def modify_main_nf(path):
 
 def test_create_patch_no_change(self):
     """Test creating a patch when there is no change to the module"""
-    # Remove modules that may cause org_path conflict
-    remove_template_modules(self)
-
-    setup_patch(self.pipeline_dir, False, self.pipeline_name)
+    setup_patch(self.pipeline_dir, False)
 
     # Try creating a patch file
     patch_obj = nf_core.modules.ModulePatch(self.pipeline_dir, GITLAB_URL, PATCH_BRANCH)
@@ -91,10 +79,7 @@ def test_create_patch_no_change(self):
 
 def test_create_patch_change(self):
     """Test creating a patch when there is a change to the module"""
-    # Remove modules that may cause org_path conflict
-    remove_template_modules(self)
-
-    setup_patch(self.pipeline_dir, True, self.pipeline_name)
+    setup_patch(self.pipeline_dir, True)
 
     # Try creating a patch file
     patch_obj = nf_core.modules.ModulePatch(self.pipeline_dir, GITLAB_URL, PATCH_BRANCH)
@@ -127,10 +112,8 @@ def test_create_patch_try_apply_successful(self):
     """
     Test creating a patch file and applying it to a new version of the the files
     """
-    # Remove modules that may cause org_path conflict
-    remove_template_modules(self)
 
-    setup_patch(self.pipeline_dir, True, self.pipeline_name)
+    setup_patch(self.pipeline_dir, True)
     module_relpath = Path("modules", REPO_NAME, BISMARK_ALIGN)
     module_path = Path(self.pipeline_dir, module_relpath)
 
@@ -196,10 +179,8 @@ def test_create_patch_try_apply_failed(self):
     """
     Test creating a patch file and applying it to a new version of the the files
     """
-    # Remove modules that may cause org_path conflict
-    remove_template_modules(self)
 
-    setup_patch(self.pipeline_dir, True, self.pipeline_name)
+    setup_patch(self.pipeline_dir, True)
     module_relpath = Path("modules", REPO_NAME, BISMARK_ALIGN)
     module_path = Path(self.pipeline_dir, module_relpath)
 
@@ -237,10 +218,8 @@ def test_create_patch_update_success(self):
     Should have the same effect as 'test_create_patch_try_apply_successful'
     but uses higher level api
     """
-    # Remove modules that may cause org_path conflict
-    remove_template_modules(self)
 
-    setup_patch(self.pipeline_dir, True, self.pipeline_name)
+    setup_patch(self.pipeline_dir, True)
     module_path = Path(self.pipeline_dir, "modules", REPO_NAME, BISMARK_ALIGN)
 
     # Try creating a patch file
@@ -301,10 +280,8 @@ def test_create_patch_update_fail(self):
     """
     Test creating a patch file and updating a module when there is a diff conflict
     """
-    # Remove modules that may cause org_path conflict
-    remove_template_modules(self)
 
-    setup_patch(self.pipeline_dir, True, self.pipeline_name)
+    setup_patch(self.pipeline_dir, True)
     module_path = Path(self.pipeline_dir, "modules", REPO_NAME, BISMARK_ALIGN)
 
     # Try creating a patch file

tests/modules/remove.py

@@ -1,7 +1,5 @@
 import os
 
-from ..utils import remove_template_modules
-
 
 def test_modules_remove_trimgalore(self):
     """Test removing TrimGalore! module after installing it"""
@@ -18,10 +16,7 @@ def test_modules_remove_trimgalore_uninstalled(self):
 
 def test_modules_remove_multiqc_from_gitlab(self):
     """Test removing multiqc module after installing it from an alternative source"""
-    # Remove modules that may cause org_path conflict
-    remove_template_modules(self)
-
     self.mods_install_gitlab.install("multiqc")
-    module_path = os.path.join(self.mods_install_gitlab.dir, "modules", "nf-core", "multiqc")
+    module_path = os.path.join(self.mods_install_gitlab.dir, "modules", "nf-core-test", "multiqc")
     assert self.mods_remove_gitlab.remove("multiqc", force=True)
     assert os.path.exists(module_path) is False

tests/modules/update.py

@@ -24,7 +24,6 @@
     GITLAB_URL,
     OLD_TRIMGALORE_BRANCH,
     OLD_TRIMGALORE_SHA,
-    remove_template_modules,
 )
 
 
@@ -45,7 +44,6 @@ def test_install_and_update(self):
 
 def test_install_at_hash_and_update(self):
     """Installs an old version of a module in the pipeline and updates it"""
-    remove_template_modules(self)
     assert self.mods_install_old.install("trimgalore")
     update_obj = ModuleUpdate(
         self.pipeline_dir, show_diff=False, update_deps=True, remote_url=GITLAB_URL, branch=OLD_TRIMGALORE_BRANCH
@@ -71,7 +69,6 @@ def test_install_at_hash_and_update(self):
 
 def test_install_at_hash_and_update_and_save_diff_to_file(self):
     """Installs an old version of a module in the pipeline and updates it"""
-    remove_template_modules(self)
     self.mods_install_old.install("trimgalore")
     patch_path = os.path.join(self.pipeline_dir, "trimgalore.patch")
     update_obj = ModuleUpdate(
@@ -112,7 +109,6 @@ def test_update_all(self):
 
 def test_update_with_config_fixed_version(self):
     """Try updating when there are entries in the .nf-core.yml"""
-    remove_template_modules(self)
     # Install trimgalore at the latest version
     assert self.mods_install_trimgalore.install("trimgalore")
 
@@ -138,7 +134,6 @@ def test_update_with_config_fixed_version(self):
 
 def test_update_with_config_dont_update(self):
     """Try updating when module is to be ignored"""
-    remove_template_modules(self)
     # Install an old version of trimgalore
     self.mods_install_old.install("trimgalore")
 
@@ -169,7 +164,6 @@ def test_update_with_config_dont_update(self):
 
 def test_update_with_config_fix_all(self):
     """Fix the version of all nf-core modules"""
-    remove_template_modules(self)
     self.mods_install_trimgalore.install("trimgalore")
 
     # Fix the version of all nf-core modules in the .nf-core.yml to an old version
@@ -193,8 +187,7 @@ def test_update_with_config_fix_all(self):
 
 def test_update_with_config_no_updates(self):
     """Don't update any nf-core modules"""
-    remove_template_modules(self)
-    self.mods_install_old.install("trimgalore")
+    assert self.mods_install_old.install("trimgalore")
     old_mod_json = ModulesJson(self.pipeline_dir).get_modules_json()
 
     # Fix the version of all nf-core modules in the .nf-core.yml to an old version
@@ -227,7 +220,6 @@ def test_update_with_config_no_updates(self):
 
 def test_update_different_branch_single_module(self):
     """Try updating a module in a specific branch"""
-    remove_template_modules(self)
     install_obj = ModuleInstall(
         self.pipeline_dir,
         prompt=False,
@@ -254,7 +246,6 @@ def test_update_different_branch_single_module(self):
 
 def test_update_different_branch_mixed_modules_main(self):
     """Try updating all modules where MultiQC is installed from main branch"""
-    remove_template_modules(self)
     # Install fastp
     assert self.mods_install_gitlab_old.install("fastp")
 
@@ -281,7 +272,6 @@ def test_update_different_branch_mixed_modules_main(self):
 
 def test_update_different_branch_mix_modules_branch_test(self):
     """Try updating all modules where MultiQC is installed from branch-test branch"""
-    remove_template_modules(self)
     # Install multiqc from the branch-test branch
     assert self.mods_install_gitlab_old.install(
         "multiqc"

tests/subworkflows/install.py

@@ -9,8 +9,8 @@
     GITLAB_BRANCH_TEST_BRANCH,
     GITLAB_REPO,
     GITLAB_SUBWORKFLOWS_BRANCH,
+    GITLAB_SUBWORKFLOWS_ORG_PATH_BRANCH,
     GITLAB_URL,
-    remove_template_modules,
     with_temporary_folder,
 )
 
@@ -63,7 +63,6 @@ def test_subworkflows_install_bam_sort_stats_samtools_twice(self):
 
 def test_subworkflows_install_from_gitlab(self):
     """Test installing a subworkflow from GitLab"""
-    remove_template_modules(self)
     assert self.subworkflow_install_gitlab.install("bam_stats_samtools") is True
     # Verify that the branch entry was added correctly
     modules_json = ModulesJson(self.pipeline_dir)
@@ -145,6 +144,11 @@ def test_subworkflows_install_tracking_added_super_subworkflow(self):
 
 
 def test_subworkflows_install_alternate_remote(self):
-    """Test installing a subworkflow from a different remote with the same organization path"""
-    self.subworkflow_install.install("bam_sort_stats_samtools")
-    assert self.subworkflow_install_gitlab.install("bam_stats_samtools") is False
+    """Test installing a module from a different remote with the same organization path"""
+    install_obj = SubworkflowInstall(
+        self.pipeline_dir, remote_url=GITLAB_URL, branch=GITLAB_SUBWORKFLOWS_ORG_PATH_BRANCH
+    )
+    # Install a subworkflow from GitLab which is also installed from GitHub with the same org_path
+    with pytest.raises(Exception) as excinfo:
+        install_obj.install("fastqc")
+        assert "Could not find a 'main.nf' or 'nextflow.config' file" in str(excinfo.value)

tests/subworkflows/list.py

@@ -2,7 +2,7 @@
 
 import nf_core.subworkflows
 
-from ..utils import GITLAB_SUBWORKFLOWS_BRANCH, GITLAB_URL, remove_template_modules
+from ..utils import GITLAB_SUBWORKFLOWS_BRANCH, GITLAB_URL
 
 
 def test_subworkflows_list_remote(self):
@@ -40,7 +40,6 @@ def test_subworkflows_install_and_list_subworkflows(self):
 
 def test_subworkflows_install_gitlab_and_list_subworkflows(self):
     """Test listing locally installed subworkflows"""
-    remove_template_modules(self)
     self.subworkflow_install_gitlab.install("bam_sort_stats_samtools")
     subworkflows_list = nf_core.subworkflows.SubworkflowList(self.pipeline_dir, remote=False)
     listed_subworkflows = subworkflows_list.list_components()