[FIX] AWS and conda (#163)

* [FIX] AWS and conda

* [CI] add conda tests

* More fixes

* remove debug

* add mamba to gh actions?

* channels?

* remove debug var

* No dev version

* Update ci.yml

* Update ci.yml

* lint

.github/workflows/ci.yml

@@ -14,12 +14,14 @@ jobs:
       NXF_ANSI_LOG: false
       CAPSULE_LOG: none
       TEST_PROFILE: ${{ matrix.test_profile }}
+      EXEC_PROFILE: ${{ matrix.exec_profile }}
 
     name: Run pipeline with test data
     # Only run on push if this is the nf-core dev branch (merged PRs)
     if: ${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/quantms') }}
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         # Nextflow versions
         include:
@@ -30,6 +32,12 @@ jobs:
           - NXF_VER: ""
             NXF_EDGE: "1"
         test_profile: ["test", "test_lfq", "test_dia", "test_localize"]
+        exec_profile: ["docker", "conda"]
+        exclude:
+          - test_profile: test_dia
+            exec_profile: conda
+          - test_profile: test_localize
+            exec_profile: conda
     steps:
       - name: Check out pipeline code
         uses: actions/checkout@v2
@@ -44,18 +52,44 @@ jobs:
           wget -qO- get.nextflow.io | bash
           sudo mv nextflow /usr/local/bin/
 
+      - name: Install mamba
+        if: matrix.exec_profile == 'conda'
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          python-version: 3.9
+          mamba-version: "*"
+          channels: conda-forge,defaults
+
+      #- name: Install micromamba as mamba
+      #  if: matrix.exec_profile == 'conda'
+      #  run: |
+      #    wget -qO- https://micromamba.snakepit.net/api/micromamba/linux-64/latest | tar -xvj bin/micromamba
+      #    mv bin/micromamba bin/mamba
+      #    echo "$(pwd)/bin" >> $GITHUB_PATH
+      #    echo "$(pwd)/mamba/bin" >> $GITHUB_PATH
+      #    ./bin/mamba shell init -s bash -p ./mamba
+
       - name: Run pipeline with test data
+        if: matrix.exec_profile != 'conda'
+        # TODO nf-core: You can customise CI pipeline run tests as required
+        # For example: adding multiple test runs with different parameters
+        # Remember that you can parallelise this by using strategy.matrix
+        run: |
+          nextflow run ${GITHUB_WORKSPACE} -profile $TEST_PROFILE,$EXEC_PROFILE --outdir ${TEST_PROFILE}_${EXEC_PROFILE}_results
+
+      - name: Run pipeline with test data in conda profile (and single-threaded)
+        if: matrix.exec_profile == 'conda'
         # TODO nf-core: You can customise CI pipeline run tests as required
         # For example: adding multiple test runs with different parameters
         # Remember that you can parallelise this by using strategy.matrix
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -profile $TEST_PROFILE,docker --outdir ${TEST_PROFILE}_results
+          nextflow run ${GITHUB_WORKSPACE} -profile $TEST_PROFILE,$EXEC_PROFILE,mambaci --outdir ${TEST_PROFILE}_${EXEC_PROFILE}_results
 
       - name: Gather failed logs
         if: failure() || cancelled()
         run: |
           mkdir failed_logs
-          failed=$(grep "FAILED" ${TEST_PROFILE}_results/pipeline_info/execution_trace.txt | cut -f 2)
+          failed=$(grep "FAILED" ${TEST_PROFILE}_${EXEC_PROFILE}_results/pipeline_info/execution_trace.txt | cut -f 2)
           while read -r line ; do cp $(ls work/${line}*/*.log) failed_logs/ | true ; done <<< "$failed"
       - uses: actions/upload-artifact@v1
         if: failure() || cancelled()
@@ -67,8 +101,8 @@ jobs:
         if: always()
         name: Upload results
         with:
-          name: ${{ env.TEST_PROFILE }}_results
-          path: ${{ env.TEST_PROFILE }}_results
+          name: ${{ env.TEST_PROFILE }}_${{ env.EXEC_PROFILE }}_results
+          path: ${{ env.TEST_PROFILE }}_${{ env.EXEC_PROFILE }}_results
       - uses: actions/upload-artifact@v1
         if: always()
         name: Upload log

.gitignore

@@ -10,3 +10,4 @@ testing*
 .idea/*
 *.log
 /build/
+results*/

conf/mambaci.config

@@ -0,0 +1,12 @@
+// We need this because mamba is completely bugged
+// https://github.com/mamba-org/mamba/issues/1429
+// and nextflow does not support micromamba
+// https://github.com/mamba-org/mamba/issues/1654
+process {
+    executor = 'local'
+    maxForks = 1
+}
+executor {
+    pollInterval = '5sec'
+    queueSize = 1
+}

conf/test.config

@@ -20,6 +20,7 @@ params {
     max_time   = '6.h'
 
     outdir = "./results_iso"
+    tracedir = "${params.outdir}/pipeline_info"
 
     // Input data
     input = 'https://raw.githubusercontent.com/nf-core/test-datasets/quantms/testdata/tmt_ci/PXD000001.sdrf.tsv'

conf/test_dia.config

@@ -20,6 +20,7 @@ params {
     max_time = 48.h
 
     outdir = './results_dia'
+    tracedir = "${params.outdir}/pipeline_info"
 
     // Input data
     input = 'https://raw.githubusercontent.com/nf-core/test-datasets/quantms/testdata/lfq_ci/PXD026600/PXD026600.sdrf.tsv'

conf/test_full.config

@@ -15,6 +15,7 @@ params {
     config_profile_description = 'Full test dataset in isotopic labelling mode to check pipeline function and sanity of results'
 
     outdir = "./results_iso_full"
+    tracedir = "${params.outdir}/pipeline_info"
 
     // Input data for full size test
     input = 'https://raw.githubusercontent.com/nf-core/test-datasets/quantms/testdata/tmt_ci/PXD000001.sdrf.tsv'

conf/test_lfq.config

@@ -20,6 +20,7 @@ params {
     max_time = 48.h
 
     outdir = "./results_lfq"
+    tracedir = "${params.outdir}/pipeline_info"
 
     // Input data
     labelling_type = "label free sample"

conf/test_localize.config

@@ -20,6 +20,7 @@ params {
     max_time = 1.h
 
     outdir = "./results_localize"
+    tracedir = "${params.outdir}/pipeline_info"
 
     // Input data
     input = 'https://raw.githubusercontent.com/nf-core/test-datasets/quantms/testdata/lfq_ci_phospho/test_phospho.sdrf'

lib/WorkflowMain.groovy

@@ -78,6 +78,20 @@ class WorkflowMain {
             System.exit(1)
         }
 
+        // Check input has been provided
+        if (!params.outdir) {
+            log.error "Please provide an outdir to the pipeline e.g. '--outdir ./results'"
+            System.exit(1)
+        }
+
+        if (params.tracedir == "null/pipeline_info")
+        {
+            log.error """Error: Your tracedir is `null/pipeline_info`, this means you probably set outdir in a way that does not affect the default
+            `\$params.outdir/pipeline_info` (e.g., by specifying outdir in a profile instead of the commandline or through a `-params-file`.
+            Either set outdir in a correct way, or redefine tracedir as well (e.g., in your profile)."""
+            System.exit(1)
+        }
+
         // check fasta database has been provided
         if (!params.database) {
             log.error "Please provide an fasta database to the pipeline e.g. '--database *.fasta'"

modules/local/openms/consensusid/main.nf

@@ -4,7 +4,7 @@ process CONSENSUSID {
     label 'process_single_thread'
     label 'openms'
 
-    conda (params.enable_conda ? "openms::openms=2.8.0" : null)
+    conda (params.enable_conda ? "bioconda::openms=2.8.0" : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/openms:2.8.0--h7ca0330_1' :
         'quay.io/biocontainers/openms:2.8.0--h7ca0330_1' }"

modules/local/openms/decoydatabase/main.nf

@@ -2,7 +2,7 @@ process DECOYDATABASE {
     label 'process_very_low'
     label 'openms'
 
-    conda (params.enable_conda ? "openms::openms=2.8.0" : null)
+    conda (params.enable_conda ? "bioconda::openms=2.8.0" : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/openms:2.8.0--h7ca0330_1' :
         'quay.io/biocontainers/openms:2.8.0--h7ca0330_1' }"

modules/local/openms/extractpsmfeatures/main.nf

@@ -3,7 +3,7 @@ process EXTRACTPSMFEATURES {
     label 'process_single_thread'
     label 'openms'
 
-    conda (params.enable_conda ? "bioconda::bumbershoot bioconda::comet-ms bioconda::crux-toolkit=3.2 bioconda::fido=1.0 conda-forge::gnuplot bioconda::luciphor2=2020_04_03 bioconda::msgf_plus=2021.03.22 bioconda::openms=2.8.0 bioconda::pepnovo=20101117 bioconda::percolator=3.5 bioconda::sirius-csifingerid=4.0.1 bioconda::thermorawfileparser=1.3.4 bioconda::xtandem=15.12.15.2 bioconda::openms-thirdparty=2.8.0" : null)
+    conda (params.enable_conda ? "bioconda::openms=2.8.0" : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/openms:2.8.0--h7ca0330_1' :
         'quay.io/biocontainers/openms:2.8.0--h7ca0330_1' }"

modules/local/openms/falsediscoveryrate/main.nf

@@ -3,7 +3,7 @@ process FALSEDISCOVERYRATE {
     label 'process_single_thread'
     label 'openms'
 
-    conda (params.enable_conda ? "openms::openms=2.8.0" : null)
+    conda (params.enable_conda ? "bioconda::openms=2.8.0" : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/openms:2.8.0--h7ca0330_1' :
         'quay.io/biocontainers/openms:2.8.0--h7ca0330_1' }"

modules/local/openms/filemerge/main.nf

@@ -3,7 +3,7 @@ process FILEMERGE {
     label 'process_single_thread'
     label 'openms'
 
-    conda (params.enable_conda ? "openms::openms=2.8.0" : null)
+    conda (params.enable_conda ? "bioconda::openms=2.8.0" : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/openms:2.8.0--h7ca0330_1' :
         'quay.io/biocontainers/openms:2.8.0--h7ca0330_1' }"

modules/local/openms/idconflictresolver/main.nf

@@ -2,7 +2,7 @@ process IDCONFLICTRESOLVER {
     label 'process_low'
     label 'openms'
 
-    conda (params.enable_conda ? "openms::openms=2.8.0" : null)
+    conda (params.enable_conda ? "bioconda::openms=2.8.0" : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/openms:2.8.0--h7ca0330_1' :
         'quay.io/biocontainers/openms:2.8.0--h7ca0330_1' }"

modules/local/openms/idfilter/main.nf

@@ -4,7 +4,7 @@ process IDFILTER {
     label 'process_single_thread'
     label 'openms'
 
-    conda (params.enable_conda ? "openms::openms=2.8.0" : null)
+    conda (params.enable_conda ? "bioconda::openms=2.8.0" : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/openms:2.8.0--h7ca0330_1' :
         'quay.io/biocontainers/openms:2.8.0--h7ca0330_1' }"

modules/local/openms/idmapper/main.nf

@@ -4,7 +4,7 @@ process IDMAPPER {
     label 'process_medium'
     label 'openms'
 
-    conda (params.enable_conda ? "openms::openms=2.8.0" : null)
+    conda (params.enable_conda ? "bioconda::openms=2.8.0" : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/openms:2.8.0--h7ca0330_1' :
         'quay.io/biocontainers/openms:2.8.0--h7ca0330_1' }"

modules/local/openms/idpep/main.nf

@@ -1,7 +1,7 @@
 process IDPEP {
     label 'process_very_low'
 
-    conda (params.enable_conda ? "openms::openms=2.8.0" : null)
+    conda (params.enable_conda ? "bioconda::openms=2.8.0" : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/openms:2.8.0--h7ca0330_1' :
         'quay.io/biocontainers/openms:2.8.0--h7ca0330_1' }"

modules/local/openms/idscoreswitcher/main.nf

@@ -3,7 +3,7 @@ process IDSCORESWITCHER {
     label 'process_very_low'
     label 'process_single_thread'
 
-    conda (params.enable_conda ? "openms::openms=2.8.0" : null)
+    conda (params.enable_conda ? "bioconda::openms=2.8.0" : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/openms:2.8.0--h7ca0330_1' :
         'quay.io/biocontainers/openms:2.8.0--h7ca0330_1' }"

modules/local/openms/indexpeptides/main.nf

@@ -1,7 +1,7 @@
 process INDEXPEPTIDES {
     label 'process_low'
 
-    conda (params.enable_conda ? "openms::openms=2.8.0" : null)
+    conda (params.enable_conda ? "bioconda::openms=2.8.0" : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/openms:2.8.0--h7ca0330_1' :
         'quay.io/biocontainers/openms:2.8.0--h7ca0330_1' }"

modules/local/openms/isobaricanalyzer/main.nf

@@ -2,7 +2,7 @@ process ISOBARICANALYZER {
     tag "$meta.id"
     label 'process_medium'
 
-    conda (params.enable_conda ? "openms::openms=2.8.0" : null)
+    conda (params.enable_conda ? "bioconda::openms=2.8.0" : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/openms:2.8.0--h7ca0330_1' :
         'quay.io/biocontainers/openms:2.8.0--h7ca0330_1' }"

modules/local/openms/msstatsconverter/main.nf

@@ -1,7 +1,7 @@
 process MSSTATSCONVERTER {
     label 'process_low'
 
-    conda (params.enable_conda ? "openms::openms=2.8.0" : null)
+    conda (params.enable_conda ? "bioconda::openms=2.8.0" : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/openms:2.8.0--h7ca0330_1' :
         'quay.io/biocontainers/openms:2.8.0--h7ca0330_1' }"

modules/local/openms/mzmlindexing/main.nf

@@ -2,7 +2,7 @@ process MZMLINDEXING {
     tag "$meta.id"
     label 'process_low'
 
-    conda (params.enable_conda ? "openms::openms=2.8.0" : null)
+    conda (params.enable_conda ? "bioconda::openms=2.8.0" : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/openms:2.8.0--h7ca0330_1' :
         'quay.io/biocontainers/openms:2.8.0--h7ca0330_1' }"

modules/local/openms/openmspeakpicker/main.nf

@@ -2,7 +2,7 @@ process OPENMSPEAKPICKER {
     tag "$meta.id"
     label 'process_low'
 
-    conda (params.enable_conda ? "openms::openms=2.8.0" : null)
+    conda (params.enable_conda ? "bioconda::openms=2.8.0" : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/openms:2.8.0--h7ca0330_1' :
         'quay.io/biocontainers/openms:2.8.0--h7ca0330_1' }"

modules/local/openms/proteininference/main.nf

@@ -1,7 +1,7 @@
 process PROTEININFERENCE {
     label 'process_medium'
 
-    conda (params.enable_conda ? "openms::openms=2.8.0" : null)
+    conda (params.enable_conda ? "bioconda::openms=2.8.0" : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/openms:2.8.0--h7ca0330_1' :
         'quay.io/biocontainers/openms:2.8.0--h7ca0330_1' }"

modules/local/openms/proteinquantifier/main.nf

@@ -1,7 +1,7 @@
 process PROTEINQUANTIFIER {
     label 'process_medium'
 
-    conda (params.enable_conda ? "openms::openms=2.8.0" : null)
+    conda (params.enable_conda ? "bioconda::openms=2.8.0" : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/openms:2.8.0--h7ca0330_1' :
         'quay.io/biocontainers/openms:2.8.0--h7ca0330_1' }"

modules/local/openms/proteomicslfq/main.nf

@@ -1,7 +1,7 @@
 process PROTEOMICSLFQ {
     label 'process_high'
 
-    conda (params.enable_conda ? "openms::openms=2.8.0.dev" : null)
+    conda (params.enable_conda ? "bioconda::openms=2.8.0" : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/openms:2.8.0--h7ca0330_1' :
         'quay.io/biocontainers/openms:2.8.0--h7ca0330_1' }"

modules/local/openms/thirdparty/luciphoradapter/main.nf

@@ -2,7 +2,7 @@ process LUCIPHORADAPTER {
     tag "$meta.id"
     label 'process_medium'
 
-    conda (params.enable_conda ? "bioconda::bumbershoot bioconda::comet-ms bioconda::crux-toolkit=3.2 bioconda::fido=1.0 conda-forge::gnuplot bioconda::luciphor2=2020_04_03 bioconda::msgf_plus=2021.03.22 bioconda::openms=2.8.0 bioconda::pepnovo=20101117 bioconda::percolator=3.5 bioconda::sirius-csifingerid=4.0.1 bioconda::thermorawfileparser=1.3.4 bioconda::xtandem=15.12.15.2 bioconda::openms-thirdparty=2.8.0" : null)
+    conda (params.enable_conda ? "bioconda::openms-thirdparty=2.8.0" : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/openms-thirdparty:2.8.0--h9ee0642_0' :
         'quay.io/biocontainers/openms-thirdparty:2.8.0--h9ee0642_0' }"
@@ -17,9 +17,12 @@ process LUCIPHORADAPTER {
     path "*.log", emit: log
 
     script:
+    // The OpenMS adapters need the actuall jar file, not the executable/shell wrapper that (bio)conda creates
     luciphor_jar = ''
-    if (workflow.containerEngine) {
+    if (workflow.containerEngine || (task.executor == "awsbatch")) {
         luciphor_jar = "-executable \$(find /usr/local/share/luciphor2-*/luciphor2.jar -maxdepth 0)"
+    } else if (params.enable_conda) {
+        luciphor_jar = "-executable \$(find \$CONDA_PREFIX/share/luciphor2-*/luciphor2.jar -maxdepth 0)"
     }
 
     def args = task.ext.args ?: ''

modules/local/openms/thirdparty/percolator/main.nf

@@ -1,7 +1,7 @@
 process PERCOLATOR {
     label 'process_medium'
 
-    conda (params.enable_conda ? "bioconda::bumbershoot bioconda::comet-ms bioconda::crux-toolkit=3.2 bioconda::fido=1.0 conda-forge::gnuplot bioconda::luciphor2=2020_04_03 bioconda::msgf_plus=2021.03.22 bioconda::openms=2.8.0 bioconda::pepnovo=20101117 bioconda::percolator=3.5 bioconda::sirius-csifingerid=4.0.1 bioconda::thermorawfileparser=1.3.4 bioconda::xtandem=15.12.15.2 bioconda::openms-thirdparty=2.8.0" : null)
+    conda (params.enable_conda ? "bioconda::openms-thirdparty=2.8.0" : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/openms-thirdparty:2.8.0--h9ee0642_0' :
         'quay.io/biocontainers/openms-thirdparty:2.8.0--h9ee0642_0' }"

modules/local/openms/thirdparty/searchenginemsgf/main.nf

@@ -16,10 +16,12 @@ process SEARCHENGINEMSGF {
     path "*.log",   emit: log
 
     script:
-    // find a way to add MSGFPlus.jar dependence
+    // The OpenMS adapters need the actuall jar file, not the executable/shell wrapper that (bio)conda creates
     msgf_jar = ''
-    if (workflow.containerEngine) {
+    if (workflow.containerEngine || (task.executor == "awsbatch")) {
         msgf_jar = "-executable \$(find /usr/local/share/msgf_plus-*/MSGFPlus.jar -maxdepth 0)"
+    } else if (params.enable_conda) {
+        msgf_jar = "-executable \$(find \$CONDA_PREFIX/share/msgf_plus-*/MSGFPlus.jar -maxdepth 0)"
     }
 
     def args = task.ext.args ?: ''

modules/local/pmultiqc/main.nf

@@ -1,7 +1,7 @@
 process PMULTIQC {
     label 'process_high'
 
-    conda (params.enable_conda ? "conda-forge::pandas_schema conda-forge::lzstring bioconda::pmultiqc=0.0.10" : null)
+    conda (params.enable_conda ? "conda-forge::pandas_schema conda-forge::lzstring bioconda::pmultiqc=0.0.11" : null)
     if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) {
         container "https://depot.galaxyproject.org/singularity/pmultiqc:0.0.11--pyhdfd78af_0"
     } else {