From 59eb9d3eafc290d16e9e74548272f2d27a0338c0 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 18 Feb 2019 14:39:57 +0100 Subject: [PATCH 01/12] Fix samtools sort taking too much memory --- main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.nf b/main.nf index c070f599c..9faa6db2f 100644 --- a/main.nf +++ b/main.nf @@ -675,7 +675,7 @@ if(params.aligner == 'hisat2'){ """ samtools sort \\ $hisat2_bam \\ - -@ ${task.cpus} $avail_mem \\ + -@ ${task.cpus} -m 2G \\ -o ${hisat2_bam.baseName}.sorted.bam samtools index ${hisat2_bam.baseName}.sorted.bam """ From dbf783acd2033cac4102a01132d3d76643359886 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 18 Feb 2019 14:40:31 +0100 Subject: [PATCH 02/12] Fix memory issues with hisat2 sorting --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 327c05ac3..62aa991c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ #### Bug fixes * Fixing HISAT2 Index Building for large reference genomes [#153](https://github.com/nf-core/rnaseq/issues/153) +* Fixing HISAT2 BAM sorting using more memory than available on the system #### Dependency Updates From 52bef548e86922a19b6c748e2b9c4bcbea2811d8 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 18 Feb 2019 19:00:22 +0100 Subject: [PATCH 03/12] Fix avail_mem usage hopefully --- main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.nf b/main.nf index 9faa6db2f..c437aaa7c 100644 --- a/main.nf +++ b/main.nf @@ -675,7 +675,7 @@ if(params.aligner == 'hisat2'){ """ samtools sort \\ $hisat2_bam \\ - -@ ${task.cpus} -m 2G \\ + -@ ${task.cpus} -m ${avail_mem} \\ -o ${hisat2_bam.baseName}.sorted.bam samtools index ${hisat2_bam.baseName}.sorted.bam """ From 3663260a05d6e0561e8ffb44abe5170c87a01e41 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 18 Feb 2019 19:04:42 +0100 Subject: [PATCH 04/12] Deduct - 100000000 to fix memory issues hopefully --- main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.nf b/main.nf index c437aaa7c..321c2887e 100644 --- a/main.nf +++ b/main.nf @@ -671,7 +671,7 @@ if(params.aligner == 'hisat2'){ file "where_are_my_files.txt" script: - def avail_mem = task.memory ? "-m ${task.memory.toBytes() / task.cpus}" : '' + def avail_mem = task.memory ? "-m ${(task.memory.toBytes() - 100000000) / task.cpus}" : '' """ samtools sort \\ $hisat2_bam \\ From 92e53ee834e92c6ad1eaa8b9e0d586d659924817 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 18 Feb 2019 19:35:58 +0100 Subject: [PATCH 05/12] remove the -m part --- main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.nf b/main.nf index 321c2887e..09f7c1d1a 100644 --- a/main.nf +++ b/main.nf @@ -675,7 +675,7 @@ if(params.aligner == 'hisat2'){ """ samtools sort \\ $hisat2_bam \\ - -@ ${task.cpus} -m ${avail_mem} \\ + -@ ${task.cpus} ${avail_mem} \\ -o ${hisat2_bam.baseName}.sorted.bam samtools index ${hisat2_bam.baseName}.sorted.bam """ From 63354e1281ea2cb16e8d9e1041c1e999301bb9b2 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 19 Feb 2019 16:13:37 +0100 Subject: [PATCH 06/12] Increase deduction --- main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.nf b/main.nf index 09f7c1d1a..49441e645 100644 --- a/main.nf +++ b/main.nf @@ -671,7 +671,7 @@ if(params.aligner == 'hisat2'){ file "where_are_my_files.txt" script: - def avail_mem = task.memory ? "-m ${(task.memory.toBytes() - 100000000) / task.cpus}" : '' + def avail_mem = task.memory ? "-m ${(task.memory.toBytes() - 1000000000) / task.cpus}" : '' """ samtools sort \\ $hisat2_bam \\ From 1f8de20f1daa9afa3b95679acb8a01b70c80983a Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Fri, 22 Feb 2019 09:16:56 +0100 Subject: [PATCH 07/12] use more cores (1 i/o, 4 sorting) --- conf/base.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/base.config b/conf/base.config index 07ece3176..bb5cd93c8 100644 --- a/conf/base.config +++ b/conf/base.config @@ -47,7 +47,7 @@ process { time = { check_max( 8.h * task.attempt, 'time' ) } } withName:hisat2_sortOutput { - cpus = { check_max( 4, 'cpus' ) } + cpus = { check_max( 5, 'cpus' ) } memory = { check_max( 32.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } From 60e6574c3e0e3cdc5964f3497d30780b5bc6af3f Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Fri, 22 Feb 2019 09:20:36 +0100 Subject: [PATCH 08/12] Use safer defaults --- main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.nf b/main.nf index 49441e645..bfc5bf54e 100644 --- a/main.nf +++ b/main.nf @@ -671,7 +671,7 @@ if(params.aligner == 'hisat2'){ file "where_are_my_files.txt" script: - def avail_mem = task.memory ? "-m ${(task.memory.toBytes() - 1000000000) / task.cpus}" : '' + def avail_mem = task.memory ? "-m ${(task.memory.toBytes() - 3000000000) / task.cpus}" : '' """ samtools sort \\ $hisat2_bam \\ From 0d8da06fa5ff333f7111b433822beec8f3ca2549 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Fri, 22 Feb 2019 09:22:15 +0100 Subject: [PATCH 09/12] Revert to using 4 cpus --- conf/base.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/base.config b/conf/base.config index bb5cd93c8..07ece3176 100644 --- a/conf/base.config +++ b/conf/base.config @@ -47,7 +47,7 @@ process { time = { check_max( 8.h * task.attempt, 'time' ) } } withName:hisat2_sortOutput { - cpus = { check_max( 5, 'cpus' ) } + cpus = { check_max( 4, 'cpus' ) } memory = { check_max( 32.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } From 3bdfa9fbc2f4171241f5c67f2342f1d54f824f88 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Fri, 22 Feb 2019 10:53:58 +0100 Subject: [PATCH 10/12] Better 5 down --- main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.nf b/main.nf index bfc5bf54e..b31b22440 100644 --- a/main.nf +++ b/main.nf @@ -671,7 +671,7 @@ if(params.aligner == 'hisat2'){ file "where_are_my_files.txt" script: - def avail_mem = task.memory ? "-m ${(task.memory.toBytes() - 3000000000) / task.cpus}" : '' + def avail_mem = task.memory ? "-m ${(task.memory.toBytes() - 5000000000) / task.cpus}" : '' """ samtools sort \\ $hisat2_bam \\ From a05c5bf72271f67e2524d4e4880cfbae5a4a9af2 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Fri, 22 Feb 2019 10:57:38 +0100 Subject: [PATCH 11/12] Use 6GB down --- main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.nf b/main.nf index b31b22440..d6bdb8b8d 100644 --- a/main.nf +++ b/main.nf @@ -671,7 +671,7 @@ if(params.aligner == 'hisat2'){ file "where_are_my_files.txt" script: - def avail_mem = task.memory ? "-m ${(task.memory.toBytes() - 5000000000) / task.cpus}" : '' + def avail_mem = task.memory ? "-m ${(task.memory.toBytes() - 6000000000) / task.cpus}" : '' """ samtools sort \\ $hisat2_bam \\ From afb19ad833050bb7f2aa816bd2018402ac131192 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 25 Feb 2019 16:57:11 +0100 Subject: [PATCH 12/12] Add check for enough memory to hisat2sort --- main.nf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/main.nf b/main.nf index d6bdb8b8d..2c4b62fe0 100644 --- a/main.nf +++ b/main.nf @@ -671,7 +671,8 @@ if(params.aligner == 'hisat2'){ file "where_are_my_files.txt" script: - def avail_mem = task.memory ? "-m ${(task.memory.toBytes() - 6000000000) / task.cpus}" : '' + def suff_mem = ("${(task.memory.toBytes() - 6000000000) / task.cpus}" > 2000000000) ? 'true' : 'false' + def avail_mem = (task.memory && suff_mem) ? "-m" + "${(task.memory.toBytes() - 6000000000) / task.cpus}" : '' """ samtools sort \\ $hisat2_bam \\