diff --git a/modules/nf-commons/src/main/nextflow/file/CopyMoveHelper.java b/modules/nf-commons/src/main/nextflow/file/CopyMoveHelper.java index 7c978e65d2..c42398e9b0 100644 --- a/modules/nf-commons/src/main/nextflow/file/CopyMoveHelper.java +++ b/modules/nf-commons/src/main/nextflow/file/CopyMoveHelper.java @@ -87,7 +87,9 @@ private static void copyFile(Path source, Path target, boolean foreign, CopyOpti return; } - try (InputStream in = Files.newInputStream(source)) { + // open the remote input stream using FULL_DOWNLOAD option, so it will error if not + // all bytes are read before closing + try (InputStream in = Files.newInputStream(source, ForeignOpenOption.FULL_DOWNLOAD)) { Files.copy(in, target); } } diff --git a/modules/nf-commons/src/main/nextflow/file/ForeignOpenOption.java b/modules/nf-commons/src/main/nextflow/file/ForeignOpenOption.java new file mode 100644 index 0000000000..6b5ebe108b --- /dev/null +++ b/modules/nf-commons/src/main/nextflow/file/ForeignOpenOption.java @@ -0,0 +1,14 @@ +package nextflow.file; + +import java.nio.file.OpenOption; + +/** + * File open options for remote (foreign) file systems + */ +public enum ForeignOpenOption implements OpenOption { + /** + * This option causes an exception to be thrown if the file {@link java.io.InputStream} + * is closed before all the data has been read. + */ + FULL_DOWNLOAD +} diff --git a/modules/nf-httpfs/src/main/nextflow/file/http/XFileSystemProvider.groovy b/modules/nf-httpfs/src/main/nextflow/file/http/XFileSystemProvider.groovy index ea3d122627..f604c9f896 100644 --- a/modules/nf-httpfs/src/main/nextflow/file/http/XFileSystemProvider.groovy +++ b/modules/nf-httpfs/src/main/nextflow/file/http/XFileSystemProvider.groovy @@ -16,6 +16,8 @@ package nextflow.file.http +import nextflow.file.ForeignOpenOption + import static nextflow.file.http.XFileSystemConfig.* import java.nio.ByteBuffer @@ -341,18 +343,21 @@ abstract class XFileSystemProvider extends FileSystemProvider { if (path.class != XPath) throw new ProviderMismatchException() + boolean requireCompletion = false; if (options.length > 0) { for (OpenOption opt: options) { // All OpenOption values except for APPEND and WRITE are allowed - if (opt == StandardOpenOption.APPEND || - opt == StandardOpenOption.WRITE) - throw new UnsupportedOperationException("'$opt' not allowed"); + if (opt == StandardOpenOption.APPEND || opt == StandardOpenOption.WRITE) + throw new UnsupportedOperationException("'$opt' not allowed") + + if (opt == ForeignOpenOption.FULL_DOWNLOAD) + requireCompletion = true } } final conn = toConnection(path) final length = conn.getContentLengthLong() - return length>0 + return length>0 && requireCompletion ? new FixedInputStream(conn.getInputStream(), length) : conn.getInputStream() }