Only apply the FixedInputStream check when staging files

Signed-off-by: Tom Sellman <[email protected]>

modules/nf-commons/src/main/nextflow/file/CopyMoveHelper.java

@@ -87,7 +87,9 @@ private static void copyFile(Path source, Path target, boolean foreign, CopyOpti
             return;
         }
 
-        try (InputStream in = Files.newInputStream(source)) {
+        // open the remote input stream using FULL_DOWNLOAD option, so it will error if not
+        // all bytes are read before closing
+        try (InputStream in = Files.newInputStream(source, ForeignOpenOption.FULL_DOWNLOAD)) {
             Files.copy(in, target);
         }
     }

modules/nf-commons/src/main/nextflow/file/ForeignOpenOption.java

@@ -0,0 +1,14 @@
+package nextflow.file;
+
+import java.nio.file.OpenOption;
+
+/**
+ * File open options for remote (foreign) file systems
+ */
+public enum ForeignOpenOption implements OpenOption {
+    /**
+     * This option causes an exception to be thrown if the file {@link java.io.InputStream}
+     * is closed before all the data has been read.
+     */
+    FULL_DOWNLOAD
+}

modules/nf-httpfs/src/main/nextflow/file/http/XFileSystemProvider.groovy

@@ -16,6 +16,8 @@
 
 package nextflow.file.http
 
+import nextflow.file.ForeignOpenOption
+
 import static nextflow.file.http.XFileSystemConfig.*
 
 import java.nio.ByteBuffer
@@ -341,18 +343,21 @@ abstract class XFileSystemProvider extends FileSystemProvider {
         if (path.class != XPath)
             throw new ProviderMismatchException()
 
+        boolean requireCompletion = false;
         if (options.length > 0) {
             for (OpenOption opt: options) {
                 // All OpenOption values except for APPEND and WRITE are allowed
-                if (opt == StandardOpenOption.APPEND ||
-                        opt == StandardOpenOption.WRITE)
-                    throw new UnsupportedOperationException("'$opt' not allowed");
+                if (opt == StandardOpenOption.APPEND || opt == StandardOpenOption.WRITE)
+                    throw new UnsupportedOperationException("'$opt' not allowed")
+
+                if (opt == ForeignOpenOption.FULL_DOWNLOAD)
+                    requireCompletion = true
             }
         }
 
         final conn = toConnection(path)
         final length = conn.getContentLengthLong()
-        return length>0
+        return length>0 && requireCompletion
             ? new FixedInputStream(conn.getInputStream(), length)
             : conn.getInputStream()
     }