Only apply the FixedInputStream check when staging files

Signed-off-by: Tom Sellman <[email protected]>

modules/nf-commons/src/main/nextflow/file/CopyMoveHelper.java

@@ -27,6 +27,7 @@
 import java.nio.file.Files;
 import java.nio.file.LinkOption;
 import java.nio.file.Path;
+import java.nio.file.ProviderMismatchException;
 import java.nio.file.SimpleFileVisitor;
 import java.nio.file.StandardCopyOption;
 import java.nio.file.attribute.BasicFileAttributes;
@@ -81,14 +82,15 @@ private static CopyOption[] convertMoveToCopyOptions(CopyOption... options)
     private static void copyFile(Path source, Path target, boolean foreign, CopyOption... options)
             throws IOException
     {
-
-        if( !foreign ) {
+        try {
+            // try to use the filesystem provider's copy implementation
             source.getFileSystem().provider().copy(source, target, options);
-            return;
-        }
+        } catch (UnsupportedOperationException | ProviderMismatchException ignore) {
 
-        try (InputStream in = Files.newInputStream(source)) {
-            Files.copy(in, target);
+            // provider doesn't support copy() for this scenario, fallback to stream copy
+            try (InputStream in = Files.newInputStream(source)) {
+                Files.copy(in, target);
+            }
         }
     }
 

modules/nf-httpfs/src/main/nextflow/file/http/XFileSystemProvider.groovy

@@ -341,18 +341,21 @@ abstract class XFileSystemProvider extends FileSystemProvider {
         if (path.class != XPath)
             throw new ProviderMismatchException()
 
+        boolean checkContentLength = false;
         if (options.length > 0) {
             for (OpenOption opt: options) {
                 // All OpenOption values except for APPEND and WRITE are allowed
-                if (opt == StandardOpenOption.APPEND ||
-                        opt == StandardOpenOption.WRITE)
-                    throw new UnsupportedOperationException("'$opt' not allowed");
+                if (opt == StandardOpenOption.APPEND || opt == StandardOpenOption.WRITE)
+                    throw new UnsupportedOperationException("'$opt' not allowed")
+
+                if (opt == XOpenOptions.CHECK_CONTENT_LENGTH)
+                    checkContentLength = true
             }
         }
 
         final conn = toConnection(path)
         final length = conn.getContentLengthLong()
-        return length>0
+        return length>0 && checkContentLength
             ? new FixedInputStream(conn.getInputStream(), length)
             : conn.getInputStream()
     }
@@ -412,7 +415,14 @@ abstract class XFileSystemProvider extends FileSystemProvider {
 
     @Override
     void copy(Path source, Path target, CopyOption... options) throws IOException {
-        throw new UnsupportedOperationException("Copy not supported by ${getScheme().toUpperCase()} file system provider")
+        // only support copy from http/ftp source to non-http/ftp target
+        if (source.getScheme() != getScheme() || target.getScheme() == getScheme()) {
+            throw new UnsupportedOperationException("Copy not supported by ${getScheme().toUpperCase()} file system provider")
+        }
+
+        try (InputStream input = newInputStream(source, XOpenOptions.CHECK_CONTENT_LENGTH)) {
+            Files.copy(input, target);
+        }
     }
 
     @Override

modules/nf-httpfs/src/main/nextflow/file/http/XOpenOptions.java

@@ -0,0 +1,11 @@
+package nextflow.file.http;
+
+import java.nio.file.OpenOption;
+
+public enum XOpenOptions implements OpenOption {
+    /**
+     * This option causes an exception to be thrown if the file {@link java.io.InputStream}
+     * is closed before all the data has been read.
+     */
+    CHECK_CONTENT_LENGTH
+}