Use existing library for filtering files using glob patterns

build.gradle

@@ -81,7 +81,7 @@ configure(javaSubprojects) {
 
     dependencies {
         compile 'log4j:log4j:1.2.14'
-        compile 'com.google.guava:guava:17.0'
+        compile 'com.google.guava:guava:18.0'
         runtime 'org.slf4j:slf4j-log4j12:1.6.6'
         testCompile 'junit:junit:4.11'
     }
@@ -150,7 +150,7 @@ def functionalTestConfiguration = {
 
 project(':scheduler:scheduler-api') {
     dependencies {
-        compile "org.objectweb.proactive:programming-annotation:${programmingVersion}"
+        compile "org.objectweb.proactive:programming-extension-dataspaces:${programmingVersion}"
 
         compile 'it.sauronsoftware.cron4j:cron4j:2.2.5'
         compile 'isorelax:isorelax:20030108'

rest/build.gradle

@@ -36,6 +36,7 @@ project(':rest:rest-api') {
         compile 'org.jboss.resteasy:resteasy-jackson-provider:3.0.6.Final'
         compile 'org.jboss.resteasy:resteasy-multipart-provider:3.0.6.Final'
         compile "org.objectweb.proactive:programming-util:${programmingVersion}"
+        compile "org.objectweb.proactive:programming-extension-dataspaces:${programmingVersion}"
     }
 }
 

rest/rest-api/src/main/java/org/ow2/proactive_grid_cloud_portal/scheduler/client/utils/Zipper.java

@@ -43,9 +43,10 @@
 import com.google.common.io.ByteStreams;
 import com.google.common.io.Closer;
 import com.google.common.io.Files;
-import org.objectweb.proactive.utils.SelectorUtils;
+import org.objectweb.proactive.extensions.dataspaces.vfs.selector.FileSelector;
 
 import java.io.*;
+import java.nio.file.Path;
 import java.util.List;
 import java.util.zip.*;
 
@@ -152,7 +153,7 @@ public static void writeZipEntry(ZipEntry zipEntry, InputStream is, ZipOutputStr
             }
         }
 
-        public static void unzip(InputStream is, File outFile) throws FileNotFoundException, IOException {
+        public static void unzip(InputStream is, File outFile) throws IOException {
             Closer closer = Closer.create();
             try {
                 ZipInputStream zis = new ZipInputStream(is);
@@ -241,28 +242,14 @@ public FileSelectionPredicate(File root, List<String> includes, List<String> exc
 
         @Override
         public boolean apply(File file) {
-            File filePathRelativeToRoot = root.toPath().relativize(file.toPath()).toFile();
+            Path pathRelativeToRoot = root.toPath().relativize(file.toPath());
+
+            FileSelector selector = new FileSelector(includes, excludes);
 
             return !file.isDirectory()
-                        && !matches(filePathRelativeToRoot, excludes)
-                        && matches(filePathRelativeToRoot, includes);
+                        && selector.matches(pathRelativeToRoot);
         }
 
-        private boolean matches(File file, List<String> exprs) {
-            if (exprs == null || exprs.isEmpty()) {
-                return false;
-            } else if (exprs.contains(file.getName())) {
-                return true;
-            } else {
-                for (String expr : exprs) {
-                    if (SelectorUtils.matchPath(expr, file.toString())) {
-                        return true;
-                    }
-                }
-            }
-
-            return false;
-        }
     }
 
 }

rest/rest-client/src/main/java/org/ow2/proactive/scheduler/rest/ds/LocalDirSource.java

@@ -46,6 +46,7 @@
 import java.util.List;
 
 import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
 
 
 public class LocalDirSource implements ILocalSource {
@@ -64,23 +65,23 @@ public LocalDirSource(String path) {
     }
 
     public void setIncludes(List<String> includes) {
-        checkArgument(includes != null && !includes.isEmpty());
-        this.includes = includes;
+        checkNotNull(includes);
+        this.includes = Lists.newArrayList(includes);
     }
 
-    public void setIncludes(String... include) {
-        checkArgument(include != null && include.length > 0);
-        this.includes = Lists.newArrayList(include);
+    public void setIncludes(String... includes) {
+        checkNotNull(includes);
+        this.includes = Lists.newArrayList(includes);
     }
 
     public void setExcludes(List<String> excludes) {
-        checkArgument(excludes != null && !excludes.isEmpty());
-        this.excludes = excludes;
+        checkNotNull(excludes);
+        this.excludes = Lists.newArrayList(excludes);
     }
 
-    public void setExcludes(String... exclude) {
-        checkArgument(exclude != null && exclude.length > 0);
-        this.excludes = Lists.newArrayList(exclude);
+    public void setExcludes(String... excludes) {
+        checkNotNull(excludes);
+        this.excludes = Lists.newArrayList(excludes);
     }
 
     @Override

rest/rest-client/src/main/java/org/ow2/proactive/scheduler/rest/ds/RemoteSource.java

@@ -43,7 +43,7 @@
 
 import java.util.List;
 
-import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
 
 
 public class RemoteSource implements IRemoteSource {
@@ -83,13 +83,13 @@ public String getPath() {
     }
 
     public void setIncludes(List<String> includes) {
-        checkArgument(includes != null && !includes.isEmpty());
-        this.includes = includes;
+        checkNotNull(includes);
+        this.includes = Lists.newArrayList(includes);
     }
 
-    public void includes(String... include) {
-        checkArgument(include != null && include.length > 0);
-        this.includes = Lists.newArrayList(include);
+    public void setIncludes(String... includes) {
+        checkNotNull(includes);
+        this.includes = Lists.newArrayList(includes);
     }
 
     @Override
@@ -98,13 +98,13 @@ public List<String> getIncludes() {
     }
 
     public void setExcludes(List<String> excludes) {
-        checkArgument(includes != null && !includes.isEmpty());
-        this.excludes = excludes;
+        checkNotNull(excludes);
+        this.excludes = Lists.newArrayList(excludes);
     }
 
-    public void setExcludes(String... exclude) {
-        checkArgument(exclude != null && exclude.length > 0);
-        this.excludes = Lists.newArrayList(exclude);
+    public void setExcludes(String... excludes) {
+        checkNotNull(excludes);
+        this.excludes = Lists.newArrayList(excludes);
     }
 
     @Override

rest/rest-client/src/test/java/functionaltests/DataTransferTest.java

@@ -85,7 +85,7 @@ public void testUploadSingleFile() throws Exception {
     public void testUploadAllFilesInDirectory() throws Exception {
         // entire folder
         File tempTextFile = tmpDir.newFile("tempFile.txt");
-        Files.write("some random text ...".getBytes(), tempTextFile);
+        Files.write("some text ...".getBytes(), tempTextFile);
 
         File tempDir = tmpDir.newFolder("tempDir");
         File tempFile = new File(tempDir, "tempFile.tmp");
@@ -108,7 +108,7 @@ public void testUploadAllFilesInDirectory() throws Exception {
     @Test
     public void testUploadSelectedFilesUsingGlobPattern() throws Exception {
         File tempTextFile = tmpDir.newFile("tempFile.txt");
-        Files.write("some random text ...".getBytes(), tempTextFile);
+        Files.write("some text ...".getBytes(), tempTextFile);
 
         File tempDir = tmpDir.newFolder("tempDir");
         File tempFile = new File(tempDir, "tempFile.tmp");
@@ -130,7 +130,7 @@ public void testUploadSelectedFilesUsingGlobPattern() throws Exception {
     @Test
     public void testUploadSelectedFilesUsingFilenames() throws Exception {
         File tempTextFile = tmpDir.newFile("tempFile.txt");
-        Files.write("some random text ...".getBytes(), tempTextFile);
+        Files.write("some text ...".getBytes(), tempTextFile);
 
         File tempDir = tmpDir.newFolder("tempDir");
         File tempFile = new File(tempDir, "tempFile.tmp");
@@ -139,7 +139,7 @@ public void testUploadSelectedFilesUsingFilenames() throws Exception {
 
         IDataSpaceClient client = clientInstance();
         LocalDirSource source = new LocalDirSource(tmpDir.getRoot());
-        source.setIncludes("tempFile.tmp");
+        source.setIncludes("**/tempFile.tmp");
         RemoteDestination dest = new RemoteDestination(USER, "testUploadSelectedFilesUsingFilenames");
         assertTrue(client.upload(source, dest));
 
@@ -181,7 +181,7 @@ public void testDownloadAllFilesInDirectory() throws Exception {
 
         File srcTextFile = new File(srcDir, "tempFile.txt");
         Files.createParentDirs(srcTextFile);
-        Files.write("some random text ...".getBytes(), srcTextFile);
+        Files.write("some text ...".getBytes(), srcTextFile);
 
         File srcTempDir = new File(srcDir, "tempDir");
         File srcTempFile = new File(srcTempDir, "tempFile.tmp");
@@ -209,7 +209,7 @@ public void testDownloadSelectedFilesUsingGlobPattern() throws Exception {
 
         File srcTextFile = new File(srcDir, "tempFile.txt");
         Files.createParentDirs(srcTextFile);
-        Files.write("some random text ...".getBytes(), srcTextFile);
+        Files.write("some text ...".getBytes(), srcTextFile);
 
         File srcTempDir = new File(srcDir, "tempDir");
         File srcTempFile = new File(srcTempDir, "tempFile.tmp");
@@ -218,7 +218,7 @@ public void testDownloadSelectedFilesUsingGlobPattern() throws Exception {
 
         File destTempDir = tmpDir.newFolder("tempDir");
         RemoteSource source = new RemoteSource(USER, "testDownloadAllFilesInDirectory");
-        source.includes("*.txt");
+        source.setIncludes("*.txt");
         LocalDestination dest = new LocalDestination(destTempDir);
         IDataSpaceClient client = clientInstance();
 
@@ -240,7 +240,7 @@ public void testListFiles() throws Exception {
 
         File srcTextFile = new File(srcDir, "tempFile.txt");
         Files.createParentDirs(srcTextFile);
-        Files.write("some random text ...".getBytes(), srcTextFile);
+        Files.write("some text ...".getBytes(), srcTextFile);
 
         File srcTempDir = new File(srcDir, "tempDir");
         File srcTempFile = new File(srcTempDir, "tempFile.tmp");
@@ -314,8 +314,6 @@ public void testCreateFolder() throws Exception {
 
     @Test(expected = Exception.class)
     public void testCreateFolderWithoutSpecifyingFileType() throws Exception {
-        URI srcDirPath = URI.create(getScheduler().getUserSpaceURIs().get(0));
-
         String folderName = "testcreatefolder";
 
         RemoteSource source = new RemoteSource(USER, folderName);

rest/rest-server/src/main/java/org/ow2/proactive_grid_cloud_portal/dataspace/FileSystem.java

@@ -41,7 +41,6 @@
 import com.google.common.io.ByteStreams;
 import com.google.common.io.Closer;
 import org.apache.commons.vfs2.*;
-import org.objectweb.proactive.extensions.dataspaces.vfs.selector.GlobPatternFileSelector;
 import org.ow2.proactive.scheduler.common.exception.NotConnectedException;
 import org.ow2.proactive.scheduler.common.exception.PermissionException;
 import org.ow2.proactive.scheduler.common.util.SchedulerProxyUserInterface;
@@ -79,7 +78,6 @@ public FileObject resolveFileInGlobalspace(String pathname) throws FileSystemExc
 
     public FileObject resolveFile(String dirPath, String pathname) throws FileSystemException {
         return fsm.resolveFile(dirPath + (dirPath.endsWith(File.separator) ? "" : File.separator) + pathname);
-
     }
 
     public FileObject createFile(String pathname) throws FileSystemException {
@@ -144,8 +142,10 @@ private static void fillFileProps(FileObject fo, Map<String, Object> properties)
     public static List<FileObject> findFiles(FileObject root, List<String> includes, List<String> excludes)
             throws FileSystemException {
         List<FileObject> files = Lists.newArrayList();
-        FileSelector selector = (isNullOrEmpty(includes) && isNullOrEmpty(excludes)) ? new AllFilesSelector()
-                : new GlobPatternFileSelector(includes, excludes);
+        FileSelector selector =
+                (isNullOrEmpty(includes) && isNullOrEmpty(excludes)) ?
+                        new AllFilesSelector() :
+                        new org.objectweb.proactive.extensions.dataspaces.vfs.selector.FileSelector(includes, excludes);
         root.findFiles(selector, true, files);
         return files;
     }
@@ -203,7 +203,7 @@ public static FileSystem create(SchedulerProxyUserInterface schedulerProxy)
         }
     }
 
-    private static class AllFilesSelector implements FileSelector {
+    private static final class AllFilesSelector implements FileSelector {
         @Override
         public boolean includeFile(FileSelectInfo selInfo) throws Exception {
             return FileType.FILE == selInfo.getFile().getName().getType();

rest/rest-smartproxy/src/main/java/org/ow2/proactive_grid_cloud_portal/smartproxy/RestSmartProxyImpl.java

@@ -37,6 +37,7 @@
 import com.google.common.base.Throwables;
 import com.google.common.collect.Lists;
 import org.apache.log4j.Logger;
+import org.objectweb.proactive.extensions.dataspaces.vfs.selector.FileSelector;
 import org.ow2.proactive.scheduler.common.Scheduler;
 import org.ow2.proactive.scheduler.common.SchedulerConstants;
 import org.ow2.proactive.scheduler.common.SchedulerEvent;
@@ -45,7 +46,6 @@
 import org.ow2.proactive.scheduler.common.job.*;
 import org.ow2.proactive.scheduler.common.task.Task;
 import org.ow2.proactive.scheduler.common.task.TaskResult;
-import org.ow2.proactive.scheduler.common.task.dataspaces.FileSelector;
 import org.ow2.proactive.scheduler.common.task.dataspaces.InputSelector;
 import org.ow2.proactive.scheduler.common.task.dataspaces.OutputSelector;
 import org.ow2.proactive.scheduler.rest.ISchedulerClient;
@@ -59,7 +59,6 @@
 
 import javax.security.auth.login.LoginException;
 import java.io.File;
-import java.util.Arrays;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -247,12 +246,9 @@ public boolean uploadInputfiles(TaskFlowJob job, String localInputFolderPath)
                 addfileSelection(is.getInputFiles(), includes, excludes);
             }
             LocalDirSource source = new LocalDirSource(localInputFolderPath);
-            if (!includes.isEmpty()) {
-                source.setIncludes(includes);
-            }
-            if (!excludes.isEmpty()) {
-                source.setExcludes(excludes);
-            }
+            source.setIncludes(includes);
+            source.setExcludes(excludes);
+
             RemoteDestination dest = new RemoteDestination(USER, remotePath);
             restDataSpaceClient.upload(source, dest);
         }
@@ -305,12 +301,9 @@ protected void downloadTaskOutputFiles(AwaitedJob awaitedjob, String jobId, Stri
         } else {
             try {
                 RemoteSource source = new RemoteSource(USER, sourceFile);
-                if (!includes.isEmpty()) {
-                    source.setIncludes(includes);
-                }
-                if (!excludes.isEmpty()) {
-                    source.setExcludes(excludes);
-                }
+                source.setIncludes(includes);
+                source.setExcludes(excludes);
+
                 File localDir = new File(localFolder);
                 LocalDestination dest = new LocalDestination(localDir);
                 restDataSpaceClient.download(source, dest);
@@ -328,13 +321,8 @@ protected void downloadTaskOutputFiles(AwaitedJob awaitedjob, String jobId, Stri
     }
 
     private void addfileSelection(FileSelector fs, List<String> includes, List<String> excludes) {
-        if (fs.getIncludes() != null && fs.getIncludes().length > 0) {
-            includes.addAll(Arrays.asList(fs.getIncludes()));
-        }
-        if (fs.getExcludes() != null && fs.getExcludes().length > 0) {
-            excludes.addAll(Arrays.asList(fs.getExcludes()));
-        }
-
+        includes.addAll(fs.getIncludes());
+        excludes.addAll(fs.getExcludes());
     }
 
     @Override
@@ -454,12 +442,9 @@ public DownloadHandler(String jobId, String taskName, String sourceFile, List<St
         public void run() {
             try {
                 RemoteSource source = new RemoteSource(USER, sourceFile);
-                if (!includes.isEmpty()) {
-                    source.setIncludes(includes);
-                }
-                if (!excludes.isEmpty()) {
-                    source.setExcludes(excludes);
-                }
+                source.setIncludes(includes);
+                source.setExcludes(excludes);
+
                 File localDir = new File(localFolder);
                 LocalDestination dest = new LocalDestination(localDir);
 

rest/rest-smartproxy/src/test/java/functionaltests/RestSmartProxyTest.java

@@ -75,8 +75,6 @@ public final class RestSmartProxyTest extends AbstractRestFuncTestCase {
     protected String pushUrl;
     protected String pullUrl;
 
-    protected static final String TEST_SESSION_NAME = "TestDSSupport";
-
     protected static final String TASK_NAME = "TestJavaTask";
 
     public final static String INPUT_FILE_BASE_NAME = "input";
@@ -296,4 +294,4 @@ public void usersUpdatedEvent(NotificationData<UserIdentification> arg0) {
 
     }
 
-}
+}