diff --git a/vfs-gcs/.classpath b/vfs-gcs/.classpath new file mode 100644 index 0000000..e43402f --- /dev/null +++ b/vfs-gcs/.classpath @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/vfs-gcs/.gitignore b/vfs-gcs/.gitignore new file mode 100644 index 0000000..731eb43 --- /dev/null +++ b/vfs-gcs/.gitignore @@ -0,0 +1,2 @@ +/target/ +/.settings/ diff --git a/vfs-gcs/.project b/vfs-gcs/.project new file mode 100644 index 0000000..b25f4a0 --- /dev/null +++ b/vfs-gcs/.project @@ -0,0 +1,23 @@ + + + maverick-vfs-gcs + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.m2e.core.maven2Builder + + + + + + org.eclipse.jdt.core.javanature + org.eclipse.m2e.core.maven2Nature + + diff --git a/vfs-gcs/pom.xml b/vfs-gcs/pom.xml new file mode 100644 index 0000000..486c378 --- /dev/null +++ b/vfs-gcs/pom.xml @@ -0,0 +1,86 @@ + + 4.0.0 + Google Compute Storage + An implementation of VFS provider for Google Storage + vfs-gcs + + com.sshtools + vfs + 3.0.0-SNAPSHOT + .. + + + src/main/java + src/test/java + target/classes + target/test-classes + ${project.artifactId} + + + . + src/main/resources + + + + + . + src/test/resources + + + + + org.apache.maven.plugins + maven-site-plugin + + + com.nativelibs4java + maven-jnaerator-plugin + 0.12-SNAPSHOT + + + + + + com.google.cloud + google-cloud-storage + 1.4.0 + + + javax.servlet + servlet-api + + + + + commons-logging + commons-logging + 1.2 + + + org.apache.commons + commons-vfs2 + ${commonsVFSVersion} + + + commons-logging + commons-logging + + + + + + + sonatype + Sonatype OSS Snapshots Repository + http://oss.sonatype.org/content/groups/public + + + + + sonatype + Sonatype OSS Snapshots Repository + http://oss.sonatype.org/content/groups/public + + + diff --git a/vfs-gcs/src/main/java/com/sshtools/vfs/gcs/GoogleFileName.java b/vfs-gcs/src/main/java/com/sshtools/vfs/gcs/GoogleFileName.java new file mode 100644 index 0000000..adda82b --- /dev/null +++ b/vfs-gcs/src/main/java/com/sshtools/vfs/gcs/GoogleFileName.java @@ -0,0 +1,24 @@ +package com.sshtools.vfs.gcs; + +import org.apache.commons.vfs2.FileName; +import org.apache.commons.vfs2.FileType; +import org.apache.commons.vfs2.provider.AbstractFileName; + +public class GoogleFileName extends AbstractFileName { + + public GoogleFileName(String absPath, FileType type) { + super("gcs", absPath, type); + } + + @Override + public FileName createName(String absPath, FileType type) { + return new GoogleFileName(absPath, type); + } + + @Override + protected void appendRootUri(StringBuilder buffer, boolean addPassword) { + buffer.append(getScheme()); + buffer.append(":"); + } + +} diff --git a/vfs-gcs/src/main/java/com/sshtools/vfs/gcs/GoogleFileObject.java b/vfs-gcs/src/main/java/com/sshtools/vfs/gcs/GoogleFileObject.java new file mode 100644 index 0000000..cb0d1b3 --- /dev/null +++ b/vfs-gcs/src/main/java/com/sshtools/vfs/gcs/GoogleFileObject.java @@ -0,0 +1,273 @@ +package com.sshtools.vfs.gcs; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.vfs2.FileObject; +import org.apache.commons.vfs2.FileType; +import org.apache.commons.vfs2.provider.AbstractFileName; +import org.apache.commons.vfs2.provider.AbstractFileObject; + +import com.google.api.gax.paging.Page; +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.Bucket; +import com.google.cloud.storage.BucketInfo; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.Storage.BlobListOption; + +public class GoogleFileObject extends AbstractFileObject { + + Blob blob = null; + Bucket bucket = null; + + String bucketName; + String bucketPath; + + protected GoogleFileObject(AbstractFileName name, GoogleStorageFileSystem fs) { + super(name, fs); + bucketName = getAbstractFileSystem().getBucketName(getName()).trim(); + bucketPath = getAbstractFileSystem().getBucketPath(getName()).trim(); + } + + protected GoogleFileObject(AbstractFileName name, GoogleStorageFileSystem fs, Bucket bucket, Blob blob) { + super(name, fs); + bucketName = getAbstractFileSystem().getBucketName(getName()).trim(); + bucketPath = getAbstractFileSystem().getBucketPath(getName()).trim(); + this.bucket = bucket; + this.blob = blob; + } + + private boolean hasBucket() { + return bucket!=null; + } + + private boolean hasObject() { + return blob!=null; + } + + @Override + protected void doAttach() throws Exception { + Storage storage = getAbstractFileSystem().setupStorage(); + + if(bucketName.length() > 0) { + if(this.bucket==null) { + this.bucket = storage.get(bucketName); + } + if(bucketPath.length() > 0) { + if(this.blob==null) { + this.blob = storage.get(bucketName, bucketPath); + if(this.blob==null) { + String parent = getParentFolder(bucketPath); + String child = lastPathElement(stripTrailingSlash(bucketPath)); + Page page; + if(parent.length() > 0) { + page = storage.list(bucketName, + BlobListOption.currentDirectory(), + BlobListOption.prefix(parent + "/")); + + } else { + page = storage.list(bucketName, BlobListOption.currentDirectory()); + } + for(Blob b : page.iterateAll()) { + if(lastPathElement(stripTrailingSlash(b.getName())).equals(child)) { + this.blob = b; + break; + } + } + } + } + } + } + } + + @Override + protected long doGetContentSize() throws Exception { + return hasObject() ? blob.getSize() : 0L; + } + + @Override + protected InputStream doGetInputStream() throws Exception { + if(!isFile()) { + throw new FileNotFoundException(); + } + Storage storage = getAbstractFileSystem().setupStorage(); + return new ReadChannelInputStream(storage.reader(blob.getBlobId())); + } + + @Override + protected FileType doGetType() throws Exception { + if(getName() instanceof GoogleFileName) { + return getName().getType(); + } + if(hasBucket()) { + if(hasObject()) { + return blob.isDirectory() ? FileType.FOLDER : FileType.FILE; + } + if(bucketPath.length()==0) { + return FileType.FOLDER; + } + return FileType.IMAGINARY; + } + return FileType.FOLDER; + } + + @Override + protected String[] doListChildren() throws Exception { + if(!isFolder()) { + throw new IOException("Object is not a directory"); + } + Storage storage = getAbstractFileSystem().setupStorage(); + List results = new ArrayList(); + if(!hasBucket()) { + Page page = storage.list(); + for(Bucket b : page.iterateAll()) { + results.add(b.getName()); + } + } else { + Page page; + if(bucketPath.length() > 0) { + page = storage.list(bucketName, BlobListOption.currentDirectory(), + BlobListOption.prefix(appendTrailingSlash(bucketPath))); + } else { + page = storage.list(bucketName, BlobListOption.currentDirectory()); + } + for(Blob b : page.iterateAll()) { + results.add(lastPathElement(stripTrailingSlash(b.getName()))); + } + } + return results.toArray(new String[0]); + } + + @Override + protected void doCreateFolder() throws Exception { + + Storage storage = getAbstractFileSystem().setupStorage(); + if(!hasBucket()) { + this.bucket = storage.create(BucketInfo.newBuilder(bucketName).build()); + } else { + this.blob = storage.create(BlobInfo.newBuilder(bucket, appendTrailingSlash(bucketPath)).build()); + } + } + + @Override + protected void doDelete() throws Exception { + if(!hasObject()) { + throw new IOException("Object is not attached"); + } + if(!blob.delete()) { + throw new IOException("Failed to delete object"); + } + } + + @Override + protected void doDetach() throws Exception { + bucket = null; + blob = null; + } + + @Override + protected long doGetLastModifiedTime() throws Exception { + if(hasObject()) { + return blob.getUpdateTime(); + } + if(hasBucket()) { + return bucket.getCreateTime(); + } else { + return 0L; + } + + } + + @Override + protected OutputStream doGetOutputStream(boolean bAppend) throws Exception { + if(!isFolder()) { + throw new IOException("Object is a directory or bucket"); + } + if(!hasBucket()) { + throw new IOException("Object must be placed in bucket"); + } + if(bucketPath.length()==0) { + throw new IOException("Object needs a path within the bucket"); + } + Storage storage = getAbstractFileSystem().setupStorage(); + if(!hasObject()) { + this.blob = storage.create(BlobInfo.newBuilder(bucket, stripTrailingSlash(bucketPath)).build()); + } + return new WriteChannelOutputStream(storage.writer(blob)); + } + + @Override + protected FileObject[] doListChildrenResolved() throws Exception { + if(!isFolder()) { + throw new IOException("Object is not a directory"); + } + Storage storage = getAbstractFileSystem().setupStorage(); + List results = new ArrayList(); + if(!hasBucket()) { + Page page = storage.list(); + for(Bucket b : page.iterateAll()) { + results.add(new GoogleFileObject(new GoogleFileName(b.getName(), FileType.FOLDER), getAbstractFileSystem())); + } + } else { + Page page; + if(bucketPath.length() > 0) { + page = storage.list(bucketName, BlobListOption.currentDirectory(), + BlobListOption.prefix(appendTrailingSlash(bucketPath))); + } else { + page = storage.list(bucketName, BlobListOption.currentDirectory()); + } + for(Blob b : page.iterateAll()) { + if(this.blob!=null && b.getName().equals(this.blob.getName())) { + continue; + } + results.add(new GoogleFileObject( + new GoogleFileName(getName().getPath() + "/" + lastPathElement(stripTrailingSlash(b.getName())), b.isDirectory() ? FileType.FOLDER : FileType.FILE), + getAbstractFileSystem(), + this.bucket != null ? bucket : storage.get(bucketName), + b)); + } + } + return results.toArray(new FileObject[0]); + } + + String getParentFolder(String name) { + name = stripTrailingSlash(name); + int idx = name.lastIndexOf('/'); + if(idx > -1) { + return name.substring(0, idx); + } + return ""; + } + + boolean hasTrailingSlash(String name) { + return name.endsWith("/"); + } + + String stripTrailingSlash(String name) { + if(name.endsWith("/")) { + return name.substring(0, name.length()-1); + } + return name; + } + + String appendTrailingSlash(String name) { + if(!name.endsWith("/")) { + return name + "/"; + } + return name; + } + + String lastPathElement(String name) { + int idx = name.lastIndexOf('/'); + if(idx > -1) { + return name.substring(idx+1); + } else { + return name; + } + } +} diff --git a/vfs-gcs/src/main/java/com/sshtools/vfs/gcs/GoogleStorageFileProvider.java b/vfs-gcs/src/main/java/com/sshtools/vfs/gcs/GoogleStorageFileProvider.java new file mode 100644 index 0000000..4ddf653 --- /dev/null +++ b/vfs-gcs/src/main/java/com/sshtools/vfs/gcs/GoogleStorageFileProvider.java @@ -0,0 +1,39 @@ +package com.sshtools.vfs.gcs; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; + +import org.apache.commons.vfs2.Capability; +import org.apache.commons.vfs2.FileName; +import org.apache.commons.vfs2.FileSystem; +import org.apache.commons.vfs2.FileSystemException; +import org.apache.commons.vfs2.FileSystemOptions; +import org.apache.commons.vfs2.provider.AbstractOriginatingFileProvider; + +public class GoogleStorageFileProvider extends AbstractOriginatingFileProvider { + + public final static Collection capabilities = Collections.unmodifiableCollection(Arrays.asList( + Capability.CREATE, + Capability.DELETE, + Capability.GET_TYPE, + Capability.GET_LAST_MODIFIED, + Capability.SET_LAST_MODIFIED_FILE, + Capability.SET_LAST_MODIFIED_FOLDER, + Capability.LIST_CHILDREN, + Capability.READ_CONTENT, + Capability.URI, + Capability.WRITE_CONTENT + )); + + public Collection getCapabilities() { + return capabilities; + } + + @Override + protected FileSystem doCreateFileSystem(FileName rootName, FileSystemOptions fileSystemOptions) + throws FileSystemException { + return new GoogleStorageFileSystem(rootName, null, fileSystemOptions); + } + +} diff --git a/vfs-gcs/src/main/java/com/sshtools/vfs/gcs/GoogleStorageFileSystem.java b/vfs-gcs/src/main/java/com/sshtools/vfs/gcs/GoogleStorageFileSystem.java new file mode 100644 index 0000000..29f2e21 --- /dev/null +++ b/vfs-gcs/src/main/java/com/sshtools/vfs/gcs/GoogleStorageFileSystem.java @@ -0,0 +1,69 @@ +package com.sshtools.vfs.gcs; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.util.Collection; + +import org.apache.commons.vfs2.Capability; +import org.apache.commons.vfs2.FileName; +import org.apache.commons.vfs2.FileObject; +import org.apache.commons.vfs2.FileSystemOptions; +import org.apache.commons.vfs2.provider.AbstractFileName; +import org.apache.commons.vfs2.provider.AbstractFileSystem; + +import com.google.auth.oauth2.ServiceAccountCredentials; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageOptions; + +public class GoogleStorageFileSystem extends AbstractFileSystem { + + Storage storage = null; + + protected GoogleStorageFileSystem(FileName rootName, FileObject parentLayer, FileSystemOptions fileSystemOptions) { + super(rootName, parentLayer, fileSystemOptions); + } + + @Override + protected FileObject createFile(AbstractFileName name) throws Exception { + return new GoogleFileObject(name, this); + } + + @Override + protected void addCapabilities(Collection caps) { + caps.addAll(GoogleStorageFileProvider.capabilities); + } + + Storage setupStorage() throws IOException { + if(storage!=null) { + return storage; + } + ServiceAccountCredentials credentials = ServiceAccountCredentials.fromStream(new ByteArrayInputStream( + GoogleStorageFileSystemConfigBuilder.getInstance().getClientIdJSON(getFileSystemOptions()).getBytes("UTF-8"))); + StorageOptions.Builder optionsBuilder = StorageOptions.newBuilder(); + optionsBuilder.setCredentials(credentials); + return storage = optionsBuilder.build().getService(); + } + + String getBucketName(FileName name) { + + String path = name.getPath(); + int idx = path.indexOf('/', 1); + if(idx > -1) { + return name.getPath().substring(1, idx); + } else { + return name.getPath().substring(1); + } + } + + String getBucketPath(FileName name) { + int idx = name.getPath().indexOf('/',1); + if(idx > -1) { + return name.getPath().substring(idx+1); + } else { + return ""; + } + } + + + +} diff --git a/vfs-gcs/src/main/java/com/sshtools/vfs/gcs/GoogleStorageFileSystemConfigBuilder.java b/vfs-gcs/src/main/java/com/sshtools/vfs/gcs/GoogleStorageFileSystemConfigBuilder.java new file mode 100644 index 0000000..3958c70 --- /dev/null +++ b/vfs-gcs/src/main/java/com/sshtools/vfs/gcs/GoogleStorageFileSystemConfigBuilder.java @@ -0,0 +1,31 @@ +package com.sshtools.vfs.gcs; + +import org.apache.commons.vfs2.FileSystem; +import org.apache.commons.vfs2.FileSystemConfigBuilder; +import org.apache.commons.vfs2.FileSystemOptions; + +public class GoogleStorageFileSystemConfigBuilder extends FileSystemConfigBuilder { + + private static final String JSON_CLIENT_ID_RESOURCE = "jsonClientIdResource"; + private final static GoogleStorageFileSystemConfigBuilder builder = new GoogleStorageFileSystemConfigBuilder(); + + public static GoogleStorageFileSystemConfigBuilder getInstance() { + return builder; + } + + private GoogleStorageFileSystemConfigBuilder() { + } + + public void setClientIdJSON(FileSystemOptions opts, String jsonClientIdResource) { + setParam(opts, JSON_CLIENT_ID_RESOURCE, jsonClientIdResource); + } + + public String getClientIdJSON(FileSystemOptions opts) { + return (String) getParam(opts, JSON_CLIENT_ID_RESOURCE); + } + + @Override + protected Class getConfigClass() { + return GoogleStorageFileSystem.class; + } +} diff --git a/vfs-gcs/src/main/java/com/sshtools/vfs/gcs/ReadChannelInputStream.java b/vfs-gcs/src/main/java/com/sshtools/vfs/gcs/ReadChannelInputStream.java new file mode 100644 index 0000000..da750fe --- /dev/null +++ b/vfs-gcs/src/main/java/com/sshtools/vfs/gcs/ReadChannelInputStream.java @@ -0,0 +1,58 @@ +package com.sshtools.vfs.gcs; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; + +import com.google.cloud.ReadChannel; + +public class ReadChannelInputStream extends InputStream { + + ReadChannel channel; + ByteBuffer bytes = ByteBuffer.allocate(64 * 1024); + + public ReadChannelInputStream(ReadChannel channel) { + this.channel = channel; + } + + @Override + public int read() throws IOException { + if(channel==null) { + return -1; + } + byte[] b = new byte[1]; + int r = read(b); + if(r > 0) { + return b[0] & 0xFF; + } + return r; + } + + public synchronized int read(byte[] buf, int off, int len) throws IOException { + if(channel==null) { + return -1; + } + int limit = bytes.limit(); + if(len < bytes.remaining()) { + bytes.limit(len); + } + int res = channel.read(bytes); + if(res < 0) { + close(); + return -1; + } + bytes.flip(); + int read = Math.min(bytes.remaining(), len); + bytes.get(buf, off, read); + bytes.compact(); + bytes.limit(limit); + return read; + } + + public synchronized void close() { + if(channel!=null) { + channel.close(); + channel = null; + } + } +} diff --git a/vfs-gcs/src/main/java/com/sshtools/vfs/gcs/WriteChannelOutputStream.java b/vfs-gcs/src/main/java/com/sshtools/vfs/gcs/WriteChannelOutputStream.java new file mode 100644 index 0000000..8bfaefe --- /dev/null +++ b/vfs-gcs/src/main/java/com/sshtools/vfs/gcs/WriteChannelOutputStream.java @@ -0,0 +1,53 @@ +package com.sshtools.vfs.gcs; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.channels.ClosedChannelException; + +import com.google.cloud.WriteChannel; + +public class WriteChannelOutputStream extends OutputStream { + + WriteChannel channel; + ByteBuffer bytes = ByteBuffer.allocate(64 * 1024); + + public WriteChannelOutputStream(WriteChannel channel) { + this.channel = channel; + } + + @Override + public void write(int b) throws IOException { + if(channel==null) { + throw new ClosedChannelException(); + } + bytes.put((byte)b); + bytes.flip(); + channel.write(bytes); + bytes.compact(); + } + + public synchronized void write(byte[] buf, int off, int len) throws IOException { + if(channel==null) { + throw new ClosedChannelException(); + } + int count = 0; + while(count < len) { + int c = Math.min(len-count, bytes.remaining()); + bytes.put(buf, off+count, c); + bytes.flip(); + while(bytes.hasRemaining()) { + channel.write(bytes); + } + bytes.compact(); + count += c; + } + } + + public synchronized void close() throws IOException { + if(channel!=null) { + channel.close(); + channel = null; + } + } +}