Add Storage Transfer Service samples for java

cloud-storage/storage-transfer/README.md

@@ -0,0 +1,50 @@
+# Transfer Service sample using Java
+
+This app creates two types of transfers using the Transfer Service tool.
+
+## Prerequisites
+
+1. Set up a project on Cloud Console.
+  1. Go to the [Cloud Console](https://cloud.google.com/console) and create or select your project.
+     You will need the project ID later.
+1. Install jar
+   `mvn install:install-file -Dfile=libstoragetransfer-v1-java-public.jar \`
+   `-DgroupId=com.google.storagetransfer.samples -DartifactId=libstoragetransfer -Dversion=1 -Dpackaging=jar`
+1. Set up gcloud for application default credentials.
+  1. `gcloud components update`
+  1. `gcloud auth login`
+  1. `gcloud config set project PROJECT_ID`
+
+## Transfer from AWS S3
+
+Creating a one-time transfer between Amazon AWS S3 and Google Cloud Storage.
+1. Set up data sink.
+  1. Go to the Cloud Console and create a bucket under Cloud Storage > Storage Browser.
+1. Set up data source.
+  1. Go to Amazon AWS S3 console and create a bucket.
+  1. Under Security Credentials, create an IAM User with access to the bucket.
+  1. Create an Access Key for the user. Note the Access Key ID and Secret Access Key.
+1. In AwsRequester.java, fill in the user-provided constants.
+1. Run with `mvn compile` and
+   `mvn exec:java -Dexec.mainClass="com.google.storagetransfer.samples.AwsRequester"`
+  1. Note the job ID in the returned Transfer Job.
+
+## Transfer to Nearline
+
+Creating a daily transfer between a regular Cloud Storage bucket and a Nearline
+Cloud Storage bucket for files untouched for 30 days.
+1. Set up data sink.
+  1. Go to the Cloud Console and create a bucket under Cloud Storage > Storage Browser.
+  1. Select Nearline for Storage Class.
+1. Set up data source.
+  1. Go to the Cloud Console and create a bucket under Cloud Storage > Storage Browser.
+1. In NearlineRequester.java, fill in the user-provided constants.
+1. Run with `mvn compile` and
+   `mvn exec:java -Dexec.mainClass="com.google.storagetransfer.samples.NearlineRequester"`
+  1. Note the job ID in the returned Transfer Job.
+
+## Checking the status of a transfer
+
+1. In RequestChecker.java, fill in the user-provided constants. Use the Job ID you recorded earlier.
+1. Run with `mvn compile` and
+   `mvn exec:java -Dexec.mainClass="com.google.storagetransfer.samples.RequestChecker"`

cloud-storage/storage-transfer/pom.xml

@@ -0,0 +1,36 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <groupId>com.google.storagetransfer.samples</groupId>
+  <artifactId>transfersample</artifactId>
+  <version>0.1</version>
+  <packaging>jar</packaging>
+
+  <name>transfersample</name>
+  <url>http://maven.apache.org</url>
+
+  <properties>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>3.8.1</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.google.apis</groupId>
+      <artifactId>google-api-services-storage</artifactId>
+      <version>v1-rev35-1.20.0</version>
+    </dependency>
+    <dependency>
+      <groupId>com.google.storagetransfer.samples</groupId>
+      <artifactId>libstoragetransfer</artifactId>
+      <version>1</version>
+    </dependency>
+  </dependencies>
+
+</project>

cloud-storage/storage-transfer/src/main/java/com/google/storagetransfer/samples/AwsRequester.java

@@ -0,0 +1,70 @@
+package com.google.storagetransfer.samples;
+
+import java.io.IOException;
+import java.util.logging.Logger;
+
+import com.google.api.services.storagetransfer_googleapis.StoragetransferGoogleapis;
+import com.google.api.services.storagetransfer_googleapis.model.AwsAccessKey;
+import com.google.api.services.storagetransfer_googleapis.model.AwsS3Data;
+import com.google.api.services.storagetransfer_googleapis.model.Date;
+import com.google.api.services.storagetransfer_googleapis.model.GcsData;
+import com.google.api.services.storagetransfer_googleapis.model.Schedule;
+import com.google.api.services.storagetransfer_googleapis.model.TimeOfDay;
+import com.google.api.services.storagetransfer_googleapis.model.TransferJob;
+import com.google.api.services.storagetransfer_googleapis.model.TransferSpec;
+
+public class AwsRequester {
+
+  // User-provided constants
+  private static final String projectId = "";
+  private static final String projectDesc = "";
+  private static final String awsSourceName = "";
+  private static final String awsAccessKeyId = "";
+  private static final String awsSecretAccessKey = "";
+  private static final String gcsSinkName = "";
+  // US Pacific Time Zone
+  private static final String startDate = "YYYY-MM-DD";
+  private static final String startTime = "HH:MM:SS"; 
+
+  private static final Logger log = Logger.getLogger(AwsRequester.class.getName());
+
+  public static void createAwsRequest(StoragetransferGoogleapis client, String projectId,
+    String desc, Date date, TimeOfDay time, String awsSourceName, String awsAccessKeyId,
+    String awsSecretAccessKey, String gcsSinkName)
+    throws InstantiationException, IllegalAccessException, IOException {
+    TransferJob t = TransferJob.class
+      .newInstance()
+      .setProjectId(projectId)
+      .setSchedule(
+        Schedule.class.newInstance()
+          .setScheduleStartDate(date)
+          .setScheduleEndDate(date)
+          .setStartTimeOfDay(time))
+      .setTransferSpec(
+        TransferSpec.class.newInstance()
+          .setAwsS3DataSource(
+            AwsS3Data.class.newInstance()
+              .setBucketName(awsSourceName)
+              .setAwsAccessKey(AwsAccessKey.class.newInstance().setAccessKeyId(awsAccessKeyId)
+              .setSecretAccessKey(awsSecretAccessKey)))
+          .setGcsDataSink(
+            GcsData.class.newInstance().setBucketName(gcsSinkName)))
+      .setJobDescription(desc).setJobState("ENABLED");
+
+    TransferJob responseT = client.transferJobs().create(t).execute();
+    log.info("Return transferJob: " + responseT.toPrettyString());
+  }
+
+  public static void main(String[] args) throws IOException, NumberFormatException,
+    InstantiationException, IllegalAccessException {
+    Date date = Date.class.newInstance().setYear(Integer.decode(startDate.split("-")[0]))
+      .setMonth(Integer.decode(startDate.split("-")[1]))
+      .setDay(Integer.decode(startDate.split("-")[2]));
+    TimeOfDay time = TimeOfDay.class.newInstance()
+      .setHours(Integer.decode(startTime.split(":")[0]))
+      .setMinutes(Integer.decode(startTime.split(":")[1]))
+      .setSeconds(Integer.decode(startTime.split(":")[2]));
+    createAwsRequest(TransferClientCreator.createStorageTransferClient(), projectId, projectDesc,
+      date, time, awsSourceName, awsAccessKeyId, awsSecretAccessKey, gcsSinkName);
+  }
+}

cloud-storage/storage-transfer/src/main/java/com/google/storagetransfer/samples/NearlineRequester.java

@@ -0,0 +1,68 @@
+package com.google.storagetransfer.samples;
+
+import java.io.IOException;
+import java.util.logging.Logger;
+
+import com.google.api.services.storagetransfer_googleapis.StoragetransferGoogleapis;
+import com.google.api.services.storagetransfer_googleapis.model.Date;
+import com.google.api.services.storagetransfer_googleapis.model.GcsData;
+import com.google.api.services.storagetransfer_googleapis.model.ObjectConditions;
+import com.google.api.services.storagetransfer_googleapis.model.Schedule;
+import com.google.api.services.storagetransfer_googleapis.model.TimeOfDay;
+import com.google.api.services.storagetransfer_googleapis.model.TransferJob;
+import com.google.api.services.storagetransfer_googleapis.model.TransferOptions;
+import com.google.api.services.storagetransfer_googleapis.model.TransferSpec;
+
+public class NearlineRequester {
+
+  // User-provided constants
+  private static final String projectId = "";
+  private static final String projectDesc = "";
+  private static final String gcsSourceName = "";
+  private static final String gcsSinkName = "";
+  // US Pacific Time Zone
+  private static final String startDate = "YYYY-MM-DD";
+  private static final String startTime = "HH:MM:SS";
+
+  private static final Logger log = Logger.getLogger(NearlineRequester.class.getName());
+
+  public static void createNearlineRequest(StoragetransferGoogleapis client, String projectId,
+    String desc, Date date, TimeOfDay time, String gcsSourceName, String gcsNearlineName)
+    throws InstantiationException,
+    IllegalAccessException, IOException {
+    TransferJob t = TransferJob.class
+      .newInstance()
+      .setProjectId(projectId)
+      .setSchedule(
+        Schedule.class.newInstance()
+          .setScheduleStartDate(date)
+          .setStartTimeOfDay(time))
+      .setTransferSpec(
+        TransferSpec.class.newInstance()
+          .setGcsDataSource(
+            GcsData.class.newInstance().setBucketName(gcsSourceName))
+          .setGcsDataSink(
+            GcsData.class.newInstance().setBucketName(gcsNearlineName))
+          .setObjectConditions(
+            ObjectConditions.class.newInstance().setMinTimeElapsedSinceLastModification("2592000s"))
+          .setTransferOptions(
+            TransferOptions.class.newInstance().setDeleteObjectsFromSourceAfterTransfer(true)))
+      .setJobDescription(desc).setJobState("ENABLED");
+
+    TransferJob responseT = client.transferJobs().create(t).execute();
+    log.info("Return transferJob: " + responseT.toPrettyString());
+  }
+
+  public static void main(String[] args) throws NumberFormatException, InstantiationException,
+    IllegalAccessException, IOException {
+    Date date = Date.class.newInstance().setYear(Integer.decode(startDate.split("-")[0]))
+      .setMonth(Integer.decode(startDate.split("-")[1]))
+      .setDay(Integer.decode(startDate.split("-")[2]));
+    TimeOfDay time = TimeOfDay.class.newInstance()
+      .setHours(Integer.decode(startTime.split(":")[0]))
+      .setMinutes(Integer.decode(startTime.split(":")[1]))
+      .setSeconds(Integer.decode(startTime.split(":")[2]));
+    createNearlineRequest(TransferClientCreator.createStorageTransferClient(), projectId,
+      projectDesc, date, time, gcsSourceName, gcsSinkName);
+  }
+}

cloud-storage/storage-transfer/src/main/java/com/google/storagetransfer/samples/RequestChecker.java

@@ -0,0 +1,28 @@
+package com.google.storagetransfer.samples;
+
+import java.io.IOException;
+import java.util.logging.Logger;
+
+import com.google.api.services.storagetransfer_googleapis.StoragetransferGoogleapis;
+import com.google.api.services.storagetransfer_googleapis.model.ListOperationsResponse;
+
+public class RequestChecker {
+
+  // User-provided constants
+  private static final String projectId = "";
+  private static final String jobId = "";
+  private static final Logger log = Logger.getLogger(RequestChecker.class.getName());
+
+  public static void checkTransfer(StoragetransferGoogleapis client, String projectId, String jobId)
+    throws InstantiationException, IllegalAccessException, IOException {
+    ListOperationsResponse r = client.transferOperations().list("transferOperations")
+      .setFilter("{\"project_id\": \"" + projectId + "\", \"job_id\": [\"" + jobId + "\"] }")
+      .execute();
+    log.info("Result of transferOperations/list: " + r.toPrettyString());
+  }
+
+  public static void main(String[] args) throws InstantiationException, IllegalAccessException,
+    IOException {
+    checkTransfer(TransferClientCreator.createStorageTransferClient(), projectId, jobId);
+  }
+}

cloud-storage/storage-transfer/src/main/java/com/google/storagetransfer/samples/RetryHttpInitializerWrapper.java

@@ -0,0 +1,83 @@
+package com.google.storagetransfer.samples;
+
+import com.google.api.client.auth.oauth2.Credential;
+import com.google.api.client.http.HttpBackOffIOExceptionHandler;
+import com.google.api.client.http.HttpBackOffUnsuccessfulResponseHandler;
+import com.google.api.client.http.HttpRequest;
+import com.google.api.client.http.HttpRequestInitializer;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpUnsuccessfulResponseHandler;
+import com.google.api.client.util.ExponentialBackOff;
+import com.google.api.client.util.Sleeper;
+import com.google.common.base.Preconditions;
+
+import java.io.IOException;
+import java.util.logging.Logger;
+
+/**
+ * RetryHttpInitializerWrapper will automatically retry upon RPC
+ * failures, preserving the auto-refresh behavior of the Google
+ * Credentials.
+ */
+public class RetryHttpInitializerWrapper implements HttpRequestInitializer {
+
+    private static final Logger LOG =
+            Logger.getLogger(RetryHttpInitializerWrapper.class.getName());
+
+    // Intercepts the request for filling in the "Authorization"
+    // header field, as well as recovering from certain unsuccessful
+    // error codes wherein the Credential must refresh its token for a
+    // retry.
+    private final Credential wrappedCredential;
+
+    // A sleeper; you can replace it with a mock in your test.
+    private final Sleeper sleeper;
+
+    public RetryHttpInitializerWrapper(Credential wrappedCredential) {
+        this(wrappedCredential, Sleeper.DEFAULT);
+    }
+
+    // Use only for testing.
+    RetryHttpInitializerWrapper(
+            Credential wrappedCredential, Sleeper sleeper) {
+        this.wrappedCredential = Preconditions.checkNotNull(wrappedCredential);
+        this.sleeper = sleeper;
+    }
+
+    public void initialize(HttpRequest request) {
+        request.setReadTimeout(2 * 60000); // 2 minutes read timeout
+        final HttpUnsuccessfulResponseHandler backoffHandler =
+                new HttpBackOffUnsuccessfulResponseHandler(
+                        new ExponentialBackOff())
+                        .setSleeper(sleeper);
+        request.setInterceptor(wrappedCredential);
+        request.setUnsuccessfulResponseHandler(
+                new HttpUnsuccessfulResponseHandler() {
+                    public boolean handleResponse(
+                            HttpRequest request,
+                            HttpResponse response,
+                            boolean supportsRetry) throws IOException {
+                        if (wrappedCredential.handleResponse(
+                                request, response, supportsRetry)) {
+                            // If credential decides it can handle it,
+                            // the return code or message indicated
+                            // something specific to authentication,
+                            // and no backoff is desired.
+                            return true;
+                        } else if (backoffHandler.handleResponse(
+                                request, response, supportsRetry)) {
+                            // Otherwise, we defer to the judgement of
+                            // our internal backoff handler.
+                            LOG.info("Retrying "
+                                    + request.getUrl().toString());
+                            return true;
+                        } else {
+                            return false;
+                        }
+                    }
+                });
+        request.setIOExceptionHandler(
+                new HttpBackOffIOExceptionHandler(new ExponentialBackOff())
+                        .setSleeper(sleeper));
+    }
+}

cloud-storage/storage-transfer/src/main/java/com/google/storagetransfer/samples/TransferClientCreator.java

@@ -0,0 +1,36 @@
+package com.google.storagetransfer.samples;
+
+import java.io.IOException;
+
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.util.Utils;
+import com.google.api.client.http.HttpRequestInitializer;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.services.storagetransfer_googleapis.StoragetransferGoogleapis;
+import com.google.api.services.storagetransfer_googleapis.StoragetransferGoogleapisScopes;
+import com.google.common.base.Preconditions;
+
+public class TransferClientCreator {
+  public static StoragetransferGoogleapis createStorageTransferClient() throws IOException {
+    return createStorageTransferClient(Utils.getDefaultTransport(), Utils.getDefaultJsonFactory());
+  }
+
+  public static StoragetransferGoogleapis createStorageTransferClient(HttpTransport httpTransport,
+    JsonFactory jsonFactory) throws IOException {
+    Preconditions.checkNotNull(httpTransport);
+    Preconditions.checkNotNull(jsonFactory);
+    GoogleCredential credential = GoogleCredential
+      .getApplicationDefault(httpTransport, jsonFactory);
+    // In some cases, you need to add the scope explicitly.
+    if (credential.createScopedRequired()) {
+      credential = credential.createScoped(StoragetransferGoogleapisScopes.all());
+    }
+    // Please use custom HttpRequestInitializer for automatic
+    // retry upon failures. We provide a simple reference
+    // implementation in the "Retry Handling" section.
+    HttpRequestInitializer initializer = new RetryHttpInitializerWrapper(credential);
+    return new StoragetransferGoogleapis.Builder(httpTransport, jsonFactory, initializer)
+      .setApplicationName("storagetransfer-sample").build();
+  }
+}