Merge pull request #2 from jenkinsci/master

Update forked s3-plugin to be up-to-date with Jenkins
martazobro · Nov 9, 2015 · b08f46f · b08f46f
2 parents e69e6df + a57c89b
commit b08f46f
Show file tree

Hide file tree

Showing 6 changed files with 114 additions and 49 deletions.
diff --git a/src/main/java/hudson/plugins/s3/Entry.java b/src/main/java/hudson/plugins/s3/Entry.java
@@ -7,6 +7,8 @@
 import hudson.util.ListBoxModel;
 import org.kohsuke.stapler.DataBoundConstructor;
 
+import java.util.List;
+
 public final class Entry implements Describable<Entry> {
 
     /**
@@ -60,10 +62,21 @@ public final class Entry implements Describable<Entry> {
      */
     public boolean flatten;
 
+    /**
+    * use GZIP to compress files
+    */
+
+    public boolean gzipFiles;
+
+    /**
+    * Metadata overrides
+    */
+    public List<MetadataPair> userMetadata;
+
     @DataBoundConstructor
     public Entry(String bucket, String sourceFile, String storageClass, String selectedRegion,
                  boolean noUploadOnFailure, boolean uploadFromSlave, boolean managedArtifacts,
-                 boolean useServerSideEncryption, boolean flatten) {
+                 boolean useServerSideEncryption, boolean flatten, boolean gzipFiles, List<MetadataPair> userMetadata) {
         this.bucket = bucket;
         this.sourceFile = sourceFile;
         this.storageClass = storageClass;
@@ -73,6 +86,8 @@ public Entry(String bucket, String sourceFile, String storageClass, String selec
         this.managedArtifacts = managedArtifacts;
         this.useServerSideEncryption = useServerSideEncryption;
         this.flatten = flatten;
+        this.gzipFiles = gzipFiles;
+        this.userMetadata = userMetadata;
     }
 
     public Descriptor<Entry> getDescriptor() {

diff --git a/src/main/java/hudson/plugins/s3/S3BucketPublisher.java b/src/main/java/hudson/plugins/s3/S3BucketPublisher.java
@@ -1,6 +1,7 @@
 package hudson.plugins.s3;
 
 import com.amazonaws.regions.Regions;
+import com.google.common.collect.Iterables;
 import hudson.Extension;
 import hudson.FilePath;
 import hudson.Launcher;
@@ -28,10 +29,7 @@
 import java.io.IOException;
 import java.io.File;
 import java.io.PrintStream;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
@@ -166,20 +164,34 @@ public boolean perform(AbstractBuild<?, ?> build,
                 String bucket = Util.replaceMacro(entry.bucket, envVars);
                 String storageClass = Util.replaceMacro(entry.storageClass, envVars);
                 String selRegion = entry.selectedRegion;
-                List<MetadataPair> escapedUserMetadata = new ArrayList<MetadataPair>();
-                for (MetadataPair metadataPair : userMetadata) {
-                    escapedUserMetadata.add(
-                        new MetadataPair(
-                            Util.replaceMacro(metadataPair.key, envVars),
-                            Util.replaceMacro(metadataPair.value, envVars))
-                    );
+
+                Map<String, String> mergedMetadata = new HashMap<String, String>();
+
+                if (userMetadata != null) {
+                    for (MetadataPair pair : userMetadata) {
+                        mergedMetadata.put(pair.key, pair.value);
+                    }
                 }
-
+
+                if (entry.userMetadata != null) {
+                    for (MetadataPair pair : entry.userMetadata) {
+                        mergedMetadata.put(pair.key, pair.value);
+                    }
+                }
+
+                Map<String, String> escapedMetadata = new HashMap<String, String>();
+
+                for (Map.Entry<String, String> mapEntry : mergedMetadata.entrySet()) {
+                    escapedMetadata.put(
+                            Util.replaceMacro(mapEntry.getKey(), envVars),
+                            Util.replaceMacro(mapEntry.getValue(), envVars));
+                }
+
                 List<FingerprintRecord> records = Lists.newArrayList();
 
                 for (FilePath src : paths) {
                     log(listener.getLogger(), "bucket=" + bucket + ", file=" + src.getName() + " region=" + selRegion + ", upload from slave=" + entry.uploadFromSlave + " managed="+ entry.managedArtifacts + " , server encryption "+entry.useServerSideEncryption);
-                    records.add(profile.upload(build, listener, bucket, src, searchPathLength, escapedUserMetadata, storageClass, selRegion, entry.uploadFromSlave, entry.managedArtifacts, entry.useServerSideEncryption, entry.flatten));
+                    records.add(profile.upload(build, listener, bucket, src, searchPathLength, escapedMetadata, storageClass, selRegion, entry.uploadFromSlave, entry.managedArtifacts, entry.useServerSideEncryption, entry.flatten, entry.gzipFiles));
                 }
                 if (entry.managedArtifacts) {
                     artifacts.addAll(records);

diff --git a/src/main/java/hudson/plugins/s3/S3Profile.java b/src/main/java/hudson/plugins/s3/S3Profile.java
@@ -9,6 +9,7 @@
 import java.net.URL;
 import java.util.Date;
 import java.util.List;
+import java.util.Map;
 import java.util.regex.Pattern;
 
 import jenkins.model.Jenkins;
@@ -166,8 +167,8 @@ public void check() throws Exception {
         getClient().listBuckets();
     }
 
-    public FingerprintRecord upload(AbstractBuild<?,?> build, final BuildListener listener, String bucketName, FilePath filePath, int searchPathLength, List<MetadataPair> userMetadata,
-            String storageClass, String selregion, boolean uploadFromSlave, boolean managedArtifacts,boolean useServerSideEncryption, boolean flatten) throws IOException, InterruptedException {
+    public FingerprintRecord upload(AbstractBuild<?,?> build, final BuildListener listener, String bucketName, FilePath filePath, int searchPathLength, Map<String, String> userMetadata,
+            String storageClass, String selregion, boolean uploadFromSlave, boolean managedArtifacts, boolean useServerSideEncryption, boolean flatten, boolean gzipFiles) throws IOException, InterruptedException {
         if (filePath.isDirectory()) {
             throw new IOException(filePath + " is a directory");
         }
@@ -190,7 +191,7 @@ public FingerprintRecord upload(AbstractBuild<?,?> build, final BuildListener li
 
         while (true) {
             try {
-                S3UploadCallable callable = new S3UploadCallable(produced, accessKey, secretKey, useRole, bucketName, dest, userMetadata, storageClass, selregion,useServerSideEncryption);
+                S3UploadCallable callable = new S3UploadCallable(produced, accessKey, secretKey, useRole, bucketName, dest, userMetadata, storageClass, selregion, useServerSideEncryption, gzipFiles);
                 if (uploadFromSlave) {
                     return filePath.act(callable);
                 } else {

diff --git a/src/main/java/hudson/plugins/s3/callable/S3UploadCallable.java b/src/main/java/hudson/plugins/s3/callable/S3UploadCallable.java
@@ -8,13 +8,14 @@
 import hudson.remoting.VirtualChannel;
 import hudson.util.Secret;
 
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
+import java.io.*;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.Date;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.zip.GZIPOutputStream;
 
 import com.amazonaws.regions.Region;
 import com.amazonaws.regions.RegionUtils;
@@ -23,25 +24,38 @@
 import com.amazonaws.services.s3.model.ObjectMetadata;
 import com.amazonaws.services.s3.model.PutObjectRequest;
 import com.amazonaws.services.s3.model.PutObjectResult;
+import org.apache.commons.io.IOUtils;
 
 public class S3UploadCallable extends AbstractS3Callable implements FileCallable<FingerprintRecord> {
+
+    private static Map<String, String> convertOldMeta(List<MetadataPair> userMeta) {
+        Map<String, String> result = new HashMap<String, String>();
+
+        for (MetadataPair pair : userMeta) {
+            result.put(pair.key, pair.value);
+        }
+
+        return result;
+    }
+
     private static final long serialVersionUID = 1L;
     private final String bucketName;
     private final Destination dest;
     private final String storageClass;
-    private final List<MetadataPair> userMetadata;
+    private final Map<String, String> userMetadata;
     private final String selregion;
     private final boolean produced;
     private final boolean useServerSideEncryption;
+    private final boolean gzipFiles;
 
     @Deprecated
     public S3UploadCallable(boolean produced, String accessKey, Secret secretKey, boolean useRole, Destination dest, List<MetadataPair> userMetadata, String storageClass,
                             String selregion, boolean useServerSideEncryption) {
-        this(produced, accessKey, secretKey, useRole, dest.bucketName, dest, userMetadata, storageClass, selregion, useServerSideEncryption);
+        this(produced, accessKey, secretKey, useRole, dest.bucketName, dest, convertOldMeta(userMetadata), storageClass, selregion, useServerSideEncryption, false);
     }
 
-    public S3UploadCallable(boolean produced, String accessKey, Secret secretKey, boolean useRole, String bucketName, Destination dest, List<MetadataPair> userMetadata, String storageClass,
-            String selregion, boolean useServerSideEncryption) {
+    public S3UploadCallable(boolean produced, String accessKey, Secret secretKey, boolean useRole, String bucketName, Destination dest, Map<String, String> userMetadata, String storageClass,
+                            String selregion, boolean useServerSideEncryption, boolean gzipFiles) {
         super(accessKey, secretKey, useRole);
         this.bucketName = bucketName;
         this.dest = dest;
@@ -50,6 +64,7 @@ public S3UploadCallable(boolean produced, String accessKey, Secret secretKey, bo
         this.selregion = selregion;
         this.produced = produced;
         this.useServerSideEncryption = useServerSideEncryption;
+        this.gzipFiles = gzipFiles;
     }
 
     public ObjectMetadata buildMetadata(FilePath filePath) throws IOException, InterruptedException {
@@ -64,21 +79,21 @@ public ObjectMetadata buildMetadata(FilePath filePath) throws IOException, Inter
             metadata.setServerSideEncryption(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
         }
 
-        for (MetadataPair metadataPair : userMetadata) {
-            String key = metadataPair.key.toLowerCase();
+        for (Map.Entry<String, String> entry : userMetadata.entrySet()) {
+            String key = entry.getKey().toLowerCase();
             if (key.equals("cache-control")) {
-                metadata.setCacheControl(metadataPair.value);
+                metadata.setCacheControl(entry.getValue());
             } else if (key.equals("expires")) {
                 try {
-                    Date expires = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss z").parse(metadataPair.value);
+                    Date expires = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss z").parse(entry.getValue());
                     metadata.setHttpExpiresDate(expires);
                 } catch (ParseException e) {
-                    metadata.addUserMetadata(metadataPair.key, metadataPair.value);
+                    metadata.addUserMetadata(entry.getKey(), entry.getValue());
                 }
             } else if (key.equals("content-encoding")) {
-                metadata.setContentEncoding(metadataPair.value);
+                metadata.setContentEncoding(entry.getValue());
             } else {
-                metadata.addUserMetadata(metadataPair.key, metadataPair.value);
+                metadata.addUserMetadata(entry.getKey(), entry.getValue());
             }
         }
         return metadata;
@@ -97,30 +112,35 @@ public FingerprintRecord invoke(File file, VirtualChannel channel) throws IOExce
     public FingerprintRecord invoke(FilePath file) throws IOException, InterruptedException {
         setRegion();
 
+        ObjectMetadata metadata = buildMetadata(file);
+
+        InputStream inputStream = file.read();
+
         File localFile = null;
-        FileOutputStream os = null;
-        boolean deleteLocalFile = false;
 
-        try {
-            if (file.isRemote()) {
-                deleteLocalFile = true;
-                localFile = File.createTempFile("s3plugin", ".bin");
-                os = new FileOutputStream(localFile);
-                file.copyTo(os);
-                os.flush();
-            } else {
-                localFile = new File(file.getRemote());
-            }
+        if (gzipFiles) {
+            localFile = File.createTempFile("s3plugin", ".bin");
+
+            OutputStream outputStream = new FileOutputStream(localFile);
+
+            outputStream = new GZIPOutputStream(outputStream, true);
+
+            IOUtils.copy(inputStream, outputStream);
+            outputStream.flush();
+            outputStream.close();
 
-            final PutObjectRequest request = new PutObjectRequest(dest.bucketName, dest.objectName, localFile)
-                .withMetadata(buildMetadata(file));
+            inputStream = new FileInputStream(localFile);
+            metadata.setContentEncoding("gzip");
+            metadata.setContentLength(localFile.length());
+        }
+
+        try {
+            final PutObjectRequest request = new PutObjectRequest(dest.bucketName, dest.objectName, inputStream, metadata)
+                .withMetadata(metadata);
             final PutObjectResult result = getClient().putObject(request);
             return new FingerprintRecord(produced, bucketName, file.getName(), result.getETag());
         } finally {
-            if (os != null) {
-                os.close();
-            }
-            if (deleteLocalFile && localFile != null) {
+            if (localFile != null) {
                 localFile.delete();
             }
         }

diff --git a/src/main/resources/hudson/plugins/s3/Entry/config.jelly b/src/main/resources/hudson/plugins/s3/Entry/config.jelly
@@ -27,5 +27,18 @@
         <f:entry field="flatten" title="Flatten directories">
 		    <f:checkbox />
         </f:entry>
+        <f:entry field="gzipFiles" title="GZIP files">
+            <f:checkbox />
+        </f:entry>
+
+        <f:entry title="Metadata tags">
+            <f:repeatableProperty field="userMetadata">
+                <f:entry title="">
+                  <div align="right">
+                    <f:repeatableDeleteButton />
+                  </div>
+                </f:entry>
+            </f:repeatableProperty>
+        </f:entry>
 
 </j:jelly>
diff --git a/src/main/resources/hudson/plugins/s3/Entry/help-gzipFiles.html b/src/main/resources/hudson/plugins/s3/Entry/help-gzipFiles.html
@@ -0,0 +1,4 @@
+<div>
+When enabled, files will be compressed with GZIP and "Content-Encoding" header
+will be set to "gzip".
+</div>