"fast-start" the delivery of gzip-compressed bodies, and support flushing of GZIPOutputStreams on JDK7+

README.md

@@ -1,18 +1,35 @@
 ## ring-gzip-middleware
 
-Gzips [Ring](http://github.com/mmcgrana/ring) responses for user agents which can handle it.
+Gzips [Ring](http://github.com/ring-clojure/ring) responses for user agents
+which can handle it.
 
 ### Usage
 
 Apply the Ring middleware function `ring.middleware.gzip/wrap-gzip` to
 your Ring handler, typically at the top level (i.e. as the last bit of
 middleware in a `->` form).
 
-
 ### Installation
 
 Add `[amalloy/ring-gzip-middleware "0.1.2"]` to your Leingingen dependencies.
 
+### Compression of seq bodies
+
+In JDK versions <=6, [`java.util.zip.GZIPOutputStream.flush()` does not actually
+flush data compressed so
+far](http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4813885), which means
+that every gzip response must be complete before any bytes hit the wire.  While
+of marginal importance when compressing static files and other resources that
+are consumed in an all-or-nothing manner (i.e. virtually everything that is sent
+in a Ring response), lazy sequences are impacted negatively by this.  In
+particular, long-polling or server-sent event responses backed by lazy
+sequences, when gzipped under <=JDK6, must be fully consumed before the client
+receives any data at all.
+
+So, _this middleware does not gzip-compress Ring seq response bodies unless the
+JDK in use is 7+_, in which case it takes advantage of the new `flush`-ability
+of `GZIPOutputStream` there.
+
 ### License
 
 Copyright (C) 2010 Michael Stephens and other contributors.

src/ring/middleware/gzip.clj

@@ -1,22 +1,55 @@
 (ns ring.middleware.gzip
-  (:require [clojure.java.io :as io])
+  (:require [clojure.java.io :as io]
+            clojure.reflect)
   (:import (java.util.zip GZIPOutputStream)
            (java.io InputStream
+                    OutputStream
                     Closeable
                     File
                     PipedInputStream
                     PipedOutputStream)))
 
+; only available on JDK7
+(def ^:private flushable-gzip?
+  (delay (->> (clojure.reflect/reflect GZIPOutputStream)
+           :members
+           (some (comp '#{[java.io.OutputStream boolean]} :parameter-types)))))
+
+; only proxying here so we can specialize io/copy (which ring uses to transfer
+; InputStream bodies to the servlet response) for reading from the result of
+; piped-gzipped-input-stream
+(defn- piped-gzipped-input-stream*
+  []
+  (proxy [PipedInputStream] []))
+
+; exactly the same as do-copy for [InputStream OutputStream], but
+; flushes the output on every chunk; this allows gzipped content to start
+; flowing to clients ASAP (a reasonable change to ring IMO)
+(defmethod @#'io/do-copy [(class (piped-gzipped-input-stream*)) OutputStream]
+  [^InputStream input ^OutputStream output opts]
+  (let [buffer (make-array Byte/TYPE (or (:buffer-size opts) 1024))]
+    (loop []
+      (let [size (.read input buffer)]
+        (when (pos? size)
+          (do (.write output buffer 0 size)
+              (.flush output)
+              (recur)))))))
+
 (defn piped-gzipped-input-stream [in]
-  (let [pipe-in (PipedInputStream.)
+  (let [pipe-in (piped-gzipped-input-stream*)
         pipe-out (PipedOutputStream. pipe-in)]
-    (future                  ; new thread to prevent blocking deadlock
-      (with-open [out (GZIPOutputStream. pipe-out)]
+    ; separate thread to prevent blocking deadlock
+    (future
+      (with-open [out (if @flushable-gzip?
+                        (GZIPOutputStream. pipe-out true)
+                        (GZIPOutputStream. pipe-out))]
         (if (seq? in)
-          (doseq [string in] (io/copy (str string) out))
+          (doseq [string in]
+            (io/copy (str string) out)
+            (.flush out))
           (io/copy in out)))
       (when (instance? Closeable in)
-        (.close in)))
+        (.close ^Closeable in)))
     pipe-in))
 
 (defn gzipped-response [resp]
@@ -34,7 +67,7 @@
                (not (get-in resp [:headers "Content-Encoding"]))
                (or
                 (and (string? body) (> (count body) 200))
-                (seq? body)
+                (and (seq? body) @flushable-gzip?)
                 (instance? InputStream body)
                 (instance? File body)))
         (let [accepts (get-in req [:headers "accept-encoding"] "")
@@ -43,4 +76,4 @@
                                          (match 3))))
             (gzipped-response resp)
             resp))
-        resp))))
+        resp))))

test/ring/middleware/gzip_test.clj

@@ -45,14 +45,22 @@
     (is (Arrays/equals (unzip (resp :body)) (.getBytes output)))))
 
 (deftest test-string-seq-gzip
-  (let [app (wrap-gzip (fn [req] {:status 200
-                                  :body (->> (partition-all 20 output)
-                                          (map (partial apply str)))
+  (let [seq-body (->> (partition-all 20 output)
+                      (map (partial apply str)))
+        app (wrap-gzip (fn [req] {:status 200
+                                  :body seq-body
                                   :headers {}}))
         resp (app (accepting "gzip"))]
     (is (= 200 (:status resp)))
-    (is (= "gzip" (encoding resp)))
-    (is (Arrays/equals (unzip (resp :body)) (.getBytes output)))))
+    (if @@#'ring.middleware.gzip/flushable-gzip?
+      (do
+        (println "Running on JDK7+, testing gzipping of seq response bodies.")
+        (is (= "gzip" (encoding resp)))
+        (is (Arrays/equals (unzip (resp :body)) (.getBytes output))))
+      (do 
+        (println "Running on <=JDK6, testing non-gzipping of seq response bodies.")
+        (is (nil? (encoding resp)))
+        (is (= seq-body (resp :body)))))))
 
 (deftest test-accepts
   (doseq [ctype ["gzip" "*" "gzip,deflate" "gzip,deflate,sdch"