Add ETag header to all blobs (#1045)

docs/api.yaml

@@ -4562,6 +4562,8 @@ paths:
       description: If a Draft Form was created with an Excel file (`.xls` or `.xlsx`),
         you can get that file back by adding `.xls` or `.xlsx` as appropriate to the
         Draft Form resource path.
+
+        This endpoint supports `ETag`, which can be used to avoid downloading the same content more than once. When an API consumer calls this endpoint, it returns a value in `ETag` header, you can pass this value in the header `If-None-Match` of subsequent requests. If the file has not been changed since the previous request, you will receive `304 Not Modified` response otherwise you'll get the latest file.
       operationId: Retrieving Draft Form XLS(X)
       parameters:
       - name: projectId
@@ -4581,6 +4583,11 @@ paths:
       responses:
         200:
           description: OK
+          headers:
+            ETag:
+              schema:
+                type: string
+                description: content version identifier
           content:
             application/xml:
               example: |
@@ -4674,6 +4681,8 @@ paths:
         (attachment with a filename or Dataset name) and `Content-Type` (based on
         the type supplied at upload time or `text/csv` in the case of a linked Dataset)
         will be given.
+
+        This endpoint supports `ETag`, which can be used to avoid downloading the same content more than once. When an API consumer calls this endpoint, it returns a value in `ETag` header, you can pass this value in the header `If-None-Match` of subsequent requests. If the file has not been changed since the previous request, you will receive `304 Not Modified` response otherwise you'll get the latest file.
       operationId: Downloading a Draft Form Attachment
       parameters:
       - name: projectId
@@ -4704,6 +4713,10 @@ paths:
             Content-Disposition:
               schema:
                 type: string
+            ETag:
+              schema:
+                type: string
+                description: content version identifier
           content:
             '{the MIME type of the attachment file itself or text/csv for a Dataset}':
               example: |
@@ -5375,6 +5388,8 @@ paths:
       description: If a Form Version was created with an Excel file (`.xls` or `.xlsx`),
         you can get that file back by adding `.xls` or `.xlsx` as appropriate to the
         Form Version resource path.
+
+        This endpoint supports `ETag` header, which can be used to avoid downloading the same content more than once. When an API consumer calls this endpoint, the endpoint returns a value in `ETag` header. If you pass that value in the `If-None-Match` header of a subsequent request, then if the file has not been changed since the previous request, you will receive `304 Not Modified` response; otherwise you'll get the latest file.
       operationId: Retrieving Form Version XLS(X)
       parameters:
       - name: projectId
@@ -5402,6 +5417,10 @@ paths:
       responses:
         200:
           description: OK
+            ETag:
+              schema:
+                type: string
+                description: content version identifier
           content:
             application/xml:
               example: |
@@ -7878,6 +7897,8 @@ paths:
       summary: Downloading an Attachment
       description: The `Content-Type` and `Content-Disposition` will be set appropriately
         based on the file itself when requesting an attachment file download.
+
+        This endpoint supports `ETag`, which can be used to avoid downloading the same content more than once. When an API consumer calls this endpoint, it returns a value in `ETag` header, you can pass this value in the header `If-None-Match` of subsequent requests. If the file has not been changed since the previous request, you will receive `304 Not Modified` response otherwise you'll get the latest file.
       operationId: Downloading an Attachment
       parameters:
       - name: projectId
@@ -7915,6 +7936,10 @@ paths:
             Content-Disposition:
               schema:
                 type: string
+            ETag:
+              schema:
+                type: string
+                description: content version identifier
           content:
             '{the MIME type of the attachment file itself}':
               example: |
@@ -8426,6 +8451,8 @@ paths:
       description: It is important to note that this endpoint returns whatever is
         _currently_ uploaded against the _particular version_ of the _Submission_.
         It will not track overwritten attachments.
+
+        This endpoint supports `ETag`, which can be used to avoid downloading the same content more than once. When an API consumer calls this endpoint, it returns a value in `ETag` header, you can pass this value in the header `If-None-Match` of subsequent requests. If the file has not been changed since the previous request, you will receive `304 Not Modified` response otherwise you'll get the latest file.
       operationId: Downloading a Version's Attachment
       parameters:
       - name: projectId
@@ -8471,6 +8498,10 @@ paths:
             Content-Disposition:
               schema:
                 type: string
+            ETag:
+              schema:
+                type: string
+                description: content version identifier
           content:
             '{the MIME type of the attachment file itself}':
               example: |

lib/resources/forms.js

@@ -12,7 +12,7 @@ const { identity } = require('ramda');
 const { Blob, Form } = require('../model/frames');
 const { ensureDef } = require('../model/frame');
 const { QueryOptions } = require('../util/db');
-const { isTrue, xml, binary, contentDisposition, withEtag } = require('../util/http');
+const { isTrue, xml, blobResponse, contentDisposition, withEtag } = require('../util/http');
 const Problem = require('../util/problem');
 const { sanitizeFieldsForOdata, setVersion } = require('../data/schema');
 const { getOrNotFound, reject, resolve, rejectIf } = require('../util/promise');
@@ -40,7 +40,7 @@ const streamAttachment = async (container, attachment, response) => {
     return reject(Problem.user.notFound());
   } else if (attachment.blobId != null) {
     const blob = await Blobs.getById(attachment.blobId).then(getOrNotFound);
-    return withEtag(`${blob.md5}`, () => binary(blob.contentType, attachment.name, blob.content));
+    return blobResponse(attachment.name, blob);
   } else {
     const dataset = await Datasets.getById(attachment.datasetId, true).then(getOrNotFound);
     const properties = await Datasets.getProperties(attachment.datasetId);
@@ -268,7 +268,7 @@ module.exports = (service, endpoint) => {
           : Blobs.getById(form.def.xlsBlobId)
             .then(getOrNotFound)
             .then(rejectIf(((blob) => blob.contentType !== excelMimeTypes[extension]), noargs(Problem.user.notFound)))
-            .then((blob) => binary(blob.contentType, `${form.xmlFormId}.${extension}`, blob.content)))));
+            .then((blob) => blobResponse(`${form.xmlFormId}.${extension}`, blob)))));
     service.get(`${base}.xls`, getXls('xls'));
     service.get(`${base}.xlsx`, getXls('xlsx'));
 

lib/resources/submissions.js

@@ -15,7 +15,7 @@ const { ensureDef } = require('../model/frame');
 const { createdMessage } = require('../formats/openrosa');
 const { getOrNotFound, getOrReject, rejectIf, reject } = require('../util/promise');
 const { QueryOptions } = require('../util/db');
-const { success, xml, isFalse, contentDisposition, binary, redirect, url } = require('../util/http');
+const { success, xml, isFalse, contentDisposition, blobResponse, redirect, url } = require('../util/http');
 const Problem = require('../util/problem');
 const { streamBriefcaseCsvs } = require('../data/briefcase');
 const { streamAttachments } = require('../data/attachments');
@@ -411,7 +411,7 @@ module.exports = (service, endpoint) => {
       getOrRedirect(Forms, Submissions, context)
         .then(([ form, submission ]) => SubmissionAttachments.getCurrentBlobByIds(form.id, submission.id, context.params.name, draft))
         .then(getOrNotFound)
-        .then((blob) => binary(blob.contentType, context.params.name, blob.content))));
+        .then((blob) => blobResponse(context.params.name, blob))));
 
     // TODO: wow audit-logging this is expensive.
     service.post(
@@ -498,7 +498,7 @@ module.exports = (service, endpoint) => {
             .then(getOrNotFound),
           Submissions.verifyVersion(form.id, params.rootId, params.instanceId, draft)
         ]))
-        .then(([ blob ]) => binary(blob.contentType, params.name, blob.content))));
+        .then(([ blob ]) => blobResponse(params.name, blob))));
 
     ////////////////////////////////////////////////////////////////////////////////
     // Diffs between all versions of a submission

lib/util/http.js

@@ -157,10 +157,15 @@ const withEtag = (serverEtag, fn) => (request, response) => {
   return fn();
 };
 
+const blobResponse = (filename, blob) => withEtag(
+  blob.md5,
+  () => binary(blob.contentType, filename, blob.content),
+);
+
 module.exports = {
   isTrue, isFalse, urlPathname,
   serialize,
-  success, contentType, xml, atom, json, contentDisposition, binary, redirect,
+  success, contentType, xml, atom, json, contentDisposition, blobResponse, redirect,
   urlWithQueryParams, url,
   withEtag
 };

test/integration/api/forms/forms.js

@@ -683,8 +683,12 @@ describe('api: /projects/:id/forms (create, read, update)', () => {
               .then(({ headers, body }) => {
                 headers['content-type'].should.equal('application/vnd.openxmlformats-officedocument.spreadsheetml.sheet');
                 headers['content-disposition'].should.equal('attachment; filename="simple2.xlsx"; filename*=UTF-8\'\'simple2.xlsx');
+                headers['etag'].should.equal('"30fdb0e9115ea7ca6702573f521814d1"'); // eslint-disable-line dot-notation
                 Buffer.compare(input, body).should.equal(0);
-              })));
+              }))
+            .then(() => asAlice.get('/v1/projects/1/forms/simple2.xlsx')
+              .set('If-None-Match', '"30fdb0e9115ea7ca6702573f521814d1"')
+              .expect(304)));
       }));
 
       it('should return the xlsx file originally provided', testService((service) => {
@@ -704,8 +708,12 @@ describe('api: /projects/:id/forms (create, read, update)', () => {
               .then(({ headers, body }) => {
                 headers['content-type'].should.equal('application/vnd.openxmlformats-officedocument.spreadsheetml.sheet');
                 headers['content-disposition'].should.equal('attachment; filename="simple2.xlsx"; filename*=UTF-8\'\'simple2.xlsx');
+                headers['etag'].should.equal('"30fdb0e9115ea7ca6702573f521814d1"'); // eslint-disable-line dot-notation
                 Buffer.compare(input, body).should.equal(0);
-              })));
+              }))
+            .then(() => asAlice.get('/v1/projects/1/forms/simple2/draft.xlsx')
+              .set('If-None-Match', '"30fdb0e9115ea7ca6702573f521814d1"')
+              .expect(304)));
       }));
 
       it('should continue to offer the xlsx file after a copy-draft', testService((service) => {
@@ -755,8 +763,15 @@ describe('api: /projects/:id/forms (create, read, update)', () => {
             .set('Content-Type', 'application/vnd.ms-excel')
             .set('X-XlsForm-FormId-Fallback', 'testformid')
             .expect(200)
-            .then(() => asAlice.get('/v1/projects/1/forms/simple2.xls').expect(200))
-            .then(() => asAlice.get('/v1/projects/1/forms/simple2.xlsx').expect(404)))));
+            .then(() => asAlice.get('/v1/projects/1/forms/simple2.xls')
+              .expect(200)
+              .then(({ headers }) => {
+                headers['etag'].should.equal('"30fdb0e9115ea7ca6702573f521814d1"'); // eslint-disable-line dot-notation
+              }))
+            .then(() => asAlice.get('/v1/projects/1/forms/simple2.xlsx').expect(404))
+            .then(() => asAlice.get('/v1/projects/1/forms/simple2.xls')
+              .set('If-None-Match', '"30fdb0e9115ea7ca6702573f521814d1"')
+              .expect(304)))));
     });
 
     describe('.xml GET', () => {

test/integration/api/submissions.js

@@ -450,8 +450,12 @@ describe('api: /submission', () => {
               .then(({ headers, body }) => {
                 headers['content-type'].should.equal('video/mp4');
                 headers['content-disposition'].should.equal('attachment; filename="my_file1.mp4"; filename*=UTF-8\'\'my_file1.mp4');
+                headers['etag'].should.equal('"75f5701abfe7de8202cecaa0ca753f29"'); // eslint-disable-line dot-notation
                 body.toString('utf8').should.equal('this is test file one');
-              }))))));
+              }))
+            .then(() => asAlice.get('/v1/projects/1/forms/binaryType/submissions/both/attachments/my_file1.mp4')
+              .set('If-None-Match', '"75f5701abfe7de8202cecaa0ca753f29"')
+              .expect(304))))));
 
     it('should successfully save additionally POSTed attachment binary data', testService((service) =>
       service.login('alice', (asAlice) =>
@@ -474,8 +478,12 @@ describe('api: /submission', () => {
                 .then(({ headers, body }) => {
                   headers['content-type'].should.equal('image/jpeg');
                   headers['content-disposition'].should.equal('attachment; filename="here_is_file2.jpg"; filename*=UTF-8\'\'here_is_file2.jpg');
+                  headers['etag'].should.equal('"25bdb03b7942881c279788575997efba"'); // eslint-disable-line dot-notation
                   body.toString('utf8').should.equal('this is test file two');
-                })))))));
+                }))
+              .then(() => asAlice.get('/v1/projects/1/forms/binaryType/submissions/both/attachments/here_is_file2.jpg')
+                .set('If-None-Match', '"25bdb03b7942881c279788575997efba"')
+                .expect(304)))))));
 
     it('should accept encrypted submissions, with attachments', testService((service) =>
       service.login('alice', (asAlice) =>