VersionId support in listObjects API

docs/API.md

@@ -221,7 +221,7 @@ minioClient.removeBucket('mybucket', function(err) {
 ```
 
 <a name="listObjects"></a>
-### listObjects(bucketName, prefix, recursive)
+### listObjects(bucketName, prefix, recursive [,listOpts])
 
 Lists all objects in a bucket.
 
@@ -233,7 +233,7 @@ __Parameters__
 | `bucketName` | _string_ | Name of the bucket. |
 | `prefix`  | _string_  |  The prefix of the objects that should be listed (optional, default `''`). |
 | `recursive`  | _bool_  | `true` indicates recursive style listing and `false` indicates directory style listing delimited by '/'. (optional, default `false`).  |
-
+| `listOpts`  | _object_  | query params to list object which can have `{IncludeVersion: _bool_ }` (optional)|
 
 __Return Value__
 
@@ -249,7 +249,8 @@ The object is of the format:
 | `obj.name` | _string_ | name of the object. |
 | `obj.prefix` | _string_ | name of the object prefix. |
 | `obj.size` | _number_ | size of the object. |
-| `obj.etag` | _string_ |etag of the object. |
+| `obj.etag` | _string_ | etag of the object. |
+| `obj.versionId` | _string_ | versionId of the object. |
 | `obj.lastModified` | _Date_ | modified time stamp. |
 
 __Example__
@@ -261,6 +262,16 @@ stream.on('data', function(obj) { console.log(obj) } )
 stream.on('error', function(err) { console.log(err) } )
 ```
 
+__Example1__
+To get Object versions
+
+```js
+var stream = minioClient.listObjects('mybucket','', true, {IncludeVersion:true})
+stream.on('data', function(obj) { console.log(obj) } )
+stream.on('error', function(err) { console.log(err) } )
+```
+
+
 <a name="listObjectsV2"></a>
 ### listObjectsV2(bucketName, prefix, recursive, startAfter)
 

examples/list-objects.js

@@ -14,8 +14,8 @@
  * limitations under the License.
  */
 
- // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are
- // dummy values, please replace them with original values.
+// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are
+// dummy values, please replace them with original values.
 
 var Minio = require('minio')
 
@@ -32,3 +32,34 @@ objectsStream.on('data', function(obj) {
 objectsStream.on('error', function(e) {
   console.log(e)
 })
+
+
+// List all object versions in bucket my-bucketname.
+var objectsStreamWithVersions = s3Client.listObjects('my-bucketname', '', true, {IncludeVersion:true})
+objectsStreamWithVersions.on('data', function(obj) {
+  console.log(obj)
+})
+objectsStreamWithVersions.on('error', function(e) {
+  console.log(e)
+})
+
+
+// Example to list only the prefixes of a bucket.
+//Non versioned bucket with Prefix listing.
+function listPrefixesOfABucket(buckName) {
+  var objectsStream=s3Client.listObjects(buckName, '', false , {})
+  var counter = 0
+  objectsStream.on('data', function (obj) {
+    console.dir(obj)
+    counter+=1
+  })
+  objectsStream.on('end',()=>{
+    console.log("Non Versioned Prefix Count:", counter)
+  })
+  objectsStream.on('error', function (e) {
+    console.log("::Error:",e)
+  })
+
+}
+
+listPrefixesOfABucket('your-bucket')

src/main/minio.js

@@ -1184,7 +1184,7 @@ export class Client {
   }
 
   // list a batch of objects
-  listObjectsQuery(bucketName, prefix, marker, delimiter, maxKeys) {
+  listObjectsQuery(bucketName, prefix, marker, listQueryOpts={}) {
     if (!isValidBucketName(bucketName)) {
       throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName)
     }
@@ -1194,29 +1194,47 @@ export class Client {
     if (!isString(marker)) {
       throw new TypeError('marker should be of type "string"')
     }
-    if (!isString(delimiter)) {
+    let{
+      Delimiter,
+      MaxKeys,
+      IncludeVersion,
+    } = listQueryOpts
+
+    if (!isObject(listQueryOpts)) {
+      throw new TypeError('listQueryOpts should be of type "object"')
+    }
+
+    if (!isString(Delimiter)) {
       throw new TypeError('delimiter should be of type "string"')
     }
-    if (!isNumber(maxKeys)) {
+    if (!isNumber(MaxKeys)) {
       throw new TypeError('maxKeys should be of type "number"')
     }
 
-    var queries = []
+    const queries = []
     // escape every value in query string, except maxKeys
     queries.push(`prefix=${uriEscape(prefix)}`)
-    queries.push(`delimiter=${uriEscape(delimiter)}`)
+    queries.push(`delimiter=${uriEscape(Delimiter)}`)
+
+    if (IncludeVersion) {
+      queries.push(`versions`)
+    }
 
     if (marker) {
       marker = uriEscape(marker)
-      queries.push(`marker=${marker}`)
+      if (IncludeVersion) {
+        queries.push(`key-marker=${marker}`)
+      } else {
+        queries.push(`marker=${marker}`)
+      }
     }
 
     // no need to escape maxKeys
-    if (maxKeys) {
-      if (maxKeys >= 1000) {
-        maxKeys = 1000
+    if (MaxKeys) {
+      if (MaxKeys >= 1000) {
+        MaxKeys = 1000
       }
-      queries.push(`max-keys=${maxKeys}`)
+      queries.push(`max-keys=${MaxKeys}`)
     }
     queries.sort()
     var query = ''
@@ -1239,15 +1257,18 @@ export class Client {
   // * `bucketName` _string_: name of the bucket
   // * `prefix` _string_: the prefix of the objects that should be listed (optional, default `''`)
   // * `recursive` _bool_: `true` indicates recursive style listing and `false` indicates directory style listing delimited by '/'. (optional, default `false`)
-  //
+  // * `listOpts _object_: query params to list object with below keys
+  // *    listOpts.MaxKeys _int_ maximum number of keys to return
+  // *    listOpts.IncludeVersion  _bool_ true|false to include versions.
   // __Return Value__
   // * `stream` _Stream_: stream emitting the objects in the bucket, the object is of the format:
   // * `obj.name` _string_: name of the object
   // * `obj.prefix` _string_: name of the object prefix
   // * `obj.size` _number_: size of the object
   // * `obj.etag` _string_: etag of the object
   // * `obj.lastModified` _Date_: modified time stamp
-  listObjects(bucketName, prefix, recursive) {
+  // * `obj.versionId` _string_: versionId of the object
+  listObjects(bucketName, prefix, recursive, listOpts={}) {
     if (prefix === undefined) prefix = ''
     if (recursive === undefined) recursive = false
     if (!isValidBucketName(bucketName)) {
@@ -1262,9 +1283,15 @@ export class Client {
     if (!isBoolean(recursive)) {
       throw new TypeError('recursive should be of type "boolean"')
     }
-    // if recursive is false set delimiter to '/'
-    var delimiter = recursive ? '' : '/'
+    if (!isObject(listOpts)) {
+      throw new TypeError('listOpts should be of type "object"')
+    }
     var marker = ''
+    const listQueryOpts={
+      Delimiter:recursive ? '' : '/', // if recursive is false set delimiter to '/'
+      MaxKeys: 1000,
+      IncludeVersion:listOpts.IncludeVersion,
+    }
     var objects = []
     var ended = false
     var readStream = Stream.Readable({objectMode: true})
@@ -1276,11 +1303,11 @@ export class Client {
       }
       if (ended) return readStream.push(null)
       // if there are no objects to push do query for the next batch of objects
-      this.listObjectsQuery(bucketName, prefix, marker, delimiter, 1000)
+      this.listObjectsQuery(bucketName, prefix, marker, listQueryOpts)
         .on('error', e => readStream.emit('error', e))
         .on('data', result => {
           if (result.isTruncated) {
-            marker = result.nextMarker
+            marker = result.nextMarker || result.versionIdMarker
           } else {
             ended = true
           }

src/main/xml-parsers.js

@@ -17,6 +17,7 @@
 import fxp from 'fast-xml-parser'
 import _ from 'lodash'
 import * as errors from './errors.js'
+import { sanitizeETag } from "./helpers"
 
 var parseXml = (xml) => {
   var result = null
@@ -279,42 +280,101 @@ export function parseCompleteMultipart(xml) {
   }
 }
 
+const formatObjInfo = content => {
+
+  let {
+    Key,
+    LastModified,
+    ETag,
+    Size,
+    VersionId,
+    IsLatest
+  } = content
+
+  const name = toArray(Key)[0]
+  const lastModified = new Date(toArray(LastModified)[0])
+  const etag = sanitizeETag(toArray(ETag)[0])
+
+  return {
+    name,
+    lastModified,
+    etag,
+    size:Size,
+    versionId:VersionId,
+    isLatest:IsLatest
+  }
+}
+
 // parse XML response for list objects in a bucket
 export function parseListObjects(xml) {
   var result = {
     objects: [],
     isTruncated: false
   }
-  var nextMarker
-  var xmlobj = parseXml(xml)
+  let isTruncated = false
+  let nextMarker, nextVersionKeyMarker
+  const xmlobj = parseXml(xml)
 
-  if (!xmlobj.ListBucketResult) {
-    throw new errors.InvalidXMLError('Missing tag: "ListBucketResult"')
+  const parseCommonPrefixesEntity = responseEntity => {
+    if(responseEntity){
+      toArray(responseEntity).forEach((commonPrefix) => {
+        const prefix = toArray(commonPrefix.Prefix)[0]
+        result.objects.push({prefix, size: 0})
+      })
+    }
   }
-  xmlobj = xmlobj.ListBucketResult
-  if (xmlobj.IsTruncated) result.isTruncated = xmlobj.IsTruncated
-  if (xmlobj.Contents) {
-    toArray(xmlobj.Contents).forEach(content => {
-      var name = toArray(content.Key)[0]
-      var lastModified = new Date(toArray(content.LastModified)[0])
-      var etag = toArray(content.ETag)[0].replace(/^"/g, '').replace(/"$/g, '')
-        .replace(/^"/g, '').replace(/"$/g, '')
-        .replace(/^"/g, '').replace(/"$/g, '')
-      var size = content.Size
-      result.objects.push({name, lastModified, etag, size})
-      nextMarker = name
-    })
+
+  const listBucketResult = xmlobj.ListBucketResult
+  const listVersionsResult=xmlobj.ListVersionsResult
+
+  if(listBucketResult){
+    if ( listBucketResult.IsTruncated) {
+      isTruncated = listBucketResult.IsTruncated
+    }
+    if (listBucketResult.Contents) {
+      toArray(listBucketResult.Contents).forEach(content => {
+        const name = toArray(content.Key)[0]
+        const lastModified = new Date(toArray(content.LastModified)[0])
+        const etag = sanitizeETag(toArray(content.ETag)[0])
+        const size = content.Size
+        result.objects.push({name, lastModified, etag, size})
+      })
+    }
+
+    if( listBucketResult.NextMarker){
+      nextMarker = listBucketResult.NextMarker
+    }
+    parseCommonPrefixesEntity(listBucketResult.CommonPrefixes)
   }
+
+  if(listVersionsResult){
+    if(listVersionsResult.IsTruncated){
+      isTruncated = listVersionsResult.IsTruncated
+    }
 
-  if (xmlobj.CommonPrefixes) {
-    toArray(xmlobj.CommonPrefixes).forEach(commonPrefix => {
-      var prefix = toArray(commonPrefix.Prefix)[0]
-      var size = 0
-      result.objects.push({prefix, size})
-    })
+    if (listVersionsResult.Version) {
+      toArray(listVersionsResult.Version).forEach(content => {
+        result.objects.push(formatObjInfo(content))
+      })
+    }
+    if (listVersionsResult.DeleteMarker) {
+      toArray(listVersionsResult.DeleteMarker).forEach(content => {
+        result.objects.push(formatObjInfo(content))
+      })
+    }
+
+    if (listVersionsResult.NextKeyMarker) {
+      nextVersionKeyMarker = listVersionsResult.NextKeyMarker
+    }
+    if (listVersionsResult.NextVersionIdMarker) {
+      result.versionIdMarker = listVersionsResult.NextVersionIdMarker
+    }
+    parseCommonPrefixesEntity(listVersionsResult.CommonPrefixes)
   }
-  if (result.isTruncated) {
-    result.nextMarker = xmlobj.NextMarker ? toArray(xmlobj.NextMarker)[0]: nextMarker
+
+  if (isTruncated) {
+    result.isTruncated= isTruncated
+    result.nextMarker = nextVersionKeyMarker || nextMarker
   }
   return result
 }
@@ -336,9 +396,7 @@ export function parseListObjectsV2(xml) {
     toArray(xmlobj.Contents).forEach(content => {
       var name = content.Key
       var lastModified = new Date(content.LastModified)
-      var etag = content.ETag.replace(/^"/g, '').replace(/"$/g, '')
-        .replace(/^"/g, '').replace(/"$/g, '')
-        .replace(/^"/g, '').replace(/"$/g, '')
+      var etag = sanitizeETag(content.ETag)
       var size = content.Size
       result.objects.push({name, lastModified, etag, size})
     })
@@ -371,9 +429,7 @@ export function parseListObjectsV2WithMetadata(xml) {
     toArray(xmlobj.Contents).forEach(content => {
       var name = content.Key
       var lastModified = new Date(content.LastModified)
-      var etag = content.ETag.replace(/^"/g, '').replace(/"$/g, '')
-        .replace(/^"/g, '').replace(/"$/g, '')
-        .replace(/^"/g, '').replace(/"$/g, '')
+      var etag = sanitizeETag(content.ETag)
       var size = content.Size
       var metadata
       if (content.UserMetadata != null) {