Skip to content

Commit

Permalink
VersionId support in listObjects API
Browse files Browse the repository at this point in the history
  • Loading branch information
@prakashsvmx
prakashsvmx committed Feb 18, 2021
1 parent da810fa commit 98c2637
Show file tree
Hide file tree
Showing 5 changed files with 267 additions and 48 deletions.
26 changes: 23 additions & 3 deletions docs/API.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -221,7 +221,7 @@ minioClient.removeBucket('mybucket', function(err) {
```

<a name="listObjects"></a>
### listObjects(bucketName, prefix, recursive)
### listObjects(bucketName, prefix, recursive [,listOpts])

Lists all objects in a bucket.

Expand All @@ -233,7 +233,7 @@ __Parameters__
| `bucketName` | _string_ | Name of the bucket. |
| `prefix` | _string_ | The prefix of the objects that should be listed (optional, default `''`). |
| `recursive` | _bool_ | `true` indicates recursive style listing and `false` indicates directory style listing delimited by '/'. (optional, default `false`). |

| `listOpts` | _object_ | query params to list object which can have `{MaxKeys: _int_ , IncludeVersion: _bool_ }` (optional)|

__Return Value__

Expand All @@ -249,7 +249,8 @@ The object is of the format:
| `obj.name` | _string_ | name of the object. |
| `obj.prefix` | _string_ | name of the object prefix. |
| `obj.size` | _number_ | size of the object. |
| `obj.etag` | _string_ |etag of the object. |
| `obj.etag` | _string_ | etag of the object. |
| `obj.versionId` | _string_ | versionId of the object. |
| `obj.lastModified` | _Date_ | modified time stamp. |

__Example__
Expand All @@ -261,6 +262,25 @@ stream.on('data', function(obj) { console.log(obj) } )
stream.on('error', function(err) { console.log(err) } )
```

__Example1__
To get Object versions

```js
var stream = minioClient.listObjects('mybucket','', true, {IncludeVersion:true})
stream.on('data', function(obj) { console.log(obj) } )
stream.on('error', function(err) { console.log(err) } )
```

__Example2__
To get Object versions along with MaxKeys

```js
var stream = minioClient.listObjects('mybucket','', true, {IncludeVersion:true, MaxKeys:3})
stream.on('data', function(obj) { console.log(obj) } )
stream.on('error', function(err) { console.log(err) } )
```


<a name="listObjectsV2"></a>
### listObjectsV2(bucketName, prefix, recursive, startAfter)

Expand Down
14 changes: 12 additions & 2 deletions examples/list-objects.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@
* limitations under the License.
*/

// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are
// dummy values, please replace them with original values.
// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are
// dummy values, please replace them with original values.

var Minio = require('minio')

Expand All @@ -32,3 +32,13 @@ objectsStream.on('data', function(obj) {
objectsStream.on('error', function(e) {
console.log(e)
})


// List all object versions in bucket my-bucketname.
var objectsStreamWithVersions = s3Client.listObjects('my-bucketname', '', true, {IncludeVersion:true})
objectsStreamWithVersions.on('data', function(obj) {
console.log(obj)
})
objectsStreamWithVersions.on('error', function(e) {
console.log(e)
})
60 changes: 47 additions & 13 deletions src/main/minio.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1184,7 +1184,7 @@ export class Client {
}

// list a batch of objects
listObjectsQuery(bucketName, prefix, marker, delimiter, maxKeys) {
listObjectsQuery(bucketName, prefix, marker, listQueryOpts={}) {
if (!isValidBucketName(bucketName)) {
throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName)
}
Expand All @@ -1194,29 +1194,48 @@ export class Client {
if (!isString(marker)) {
throw new TypeError('marker should be of type "string"')
}
if (!isString(delimiter)) {
let{
Delimiter,
MaxKeys,
IncludeVersion,
} = listQueryOpts

if (!isObject(listQueryOpts)) {
throw new TypeError('listQueryOpts should be of type "object"')
}


if (!isString(Delimiter)) {
throw new TypeError('delimiter should be of type "string"')
}
if (!isNumber(maxKeys)) {
if (!isNumber(MaxKeys)) {
throw new TypeError('maxKeys should be of type "number"')
}

var queries = []
const queries = []
// escape every value in query string, except maxKeys
queries.push(`prefix=${uriEscape(prefix)}`)
queries.push(`delimiter=${uriEscape(delimiter)}`)
queries.push(`delimiter=${uriEscape(Delimiter)}`)

if (IncludeVersion) {
queries.push(`versions`)
}

if (marker) {
marker = uriEscape(marker)
queries.push(`marker=${marker}`)
if (IncludeVersion) {
queries.push(`key-marker=${marker}`)
} else {
queries.push(`marker=${marker}`)
}
}

// no need to escape maxKeys
if (maxKeys) {
if (maxKeys >= 1000) {
maxKeys = 1000
if (MaxKeys) {
if (MaxKeys >= 1000) {
MaxKeys = 1000
}
queries.push(`max-keys=${maxKeys}`)
queries.push(`max-keys=${MaxKeys}`)
}
queries.sort()
var query = ''
Expand All @@ -1239,15 +1258,18 @@ export class Client {
// * `bucketName` _string_: name of the bucket
// * `prefix` _string_: the prefix of the objects that should be listed (optional, default `''`)
// * `recursive` _bool_: `true` indicates recursive style listing and `false` indicates directory style listing delimited by '/'. (optional, default `false`)
//
// * `listOpts _object_: query params to list object with below keys
// * listOpts.MaxKeys _int_ maximum number of keys to return
// * listOpts.IncludeVersion _bool_ true|false to include versions.
// __Return Value__
// * `stream` _Stream_: stream emitting the objects in the bucket, the object is of the format:
// * `obj.name` _string_: name of the object
// * `obj.prefix` _string_: name of the object prefix
// * `obj.size` _number_: size of the object
// * `obj.etag` _string_: etag of the object
// * `obj.lastModified` _Date_: modified time stamp
listObjects(bucketName, prefix, recursive) {
// * `obj.versionId` _string_: versionId of the object
listObjects(bucketName, prefix, recursive, listOpts={}) {
if (prefix === undefined) prefix = ''
if (recursive === undefined) recursive = false
if (!isValidBucketName(bucketName)) {
Expand All @@ -1262,9 +1284,21 @@ export class Client {
if (!isBoolean(recursive)) {
throw new TypeError('recursive should be of type "boolean"')
}
if (!isObject(listOpts)) {
throw new TypeError('listOpts should be of type "object"')
}
// if recursive is false set delimiter to '/'
var delimiter = recursive ? '' : '/'

var marker = ''
const maxKeys = listOpts.MaxKeys || 1000
const includeVersion=listOpts.IncludeVersion

const listQueryOpts={
Delimiter:delimiter,
MaxKeys:maxKeys,
IncludeVersion:includeVersion,
}
var objects = []
var ended = false
var readStream = Stream.Readable({objectMode: true})
Expand All @@ -1276,7 +1310,7 @@ export class Client {
}
if (ended) return readStream.push(null)
// if there are no objects to push do query for the next batch of objects
this.listObjectsQuery(bucketName, prefix, marker, delimiter, 1000)
this.listObjectsQuery(bucketName, prefix, marker, listQueryOpts)
.on('error', e => readStream.emit('error', e))
.on('data', result => {
if (result.isTruncated) {
Expand Down
89 changes: 68 additions & 21 deletions src/main/xml-parsers.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import fxp from 'fast-xml-parser'
import _ from 'lodash'
import * as errors from './errors.js'
import { sanitizeETag } from "./helpers"

var parseXml = (xml) => {
var result = null
Expand Down Expand Up @@ -279,33 +280,83 @@ export function parseCompleteMultipart(xml) {
}
}

const formatObjInfo = content => {

let {
Key,
LastModified,
ETag,
Size,
VersionId,
IsLatest
} = content

const name = toArray(Key)[0]
const lastModified = new Date(toArray(LastModified)[0])
const etag = sanitizeETag(toArray(ETag)[0])

return {
name,
lastModified,
etag,
size:Size,
versionId:VersionId,
isLatest:IsLatest
}
}

// parse XML response for list objects in a bucket
export function parseListObjects(xml) {
var result = {
objects: [],
isTruncated: false
}
var nextMarker
var nextMarker, nextVersionKeyMarker
var xmlobj = parseXml(xml)

if (!xmlobj.ListBucketResult) {
throw new errors.InvalidXMLError('Missing tag: "ListBucketResult"')
const listBucketResult = xmlobj.ListBucketResult
const listVersionsResult=xmlobj.ListVersionsResult

if (listBucketResult && listBucketResult.IsTruncated) {
result.isTruncated = listBucketResult.IsTruncated
}
xmlobj = xmlobj.ListBucketResult
if (xmlobj.IsTruncated) result.isTruncated = xmlobj.IsTruncated
if (xmlobj.Contents) {
toArray(xmlobj.Contents).forEach(content => {
var name = toArray(content.Key)[0]
var lastModified = new Date(toArray(content.LastModified)[0])
var etag = toArray(content.ETag)[0].replace(/^"/g, '').replace(/"$/g, '')
.replace(/^&quot;/g, '').replace(/&quot;$/g, '')
.replace(/^&#34;/g, '').replace(/&#34;$/g, '')
var size = content.Size
if (listVersionsResult && listVersionsResult.IsTruncated) {
result.isTruncated = listVersionsResult.IsTruncated
}

if (listBucketResult && listBucketResult.Contents) {
toArray(listBucketResult.Contents).forEach(content => {
const name = toArray(content.Key)[0]
const lastModified = new Date(toArray(content.LastModified)[0])
const etag = sanitizeETag(toArray(content.ETag)[0])
const size = content.Size
result.objects.push({name, lastModified, etag, size})
nextMarker = name
if (!nextMarker) {
nextMarker = name
}
})
}

if (listVersionsResult && listVersionsResult.Version) {
toArray(listVersionsResult.Version).forEach(content => {
result.objects.push(formatObjInfo(content))
})
}

if (listVersionsResult && listVersionsResult.DeleteMarker) {
toArray(listVersionsResult.DeleteMarker).forEach(content => {
result.objects.push(formatObjInfo(content))
})
}

if (listVersionsResult && listVersionsResult.NextKeyMarker) {
nextVersionKeyMarker = listVersionsResult.NextKeyMarker
}
if (listVersionsResult && listVersionsResult.NextVersionIdMarker) {
result.versionIdMarker = listVersionsResult.NextVersionIdMarker
}


if (xmlobj.CommonPrefixes) {
toArray(xmlobj.CommonPrefixes).forEach(commonPrefix => {
var prefix = toArray(commonPrefix.Prefix)[0]
Expand All @@ -314,7 +365,7 @@ export function parseListObjects(xml) {
})
}
if (result.isTruncated) {
result.nextMarker = xmlobj.NextMarker ? toArray(xmlobj.NextMarker)[0]: nextMarker
result.nextMarker = nextVersionKeyMarker || nextMarker
}
return result
}
Expand All @@ -336,9 +387,7 @@ export function parseListObjectsV2(xml) {
toArray(xmlobj.Contents).forEach(content => {
var name = content.Key
var lastModified = new Date(content.LastModified)
var etag = content.ETag.replace(/^"/g, '').replace(/"$/g, '')
.replace(/^&quot;/g, '').replace(/&quot;$/g, '')
.replace(/^&#34;/g, '').replace(/&#34;$/g, '')
var etag = sanitizeETag(content.ETag)
var size = content.Size
result.objects.push({name, lastModified, etag, size})
})
Expand Down Expand Up @@ -371,9 +420,7 @@ export function parseListObjectsV2WithMetadata(xml) {
toArray(xmlobj.Contents).forEach(content => {
var name = content.Key
var lastModified = new Date(content.LastModified)
var etag = content.ETag.replace(/^"/g, '').replace(/"$/g, '')
.replace(/^&quot;/g, '').replace(/&quot;$/g, '')
.replace(/^&#34;/g, '').replace(/&#34;$/g, '')
var etag = sanitizeETag(content.ETag)
var size = content.Size
var metadata
if (content.UserMetadata != null) {
Expand Down
Loading

0 comments on commit 98c2637

Please sign in to comment.