Skip to content

Commit

Permalink
VersionId support in listObjects API (#903)
Browse files Browse the repository at this point in the history
  • Loading branch information
@prakashsvmx
prakashsvmx authored Mar 4, 2021
1 parent da810fa commit bb7c645
Show file tree
Hide file tree
Showing 5 changed files with 290 additions and 64 deletions.
18 changes: 15 additions & 3 deletions docs/API.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -221,7 +221,7 @@ minioClient.removeBucket('mybucket', function(err) {
```

<a name="listObjects"></a>
### listObjects(bucketName, prefix, recursive)
### listObjects(bucketName, prefix, recursive [,listOpts])

Lists all objects in a bucket.

Expand All @@ -233,7 +233,7 @@ __Parameters__
| `bucketName` | _string_ | Name of the bucket. |
| `prefix` | _string_ | The prefix of the objects that should be listed (optional, default `''`). |
| `recursive` | _bool_ | `true` indicates recursive style listing and `false` indicates directory style listing delimited by '/'. (optional, default `false`). |

| `listOpts` | _object_ | query params to list object which can have `{IncludeVersion: _bool_ }` (optional)|

__Return Value__

Expand All @@ -249,7 +249,9 @@ The object is of the format:
| `obj.name` | _string_ | name of the object. |
| `obj.prefix` | _string_ | name of the object prefix. |
| `obj.size` | _number_ | size of the object. |
| `obj.etag` | _string_ |etag of the object. |
| `obj.etag` | _string_ | etag of the object. |
| `obj.versionId` | _string_ | versionId of the object. |
| `obj.isDeleteMarker` | _boolean_ | true if it is a delete marker. |
| `obj.lastModified` | _Date_ | modified time stamp. |

__Example__
Expand All @@ -261,6 +263,16 @@ stream.on('data', function(obj) { console.log(obj) } )
stream.on('error', function(err) { console.log(err) } )
```

__Example1__
To get Object versions

```js
var stream = minioClient.listObjects('mybucket','', true, {IncludeVersion:true})
stream.on('data', function(obj) { console.log(obj) } )
stream.on('error', function(err) { console.log(err) } )
```


<a name="listObjectsV2"></a>
### listObjectsV2(bucketName, prefix, recursive, startAfter)

Expand Down
36 changes: 34 additions & 2 deletions examples/list-objects.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@
* limitations under the License.
*/

// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are
// dummy values, please replace them with original values.
// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are
// dummy values, please replace them with original values.

var Minio = require('minio')

Expand All @@ -32,3 +32,35 @@ objectsStream.on('data', function(obj) {
objectsStream.on('error', function(e) {
console.log(e)
})


// List all object versions in bucket my-bucketname.
var objectsStreamWithVersions = s3Client.listObjects('my-bucketname', '', true, {IncludeVersion:true})
objectsStreamWithVersions.on('data', function(obj) {
console.log(obj)
})
objectsStreamWithVersions.on('error', function(e) {
console.log(e)
})


// Example to list only the prefixes of a bucket.
//Non versioned bucket with Prefix listing.
function listPrefixesOfABucket(buckName) {
var objectsStream=s3Client.listObjects(buckName, '', false , {})
var counter = 0
objectsStream.on('data', function (obj) {
if(obj.prefix) {
counter += 1
}
})
objectsStream.on('end',()=>{
console.log("Non Versioned Prefix Count:", counter)
})
objectsStream.on('error', function (e) {
console.log("::Error:",e)
})

}

listPrefixesOfABucket('your-bucket')
64 changes: 46 additions & 18 deletions src/main/minio.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1184,7 +1184,7 @@ export class Client {
}

// list a batch of objects
listObjectsQuery(bucketName, prefix, marker, delimiter, maxKeys) {
listObjectsQuery(bucketName, prefix, marker, listQueryOpts={}) {
if (!isValidBucketName(bucketName)) {
throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName)
}
Expand All @@ -1194,29 +1194,47 @@ export class Client {
if (!isString(marker)) {
throw new TypeError('marker should be of type "string"')
}
if (!isString(delimiter)) {
throw new TypeError('delimiter should be of type "string"')
let{
Delimiter,
MaxKeys,
IncludeVersion,
} = listQueryOpts

if (!isObject(listQueryOpts)) {
throw new TypeError('listQueryOpts should be of type "object"')
}
if (!isNumber(maxKeys)) {
throw new TypeError('maxKeys should be of type "number"')

if (!isString(Delimiter)) {
throw new TypeError('Delimiter should be of type "string"')
}
if (!isNumber(MaxKeys)) {
throw new TypeError('MaxKeys should be of type "number"')
}

var queries = []
const queries = []
// escape every value in query string, except maxKeys
queries.push(`prefix=${uriEscape(prefix)}`)
queries.push(`delimiter=${uriEscape(delimiter)}`)
queries.push(`delimiter=${uriEscape(Delimiter)}`)

if (IncludeVersion) {
queries.push(`versions`)
}

if (marker) {
marker = uriEscape(marker)
queries.push(`marker=${marker}`)
if (IncludeVersion) {
queries.push(`key-marker=${marker}`)
} else {
queries.push(`marker=${marker}`)
}
}

// no need to escape maxKeys
if (maxKeys) {
if (maxKeys >= 1000) {
maxKeys = 1000
if (MaxKeys) {
if (MaxKeys >= 1000) {
MaxKeys = 1000
}
queries.push(`max-keys=${maxKeys}`)
queries.push(`max-keys=${MaxKeys}`)
}
queries.sort()
var query = ''
Expand All @@ -1239,15 +1257,19 @@ export class Client {
// * `bucketName` _string_: name of the bucket
// * `prefix` _string_: the prefix of the objects that should be listed (optional, default `''`)
// * `recursive` _bool_: `true` indicates recursive style listing and `false` indicates directory style listing delimited by '/'. (optional, default `false`)
//
// * `listOpts _object_: query params to list object with below keys
// * listOpts.MaxKeys _int_ maximum number of keys to return
// * listOpts.IncludeVersion _bool_ true|false to include versions.
// __Return Value__
// * `stream` _Stream_: stream emitting the objects in the bucket, the object is of the format:
// * `obj.name` _string_: name of the object
// * `obj.prefix` _string_: name of the object prefix
// * `obj.size` _number_: size of the object
// * `obj.etag` _string_: etag of the object
// * `obj.lastModified` _Date_: modified time stamp
listObjects(bucketName, prefix, recursive) {
// * `obj.isDeleteMarker` _boolean_: true if it is a delete marker
// * `obj.versionId` _string_: versionId of the object
listObjects(bucketName, prefix, recursive, listOpts={}) {
if (prefix === undefined) prefix = ''
if (recursive === undefined) recursive = false
if (!isValidBucketName(bucketName)) {
Expand All @@ -1262,9 +1284,15 @@ export class Client {
if (!isBoolean(recursive)) {
throw new TypeError('recursive should be of type "boolean"')
}
// if recursive is false set delimiter to '/'
var delimiter = recursive ? '' : '/'
if (!isObject(listOpts)) {
throw new TypeError('listOpts should be of type "object"')
}
var marker = ''
const listQueryOpts={
Delimiter:recursive ? '' : '/', // if recursive is false set delimiter to '/'
MaxKeys: 1000,
IncludeVersion:listOpts.IncludeVersion,
}
var objects = []
var ended = false
var readStream = Stream.Readable({objectMode: true})
Expand All @@ -1276,11 +1304,11 @@ export class Client {
}
if (ended) return readStream.push(null)
// if there are no objects to push do query for the next batch of objects
this.listObjectsQuery(bucketName, prefix, marker, delimiter, 1000)
this.listObjectsQuery(bucketName, prefix, marker, listQueryOpts)
.on('error', e => readStream.emit('error', e))
.on('data', result => {
if (result.isTruncated) {
marker = result.nextMarker
marker = result.nextMarker || result.versionIdMarker
} else {
ended = true
}
Expand Down
123 changes: 92 additions & 31 deletions src/main/xml-parsers.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import fxp from 'fast-xml-parser'
import _ from 'lodash'
import * as errors from './errors.js'
import { sanitizeETag, isObject } from "./helpers"

var parseXml = (xml) => {
var result = null
Expand Down Expand Up @@ -279,42 +280,106 @@ export function parseCompleteMultipart(xml) {
}
}

const formatObjInfo = (content, opts={}) => {

let {
Key,
LastModified,
ETag,
Size,
VersionId,
IsLatest
} = content

if(!isObject(opts)){
opts = {}
}

const name = toArray(Key)[0]
const lastModified = new Date(toArray(LastModified)[0])
const etag = sanitizeETag(toArray(ETag)[0])

return {
name,
lastModified,
etag,
size:Size,
versionId:VersionId,
isLatest:IsLatest,
isDeleteMarker:opts.IsDeleteMarker ? opts.IsDeleteMarker: false
}
}

// parse XML response for list objects in a bucket
export function parseListObjects(xml) {
var result = {
objects: [],
isTruncated: false
}
var nextMarker
var xmlobj = parseXml(xml)
let isTruncated = false
let nextMarker, nextVersionKeyMarker
const xmlobj = parseXml(xml)

if (!xmlobj.ListBucketResult) {
throw new errors.InvalidXMLError('Missing tag: "ListBucketResult"')
const parseCommonPrefixesEntity = responseEntity => {
if(responseEntity){
toArray(responseEntity).forEach((commonPrefix) => {
const prefix = toArray(commonPrefix.Prefix)[0]
result.objects.push({prefix, size: 0})
})
}
}
xmlobj = xmlobj.ListBucketResult
if (xmlobj.IsTruncated) result.isTruncated = xmlobj.IsTruncated
if (xmlobj.Contents) {
toArray(xmlobj.Contents).forEach(content => {
var name = toArray(content.Key)[0]
var lastModified = new Date(toArray(content.LastModified)[0])
var etag = toArray(content.ETag)[0].replace(/^"/g, '').replace(/"$/g, '')
.replace(/^&quot;/g, '').replace(/&quot;$/g, '')
.replace(/^&#34;/g, '').replace(/&#34;$/g, '')
var size = content.Size
result.objects.push({name, lastModified, etag, size})
nextMarker = name
})

const listBucketResult = xmlobj.ListBucketResult
const listVersionsResult=xmlobj.ListVersionsResult

if(listBucketResult){
if ( listBucketResult.IsTruncated) {
isTruncated = listBucketResult.IsTruncated
}
if (listBucketResult.Contents) {
toArray(listBucketResult.Contents).forEach(content => {
const name = toArray(content.Key)[0]
const lastModified = new Date(toArray(content.LastModified)[0])
const etag = sanitizeETag(toArray(content.ETag)[0])
const size = content.Size
result.objects.push({name, lastModified, etag, size})
})
}

if( listBucketResult.NextMarker){
nextMarker = listBucketResult.NextMarker
}
parseCommonPrefixesEntity(listBucketResult.CommonPrefixes)
}

if(listVersionsResult){
if(listVersionsResult.IsTruncated){
isTruncated = listVersionsResult.IsTruncated
}

if (xmlobj.CommonPrefixes) {
toArray(xmlobj.CommonPrefixes).forEach(commonPrefix => {
var prefix = toArray(commonPrefix.Prefix)[0]
var size = 0
result.objects.push({prefix, size})
})
if (listVersionsResult.Version) {
toArray(listVersionsResult.Version).forEach(content => {
result.objects.push(formatObjInfo(content))
})
}
if (listVersionsResult.DeleteMarker) {
toArray(listVersionsResult.DeleteMarker).forEach(content => {
result.objects.push(formatObjInfo(content, {IsDeleteMarker:true}))
})
}

if (listVersionsResult.NextKeyMarker) {
nextVersionKeyMarker = listVersionsResult.NextKeyMarker
}
if (listVersionsResult.NextVersionIdMarker) {
result.versionIdMarker = listVersionsResult.NextVersionIdMarker
}
parseCommonPrefixesEntity(listVersionsResult.CommonPrefixes)
}
if (result.isTruncated) {
result.nextMarker = xmlobj.NextMarker ? toArray(xmlobj.NextMarker)[0]: nextMarker

result.isTruncated= isTruncated
if (isTruncated) {
result.nextMarker = nextVersionKeyMarker || nextMarker
}
return result
}
Expand All @@ -336,9 +401,7 @@ export function parseListObjectsV2(xml) {
toArray(xmlobj.Contents).forEach(content => {
var name = content.Key
var lastModified = new Date(content.LastModified)
var etag = content.ETag.replace(/^"/g, '').replace(/"$/g, '')
.replace(/^&quot;/g, '').replace(/&quot;$/g, '')
.replace(/^&#34;/g, '').replace(/&#34;$/g, '')
var etag = sanitizeETag(content.ETag)
var size = content.Size
result.objects.push({name, lastModified, etag, size})
})
Expand Down Expand Up @@ -371,9 +434,7 @@ export function parseListObjectsV2WithMetadata(xml) {
toArray(xmlobj.Contents).forEach(content => {
var name = content.Key
var lastModified = new Date(content.LastModified)
var etag = content.ETag.replace(/^"/g, '').replace(/"$/g, '')
.replace(/^&quot;/g, '').replace(/&quot;$/g, '')
.replace(/^&#34;/g, '').replace(/&#34;$/g, '')
var etag = sanitizeETag(content.ETag)
var size = content.Size
var metadata
if (content.UserMetadata != null) {
Expand Down
Loading

0 comments on commit bb7c645

Please sign in to comment.