From 98c26371de9a722a697806ad6c3c7f9cfacf2559 Mon Sep 17 00:00:00 2001 From: prakashsvmx Date: Thu, 18 Feb 2021 12:21:42 +0530 Subject: [PATCH 1/7] VersionId support in listObjects API --- docs/API.md | 26 ++++- examples/list-objects.js | 14 ++- src/main/minio.js | 60 ++++++++--- src/main/xml-parsers.js | 89 +++++++++++++---- src/test/functional/functional-tests.js | 126 ++++++++++++++++++++++-- 5 files changed, 267 insertions(+), 48 deletions(-) diff --git a/docs/API.md b/docs/API.md index 57df1623..2b1ed09a 100644 --- a/docs/API.md +++ b/docs/API.md @@ -221,7 +221,7 @@ minioClient.removeBucket('mybucket', function(err) { ``` -### listObjects(bucketName, prefix, recursive) +### listObjects(bucketName, prefix, recursive [,listOpts]) Lists all objects in a bucket. @@ -233,7 +233,7 @@ __Parameters__ | `bucketName` | _string_ | Name of the bucket. | | `prefix` | _string_ | The prefix of the objects that should be listed (optional, default `''`). | | `recursive` | _bool_ | `true` indicates recursive style listing and `false` indicates directory style listing delimited by '/'. (optional, default `false`). | - +| `listOpts` | _object_ | query params to list object which can have `{MaxKeys: _int_ , IncludeVersion: _bool_ }` (optional)| __Return Value__ @@ -249,7 +249,8 @@ The object is of the format: | `obj.name` | _string_ | name of the object. | | `obj.prefix` | _string_ | name of the object prefix. | | `obj.size` | _number_ | size of the object. | -| `obj.etag` | _string_ |etag of the object. | +| `obj.etag` | _string_ | etag of the object. | +| `obj.versionId` | _string_ | versionId of the object. | | `obj.lastModified` | _Date_ | modified time stamp. | __Example__ @@ -261,6 +262,25 @@ stream.on('data', function(obj) { console.log(obj) } ) stream.on('error', function(err) { console.log(err) } ) ``` +__Example1__ +To get Object versions + +```js +var stream = minioClient.listObjects('mybucket','', true, {IncludeVersion:true}) +stream.on('data', function(obj) { console.log(obj) } ) +stream.on('error', function(err) { console.log(err) } ) +``` + +__Example2__ +To get Object versions along with MaxKeys + +```js +var stream = minioClient.listObjects('mybucket','', true, {IncludeVersion:true, MaxKeys:3}) +stream.on('data', function(obj) { console.log(obj) } ) +stream.on('error', function(err) { console.log(err) } ) +``` + + ### listObjectsV2(bucketName, prefix, recursive, startAfter) diff --git a/examples/list-objects.js b/examples/list-objects.js index 72b22db8..a4d8b449 100644 --- a/examples/list-objects.js +++ b/examples/list-objects.js @@ -14,8 +14,8 @@ * limitations under the License. */ - // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are - // dummy values, please replace them with original values. +// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are +// dummy values, please replace them with original values. var Minio = require('minio') @@ -32,3 +32,13 @@ objectsStream.on('data', function(obj) { objectsStream.on('error', function(e) { console.log(e) }) + + +// List all object versions in bucket my-bucketname. +var objectsStreamWithVersions = s3Client.listObjects('my-bucketname', '', true, {IncludeVersion:true}) +objectsStreamWithVersions.on('data', function(obj) { + console.log(obj) +}) +objectsStreamWithVersions.on('error', function(e) { + console.log(e) +}) diff --git a/src/main/minio.js b/src/main/minio.js index baf842ac..849441d6 100644 --- a/src/main/minio.js +++ b/src/main/minio.js @@ -1184,7 +1184,7 @@ export class Client { } // list a batch of objects - listObjectsQuery(bucketName, prefix, marker, delimiter, maxKeys) { + listObjectsQuery(bucketName, prefix, marker, listQueryOpts={}) { if (!isValidBucketName(bucketName)) { throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName) } @@ -1194,29 +1194,48 @@ export class Client { if (!isString(marker)) { throw new TypeError('marker should be of type "string"') } - if (!isString(delimiter)) { + let{ + Delimiter, + MaxKeys, + IncludeVersion, + } = listQueryOpts + + if (!isObject(listQueryOpts)) { + throw new TypeError('listQueryOpts should be of type "object"') + } + + + if (!isString(Delimiter)) { throw new TypeError('delimiter should be of type "string"') } - if (!isNumber(maxKeys)) { + if (!isNumber(MaxKeys)) { throw new TypeError('maxKeys should be of type "number"') } - var queries = [] + const queries = [] // escape every value in query string, except maxKeys queries.push(`prefix=${uriEscape(prefix)}`) - queries.push(`delimiter=${uriEscape(delimiter)}`) + queries.push(`delimiter=${uriEscape(Delimiter)}`) + + if (IncludeVersion) { + queries.push(`versions`) + } if (marker) { marker = uriEscape(marker) - queries.push(`marker=${marker}`) + if (IncludeVersion) { + queries.push(`key-marker=${marker}`) + } else { + queries.push(`marker=${marker}`) + } } // no need to escape maxKeys - if (maxKeys) { - if (maxKeys >= 1000) { - maxKeys = 1000 + if (MaxKeys) { + if (MaxKeys >= 1000) { + MaxKeys = 1000 } - queries.push(`max-keys=${maxKeys}`) + queries.push(`max-keys=${MaxKeys}`) } queries.sort() var query = '' @@ -1239,7 +1258,9 @@ export class Client { // * `bucketName` _string_: name of the bucket // * `prefix` _string_: the prefix of the objects that should be listed (optional, default `''`) // * `recursive` _bool_: `true` indicates recursive style listing and `false` indicates directory style listing delimited by '/'. (optional, default `false`) - // + // * `listOpts _object_: query params to list object with below keys + // * listOpts.MaxKeys _int_ maximum number of keys to return + // * listOpts.IncludeVersion _bool_ true|false to include versions. // __Return Value__ // * `stream` _Stream_: stream emitting the objects in the bucket, the object is of the format: // * `obj.name` _string_: name of the object @@ -1247,7 +1268,8 @@ export class Client { // * `obj.size` _number_: size of the object // * `obj.etag` _string_: etag of the object // * `obj.lastModified` _Date_: modified time stamp - listObjects(bucketName, prefix, recursive) { + // * `obj.versionId` _string_: versionId of the object + listObjects(bucketName, prefix, recursive, listOpts={}) { if (prefix === undefined) prefix = '' if (recursive === undefined) recursive = false if (!isValidBucketName(bucketName)) { @@ -1262,9 +1284,21 @@ export class Client { if (!isBoolean(recursive)) { throw new TypeError('recursive should be of type "boolean"') } + if (!isObject(listOpts)) { + throw new TypeError('listOpts should be of type "object"') + } // if recursive is false set delimiter to '/' var delimiter = recursive ? '' : '/' + var marker = '' + const maxKeys = listOpts.MaxKeys || 1000 + const includeVersion=listOpts.IncludeVersion + + const listQueryOpts={ + Delimiter:delimiter, + MaxKeys:maxKeys, + IncludeVersion:includeVersion, + } var objects = [] var ended = false var readStream = Stream.Readable({objectMode: true}) @@ -1276,7 +1310,7 @@ export class Client { } if (ended) return readStream.push(null) // if there are no objects to push do query for the next batch of objects - this.listObjectsQuery(bucketName, prefix, marker, delimiter, 1000) + this.listObjectsQuery(bucketName, prefix, marker, listQueryOpts) .on('error', e => readStream.emit('error', e)) .on('data', result => { if (result.isTruncated) { diff --git a/src/main/xml-parsers.js b/src/main/xml-parsers.js index 4ec5ccdf..22345fd8 100644 --- a/src/main/xml-parsers.js +++ b/src/main/xml-parsers.js @@ -17,6 +17,7 @@ import fxp from 'fast-xml-parser' import _ from 'lodash' import * as errors from './errors.js' +import { sanitizeETag } from "./helpers" var parseXml = (xml) => { var result = null @@ -279,33 +280,83 @@ export function parseCompleteMultipart(xml) { } } +const formatObjInfo = content => { + + let { + Key, + LastModified, + ETag, + Size, + VersionId, + IsLatest + } = content + + const name = toArray(Key)[0] + const lastModified = new Date(toArray(LastModified)[0]) + const etag = sanitizeETag(toArray(ETag)[0]) + + return { + name, + lastModified, + etag, + size:Size, + versionId:VersionId, + isLatest:IsLatest + } +} + // parse XML response for list objects in a bucket export function parseListObjects(xml) { var result = { objects: [], isTruncated: false } - var nextMarker + var nextMarker, nextVersionKeyMarker var xmlobj = parseXml(xml) - if (!xmlobj.ListBucketResult) { - throw new errors.InvalidXMLError('Missing tag: "ListBucketResult"') + const listBucketResult = xmlobj.ListBucketResult + const listVersionsResult=xmlobj.ListVersionsResult + + if (listBucketResult && listBucketResult.IsTruncated) { + result.isTruncated = listBucketResult.IsTruncated } - xmlobj = xmlobj.ListBucketResult - if (xmlobj.IsTruncated) result.isTruncated = xmlobj.IsTruncated - if (xmlobj.Contents) { - toArray(xmlobj.Contents).forEach(content => { - var name = toArray(content.Key)[0] - var lastModified = new Date(toArray(content.LastModified)[0]) - var etag = toArray(content.ETag)[0].replace(/^"/g, '').replace(/"$/g, '') - .replace(/^"/g, '').replace(/"$/g, '') - .replace(/^"/g, '').replace(/"$/g, '') - var size = content.Size + if (listVersionsResult && listVersionsResult.IsTruncated) { + result.isTruncated = listVersionsResult.IsTruncated + } + + if (listBucketResult && listBucketResult.Contents) { + toArray(listBucketResult.Contents).forEach(content => { + const name = toArray(content.Key)[0] + const lastModified = new Date(toArray(content.LastModified)[0]) + const etag = sanitizeETag(toArray(content.ETag)[0]) + const size = content.Size result.objects.push({name, lastModified, etag, size}) - nextMarker = name + if (!nextMarker) { + nextMarker = name + } }) } + if (listVersionsResult && listVersionsResult.Version) { + toArray(listVersionsResult.Version).forEach(content => { + result.objects.push(formatObjInfo(content)) + }) + } + + if (listVersionsResult && listVersionsResult.DeleteMarker) { + toArray(listVersionsResult.DeleteMarker).forEach(content => { + result.objects.push(formatObjInfo(content)) + }) + } + + if (listVersionsResult && listVersionsResult.NextKeyMarker) { + nextVersionKeyMarker = listVersionsResult.NextKeyMarker + } + if (listVersionsResult && listVersionsResult.NextVersionIdMarker) { + result.versionIdMarker = listVersionsResult.NextVersionIdMarker + } + + if (xmlobj.CommonPrefixes) { toArray(xmlobj.CommonPrefixes).forEach(commonPrefix => { var prefix = toArray(commonPrefix.Prefix)[0] @@ -314,7 +365,7 @@ export function parseListObjects(xml) { }) } if (result.isTruncated) { - result.nextMarker = xmlobj.NextMarker ? toArray(xmlobj.NextMarker)[0]: nextMarker + result.nextMarker = nextVersionKeyMarker || nextMarker } return result } @@ -336,9 +387,7 @@ export function parseListObjectsV2(xml) { toArray(xmlobj.Contents).forEach(content => { var name = content.Key var lastModified = new Date(content.LastModified) - var etag = content.ETag.replace(/^"/g, '').replace(/"$/g, '') - .replace(/^"/g, '').replace(/"$/g, '') - .replace(/^"/g, '').replace(/"$/g, '') + var etag = sanitizeETag(content.ETag) var size = content.Size result.objects.push({name, lastModified, etag, size}) }) @@ -371,9 +420,7 @@ export function parseListObjectsV2WithMetadata(xml) { toArray(xmlobj.Contents).forEach(content => { var name = content.Key var lastModified = new Date(content.LastModified) - var etag = content.ETag.replace(/^"/g, '').replace(/"$/g, '') - .replace(/^"/g, '').replace(/"$/g, '') - .replace(/^"/g, '').replace(/"$/g, '') + var etag = sanitizeETag(content.ETag) var size = content.Size var metadata if (content.UserMetadata != null) { diff --git a/src/test/functional/functional-tests.js b/src/test/functional/functional-tests.js index 01fde90e..b0152b6a 100644 --- a/src/test/functional/functional-tests.js +++ b/src/test/functional/functional-tests.js @@ -111,7 +111,15 @@ describe('functional tests', function() { } var tmpDir = os.tmpdir() - + + function readableStream(data) { + var s = new stream.Readable() + s._read = () => {} + s.push(data) + s.push(null) + return s + } + var traceStream // FUNCTIONAL_TEST_TRACE env variable contains the path to which trace @@ -1117,14 +1125,6 @@ describe('functional tests', function() { }) }) - function readableStream(data) { - var s = new stream.Readable() - s._read = () => {} - s.push(data) - s.push(null) - return s - } - describe('removeObjects', function() { var listObjectPrefix = 'miniojsPrefix' var listObjectsNum = 10 @@ -1466,4 +1466,112 @@ describe('functional tests', function() { }) }) + + + describe('Versioning Supported listObjects', function() { + const versionedBucketName = "minio-js-test-version-list" + uuid.v4() + const prefixName = "Prefix1" + const versionedObjectName ="datafile-versioned-list-100-kB" + const objVersionIdCounter = [1,2,3,4,5]// This should track adding 5 versions of the same object. + let listObjectsNum = objVersionIdCounter.length + let objArray = [] + let listPrefixArray = [] + let isVersioningSupported=false + + const objNameWithPrefix = `${prefixName}/${versionedObjectName}` + + before((done) => client.makeBucket(versionedBucketName, '', ()=>{ + client.setBucketVersioning(versionedBucketName,{Status:"Enabled"},(err)=>{ + if (err && err.code === 'NotImplemented') return done() + if (err) return done(err) + isVersioningSupported = true + done() + }) + + })) + after((done) => client.removeBucket(versionedBucketName, done)) + + + step(`putObject(bucketName, objectName, stream, size, metaData, callback)_bucketName:${versionedBucketName}, stream:1b, size:1_Create ${listObjectsNum} objects`, done => { + if(isVersioningSupported) { + objVersionIdCounter.forEach((versionCounter, index)=>{ + client.putObject(versionedBucketName, objNameWithPrefix, readableStream(_1byte), _1byte.length, {}, (e,data)=>{ + objArray.push(data) + if(index+1 === objVersionIdCounter.length) + done() + }) + }) + }else { + done() + } + }) + + step(`listObjects(bucketName, prefix, recursive)_bucketName:${versionedBucketName}, prefix: '', recursive:true_`, done => { + if(isVersioningSupported) { + client.listObjects(versionedBucketName, '', true, {IncludeVersion: true}) + .on('error', done) + .on('end', () => { + if (_.isEqual(objArray.length, listPrefixArray.length)) return done() + return done(new Error(`listObjects lists ${listPrefixArray.length} objects, expected ${listObjectsNum}`)) + }) + .on('data', data => { + listPrefixArray.push(data) + }) + } else { + done() + } + }) + + step(`listObjects(bucketName, prefix, recursive)_bucketName:${versionedBucketName}, prefix: ${prefixName}, recursive:true_`, done => { + if(isVersioningSupported) { + listPrefixArray=[] + client.listObjects(versionedBucketName, prefixName, true, {IncludeVersion: true}) + .on('error', done) + .on('end', () => { + if (_.isEqual(objArray.length, listPrefixArray.length)) return done() + return done(new Error(`listObjects lists ${listPrefixArray.length} objects, expected ${listObjectsNum}`)) + }) + .on('data', data => { + listPrefixArray.push(data) + }) + } else { + done() + } + }) + + step(`listObjects(bucketName, prefix, recursive, listOpts)_bucketName:${versionedBucketName}, prefix: ${prefixName}, recursive:true_ ,{IncludeVersion: true, MaxKeys:2}`, done => { + if(isVersioningSupported) { + const maxKeysList=[] + client.listObjects(versionedBucketName, '', true, {IncludeVersion: true, MaxKeys: 2}) + .on('error', done) + .on('end', () => { + if (maxKeysList.length === 2) return done() + return done(new Error(`listObjects lists ${maxKeysList.length} objects, expected ${2}`)) + }) + .on('data', data => { + maxKeysList.push(data) + }) + } else { + done() + } + }) + + + step(`removeObject(bucketName, objectName, removeOpts)_bucketName:${versionedBucketName}_Remove ${listObjectsNum} objects`, done => { + if(isVersioningSupported) { + listPrefixArray.forEach((item, index)=>{ + client.removeObject(versionedBucketName, objNameWithPrefix, {versionId: item.versionId}, (res)=>{ + if(!res) { + if (index+1 === listPrefixArray.length) { + done() + } + } + }) + }) + }else { + done() + } + }) + }) + }) From 0fda269e6df32ca3994291ce477b26357e3d4ff1 Mon Sep 17 00:00:00 2001 From: prakashsvmx Date: Tue, 23 Feb 2021 13:03:54 +0530 Subject: [PATCH 2/7] Address few review comments --- src/main/minio.js | 13 ++------ src/main/xml-parsers.js | 73 +++++++++++++++++++++-------------------- 2 files changed, 41 insertions(+), 45 deletions(-) diff --git a/src/main/minio.js b/src/main/minio.js index 849441d6..2ef99871 100644 --- a/src/main/minio.js +++ b/src/main/minio.js @@ -1203,7 +1203,6 @@ export class Client { if (!isObject(listQueryOpts)) { throw new TypeError('listQueryOpts should be of type "object"') } - if (!isString(Delimiter)) { throw new TypeError('delimiter should be of type "string"') @@ -1287,17 +1286,11 @@ export class Client { if (!isObject(listOpts)) { throw new TypeError('listOpts should be of type "object"') } - // if recursive is false set delimiter to '/' - var delimiter = recursive ? '' : '/' - var marker = '' - const maxKeys = listOpts.MaxKeys || 1000 - const includeVersion=listOpts.IncludeVersion - const listQueryOpts={ - Delimiter:delimiter, - MaxKeys:maxKeys, - IncludeVersion:includeVersion, + Delimiter:recursive ? '' : '/', // if recursive is false set delimiter to '/' + MaxKeys: listOpts.MaxKeys || 1000, + IncludeVersion:listOpts.IncludeVersion, } var objects = [] var ended = false diff --git a/src/main/xml-parsers.js b/src/main/xml-parsers.js index 22345fd8..3f6bee74 100644 --- a/src/main/xml-parsers.js +++ b/src/main/xml-parsers.js @@ -316,47 +316,50 @@ export function parseListObjects(xml) { const listBucketResult = xmlobj.ListBucketResult const listVersionsResult=xmlobj.ListVersionsResult - - if (listBucketResult && listBucketResult.IsTruncated) { - result.isTruncated = listBucketResult.IsTruncated - } - if (listVersionsResult && listVersionsResult.IsTruncated) { - result.isTruncated = listVersionsResult.IsTruncated - } - if (listBucketResult && listBucketResult.Contents) { - toArray(listBucketResult.Contents).forEach(content => { - const name = toArray(content.Key)[0] - const lastModified = new Date(toArray(content.LastModified)[0]) - const etag = sanitizeETag(toArray(content.ETag)[0]) - const size = content.Size - result.objects.push({name, lastModified, etag, size}) - if (!nextMarker) { - nextMarker = name - } - }) - } - - if (listVersionsResult && listVersionsResult.Version) { - toArray(listVersionsResult.Version).forEach(content => { - result.objects.push(formatObjInfo(content)) - }) + if(listBucketResult){ + if ( listBucketResult.IsTruncated) { + result.isTruncated = listBucketResult.IsTruncated + } + if (listBucketResult.Contents) { + toArray(listBucketResult.Contents).forEach(content => { + const name = toArray(content.Key)[0] + const lastModified = new Date(toArray(content.LastModified)[0]) + const etag = sanitizeETag(toArray(content.ETag)[0]) + const size = content.Size + result.objects.push({name, lastModified, etag, size}) + if (!nextMarker) { + nextMarker = name + } + }) + } } + + if(listVersionsResult){ + if(listVersionsResult.IsTruncated){ + result.isTruncated = listVersionsResult.IsTruncated + } - if (listVersionsResult && listVersionsResult.DeleteMarker) { - toArray(listVersionsResult.DeleteMarker).forEach(content => { - result.objects.push(formatObjInfo(content)) - }) - } + if (listVersionsResult.Version) { + toArray(listVersionsResult.Version).forEach(content => { + result.objects.push(formatObjInfo(content)) + }) + } + if (listVersionsResult.DeleteMarker) { + toArray(listVersionsResult.DeleteMarker).forEach(content => { + result.objects.push(formatObjInfo(content)) + }) + } - if (listVersionsResult && listVersionsResult.NextKeyMarker) { - nextVersionKeyMarker = listVersionsResult.NextKeyMarker - } - if (listVersionsResult && listVersionsResult.NextVersionIdMarker) { - result.versionIdMarker = listVersionsResult.NextVersionIdMarker + if (listVersionsResult.NextKeyMarker) { + nextVersionKeyMarker = listVersionsResult.NextKeyMarker + } + if (listVersionsResult.NextVersionIdMarker) { + result.versionIdMarker = listVersionsResult.NextVersionIdMarker + } + } - if (xmlobj.CommonPrefixes) { toArray(xmlobj.CommonPrefixes).forEach(commonPrefix => { var prefix = toArray(commonPrefix.Prefix)[0] From 19dce359e4e39abc2f14e157baaa9972ddeabd5a Mon Sep 17 00:00:00 2001 From: prakashsvmx Date: Tue, 23 Feb 2021 20:01:57 +0530 Subject: [PATCH 3/7] Address CommonPrefix related parsing --- examples/list-objects.js | 21 +++++++++++++++++++++ src/main/xml-parsers.js | 31 ++++++++++++++++++------------- 2 files changed, 39 insertions(+), 13 deletions(-) diff --git a/examples/list-objects.js b/examples/list-objects.js index a4d8b449..aa46f5b0 100644 --- a/examples/list-objects.js +++ b/examples/list-objects.js @@ -42,3 +42,24 @@ objectsStreamWithVersions.on('data', function(obj) { objectsStreamWithVersions.on('error', function(e) { console.log(e) }) + + +// Example to list only the prefixes of a bucket. +//Non versioned bucket with Prefix listing. +function listPrefixesOfABucket(buckName) { + var objectsStream=s3Client.listObjects(buckName, '', false , {}) + var counter = 0 + objectsStream.on('data', function (obj) { + console.dir(obj) + counter+=1 + }) + objectsStream.on('end',()=>{ + console.log("Non Versioned Prefix Count:", counter) + }) + objectsStream.on('error', function (e) { + console.log("::Error:",e) + }) + +} + +listPrefixesOfABucket('your-bucket') \ No newline at end of file diff --git a/src/main/xml-parsers.js b/src/main/xml-parsers.js index 3f6bee74..fb63674a 100644 --- a/src/main/xml-parsers.js +++ b/src/main/xml-parsers.js @@ -311,15 +311,25 @@ export function parseListObjects(xml) { objects: [], isTruncated: false } - var nextMarker, nextVersionKeyMarker - var xmlobj = parseXml(xml) + let isTruncated = false + let nextMarker, nextVersionKeyMarker + const xmlobj = parseXml(xml) + + const parseCommonPrefixesEntity = responseEntity => { + if(responseEntity){ + toArray(responseEntity).forEach((commonPrefix) => { + const prefix = toArray(commonPrefix.Prefix)[0] + result.objects.push({prefix, size: 0}) + }) + } + } const listBucketResult = xmlobj.ListBucketResult const listVersionsResult=xmlobj.ListVersionsResult if(listBucketResult){ if ( listBucketResult.IsTruncated) { - result.isTruncated = listBucketResult.IsTruncated + isTruncated = listBucketResult.IsTruncated } if (listBucketResult.Contents) { toArray(listBucketResult.Contents).forEach(content => { @@ -333,11 +343,12 @@ export function parseListObjects(xml) { } }) } + parseCommonPrefixesEntity(listBucketResult.CommonPrefixes) } if(listVersionsResult){ if(listVersionsResult.IsTruncated){ - result.isTruncated = listVersionsResult.IsTruncated + isTruncated = listVersionsResult.IsTruncated } if (listVersionsResult.Version) { @@ -357,17 +368,11 @@ export function parseListObjects(xml) { if (listVersionsResult.NextVersionIdMarker) { result.versionIdMarker = listVersionsResult.NextVersionIdMarker } - + parseCommonPrefixesEntity(listVersionsResult.CommonPrefixes) } - if (xmlobj.CommonPrefixes) { - toArray(xmlobj.CommonPrefixes).forEach(commonPrefix => { - var prefix = toArray(commonPrefix.Prefix)[0] - var size = 0 - result.objects.push({prefix, size}) - }) - } - if (result.isTruncated) { + if (isTruncated) { + result.isTruncated= isTruncated result.nextMarker = nextVersionKeyMarker || nextMarker } return result From 7a65ce4e8d0f665e1043a3540e382ed5c002b5f2 Mon Sep 17 00:00:00 2001 From: prakashsvmx Date: Wed, 24 Feb 2021 13:51:40 +0530 Subject: [PATCH 4/7] Fix recursive listing --- src/main/minio.js | 2 +- src/main/xml-parsers.js | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/minio.js b/src/main/minio.js index 2ef99871..992a9ddc 100644 --- a/src/main/minio.js +++ b/src/main/minio.js @@ -1307,7 +1307,7 @@ export class Client { .on('error', e => readStream.emit('error', e)) .on('data', result => { if (result.isTruncated) { - marker = result.nextMarker + marker = result.nextMarker || result.versionIdMarker } else { ended = true } diff --git a/src/main/xml-parsers.js b/src/main/xml-parsers.js index fb63674a..f75ffad3 100644 --- a/src/main/xml-parsers.js +++ b/src/main/xml-parsers.js @@ -338,11 +338,12 @@ export function parseListObjects(xml) { const etag = sanitizeETag(toArray(content.ETag)[0]) const size = content.Size result.objects.push({name, lastModified, etag, size}) - if (!nextMarker) { - nextMarker = name - } }) } + + if( listBucketResult.NextMarker){ + nextMarker = listBucketResult.NextMarker + } parseCommonPrefixesEntity(listBucketResult.CommonPrefixes) } From ea3587075e08fc955f9b41a303e35ab2e4254dac Mon Sep 17 00:00:00 2001 From: prakashsvmx Date: Thu, 25 Feb 2021 10:21:37 +0530 Subject: [PATCH 5/7] Revert max-keys support --- docs/API.md | 11 +------ src/main/minio.js | 2 +- src/test/functional/functional-tests.js | 39 ++++++++----------------- 3 files changed, 14 insertions(+), 38 deletions(-) diff --git a/docs/API.md b/docs/API.md index 2b1ed09a..a4da8812 100644 --- a/docs/API.md +++ b/docs/API.md @@ -233,7 +233,7 @@ __Parameters__ | `bucketName` | _string_ | Name of the bucket. | | `prefix` | _string_ | The prefix of the objects that should be listed (optional, default `''`). | | `recursive` | _bool_ | `true` indicates recursive style listing and `false` indicates directory style listing delimited by '/'. (optional, default `false`). | -| `listOpts` | _object_ | query params to list object which can have `{MaxKeys: _int_ , IncludeVersion: _bool_ }` (optional)| +| `listOpts` | _object_ | query params to list object which can have `{IncludeVersion: _bool_ }` (optional)| __Return Value__ @@ -271,15 +271,6 @@ stream.on('data', function(obj) { console.log(obj) } ) stream.on('error', function(err) { console.log(err) } ) ``` -__Example2__ -To get Object versions along with MaxKeys - -```js -var stream = minioClient.listObjects('mybucket','', true, {IncludeVersion:true, MaxKeys:3}) -stream.on('data', function(obj) { console.log(obj) } ) -stream.on('error', function(err) { console.log(err) } ) -``` - ### listObjectsV2(bucketName, prefix, recursive, startAfter) diff --git a/src/main/minio.js b/src/main/minio.js index 992a9ddc..c9ad9d89 100644 --- a/src/main/minio.js +++ b/src/main/minio.js @@ -1289,7 +1289,7 @@ export class Client { var marker = '' const listQueryOpts={ Delimiter:recursive ? '' : '/', // if recursive is false set delimiter to '/' - MaxKeys: listOpts.MaxKeys || 1000, + MaxKeys: 1000, IncludeVersion:listOpts.IncludeVersion, } var objects = [] diff --git a/src/test/functional/functional-tests.js b/src/test/functional/functional-tests.js index b0152b6a..09ef4374 100644 --- a/src/test/functional/functional-tests.js +++ b/src/test/functional/functional-tests.js @@ -61,7 +61,7 @@ describe('functional tests', function() { } else { playConfig.useSSL = true } - + // dataDir is falsy if we need to generate data on the fly. Otherwise, it will be // a directory with files to read from, i.e. /mint/data. var dataDir = process.env['MINT_DATA_DIR'] @@ -1466,7 +1466,6 @@ describe('functional tests', function() { }) }) - describe('Versioning Supported listObjects', function() { const versionedBucketName = "minio-js-test-version-list" + uuid.v4() @@ -1494,11 +1493,14 @@ describe('functional tests', function() { step(`putObject(bucketName, objectName, stream, size, metaData, callback)_bucketName:${versionedBucketName}, stream:1b, size:1_Create ${listObjectsNum} objects`, done => { if(isVersioningSupported) { - objVersionIdCounter.forEach((versionCounter, index)=>{ + let count=1 + objVersionIdCounter.forEach(()=>{ client.putObject(versionedBucketName, objNameWithPrefix, readableStream(_1byte), _1byte.length, {}, (e,data)=>{ objArray.push(data) - if(index+1 === objVersionIdCounter.length) + if(count === objVersionIdCounter.length) { done() + } + count +=1 }) }) }else { @@ -1539,33 +1541,16 @@ describe('functional tests', function() { } }) - step(`listObjects(bucketName, prefix, recursive, listOpts)_bucketName:${versionedBucketName}, prefix: ${prefixName}, recursive:true_ ,{IncludeVersion: true, MaxKeys:2}`, done => { - if(isVersioningSupported) { - const maxKeysList=[] - client.listObjects(versionedBucketName, '', true, {IncludeVersion: true, MaxKeys: 2}) - .on('error', done) - .on('end', () => { - if (maxKeysList.length === 2) return done() - return done(new Error(`listObjects lists ${maxKeysList.length} objects, expected ${2}`)) - }) - .on('data', data => { - maxKeysList.push(data) - }) - } else { - done() - } - }) - step(`removeObject(bucketName, objectName, removeOpts)_bucketName:${versionedBucketName}_Remove ${listObjectsNum} objects`, done => { if(isVersioningSupported) { - listPrefixArray.forEach((item, index)=>{ - client.removeObject(versionedBucketName, objNameWithPrefix, {versionId: item.versionId}, (res)=>{ - if(!res) { - if (index+1 === listPrefixArray.length) { - done() - } + let count=1 + listPrefixArray.forEach((item)=>{ + client.removeObject(versionedBucketName ,item.name, {versionId: item.versionId}, ()=>{ + if (count === listPrefixArray.length) { + done() } + count +=1 }) }) }else { From e792826047e91b57082ab118f2761549c945b47c Mon Sep 17 00:00:00 2001 From: prakashsvmx Date: Thu, 25 Feb 2021 18:22:52 +0530 Subject: [PATCH 6/7] Address review comments --- examples/list-objects.js | 5 +++-- src/main/minio.js | 4 ++-- src/main/xml-parsers.js | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/examples/list-objects.js b/examples/list-objects.js index aa46f5b0..57c17e1b 100644 --- a/examples/list-objects.js +++ b/examples/list-objects.js @@ -50,8 +50,9 @@ function listPrefixesOfABucket(buckName) { var objectsStream=s3Client.listObjects(buckName, '', false , {}) var counter = 0 objectsStream.on('data', function (obj) { - console.dir(obj) - counter+=1 + if(obj.prefix) { + counter += 1 + } }) objectsStream.on('end',()=>{ console.log("Non Versioned Prefix Count:", counter) diff --git a/src/main/minio.js b/src/main/minio.js index c9ad9d89..3eb80917 100644 --- a/src/main/minio.js +++ b/src/main/minio.js @@ -1205,10 +1205,10 @@ export class Client { } if (!isString(Delimiter)) { - throw new TypeError('delimiter should be of type "string"') + throw new TypeError('Delimiter should be of type "string"') } if (!isNumber(MaxKeys)) { - throw new TypeError('maxKeys should be of type "number"') + throw new TypeError('MaxKeys should be of type "number"') } const queries = [] diff --git a/src/main/xml-parsers.js b/src/main/xml-parsers.js index f75ffad3..16c4064d 100644 --- a/src/main/xml-parsers.js +++ b/src/main/xml-parsers.js @@ -372,8 +372,8 @@ export function parseListObjects(xml) { parseCommonPrefixesEntity(listVersionsResult.CommonPrefixes) } + result.isTruncated= isTruncated if (isTruncated) { - result.isTruncated= isTruncated result.nextMarker = nextVersionKeyMarker || nextMarker } return result From ba33af5527d484884764a180e6bcaf05a5d3e51f Mon Sep 17 00:00:00 2001 From: prakashsvmx Date: Mon, 1 Mar 2021 10:30:21 +0530 Subject: [PATCH 7/7] Add delete marker flag --- docs/API.md | 1 + src/main/minio.js | 1 + src/main/xml-parsers.js | 13 +++++++++---- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/docs/API.md b/docs/API.md index a4da8812..424ebc4c 100644 --- a/docs/API.md +++ b/docs/API.md @@ -251,6 +251,7 @@ The object is of the format: | `obj.size` | _number_ | size of the object. | | `obj.etag` | _string_ | etag of the object. | | `obj.versionId` | _string_ | versionId of the object. | +| `obj.isDeleteMarker` | _boolean_ | true if it is a delete marker. | | `obj.lastModified` | _Date_ | modified time stamp. | __Example__ diff --git a/src/main/minio.js b/src/main/minio.js index 3eb80917..85027262 100644 --- a/src/main/minio.js +++ b/src/main/minio.js @@ -1267,6 +1267,7 @@ export class Client { // * `obj.size` _number_: size of the object // * `obj.etag` _string_: etag of the object // * `obj.lastModified` _Date_: modified time stamp + // * `obj.isDeleteMarker` _boolean_: true if it is a delete marker // * `obj.versionId` _string_: versionId of the object listObjects(bucketName, prefix, recursive, listOpts={}) { if (prefix === undefined) prefix = '' diff --git a/src/main/xml-parsers.js b/src/main/xml-parsers.js index 16c4064d..0cdfd0ef 100644 --- a/src/main/xml-parsers.js +++ b/src/main/xml-parsers.js @@ -17,7 +17,7 @@ import fxp from 'fast-xml-parser' import _ from 'lodash' import * as errors from './errors.js' -import { sanitizeETag } from "./helpers" +import { sanitizeETag, isObject } from "./helpers" var parseXml = (xml) => { var result = null @@ -280,7 +280,7 @@ export function parseCompleteMultipart(xml) { } } -const formatObjInfo = content => { +const formatObjInfo = (content, opts={}) => { let { Key, @@ -290,6 +290,10 @@ const formatObjInfo = content => { VersionId, IsLatest } = content + + if(!isObject(opts)){ + opts = {} + } const name = toArray(Key)[0] const lastModified = new Date(toArray(LastModified)[0]) @@ -301,7 +305,8 @@ const formatObjInfo = content => { etag, size:Size, versionId:VersionId, - isLatest:IsLatest + isLatest:IsLatest, + isDeleteMarker:opts.IsDeleteMarker ? opts.IsDeleteMarker: false } } @@ -359,7 +364,7 @@ export function parseListObjects(xml) { } if (listVersionsResult.DeleteMarker) { toArray(listVersionsResult.DeleteMarker).forEach(content => { - result.objects.push(formatObjInfo(content)) + result.objects.push(formatObjInfo(content, {IsDeleteMarker:true})) }) }