From f69303d5641e5549d22d2937a6904b2de935f751 Mon Sep 17 00:00:00 2001 From: prakashsvmx Date: Wed, 17 Feb 2021 13:16:48 +0530 Subject: [PATCH] Update functional tests --- docs/API.md | 26 ++++++- examples/list-objects.js | 14 +++- src/main/helpers.js | 6 ++ src/main/minio.js | 60 ++++++++++++---- src/main/xml-parsers.js | 89 ++++++++++++++++++------ src/test/functional/functional-tests.js | 91 +++++++++++++++++++++++++ 6 files changed, 247 insertions(+), 39 deletions(-) diff --git a/docs/API.md b/docs/API.md index e49aa6716..113d3caff 100644 --- a/docs/API.md +++ b/docs/API.md @@ -221,7 +221,7 @@ minioClient.removeBucket('mybucket', function(err) { ``` -### listObjects(bucketName, prefix, recursive) +### listObjects(bucketName, prefix, recursive [,listOpts]) Lists all objects in a bucket. @@ -233,7 +233,7 @@ __Parameters__ | `bucketName` | _string_ | Name of the bucket. | | `prefix` | _string_ | The prefix of the objects that should be listed (optional, default `''`). | | `recursive` | _bool_ | `true` indicates recursive style listing and `false` indicates directory style listing delimited by '/'. (optional, default `false`). | - +| `listOpts` | _object_ | query params to list object which can have `{MaxKeys: _int_ , IncludeVersion: _bool_ }` (optional)| __Return Value__ @@ -249,7 +249,8 @@ The object is of the format: | `obj.name` | _string_ | name of the object. | | `obj.prefix` | _string_ | name of the object prefix. | | `obj.size` | _number_ | size of the object. | -| `obj.etag` | _string_ |etag of the object. | +| `obj.etag` | _string_ | etag of the object. | +| `obj.versionId` | _string_ | versionId of the object. | | `obj.lastModified` | _Date_ | modified time stamp. | __Example__ @@ -261,6 +262,25 @@ stream.on('data', function(obj) { console.log(obj) } ) stream.on('error', function(err) { console.log(err) } ) ``` +__Example1__ +To get Object versions + +```js +var stream = minioClient.listObjects('mybucket','', true, {IncludeVersion:true}) +stream.on('data', function(obj) { console.log(obj) } ) +stream.on('error', function(err) { console.log(err) } ) +``` + +__Example2__ +To get Object versions along with MaxKeys + +```js +var stream = minioClient.listObjects('mybucket','', true, {IncludeVersion:true, MaxKeys:3}) +stream.on('data', function(obj) { console.log(obj) } ) +stream.on('error', function(err) { console.log(err) } ) +``` + + ### listObjectsV2(bucketName, prefix, recursive, startAfter) diff --git a/examples/list-objects.js b/examples/list-objects.js index 72b22db8d..a4d8b449d 100644 --- a/examples/list-objects.js +++ b/examples/list-objects.js @@ -14,8 +14,8 @@ * limitations under the License. */ - // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are - // dummy values, please replace them with original values. +// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are +// dummy values, please replace them with original values. var Minio = require('minio') @@ -32,3 +32,13 @@ objectsStream.on('data', function(obj) { objectsStream.on('error', function(e) { console.log(e) }) + + +// List all object versions in bucket my-bucketname. +var objectsStreamWithVersions = s3Client.listObjects('my-bucketname', '', true, {IncludeVersion:true}) +objectsStreamWithVersions.on('data', function(obj) { + console.log(obj) +}) +objectsStreamWithVersions.on('error', function(e) { + console.log(e) +}) diff --git a/src/main/helpers.js b/src/main/helpers.js index 61b48e677..ab7bcc6fc 100644 --- a/src/main/helpers.js +++ b/src/main/helpers.js @@ -362,4 +362,10 @@ export function extractMetadata(metaData) { export function getVersionId(headers={}){ const versionIdValue = headers["x-amz-version-id"] return versionIdValue || null +} + +export function sanitizeETag(etag='') { + const replaceChars = {'"':'','"':'','"':'','"':'','"':''} + return etag.replace(/^("|"|")|("|"|")$/g, m => replaceChars[m]) + } \ No newline at end of file diff --git a/src/main/minio.js b/src/main/minio.js index 9575ee605..e21f4a0f9 100644 --- a/src/main/minio.js +++ b/src/main/minio.js @@ -1159,7 +1159,7 @@ export class Client { } // list a batch of objects - listObjectsQuery(bucketName, prefix, marker, delimiter, maxKeys) { + listObjectsQuery(bucketName, prefix, marker, listQueryOpts={}) { if (!isValidBucketName(bucketName)) { throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName) } @@ -1169,29 +1169,48 @@ export class Client { if (!isString(marker)) { throw new TypeError('marker should be of type "string"') } - if (!isString(delimiter)) { + let{ + Delimiter, + MaxKeys, + IncludeVersion, + } = listQueryOpts + + if (!isObject(listQueryOpts)) { + throw new TypeError('listQueryOpts should be of type "object"') + } + + + if (!isString(Delimiter)) { throw new TypeError('delimiter should be of type "string"') } - if (!isNumber(maxKeys)) { + if (!isNumber(MaxKeys)) { throw new TypeError('maxKeys should be of type "number"') } - var queries = [] + const queries = [] // escape every value in query string, except maxKeys queries.push(`prefix=${uriEscape(prefix)}`) - queries.push(`delimiter=${uriEscape(delimiter)}`) + queries.push(`delimiter=${uriEscape(Delimiter)}`) + + if (IncludeVersion) { + queries.push(`versions`) + } if (marker) { marker = uriEscape(marker) - queries.push(`marker=${marker}`) + if (IncludeVersion) { + queries.push(`key-marker=${marker}`) + } else { + queries.push(`marker=${marker}`) + } } // no need to escape maxKeys - if (maxKeys) { - if (maxKeys >= 1000) { - maxKeys = 1000 + if (MaxKeys) { + if (MaxKeys >= 1000) { + MaxKeys = 1000 } - queries.push(`max-keys=${maxKeys}`) + queries.push(`max-keys=${MaxKeys}`) } queries.sort() var query = '' @@ -1214,7 +1233,9 @@ export class Client { // * `bucketName` _string_: name of the bucket // * `prefix` _string_: the prefix of the objects that should be listed (optional, default `''`) // * `recursive` _bool_: `true` indicates recursive style listing and `false` indicates directory style listing delimited by '/'. (optional, default `false`) - // + // * `listOpts _object_: query params to list object with below keys + // * listOpts.MaxKeys _int_ maximum number of keys to return + // * listOpts.IncludeVersion _bool_ true|false to include versions. // __Return Value__ // * `stream` _Stream_: stream emitting the objects in the bucket, the object is of the format: // * `obj.name` _string_: name of the object @@ -1222,7 +1243,8 @@ export class Client { // * `obj.size` _number_: size of the object // * `obj.etag` _string_: etag of the object // * `obj.lastModified` _Date_: modified time stamp - listObjects(bucketName, prefix, recursive) { + // * `obj.versionId` _string_: versionId of the object + listObjects(bucketName, prefix, recursive, listOpts={}) { if (prefix === undefined) prefix = '' if (recursive === undefined) recursive = false if (!isValidBucketName(bucketName)) { @@ -1237,9 +1259,21 @@ export class Client { if (!isBoolean(recursive)) { throw new TypeError('recursive should be of type "boolean"') } + if (!isObject(listOpts)) { + throw new TypeError('listOpts should be of type "object"') + } // if recursive is false set delimiter to '/' var delimiter = recursive ? '' : '/' + var marker = '' + const maxKeys = listOpts.MaxKeys || 1000 + const includeVersion=listOpts.IncludeVersion + + const listQueryOpts={ + Delimiter:delimiter, + MaxKeys:maxKeys, + IncludeVersion:includeVersion, + } var objects = [] var ended = false var readStream = Stream.Readable({objectMode: true}) @@ -1251,7 +1285,7 @@ export class Client { } if (ended) return readStream.push(null) // if there are no objects to push do query for the next batch of objects - this.listObjectsQuery(bucketName, prefix, marker, delimiter, 1000) + this.listObjectsQuery(bucketName, prefix, marker, listQueryOpts) .on('error', e => readStream.emit('error', e)) .on('data', result => { if (result.isTruncated) { diff --git a/src/main/xml-parsers.js b/src/main/xml-parsers.js index 4ec5ccdf2..22345fd83 100644 --- a/src/main/xml-parsers.js +++ b/src/main/xml-parsers.js @@ -17,6 +17,7 @@ import fxp from 'fast-xml-parser' import _ from 'lodash' import * as errors from './errors.js' +import { sanitizeETag } from "./helpers" var parseXml = (xml) => { var result = null @@ -279,33 +280,83 @@ export function parseCompleteMultipart(xml) { } } +const formatObjInfo = content => { + + let { + Key, + LastModified, + ETag, + Size, + VersionId, + IsLatest + } = content + + const name = toArray(Key)[0] + const lastModified = new Date(toArray(LastModified)[0]) + const etag = sanitizeETag(toArray(ETag)[0]) + + return { + name, + lastModified, + etag, + size:Size, + versionId:VersionId, + isLatest:IsLatest + } +} + // parse XML response for list objects in a bucket export function parseListObjects(xml) { var result = { objects: [], isTruncated: false } - var nextMarker + var nextMarker, nextVersionKeyMarker var xmlobj = parseXml(xml) - if (!xmlobj.ListBucketResult) { - throw new errors.InvalidXMLError('Missing tag: "ListBucketResult"') + const listBucketResult = xmlobj.ListBucketResult + const listVersionsResult=xmlobj.ListVersionsResult + + if (listBucketResult && listBucketResult.IsTruncated) { + result.isTruncated = listBucketResult.IsTruncated } - xmlobj = xmlobj.ListBucketResult - if (xmlobj.IsTruncated) result.isTruncated = xmlobj.IsTruncated - if (xmlobj.Contents) { - toArray(xmlobj.Contents).forEach(content => { - var name = toArray(content.Key)[0] - var lastModified = new Date(toArray(content.LastModified)[0]) - var etag = toArray(content.ETag)[0].replace(/^"/g, '').replace(/"$/g, '') - .replace(/^"/g, '').replace(/"$/g, '') - .replace(/^"/g, '').replace(/"$/g, '') - var size = content.Size + if (listVersionsResult && listVersionsResult.IsTruncated) { + result.isTruncated = listVersionsResult.IsTruncated + } + + if (listBucketResult && listBucketResult.Contents) { + toArray(listBucketResult.Contents).forEach(content => { + const name = toArray(content.Key)[0] + const lastModified = new Date(toArray(content.LastModified)[0]) + const etag = sanitizeETag(toArray(content.ETag)[0]) + const size = content.Size result.objects.push({name, lastModified, etag, size}) - nextMarker = name + if (!nextMarker) { + nextMarker = name + } }) } + if (listVersionsResult && listVersionsResult.Version) { + toArray(listVersionsResult.Version).forEach(content => { + result.objects.push(formatObjInfo(content)) + }) + } + + if (listVersionsResult && listVersionsResult.DeleteMarker) { + toArray(listVersionsResult.DeleteMarker).forEach(content => { + result.objects.push(formatObjInfo(content)) + }) + } + + if (listVersionsResult && listVersionsResult.NextKeyMarker) { + nextVersionKeyMarker = listVersionsResult.NextKeyMarker + } + if (listVersionsResult && listVersionsResult.NextVersionIdMarker) { + result.versionIdMarker = listVersionsResult.NextVersionIdMarker + } + + if (xmlobj.CommonPrefixes) { toArray(xmlobj.CommonPrefixes).forEach(commonPrefix => { var prefix = toArray(commonPrefix.Prefix)[0] @@ -314,7 +365,7 @@ export function parseListObjects(xml) { }) } if (result.isTruncated) { - result.nextMarker = xmlobj.NextMarker ? toArray(xmlobj.NextMarker)[0]: nextMarker + result.nextMarker = nextVersionKeyMarker || nextMarker } return result } @@ -336,9 +387,7 @@ export function parseListObjectsV2(xml) { toArray(xmlobj.Contents).forEach(content => { var name = content.Key var lastModified = new Date(content.LastModified) - var etag = content.ETag.replace(/^"/g, '').replace(/"$/g, '') - .replace(/^"/g, '').replace(/"$/g, '') - .replace(/^"/g, '').replace(/"$/g, '') + var etag = sanitizeETag(content.ETag) var size = content.Size result.objects.push({name, lastModified, etag, size}) }) @@ -371,9 +420,7 @@ export function parseListObjectsV2WithMetadata(xml) { toArray(xmlobj.Contents).forEach(content => { var name = content.Key var lastModified = new Date(content.LastModified) - var etag = content.ETag.replace(/^"/g, '').replace(/"$/g, '') - .replace(/^"/g, '').replace(/"$/g, '') - .replace(/^"/g, '').replace(/"$/g, '') + var etag = sanitizeETag(content.ETag) var size = content.Size var metadata if (content.UserMetadata != null) { diff --git a/src/test/functional/functional-tests.js b/src/test/functional/functional-tests.js index 58a097d73..b8287bf8e 100644 --- a/src/test/functional/functional-tests.js +++ b/src/test/functional/functional-tests.js @@ -1368,4 +1368,95 @@ describe('functional tests', function() { }) }) + + + describe('Versioning Supported listObjects', function() { + const versionedBucketName = "minio-js-test-version-list" + uuid.v4() + const prefixName = "Prefix1" + const versionedObjectName ="datafile-versioned-list-100-kB" + const objVersionIdCounter = [1,2,3,4,5]// This should track adding 5 versions of the same object. + let listObjectsNum = objVersionIdCounter.length + let objArray = [] + let listPrefixArray = [] + let isVersioningSupported=false + + const objNameWithPrefix = `${prefixName}/${versionedObjectName}` + + before((done) => client.makeBucket(versionedBucketName, '', ()=>{ + client.setBucketVersioning(versionedBucketName,{Status:"Enabled"},(err)=>{ + if (err && err.code === 'NotImplemented') return done() + if (err) return done(err) + isVersioningSupported = true + done() + }) + + })) + after((done) => client.removeBucket(versionedBucketName, done)) + + + step(`putObject(bucketName, objectName, stream, size, metaData, callback)_bucketName:${versionedBucketName}, stream:1b, size:1_Create ${listObjectsNum} objects`, done => { + if(isVersioningSupported) { + objVersionIdCounter.forEach((versionCounter, index)=>{ + client.putObject(versionedBucketName, objNameWithPrefix, readableStream(_1byte), _1byte.length, {}, (e,data)=>{ + objArray.push(data) + if(index+1 === objVersionIdCounter.length) + done() + }) + }) + }else { + done() + } + }) + + step(`listObjects(bucketName, prefix, recursive)_bucketName:${versionedBucketName}, prefix: ${prefixName}, recursive:true_`, done => { + if(isVersioningSupported) { + client.listObjects(versionedBucketName, '', true, {IncludeVersion: true}) + .on('error', done) + .on('end', () => { + if (_.isEqual(objArray.length, listPrefixArray.length)) return done() + return done(new Error(`listObjects lists ${listPrefixArray.length} objects, expected ${listObjectsNum}`)) + }) + .on('data', data => { + listPrefixArray.push(data) + }) + } else { + done() + } + }) + + step(`listObjects(bucketName, prefix, recursive, listOpts)_bucketName:${versionedBucketName}, prefix: ${prefixName}, recursive:true_ ,{IncludeVersion: true, MaxKeys:2}`, done => { + if(isVersioningSupported) { + const maxKeysList=[] + client.listObjects(versionedBucketName, '', true, {IncludeVersion: true, MaxKeys: 2}) + .on('error', done) + .on('end', () => { + if (maxKeysList.length === 2) return done() + return done(new Error(`listObjects lists ${maxKeysList.length} objects, expected ${2}`)) + }) + .on('data', data => { + maxKeysList.push(data) + }) + } else { + done() + } + }) + + + step(`removeObject(bucketName, objectName, removeOpts)_bucketName:${versionedBucketName}_Remove ${listObjectsNum} objects`, done => { + if(isVersioningSupported) { + listPrefixArray.forEach((item, index)=>{ + client.removeObject(versionedBucketName, objNameWithPrefix, {versionId: item.versionId}, (res)=>{ + if(!res) { + if (index+1 === listPrefixArray.length) { + done() + } + } + }) + }) + }else { + done() + } + }) + }) + })