Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor: completeMultipartUpload to TypeScript #1229

Merged
merged 10 commits into from
Nov 21, 2023
69 changes: 68 additions & 1 deletion src/internal/client.ts
Original file line number Diff line number Diff line change
@@ -65,7 +65,7 @@ import type {
} from './type.ts'
import type { ListMultipartResult, UploadedPart } from './xml-parser.ts'
import * as xmlParsers from './xml-parser.ts'
import { parseInitiateMultipart, parseObjectLegalHoldConfig } from './xml-parser.ts'
import { parseCompleteMultipart, parseInitiateMultipart, parseObjectLegalHoldConfig } from './xml-parser.ts'

const xml = new xml2js.Builder({ renderOpts: { pretty: false }, headless: true })

@@ -1139,6 +1139,73 @@ export class TypedClient {
await this.makeRequestAsyncOmit(requestOptions, '', [204])
}

/**
* this call will aggregate the parts on the server into a single object.
*/
async completeMultipartUpload(
bucketName: string,
objectName: string,
uploadId: string,
etags: {
part: number
etag?: string
}[],
): Promise<{ etag: string; versionId: string | null }> {
if (!isValidBucketName(bucketName)) {
throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName)
}
if (!isValidObjectName(objectName)) {
throw new errors.InvalidObjectNameError(`Invalid object name: ${objectName}`)
}
if (!isString(uploadId)) {
throw new TypeError('uploadId should be of type "string"')
}
if (!isObject(etags)) {
throw new TypeError('etags should be of type "Array"')
}

if (!uploadId) {
throw new errors.InvalidArgumentError('uploadId cannot be empty')
}

const method = 'POST'
const query = `uploadId=${uriEscape(uploadId)}`

const builder = new xml2js.Builder()
const payload = builder.buildObject({
CompleteMultipartUpload: {
$: {
xmlns: 'http://s3.amazonaws.com/doc/2006-03-01/',
},
Part: etags.map((etag) => {
return {
PartNumber: etag.part,
ETag: etag.etag,
}
}),
},
})

const res = await this.makeRequestAsync({ method, bucketName, objectName, query }, payload)
const body = await readAsBuffer(res)
const result = parseCompleteMultipart(body.toString())
if (!result) {
throw new Error('BUG: failed to parse server response')
}

if (result.errCode) {
// Multipart Complete API returns an error XML after a 200 http status
throw new errors.S3Error(result.errMessage)
}

return {
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
etag: result.etag as string,
versionId: getVersionId(res.headers as ResponseHeader),
}
}

/**
* Get part-info of all parts of an incomplete upload specified by uploadId.
*/
24 changes: 24 additions & 0 deletions src/internal/xml-parser.ts
Original file line number Diff line number Diff line change
@@ -269,6 +269,30 @@ export function parseTagging(xml: string) {
return result
}

// parse XML response when a multipart upload is completed
export function parseCompleteMultipart(xml: string) {
const xmlobj = parseXml(xml).CompleteMultipartUploadResult
if (xmlobj.Location) {
const location = toArray(xmlobj.Location)[0]
const bucket = toArray(xmlobj.Bucket)[0]
const key = xmlobj.Key
const etag = xmlobj.ETag.replace(/^"/g, '')
.replace(/"$/g, '')
.replace(/^&quot;/g, '')
.replace(/&quot;$/g, '')
.replace(/^&#34;/g, '')
.replace(/&#34;$/g, '')

return { location, bucket, key, etag }
}
// Complete Multipart can return XML Error after a 200 OK response
if (xmlobj.Code && xmlobj.Message) {
const errCode = toArray(xmlobj.Code)[0]
const errMessage = toArray(xmlobj.Message)[0]
return { errCode, errMessage }
}
}

type UploadID = unknown

export type ListMultipartResult = {
68 changes: 4 additions & 64 deletions src/minio.js
Original file line number Diff line number Diff line change
@@ -1216,69 +1216,6 @@ export class Client extends TypedClient {
})
}

// Complete the multipart upload. After all the parts are uploaded issuing
// this call will aggregate the parts on the server into a single object.
completeMultipartUpload(bucketName, objectName, uploadId, etags, cb) {
if (!isValidBucketName(bucketName)) {
throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName)
}
if (!isValidObjectName(objectName)) {
throw new errors.InvalidObjectNameError(`Invalid object name: ${objectName}`)
}
if (!isString(uploadId)) {
throw new TypeError('uploadId should be of type "string"')
}
if (!isObject(etags)) {
throw new TypeError('etags should be of type "Array"')
}
if (!isFunction(cb)) {
throw new TypeError('cb should be of type "function"')
}

if (!uploadId) {
throw new errors.InvalidArgumentError('uploadId cannot be empty')
}

var method = 'POST'
var query = `uploadId=${uriEscape(uploadId)}`

const builder = new xml2js.Builder()
const payload = builder.buildObject({
CompleteMultipartUpload: {
$: {
xmlns: 'http://s3.amazonaws.com/doc/2006-03-01/',
},
Part: etags.map((etag) => {
return {
PartNumber: etag.part,
ETag: etag.etag,
}
}),
},
})

this.makeRequest({ method, bucketName, objectName, query }, payload, [200], '', true, (e, response) => {
if (e) {
return cb(e)
}
var transformer = transformers.getCompleteMultipartTransformer()
pipesetup(response, transformer)
.on('error', (e) => cb(e))
.on('data', (result) => {
if (result.errCode) {
// Multipart Complete API returns an error XML after a 200 http status
cb(new errors.S3Error(result.errMessage))
} else {
const completeMultipartResult = {
etag: result.etag,
versionId: getVersionId(response.headers),
}
cb(null, completeMultipartResult)
}
})
})
}

// Find uploadId of an incomplete upload.
findUploadId(bucketName, objectName, cb) {
if (!isValidBucketName(bucketName)) {
@@ -2002,7 +1939,10 @@ export class Client extends TypedClient {
return
}
const partsDone = res.map((partCopy) => ({ etag: partCopy.etag, part: partCopy.part }))
return me.completeMultipartUpload(destObjConfig.Bucket, destObjConfig.Object, uploadId, partsDone, cb)
return me.completeMultipartUpload(destObjConfig.Bucket, destObjConfig.Object, uploadId, partsDone).then(
(result) => cb(null, result),
(err) => cb(err),
)
})
}

26 changes: 12 additions & 14 deletions src/object-uploader.js
Original file line number Diff line number Diff line change
@@ -77,8 +77,7 @@ export class ObjectUploader extends Transform {
if (this.partNumber == 1 && chunk.length < this.partSize) {
// PUT the chunk in a single request — use an empty query.
let options = {
method,
// Set user metadata as this is not a multipart upload
method, // Set user metadata as this is not a multipart upload
headers: Object.assign({}, this.metaData, headers),
query: '',
bucketName: this.bucketName,
@@ -267,19 +266,18 @@ export class ObjectUploader extends Transform {

// This is called when all of the chunks uploaded successfully, thus
// completing the multipart upload.
this.client.completeMultipartUpload(this.bucketName, this.objectName, this.id, this.etags, (err, etag) => {
if (err) {
return callback(err)
}

// Call our callback on the next tick to allow the streams infrastructure
// to finish what its doing before we continue.
process.nextTick(() => {
this.callback(null, etag)
})
this.client.completeMultipartUpload(this.bucketName, this.objectName, this.id, this.etags).then(
(etag) => {
// Call our callback on the next tick to allow the streams infrastructure
// to finish what its doing before we continue.
process.nextTick(() => {
this.callback(null, etag)
})

callback()
})
callback()
},
(err) => callback(err),
)
}
}

5 changes: 0 additions & 5 deletions src/transformers.js
Original file line number Diff line number Diff line change
@@ -116,11 +116,6 @@ export function getListObjectsV2WithMetadataTransformer() {
return getConcater(xmlParsers.parseListObjectsV2WithMetadata)
}

// Parses completeMultipartUpload response.
export function getCompleteMultipartTransformer() {
return getConcater(xmlParsers.parseCompleteMultipart)
}

// Parses GET/SET BucketNotification response
export function getBucketNotificationTransformer() {
return getConcater(xmlParsers.parseBucketNotification)
24 changes: 0 additions & 24 deletions src/xml-parsers.js
Original file line number Diff line number Diff line change
@@ -135,30 +135,6 @@ export function parseBucketNotification(xml) {
return result
}

// parse XML response when a multipart upload is completed
export function parseCompleteMultipart(xml) {
var xmlobj = parseXml(xml).CompleteMultipartUploadResult
if (xmlobj.Location) {
var location = toArray(xmlobj.Location)[0]
var bucket = toArray(xmlobj.Bucket)[0]
var key = xmlobj.Key
var etag = xmlobj.ETag.replace(/^"/g, '')
.replace(/"$/g, '')
.replace(/^&quot;/g, '')
.replace(/&quot;$/g, '')
.replace(/^&#34;/g, '')
.replace(/&#34;$/g, '')

return { location, bucket, key, etag }
}
// Complete Multipart can return XML Error after a 200 OK response
if (xmlobj.Code && xmlobj.Message) {
var errCode = toArray(xmlobj.Code)[0]
var errMessage = toArray(xmlobj.Message)[0]
return { errCode, errMessage }
}
}

const formatObjInfo = (content, opts = {}) => {
let { Key, LastModified, ETag, Size, VersionId, IsLatest } = content