From d22ae69d7fdd1e73f2eaeee99e9b62106e5e4e55 Mon Sep 17 00:00:00 2001 From: DenisaCG Date: Mon, 20 Jan 2025 19:31:05 +0100 Subject: [PATCH 1/6] update directory check in all functionalities --- src/s3.ts | 36 +++++++++++++++++++++++------------- src/s3contents.ts | 8 +++++--- 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/src/s3.ts b/src/s3.ts index 0254e22..64fde13 100644 --- a/src/s3.ts +++ b/src/s3.ts @@ -106,14 +106,16 @@ export const listS3Contents = async ( c.Key !== path + '/' && c.Key !== root + '/' + path + '/' ) { - const fileName = c - .Key!.replace( - (root ? root + '/' : '') + (path ? path + '/' : ''), - '' - ) - .split('/')[0]; + let fileName = c.Key!.replace( + (root ? root + '/' : '') + (path ? path + '/' : ''), + '' + ); + const isDir: boolean = + fileName === fileName.split('/')[0] ? false : true; + fileName = fileName.split('/')[0]; const [fileType, fileMimeType, fileFormat] = Private.getFileType( PathExt.extname(PathExt.basename(fileName)), + isDir, registeredFileTypes ); @@ -184,6 +186,7 @@ export const getS3FileContents = async ( const date: string = response.LastModified!.toISOString(); const [fileType, fileMimeType, fileFormat] = Private.getFileType( PathExt.extname(PathExt.basename(path)), + false, registeredFileTypes ); @@ -243,12 +246,14 @@ export const createS3Object = async ( path: string, body: string | Blob, registeredFileTypes: IRegisteredFileTypes, + isDir: boolean, options?: Partial ): Promise => { path = PathExt.join(root, path); const [fileType, fileMimeType, fileFormat] = Private.getFileType( PathExt.extname(PathExt.basename(name)), + isDir, registeredFileTypes ); @@ -403,7 +408,6 @@ export const renameS3Objects = async ( oldLocalPath = PathExt.join(root, oldLocalPath); const isDir: boolean = await isDirectory(s3Client, bucketName, oldLocalPath); - if (isDir) { newLocalPath = newLocalPath.substring(0, newLocalPath.length - 1); } @@ -412,6 +416,7 @@ export const renameS3Objects = async ( const [fileType, fileMimeType, fileFormat] = Private.getFileType( PathExt.extname(PathExt.basename(newFileName)), + isDir, registeredFileTypes ); @@ -560,6 +565,7 @@ export const copyS3Objects = async ( const [fileType, fileMimeType, fileFormat] = Private.getFileType( PathExt.extname(PathExt.basename(name)), + isDir, registeredFileTypes ); @@ -669,18 +675,22 @@ export async function isDirectory( namespace Private { /** * Helping function to define file type, mimetype and format based on file extension. - * @param extension file extension (e.g.: txt, ipynb, csv) - * @returns + * @param extension: File extension (e.g.: txt, ipynb, csv) + * @param isDir: Boolean showing if the object is a directory or a file + * @param registeredFileTypes: The list containing all registered file types. + * @returns The object type, mimetype and format. */ export function getFileType( extension: string, + isDir: boolean, registeredFileTypes: IRegisteredFileTypes ) { - let fileType: string = 'text'; - let fileMimetype: string = 'text/plain'; - let fileFormat: string = 'text'; + let fileType: string = isDir === false ? 'text' : 'directory'; + let fileMimetype: string = + isDir === false ? 'text/plain' : 'text/directory'; + let fileFormat: string = isDir === false ? 'text' : 'json'; - if (registeredFileTypes[extension]) { + if (isDir === false && registeredFileTypes[extension]) { fileType = registeredFileTypes[extension].fileType; fileMimetype = registeredFileTypes[extension].fileMimeTypes[0]; fileFormat = registeredFileTypes[extension].fileFormat; diff --git a/src/s3contents.ts b/src/s3contents.ts index 7eba626..4c3861a 100644 --- a/src/s3contents.ts +++ b/src/s3contents.ts @@ -287,10 +287,10 @@ export class Drive implements Contents.IDrive { this._registeredFileTypes ); } else { - const currentPath = PathExt.basename(path); + const isDir = await isDirectory(this._s3Client, this._name, path); // listing contents of a folder - if (PathExt.extname(currentPath) === '') { + if (isDir === true) { data = await listS3Contents( this._s3Client, this._name, @@ -345,7 +345,8 @@ export class Drive implements Contents.IDrive { name, options.path ? PathExt.join(options.path, name) : name, '', // create new file with empty body, - this.registeredFileTypes + this.registeredFileTypes, + options.type === 'directory' ? true : false ); } else { console.warn('Type of new element is undefined'); @@ -549,6 +550,7 @@ export class Drive implements Contents.IDrive { localPath, options.content, this._registeredFileTypes, + true, options ); From 7e9f942af565b45404de48e2bc022a73de5b0833 Mon Sep 17 00:00:00 2001 From: DenisaCG Date: Mon, 20 Jan 2025 23:09:02 +0100 Subject: [PATCH 2/6] refactor get functionality --- src/s3.ts | 37 +++++++++++++++++++++++++------------ src/s3contents.ts | 31 ++++++++----------------------- 2 files changed, 33 insertions(+), 35 deletions(-) diff --git a/src/s3.ts b/src/s3.ts index 64fde13..a83da95 100644 --- a/src/s3.ts +++ b/src/s3.ts @@ -83,6 +83,7 @@ export const listS3Contents = async ( registeredFileTypes: IRegisteredFileTypes, path?: string ): Promise => { + let isFile: boolean = false; const fileList: IContentsList = {}; const prefix = path ? PathExt.join(root, path) : root; @@ -133,25 +134,37 @@ export const listS3Contents = async ( }; } }); + } else { + isFile = true; + data = await getS3FileContents( + s3Client, + bucketName, + root, + path!, + registeredFileTypes + ); } + if (isTruncated) { isTruncated = IsTruncated; } command.input.ContinuationToken = NextContinuationToken; } - data = { - name: path ? PathExt.basename(path) : bucketName, - path: path ? path + '/' : bucketName, - last_modified: '', - created: '', - content: Object.values(fileList), - format: 'json', - mimetype: '', - size: undefined, - writable: true, - type: 'directory' - }; + if (isFile === false) { + data = { + name: path ? PathExt.basename(path) : bucketName, + path: path ? path + '/' : bucketName, + last_modified: '', + created: '', + content: Object.values(fileList), + format: 'json', + mimetype: '', + size: undefined, + writable: true, + type: 'directory' + }; + } return data; }; diff --git a/src/s3contents.ts b/src/s3contents.ts index 4c3861a..adfd43d 100644 --- a/src/s3contents.ts +++ b/src/s3contents.ts @@ -19,7 +19,6 @@ import { renameS3Objects, listS3Contents, IRegisteredFileTypes, - getS3FileContents, isDirectory } from './s3'; @@ -287,28 +286,14 @@ export class Drive implements Contents.IDrive { this._registeredFileTypes ); } else { - const isDir = await isDirectory(this._s3Client, this._name, path); - - // listing contents of a folder - if (isDir === true) { - data = await listS3Contents( - this._s3Client, - this._name, - this.root, - this.registeredFileTypes, - path - ); - } - // getting the contents of a specific file - else { - data = await getS3FileContents( - this._s3Client, - this._name, - this._root, - path, - this.registeredFileTypes - ); - } + // listing the contents of a directory or retriving the contents of a file + data = await listS3Contents( + this._s3Client, + this._name, + this.root, + this.registeredFileTypes, + path + ); } Contents.validateContentsModel(data); From 34a5ce346c7cee91cc8d6410f0ac5f190928dd10 Mon Sep 17 00:00:00 2001 From: DenisaCG Date: Mon, 20 Jan 2025 23:13:05 +0100 Subject: [PATCH 3/6] iterate on get functionality --- src/s3contents.ts | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/src/s3contents.ts b/src/s3contents.ts index adfd43d..ebb1c1f 100644 --- a/src/s3contents.ts +++ b/src/s3contents.ts @@ -277,24 +277,14 @@ export class Drive implements Contents.IDrive { this._isRootFormatted = true; } - // getting the list of files from the root - if (!path) { - data = await listS3Contents( - this._s3Client, - this._name, - this._root, - this._registeredFileTypes - ); - } else { - // listing the contents of a directory or retriving the contents of a file - data = await listS3Contents( - this._s3Client, - this._name, - this.root, - this.registeredFileTypes, - path - ); - } + // listing the contents of a directory or retriving the contents of a file + data = await listS3Contents( + this._s3Client, + this._name, + this.root, + this.registeredFileTypes, + path + ); Contents.validateContentsModel(data); return data; From 939a9e8e9833021e303f786681d1e3dcf5704afd Mon Sep 17 00:00:00 2001 From: DenisaCG Date: Tue, 21 Jan 2025 13:09:52 +0100 Subject: [PATCH 4/6] fix directory copy functionality --- src/s3.ts | 3 ++- src/s3contents.ts | 24 ++++++++++++++---------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/src/s3.ts b/src/s3.ts index a83da95..889616b 100644 --- a/src/s3.ts +++ b/src/s3.ts @@ -674,7 +674,8 @@ export async function isDirectory( // listing contents given a path, to check if it is a directory const command = new ListObjectsV2Command({ Bucket: bucketName, - Prefix: objectPath + '/' + Prefix: + objectPath[objectPath.length - 1] === '/' ? objectPath : objectPath + '/' }); const { Contents } = await s3Client.send(command); diff --git a/src/s3contents.ts b/src/s3contents.ts index ebb1c1f..29b792c 100644 --- a/src/s3contents.ts +++ b/src/s3contents.ts @@ -423,10 +423,15 @@ export class Drive implements Contents.IDrive { options: Contents.ICreateOptions = {} ): Promise { let newFileName = PathExt.basename(newLocalPath); + const isDir: boolean = await isDirectory( + this._s3Client, + this._name, + oldLocalPath + ); try { await checkS3Object(this._s3Client, this._name, this._root, newLocalPath); - newFileName = await this.incrementName(newLocalPath, this._name); + newFileName = await this.incrementName(newLocalPath, this._name, isDir); } catch (error) { // HEAD request failed for this file name, continue, as name doesn't already exist. } finally { @@ -457,19 +462,14 @@ export class Drive implements Contents.IDrive { * * @param bucketName - The name of the bucket where content is moved. * - * @param root - The root of the bucket, if it exists. + * @param isDir - Whether the object is a directory or a file. */ - async incrementName(localPath: string, bucketName: string) { - const isDir: boolean = await isDirectory( - this._s3Client, - bucketName, - localPath - ); + async incrementName(localPath: string, bucketName: string, isDir: boolean) { let fileExtension: string = ''; let originalName: string = ''; // check if we are dealing with a directory - if (isDir) { + if (isDir === true) { localPath = localPath.substring(0, localPath.length - 1); originalName = PathExt.basename(localPath); } @@ -571,7 +571,11 @@ export class Drive implements Contents.IDrive { (isDir ? '/' : ''); // getting incremented name of Copy in case of duplicates - const incrementedName = await this.incrementName(newFilePath, bucketName); + const incrementedName = await this.incrementName( + newFilePath, + bucketName, + isDir + ); return incrementedName; } From bb14e95617851e246e0f7f2ce3cbed852def6d66 Mon Sep 17 00:00:00 2001 From: DenisaCG Date: Tue, 21 Jan 2025 13:12:21 +0100 Subject: [PATCH 5/6] refactor rename functionality to add isDir parameter --- src/s3.ts | 3 ++- src/s3contents.ts | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/s3.ts b/src/s3.ts index 889616b..3664b20 100644 --- a/src/s3.ts +++ b/src/s3.ts @@ -404,6 +404,7 @@ export const checkS3Object = async ( * @param oldLocalPath: The old path of the object. * @param newLocalPath: The new path of the object. * @param newFileName: The new object name. + * @param isDir: Whether the object is a directory or a file. * @param registeredFileTypes: The list containing all registered file types. * * @returns A promise which resolves with the new object contents model. @@ -415,12 +416,12 @@ export const renameS3Objects = async ( oldLocalPath: string, newLocalPath: string, newFileName: string, + isDir: boolean, registeredFileTypes: IRegisteredFileTypes ): Promise => { newLocalPath = PathExt.join(root, newLocalPath); oldLocalPath = PathExt.join(root, oldLocalPath); - const isDir: boolean = await isDirectory(s3Client, bucketName, oldLocalPath); if (isDir) { newLocalPath = newLocalPath.substring(0, newLocalPath.length - 1); } diff --git a/src/s3contents.ts b/src/s3contents.ts index 29b792c..1f21813 100644 --- a/src/s3contents.ts +++ b/src/s3contents.ts @@ -442,6 +442,7 @@ export class Drive implements Contents.IDrive { oldLocalPath, newLocalPath, newFileName, + isDir, this._registeredFileTypes ); } From ef702d04ce792fc88be07fb9e44d12bb3fd606c1 Mon Sep 17 00:00:00 2001 From: DenisaCG Date: Tue, 21 Jan 2025 13:54:17 +0100 Subject: [PATCH 6/6] fix directory copy functionality information retrieval --- src/s3.ts | 53 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/src/s3.ts b/src/s3.ts index 3664b20..14de05b 100644 --- a/src/s3.ts +++ b/src/s3.ts @@ -583,26 +583,41 @@ export const copyS3Objects = async ( registeredFileTypes ); - // retrieve information of new file - const newFileContents = await s3Client.send( - new GetObjectCommand({ - Bucket: newBucketName ?? bucketName, - Key: name + (suffix ? suffix : '') - }) - ); + try { + const newFileContents = await s3Client.send( + new GetObjectCommand({ + Bucket: newBucketName ?? bucketName, + Key: name + (suffix ? suffix : '') + }) + ); - data = { - name: PathExt.basename(name), - path: name, - last_modified: newFileContents.LastModified!.toISOString(), - created: new Date().toISOString(), - content: await newFileContents.Body!.transformToString(), - format: fileFormat as Contents.FileFormat, - mimetype: fileMimeType, - size: newFileContents.ContentLength!, - writable: true, - type: fileType - }; + data = { + name: PathExt.basename(name), + path: name, + last_modified: newFileContents.LastModified!.toISOString(), + created: new Date().toISOString(), + content: await newFileContents.Body!.transformToString(), + format: fileFormat as Contents.FileFormat, + mimetype: fileMimeType, + size: newFileContents.ContentLength!, + writable: true, + type: fileType + }; + } catch { + // object directory itself doesn't exist + data = { + name: PathExt.basename(name), + path: name, + last_modified: new Date().toISOString(), + created: new Date().toISOString(), + content: [], + format: fileFormat as Contents.FileFormat, + mimetype: fileMimeType, + size: 0, + writable: true, + type: fileType + }; + } return data; };