Skip to content

Commit

Permalink
Merge pull request #35 from QuantStack/fileType
Browse files Browse the repository at this point in the history 
Update `getFileType` function
  • Loading branch information
@DenisaCG
DenisaCG authored Jan 22, 2025
2 parents c4dee63 + ef702d0 commit 205ab5e
Show file tree
Hide file tree
Showing 2 changed files with 112 additions and 90 deletions.
132 changes: 86 additions & 46 deletions src/s3.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ export const listS3Contents = async (
registeredFileTypes: IRegisteredFileTypes,
path?: string
): Promise<Contents.IModel> => {
let isFile: boolean = false;
const fileList: IContentsList = {};
const prefix = path ? PathExt.join(root, path) : root;

Expand All @@ -106,14 +107,16 @@ export const listS3Contents = async (
c.Key !== path + '/' &&
c.Key !== root + '/' + path + '/'
) {
const fileName = c
.Key!.replace(
(root ? root + '/' : '') + (path ? path + '/' : ''),
''
)
.split('/')[0];
let fileName = c.Key!.replace(
(root ? root + '/' : '') + (path ? path + '/' : ''),
''
);
const isDir: boolean =
fileName === fileName.split('/')[0] ? false : true;
fileName = fileName.split('/')[0];
const [fileType, fileMimeType, fileFormat] = Private.getFileType(
PathExt.extname(PathExt.basename(fileName)),
isDir,
registeredFileTypes
);

Expand All @@ -131,25 +134,37 @@ export const listS3Contents = async (
};
}
});
} else {
isFile = true;
data = await getS3FileContents(
s3Client,
bucketName,
root,
path!,
registeredFileTypes
);
}

if (isTruncated) {
isTruncated = IsTruncated;
}
command.input.ContinuationToken = NextContinuationToken;
}

data = {
name: path ? PathExt.basename(path) : bucketName,
path: path ? path + '/' : bucketName,
last_modified: '',
created: '',
content: Object.values(fileList),
format: 'json',
mimetype: '',
size: undefined,
writable: true,
type: 'directory'
};
if (isFile === false) {
data = {
name: path ? PathExt.basename(path) : bucketName,
path: path ? path + '/' : bucketName,
last_modified: '',
created: '',
content: Object.values(fileList),
format: 'json',
mimetype: '',
size: undefined,
writable: true,
type: 'directory'
};
}

return data;
};
Expand Down Expand Up @@ -184,6 +199,7 @@ export const getS3FileContents = async (
const date: string = response.LastModified!.toISOString();
const [fileType, fileMimeType, fileFormat] = Private.getFileType(
PathExt.extname(PathExt.basename(path)),
false,
registeredFileTypes
);

Expand Down Expand Up @@ -243,12 +259,14 @@ export const createS3Object = async (
path: string,
body: string | Blob,
registeredFileTypes: IRegisteredFileTypes,
isDir: boolean,
options?: Partial<Contents.IModel>
): Promise<Contents.IModel> => {
path = PathExt.join(root, path);

const [fileType, fileMimeType, fileFormat] = Private.getFileType(
PathExt.extname(PathExt.basename(name)),
isDir,
registeredFileTypes
);

Expand Down Expand Up @@ -386,6 +404,7 @@ export const checkS3Object = async (
* @param oldLocalPath: The old path of the object.
* @param newLocalPath: The new path of the object.
* @param newFileName: The new object name.
* @param isDir: Whether the object is a directory or a file.
* @param registeredFileTypes: The list containing all registered file types.
*
* @returns A promise which resolves with the new object contents model.
Expand All @@ -397,13 +416,12 @@ export const renameS3Objects = async (
oldLocalPath: string,
newLocalPath: string,
newFileName: string,
isDir: boolean,
registeredFileTypes: IRegisteredFileTypes
): Promise<Contents.IModel> => {
newLocalPath = PathExt.join(root, newLocalPath);
oldLocalPath = PathExt.join(root, oldLocalPath);

const isDir: boolean = await isDirectory(s3Client, bucketName, oldLocalPath);

if (isDir) {
newLocalPath = newLocalPath.substring(0, newLocalPath.length - 1);
}
Expand All @@ -412,6 +430,7 @@ export const renameS3Objects = async (

const [fileType, fileMimeType, fileFormat] = Private.getFileType(
PathExt.extname(PathExt.basename(newFileName)),
isDir,
registeredFileTypes
);

Expand Down Expand Up @@ -560,29 +579,45 @@ export const copyS3Objects = async (

const [fileType, fileMimeType, fileFormat] = Private.getFileType(
PathExt.extname(PathExt.basename(name)),
isDir,
registeredFileTypes
);

// retrieve information of new file
const newFileContents = await s3Client.send(
new GetObjectCommand({
Bucket: newBucketName ?? bucketName,
Key: name + (suffix ? suffix : '')
})
);
try {
const newFileContents = await s3Client.send(
new GetObjectCommand({
Bucket: newBucketName ?? bucketName,
Key: name + (suffix ? suffix : '')
})
);

data = {
name: PathExt.basename(name),
path: name,
last_modified: newFileContents.LastModified!.toISOString(),
created: new Date().toISOString(),
content: await newFileContents.Body!.transformToString(),
format: fileFormat as Contents.FileFormat,
mimetype: fileMimeType,
size: newFileContents.ContentLength!,
writable: true,
type: fileType
};
data = {
name: PathExt.basename(name),
path: name,
last_modified: newFileContents.LastModified!.toISOString(),
created: new Date().toISOString(),
content: await newFileContents.Body!.transformToString(),
format: fileFormat as Contents.FileFormat,
mimetype: fileMimeType,
size: newFileContents.ContentLength!,
writable: true,
type: fileType
};
} catch {
// object directory itself doesn't exist
data = {
name: PathExt.basename(name),
path: name,
last_modified: new Date().toISOString(),
created: new Date().toISOString(),
content: [],
format: fileFormat as Contents.FileFormat,
mimetype: fileMimeType,
size: 0,
writable: true,
type: fileType
};
}

return data;
};
Expand Down Expand Up @@ -655,7 +690,8 @@ export async function isDirectory(
// listing contents given a path, to check if it is a directory
const command = new ListObjectsV2Command({
Bucket: bucketName,
Prefix: objectPath + '/'
Prefix:
objectPath[objectPath.length - 1] === '/' ? objectPath : objectPath + '/'
});

const { Contents } = await s3Client.send(command);
Expand All @@ -669,18 +705,22 @@ export async function isDirectory(
namespace Private {
/**
* Helping function to define file type, mimetype and format based on file extension.
* @param extension file extension (e.g.: txt, ipynb, csv)
* @returns
* @param extension: File extension (e.g.: txt, ipynb, csv)
* @param isDir: Boolean showing if the object is a directory or a file
* @param registeredFileTypes: The list containing all registered file types.
* @returns The object type, mimetype and format.
*/
export function getFileType(
extension: string,
isDir: boolean,
registeredFileTypes: IRegisteredFileTypes
) {
let fileType: string = 'text';
let fileMimetype: string = 'text/plain';
let fileFormat: string = 'text';
let fileType: string = isDir === false ? 'text' : 'directory';
let fileMimetype: string =
isDir === false ? 'text/plain' : 'text/directory';
let fileFormat: string = isDir === false ? 'text' : 'json';

if (registeredFileTypes[extension]) {
if (isDir === false && registeredFileTypes[extension]) {
fileType = registeredFileTypes[extension].fileType;
fileMimetype = registeredFileTypes[extension].fileMimeTypes[0];
fileFormat = registeredFileTypes[extension].fileFormat;
Expand Down
70 changes: 26 additions & 44 deletions src/s3contents.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ import {
renameS3Objects,
listS3Contents,
IRegisteredFileTypes,
getS3FileContents,
isDirectory
} from './s3';

Expand Down Expand Up @@ -278,38 +277,14 @@ export class Drive implements Contents.IDrive {
this._isRootFormatted = true;
}

// getting the list of files from the root
if (!path) {
data = await listS3Contents(
this._s3Client,
this._name,
this._root,
this._registeredFileTypes
);
} else {
const currentPath = PathExt.basename(path);

// listing contents of a folder
if (PathExt.extname(currentPath) === '') {
data = await listS3Contents(
this._s3Client,
this._name,
this.root,
this.registeredFileTypes,
path
);
}
// getting the contents of a specific file
else {
data = await getS3FileContents(
this._s3Client,
this._name,
this._root,
path,
this.registeredFileTypes
);
}
}
// listing the contents of a directory or retriving the contents of a file
data = await listS3Contents(
this._s3Client,
this._name,
this.root,
this.registeredFileTypes,
path
);

Contents.validateContentsModel(data);
return data;
Expand Down Expand Up @@ -345,7 +320,8 @@ export class Drive implements Contents.IDrive {
name,
options.path ? PathExt.join(options.path, name) : name,
'', // create new file with empty body,
this.registeredFileTypes
this.registeredFileTypes,
options.type === 'directory' ? true : false
);
} else {
console.warn('Type of new element is undefined');
Expand Down Expand Up @@ -447,10 +423,15 @@ export class Drive implements Contents.IDrive {
options: Contents.ICreateOptions = {}
): Promise<Contents.IModel> {
let newFileName = PathExt.basename(newLocalPath);
const isDir: boolean = await isDirectory(
this._s3Client,
this._name,
oldLocalPath
);

try {
await checkS3Object(this._s3Client, this._name, this._root, newLocalPath);
newFileName = await this.incrementName(newLocalPath, this._name);
newFileName = await this.incrementName(newLocalPath, this._name, isDir);
} catch (error) {
// HEAD request failed for this file name, continue, as name doesn't already exist.
} finally {
Expand All @@ -461,6 +442,7 @@ export class Drive implements Contents.IDrive {
oldLocalPath,
newLocalPath,
newFileName,
isDir,
this._registeredFileTypes
);
}
Expand All @@ -481,19 +463,14 @@ export class Drive implements Contents.IDrive {
*
* @param bucketName - The name of the bucket where content is moved.
*
* @param root - The root of the bucket, if it exists.
* @param isDir - Whether the object is a directory or a file.
*/
async incrementName(localPath: string, bucketName: string) {
const isDir: boolean = await isDirectory(
this._s3Client,
bucketName,
localPath
);
async incrementName(localPath: string, bucketName: string, isDir: boolean) {
let fileExtension: string = '';
let originalName: string = '';

// check if we are dealing with a directory
if (isDir) {
if (isDir === true) {
localPath = localPath.substring(0, localPath.length - 1);
originalName = PathExt.basename(localPath);
}
Expand Down Expand Up @@ -549,6 +526,7 @@ export class Drive implements Contents.IDrive {
localPath,
options.content,
this._registeredFileTypes,
true,
options
);

Expand Down Expand Up @@ -594,7 +572,11 @@ export class Drive implements Contents.IDrive {
(isDir ? '/' : '');

// getting incremented name of Copy in case of duplicates
const incrementedName = await this.incrementName(newFilePath, bucketName);
const incrementedName = await this.incrementName(
newFilePath,
bucketName,
isDir
);

return incrementedName;
}
Expand Down

0 comments on commit 205ab5e

Please sign in to comment.