diff --git a/src/clients/emrServerlessClient.ts b/src/clients/emrServerlessClient.ts index 339de00..67472be 100644 --- a/src/clients/emrServerlessClient.ts +++ b/src/clients/emrServerlessClient.ts @@ -84,7 +84,7 @@ export class DefaultEMRServerlessClient { return jobRuns; } - public async startJobRun(applicationId: string, executionRoleARN: string, entryPoint: string): Promise { + public async startJobRun(applicationId: string, executionRoleARN: string, entryPoint: string, logPrefix: string): Promise { this.globals.outputChannel.appendLine( `EMR Serverless: Starting job run (${applicationId}).` ); @@ -100,6 +100,14 @@ export class DefaultEMRServerlessClient { } }; + if (logPrefix) { + jobRunParams.configurationOverrides = { + monitoringConfiguration: { + s3MonitoringConfiguration: {logUri: logPrefix} + } + }; + } + try { const result = await emr.send( new StartJobRunCommand(jobRunParams) diff --git a/src/commands/emrDeploy.ts b/src/commands/emrDeploy.ts index b546866..ea3f4cc 100644 --- a/src/commands/emrDeploy.ts +++ b/src/commands/emrDeploy.ts @@ -46,11 +46,15 @@ interface State { s3LogTargetURI: string; srcScriptURI: string; } + +const TOTAL_STEPS = 5; + export class EMRServerlessDeploy { context: vscode.ExtensionContext; title: string; previousAppID: string | undefined; previousS3TargetURI: string | undefined; + previousS3LogTargetURI: string | undefined; previousJobRoleARN: string | undefined; @@ -65,6 +69,7 @@ export class EMRServerlessDeploy { this.previousAppID = undefined; this.previousS3TargetURI = undefined; + this.previousS3LogTargetURI = undefined; this.previousJobRoleARN = undefined; } @@ -89,7 +94,7 @@ export class EMRServerlessDeploy { const pick = await input.showInputBox({ title: this.title, step: 1, - totalSteps: 4, + totalSteps: TOTAL_STEPS, value: defaultTarget, prompt: "Provide an S3 URI where you want to upload your code.", validate: this.validateBucketURI, @@ -99,6 +104,34 @@ export class EMRServerlessDeploy { state.s3TargetURI = pick.valueOf(); this.previousS3TargetURI = state.s3TargetURI; + return (input: MultiStepInput) => this.insertS3LogTargetURI(input, state); + } + + + async insertS3LogTargetURI( + input: MultiStepInput, + state: Partial + ) { + let defaultTarget = "s3://bucket-name/logs/"; + if (this.previousS3LogTargetURI !== undefined) { + defaultTarget = this.previousS3LogTargetURI; + } else if (state.s3TargetURI) { + let codeBucket = this.extractBucketName(state.s3TargetURI!); + defaultTarget = `s3://${codeBucket}/logs/`; + } + const pick = await input.showInputBox({ + title: this.title, + step: 2, + totalSteps: TOTAL_STEPS, + value: defaultTarget, + prompt: "Provide an S3 URI for Spark logs (leave blank to disable).", + validate: this.validateOptionalBucketURI.bind(this), + shouldResume: this.shouldResume, + ignoreFocusOut: true, + }); + + state.s3LogTargetURI = pick.valueOf(); + this.previousS3LogTargetURI = state.s3LogTargetURI; return (input: MultiStepInput) => this.insertJobRoleARN(input, state); } @@ -109,8 +142,8 @@ export class EMRServerlessDeploy { let defaultJobRole = this.previousJobRoleARN ? this.previousJobRoleARN : "arn:aws:iam::xxx:role/job-role"; const pick = await input.showInputBox({ title: this.title, - step: 2, - totalSteps: 4, + step: 3, + totalSteps: TOTAL_STEPS, value: defaultJobRole, prompt: "Provide an IAM Role that has access to the resources for your job.", @@ -132,8 +165,8 @@ export class EMRServerlessDeploy { // TODO: Populate the list of application IDs automatically const pick = await input.showInputBox({ title: this.title, - step: 3, - totalSteps: 4, + step: 4, + totalSteps: TOTAL_STEPS, value: defaultAppId, prompt: "Provide the EMR Serverless Application ID.", validate: this.validateApplicationID, @@ -156,6 +189,14 @@ export class EMRServerlessDeploy { } } + async validateOptionalBucketURI(uri: string): Promise { + if (uri === "" || uri === undefined) { + return undefined; + } + + return this.validateBucketURI(uri); + } + async validateBucketURI(uri: string): Promise { if (!uri.startsWith("s3://")) { return "S3 location must start with s3://"; @@ -163,6 +204,10 @@ export class EMRServerlessDeploy { return undefined; } + extractBucketName(uri: string): string { + return uri.split("/")[2]; + } + async validateJobRole(uri: string): Promise { if (!uri.startsWith("arn:aws:iam::")) { return "Job role must be a full ARN: arn:aws:iam:::role/"; @@ -208,7 +253,8 @@ export class EMRServerlessDeploy { state.applicationID, state.jobRoleARN, state.srcScriptURI, - state.s3TargetURI + state.s3TargetURI, + state.s3LogTargetURI, ); } // Do I do a "deploy" and "run" @@ -218,7 +264,8 @@ export class EMRServerlessDeploy { applicationID: string, executionRoleARN: string, sourceFile: string, - s3TargetURI: string + s3TargetURI: string, + s3LogTargetURI: string, ) { const data = fs.readFileSync(sourceFile); const bucketName = s3TargetURI.split("/")[2]; @@ -228,7 +275,7 @@ export class EMRServerlessDeploy { await this.s3.uploadFile(bucketName, fullS3Key, data); - this.emr.startJobRun(applicationID, executionRoleARN,fullS3Path); + this.emr.startJobRun(applicationID, executionRoleARN,fullS3Path, s3LogTargetURI); vscode.window.showInformationMessage("Your job has been submitted, refresh the EMR Serverless view to keep an eye on it."); }