diff --git a/extension/package.json b/extension/package.json index 0029491034..96b91cd27b 100644 --- a/extension/package.json +++ b/extension/package.json @@ -238,6 +238,15 @@ "dark": "resources/dark/run-experiment.svg" } }, + { + "title": "%command.modifyExperimentParamsResetAndRun%", + "command": "dvc.modifyExperimentParamsResetAndRun", + "category": "DVC", + "icon": { + "light": "resources/light/run-experiment.svg", + "dark": "resources/dark/run-experiment.svg" + } + }, { "title": "%command.queueExperiment%", "command": "dvc.queueExperiment", @@ -415,6 +424,15 @@ "dark": "resources/dark/run-experiment.svg" } }, + { + "title": "%command.views.experimentsTree.resetRunExperiment%", + "command": "dvc.views.experimentsTree.resetRunExperiment", + "category": "DVC", + "icon": { + "light": "resources/light/run-experiment.svg", + "dark": "resources/dark/run-experiment.svg" + } + }, { "title": "%command.views.experimentsTree.selectExperiments%", "command": "dvc.views.experimentsTree.selectExperiments", @@ -508,11 +526,11 @@ }, { "command": "dvc.applyExperiment", - "when": "dvc.commands.available && dvc.project.available" + "when": "dvc.commands.available && dvc.project.available && !dvc.runner.running" }, { "command": "dvc.branchExperiment", - "when": "dvc.commands.available && dvc.project.available" + "when": "dvc.commands.available && dvc.project.available && !dvc.runner.running" }, { "command": "dvc.checkout", @@ -552,7 +570,7 @@ }, { "command": "dvc.experimentGarbageCollect", - "when": "dvc.commands.available && dvc.project.available" + "when": "dvc.commands.available && dvc.project.available && !dvc.runner.running" }, { "command": "dvc.findInFolder", @@ -596,23 +614,27 @@ }, { "command": "dvc.modifyExperimentParamsAndQueue", - "when": "dvc.commands.available && dvc.project.available" + "when": "dvc.commands.available && dvc.project.available && !dvc.runner.running" }, { "command": "dvc.modifyExperimentParamsAndRun", - "when": "dvc.commands.available && dvc.project.available" + "when": "dvc.commands.available && dvc.project.available && !dvc.runner.running" + }, + { + "command": "dvc.modifyExperimentParamsResetAndRun", + "when": "dvc.commands.available && dvc.project.available && !dvc.runner.running" }, { "command": "dvc.queueExperiment", - "when": "dvc.commands.available && dvc.project.available" + "when": "dvc.commands.available && dvc.project.available && !dvc.runner.running" }, { "command": "dvc.removeExperiment", - "when": "dvc.commands.available && dvc.project.available" + "when": "dvc.commands.available && dvc.project.available && !dvc.runner.running" }, { "command": "dvc.removeExperimentQueue", - "when": "dvc.commands.available && dvc.project.available" + "when": "dvc.commands.available && dvc.project.available && !dvc.runner.running" }, { "command": "dvc.removeExperimentsTableFilters", @@ -624,7 +646,7 @@ }, { "command": "dvc.removeQueuedExperiment", - "when": "dvc.commands.available && dvc.project.available" + "when": "dvc.commands.available && dvc.project.available && !dvc.runner.running" }, { "command": "dvc.removeTarget", @@ -640,15 +662,15 @@ }, { "command": "dvc.runExperiment", - "when": "dvc.commands.available && dvc.project.available" + "when": "dvc.commands.available && dvc.project.available && !dvc.runner.running" }, { "command": "dvc.runQueuedExperiments", - "when": "dvc.commands.available && dvc.project.available" + "when": "dvc.commands.available && dvc.project.available && !dvc.runner.running" }, { "command": "dvc.runResetExperiment", - "when": "dvc.commands.available && dvc.project.available" + "when": "dvc.commands.available && dvc.project.available && !dvc.runner.running" }, { "command": "dvc.selectForCompare", @@ -959,10 +981,15 @@ "when": "view == dvc.views.experimentsTree && dvc.commands.available && viewItem =~ /^(workspace|branch|experiment|queued)$/ && !dvc.runner.running" }, { - "command": "dvc.views.experimentsTree.queueExperiment", + "command": "dvc.views.experimentsTree.resetRunExperiment", "group": "1_do@2", "when": "view == dvc.views.experimentsTree && dvc.commands.available && viewItem =~ /^(workspace|branch|experiment|queued)$/ && !dvc.runner.running" }, + { + "command": "dvc.views.experimentsTree.queueExperiment", + "group": "1_do@3", + "when": "view == dvc.views.experimentsTree && dvc.commands.available && viewItem =~ /^(workspace|branch|experiment|queued)$/ && !dvc.runner.running" + }, { "command": "dvc.views.experimentsTree.selectExperiments", "group": "inline", @@ -1007,14 +1034,29 @@ }, { "command": "dvc.runExperiment", - "when": "view == dvc.views.experimentsTree && !dvc.experiments.webviewActive", + "when": "view == dvc.views.experimentsTree && !dvc.experiments.webviewActive && !dvc.runner.running", "group": "1_run@1" }, { "command": "dvc.runExperiment", - "when": "view == dvc.views.experimentsTree && dvc.experiments.webviewActive", + "when": "view == dvc.views.experimentsTree && dvc.experiments.webviewActive && !dvc.runner.running", + "group": "navigation@1" + }, + { + "command": "dvc.stopRunningExperiment", + "when": "view == dvc.views.experimentsTree && !dvc.experiments.webviewActive && dvc.runner.running", + "group": "1_run@1" + }, + { + "command": "dvc.stopRunningExperiment", + "when": "view == dvc.views.experimentsTree && dvc.experiments.webviewActive && dvc.runner.running", "group": "navigation@1" }, + { + "command": "dvc.runResetExperiment", + "when": "view == dvc.views.experimentsTree && !dvc.runner.running", + "group": "1_run@2" + }, { "command": "dvc.showPlots", "when": "view == dvc.views.experimentsTree && !dvc.plots.webviewActive", @@ -1028,17 +1070,27 @@ { "command": "dvc.runQueuedExperiments", "when": "view == dvc.views.experimentsTree && !dvc.runner.running", - "group": "1_run@2" + "group": "1_run@3" }, { - "command": "dvc.queueExperiment", + "command": "dvc.modifyExperimentParamsAndRun", "when": "view == dvc.views.experimentsTree && !dvc.runner.running", - "group": "2_queue@1" + "group": "2_modify@1" + }, + { + "command": "dvc.modifyExperimentParamsResetAndRun", + "when": "view == dvc.views.experimentsTree && !dvc.runner.running", + "group": "2_modify@2" }, { "command": "dvc.modifyExperimentParamsAndQueue", "when": "view == dvc.views.experimentsTree && !dvc.runner.running", - "group": "2_queue@2" + "group": "2_modify@3" + }, + { + "command": "dvc.queueExperiment", + "when": "view == dvc.views.experimentsTree && !dvc.runner.running", + "group": "3_queue@1" }, { "command": "dvc.views.experimentsTree.autoApplyFilters", diff --git a/extension/package.nls.json b/extension/package.nls.json index b4f556d2bf..55004e749b 100644 --- a/extension/package.nls.json +++ b/extension/package.nls.json @@ -30,6 +30,7 @@ "command.pushTarget": "Push", "command.modifyExperimentParamsAndQueue": "Modify Experiment Param(s) and Queue", "command.modifyExperimentParamsAndRun": "Modify Experiment Param(s) and Run", + "command.modifyExperimentParamsResetAndRun": "Modify Experiment Param(s) Reset and Run", "command.queueExperiment": "Queue Experiment", "command.removeExperiment": "Remove Experiment", "command.removeExperimentQueue": "Remove All Queued Experiments", @@ -60,6 +61,7 @@ "command.views.experimentsTree.queueExperiment": "Modify Param(s) and Queue", "command.views.experimentsTree.removeExperiment": "Remove", "command.views.experimentsTree.runExperiment": "Modify Param(s) and Run", + "command.views.experimentsTree.resetRunExperiment": "Modify Param(s) Reset and Run", "command.views.experimentsTree.selectExperiments": "Select Experiments to Display in Plots", "command.views.plotsPathsTree.selectPlots": "Select Plots to Display", "config.doNotRecommendRedHatExtension.description": "Do not prompt to install the Red Hat YAML extension to assist with DVC YAML schema validation.", diff --git a/extension/src/commands/external.ts b/extension/src/commands/external.ts index f7d9ace914..884f4c4658 100644 --- a/extension/src/commands/external.ts +++ b/extension/src/commands/external.ts @@ -37,6 +37,7 @@ export enum RegisteredCommands { EXPERIMENT_TREE_QUEUE = 'dvc.views.experimentsTree.queueExperiment', EXPERIMENT_TREE_REMOVE = 'dvc.views.experimentsTree.removeExperiment', EXPERIMENT_TREE_RUN = 'dvc.views.experimentsTree.runExperiment', + EXPERIMENT_TREE_RUN_RESET = 'dvc.views.experimentsTree.resetRunExperiment', EXPERIMENT_SELECT = 'dvc.views.experimentsTree.selectExperiments', EXPERIMENT_SHOW = 'dvc.showExperiments', EXPERIMENT_SORT_ADD = 'dvc.addExperimentsTableSort', @@ -46,6 +47,7 @@ export enum RegisteredCommands { EXPERIMENT_TOGGLE = 'dvc.views.experimentsTree.toggleStatus', MODIFY_EXPERIMENT_PARAMS_AND_QUEUE = 'dvc.modifyExperimentParamsAndQueue', MODIFY_EXPERIMENT_PARAMS_AND_RUN = 'dvc.modifyExperimentParamsAndRun', + MODIFY_EXPERIMENT_PARAMS_RESET_AND_RUN = 'dvc.modifyExperimentParamsResetAndRun', STOP_EXPERIMENT = 'dvc.stopRunningExperiment', PLOTS_PATH_TOGGLE = 'dvc.views.plotsPathsTree.toggleStatus', diff --git a/extension/src/experiments/commands/register.ts b/extension/src/experiments/commands/register.ts index 9dc418b486..5c9ccdd1ef 100644 --- a/extension/src/experiments/commands/register.ts +++ b/extension/src/experiments/commands/register.ts @@ -39,6 +39,16 @@ const registerExperimentCwdCommands = ( ) ) + internalCommands.registerExternalCommand( + RegisteredCommands.MODIFY_EXPERIMENT_PARAMS_RESET_AND_RUN, + () => + experiments.pauseUpdatesThenRun(() => + experiments.modifyExperimentParamsAndRun( + AvailableCommands.EXPERIMENT_RUN_RESET + ) + ) + ) + internalCommands.registerExternalCliCommand( RegisteredCliCommands.EXPERIMENT_REMOVE_QUEUE, () => diff --git a/extension/src/experiments/index.ts b/extension/src/experiments/index.ts index bc7008c9c1..2f4d6c1203 100644 --- a/extension/src/experiments/index.ts +++ b/extension/src/experiments/index.ts @@ -403,6 +403,12 @@ export class Experiments extends BaseRepository { AvailableCommands.EXPERIMENT_RUN, message.payload ) + case MessageFromWebviewType.VARY_EXPERIMENT_PARAMS_RESET_AND_RUN: + return this.modifyExperimentParamsAndRun( + AvailableCommands.EXPERIMENT_RUN_RESET, + message.payload + ) + case MessageFromWebviewType.REMOVE_EXPERIMENT: return this.removeExperiment(message.payload) default: diff --git a/extension/src/experiments/model/tree.ts b/extension/src/experiments/model/tree.ts index 4f9e6d5fb6..6734066d57 100644 --- a/extension/src/experiments/model/tree.ts +++ b/extension/src/experiments/model/tree.ts @@ -164,6 +164,16 @@ export class ExperimentsTree ) ) + internalCommands.registerExternalCommand( + RegisteredCommands.EXPERIMENT_TREE_RUN_RESET, + ({ dvcRoot, id }: ExperimentItem) => + this.experiments.modifyExperimentParamsAndRun( + AvailableCommands.EXPERIMENT_RUN_RESET, + dvcRoot, + id + ) + ) + internalCommands.registerExternalCommand( RegisteredCommands.EXPERIMENT_TREE_REMOVE, ({ dvcRoot, id }: ExperimentItem) => diff --git a/extension/src/telemetry/constants.ts b/extension/src/telemetry/constants.ts index d90f963c4c..3a34920584 100644 --- a/extension/src/telemetry/constants.ts +++ b/extension/src/telemetry/constants.ts @@ -121,10 +121,12 @@ export interface IEventNamePropertyMapping { [EventName.EXPERIMENT_TREE_QUEUE]: undefined [EventName.EXPERIMENT_TREE_REMOVE]: undefined [EventName.EXPERIMENT_TREE_RUN]: undefined + [EventName.EXPERIMENT_TREE_RUN_RESET]: undefined [EventName.EXPERIMENT_TOGGLE]: undefined [EventName.QUEUE_EXPERIMENT]: undefined [EventName.MODIFY_EXPERIMENT_PARAMS_AND_QUEUE]: undefined [EventName.MODIFY_EXPERIMENT_PARAMS_AND_RUN]: undefined + [EventName.MODIFY_EXPERIMENT_PARAMS_RESET_AND_RUN]: undefined [EventName.STOP_EXPERIMENT]: { stopped: boolean; wasRunning: boolean } [EventName.PLOTS_PATH_TOGGLE]: undefined diff --git a/extension/src/test/suite/experiments/index.test.ts b/extension/src/test/suite/experiments/index.test.ts index b42c7cf157..442759b949 100644 --- a/extension/src/test/suite/experiments/index.test.ts +++ b/extension/src/test/suite/experiments/index.test.ts @@ -453,6 +453,38 @@ suite('Experiments Test Suite', () => { ) }) + it("should be able to handle a message to modify an experiment's params reset and run a new experiment", async () => { + const { experiments, mockExecuteCommand } = + setupExperimentsAndMockCommands() + + const mockModifiedParams = [ + '-S', + 'params.yaml:lr=0.0001', + '-S', + 'params.yaml:weight_decay=0.2' + ] + + stub(experiments, 'pickAndModifyParams').resolves(mockModifiedParams) + + const webview = await experiments.showWebview() + const mockMessageReceived = getMessageReceivedEmitter(webview) + const mockExperimentId = 'mock-experiment-id' + const tableChangePromise = experimentsUpdatedEvent(experiments) + + mockMessageReceived.fire({ + payload: mockExperimentId, + type: MessageFromWebviewType.VARY_EXPERIMENT_PARAMS_RESET_AND_RUN + }) + + await tableChangePromise + expect(mockExecuteCommand).to.be.calledOnce + expect(mockExecuteCommand).to.be.calledWithExactly( + AvailableCommands.EXPERIMENT_RUN_RESET, + dvcDemoPath, + ...mockModifiedParams + ) + }) + it('should be able to handle a message to remove an experiment', async () => { const { experiments, mockExecuteCommand } = setupExperimentsAndMockCommands() diff --git a/extension/src/test/suite/experiments/model/tree.test.ts b/extension/src/test/suite/experiments/model/tree.test.ts index 4ee8766f5a..5b22e78545 100644 --- a/extension/src/test/suite/experiments/model/tree.test.ts +++ b/extension/src/test/suite/experiments/model/tree.test.ts @@ -580,4 +580,69 @@ suite('Experiments Tree Test Suite', () => { ) }) }) + + it('should be able to reset and run a new experiment from an existing one with dvc.views.experimentsTree.resetRunExperiment', async () => { + const baseExperimentId = 'workspace' + + const { cliRunner, experiments, experimentsModel } = + buildExperiments(disposable) + + await experiments.isReady() + + const mockRunExperimentReset = stub( + cliRunner, + 'runExperimentReset' + ).resolves(undefined) + + const mockGetOnlyOrPickProject = stub( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (WorkspaceExperiments as any).prototype, + 'getOnlyOrPickProject' + ) + stub(WorkspaceExperiments.prototype, 'getRepository').returns(experiments) + + const getParamsSpy = spy(experimentsModel, 'getExperimentParams') + + const mockShowQuickPick = stub(window, 'showQuickPick') as SinonStub< + [items: readonly QuickPickItem[], options: QuickPickOptionsWithTitle], + Thenable | undefined> + > + mockShowQuickPick.resolves([ + { + label: 'params.yaml:dropout', + value: { path: 'params.yaml:dropout', value: 0.1 } + }, + { + label: 'params.yaml:process.threshold', + value: { path: 'params.yaml:process.threshold', value: 0.8 } + } + ] as QuickPickItemWithValue[]) + + stub(window, 'showInputBox') + .onFirstCall() + .resolves('0.11') + .onSecondCall() + .resolves('0.82') + + await commands.executeCommand( + RegisteredCommands.EXPERIMENT_TREE_RUN_RESET, + { + dvcRoot: dvcDemoPath, + id: baseExperimentId + } + ) + + expect(mockGetOnlyOrPickProject).not.to.be.called + expect(getParamsSpy).to.be.calledOnce + expect(getParamsSpy).to.be.calledWithExactly(baseExperimentId) + expect(mockShowQuickPick).to.be.calledOnce + expect(mockRunExperimentReset).to.be.calledOnce + expect(mockRunExperimentReset).to.be.calledWith( + dvcDemoPath, + '-S', + 'params.yaml:dropout=0.11', + '-S', + 'params.yaml:process.threshold=0.82' + ) + }) }) diff --git a/extension/src/test/suite/experiments/util.ts b/extension/src/test/suite/experiments/util.ts index 4c603b6746..ee2f8626e8 100644 --- a/extension/src/test/suite/experiments/util.ts +++ b/extension/src/test/suite/experiments/util.ts @@ -44,6 +44,7 @@ export const buildExperiments = ( const { cliExecutor, cliReader, + cliRunner, internalCommands, messageSpy, mockExperimentShow, @@ -70,6 +71,7 @@ export const buildExperiments = ( return { cliExecutor, cliReader, + cliRunner, experiments, // eslint-disable-next-line @typescript-eslint/no-explicit-any experimentsModel: (experiments as any).experiments, diff --git a/extension/src/webview/contract.ts b/extension/src/webview/contract.ts index 2d508779df..0b0e329921 100644 --- a/extension/src/webview/contract.ts +++ b/extension/src/webview/contract.ts @@ -30,7 +30,8 @@ export enum MessageFromWebviewType { TOGGLE_METRIC = 'toggle-metric', TOGGLE_PLOTS_SECTION = 'toggle-plots-section', VARY_EXPERIMENT_PARAMS_AND_QUEUE = 'vary-experiment-params-and-queue', - VARY_EXPERIMENT_PARAMS_AND_RUN = 'vary-experiment-params-and-run' + VARY_EXPERIMENT_PARAMS_AND_RUN = 'vary-experiment-params-and-run', + VARY_EXPERIMENT_PARAMS_RESET_AND_RUN = 'vary-experiment-params-reset-and-run' } export type ColumnResizePayload = { @@ -76,6 +77,10 @@ export type MessageFromWebview = type: MessageFromWebviewType.VARY_EXPERIMENT_PARAMS_AND_RUN payload: string } + | { + type: MessageFromWebviewType.VARY_EXPERIMENT_PARAMS_RESET_AND_RUN + payload: string + } | { type: MessageFromWebviewType.REMOVE_EXPERIMENT payload: string diff --git a/webview/src/experiments/components/table/Row.tsx b/webview/src/experiments/components/table/Row.tsx index 640e5edfed..a8d69b3083 100644 --- a/webview/src/experiments/components/table/Row.tsx +++ b/webview/src/experiments/components/table/Row.tsx @@ -67,24 +67,29 @@ export const RowContextMenu: React.FC = ({ ) ]) - pushIf(depth <= 1 || isWorkspace, [ + pushIf(depth === 1, [ experimentMenuOption( id, - 'Modify and Queue', - MessageFromWebviewType.VARY_EXPERIMENT_PARAMS_AND_QUEUE - ), + 'Remove', + MessageFromWebviewType.REMOVE_EXPERIMENT + ) + ]) + + pushIf(depth <= 1 || isWorkspace, [ experimentMenuOption( id, 'Modify and Run', MessageFromWebviewType.VARY_EXPERIMENT_PARAMS_AND_RUN - ) - ]) - - pushIf(depth === 1, [ + ), experimentMenuOption( id, - 'Remove', - MessageFromWebviewType.REMOVE_EXPERIMENT + 'Modify Reset and Run', + MessageFromWebviewType.VARY_EXPERIMENT_PARAMS_RESET_AND_RUN + ), + experimentMenuOption( + id, + 'Modify and Queue', + MessageFromWebviewType.VARY_EXPERIMENT_PARAMS_AND_QUEUE ) ])