From 1e95cb902e68ed9e2e3af64d709f4890364de0ef Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Tue, 19 Mar 2024 14:17:46 +1100 Subject: [PATCH] Keep notebook model in sync with the ipynb json (#208052) --- extensions/ipynb/src/cellIdService.ts | 129 ---- extensions/ipynb/src/common.ts | 2 +- extensions/ipynb/src/ipynbMain.ts | 4 +- .../ipynb/src/notebookModelStoreSync.ts | 223 +++++++ extensions/ipynb/src/serializers.ts | 47 +- .../src/test/notebookModelStoreSync.test.ts | 551 ++++++++++++++++++ 6 files changed, 807 insertions(+), 149 deletions(-) delete mode 100644 extensions/ipynb/src/cellIdService.ts create mode 100644 extensions/ipynb/src/notebookModelStoreSync.ts create mode 100644 extensions/ipynb/src/test/notebookModelStoreSync.test.ts diff --git a/extensions/ipynb/src/cellIdService.ts b/extensions/ipynb/src/cellIdService.ts deleted file mode 100644 index 43c769bca513b..0000000000000 --- a/extensions/ipynb/src/cellIdService.ts +++ /dev/null @@ -1,129 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { ExtensionContext, NotebookDocument, NotebookDocumentChangeEvent, NotebookEdit, workspace, WorkspaceEdit } from 'vscode'; -import { getCellMetadata } from './serializers'; -import { CellMetadata } from './common'; -import { getNotebookMetadata } from './notebookSerializer'; -import type * as nbformat from '@jupyterlab/nbformat'; - -/** - * Ensure all new cells in notebooks with nbformat >= 4.5 have an id. - * Details of the spec can be found here https://jupyter.org/enhancement-proposals/62-cell-id/cell-id.html# - */ -export function ensureAllNewCellsHaveCellIds(context: ExtensionContext) { - workspace.onDidChangeNotebookDocument(onDidChangeNotebookCells, undefined, context.subscriptions); -} - -function onDidChangeNotebookCells(e: NotebookDocumentChangeEvent) { - const nbMetadata = getNotebookMetadata(e.notebook); - if (!isCellIdRequired(nbMetadata)) { - return; - } - e.contentChanges.forEach(change => { - change.addedCells.forEach(cell => { - const cellMetadata = getCellMetadata(cell); - if (cellMetadata?.id) { - return; - } - const id = generateCellId(e.notebook); - const edit = new WorkspaceEdit(); - // Don't edit the metadata directly, always get a clone (prevents accidental singletons and directly editing the objects). - const updatedMetadata: CellMetadata = { ...JSON.parse(JSON.stringify(cellMetadata || {})) }; - updatedMetadata.id = id; - edit.set(cell.notebook.uri, [NotebookEdit.updateCellMetadata(cell.index, { ...(cell.metadata), custom: updatedMetadata })]); - workspace.applyEdit(edit); - }); - }); -} - -/** - * Cell ids are required in notebooks only in notebooks with nbformat >= 4.5 - */ -function isCellIdRequired(metadata: Pick, 'nbformat' | 'nbformat_minor'>) { - if ((metadata.nbformat || 0) >= 5) { - return true; - } - if ((metadata.nbformat || 0) === 4 && (metadata.nbformat_minor || 0) >= 5) { - return true; - } - return false; -} - -function generateCellId(notebook: NotebookDocument) { - while (true) { - // Details of the id can be found here https://jupyter.org/enhancement-proposals/62-cell-id/cell-id.html#adding-an-id-field, - // & here https://jupyter.org/enhancement-proposals/62-cell-id/cell-id.html#updating-older-formats - const id = generateUuid().replace(/-/g, '').substring(0, 8); - let duplicate = false; - for (let index = 0; index < notebook.cellCount; index++) { - const cell = notebook.cellAt(index); - const existingId = getCellMetadata(cell)?.id; - if (!existingId) { - continue; - } - if (existingId === id) { - duplicate = true; - break; - } - } - if (!duplicate) { - return id; - } - } -} - - -/** - * Copied from src/vs/base/common/uuid.ts - */ -function generateUuid() { - // use `randomValues` if possible - function getRandomValues(bucket: Uint8Array): Uint8Array { - for (let i = 0; i < bucket.length; i++) { - bucket[i] = Math.floor(Math.random() * 256); - } - return bucket; - } - - // prep-work - const _data = new Uint8Array(16); - const _hex: string[] = []; - for (let i = 0; i < 256; i++) { - _hex.push(i.toString(16).padStart(2, '0')); - } - - // get data - getRandomValues(_data); - - // set version bits - _data[6] = (_data[6] & 0x0f) | 0x40; - _data[8] = (_data[8] & 0x3f) | 0x80; - - // print as string - let i = 0; - let result = ''; - result += _hex[_data[i++]]; - result += _hex[_data[i++]]; - result += _hex[_data[i++]]; - result += _hex[_data[i++]]; - result += '-'; - result += _hex[_data[i++]]; - result += _hex[_data[i++]]; - result += '-'; - result += _hex[_data[i++]]; - result += _hex[_data[i++]]; - result += '-'; - result += _hex[_data[i++]]; - result += _hex[_data[i++]]; - result += '-'; - result += _hex[_data[i++]]; - result += _hex[_data[i++]]; - result += _hex[_data[i++]]; - result += _hex[_data[i++]]; - result += _hex[_data[i++]]; - result += _hex[_data[i++]]; - return result; -} diff --git a/extensions/ipynb/src/common.ts b/extensions/ipynb/src/common.ts index d5ff5f86069ea..b7b454ac03fa8 100644 --- a/extensions/ipynb/src/common.ts +++ b/extensions/ipynb/src/common.ts @@ -58,5 +58,5 @@ export interface CellMetadata { /** * Stores cell metadata. */ - metadata?: Partial; + metadata?: Partial & { vscode?: { languageId?: string } }; } diff --git a/extensions/ipynb/src/ipynbMain.ts b/extensions/ipynb/src/ipynbMain.ts index c256e3b4f6522..55fd2b62c849a 100644 --- a/extensions/ipynb/src/ipynbMain.ts +++ b/extensions/ipynb/src/ipynbMain.ts @@ -5,7 +5,7 @@ import * as vscode from 'vscode'; import { NotebookSerializer } from './notebookSerializer'; -import { ensureAllNewCellsHaveCellIds } from './cellIdService'; +import { activate as keepNotebookModelStoreInSync } from './notebookModelStoreSync'; import { notebookImagePasteSetup } from './notebookImagePaste'; import { AttachmentCleaner } from './notebookAttachmentCleaner'; @@ -30,7 +30,7 @@ type NotebookMetadata = { export function activate(context: vscode.ExtensionContext) { const serializer = new NotebookSerializer(context); - ensureAllNewCellsHaveCellIds(context); + keepNotebookModelStoreInSync(context); context.subscriptions.push(vscode.workspace.registerNotebookSerializer('jupyter-notebook', serializer, { transientOutputs: false, transientCellMetadata: { diff --git a/extensions/ipynb/src/notebookModelStoreSync.ts b/extensions/ipynb/src/notebookModelStoreSync.ts new file mode 100644 index 0000000000000..535a76c3dfbe1 --- /dev/null +++ b/extensions/ipynb/src/notebookModelStoreSync.ts @@ -0,0 +1,223 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { ExtensionContext, NotebookCellKind, NotebookDocument, NotebookDocumentChangeEvent, NotebookEdit, workspace, WorkspaceEdit, type NotebookCell, type NotebookDocumentWillSaveEvent } from 'vscode'; +import { getCellMetadata, getVSCodeCellLanguageId, removeVSCodeCellLanguageId, setVSCodeCellLanguageId } from './serializers'; +import { CellMetadata } from './common'; +import { getNotebookMetadata } from './notebookSerializer'; +import type * as nbformat from '@jupyterlab/nbformat'; + +const noop = () => { + // +}; + +/** + * Code here is used to ensure the Notebook Model is in sync the the ipynb JSON file. + * E.g. assume you add a new cell, this new cell will not have any metadata at all. + * However when we save the ipynb, the metadata will be an empty object `{}`. + * Now thats completely different from the metadata os being `empty/undefined` in the model. + * As a result, when looking at things like diff view or accessing metadata, we'll see differences. +* +* This code ensures that the model is in sync with the ipynb file. +*/ +export const pendingNotebookCellModelUpdates = new WeakMap>>(); +export function activate(context: ExtensionContext) { + workspace.onDidChangeNotebookDocument(onDidChangeNotebookCells, undefined, context.subscriptions); + workspace.onWillSaveNotebookDocument(waitForPendingModelUpdates, undefined, context.subscriptions); +} + +function isSupportedNotebook(notebook: NotebookDocument) { + return notebook.notebookType === 'jupyter-notebook' || notebook.notebookType === 'interactive'; +} + +function waitForPendingModelUpdates(e: NotebookDocumentWillSaveEvent) { + if (!isSupportedNotebook(e.notebook)) { + return; + } + + const promises = pendingNotebookCellModelUpdates.get(e.notebook); + if (!promises) { + return; + } + e.waitUntil(Promise.all(promises)); +} + +function cleanup(notebook: NotebookDocument, promise: PromiseLike) { + const pendingUpdates = pendingNotebookCellModelUpdates.get(notebook); + if (pendingUpdates) { + pendingUpdates.delete(promise); + if (!pendingUpdates.size) { + pendingNotebookCellModelUpdates.delete(notebook); + } + } +} +function trackAndUpdateCellMetadata(notebook: NotebookDocument, cell: NotebookCell, metadata: CellMetadata & { vscode?: { languageId: string } }) { + const pendingUpdates = pendingNotebookCellModelUpdates.get(notebook) ?? new Set>(); + pendingNotebookCellModelUpdates.set(notebook, pendingUpdates); + const edit = new WorkspaceEdit(); + edit.set(cell.notebook.uri, [NotebookEdit.updateCellMetadata(cell.index, { ...(cell.metadata), custom: metadata })]); + const promise = workspace.applyEdit(edit).then(noop, noop); + pendingUpdates.add(promise); + const clean = () => cleanup(notebook, promise); + promise.then(clean, clean); +} + +function onDidChangeNotebookCells(e: NotebookDocumentChangeEvent) { + if (!isSupportedNotebook(e.notebook)) { + return; + } + + const notebook = e.notebook; + const notebookMetadata = getNotebookMetadata(e.notebook); + + // use the preferred language from document metadata or the first cell language as the notebook preferred cell language + const preferredCellLanguage = notebookMetadata.metadata?.language_info?.name; + + // When we change the language of a cell, + // Ensure the metadata in the notebook cell has been updated as well, + // Else model will be out of sync with ipynb https://github.com/microsoft/vscode/issues/207968#issuecomment-2002858596 + e.cellChanges.forEach(e => { + if (!preferredCellLanguage || e.cell.kind !== NotebookCellKind.Code) { + return; + } + const languageIdInMetadata = getVSCodeCellLanguageId(getCellMetadata(e.cell)); + if (e.cell.document.languageId !== preferredCellLanguage && e.cell.document.languageId !== languageIdInMetadata) { + const metadata: CellMetadata = JSON.parse(JSON.stringify(getCellMetadata(e.cell))); + metadata.metadata = metadata.metadata || {}; + setVSCodeCellLanguageId(metadata, e.cell.document.languageId); + trackAndUpdateCellMetadata(notebook, e.cell, metadata); + + } else if (e.cell.document.languageId === preferredCellLanguage && languageIdInMetadata) { + const metadata: CellMetadata = JSON.parse(JSON.stringify(getCellMetadata(e.cell))); + metadata.metadata = metadata.metadata || {}; + removeVSCodeCellLanguageId(metadata); + trackAndUpdateCellMetadata(notebook, e.cell, metadata); + } else if (e.cell.document.languageId === preferredCellLanguage && e.cell.document.languageId === languageIdInMetadata) { + const metadata: CellMetadata = JSON.parse(JSON.stringify(getCellMetadata(e.cell))); + metadata.metadata = metadata.metadata || {}; + removeVSCodeCellLanguageId(metadata); + trackAndUpdateCellMetadata(notebook, e.cell, metadata); + } + }); + + // Ensure all new cells in notebooks with nbformat >= 4.5 have an id. + // Details of the spec can be found here https://jupyter.org/enhancement-proposals/62-cell-id/cell-id.html# + e.contentChanges.forEach(change => { + change.addedCells.forEach(cell => { + // When ever a cell is added, always update the metadata + // as metadata is always an empty `{}` in ipynb JSON file + const cellMetadata = getCellMetadata(cell); + + // Avoid updating the metadata if it's not required. + if (cellMetadata.metadata) { + if (!isCellIdRequired(notebookMetadata)) { + return; + } + if (isCellIdRequired(notebookMetadata) && cellMetadata?.id) { + return; + } + } + + // Don't edit the metadata directly, always get a clone (prevents accidental singletons and directly editing the objects). + const metadata: CellMetadata = { ...JSON.parse(JSON.stringify(cellMetadata || {})) }; + metadata.metadata = metadata.metadata || {}; + + if (isCellIdRequired(notebookMetadata) && !cellMetadata?.id) { + metadata.id = generateCellId(e.notebook); + } + trackAndUpdateCellMetadata(notebook, cell, metadata); + }); + }); +} + + +/** + * Cell ids are required in notebooks only in notebooks with nbformat >= 4.5 + */ +function isCellIdRequired(metadata: Pick, 'nbformat' | 'nbformat_minor'>) { + if ((metadata.nbformat || 0) >= 5) { + return true; + } + if ((metadata.nbformat || 0) === 4 && (metadata.nbformat_minor || 0) >= 5) { + return true; + } + return false; +} + +function generateCellId(notebook: NotebookDocument) { + while (true) { + // Details of the id can be found here https://jupyter.org/enhancement-proposals/62-cell-id/cell-id.html#adding-an-id-field, + // & here https://jupyter.org/enhancement-proposals/62-cell-id/cell-id.html#updating-older-formats + const id = generateUuid().replace(/-/g, '').substring(0, 8); + let duplicate = false; + for (let index = 0; index < notebook.cellCount; index++) { + const cell = notebook.cellAt(index); + const existingId = getCellMetadata(cell)?.id; + if (!existingId) { + continue; + } + if (existingId === id) { + duplicate = true; + break; + } + } + if (!duplicate) { + return id; + } + } +} + + +/** + * Copied from src/vs/base/common/uuid.ts + */ +function generateUuid() { + // use `randomValues` if possible + function getRandomValues(bucket: Uint8Array): Uint8Array { + for (let i = 0; i < bucket.length; i++) { + bucket[i] = Math.floor(Math.random() * 256); + } + return bucket; + } + + // prep-work + const _data = new Uint8Array(16); + const _hex: string[] = []; + for (let i = 0; i < 256; i++) { + _hex.push(i.toString(16).padStart(2, '0')); + } + + // get data + getRandomValues(_data); + + // set version bits + _data[6] = (_data[6] & 0x0f) | 0x40; + _data[8] = (_data[8] & 0x3f) | 0x80; + + // print as string + let i = 0; + let result = ''; + result += _hex[_data[i++]]; + result += _hex[_data[i++]]; + result += _hex[_data[i++]]; + result += _hex[_data[i++]]; + result += '-'; + result += _hex[_data[i++]]; + result += _hex[_data[i++]]; + result += '-'; + result += _hex[_data[i++]]; + result += _hex[_data[i++]]; + result += '-'; + result += _hex[_data[i++]]; + result += _hex[_data[i++]]; + result += '-'; + result += _hex[_data[i++]]; + result += _hex[_data[i++]]; + result += _hex[_data[i++]]; + result += _hex[_data[i++]]; + result += _hex[_data[i++]]; + result += _hex[_data[i++]]; + return result; +} diff --git a/extensions/ipynb/src/serializers.ts b/extensions/ipynb/src/serializers.ts index ce43bbae3fc64..e5843239d5913 100644 --- a/extensions/ipynb/src/serializers.ts +++ b/extensions/ipynb/src/serializers.ts @@ -5,7 +5,7 @@ import type * as nbformat from '@jupyterlab/nbformat'; import { NotebookCell, NotebookCellData, NotebookCellKind, NotebookCellOutput } from 'vscode'; -import { CellOutputMetadata } from './common'; +import { CellOutputMetadata, type CellMetadata } from './common'; import { textMimeTypes } from './deserializers'; const textDecoder = new TextDecoder(); @@ -54,28 +54,41 @@ export function sortObjectPropertiesRecursively(obj: any): any { return obj; } -export function getCellMetadata(cell: NotebookCell | NotebookCellData) { - return { +export function getCellMetadata(cell: NotebookCell | NotebookCellData): CellMetadata { + const metadata = { // it contains the cell id, and the cell metadata, along with other nb cell metadata - ...(cell.metadata?.custom ?? {}), - // promote the cell attachments to the top level - attachments: cell.metadata?.custom?.attachments ?? cell.metadata?.attachments + ...(cell.metadata?.custom ?? {}) }; + + // promote the cell attachments to the top level + const attachments = cell.metadata?.custom?.attachments ?? cell.metadata?.attachments; + if (attachments) { + metadata.attachments = attachments; + } + return metadata; +} + +export function getVSCodeCellLanguageId(metadata: CellMetadata): string | undefined { + return metadata.metadata?.vscode?.languageId; +} +export function setVSCodeCellLanguageId(metadata: CellMetadata, languageId: string) { + metadata.metadata = metadata.metadata || {}; + metadata.metadata.vscode = { languageId }; +} +export function removeVSCodeCellLanguageId(metadata: CellMetadata) { + if (metadata.metadata?.vscode) { + delete metadata.metadata.vscode; + } } function createCodeCellFromNotebookCell(cell: NotebookCellData, preferredLanguage: string | undefined): nbformat.ICodeCell { - const cellMetadata = getCellMetadata(cell); - let metadata = cellMetadata?.metadata || {}; // This cannot be empty. + const cellMetadata: CellMetadata = JSON.parse(JSON.stringify(getCellMetadata(cell))); + cellMetadata.metadata = cellMetadata.metadata || {}; // This cannot be empty. if (cell.languageId !== preferredLanguage) { - metadata = { - ...metadata, - vscode: { - languageId: cell.languageId - } - }; - } else if (metadata.vscode) { + setVSCodeCellLanguageId(cellMetadata, cell.languageId); + } else { // cell current language is the same as the preferred cell language in the document, flush the vscode custom language id metadata - delete metadata.vscode; + removeVSCodeCellLanguageId(cellMetadata); } const codeCell: any = { @@ -83,7 +96,7 @@ function createCodeCellFromNotebookCell(cell: NotebookCellData, preferredLanguag execution_count: cell.executionSummary?.executionOrder ?? null, source: splitMultilineString(cell.value.replace(/\r\n/g, '\n')), outputs: (cell.outputs || []).map(translateCellDisplayOutput), - metadata: metadata + metadata: cellMetadata.metadata }; if (cellMetadata?.id) { codeCell.id = cellMetadata.id; diff --git a/extensions/ipynb/src/test/notebookModelStoreSync.test.ts b/extensions/ipynb/src/test/notebookModelStoreSync.test.ts new file mode 100644 index 0000000000000..8c1370703e129 --- /dev/null +++ b/extensions/ipynb/src/test/notebookModelStoreSync.test.ts @@ -0,0 +1,551 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as assert from 'assert'; +import * as sinon from 'sinon'; +import { CancellationTokenSource, Disposable, EventEmitter, ExtensionContext, NotebookCellKind, NotebookDocumentChangeEvent, NotebookDocumentWillSaveEvent, NotebookEdit, NotebookRange, TextDocumentSaveReason, workspace, type CancellationToken, type NotebookCell, type NotebookDocument, type WorkspaceEdit, type WorkspaceEditMetadata } from 'vscode'; +import { activate } from '../notebookModelStoreSync'; + +suite('Notebook Model Store Sync', () => { + let disposables: Disposable[] = []; + let onDidChangeNotebookDocument: EventEmitter; + let onWillSaveNotebookDocument: AsyncEmitter; + let notebook: NotebookDocument; + let token: CancellationTokenSource; + let editsApplied: WorkspaceEdit[] = []; + let pendingPromises: Promise[] = []; + let cellMetadataUpdates: NotebookEdit[] = []; + let applyEditStub: sinon.SinonStub<[edit: WorkspaceEdit, metadata?: WorkspaceEditMetadata | undefined], Thenable>; + setup(() => { + disposables = []; + notebook = { + notebookType: '', + metadata: {} + } as NotebookDocument; + token = new CancellationTokenSource(); + disposables.push(token); + sinon.stub(notebook, 'notebookType').get(() => 'jupyter-notebook'); + applyEditStub = sinon.stub(workspace, 'applyEdit').callsFake((edit: WorkspaceEdit) => { + editsApplied.push(edit); + return Promise.resolve(true); + }); + const context = { subscriptions: [] as Disposable[] } as ExtensionContext; + onDidChangeNotebookDocument = new EventEmitter(); + disposables.push(onDidChangeNotebookDocument); + onWillSaveNotebookDocument = new AsyncEmitter(); + + sinon.stub(NotebookEdit, 'updateCellMetadata').callsFake((index, metadata) => { + const edit = (NotebookEdit.updateCellMetadata as any).wrappedMethod.call(NotebookEdit, index, metadata); + cellMetadataUpdates.push(edit); + return edit; + } + ); + sinon.stub(workspace, 'onDidChangeNotebookDocument').callsFake(cb => + onDidChangeNotebookDocument.event(cb) + ); + sinon.stub(workspace, 'onWillSaveNotebookDocument').callsFake(cb => + onWillSaveNotebookDocument.event(cb) + ); + activate(context); + }); + teardown(async () => { + await Promise.allSettled(pendingPromises); + editsApplied = []; + pendingPromises = []; + cellMetadataUpdates = []; + disposables.forEach(d => d.dispose()); + disposables = []; + sinon.restore(); + }); + + test('Empty cell will not result in any updates', async () => { + const e: NotebookDocumentChangeEvent = { + notebook, + metadata: undefined, + contentChanges: [], + cellChanges: [] + }; + + onDidChangeNotebookDocument.fire(e); + + assert.strictEqual(editsApplied.length, 0); + }); + test('Adding cell for non Jupyter Notebook will not result in any updates', async () => { + sinon.stub(notebook, 'notebookType').get(() => 'some-other-type'); + const cell: NotebookCell = { + document: {} as any, + executionSummary: {}, + index: 0, + kind: NotebookCellKind.Code, + metadata: {}, + notebook, + outputs: [] + }; + const e: NotebookDocumentChangeEvent = { + notebook, + metadata: undefined, + contentChanges: [ + { + range: new NotebookRange(0, 0), + removedCells: [], + addedCells: [cell] + } + ], + cellChanges: [] + }; + + onDidChangeNotebookDocument.fire(e); + + assert.strictEqual(editsApplied.length, 0); + assert.strictEqual(cellMetadataUpdates.length, 0); + }); + test('Adding cell will result in an update to the metadata', async () => { + const cell: NotebookCell = { + document: {} as any, + executionSummary: {}, + index: 0, + kind: NotebookCellKind.Code, + metadata: {}, + notebook, + outputs: [] + }; + const e: NotebookDocumentChangeEvent = { + notebook, + metadata: undefined, + contentChanges: [ + { + range: new NotebookRange(0, 0), + removedCells: [], + addedCells: [cell] + } + ], + cellChanges: [] + }; + + onDidChangeNotebookDocument.fire(e); + + assert.strictEqual(editsApplied.length, 1); + assert.strictEqual(cellMetadataUpdates.length, 1); + const newMetadata = cellMetadataUpdates[0].newCellMetadata; + assert.deepStrictEqual(newMetadata, { custom: { metadata: {} } }); + }); + test('Add cell id if nbformat is 4.5', async () => { + sinon.stub(notebook, 'metadata').get(() => ({ custom: { nbformat: 4, nbformat_minor: 5 } })); + const cell: NotebookCell = { + document: {} as any, + executionSummary: {}, + index: 0, + kind: NotebookCellKind.Code, + metadata: {}, + notebook, + outputs: [] + }; + const e: NotebookDocumentChangeEvent = { + notebook, + metadata: undefined, + contentChanges: [ + { + range: new NotebookRange(0, 0), + removedCells: [], + addedCells: [cell] + } + ], + cellChanges: [] + }; + + onDidChangeNotebookDocument.fire(e); + + assert.strictEqual(editsApplied.length, 1); + assert.strictEqual(cellMetadataUpdates.length, 1); + const newMetadata = cellMetadataUpdates[0].newCellMetadata || {}; + assert.strictEqual(Object.keys(newMetadata).length, 1); + assert.strictEqual(Object.keys(newMetadata.custom).length, 2); + assert.deepStrictEqual(newMetadata.custom.metadata, {}); + assert.ok(newMetadata.custom.id); + }); + test('Do not add cell id if one already exists', async () => { + sinon.stub(notebook, 'metadata').get(() => ({ custom: { nbformat: 4, nbformat_minor: 5 } })); + const cell: NotebookCell = { + document: {} as any, + executionSummary: {}, + index: 0, + kind: NotebookCellKind.Code, + metadata: { + custom: { + id: '1234' + } + }, + notebook, + outputs: [] + }; + const e: NotebookDocumentChangeEvent = { + notebook, + metadata: undefined, + contentChanges: [ + { + range: new NotebookRange(0, 0), + removedCells: [], + addedCells: [cell] + } + ], + cellChanges: [] + }; + + onDidChangeNotebookDocument.fire(e); + + assert.strictEqual(editsApplied.length, 1); + assert.strictEqual(cellMetadataUpdates.length, 1); + const newMetadata = cellMetadataUpdates[0].newCellMetadata || {}; + assert.strictEqual(Object.keys(newMetadata).length, 1); + assert.strictEqual(Object.keys(newMetadata.custom).length, 2); + assert.deepStrictEqual(newMetadata.custom.metadata, {}); + assert.strictEqual(newMetadata.custom.id, '1234'); + }); + test('Do not perform any updates if cell id and metadata exists', async () => { + sinon.stub(notebook, 'metadata').get(() => ({ custom: { nbformat: 4, nbformat_minor: 5 } })); + const cell: NotebookCell = { + document: {} as any, + executionSummary: {}, + index: 0, + kind: NotebookCellKind.Code, + metadata: { + custom: { + id: '1234', + metadata: {} + } + }, + notebook, + outputs: [] + }; + const e: NotebookDocumentChangeEvent = { + notebook, + metadata: undefined, + contentChanges: [ + { + range: new NotebookRange(0, 0), + removedCells: [], + addedCells: [cell] + } + ], + cellChanges: [] + }; + + onDidChangeNotebookDocument.fire(e); + + assert.strictEqual(editsApplied.length, 0); + assert.strictEqual(cellMetadataUpdates.length, 0); + }); + test('Store language id in custom metadata, whilst preserving existing metadata', async () => { + sinon.stub(notebook, 'metadata').get(() => ({ + custom: { + nbformat: 4, nbformat_minor: 5, + metadata: { + language_info: { name: 'python' } + } + } + })); + const cell: NotebookCell = { + document: { + languageId: 'javascript' + } as any, + executionSummary: {}, + index: 0, + kind: NotebookCellKind.Code, + metadata: { + custom: { + id: '1234', + metadata: { + collapsed: true, scrolled: true + } + } + }, + notebook, + outputs: [] + }; + const e: NotebookDocumentChangeEvent = { + notebook, + metadata: undefined, + contentChanges: [], + cellChanges: [ + { + cell, + document: undefined, + metadata: undefined, + outputs: undefined, + executionSummary: undefined + } + ] + }; + + onDidChangeNotebookDocument.fire(e); + + assert.strictEqual(editsApplied.length, 1); + assert.strictEqual(cellMetadataUpdates.length, 1); + const newMetadata = cellMetadataUpdates[0].newCellMetadata || {}; + assert.strictEqual(Object.keys(newMetadata).length, 1); + assert.strictEqual(Object.keys(newMetadata.custom).length, 2); + assert.deepStrictEqual(newMetadata.custom.metadata, { collapsed: true, scrolled: true, vscode: { languageId: 'javascript' } }); + assert.strictEqual(newMetadata.custom.id, '1234'); + }); + test('No changes when language is javascript', async () => { + sinon.stub(notebook, 'metadata').get(() => ({ + custom: { + nbformat: 4, nbformat_minor: 5, + metadata: { + language_info: { name: 'javascript' } + } + } + })); + const cell: NotebookCell = { + document: { + languageId: 'javascript' + } as any, + executionSummary: {}, + index: 0, + kind: NotebookCellKind.Code, + metadata: { + custom: { + id: '1234', + metadata: { + collapsed: true, scrolled: true + } + } + }, + notebook, + outputs: [] + }; + const e: NotebookDocumentChangeEvent = { + notebook, + metadata: undefined, + contentChanges: [], + cellChanges: [ + { + cell, + document: undefined, + metadata: undefined, + outputs: undefined, + executionSummary: undefined + } + ] + }; + + onDidChangeNotebookDocument.fire(e); + + assert.strictEqual(editsApplied.length, 0); + assert.strictEqual(cellMetadataUpdates.length, 0); + }); + test('Remove language from metadata when cell language matches kernel language', async () => { + sinon.stub(notebook, 'metadata').get(() => ({ + custom: { + nbformat: 4, nbformat_minor: 5, + metadata: { + language_info: { name: 'javascript' } + } + } + })); + const cell: NotebookCell = { + document: { + languageId: 'javascript' + } as any, + executionSummary: {}, + index: 0, + kind: NotebookCellKind.Code, + metadata: { + custom: { + id: '1234', + metadata: { + vscode: { languageId: 'python' }, + collapsed: true, scrolled: true + } + } + }, + notebook, + outputs: [] + }; + const e: NotebookDocumentChangeEvent = { + notebook, + metadata: undefined, + contentChanges: [], + cellChanges: [ + { + cell, + document: undefined, + metadata: undefined, + outputs: undefined, + executionSummary: undefined + } + ] + }; + + onDidChangeNotebookDocument.fire(e); + + assert.strictEqual(editsApplied.length, 1); + assert.strictEqual(cellMetadataUpdates.length, 1); + const newMetadata = cellMetadataUpdates[0].newCellMetadata || {}; + assert.strictEqual(Object.keys(newMetadata).length, 1); + assert.strictEqual(Object.keys(newMetadata.custom).length, 2); + assert.deepStrictEqual(newMetadata.custom.metadata, { collapsed: true, scrolled: true }); + assert.strictEqual(newMetadata.custom.id, '1234'); + }); + test('Update language in metadata', async () => { + sinon.stub(notebook, 'metadata').get(() => ({ + custom: { + nbformat: 4, nbformat_minor: 5, + metadata: { + language_info: { name: 'javascript' } + } + } + })); + const cell: NotebookCell = { + document: { + languageId: 'powershell' + } as any, + executionSummary: {}, + index: 0, + kind: NotebookCellKind.Code, + metadata: { + custom: { + id: '1234', + metadata: { + vscode: { languageId: 'python' }, + collapsed: true, scrolled: true + } + } + }, + notebook, + outputs: [] + }; + const e: NotebookDocumentChangeEvent = { + notebook, + metadata: undefined, + contentChanges: [], + cellChanges: [ + { + cell, + document: undefined, + metadata: undefined, + outputs: undefined, + executionSummary: undefined + } + ] + }; + + onDidChangeNotebookDocument.fire(e); + + assert.strictEqual(editsApplied.length, 1); + assert.strictEqual(cellMetadataUpdates.length, 1); + const newMetadata = cellMetadataUpdates[0].newCellMetadata || {}; + assert.strictEqual(Object.keys(newMetadata).length, 1); + assert.strictEqual(Object.keys(newMetadata.custom).length, 2); + assert.deepStrictEqual(newMetadata.custom.metadata, { collapsed: true, scrolled: true, vscode: { languageId: 'powershell' } }); + assert.strictEqual(newMetadata.custom.id, '1234'); + }); + + test('Will save event without any changes', async () => { + await onWillSaveNotebookDocument.fireAsync({ notebook, reason: TextDocumentSaveReason.Manual }, token.token); + }); + test('Wait for pending updates to complete when saving', async () => { + let resolveApplyEditPromise: (value: boolean) => void; + const promise = new Promise((resolve) => resolveApplyEditPromise = resolve); + applyEditStub.restore(); + sinon.stub(workspace, 'applyEdit').callsFake((edit: WorkspaceEdit) => { + editsApplied.push(edit); + return promise; + }); + + const cell: NotebookCell = { + document: {} as any, + executionSummary: {}, + index: 0, + kind: NotebookCellKind.Code, + metadata: {}, + notebook, + outputs: [] + }; + const e: NotebookDocumentChangeEvent = { + notebook, + metadata: undefined, + contentChanges: [ + { + range: new NotebookRange(0, 0), + removedCells: [], + addedCells: [cell] + } + ], + cellChanges: [] + }; + + onDidChangeNotebookDocument.fire(e); + + assert.strictEqual(editsApplied.length, 1); + assert.strictEqual(cellMetadataUpdates.length, 1); + + // Try to save. + let saveCompleted = false; + const saved = onWillSaveNotebookDocument.fireAsync({ + notebook, + reason: TextDocumentSaveReason.Manual + }, token.token); + saved.finally(() => saveCompleted = true); + await new Promise((resolve) => setTimeout(resolve, 10)); + + // Verify we have not yet completed saving. + assert.strictEqual(saveCompleted, false); + + resolveApplyEditPromise!(true); + await new Promise((resolve) => setTimeout(resolve, 1)); + + // Should have completed saving. + saved.finally(() => saveCompleted = true); + }); + + interface IWaitUntil { + token: CancellationToken; + waitUntil(thenable: Promise): void; + } + + interface IWaitUntil { + token: CancellationToken; + waitUntil(thenable: Promise): void; + } + type IWaitUntilData = Omit, 'token'>; + + class AsyncEmitter { + private listeners: ((d: T) => void)[] = []; + get event(): (listener: (e: T) => any, thisArgs?: any, disposables?: Disposable[]) => Disposable { + + return (listener, thisArgs, _disposables) => { + this.listeners.push(listener.bind(thisArgs)); + return { + dispose: () => { + // + } + }; + }; + } + dispose() { + this.listeners = []; + } + async fireAsync(data: IWaitUntilData, token: CancellationToken): Promise { + if (!this.listeners.length) { + return; + } + + const promises: Promise[] = []; + this.listeners.forEach(cb => { + const event = { + ...data, + token, + waitUntil: (thenable: Promise) => { + promises.push(thenable); + } + } as T; + cb(event); + }); + + await Promise.all(promises); + } + } +});