Use ipynb extension to serialize and drop ICell (#15383)

* Store isInteractiveWindowMessageCell at root level

* oops

* Fixes

* Use ipynb extension to serialize and drop ICell

* oops

* Misc

* misc

* Revert changes

src/interactive-window/commands/commandRegistry.ts

@@ -619,8 +619,8 @@ export class CommandRegistry implements IDisposable, IExtensionSyncActivationSer
                     await this.waitForStatus(
                         async () => {
                             if (uri) {
-                                const notebook = await this.jupyterExporter?.translateToNotebook(cells);
-                                await this.fileSystem.writeFile(uri, JSON.stringify(notebook, undefined, 1));
+                                const notebook = await this.jupyterExporter?.serialize(cells);
+                                await this.fileSystem.writeFile(uri, notebook || '');
                             }
                         },
                         DataScience.exportingFormat,
@@ -672,8 +672,8 @@ export class CommandRegistry implements IDisposable, IExtensionSyncActivationSer
                     await this.waitForStatus(
                         async () => {
                             if (uri) {
-                                const notebook = await this.jupyterExporter?.translateToNotebook(cells);
-                                await this.fileSystem.writeFile(uri, JSON.stringify(notebook, undefined, 1));
+                                const notebook = await this.jupyterExporter?.serialize(cells);
+                                await this.fileSystem.writeFile(uri, notebook || '');
                             }
                         },
                         DataScience.exportingFormat,

src/interactive-window/editor-integration/cellFactory.ts

@@ -1,48 +1,15 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-import type * as nbformat from '@jupyterlab/nbformat';
-import { NotebookCellKind, NotebookDocument, Range, TextDocument, Uri } from 'vscode';
+import { NotebookCellData, NotebookCellKind, NotebookDocument, Range, TextDocument } from 'vscode';
 import { CellMatcher } from './cellMatcher';
-import { ICell, ICellRange, IJupyterSettings } from '../../platform/common/types';
+import { ICellRange, IJupyterSettings } from '../../platform/common/types';
 import { noop } from '../../platform/common/utils/misc';
 import { createJupyterCellFromVSCNotebookCell } from '../../kernels/execution/helpers';
 import { appendLineFeed, parseForComments, generateMarkdownFromCodeLines } from '../../platform/common/utils';
 import { splitLines } from '../../platform/common/helpers';
 import { isSysInfoCell } from '../systemInfoCell';
-
-export function createCodeCell(): nbformat.ICodeCell;
-// eslint-disable-next-line @typescript-eslint/unified-signatures
-export function createCodeCell(code: string): nbformat.ICodeCell;
-export function createCodeCell(code: string[], outputs: nbformat.IOutput[]): nbformat.ICodeCell;
-// eslint-disable-next-line @typescript-eslint/unified-signatures
-export function createCodeCell(code: string[], magicCommandsAsComments: boolean): nbformat.ICodeCell;
-export function createCodeCell(code?: string | string[], options?: boolean | nbformat.IOutput[]): nbformat.ICodeCell {
-    const magicCommandsAsComments = typeof options === 'boolean' ? options : false;
-    const outputs = typeof options === 'boolean' ? [] : options || [];
-    code = code || '';
-    // If we get a string, then no need to append line feeds. Leave as is (to preserve existing functionality).
-    // If we get an array, the append a linefeed.
-    const source = Array.isArray(code)
-        ? appendLineFeed(code, '\n', magicCommandsAsComments ? uncommentMagicCommands : undefined)
-        : code;
-    return {
-        cell_type: 'code',
-        execution_count: null,
-        metadata: {},
-        outputs,
-        source
-    };
-}
-
-export function createMarkdownCell(code: string | string[], useSourceAsIs: boolean = false): nbformat.IMarkdownCell {
-    code = Array.isArray(code) ? code : [code];
-    return {
-        cell_type: 'markdown',
-        metadata: {},
-        source: useSourceAsIs ? code : generateMarkdownFromCodeLines(code)
-    };
-}
+import { getCellMetadata } from '../../platform/common/utils/jupyter';
 
 export function uncommentMagicCommands(line: string): string {
     // Uncomment lines that are shell assignments (starting with #!),
@@ -62,32 +29,23 @@ export function uncommentMagicCommands(line: string): string {
     }
 }
 
-function generateCodeCell(code: string[], uri: Uri | undefined, magicCommandsAsComments: boolean): ICell {
-    // Code cells start out with just source and no outputs.
-    return {
-        data: createCodeCell(code, magicCommandsAsComments),
-        uri
-    };
+function generateCodeCell(code: string[]) {
+    return new NotebookCellData(NotebookCellKind.Code, code.join('\n'), 'python');
 }
 
-function generateMarkdownCell(code: string[], uri: Uri | undefined, useSourceAsIs = false): ICell {
-    return {
-        uri,
-        data: createMarkdownCell(code, useSourceAsIs)
-    };
+function generateMarkdownCell(code: string[]) {
+    return new NotebookCellData(NotebookCellKind.Markup, generateMarkdownFromCodeLines(code).join('\n'), 'markdown');
 }
 
 export function generateCells(
     settings: IJupyterSettings | undefined,
     code: string,
-    uri: Uri | undefined,
     splitMarkdown: boolean
-): ICell[] {
+): NotebookCellData[] {
     // Determine if we have a markdown cell/ markdown and code cell combined/ or just a code cell
     const split = splitLines(code, { trim: false });
     const firstLine = split[0];
     const matcher = new CellMatcher(settings);
-    const { magicCommandsAsComments = false } = settings || {};
     if (matcher.isMarkdown(firstLine)) {
         // We have at least one markdown. We might have to split it if there any lines that don't begin
         // with # or are inside a multiline comment
@@ -105,16 +63,16 @@ export function generateCells(
         if (firstNonMarkdown >= 0) {
             // Make sure if we split, the second cell has a new id. It's a new submission.
             return [
-                generateMarkdownCell(split.slice(0, firstNonMarkdown), uri),
-                generateCodeCell(split.slice(firstNonMarkdown), uri, magicCommandsAsComments)
+                generateMarkdownCell(split.slice(0, firstNonMarkdown)),
+                generateCodeCell(split.slice(firstNonMarkdown))
             ];
         } else {
             // Just a single markdown cell
-            return [generateMarkdownCell(split, uri)];
+            return [generateMarkdownCell(split)];
         }
     } else {
         // Just code
-        return [generateCodeCell(split, uri, magicCommandsAsComments)];
+        return [generateCodeCell(split)];
     }
 }
 
@@ -158,22 +116,22 @@ export function generateCellRangesFromDocument(document: TextDocument, settings?
     return cells;
 }
 
-export function generateCellsFromDocument(document: TextDocument, settings?: IJupyterSettings): ICell[] {
+export function generateCellsFromDocument(document: TextDocument, settings?: IJupyterSettings): NotebookCellData[] {
     const ranges = generateCellRangesFromDocument(document, settings);
 
     // For each one, get its text and turn it into a cell
     return Array.prototype.concat(
         ...ranges.map((cr) => {
             const code = document.getText(cr.range);
-            return generateCells(settings, code, document.uri, false);
+            return generateCells(settings, code, false);
         })
     );
 }
 
 export function generateCellsFromNotebookDocument(
     notebookDocument: NotebookDocument,
     magicCommandsAsComments: boolean
-): ICell[] {
+): NotebookCellData[] {
     return notebookDocument
         .getCells()
         .filter((cell) => !isSysInfoCell(cell))
@@ -188,9 +146,15 @@ export function generateCellsFromNotebookDocument(
                 cell.kind === NotebookCellKind.Code
                     ? appendLineFeed(code, '\n', magicCommandsAsComments ? uncommentMagicCommands : undefined)
                     : appendLineFeed(code);
-            return {
-                data,
-                file: ''
-            };
+            const cellData = new NotebookCellData(
+                cell.kind,
+                code.join('\n'),
+                cell.kind === NotebookCellKind.Code ? cell.document.languageId : 'markdown'
+            );
+            if (cell.kind === NotebookCellKind.Code) {
+                cellData.outputs = [...cell.outputs];
+            }
+            cellData.metadata = { custom: getCellMetadata(cell) };
+            return cellData;
         });
 }

src/interactive-window/editor-integration/cellFactory.unit.test.ts

@@ -2,38 +2,38 @@
 // Licensed under the MIT License.
 
 import { assert } from 'chai';
-import * as vscode from 'vscode';
 import { generateCells } from './cellFactory';
-import { splitLines } from '../../platform/common/helpers';
+import { splitLines as originalSplitLines } from '../../platform/common/helpers';
 import { removeLinesFromFrontAndBack, stripComments } from '../../platform/common/utils';
 
+const splitCode = (s: string) => originalSplitLines(s, { removeEmptyEntries: false, trim: false });
+const splitMarkdown = (s: string) => originalSplitLines(s, { removeEmptyEntries: false, trim: false });
 /* eslint-disable  */
 suite('CellFactory', () => {
     test('parsing cells', () => {
-        const uri = vscode.Uri.parse('file://foo.py');
-        let cells = generateCells(undefined, '#%%\na=1\na', uri, true);
+        let cells = generateCells(undefined, '#%%\na=1\na', true);
         assert.equal(cells.length, 1, 'Simple cell, not right number found');
-        cells = generateCells(undefined, '#%% [markdown]\na=1\na', uri, true);
+        cells = generateCells(undefined, '#%% [markdown]\na=1\na', true);
         assert.equal(cells.length, 2, 'Split cell, not right number found');
-        cells = generateCells(undefined, '#%% [markdown]\n# #a=1\n#a', uri, true);
+        cells = generateCells(undefined, '#%% [markdown]\n# #a=1\n#a', true);
         assert.equal(cells.length, 1, 'Markdown split wrong');
-        assert.equal(cells[0].data.cell_type, 'markdown', 'Markdown cell not generated');
-        cells = generateCells(undefined, "#%% [markdown]\n'''\n# a\nb\n'''", uri, true);
+        assert.equal(cells[0].languageId, 'markdown', 'Markdown cell not generated');
+        cells = generateCells(undefined, "#%% [markdown]\n'''\n# a\nb\n'''", true);
         assert.equal(cells.length, 1, 'Markdown cell multline failed');
-        assert.equal(cells[0].data.cell_type, 'markdown', 'Markdown cell not generated');
-        assert.equal(cells[0].data.source.length, 2, 'Lines for markdown not emitted');
-        cells = generateCells(undefined, '#%% [markdown]\n"""\n# a\nb\n"""', uri, true);
+        assert.equal(cells[0].languageId, 'markdown', 'Markdown cell not generated');
+        assert.equal(splitMarkdown(cells[0].value).length, 3, 'Lines for markdown not emitted');
+        cells = generateCells(undefined, '#%% [markdown]\n"""\n# a\nb\n"""', true);
         assert.equal(cells.length, 1, 'Markdown cell multline failed');
-        assert.equal(cells[0].data.cell_type, 'markdown', 'Markdown cell not generated');
-        assert.equal(cells[0].data.source.length, 2, 'Lines for markdown not emitted');
-        cells = generateCells(undefined, '#%% \n"""\n# a\nb\n"""', uri, true);
+        assert.equal(cells[0].languageId, 'markdown', 'Markdown cell not generated');
+        assert.equal(splitMarkdown(cells[0].value).length, 3, 'Lines for markdown not emitted');
+        cells = generateCells(undefined, '#%% \n"""\n# a\nb\n"""', true);
         assert.equal(cells.length, 1, 'Code cell multline failed');
-        assert.equal(cells[0].data.cell_type, 'code', 'Code cell not generated');
-        assert.equal(cells[0].data.source.length, 5, 'Lines for cell not emitted');
-        cells = generateCells(undefined, '#%% [markdown] \n"""# a\nb\n"""', uri, true);
+        assert.equal(cells[0].languageId, 'python', 'Code cell not generated');
+        assert.equal(splitCode(cells[0].value).length, 5, 'Lines for cell not emitted');
+        cells = generateCells(undefined, '#%% [markdown] \n"""# a\nb\n"""', true);
         assert.equal(cells.length, 1, 'Markdown cell multline failed');
-        assert.equal(cells[0].data.cell_type, 'markdown', 'Markdown cell not generated');
-        assert.equal(cells[0].data.source.length, 2, 'Lines for cell not emitted');
+        assert.equal(cells[0].languageId, 'markdown', 'Markdown cell not generated');
+        assert.equal(splitMarkdown(cells[0].value).length, 3, 'Lines for cell not emitted');
 
         // eslint-disable-next-line no-multi-str
         const multilineCode = `#%%
@@ -58,16 +58,14 @@ Suspendisse ornare interdum velit. Suspendisse potenti.
 Morbi molestie lacinia sapien nec porttitor. Nam at vestibulum nisi.
 """ print('bob')`;
 
-        cells = generateCells(undefined, multilineCode, uri, true);
+        cells = generateCells(undefined, multilineCode, true);
         assert.equal(cells.length, 1, 'code cell multline failed');
-        assert.equal(cells[0].data.cell_type, 'code', 'Code cell not generated');
-        assert.equal(cells[0].data.source.length, 10, 'Lines for cell not emitted');
-        cells = generateCells(undefined, multilineTwo, uri, true);
+        assert.equal(cells[0].languageId, 'python', 'Code cell not generated');
+        assert.equal(splitCode(cells[0].value).length, 10, 'Lines for cell not emitted');
+        cells = generateCells(undefined, multilineTwo, true);
         assert.equal(cells.length, 1, 'code cell multline failed');
-        assert.equal(cells[0].data.cell_type, 'code', 'Code cell not generated');
-        assert.equal(cells[0].data.source.length, 10, 'Lines for cell not emitted');
-        // eslint-disable-next-line no-multi-str
-        assert.equal(cells[0].data.source[9], `""" print('bob')`, 'Lines for cell not emitted');
+        assert.equal(cells[0].languageId, 'python', 'Code cell not generated');
+        assert.equal(splitCode(cells[0].value).length, 10, 'Lines for cell not emitted');
         // eslint-disable-next-line no-multi-str
         const multilineMarkdown = `#%% [markdown]
 # ## Block of Interest
@@ -90,11 +88,12 @@ Morbi molestie lacinia sapien nec porttitor. Nam at vestibulum nisi.
 #          - Item 1-a-3-c
 #
 #   2. Item 2`;
-        cells = generateCells(undefined, multilineMarkdown, uri, true);
+        cells = generateCells(undefined, multilineMarkdown, true);
         assert.equal(cells.length, 1, 'markdown cell multline failed');
-        assert.equal(cells[0].data.cell_type, 'markdown', 'markdown cell not generated');
-        assert.equal(cells[0].data.source.length, 20, 'Lines for cell not emitted');
-        assert.equal(cells[0].data.source[17], '          - Item 1-a-3-c\n', 'Lines for markdown not emitted');
+        assert.equal(cells[0].languageId, 'markdown', 'markdown cell not generated');
+        assert.equal(splitMarkdown(cells[0].value).length, 39, 'Lines for cell not emitted');
+        console.error(`"${cells[0].value}"`);
+        assert.equal(splitMarkdown(cells[0].value)[34], '          - Item 1-a-3-c', 'Lines for markdown not emitted');
 
         // eslint-disable-next-line no-multi-str
         const multilineQuoteWithOtherDelimiter = `#%% [markdown]
@@ -104,11 +103,11 @@ Morbi molestie lacinia sapien nec porttitor. Nam at vestibulum nisi.
 """ Not a comment delimiter
 '''
 `;
-        cells = generateCells(undefined, multilineQuoteWithOtherDelimiter, uri, true);
+        cells = generateCells(undefined, multilineQuoteWithOtherDelimiter, true);
         assert.equal(cells.length, 1, 'markdown cell multline failed');
-        assert.equal(cells[0].data.cell_type, 'markdown', 'markdown cell not generated');
-        assert.equal(cells[0].data.source.length, 3, 'Lines for cell not emitted');
-        assert.equal(cells[0].data.source[2], '""" Not a comment delimiter', 'Lines for markdown not emitted');
+        assert.equal(cells[0].languageId, 'markdown', 'markdown cell not generated');
+        assert.equal(splitCode(cells[0].value).length, 5, 'Lines for cell not emitted');
+        assert.equal(splitCode(cells[0].value)[4], '""" Not a comment delimiter', 'Lines for markdown not emitted');
 
         // eslint-disable-next-line no-multi-str
         const multilineQuoteInFunc = `#%%
@@ -120,13 +119,13 @@ def download(url, filename):
         for data in response.iter_content():
             handle.write(data)
 `;
-        cells = generateCells(undefined, multilineQuoteInFunc, uri, true);
+        cells = generateCells(undefined, multilineQuoteInFunc, true);
         assert.equal(cells.length, 1, 'cell multline failed');
-        assert.equal(cells[0].data.cell_type, 'code', 'code cell not generated');
-        assert.equal(cells[0].data.source.length, 9, 'Lines for cell not emitted');
+        assert.equal(cells[0].languageId, 'python', 'code cell not generated');
+        assert.equal(splitCode(cells[0].value).length, 9, 'Lines for cell not emitted');
         assert.equal(
-            cells[0].data.source[3],
-            '    """ utility function to download a file """\n',
+            splitCode(cells[0].value)[3],
+            '    """ utility function to download a file """',
             'Lines for cell not emitted'
         );
 
@@ -141,21 +140,21 @@ class Pizza(object):
         self.rating = rating
         `;
 
-        cells = generateCells(undefined, multilineMarkdownWithCell, uri, true);
+        cells = generateCells(undefined, multilineMarkdownWithCell, true);
         assert.equal(cells.length, 2, 'cell split failed');
-        assert.equal(cells[0].data.cell_type, 'markdown', 'markdown cell not generated');
-        assert.equal(cells[0].data.source.length, 1, 'Lines for markdown not emitted');
-        assert.equal(cells[1].data.cell_type, 'code', 'code cell not generated');
-        assert.equal(cells[1].data.source.length, 7, 'Lines for code not emitted');
-        assert.equal(cells[1].data.source[3], '        self.toppings = toppings\n', 'Lines for cell not emitted');
+        assert.equal(cells[0].languageId, 'markdown', 'markdown cell not generated');
+        assert.equal(splitCode(cells[0].value).length, 1, 'Lines for markdown not emitted');
+        assert.equal(cells[1].languageId, 'python', 'code cell not generated');
+        assert.equal(splitCode(cells[1].value).length, 7, 'Lines for code not emitted');
+        assert.equal(splitCode(cells[1].value)[3], '        self.toppings = toppings', 'Lines for cell not emitted');
 
         // Non comments tests
         let nonComments = stripComments(multilineCode);
         assert.ok(nonComments.startsWith('myvar = """ # Lorem Ipsum'), 'Variable set to multiline string not working');
         nonComments = stripComments(multilineTwo);
         assert.equal(nonComments, '', 'Multline comment is not being stripped');
         nonComments = stripComments(multilineQuoteInFunc);
-        assert.equal(splitLines(nonComments).length, 6, 'Splitting quote in func wrong number of lines');
+        assert.equal(splitCode(nonComments).length, 8, 'Splitting quote in func wrong number of lines');
     });
 
     test('Line removal', () => {

src/interactive-window/types.ts

@@ -13,7 +13,7 @@ import {
 } from 'vscode';
 import { IDebuggingManager } from '../notebooks/debugger/debuggingTypes';
 import { IKernel, KernelConnectionMetadata } from '../kernels/types';
-import { Resource, InteractiveWindowMode, ICell } from '../platform/common/types';
+import { Resource, InteractiveWindowMode } from '../platform/common/types';
 import { IFileGeneratedCodes } from './editor-integration/types';
 import { IVSCodeNotebookController } from '../notebooks/controllers/types';
 
@@ -102,8 +102,8 @@ export interface IInteractiveWindow extends IInteractiveBase {
     expandAllCells(): Promise<void>;
     collapseAllCells(): Promise<void>;
     scrollToCell(id: string): void;
-    exportAs(cells?: ICell[]): void;
-    export(cells?: ICell[]): void;
+    exportAs(): void;
+    export(): void;
 }
 
 export interface IInteractiveWindowCache {