Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

custom in metadata is optional based on ipynb ext #15398

Merged
merged 19 commits into from
Mar 20, 2024
Merged

custom in metadata is optional based on ipynb ext #15398

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
4edba53
custom in metadata is optional based on ipynb ext
DonJayamanne Mar 20, 2024
bb6bd7c
Misc changes
DonJayamanne Mar 20, 2024
d3b25e0
Misc
DonJayamanne Mar 20, 2024
f17243b
Fixces
DonJayamanne Mar 20, 2024
4806ff1
Misc
DonJayamanne Mar 20, 2024
c426f54
misc
DonJayamanne Mar 20, 2024
b7eb932
More logging
DonJayamanne Mar 20, 2024
79cab66
misc
DonJayamanne Mar 20, 2024
c6a52c8
test again
DonJayamanne Mar 20, 2024
5933f5f
even more logging
DonJayamanne Mar 20, 2024
a0bbf30
more logging
DonJayamanne Mar 20, 2024
9ebeb3d
misc
DonJayamanne Mar 20, 2024
69a5bc7
misc
DonJayamanne Mar 20, 2024
a5a0bf4
misc
DonJayamanne Mar 20, 2024
6c05f6a
misc
DonJayamanne Mar 20, 2024
5ba2408
test
DonJayamanne Mar 20, 2024
6154301
fix tests
DonJayamanne Mar 20, 2024
4f3efaf
Misc
DonJayamanne Mar 20, 2024
37f3a34
always disable custom metadata on ci
DonJayamanne Mar 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/interactive-window/editor-integration/cellFactory.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import { NotebookCellData, NotebookCellKind, NotebookDocument, Range, TextDocume
import { CellMatcher } from './cellMatcher';
import { ICellRange, IJupyterSettings } from '../../platform/common/types';
import { noop } from '../../platform/common/utils/misc';
import { parseForComments, generateMarkdownFromCodeLines } from '../../platform/common/utils';
import { parseForComments, generateMarkdownFromCodeLines, useCustomMetadata } from '../../platform/common/utils';
import { splitLines } from '../../platform/common/helpers';
import { isSysInfoCell } from '../systemInfoCell';
import { getCellMetadata } from '../../platform/common/utils';
Expand Down Expand Up @@ -145,7 +145,7 @@ export function generateCellsFromNotebookDocument(notebookDocument: NotebookDocu
if (cell.kind === NotebookCellKind.Code) {
cellData.outputs = [...cell.outputs];
}
cellData.metadata = { custom: getCellMetadata(cell) };
cellData.metadata = useCustomMetadata() ? { custom: getCellMetadata(cell) } : getCellMetadata(cell);
return cellData;
});
}
1,713 changes: 868 additions & 845 deletions src/kernels/execution/cellExecutionMessageHandler.unit.test.ts
View file
Open in desktop

Large diffs are not rendered by default.

549 changes: 290 additions & 259 deletions src/notebooks/controllers/preferredKernelConnectionService.unit.test.ts
View file
Open in desktop

Large diffs are not rendered by default.

137 changes: 92 additions & 45 deletions src/platform/common/utils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,9 @@ import {
TextDocument,
Uri,
WorkspaceEdit,
extensions,
workspace,
type NotebookCell,
type NotebookCellData
type NotebookCell
} from 'vscode';
import {
InteractiveWindowView,
Expand All @@ -22,6 +22,7 @@ import {
WIDGET_STATE_MIMETYPE
} from './constants';
import { splitLines } from './helpers';
import { noop } from './utils/misc';

// Can't figure out a better way to do this. Enumerate
// the allowed keys of different output formats.
Expand Down Expand Up @@ -168,47 +169,71 @@ export type NotebookMetadata = nbformat.INotebookMetadata & {
};

export function getNotebookMetadata(document: NotebookDocument | NotebookData): NotebookMetadata | undefined {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const notebookContent: undefined | Partial<nbformat.INotebookContent> = document.metadata?.custom as any;
// Create a clone.
return JSON.parse(JSON.stringify(notebookContent?.metadata || {}));
if (useCustomMetadata()) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const notebookContent: undefined | Partial<nbformat.INotebookContent> = document.metadata?.custom as any;
// Create a clone.
return JSON.parse(JSON.stringify(notebookContent?.metadata || {}));
} else {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const notebookContent: undefined | Partial<nbformat.INotebookContent> = document.metadata as any;
// Create a clone.
return JSON.parse(JSON.stringify(notebookContent?.metadata || {}));
}
}

export function getNotebookFormat(document: NotebookDocument): {
nbformat: number | undefined;
nbformat_minor: number | undefined;
} {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const notebookContent: undefined | Partial<nbformat.INotebookContent> = document.metadata?.custom as any;
// Create a clone.
return {
nbformat: notebookContent?.nbformat,
nbformat_minor: notebookContent?.nbformat_minor
};
if (useCustomMetadata()) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const notebookContent: undefined | Partial<nbformat.INotebookContent> = document.metadata?.custom as any;
// Create a clone.
return {
nbformat: notebookContent?.nbformat,
nbformat_minor: notebookContent?.nbformat_minor
};
} else {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const notebookContent: undefined | Partial<nbformat.INotebookContent> = document.metadata as any;
// Create a clone.
return {
nbformat: notebookContent?.nbformat,
nbformat_minor: notebookContent?.nbformat_minor
};
}
}

export async function updateNotebookMetadata(document: NotebookDocument, metadata: NotebookMetadata) {
const edit = new WorkspaceEdit();
// Create a clone.
const docMetadata = JSON.parse(
JSON.stringify(
(document.metadata as {
custom?: Exclude<Partial<nbformat.INotebookContent>, 'cells'>;
}) || { custom: {} }
)
);

docMetadata.custom = docMetadata.custom || {};
docMetadata.custom.metadata = metadata;

edit.set(document.uri, [
NotebookEdit.updateNotebookMetadata(
sortObjectPropertiesRecursively({
...(document.metadata || {}),
custom: docMetadata.custom
})
)
]);
if (useCustomMetadata()) {
// Create a clone.
const docMetadata: {
custom?: Exclude<Partial<nbformat.INotebookContent>, 'cells'>;
} = JSON.parse(JSON.stringify(document.metadata || { custom: {} }));

docMetadata.custom = docMetadata.custom || {};
docMetadata.custom.metadata = metadata;

edit.set(document.uri, [
NotebookEdit.updateNotebookMetadata(
sortObjectPropertiesRecursively({
...(document.metadata || {}),
custom: docMetadata.custom
})
)
]);
} else {
edit.set(document.uri, [
NotebookEdit.updateNotebookMetadata(
sortObjectPropertiesRecursively({
...(document.metadata || {}),
metadata
})
)
]);
}
await workspace.applyEdit(edit);
}

Expand Down Expand Up @@ -430,21 +455,43 @@ type JupyterCellMetadata = Pick<nbformat.IRawCell, 'id' | 'metadata' | 'attachme
// eslint-disable-next-line @typescript-eslint/no-explicit-any
Record<string, any>;

export function getCellMetadata(cell: NotebookCell | NotebookCellData): JupyterCellMetadata {
const metadata: JupyterCellMetadata = cell.metadata?.custom || {};
const cellMetadata = metadata as nbformat.IRawCell;
// metadata property is never optional.
cellMetadata.metadata = cellMetadata.metadata || {};
export function getCellMetadata(cell: NotebookCell): JupyterCellMetadata {
if (useCustomMetadata()) {
const metadata: JupyterCellMetadata = cell.metadata.custom || {};
const cellMetadata = metadata as nbformat.IRawCell;
// metadata property is never optional.
cellMetadata.metadata = cellMetadata.metadata || {};

return metadata;
return metadata;
} else {
const metadata: JupyterCellMetadata = cell.metadata.metadata || {};
// metadata property is never optional.
metadata.metadata = metadata.metadata || {};
return metadata;
}
}

// function useCustomMetadata() {
// if (extensions.getExtension('vscode.ipynb')?.exports.dropCustomMetadata) {
// return false;
// }
// return true;
// }
export function useCustomMetadata() {
const ext = extensions.getExtension<{ dropCustomMetadata: boolean }>('vscode.ipynb');
if (ext && typeof ext.exports.dropCustomMetadata === 'boolean') {
return ext.exports.dropCustomMetadata ? false : true;
}
try {
// Means ipynb extension has not yet been activated.
// Does not matter, we can just check the setting.
return !workspace.getConfiguration('jupyter', undefined).get<boolean>('experimental.dropCustomMetadata', false);
} catch {
// This happens in unit tests, in this case just return `true`.
return true;
}
}

export async function activateIPynbExtension() {
const ext = extensions.getExtension<{ dropCustomMetadata: boolean }>('vscode.ipynb');
if (ext && ext.isActive === false) {
await ext.activate().then(noop, noop);
}
}

/**
* Sort the JSON to minimize unnecessary SCM changes.
Expand Down
84 changes: 55 additions & 29 deletions src/standalone/executionAnalysis/symbols.vscode.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import { anything, instance, mock, when } from 'ts-mockito';
import { CellAnalysis, ICellExecution, ILocationWithReferenceKind, NotebookDocumentSymbolTracker } from './symbols';
import { PylanceExtension } from './common';
import { activatePylance } from './pylance';
import { useCustomMetadata } from '../../platform/common/utils';

function withNotebookCells(data: [string, string][], fileName: string) {
const cells: vscode.NotebookCell[] = data.map((cellDto) => {
Expand Down Expand Up @@ -494,18 +495,27 @@ function closeAllEditors(): Thenable<any> {

(vscode.extensions.getExtension(PylanceExtension) ? suite : suite.skip)('Cell Analysis - Pylance', () => {
test('Advanced type dependencies', async () => {
const document = await vscode.workspace.openNotebookDocument(
'jupyter-notebook',
new vscode.NotebookData([
new vscode.NotebookCellData(vscode.NotebookCellKind.Code, 'import pandas as pd', 'python'),
new vscode.NotebookCellData(vscode.NotebookCellKind.Code, 'df = pd.DataFrame()', 'python'),
new vscode.NotebookCellData(vscode.NotebookCellKind.Code, 'mylist = [1, 2, 3, 4]', 'python'),
new vscode.NotebookCellData(vscode.NotebookCellKind.Code, 'mylist2 = [2, 3, 4, 5]', 'python'),
new vscode.NotebookCellData(vscode.NotebookCellKind.Code, 'print(mylist)', 'python')
])
);
const nb = new vscode.NotebookData([
new vscode.NotebookCellData(vscode.NotebookCellKind.Code, 'import pandas as pd', 'python'),
new vscode.NotebookCellData(vscode.NotebookCellKind.Code, 'df = pd.DataFrame()', 'python'),
new vscode.NotebookCellData(vscode.NotebookCellKind.Code, 'mylist = [1, 2, 3, 4]', 'python'),
new vscode.NotebookCellData(vscode.NotebookCellKind.Code, 'mylist2 = [2, 3, 4, 5]', 'python'),
new vscode.NotebookCellData(vscode.NotebookCellKind.Code, 'print(mylist)', 'python')
]);

// Temporary, until Pylance is fixed
if (!useCustomMetadata()) {
nb.metadata = {
custom: {
metadata: {
cellLanguage: 'python'
}
}
};
}
const document = await vscode.workspace.openNotebookDocument('jupyter-notebook', nb);

const editor = await vscode.window.showNotebookDocument(document);
const editor = await await vscode.window.showNotebookDocument(document);
const referencesProvider = await activatePylance();
if (!referencesProvider) {
assert.fail('Pylance not found');
Expand Down Expand Up @@ -540,15 +550,23 @@ function closeAllEditors(): Thenable<any> {
});

test('Advanced type dependencies 2', async () => {
const document = await vscode.workspace.openNotebookDocument(
'jupyter-notebook',
new vscode.NotebookData([
new vscode.NotebookCellData(vscode.NotebookCellKind.Code, 'import numpy as np', 'python'),
new vscode.NotebookCellData(vscode.NotebookCellKind.Code, 'arr = np.array([1, 2, 3, 4])', 'python'),
new vscode.NotebookCellData(vscode.NotebookCellKind.Code, 'arr2 = np.array([2, 3, 4, 5])', 'python'),
new vscode.NotebookCellData(vscode.NotebookCellKind.Code, 'print(arr)', 'python')
])
);
const nb = new vscode.NotebookData([
new vscode.NotebookCellData(vscode.NotebookCellKind.Code, 'import numpy as np', 'python'),
new vscode.NotebookCellData(vscode.NotebookCellKind.Code, 'arr = np.array([1, 2, 3, 4])', 'python'),
new vscode.NotebookCellData(vscode.NotebookCellKind.Code, 'arr2 = np.array([2, 3, 4, 5])', 'python'),
new vscode.NotebookCellData(vscode.NotebookCellKind.Code, 'print(arr)', 'python')
]);
// Temporary, until Pylance is fixed
if (!useCustomMetadata()) {
nb.metadata = {
custom: {
metadata: {
cellLanguage: 'python'
}
}
};
}
const document = await vscode.workspace.openNotebookDocument('jupyter-notebook', nb);
const editor = await vscode.window.showNotebookDocument(document);
const referencesProvider = await activatePylance();
if (!referencesProvider) {
Expand Down Expand Up @@ -584,15 +602,23 @@ function closeAllEditors(): Thenable<any> {
});

test('Advanced type dependencies 3', async () => {
const document = await vscode.workspace.openNotebookDocument(
'jupyter-notebook',
new vscode.NotebookData([
new vscode.NotebookCellData(vscode.NotebookCellKind.Code, 'import matplotlib.pyplot as plt', 'python'),
new vscode.NotebookCellData(vscode.NotebookCellKind.Code, 'x = [1, 2, 3, 4]', 'python'),
new vscode.NotebookCellData(vscode.NotebookCellKind.Code, 'y = [2, 3, 4, 5]', 'python'),
new vscode.NotebookCellData(vscode.NotebookCellKind.Code, 'plt.plot(x, y)', 'python')
])
);
const nb = new vscode.NotebookData([
new vscode.NotebookCellData(vscode.NotebookCellKind.Code, 'import matplotlib.pyplot as plt', 'python'),
new vscode.NotebookCellData(vscode.NotebookCellKind.Code, 'x = [1, 2, 3, 4]', 'python'),
new vscode.NotebookCellData(vscode.NotebookCellKind.Code, 'y = [2, 3, 4, 5]', 'python'),
new vscode.NotebookCellData(vscode.NotebookCellKind.Code, 'plt.plot(x, y)', 'python')
]);
// Temporary, until Pylance is fixed
if (!useCustomMetadata()) {
nb.metadata = {
custom: {
metadata: {
cellLanguage: 'python'
}
}
};
}
const document = await vscode.workspace.openNotebookDocument('jupyter-notebook', nb);
const editor = await vscode.window.showNotebookDocument(document);
const referencesProvider = await activatePylance();
if (!referencesProvider) {
Expand Down
Loading
Loading