From 1f76cbfda85dcfaf02ba995d4fb10479297b1d92 Mon Sep 17 00:00:00 2001 From: James Boyden Date: Thu, 16 Jan 2025 15:30:37 +0000 Subject: [PATCH 01/13] Escape markdown -> rich text. TODO: other way --- src/rich-text/schema.ts | 6 ++++ src/shared/markdown-it/preserve-escape.ts | 34 +++++++++++++++++++++++ src/shared/markdown-parser.ts | 9 ++++++ src/shared/markdown-serializer.ts | 6 ++++ 4 files changed, 55 insertions(+) create mode 100644 src/shared/markdown-it/preserve-escape.ts diff --git a/src/rich-text/schema.ts b/src/rich-text/schema.ts index cbf52063..49929368 100644 --- a/src/rich-text/schema.ts +++ b/src/rich-text/schema.ts @@ -518,6 +518,12 @@ const marks: { sup: genHtmlInlineMarkSpec({}, "sup"), sub: genHtmlInlineMarkSpec({}, "sub"), + + escape: { + toDOM() { + return ["span"]; + } + } }; // for *every* mark, add in support for the `markup` attribute diff --git a/src/shared/markdown-it/preserve-escape.ts b/src/shared/markdown-it/preserve-escape.ts new file mode 100644 index 00000000..ae90f32e --- /dev/null +++ b/src/shared/markdown-it/preserve-escape.ts @@ -0,0 +1,34 @@ +import MarkdownIt, {StateInline} from "markdown-it"; + +function buildPreserveEscapeFn(md: MarkdownIt): MarkdownIt.ParserInline.RuleInline { + const [escapeFn] = md.inline.ruler.getRules('') + .filter(r => r.name === "escape"); + + const noop = (): boolean => false; + //The "escape" rule has been disabled or otherwise removed; so there's nothing to replace here. + if(escapeFn.length === 0){ + return noop; + } + return function preserveEscapeFn(state: StateInline, silent: boolean): boolean { + const escRet = escapeFn(state, silent); + + //If the rule did nothing (returned false or is running in silent mode) there's nothing to fix + if(silent || escRet === false) return escRet; + + //The escape rule, if executed, always adds a 'text_special' node to the end, and we're going to work on that. + const [escapeToken] = state.tokens.slice(-1); + + //Now we want to retag the type so that + // - the escape token is ignored by the text_merge + // - We can enact custom rendering later + escapeToken.type = 'escape'; + console.log(escapeToken); + + return escRet; + } +} + +export function preserve_escape(md: MarkdownIt): void { + const preserveEscapeTokens = buildPreserveEscapeFn(md); + md.inline.ruler.at("escape", preserveEscapeTokens); +} diff --git a/src/shared/markdown-parser.ts b/src/shared/markdown-parser.ts index 15b26e92..051411c2 100644 --- a/src/shared/markdown-parser.ts +++ b/src/shared/markdown-parser.ts @@ -11,6 +11,7 @@ import { stackLanguageComments } from "./markdown-it/stack-language-comments"; import { tagLinks } from "./markdown-it/tag-link"; import { tight_list } from "./markdown-it/tight-list"; import type { CommonmarkParserFeatures } from "./view"; +import {preserve_escape} from "./markdown-it/preserve-escape"; // extend the default markdown parser's tokens and add our own const customMarkdownParserTokens: MarkdownParser["tokens"] = { @@ -150,6 +151,11 @@ const customMarkdownParserTokens: MarkdownParser["tokens"] = { code_inline_split: { mark: "code", }, + + escape : { + mark: "escape", + noCloseToken: true + } }; // add tag attribute support to all the marks like we did in schema @@ -320,6 +326,9 @@ export function createDefaultMarkdownItInstance( // ensure we can tell the difference between the different types of hardbreaks defaultMarkdownItInstance.use(hardbreak_markup); + // ensure that we preserve escape characters inside some block contexts + defaultMarkdownItInstance.use(preserve_escape); + // TODO should always exist, so remove the check once the param is made non-optional externalPluginProvider?.alterMarkdownIt(defaultMarkdownItInstance); diff --git a/src/shared/markdown-serializer.ts b/src/shared/markdown-serializer.ts index 1c42cff3..568dd07d 100644 --- a/src/shared/markdown-serializer.ts +++ b/src/shared/markdown-serializer.ts @@ -670,6 +670,12 @@ const customMarkdownSerializerMarks: MarkdownSerializerMarks = { mixable: true, expelEnclosingWhitespace: true, }), + escape: { + open: '\\', + close: '', + mixable: false, + escape: false,// That's what we're doing! + } }; // export our custom serializer using the extended nodes/marks taken from the default schema From dbefff60275d525c5d8a12ab937db70b3cd787ad Mon Sep 17 00:00:00 2001 From: James Boyden Date: Thu, 16 Jan 2025 16:37:38 +0000 Subject: [PATCH 02/13] Catch any existing `escape` blocks, or catch and escape on parse --- src/rich-text/schema.ts | 5 ++++- src/shared/markdown-serializer.ts | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/rich-text/schema.ts b/src/rich-text/schema.ts index 49929368..51aa866c 100644 --- a/src/rich-text/schema.ts +++ b/src/rich-text/schema.ts @@ -520,8 +520,11 @@ const marks: { sub: genHtmlInlineMarkSpec({}, "sub"), escape: { + parseDOM: [ + { style: "span.escaped" } + ], toDOM() { - return ["span"]; + return ["span", { class: "escaped" }]; } } }; diff --git a/src/shared/markdown-serializer.ts b/src/shared/markdown-serializer.ts index 568dd07d..d41a7382 100644 --- a/src/shared/markdown-serializer.ts +++ b/src/shared/markdown-serializer.ts @@ -317,7 +317,7 @@ const defaultMarkdownSerializerNodes: MarkdownSerializerNodes = { } } }, - text(state, node) { + text(state, node, parent) { const linkMark = node.marks.find( (m) => m.type === m.type.schema.marks.link ); @@ -333,7 +333,7 @@ const defaultMarkdownSerializerNodes: MarkdownSerializerNodes = { // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call - const startOfLine: boolean = state.atBlank() || state.closed; + const startOfLine: boolean = state.atBlank() || state.atBlockStart || state.closed; // escape the text using the built in escape code let escapedText = state.esc(node.text, startOfLine); From 9ba6f9a1e27318e710659baa3b06e9686e61f125 Mon Sep 17 00:00:00 2001 From: James Boyden Date: Thu, 16 Jan 2025 17:25:22 +0000 Subject: [PATCH 03/13] chore(lint): cleaning up vestigial dev nonsense --- src/shared/markdown-it/preserve-escape.ts | 3 +-- src/shared/markdown-serializer.ts | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/shared/markdown-it/preserve-escape.ts b/src/shared/markdown-it/preserve-escape.ts index ae90f32e..095cf1b3 100644 --- a/src/shared/markdown-it/preserve-escape.ts +++ b/src/shared/markdown-it/preserve-escape.ts @@ -21,8 +21,7 @@ function buildPreserveEscapeFn(md: MarkdownIt): MarkdownIt.ParserInline.RuleInli //Now we want to retag the type so that // - the escape token is ignored by the text_merge // - We can enact custom rendering later - escapeToken.type = 'escape'; - console.log(escapeToken); + escapeToken.type = 'escape' return escRet; } diff --git a/src/shared/markdown-serializer.ts b/src/shared/markdown-serializer.ts index d41a7382..6972742f 100644 --- a/src/shared/markdown-serializer.ts +++ b/src/shared/markdown-serializer.ts @@ -317,7 +317,7 @@ const defaultMarkdownSerializerNodes: MarkdownSerializerNodes = { } } }, - text(state, node, parent) { + text(state, node) { const linkMark = node.marks.find( (m) => m.type === m.type.schema.marks.link ); From 8c7a3e02b034596fe20bb685c2236ae15728c152 Mon Sep 17 00:00:00 2001 From: Dan Cormier Date: Fri, 17 Jan 2025 16:33:23 -0500 Subject: [PATCH 04/13] Apply formatting --- src/rich-text/schema.ts | 8 +++----- src/shared/markdown-it/preserve-escape.ts | 24 ++++++++++++++--------- src/shared/markdown-parser.ts | 8 ++++---- src/shared/markdown-serializer.ts | 19 ++++++++++-------- 4 files changed, 33 insertions(+), 26 deletions(-) diff --git a/src/rich-text/schema.ts b/src/rich-text/schema.ts index 51aa866c..23ac2679 100644 --- a/src/rich-text/schema.ts +++ b/src/rich-text/schema.ts @@ -520,13 +520,11 @@ const marks: { sub: genHtmlInlineMarkSpec({}, "sub"), escape: { - parseDOM: [ - { style: "span.escaped" } - ], + parseDOM: [{ style: "span.escaped" }], toDOM() { return ["span", { class: "escaped" }]; - } - } + }, + }, }; // for *every* mark, add in support for the `markup` attribute diff --git a/src/shared/markdown-it/preserve-escape.ts b/src/shared/markdown-it/preserve-escape.ts index 095cf1b3..eec33d04 100644 --- a/src/shared/markdown-it/preserve-escape.ts +++ b/src/shared/markdown-it/preserve-escape.ts @@ -1,19 +1,25 @@ -import MarkdownIt, {StateInline} from "markdown-it"; +import MarkdownIt, { StateInline } from "markdown-it"; -function buildPreserveEscapeFn(md: MarkdownIt): MarkdownIt.ParserInline.RuleInline { - const [escapeFn] = md.inline.ruler.getRules('') - .filter(r => r.name === "escape"); +function buildPreserveEscapeFn( + md: MarkdownIt +): MarkdownIt.ParserInline.RuleInline { + const [escapeFn] = md.inline.ruler + .getRules("") + .filter((r) => r.name === "escape"); const noop = (): boolean => false; //The "escape" rule has been disabled or otherwise removed; so there's nothing to replace here. - if(escapeFn.length === 0){ + if (escapeFn.length === 0) { return noop; } - return function preserveEscapeFn(state: StateInline, silent: boolean): boolean { + return function preserveEscapeFn( + state: StateInline, + silent: boolean + ): boolean { const escRet = escapeFn(state, silent); //If the rule did nothing (returned false or is running in silent mode) there's nothing to fix - if(silent || escRet === false) return escRet; + if (silent || escRet === false) return escRet; //The escape rule, if executed, always adds a 'text_special' node to the end, and we're going to work on that. const [escapeToken] = state.tokens.slice(-1); @@ -21,10 +27,10 @@ function buildPreserveEscapeFn(md: MarkdownIt): MarkdownIt.ParserInline.RuleInli //Now we want to retag the type so that // - the escape token is ignored by the text_merge // - We can enact custom rendering later - escapeToken.type = 'escape' + escapeToken.type = "escape"; return escRet; - } + }; } export function preserve_escape(md: MarkdownIt): void { diff --git a/src/shared/markdown-parser.ts b/src/shared/markdown-parser.ts index 051411c2..11fdd2f0 100644 --- a/src/shared/markdown-parser.ts +++ b/src/shared/markdown-parser.ts @@ -11,7 +11,7 @@ import { stackLanguageComments } from "./markdown-it/stack-language-comments"; import { tagLinks } from "./markdown-it/tag-link"; import { tight_list } from "./markdown-it/tight-list"; import type { CommonmarkParserFeatures } from "./view"; -import {preserve_escape} from "./markdown-it/preserve-escape"; +import { preserve_escape } from "./markdown-it/preserve-escape"; // extend the default markdown parser's tokens and add our own const customMarkdownParserTokens: MarkdownParser["tokens"] = { @@ -152,10 +152,10 @@ const customMarkdownParserTokens: MarkdownParser["tokens"] = { mark: "code", }, - escape : { + escape: { mark: "escape", - noCloseToken: true - } + noCloseToken: true, + }, }; // add tag attribute support to all the marks like we did in schema diff --git a/src/shared/markdown-serializer.ts b/src/shared/markdown-serializer.ts index 6972742f..df4e9dd4 100644 --- a/src/shared/markdown-serializer.ts +++ b/src/shared/markdown-serializer.ts @@ -330,10 +330,13 @@ const defaultMarkdownSerializerNodes: MarkdownSerializerNodes = { ) { text = linkMark.attrs.href as string; } else { - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-expect-error - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call - const startOfLine: boolean = state.atBlank() || state.atBlockStart || state.closed; + /* eslint-disable @typescript-eslint/ban-ts-comment, @typescript-eslint/no-unsafe-assignment */ + const startOfLine: boolean = + // @ts-expect-error + // eslint-disable-next-line + state.atBlank() || state.atBlockStart || state.closed; + /* eslint-enable @typescript-eslint/ban-ts-comment, @typescript-eslint/no-unsafe-assignment */ + // escape the text using the built in escape code let escapedText = state.esc(node.text, startOfLine); @@ -671,11 +674,11 @@ const customMarkdownSerializerMarks: MarkdownSerializerMarks = { expelEnclosingWhitespace: true, }), escape: { - open: '\\', - close: '', + open: "\\", + close: "", mixable: false, - escape: false,// That's what we're doing! - } + escape: false, // That's what we're doing! + }, }; // export our custom serializer using the extended nodes/marks taken from the default schema From 5942fc40ddd892be5eeb4cc3b62d3811443e52d3 Mon Sep 17 00:00:00 2001 From: James Boyden Date: Mon, 3 Feb 2025 14:59:37 +0000 Subject: [PATCH 05/13] Escape tests, roundtrip tests --- src/rich-text/schema.ts | 2 +- src/shared/markdown-it/preserve-escape.ts | 2 +- .../markdown-it/preserve-escape.test.ts | 52 +++++++++++++++++ test/shared/roundtrip.e2e.test.ts | 56 +++++++++++++++++++ 4 files changed, 110 insertions(+), 2 deletions(-) create mode 100644 test/shared/markdown-it/preserve-escape.test.ts create mode 100644 test/shared/roundtrip.e2e.test.ts diff --git a/src/rich-text/schema.ts b/src/rich-text/schema.ts index 23ac2679..61cfd14c 100644 --- a/src/rich-text/schema.ts +++ b/src/rich-text/schema.ts @@ -520,7 +520,7 @@ const marks: { sub: genHtmlInlineMarkSpec({}, "sub"), escape: { - parseDOM: [{ style: "span.escaped" }], + parseDOM: [{ tag: "span.escaped" }], toDOM() { return ["span", { class: "escaped" }]; }, diff --git a/src/shared/markdown-it/preserve-escape.ts b/src/shared/markdown-it/preserve-escape.ts index eec33d04..5835a023 100644 --- a/src/shared/markdown-it/preserve-escape.ts +++ b/src/shared/markdown-it/preserve-escape.ts @@ -9,7 +9,7 @@ function buildPreserveEscapeFn( const noop = (): boolean => false; //The "escape" rule has been disabled or otherwise removed; so there's nothing to replace here. - if (escapeFn.length === 0) { + if (!escapeFn) { return noop; } return function preserveEscapeFn( diff --git a/test/shared/markdown-it/preserve-escape.test.ts b/test/shared/markdown-it/preserve-escape.test.ts new file mode 100644 index 00000000..a0ee7131 --- /dev/null +++ b/test/shared/markdown-it/preserve-escape.test.ts @@ -0,0 +1,52 @@ +import { preserve_escape } from "../../../src/shared/markdown-it/preserve-escape"; +import MarkdownIt, {Token} from "markdown-it"; + +describe('preserve-escape', () => { + const preserved = new MarkdownIt("default").use(preserve_escape); + + const renderedInline = (rendered: Token[]) => + rendered.find((t) => t.type === "inline"); + + const renderedText = (rendered: Token[]) => + renderedInline(rendered) + .children.find((t) => t.type === "text"); + + const renderedEscape = (rendered: Token[]) => + renderedInline(rendered) + .children.find((t) => t.type === 'escape'); + + it("should render input exactly if nothing to escape", () => { + const markdown = `test`; + + const rendered = preserved.parse(markdown, {}); + expect(renderedText(rendered).content).toBe(markdown); + }) + + it("should render input exactly if the escape rule is disabled", () => { + const disabled = new MarkdownIt("default") + .disable("escape") + .use(preserve_escape); + const markdown = String.raw`\# this is just some text`; + const rendered = disabled.parse(markdown, {}) + expect(renderedText(rendered).content).toBe(markdown); + }) + + it("should preserve escaped characters", () => { + const markdown = String.raw`\# this is just some text`; + + //First, validate that rendering the escaped string removes the escape + const base = new MarkdownIt("default"); + const comparison = base.parse(markdown, {}); + expect(renderedText(comparison).content).toBe('# this is just some text'); + + //Next, we want to prove we've got an extra node that has the preserved, escaped character + const rendered = preserved.parse(markdown, {}); + const escapeNode = renderedEscape(rendered); + expect(escapeNode.type).toBe("escape") + expect(escapeNode.content).toBe("#") + expect(escapeNode.markup).toBe(String.raw`\#`) + + //Lastly, that the rest of the string is its own text node + expect(renderedText(rendered).content).toBe(" this is just some text"); + }); +}); diff --git a/test/shared/roundtrip.e2e.test.ts b/test/shared/roundtrip.e2e.test.ts new file mode 100644 index 00000000..d392cb8f --- /dev/null +++ b/test/shared/roundtrip.e2e.test.ts @@ -0,0 +1,56 @@ +import { test, expect, Page } from "@playwright/test"; +import { + switchMode, + clearEditor, + editorSelector, + enterTextAsMarkdown +} from "../e2e-helpers"; + +test.describe.serial("roundtrip tests", () => { + let page: Page; + test.beforeAll(async ({browser}) => { + page = await browser.newPage(); + await page.goto("/empty.html"); + await switchMode(page, "markdown"); + }); + + test.afterAll(async () => { + await page.close(); + }); + + for (const [markdown] of [ + //Basic commonmark + ['plain'], + ['*italic*'], + ['_italic_'], + ['**bold**'], + ['__bold__'], + ['# H1' ], + ['## H2' ], + ['[link](http://www.example.com)' ], + ['![Image](http://www.example.com/pretty.png)' ], + ['> blockquote' ], + ['* List Item' ], + ['- List Item' ], + ['1. List Item' ], + ['2) List Item' ], + ['lol\n\n---\n\nlmao' ], + ['lol\n\n***\n\nlmao' ], + ['`code`' ], + //TODO: Codeblock does weird things roundtripping: Adds an extra space + //['```javascript\ncodeblock\n```' ], + + //Escape character + [String.raw`\# not a header`], + [String.raw`- \# list item (not header)`] + ] as const) { + test(`should make markdown -> richtext -> markdown round trip '${markdown}'`, async () => { + await clearEditor(page); + await enterTextAsMarkdown(page, markdown); + await switchMode(page, "markdown"); + + const text = await page.innerText(editorSelector); + expect(text).toBe(markdown); + }) + } +}); From 67d3ad1ef44e1e14f9111ed2c878805a7bd1df26 Mon Sep 17 00:00:00 2001 From: James Boyden Date: Mon, 3 Feb 2025 15:03:40 +0000 Subject: [PATCH 06/13] Linting --- .../markdown-it/preserve-escape.test.ts | 26 +++++----- test/shared/roundtrip.e2e.test.ts | 48 +++++++++---------- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/test/shared/markdown-it/preserve-escape.test.ts b/test/shared/markdown-it/preserve-escape.test.ts index a0ee7131..eb10847e 100644 --- a/test/shared/markdown-it/preserve-escape.test.ts +++ b/test/shared/markdown-it/preserve-escape.test.ts @@ -1,35 +1,33 @@ import { preserve_escape } from "../../../src/shared/markdown-it/preserve-escape"; -import MarkdownIt, {Token} from "markdown-it"; +import MarkdownIt, { Token } from "markdown-it"; -describe('preserve-escape', () => { +describe("preserve-escape", () => { const preserved = new MarkdownIt("default").use(preserve_escape); const renderedInline = (rendered: Token[]) => rendered.find((t) => t.type === "inline"); const renderedText = (rendered: Token[]) => - renderedInline(rendered) - .children.find((t) => t.type === "text"); + renderedInline(rendered).children.find((t) => t.type === "text"); const renderedEscape = (rendered: Token[]) => - renderedInline(rendered) - .children.find((t) => t.type === 'escape'); + renderedInline(rendered).children.find((t) => t.type === "escape"); it("should render input exactly if nothing to escape", () => { const markdown = `test`; const rendered = preserved.parse(markdown, {}); expect(renderedText(rendered).content).toBe(markdown); - }) + }); it("should render input exactly if the escape rule is disabled", () => { const disabled = new MarkdownIt("default") .disable("escape") .use(preserve_escape); const markdown = String.raw`\# this is just some text`; - const rendered = disabled.parse(markdown, {}) + const rendered = disabled.parse(markdown, {}); expect(renderedText(rendered).content).toBe(markdown); - }) + }); it("should preserve escaped characters", () => { const markdown = String.raw`\# this is just some text`; @@ -37,14 +35,16 @@ describe('preserve-escape', () => { //First, validate that rendering the escaped string removes the escape const base = new MarkdownIt("default"); const comparison = base.parse(markdown, {}); - expect(renderedText(comparison).content).toBe('# this is just some text'); + expect(renderedText(comparison).content).toBe( + "# this is just some text" + ); //Next, we want to prove we've got an extra node that has the preserved, escaped character const rendered = preserved.parse(markdown, {}); const escapeNode = renderedEscape(rendered); - expect(escapeNode.type).toBe("escape") - expect(escapeNode.content).toBe("#") - expect(escapeNode.markup).toBe(String.raw`\#`) + expect(escapeNode.type).toBe("escape"); + expect(escapeNode.content).toBe("#"); + expect(escapeNode.markup).toBe(String.raw`\#`); //Lastly, that the rest of the string is its own text node expect(renderedText(rendered).content).toBe(" this is just some text"); diff --git a/test/shared/roundtrip.e2e.test.ts b/test/shared/roundtrip.e2e.test.ts index d392cb8f..60806f55 100644 --- a/test/shared/roundtrip.e2e.test.ts +++ b/test/shared/roundtrip.e2e.test.ts @@ -3,15 +3,15 @@ import { switchMode, clearEditor, editorSelector, - enterTextAsMarkdown + enterTextAsMarkdown, } from "../e2e-helpers"; test.describe.serial("roundtrip tests", () => { let page: Page; - test.beforeAll(async ({browser}) => { - page = await browser.newPage(); - await page.goto("/empty.html"); - await switchMode(page, "markdown"); + test.beforeAll(async ({ browser }) => { + page = await browser.newPage(); + await page.goto("/empty.html"); + await switchMode(page, "markdown"); }); test.afterAll(async () => { @@ -20,29 +20,29 @@ test.describe.serial("roundtrip tests", () => { for (const [markdown] of [ //Basic commonmark - ['plain'], - ['*italic*'], - ['_italic_'], - ['**bold**'], - ['__bold__'], - ['# H1' ], - ['## H2' ], - ['[link](http://www.example.com)' ], - ['![Image](http://www.example.com/pretty.png)' ], - ['> blockquote' ], - ['* List Item' ], - ['- List Item' ], - ['1. List Item' ], - ['2) List Item' ], - ['lol\n\n---\n\nlmao' ], - ['lol\n\n***\n\nlmao' ], - ['`code`' ], + ["plain"], + ["*italic*"], + ["_italic_"], + ["**bold**"], + ["__bold__"], + ["# H1"], + ["## H2"], + ["[link](http://www.example.com)"], + ["![Image](http://www.example.com/pretty.png)"], + ["> blockquote"], + ["* List Item"], + ["- List Item"], + ["1. List Item"], + ["2) List Item"], + ["lol\n\n---\n\nlmao"], + ["lol\n\n***\n\nlmao"], + ["`code`"], //TODO: Codeblock does weird things roundtripping: Adds an extra space //['```javascript\ncodeblock\n```' ], //Escape character [String.raw`\# not a header`], - [String.raw`- \# list item (not header)`] + [String.raw`- \# list item (not header)`], ] as const) { test(`should make markdown -> richtext -> markdown round trip '${markdown}'`, async () => { await clearEditor(page); @@ -51,6 +51,6 @@ test.describe.serial("roundtrip tests", () => { const text = await page.innerText(editorSelector); expect(text).toBe(markdown); - }) + }); } }); From 25fe87406d43efbfad14c9b3ebf51bceb7edfe76 Mon Sep 17 00:00:00 2001 From: James Boyden Date: Mon, 3 Feb 2025 16:33:47 +0000 Subject: [PATCH 07/13] `escape` isn't as cut and dry, so be specific in our operations --- src/shared/markdown-it/preserve-escape.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/shared/markdown-it/preserve-escape.ts b/src/shared/markdown-it/preserve-escape.ts index 5835a023..3ed3475d 100644 --- a/src/shared/markdown-it/preserve-escape.ts +++ b/src/shared/markdown-it/preserve-escape.ts @@ -21,13 +21,15 @@ function buildPreserveEscapeFn( //If the rule did nothing (returned false or is running in silent mode) there's nothing to fix if (silent || escRet === false) return escRet; - //The escape rule, if executed, always adds a 'text_special' node to the end, and we're going to work on that. - const [escapeToken] = state.tokens.slice(-1); + //The escape rule, if executed, always adds a 'text_special' node with 'escape', and we're going to work on that. + const escapeToken = state.tokens.findLast((t) => t.info == 'escape') - //Now we want to retag the type so that - // - the escape token is ignored by the text_merge - // - We can enact custom rendering later - escapeToken.type = "escape"; + if(escapeToken) { + //Now we want to retag the type so that + // - the escape token is ignored by the text_merge + // - We can enact custom rendering later + escapeToken.type = "escape"; + } return escRet; }; From d4fc55f8853bc0e2ce2a0b96a392ba863924e98d Mon Sep 17 00:00:00 2001 From: James Boyden Date: Mon, 3 Feb 2025 16:34:28 +0000 Subject: [PATCH 08/13] The linting just does not stop --- src/shared/markdown-it/preserve-escape.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/shared/markdown-it/preserve-escape.ts b/src/shared/markdown-it/preserve-escape.ts index 3ed3475d..242c87ec 100644 --- a/src/shared/markdown-it/preserve-escape.ts +++ b/src/shared/markdown-it/preserve-escape.ts @@ -22,9 +22,9 @@ function buildPreserveEscapeFn( if (silent || escRet === false) return escRet; //The escape rule, if executed, always adds a 'text_special' node with 'escape', and we're going to work on that. - const escapeToken = state.tokens.findLast((t) => t.info == 'escape') + const escapeToken = state.tokens.findLast((t) => t.info == "escape"); - if(escapeToken) { + if (escapeToken) { //Now we want to retag the type so that // - the escape token is ignored by the text_merge // - We can enact custom rendering later From 1eed711be72329fe69ada29704206f69e4000e27 Mon Sep 17 00:00:00 2001 From: James Boyden Date: Mon, 3 Feb 2025 16:39:41 +0000 Subject: [PATCH 09/13] I don't think we're doing anything special here, so lets bump --- .github/workflows/main.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5fc2c513..a3f206ee 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -10,9 +10,9 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup Node.js environment - uses: actions/setup-node@v3 + uses: actions/setup-node@v4 with: node-version: lts/* cache: "npm" @@ -25,10 +25,10 @@ jobs: unit-test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup Node.js environment - uses: actions/setup-node@v3 + uses: actions/setup-node@v4 with: node-version: lts/* cache: "npm" @@ -41,10 +41,10 @@ jobs: e2e-test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup Node.js environment - uses: actions/setup-node@v3 + uses: actions/setup-node@v4 with: node-version: lts/* cache: "npm" @@ -58,7 +58,7 @@ jobs: - name: Run E2E tests run: npm run test:e2e - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: failure() with: name: playwright-test-results From ee5b30c06075129b87c37ce777d49877b91d1c60 Mon Sep 17 00:00:00 2001 From: James Boyden Date: Mon, 3 Feb 2025 16:45:13 +0000 Subject: [PATCH 10/13] Add a doc comment --- src/shared/markdown-it/preserve-escape.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/shared/markdown-it/preserve-escape.ts b/src/shared/markdown-it/preserve-escape.ts index 242c87ec..caddc0fa 100644 --- a/src/shared/markdown-it/preserve-escape.ts +++ b/src/shared/markdown-it/preserve-escape.ts @@ -35,6 +35,10 @@ function buildPreserveEscapeFn( }; } +/*** + * Preserves `text_special` nodes marked as `escape` by changing their type to `escape` + * allowing us to preserve the escape characters and parse them downstream. + */ export function preserve_escape(md: MarkdownIt): void { const preserveEscapeTokens = buildPreserveEscapeFn(md); md.inline.ruler.at("escape", preserveEscapeTokens); From 036b5bcc5d42d114934fb451d670e1e61efcc372 Mon Sep 17 00:00:00 2001 From: James Boyden Date: Tue, 4 Feb 2025 10:14:15 +0000 Subject: [PATCH 11/13] Stripping down to basics. YAGNI! --- src/shared/markdown-it/preserve-escape.ts | 45 ---------------- src/shared/markdown-parser.ts | 9 ---- .../markdown-it/preserve-escape.test.ts | 52 ------------------- test/shared/roundtrip.e2e.test.ts | 2 +- 4 files changed, 1 insertion(+), 107 deletions(-) delete mode 100644 src/shared/markdown-it/preserve-escape.ts delete mode 100644 test/shared/markdown-it/preserve-escape.test.ts diff --git a/src/shared/markdown-it/preserve-escape.ts b/src/shared/markdown-it/preserve-escape.ts deleted file mode 100644 index caddc0fa..00000000 --- a/src/shared/markdown-it/preserve-escape.ts +++ /dev/null @@ -1,45 +0,0 @@ -import MarkdownIt, { StateInline } from "markdown-it"; - -function buildPreserveEscapeFn( - md: MarkdownIt -): MarkdownIt.ParserInline.RuleInline { - const [escapeFn] = md.inline.ruler - .getRules("") - .filter((r) => r.name === "escape"); - - const noop = (): boolean => false; - //The "escape" rule has been disabled or otherwise removed; so there's nothing to replace here. - if (!escapeFn) { - return noop; - } - return function preserveEscapeFn( - state: StateInline, - silent: boolean - ): boolean { - const escRet = escapeFn(state, silent); - - //If the rule did nothing (returned false or is running in silent mode) there's nothing to fix - if (silent || escRet === false) return escRet; - - //The escape rule, if executed, always adds a 'text_special' node with 'escape', and we're going to work on that. - const escapeToken = state.tokens.findLast((t) => t.info == "escape"); - - if (escapeToken) { - //Now we want to retag the type so that - // - the escape token is ignored by the text_merge - // - We can enact custom rendering later - escapeToken.type = "escape"; - } - - return escRet; - }; -} - -/*** - * Preserves `text_special` nodes marked as `escape` by changing their type to `escape` - * allowing us to preserve the escape characters and parse them downstream. - */ -export function preserve_escape(md: MarkdownIt): void { - const preserveEscapeTokens = buildPreserveEscapeFn(md); - md.inline.ruler.at("escape", preserveEscapeTokens); -} diff --git a/src/shared/markdown-parser.ts b/src/shared/markdown-parser.ts index 11fdd2f0..15b26e92 100644 --- a/src/shared/markdown-parser.ts +++ b/src/shared/markdown-parser.ts @@ -11,7 +11,6 @@ import { stackLanguageComments } from "./markdown-it/stack-language-comments"; import { tagLinks } from "./markdown-it/tag-link"; import { tight_list } from "./markdown-it/tight-list"; import type { CommonmarkParserFeatures } from "./view"; -import { preserve_escape } from "./markdown-it/preserve-escape"; // extend the default markdown parser's tokens and add our own const customMarkdownParserTokens: MarkdownParser["tokens"] = { @@ -151,11 +150,6 @@ const customMarkdownParserTokens: MarkdownParser["tokens"] = { code_inline_split: { mark: "code", }, - - escape: { - mark: "escape", - noCloseToken: true, - }, }; // add tag attribute support to all the marks like we did in schema @@ -326,9 +320,6 @@ export function createDefaultMarkdownItInstance( // ensure we can tell the difference between the different types of hardbreaks defaultMarkdownItInstance.use(hardbreak_markup); - // ensure that we preserve escape characters inside some block contexts - defaultMarkdownItInstance.use(preserve_escape); - // TODO should always exist, so remove the check once the param is made non-optional externalPluginProvider?.alterMarkdownIt(defaultMarkdownItInstance); diff --git a/test/shared/markdown-it/preserve-escape.test.ts b/test/shared/markdown-it/preserve-escape.test.ts deleted file mode 100644 index eb10847e..00000000 --- a/test/shared/markdown-it/preserve-escape.test.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { preserve_escape } from "../../../src/shared/markdown-it/preserve-escape"; -import MarkdownIt, { Token } from "markdown-it"; - -describe("preserve-escape", () => { - const preserved = new MarkdownIt("default").use(preserve_escape); - - const renderedInline = (rendered: Token[]) => - rendered.find((t) => t.type === "inline"); - - const renderedText = (rendered: Token[]) => - renderedInline(rendered).children.find((t) => t.type === "text"); - - const renderedEscape = (rendered: Token[]) => - renderedInline(rendered).children.find((t) => t.type === "escape"); - - it("should render input exactly if nothing to escape", () => { - const markdown = `test`; - - const rendered = preserved.parse(markdown, {}); - expect(renderedText(rendered).content).toBe(markdown); - }); - - it("should render input exactly if the escape rule is disabled", () => { - const disabled = new MarkdownIt("default") - .disable("escape") - .use(preserve_escape); - const markdown = String.raw`\# this is just some text`; - const rendered = disabled.parse(markdown, {}); - expect(renderedText(rendered).content).toBe(markdown); - }); - - it("should preserve escaped characters", () => { - const markdown = String.raw`\# this is just some text`; - - //First, validate that rendering the escaped string removes the escape - const base = new MarkdownIt("default"); - const comparison = base.parse(markdown, {}); - expect(renderedText(comparison).content).toBe( - "# this is just some text" - ); - - //Next, we want to prove we've got an extra node that has the preserved, escaped character - const rendered = preserved.parse(markdown, {}); - const escapeNode = renderedEscape(rendered); - expect(escapeNode.type).toBe("escape"); - expect(escapeNode.content).toBe("#"); - expect(escapeNode.markup).toBe(String.raw`\#`); - - //Lastly, that the rest of the string is its own text node - expect(renderedText(rendered).content).toBe(" this is just some text"); - }); -}); diff --git a/test/shared/roundtrip.e2e.test.ts b/test/shared/roundtrip.e2e.test.ts index 60806f55..398f9d44 100644 --- a/test/shared/roundtrip.e2e.test.ts +++ b/test/shared/roundtrip.e2e.test.ts @@ -44,7 +44,7 @@ test.describe.serial("roundtrip tests", () => { [String.raw`\# not a header`], [String.raw`- \# list item (not header)`], ] as const) { - test(`should make markdown -> richtext -> markdown round trip '${markdown}'`, async () => { + test(`should make markdown -> richtext -> markdown round trip '${JSON.stringify(markdown)}'`, async () => { await clearEditor(page); await enterTextAsMarkdown(page, markdown); await switchMode(page, "markdown"); From 37acf1793b890ec15479327b1933c4535cef8d2f Mon Sep 17 00:00:00 2001 From: James Boyden Date: Tue, 4 Feb 2025 10:15:09 +0000 Subject: [PATCH 12/13] Vestigial mark definition --- src/shared/markdown-serializer.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/shared/markdown-serializer.ts b/src/shared/markdown-serializer.ts index df4e9dd4..ccf2cab6 100644 --- a/src/shared/markdown-serializer.ts +++ b/src/shared/markdown-serializer.ts @@ -673,12 +673,6 @@ const customMarkdownSerializerMarks: MarkdownSerializerMarks = { mixable: true, expelEnclosingWhitespace: true, }), - escape: { - open: "\\", - close: "", - mixable: false, - escape: false, // That's what we're doing! - }, }; // export our custom serializer using the extended nodes/marks taken from the default schema From 415239c71da8d1993976063c69b418618277255d Mon Sep 17 00:00:00 2001 From: James Boyden Date: Tue, 4 Feb 2025 10:15:51 +0000 Subject: [PATCH 13/13] Vestigial mark definition (2) --- src/rich-text/schema.ts | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/rich-text/schema.ts b/src/rich-text/schema.ts index 61cfd14c..cbf52063 100644 --- a/src/rich-text/schema.ts +++ b/src/rich-text/schema.ts @@ -518,13 +518,6 @@ const marks: { sup: genHtmlInlineMarkSpec({}, "sup"), sub: genHtmlInlineMarkSpec({}, "sub"), - - escape: { - parseDOM: [{ tag: "span.escaped" }], - toDOM() { - return ["span", { class: "escaped" }]; - }, - }, }; // for *every* mark, add in support for the `markup` attribute