` / ``
+// from rescanning the rest of the input on every attempt.
+const BACKTICK_ENTITY_RE = /�*96;|�*60;/gi;
+const NBSP_ENTITY = ' |�*160;|�*a0;';
+function fenceProtectedRe(html) {
+ const parts = [];
+ if (/<\/pre/i.test(html)) parts.push('');
+ if (/<\/code/i.test(html)) parts.push(']*>[\\s\\S]*?<\\/code>');
+ parts.push('<[A-Za-z/!][^>]*>');
+ return new RegExp(parts.join('|'), 'gi');
+}
+function escapeLiteralFences(html) {
+ const escapeProse = (text) => escapeFenceLikeText(text.replace(BACKTICK_ENTITY_RE, '`'), NBSP_ENTITY);
+ let out = '';
+ let last = 0;
+ for (const m of html.matchAll(fenceProtectedRe(html))) {
+ out += escapeProse(html.slice(last, m.index)) + m[0];
+ last = m.index + m[0].length;
+ }
+ return out + escapeProse(html.slice(last));
+}
+
function htmlToMarkdown(html) {
- let markdown = escapeSentinels(html);
+ let markdown = escapeLiteralFences(escapeSentinels(html));
markdown = markdown.replace(/]*(?:\/>|>\s*<\/time>)/g, '$1');
diff --git a/lib/markdown-cleanup.js b/lib/markdown-cleanup.js
index ed2283b..5e06f7a 100644
--- a/lib/markdown-cleanup.js
+++ b/lib/markdown-cleanup.js
@@ -30,6 +30,33 @@ function fenceLength(decodedBody) {
return Math.max(3, max + 1);
}
+// A prose line that opens with a run of 3+ backticks looks like a code fence.
+// splitOnFences would pair it with the next real fence, so the real code body
+// would leave the fence and go through whitespace cleanup — silently changing
+// the code. Backslash-escaping every backtick of the run keeps the line
+// literal text, which is also what the author meant: converter-emitted fences
+// are the only lines that may open one. (Escaping only the first would leave
+// the rest free to pair with later backticks into a code span on write-back.)
+// Apply to prose only, never to code bodies or code spans. A run at the start
+// of the text or of a line is escaped; text that begins at an inline-element
+// boundary counts as a line start, where escaping is harmless. Mid-line runs
+// cannot open a fence and stay as written. Plain spaces, tabs and NBSP may
+// indent the run (`extraIndent` adds more alternatives, e.g. HTML entities).
+const FENCE_LIKE_RES = new Map();
+function fenceLikeRe(extraIndent) {
+ if (!FENCE_LIKE_RES.has(extraIndent)) {
+ const indent = extraIndent ? `[ \\t\\u00a0]|${extraIndent}` : '[ \\t\\u00a0]';
+ FENCE_LIKE_RES.set(extraIndent, new RegExp(`(^|\\n)((?:${indent})*)(\`{3,})`, 'gi'));
+ }
+ return FENCE_LIKE_RES.get(extraIndent);
+}
+function escapeFenceLikeText(text, extraIndent = '') {
+ return text.replace(
+ fenceLikeRe(extraIndent),
+ (_, lead, indent, ticks) => `${lead}${indent}${'\\`'.repeat(ticks.length)}`,
+ );
+}
+
// List-item continuation indent emitted by StorageWalker and
// html-to-markdown. A private-use codepoint (not matched by \s or [ \t]) so
// the whitespace cleanup chain below cannot strip or collapse it; each
@@ -267,6 +294,7 @@ module.exports = {
stripListIndent,
finalizeSentinels,
fenceLength,
+ escapeFenceLikeText,
splitOnFences,
isFenceOpenLine,
collapseBlankLines,
diff --git a/lib/storage-walker.js b/lib/storage-walker.js
index 50eecc9..3390bd2 100644
--- a/lib/storage-walker.js
+++ b/lib/storage-walker.js
@@ -8,6 +8,7 @@ const {
stripListIndent,
finalizeSentinels,
fenceLength,
+ escapeFenceLikeText,
isFenceOpenLine,
splitOnFences,
collapseBlankLines,
@@ -857,9 +858,10 @@ class StorageWalker {
renderText(text) {
const decodedText = decodeEntities(text);
- return this._markdownLinkLabelDepth > 0 && this._markdownCodeSpanDepth === 0
+ if (this._markdownCodeSpanDepth > 0) return decodedText;
+ return this._markdownLinkLabelDepth > 0
? this.escapeMarkdownText(decodedText)
- : decodedText;
+ : escapeFenceLikeText(decodedText);
}
renderCodeSpan(content) {
diff --git a/tests/html-to-markdown.test.js b/tests/html-to-markdown.test.js
index 0ada337..3a0ff49 100644
--- a/tests/html-to-markdown.test.js
+++ b/tests/html-to-markdown.test.js
@@ -178,10 +178,10 @@ describe('htmlToMarkdown', () => {
});
test('literal backticks leading an item keep a plain-space marker', () => {
- // Joining the marker with LIST_INDENT would make splitOnFences pair
- // this literal with the following real fence.
+ // The literal is escaped, and joining the marker with LIST_INDENT
+ // would make splitOnFences pair it with the following real fence.
const html = '- ```
x y
c
';
- expect(htmlToMarkdown(html)).toBe('- ```\n\nx y\n\n```\n c\n```');
+ expect(htmlToMarkdown(html)).toBe('- \\`\\`\\`\n\nx y\n\n```\n c\n```');
});
test('pretty-printed HTML with whitespace between tags', () => {
@@ -476,7 +476,7 @@ describe('htmlToMarkdown', () => {
test('a line whose info string contains a backtick does not open a fence (#244)', () => {
const out = htmlToMarkdown('```js `x`
a b
c d
');
- expect(out).toBe('```js `x`\n\na b\n\n```\nc d\n```');
+ expect(out).toBe('\\`\\`\\`js `x`\n\na b\n\n```\nc d\n```');
});
test('a fence in a list item does not disturb a later top-level fence (#238, #243)', () => {
@@ -484,4 +484,58 @@ describe('htmlToMarkdown', () => {
expect(out).toContain('```\nq r\n```');
expect(out).toBe('- ```\n a b\n ```\n\nx\n\n```\nq r\n```');
});
+ describe('literal fence-like text (#254)', () => {
+ const PRE = ' indented\n code here
';
+ const FENCED = '```\n indented\n code here\n```';
+
+ test('a literal ``` after a nested list does not capture the next code block', () => {
+ const html = `- a
- b
\`\`\`
${PRE}`;
+ expect(htmlToMarkdown(html)).toBe(`- a\n - b\n\n \\\`\\\`\\\`\n\n${FENCED}`);
+ });
+
+ test('a top-level ```
does not capture the next code block', () => {
+ expect(htmlToMarkdown(`\`\`\`
${PRE}`)).toBe(`\\\`\\\`\\\`\n\n${FENCED}`);
+ });
+
+ test('backticks written as entities are treated the same way', () => {
+ expect(htmlToMarkdown(````
${PRE}`)).toBe(`\\\`\\\`\\\`\n\n${FENCED}`);
+ expect(htmlToMarkdown(````
${PRE}`)).toBe(`\\\`\\\`\\\`\n\n${FENCED}`);
+ });
+
+ test('a literal fence line after a
is escaped', () => {
+ expect(htmlToMarkdown(`x
\`\`\`
${PRE}`)).toBe(`x\n\\\`\\\`\\\`\n\n${FENCED}`);
+ });
+
+ test.each([
+ [' ', 'x
```
', 'x\n'],
+ [' ', ' ```
', ''],
+ [' ', ' ```
', ''],
+ ])('a literal fence line indented by %s is escaped', (_label, prose, lead) => {
+ const out = htmlToMarkdown(`${prose}${PRE}`);
+ expect(out).toBe(`${lead}\\\`\\\`\\\`\n\n${FENCED}`);
+ });
+
+ test('a bare < in prose is not mistaken for a tag that hides a literal fence', () => {
+ expect(htmlToMarkdown(`a < b\n\`\`\`\nc
${PRE}`)).toBe(`a < b\n\\\`\\\`\\\`\nc\n\n${FENCED}`);
+ });
+
+ test('unclosed tags do not make the escape pass quadratic', () => {
+ const start = Date.now();
+ htmlToMarkdown('x'.repeat(40000));
+ expect(Date.now() - start).toBeLessThan(1000);
+ });
+
+ test('backticks inside a code block body are left untouched', () => {
+ const html = '```\n inner\n```
';
+ expect(htmlToMarkdown(html)).toBe('````\n```\n inner\n```\n````');
+ });
+
+ test('backticks in prose that cannot open a fence are left as written', () => {
+ expect(htmlToMarkdown('use ```js fences
')).toBe('use ```js fences');
+ });
+
+ test('backticks inside an attribute value are not touched', () => {
+ expect(htmlToMarkdown('')).toBe('[x](https://example.com/a```b)');
+ });
+ });
});
diff --git a/tests/macro-converter.test.js b/tests/macro-converter.test.js
index 1a5aafe..459e780 100644
--- a/tests/macro-converter.test.js
+++ b/tests/macro-converter.test.js
@@ -2184,21 +2184,23 @@ describe('MacroConverter storageToMarkdown code inside quotes and callouts (#244
test('a bare ``` paragraph in a callout does not swallow the code blocks after it', () => {
const storage = macro('info', '```
') + codeMacro('', 'one 1') + 'x y
' + codeMacro('', 'two 2');
expect(converter.storageToMarkdown(storage))
- .toBe('> **INFO**\n> ```\n\n```\none 1\n```\n\nx y\n\n```\ntwo 2\n```');
+ .toBe('> **INFO**\n> \\`\\`\\`\n\n```\none 1\n```\n\nx y\n\n```\ntwo 2\n```');
});
test('a bare ``` paragraph in a callout does not pair with a fence in a later callout', () => {
const storage = macro('info', '```
') + 'mid text
' + macro('note', codeMacro('js', 'keep 1'));
expect(converter.storageToMarkdown(storage))
- .toBe('> **INFO**\n> ```\n\nmid text\n\n> **NOTE**\n> ```js\n> keep 1\n> ```');
+ .toBe('> **INFO**\n> \\`\\`\\`\n\nmid text\n\n> **NOTE**\n> ```js\n> keep 1\n> ```');
});
test('indented loose text between callouts is cleaned, not turned into code on write-back', () => {
const storage = macro('info', '```
') + 'loose\n indented text\n' + macro('note', codeMacro('js', 'keep 1'));
const md = converter.storageToMarkdown(storage);
- expect(md).toBe('> **INFO**\n> ```\nloose\nindented text\n\n> **NOTE**\n> ```js\n> keep 1\n> ```');
+ expect(md).toBe('> **INFO**\n> \\`\\`\\`\nloose\nindented text\n\n> **NOTE**\n> ```js\n> keep 1\n> ```');
const storage1 = converter.markdownToStorage(md);
- expect(storage1).toContain('loose\nindented text
');
+ // The escaped literal stays prose; the loose text continues that paragraph
+ // (no blank line separates them) and is not turned into code.
+ expect(storage1).toContain('```\nloose\nindented text
');
expect(storage1).toContain('');
});
@@ -2524,7 +2526,7 @@ describe('MacroConverter storageToMarkdown
in flattened contexts (#242)',
['-', '\\-'],
['===', '\\==='],
['* * *', '\\* * *'],
- ['```js', '\\```js'],
+ ['```js', '\\`\\`\\`js'],
['~~~', '\\~~~'],
['--- | ---', '\\--- | ---'],
['|:---|---:|', '\\|:---|---:|'],
@@ -2633,3 +2635,70 @@ describe('MacroConverter storageToMarkdown
in flattened contexts (#242)',
});
});
});
+
+describe('MacroConverter storageToMarkdown literal fence-like text (#254)', () => {
+ const converter = new MacroConverter({ isCloud: true });
+ const codeMacro = (lang, body) =>
+ `${lang} ${body} ```
does not capture the next code macro', () => {
+ expect(converter.storageToMarkdown(`\`\`\`
${code}`)).toBe(`\\\`\\\`\\\`\n\n${FENCED}`);
+ });
+
+ test('a literal ``` after a nested list does not capture the next code macro', () => {
+ const storage = `- a
- b
\`\`\`
${code}`;
+ expect(converter.storageToMarkdown(storage)).toBe(`- a\n - b\n\n \\\`\\\`\\\`\n\n${FENCED}`);
+ });
+
+ test('a literal fence line after a
is escaped', () => {
+ expect(converter.storageToMarkdown(`x
\`\`\`
${code}`)).toBe(`x\n\\\`\\\`\\\`\n\n${FENCED}`);
+ });
+
+ test('backticks written as entities or CDATA are treated the same way', () => {
+ expect(converter.storageToMarkdown(````
${code}`)).toBe(`\\\`\\\`\\\`\n\n${FENCED}`);
+ expect(converter.storageToMarkdown(`${code}`)).toBe(`\\\`\\\`\\\`\n\n${FENCED}`);
+ });
+
+ test('a literal NBSP before the fence-like run does not hide it', () => {
+ expect(converter.storageToMarkdown(`\u00a0\`\`\`
${code}`)).toBe(`\\\`\\\`\\\`\n\n${FENCED}`);
+ expect(converter.storageToMarkdown(` \`\`\`
${code}`)).toBe(`\\\`\\\`\\\`\n\n${FENCED}`);
+ });
+
+ test.each([
+ ['four backticks then a matching run', '````
b ``` c
', '````\nb ``` c
'],
+ ['a run followed by a shorter one', '```
use `` here
', '```\nuse `` here
'],
+ ['a run and a code-span-like pair', '``` ``
', '``` ``
'],
+ ])('every backtick of the run is escaped, so %s stays literal on write-back', (_label, storage, expected) => {
+ const md = converter.storageToMarkdown(storage);
+ expect(converter.markdownToStorage(md)).toBe(`${expected}\n`);
+ });
+
+ test('a literal ``` inside a callout stays escaped and the code after it is untouched', () => {
+ expect(converter.storageToMarkdown(macro('info', '```
') + code))
+ .toBe(`> **INFO**\n> \\\`\\\`\\\`\n\n${FENCED}`);
+ });
+
+ test('the code macro body is never escaped', () => {
+ expect(converter.storageToMarkdown(codeMacro('', '```\n inner\n```')))
+ .toBe('````\n```\n inner\n```\n````');
+ });
+
+ test('code spans and mid-line backticks are left as written', () => {
+ expect(converter.storageToMarkdown('```
')).toBe('```` ``` ````');
+ expect(converter.storageToMarkdown('use ```js fences
')).toBe('use ```js fences');
+ });
+
+ test('round-trip keeps the literal text as prose and the code body byte-exact', () => {
+ const md = converter.storageToMarkdown(`\`\`\`
${code}`);
+ const storage1 = converter.markdownToStorage(md);
+ expect(storage1).toContain('```
');
+ expect(storage1).toContain('');
+ // Writing back labels a language-less fence `text`; that is unrelated here.
+ expect(converter.storageToMarkdown(storage1))
+ .toBe('\\`\\`\\`\n\n```text\n indented\n code here\n```');
+ });
+});
diff --git a/tests/markdown-cleanup.test.js b/tests/markdown-cleanup.test.js
index b30aa12..5b6556f 100644
--- a/tests/markdown-cleanup.test.js
+++ b/tests/markdown-cleanup.test.js
@@ -5,6 +5,7 @@ const {
escapeSentinels,
finalizeSentinels,
fenceLength,
+ escapeFenceLikeText,
splitOnFences,
isFenceOpenLine,
cleanupOutsideFence,
@@ -297,3 +298,39 @@ describe('markdown-cleanup QUOTE_MARK sentinel (#244)', () => {
expect(finalizeSentinels(`${I}${HARD_BREAK}${escapeSentinels(literal)}`)).toBe(` \n${literal}`);
});
});
+
+describe('escapeFenceLikeText', () => {
+ test.each([
+ ['```', '\\`\\`\\`'],
+ ['````', '\\`\\`\\`\\`'],
+ ['```js', '\\`\\`\\`js'],
+ [' ```', ' \\`\\`\\`'],
+ ['a\n```\nb', 'a\n\\`\\`\\`\nb'],
+ ['```\n```', '\\`\\`\\`\n\\`\\`\\`'],
+ ])('escapes a line-leading run of backticks: %j', (input, expected) => {
+ expect(escapeFenceLikeText(input)).toBe(expected);
+ });
+
+ test.each([
+ 'use ```js fences',
+ 'two `` backticks',
+ 'one ` backtick',
+ 'no backticks',
+ '',
+ ])('leaves text without a line-leading 3+ run alone: %j', (input) => {
+ expect(escapeFenceLikeText(input)).toBe(input);
+ });
+
+ test('NBSP counts as indentation, and extra indent alternatives can be supplied', () => {
+ expect(escapeFenceLikeText('\u00a0```')).toBe('\u00a0\\`\\`\\`');
+ expect(escapeFenceLikeText(' ```')).toBe(' ```');
+ expect(escapeFenceLikeText(' ```', ' ')).toBe(' \\`\\`\\`');
+ });
+
+ test('an escaped line can no longer open a fence', () => {
+ const text = `${escapeFenceLikeText('```')}\n\n\`\`\`\n code\n\`\`\``;
+ const segments = splitOnFences(text);
+ expect(segments).toHaveLength(3);
+ expect(segments[1]).toBe('```\n code\n```');
+ });
+});