fix(converter): escape literal fence-like prose so it cannot capture code blocks

lib/html-to-markdown.js

@@ -3,6 +3,7 @@ const {
   escapeSentinels,
   finalizeSentinels,
   fenceLength,
+  escapeFenceLikeText,
   splitOnFences,
   cleanupWithFences,
 } = require('./markdown-cleanup');
@@ -196,8 +197,36 @@ function convertLists(src) {
   return out + src.slice(copied);
 }
 
+// Code regions and tags (attribute values may hold backticks) are left alone;
+// only the prose between them is escaped. Backtick entities in prose are
+// decoded first, since the entity pass runs after the fence-aware cleanup and
+// would otherwise reintroduce a literal fence line, and NBSP entities count as
+// indentation for the same reason. A tag must start with a letter, `/` or `!`
+// so prose like `a < b` is not mistaken for one. The code-region patterns are
+// only used when a closing tag exists, which keeps unclosed `<pre>` / `<code>`
+// from rescanning the rest of the input on every attempt.
+const BACKTICK_ENTITY_RE = /&#0*96;|&#x0*60;/gi;
+const NBSP_ENTITY = '&nbsp;|&#0*160;|&#x0*a0;';
+function fenceProtectedRe(html) {
+  const parts = [];
+  if (/<\/pre/i.test(html)) parts.push('<pre(?![\\w-])[\\s\\S]*?<\\/pre>');
+  if (/<\/code/i.test(html)) parts.push('<code(?![\\w-])[^>]*>[\\s\\S]*?<\\/code>');
+  parts.push('<[A-Za-z/!][^>]*>');
+  return new RegExp(parts.join('|'), 'gi');
+}
+function escapeLiteralFences(html) {
+  const escapeProse = (text) => escapeFenceLikeText(text.replace(BACKTICK_ENTITY_RE, '`'), NBSP_ENTITY);
+  let out = '';
+  let last = 0;
+  for (const m of html.matchAll(fenceProtectedRe(html))) {
+    out += escapeProse(html.slice(last, m.index)) + m[0];
+    last = m.index + m[0].length;
+  }
+  return out + escapeProse(html.slice(last));
+}
+
 function htmlToMarkdown(html) {
-  let markdown = escapeSentinels(html);
+  let markdown = escapeLiteralFences(escapeSentinels(html));
 
   markdown = markdown.replace(/<time\s+datetime="([^"]+)"[^>]*(?:\/>|>\s*<\/time>)/g, '$1');
 

lib/markdown-cleanup.js

@@ -30,6 +30,33 @@ function fenceLength(decodedBody) {
   return Math.max(3, max + 1);
 }
 
+// A prose line that opens with a run of 3+ backticks looks like a code fence.
+// splitOnFences would pair it with the next real fence, so the real code body
+// would leave the fence and go through whitespace cleanup — silently changing
+// the code. Backslash-escaping every backtick of the run keeps the line
+// literal text, which is also what the author meant: converter-emitted fences
+// are the only lines that may open one. (Escaping only the first would leave
+// the rest free to pair with later backticks into a code span on write-back.)
+// Apply to prose only, never to code bodies or code spans. A run at the start
+// of the text or of a line is escaped; text that begins at an inline-element
+// boundary counts as a line start, where escaping is harmless. Mid-line runs
+// cannot open a fence and stay as written. Plain spaces, tabs and NBSP may
+// indent the run (`extraIndent` adds more alternatives, e.g. HTML entities).
+const FENCE_LIKE_RES = new Map();
+function fenceLikeRe(extraIndent) {
+  if (!FENCE_LIKE_RES.has(extraIndent)) {
+    const indent = extraIndent ? `[ \\t\\u00a0]|${extraIndent}` : '[ \\t\\u00a0]';
+    FENCE_LIKE_RES.set(extraIndent, new RegExp(`(^|\\n)((?:${indent})*)(\`{3,})`, 'gi'));
+  }
+  return FENCE_LIKE_RES.get(extraIndent);
+}
+function escapeFenceLikeText(text, extraIndent = '') {
+  return text.replace(
+    fenceLikeRe(extraIndent),
+    (_, lead, indent, ticks) => `${lead}${indent}${'\\`'.repeat(ticks.length)}`,
+  );
+}
+
 // List-item continuation indent emitted by StorageWalker and
 // html-to-markdown. A private-use codepoint (not matched by \s or [ \t]) so
 // the whitespace cleanup chain below cannot strip or collapse it; each
@@ -267,6 +294,7 @@ module.exports = {
   stripListIndent,
   finalizeSentinels,
   fenceLength,
+  escapeFenceLikeText,
   splitOnFences,
   isFenceOpenLine,
   collapseBlankLines,

lib/storage-walker.js

@@ -8,6 +8,7 @@ const {
   stripListIndent,
   finalizeSentinels,
   fenceLength,
+  escapeFenceLikeText,
   isFenceOpenLine,
   splitOnFences,
   collapseBlankLines,
@@ -857,9 +858,10 @@ class StorageWalker {
 
   renderText(text) {
     const decodedText = decodeEntities(text);
-    return this._markdownLinkLabelDepth > 0 && this._markdownCodeSpanDepth === 0
+    if (this._markdownCodeSpanDepth > 0) return decodedText;
+    return this._markdownLinkLabelDepth > 0
       ? this.escapeMarkdownText(decodedText)
-      : decodedText;
+      : escapeFenceLikeText(decodedText);
   }
 
   renderCodeSpan(content) {

tests/html-to-markdown.test.js

@@ -178,10 +178,10 @@ describe('htmlToMarkdown', () => {
       });
 
       test('literal backticks leading an item keep a plain-space marker', () => {
-        // Joining the marker with LIST_INDENT would make splitOnFences pair
-        // this literal with the following real fence.
+        // The literal is escaped, and joining the marker with LIST_INDENT
+        // would make splitOnFences pair it with the following real fence.
         const html = '<ul><li>```</li></ul><p>x   y</p><pre><code>  c</code></pre>';
-        expect(htmlToMarkdown(html)).toBe('- ```\n\nx y\n\n```\n  c\n```');
+        expect(htmlToMarkdown(html)).toBe('- \\`\\`\\`\n\nx y\n\n```\n  c\n```');
       });
 
       test('pretty-printed HTML with whitespace between tags', () => {
@@ -476,12 +476,66 @@ describe('htmlToMarkdown', () => {
 
   test('a line whose info string contains a backtick does not open a fence (#244)', () => {
     const out = htmlToMarkdown('<p>```js `x`</p><p>a    b</p><pre><code>c    d</code></pre>');
-    expect(out).toBe('```js `x`\n\na b\n\n```\nc    d\n```');
+    expect(out).toBe('\\`\\`\\`js `x`\n\na b\n\n```\nc    d\n```');
   });
 
   test('a fence in a list item does not disturb a later top-level fence (#238, #243)', () => {
     const out = htmlToMarkdown('<ul><li><pre><code>a  b</code></pre></li></ul><p>x</p><pre><code>q   r</code></pre>');
     expect(out).toContain('```\nq   r\n```');
     expect(out).toBe('- ```\n  a  b\n  ```\n\nx\n\n```\nq   r\n```');
   });
+  describe('literal fence-like text (#254)', () => {
+    const PRE = '<pre><code>  indented\n    code   here</code></pre>';
+    const FENCED = '```\n  indented\n    code   here\n```';
+
+    test('a literal ``` after a nested list does not capture the next code block', () => {
+      const html = `<ul><li>a<ul><li>b</li></ul>\`\`\`</li></ul>${PRE}`;
+      expect(htmlToMarkdown(html)).toBe(`- a\n  - b\n\n  \\\`\\\`\\\`\n\n${FENCED}`);
+    });
+
+    test('a top-level <p>```</p> does not capture the next code block', () => {
+      expect(htmlToMarkdown(`<p>\`\`\`</p>${PRE}`)).toBe(`\\\`\\\`\\\`\n\n${FENCED}`);
+    });
+
+    test('backticks written as entities are treated the same way', () => {
+      expect(htmlToMarkdown(`<p>&#96;&#96;&#96;</p>${PRE}`)).toBe(`\\\`\\\`\\\`\n\n${FENCED}`);
+      expect(htmlToMarkdown(`<p>&#x60;&#x60;&#x60;</p>${PRE}`)).toBe(`\\\`\\\`\\\`\n\n${FENCED}`);
+    });
+
+    test('a literal fence line after a <br> is escaped', () => {
+      expect(htmlToMarkdown(`<p>x<br/>\`\`\`</p>${PRE}`)).toBe(`x\n\\\`\\\`\\\`\n\n${FENCED}`);
+    });
+
+    test.each([
+      ['&nbsp;', '<p>x<br>&nbsp;```</p>', 'x\n'],
+      ['&#160;', '<p>&#160;```</p>', ''],
+      ['&#xA0;', '<p>&#xA0;```</p>', ''],
+    ])('a literal fence line indented by %s is escaped', (_label, prose, lead) => {
+      const out = htmlToMarkdown(`${prose}${PRE}`);
+      expect(out).toBe(`${lead}\\\`\\\`\\\`\n\n${FENCED}`);
+    });
+
+    test('a bare < in prose is not mistaken for a tag that hides a literal fence', () => {
+      expect(htmlToMarkdown(`<p>a < b\n\`\`\`\nc</p>${PRE}`)).toBe(`a < b\n\\\`\\\`\\\`\nc\n\n${FENCED}`);
+    });
+
+    test('unclosed <pre> tags do not make the escape pass quadratic', () => {
+      const start = Date.now();
+      htmlToMarkdown('<pre>x'.repeat(40000));
+      expect(Date.now() - start).toBeLessThan(1000);
+    });
+
+    test('backticks inside a code block body are left untouched', () => {
+      const html = '<pre><code>```\n  inner\n```</code></pre>';
+      expect(htmlToMarkdown(html)).toBe('````\n```\n  inner\n```\n````');
+    });
+
+    test('backticks in prose that cannot open a fence are left as written', () => {
+      expect(htmlToMarkdown('<p>use ```js fences</p>')).toBe('use ```js fences');
+    });
+
+    test('backticks inside an attribute value are not touched', () => {
+      expect(htmlToMarkdown('<p><a href="https://example.com/a```b">x</a></p>')).toBe('[x](https://example.com/a```b)');
+    });
+  });
 });

tests/macro-converter.test.js

@@ -2184,21 +2184,23 @@ describe('MacroConverter storageToMarkdown code inside quotes and callouts (#244
   test('a bare ``` paragraph in a callout does not swallow the code blocks after it', () => {
     const storage = macro('info', '<p>```</p>') + codeMacro('', 'one   1') + '<p>x     y</p>' + codeMacro('', 'two   2');
     expect(converter.storageToMarkdown(storage))
-      .toBe('> **INFO**\n> ```\n\n```\none   1\n```\n\nx y\n\n```\ntwo   2\n```');
+      .toBe('> **INFO**\n> \\`\\`\\`\n\n```\none   1\n```\n\nx y\n\n```\ntwo   2\n```');
   });
 
   test('a bare ``` paragraph in a callout does not pair with a fence in a later callout', () => {
     const storage = macro('info', '<p>```</p>') + '<p>mid     text</p>' + macro('note', codeMacro('js', 'keep   1'));
     expect(converter.storageToMarkdown(storage))
-      .toBe('> **INFO**\n> ```\n\nmid text\n\n> **NOTE**\n> ```js\n> keep   1\n> ```');
+      .toBe('> **INFO**\n> \\`\\`\\`\n\nmid text\n\n> **NOTE**\n> ```js\n> keep   1\n> ```');
   });
 
   test('indented loose text between callouts is cleaned, not turned into code on write-back', () => {
     const storage = macro('info', '<p>```</p>') + 'loose\n        indented    text\n' + macro('note', codeMacro('js', 'keep   1'));
     const md = converter.storageToMarkdown(storage);
-    expect(md).toBe('> **INFO**\n> ```\nloose\nindented text\n\n> **NOTE**\n> ```js\n> keep   1\n> ```');
+    expect(md).toBe('> **INFO**\n> \\`\\`\\`\nloose\nindented text\n\n> **NOTE**\n> ```js\n> keep   1\n> ```');
     const storage1 = converter.markdownToStorage(md);
-    expect(storage1).toContain('<p>loose\nindented text</p>');
+    // The escaped literal stays prose; the loose text continues that paragraph
+    // (no blank line separates them) and is not turned into code.
+    expect(storage1).toContain('<p>```\nloose\nindented text</p>');
     expect(storage1).toContain('<![CDATA[keep   1]]>');
   });
 
@@ -2524,7 +2526,7 @@ describe('MacroConverter storageToMarkdown <br/> in flattened contexts (#242)',
       ['-', '\\-'],
       ['===', '\\==='],
       ['* * *', '\\* * *'],
-      ['```js', '\\```js'],
+      ['```js', '\\`\\`\\`js'],
       ['~~~', '\\~~~'],
       ['--- | ---', '\\--- | ---'],
       ['|:---|---:|', '\\|:---|---:|'],
@@ -2633,3 +2635,70 @@ describe('MacroConverter storageToMarkdown <br/> in flattened contexts (#242)',
     });
   });
 });
+
+describe('MacroConverter storageToMarkdown literal fence-like text (#254)', () => {
+  const converter = new MacroConverter({ isCloud: true });
+  const codeMacro = (lang, body) =>
+    `<ac:structured-macro ac:name="code"><ac:parameter ac:name="language">${lang}</ac:parameter><ac:plain-text-body><![CDATA[${body}]]></ac:plain-text-body></ac:structured-macro>`;
+  const macro = (name, body) =>
+    `<ac:structured-macro ac:name="${name}"><ac:rich-text-body>${body}</ac:rich-text-body></ac:structured-macro>`;
+  const code = codeMacro('', '  indented\n    code   here');
+  const FENCED = '```\n  indented\n    code   here\n```';
+
+  test('a <p>```</p> does not capture the next code macro', () => {
+    expect(converter.storageToMarkdown(`<p>\`\`\`</p>${code}`)).toBe(`\\\`\\\`\\\`\n\n${FENCED}`);
+  });
+
+  test('a literal ``` after a nested list does not capture the next code macro', () => {
+    const storage = `<ul><li>a<ul><li>b</li></ul>\`\`\`</li></ul>${code}`;
+    expect(converter.storageToMarkdown(storage)).toBe(`- a\n  - b\n\n  \\\`\\\`\\\`\n\n${FENCED}`);
+  });
+
+  test('a literal fence line after a <br/> is escaped', () => {
+    expect(converter.storageToMarkdown(`<p>x<br/>\`\`\`</p>${code}`)).toBe(`x\n\\\`\\\`\\\`\n\n${FENCED}`);
+  });
+
+  test('backticks written as entities or CDATA are treated the same way', () => {
+    expect(converter.storageToMarkdown(`<p>&#96;&#96;&#96;</p>${code}`)).toBe(`\\\`\\\`\\\`\n\n${FENCED}`);
+    expect(converter.storageToMarkdown(`<p><![CDATA[\`\`\`]]></p>${code}`)).toBe(`\\\`\\\`\\\`\n\n${FENCED}`);
+  });
+
+  test('a literal NBSP before the fence-like run does not hide it', () => {
+    expect(converter.storageToMarkdown(`<p>\u00a0\`\`\`</p>${code}`)).toBe(`\\\`\\\`\\\`\n\n${FENCED}`);
+    expect(converter.storageToMarkdown(`<p>&nbsp;\`\`\`</p>${code}`)).toBe(`\\\`\\\`\\\`\n\n${FENCED}`);
+  });
+
+  test.each([
+    ['four backticks then a matching run', '<p>````<br/>b ``` c</p>', '<p>````\nb ``` c</p>'],
+    ['a run followed by a shorter one', '<p>```<br/>use `` here</p>', '<p>```\nuse `` here</p>'],
+    ['a run and a code-span-like pair', '<p>``` ``</p>', '<p>``` ``</p>'],
+  ])('every backtick of the run is escaped, so %s stays literal on write-back', (_label, storage, expected) => {
+    const md = converter.storageToMarkdown(storage);
+    expect(converter.markdownToStorage(md)).toBe(`${expected}\n`);
+  });
+
+  test('a literal ``` inside a callout stays escaped and the code after it is untouched', () => {
+    expect(converter.storageToMarkdown(macro('info', '<p>```</p>') + code))
+      .toBe(`> **INFO**\n> \\\`\\\`\\\`\n\n${FENCED}`);
+  });
+
+  test('the code macro body is never escaped', () => {
+    expect(converter.storageToMarkdown(codeMacro('', '```\n  inner\n```')))
+      .toBe('````\n```\n  inner\n```\n````');
+  });
+
+  test('code spans and mid-line backticks are left as written', () => {
+    expect(converter.storageToMarkdown('<p><code>```</code></p>')).toBe('```` ``` ````');
+    expect(converter.storageToMarkdown('<p>use ```js fences</p>')).toBe('use ```js fences');
+  });
+
+  test('round-trip keeps the literal text as prose and the code body byte-exact', () => {
+    const md = converter.storageToMarkdown(`<p>\`\`\`</p>${code}`);
+    const storage1 = converter.markdownToStorage(md);
+    expect(storage1).toContain('<p>```</p>');
+    expect(storage1).toContain('<![CDATA[  indented\n    code   here]]>');
+    // Writing back labels a language-less fence `text`; that is unrelated here.
+    expect(converter.storageToMarkdown(storage1))
+      .toBe('\\`\\`\\`\n\n```text\n  indented\n    code   here\n```');
+  });
+});

tests/markdown-cleanup.test.js

@@ -5,6 +5,7 @@ const {
   escapeSentinels,
   finalizeSentinels,
   fenceLength,
+  escapeFenceLikeText,
   splitOnFences,
   isFenceOpenLine,
   cleanupOutsideFence,
@@ -297,3 +298,39 @@ describe('markdown-cleanup QUOTE_MARK sentinel (#244)', () => {
     expect(finalizeSentinels(`${I}${HARD_BREAK}${escapeSentinels(literal)}`)).toBe(` \n${literal}`);
   });
 });
+
+describe('escapeFenceLikeText', () => {
+  test.each([
+    ['```', '\\`\\`\\`'],
+    ['````', '\\`\\`\\`\\`'],
+    ['```js', '\\`\\`\\`js'],
+    ['  ```', '  \\`\\`\\`'],
+    ['a\n```\nb', 'a\n\\`\\`\\`\nb'],
+    ['```\n```', '\\`\\`\\`\n\\`\\`\\`'],
+  ])('escapes a line-leading run of backticks: %j', (input, expected) => {
+    expect(escapeFenceLikeText(input)).toBe(expected);
+  });
+
+  test.each([
+    'use ```js fences',
+    'two `` backticks',
+    'one ` backtick',
+    'no backticks',
+    '',
+  ])('leaves text without a line-leading 3+ run alone: %j', (input) => {
+    expect(escapeFenceLikeText(input)).toBe(input);
+  });
+
+  test('NBSP counts as indentation, and extra indent alternatives can be supplied', () => {
+    expect(escapeFenceLikeText('\u00a0```')).toBe('\u00a0\\`\\`\\`');
+    expect(escapeFenceLikeText(' ```')).toBe(' ```');
+    expect(escapeFenceLikeText(' ```', ' ')).toBe(' \\`\\`\\`');
+  });
+
+  test('an escaped line can no longer open a fence', () => {
+    const text = `${escapeFenceLikeText('```')}\n\n\`\`\`\n  code\n\`\`\``;
+    const segments = splitOnFences(text);
+    expect(segments).toHaveLength(3);
+    expect(segments[1]).toBe('```\n  code\n```');
+  });
+});