diff --git a/docs/changelog.md b/docs/changelog.md index 94b4a2f5..851b3f26 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -23,6 +23,11 @@ See the [Contributing Guide](contributing.md) for details. and closing backticks did not match (#1668). * Escape the language name in the `codehilite` extension's non-Pygments output to maintain consistency with Pygments output. +* Treat block-level raw HTML tags which are indented more than three spaces as + raw HTML blocks when they are nested within a containing block (a list item, + admonition, etc.), instead of incorrectly splitting them into separate + paragraphs at blank lines. Raw HTML blocks within indented code blocks are + still treated as code (#1096). ## [3.11.2] - 2026-10-19 diff --git a/markdown/extensions/md_in_html.py b/markdown/extensions/md_in_html.py index 1a4ec0e2..45494466 100644 --- a/markdown/extensions/md_in_html.py +++ b/markdown/extensions/md_in_html.py @@ -122,7 +122,7 @@ def handle_starttag(self, tag, attrs): if ( tag in self.block_level_tags and - (self.at_line_start() or self.intail or self.mdstarted and self.mdstarted[-1]) + (self.at_line_start() or self.indented_line_start() or self.intail or self.mdstarted and self.mdstarted[-1]) ): # Valueless attribute (ex: ``) results in `[('checked', None)]`. # Convert to `{'checked': 'checked'}`. @@ -279,7 +279,7 @@ def handle_empty_tag(self, data, is_block): self.handle_data(data) def parse_pi(self, i: int) -> int: - if self.at_line_start() or self.intail or self.mdstack: + if self.at_line_start() or self.indented_line_start() or self.intail or self.mdstack: # The same override exists in `HTMLExtractor` without the check # for `mdstack`. Therefore, use parent of `HTMLExtractor` instead. return super(HTMLExtractor, self).parse_pi(i) @@ -289,7 +289,7 @@ def parse_pi(self, i: int) -> int: return i + 2 def parse_html_declaration(self, i: int) -> int: - if self.at_line_start() or self.intail or self.mdstack: + if self.at_line_start() or self.indented_line_start() or self.intail or self.mdstack: if self.rawdata[i:i+3] == ' int: @@ -202,6 +204,76 @@ def at_line_start(self) -> bool: # Confirm up to first 3 chars are whitespace return self.rawdata[self.line_offset:self.line_offset + self.offset].strip() == '' + def indented_line_start(self) -> bool: + """ + Returns True if current position is at the start of a line which is + preceded by more than three spaces of indentation. + + A block-level tag preceded by such indentation may be nested within a + containing block (a list item, admonition, etc.) rather than being part + of an indented code block. The indentation is preserved on the line + which holds the placeholder so the block parser can make that decision. + """ + if self.offset <= 3: + return False + return self.rawdata[self.line_offset:self.line_offset + self.offset].strip() == '' + + def strip_leading_indent(self) -> None: + """ + Remove the whitespace preceding the current position from `cleandoc`. + + The whitespace is re-added to the placeholder line when the raw block + is stashed (see `get_stash_prefix`). This ensures that the placeholder + retains the indentation of the original start tag. + """ + remaining = self.offset + while remaining and self.cleandoc: + chunk = self.cleandoc[-1] + cut = len(chunk) - len(chunk.rstrip(' ')) + if not cut: + break + take = min(cut, remaining) + self.cleandoc[-1] = chunk[:-take] + remaining -= take + + def get_stash_prefix(self) -> str: + """ Returns the indentation to prefix the raw block placeholder with. """ + return ' ' * self.inraw_indent + + def dedent_raw(self, text: str) -> str: + """ + Remove the indentation of the raw block's start tag from each line of + the raw block's content (after the first line). + """ + if not self.inraw_indent: + return text + lines = text.split('\n') + for i in range(1, len(lines)): + line = lines[i] + j = 0 + while j < self.inraw_indent and j < len(line) and line[j] == ' ': + j += 1 + lines[i] = line[j:] + return '\n'.join(lines) + + def stash_raw_block(self) -> None: + """ + Store the current raw block in the `htmlStash` and insert its + placeholder into `cleandoc`. + + For an indented raw block, the placeholder line preserves the + indentation of the start tag and the stashed content records that + indentation (see [`IndentedRawHtml`][markdown.util.IndentedRawHtml]). + """ + content = self.dedent_raw(''.join(self._cache)) + if self.inraw_indent: + html = util.IndentedRawHtml(content, self.inraw_indent) + else: + html = content + self.cleandoc.append(self.get_stash_prefix() + self.md.htmlStash.store(html)) + self._cache = [] + self.inraw_indent = 0 + def get_endtag_text(self, tag: str) -> str: """ Returns the text of the end tag. @@ -223,10 +295,20 @@ def handle_starttag(self, tag: str, attrs: Sequence[tuple[str, str]]): self.handle_startendtag(tag, attrs) return - if self.md.is_block_level(tag) and (self.intail or (self.at_line_start() and not self.inraw)): + if self.md.is_block_level(tag) and ( + self.intail or ((self.at_line_start() or self.indented_line_start()) and not self.inraw) + ): # Started a new raw block. Prepare stack. self.inraw = True - self.cleandoc.append('\n') + if self.intail or self.at_line_start(): + self.inraw_indent = 0 + self.cleandoc.append('\n') + else: + # The start tag is indented. Preserve the indentation on the + # placeholder line so the block parser can determine whether + # this is a nested raw block or an indented code block. + self.inraw_indent = self.offset + self.strip_leading_indent() text = self.get_starttag_text() if self.inraw: @@ -254,15 +336,22 @@ def handle_endtag(self, tag: str): if blank_line_re.match(self.rawdata[self.line_offset + self.offset + len(text):]): # Preserve blank line and end of raw block. self._cache.append('\n') + if self.inraw_indent: + # The raw block was indented. The blank line which + # follows in the source separates the placeholder from + # any following content, so no extra newlines are + # needed here. + self.inraw = False + self.stash_raw_block() + return else: # More content exists after `endtag`. self.intail = True # Reset stack. self.inraw = False - self.cleandoc.append(self.md.htmlStash.store(''.join(self._cache))) + self.stash_raw_block() # Insert blank line between this and next line. self.cleandoc.append('\n\n') - self._cache = [] else: self.cleandoc.append(text) self.close_inline_tag(tag) @@ -317,6 +406,27 @@ def handle_empty_tag(self, data: str, is_block: bool): self.cleandoc.append(self.md.htmlStash.store(data)) # Insert blank line between this and next line. self.cleandoc.append('\n\n') + elif self.indented_line_start() and is_block: + # Handle this as a standalone indented raw block. The indentation + # is preserved on the placeholder line so the block parser can + # determine whether this is nested content or a code block. + self.inraw_indent = self.offset + if blank_line_re.match(self.rawdata[self.line_offset + self.offset + len(data):]): + # Preserve blank line after tag in raw block. The blank line + # which follows in the source separates the placeholder from + # any following content, so no extra newlines are needed here. + data += '\n' + self.strip_leading_indent() + self._cache.append(data) + self.stash_raw_block() + else: + # More content exists after tag. + self.intail = True + self.strip_leading_indent() + self._cache.append(data) + self.stash_raw_block() + # Insert blank line between this and next line. + self.cleandoc.append('\n\n') else: self.cleandoc.append(data) @@ -347,7 +457,7 @@ def unknown_decl(self, data: str): self.handle_empty_tag(' int: - if self.at_line_start() or self.intail: + if self.at_line_start() or self.indented_line_start() or self.intail: return super().parse_pi(i) # This is not the beginning of a raw block so treat as plain data # and avoid consuming any tags which may follow (see #1066). @@ -369,7 +479,7 @@ def parse_comment(self, i, report=True): return match.end() def parse_html_declaration(self, i: int) -> int: - if self.at_line_start() or self.intail: + if self.at_line_start() or self.indented_line_start() or self.intail: if self.rawdata[i:i+3] == ' util.Registry[Treeprocessor]: """ Build the default `treeprocessors` for Markdown. """ treeprocessors = util.Registry() + treeprocessors.register(RawHtmlInCodeTreeprocessor(md), 'raw_html_in_code', 35) treeprocessors.register(InlineProcessor(md), 'inline', 20) treeprocessors.register(PrettifyTreeprocessor(md), 'prettify', 10) treeprocessors.register(UnescapeTreeprocessor(md), 'unescape', 0) @@ -481,3 +482,63 @@ def run(self, root: etree.Element) -> None: # Unescape attribute values for key, value in elem.items(): elem.set(key, self.unescape(value)) + + +class RawHtmlInCodeTreeprocessor(Treeprocessor): + """ Restore indented raw HTML blocks which ended up in a code block. """ + + PLACEHOLDER_RE = re.compile(util.HTML_PLACEHOLDER % r'([0-9]+)') + + def run(self, root: etree.Element) -> None: + """ + Replace raw HTML placeholders within code blocks with escaped text. + + A block level tag which is indented by more than `tab_length` spaces + may be nested within a containing block (a list item, an admonition, + etc.) rather than being part of an indented code block. Which of the + two applies can only be determined once the document has been parsed + into blocks, so the raw block is stashed and its placeholder retains + the indentation (see + [`HTMLExtractor`][markdown.htmlparser.HTMLExtractor]). When the block + parser determines that the placeholder is in fact code, the stashed + content is restored here as escaped text so that it is displayed + rather than rendered as markup. + + This runs before the `hilite` tree processor of the `codehilite` + extension (priority `30`) so that the extension receives the code as + text and can apply its own escaping to it. + """ + if not self.md.htmlStash.html_counter: + return + for block in root.iter('pre'): + if len(block) == 1 and block[0].tag == 'code' and block[0].text: + block[0].text = util.AtomicString( + self.PLACEHOLDER_RE.sub(self.restore, block[0].text) + ) + + def restore(self, m: re.Match[str]) -> str: + """ Return the escaped content of an indented raw HTML block. """ + key = int(m.group(1)) + if key >= self.md.htmlStash.html_counter: + return m.group(0) + entry = self.md.htmlStash.rawHtmlBlocks[key] + if not isinstance(entry, util.IndentedRawHtml): + # Only indented raw blocks are expected to occur within code. + return m.group(0) + html = str(entry) + # The code block only strips `tab_length` spaces from each line while + # the start tag of the raw block was indented by `entry.indent` spaces. + # Restore the difference on the lines which follow the start tag. The + # indentation of the start tag itself is retained by the placeholder + # line, which the code block only strips `tab_length` spaces from. + extra = entry.indent - self.md.tab_length + if extra > 0: + lines = html.split('\n') + html = '\n'.join([lines[0]] + [ + ' ' * extra + line if line else line + for line in lines[1:] + ]) + if html.endswith('\n'): + # The code block adds the trailing newline itself. + html = html[:-1] + return util.code_escape(html) diff --git a/markdown/util.py b/markdown/util.py index 29ffa247..b0786c80 100644 --- a/markdown/util.py +++ b/markdown/util.py @@ -210,6 +210,27 @@ class TagData(TypedDict): right_index: int +class IndentedRawHtml(str): + """ + A raw HTML block which was extracted from a line indented by more than + `tab_length` spaces. + + The content is stored with the indentation of its start tag removed and + the `indent` attribute preserves the number of spaces which were removed. + Consumers (such as `RawHtmlPostprocessor`) can use `indent` to restore + the appropriate indentation for the context in which the placeholder ends + up (for example, within an indented code block). Otherwise it behaves as + a normal string. + """ + + def __new__(cls, html: str, indent: int): + obj = super().__new__(cls, html) + obj.indent = indent + return obj + + indent: int + + class HtmlStash: """ This class is used for stashing HTML objects that we extract diff --git a/tests/test_syntax/blocks/test_html_blocks.py b/tests/test_syntax/blocks/test_html_blocks.py index e68f6d76..45caeb5e 100644 --- a/tests/test_syntax/blocks/test_html_blocks.py +++ b/tests/test_syntax/blocks/test_html_blocks.py @@ -2005,3 +2005,67 @@ def test_unterminated_endtag(self): ''' ) ) + + +class TestIndentedHtmlBlocks(TestCase): + """ Raw HTML nested in indented blocks (issue #1096). """ + + def test_indented_code_block_with_blank_lines(self): + # Ensure an indented HTML block with blank lines remains a code block. + # See #1096. + self.assertMarkdownRenders( + self.dedent( + """ + A paragraph. + +
+ code + + more code +
+ """ + ), + self.dedent( + """ +

A paragraph.

+ 
<div>
+                code
+
+                more code
+                </div>
+
+ """ + ) + ) + + def test_raw_indented_in_list_item(self): + # A raw HTML block nested in a list item should not be split into + # separate paragraphs at blank lines. See #1096. + self.assertMarkdownRenders( + self.dedent( + """ + * A list item with a nested raw HTML block. + +
+ Some text + + Some more text +
+ """ + ), + self.dedent( + """ + + """ + ) + ) diff --git a/tests/test_syntax/extensions/test_admonition.py b/tests/test_syntax/extensions/test_admonition.py index 268633dd..ac4d25b8 100644 --- a/tests/test_syntax/extensions/test_admonition.py +++ b/tests/test_syntax/extensions/test_admonition.py @@ -263,3 +263,33 @@ def test_admonition_first_indented(self): ), extensions=['admonition'] ) + + def test_with_raw_html_block(self): + # A raw HTML block nested in an admonition should not be split into + # separate paragraphs at blank lines. See #1096. + self.assertMarkdownRenders( + self.dedent( + ''' + !!! note "Admonition" +
+ Some text + + Some more text +
+ ''' + ), + self.dedent( + ''' +
+

Admonition

+
+ Some text + + Some more text +
+ +
+ ''' + ), + extensions=['admonition'] + ) diff --git a/tests/test_syntax/extensions/test_md_in_html.py b/tests/test_syntax/extensions/test_md_in_html.py index c89ae6f8..d9767d6d 100644 --- a/tests/test_syntax/extensions/test_md_in_html.py +++ b/tests/test_syntax/extensions/test_md_in_html.py @@ -1587,10 +1587,80 @@ def test_registered_custom_element_markdown_attr(self): self.assertEqual(md.convert(src), '\n

asdf

\n') +class TestIndentedHtmlBlockswMdInHTML(TestCase): + """ Raw HTML nested in indented blocks with md_in_html enabled (issue #1096). """ + + default_kwargs = {'extensions': ['md_in_html']} + + def test_indented_code_block_with_blank_lines(self): + # An indented HTML block with blank lines remains a code block, even + # with md_in_html enabled. See #1096. + self.assertMarkdownRenders( + self.dedent( + """ + A paragraph. + +
+ code + + more code +
+ """ + ), + self.dedent( + """ +

A paragraph.

+ 
<div>
+                code
+
+                more code
+                </div>
+
+ """ + ) + ) + + def test_raw_indented_in_list_item(self): + # A raw HTML block nested in a list item should not be split into + # separate paragraphs at blank lines, even with md_in_html enabled. + # See #1096. + self.assertMarkdownRenders( + self.dedent( + """ + * A list item with a nested raw HTML block. + +
+ Some text + + Some more text +
+ """ + ), + self.dedent( + """ + + """ + ) + ) + + def load_tests(loader, tests, pattern): """ Ensure `TestHTMLBlocks` doesn't get run twice by excluding it here. """ suite = TestSuite() - for test_class in [TestDefaultwMdInHTML, TestMdInHTML, TestMarkdownInHTMLPostProcessor]: + for test_class in [ + TestDefaultwMdInHTML, TestMdInHTML, TestMarkdownInHTMLPostProcessor, + TestIndentedHtmlBlockswMdInHTML + ]: tests = loader.loadTestsFromTestCase(test_class) suite.addTests(tests) return suite