Convert deeply nested documents without recursion (#276)

markdownify/__init__.py

@@ -173,6 +173,18 @@ def _next_block_content_sibling(el):
     return None
 
 
+class _TagFrame(object):
+    """Bookkeeping for a tag whose children are still being converted."""
+    __slots__ = ('node', 'parent_tags', 'parent_tags_for_children', 'children', 'child_strings')
+
+    def __init__(self, node, parent_tags, parent_tags_for_children, children):
+        self.node = node
+        self.parent_tags = parent_tags
+        self.parent_tags_for_children = parent_tags_for_children
+        self.children = children
+        self.child_strings = []
+
+
 class MarkdownConverter(object):
     class DefaultOptions:
         autolinks = True
@@ -236,6 +248,42 @@ def process_tag(self, node, parent_tags=None):
         if parent_tags is None:
             parent_tags = set()
 
+        # Subclasses may override process_element/process_tag to customize how each
+        # element is processed; keep calling them for each child in that case.
+        # Otherwise, descend into child tags using an explicit stack instead of
+        # recursion, so the nesting depth is not bound by the recursion limit.
+        descend_inline = (
+            type(self).process_element is MarkdownConverter.process_element
+            and type(self).process_tag is MarkdownConverter.process_tag
+        )
+
+        stack = [self._open_tag(node, parent_tags)]
+        ancestor_ids = {id(node)}
+        while True:
+            frame = stack[-1]
+            child = next(frame.children, None)
+            if child is None:
+                stack.pop()
+                ancestor_ids.discard(id(frame.node))
+                text = self._close_tag(frame)
+                if not stack:
+                    return text
+                stack[-1].child_strings.append(text)
+            elif descend_inline and isinstance(child, Tag):
+                if id(child) in ancestor_ids:
+                    # A cyclic tree (a descendant that references an ancestor)
+                    # would never finish; skip the repeated tag.
+                    continue
+                stack.append(self._open_tag(child, frame.parent_tags_for_children))
+                ancestor_ids.add(id(child))
+            else:
+                frame.child_strings.append(
+                    self.process_element(child, parent_tags=frame.parent_tags_for_children)
+                )
+
+    def _open_tag(self, node, parent_tags):
+        """Start converting a tag: select the children to convert and build the parent
+        context to propagate down into them."""
         # Collect child elements to process, ignoring whitespace-only text elements
         # adjacent to the inner/outer boundaries of block elements.
         should_remove_inside = should_remove_whitespace_inside(node)
@@ -283,14 +331,15 @@ def _can_ignore(el):
         if node.name in {'pre', 'code', 'kbd', 'samp'}:
             parent_tags_for_children.add('_noformat')
 
-        # Convert the children elements into a list of result strings.
-        child_strings = [
-            self.process_element(el, parent_tags=parent_tags_for_children)
-            for el in children_to_convert
-        ]
+        return _TagFrame(node, parent_tags, parent_tags_for_children, iter(children_to_convert))
+
+    def _close_tag(self, frame):
+        """Finish converting a tag: join the converted children and apply the tag's
+        conversion function."""
+        node = frame.node
 
         # Remove empty string values.
-        child_strings = [s for s in child_strings if s]
+        child_strings = [s for s in frame.child_strings if s]
 
         # Collapse newlines at child element boundaries, if needed.
         if node.name == 'pre' or node.find_parent('pre'):
@@ -321,7 +370,7 @@ def _can_ignore(el):
         # apply this tag's final conversion function
         convert_fn = self.get_conv_fn_cached(node.name)
         if convert_fn is not None:
-            text = convert_fn(node, text, parent_tags=parent_tags)
+            text = convert_fn(node, text, parent_tags=frame.parent_tags)
 
         return text
 

tests/test_advanced.py

@@ -1,3 +1,9 @@
+import sys
+
+from bs4 import BeautifulSoup
+
+from markdownify import MarkdownConverter
+
 from .utils import md
 
 
@@ -37,3 +43,18 @@ def test_code_with_tricky_content():
 def test_special_tags():
     assert md('<!DOCTYPE html>') == ''
     assert md('<![CDATA[foobar]]>') == 'foobar'
+
+
+def test_deeply_nested():
+    # Long chains of nested tags (e.g. quoted replies wrapped in <div>s by mail
+    # clients) must not run into the interpreter's recursion limit.
+    depth = sys.getrecursionlimit()
+    assert md('<div>' * depth + 'hello' + '</div>' * depth) == '\n\nhello\n\n'
+
+
+def test_cyclic_tree():
+    # A descendant that references an ancestor (seen from some PDF-to-HTML
+    # pipelines) must not make the conversion loop forever.
+    soup = BeautifulSoup('<div><p>hello</p></div>', 'html.parser')
+    soup.find('p').contents.append(soup.find('div'))
+    assert MarkdownConverter().convert_soup(soup) == 'hello'

tests/test_custom_converter.py

@@ -42,3 +42,22 @@ def test_soup():
     html = '<b>test</b>'
     soup = BeautifulSoup(html, 'html.parser')
     assert MarkdownConverter().convert_soup(soup) == '**test**'
+
+
+class TrackingConverter(MarkdownConverter):
+    """
+    Create a custom MarkdownConverter that records every tag it processes
+    """
+    def __init__(self, **options):
+        super().__init__(**options)
+        self.tag_names = []
+
+    def process_tag(self, node, parent_tags=None):
+        self.tag_names.append(node.name)
+        return super().process_tag(node, parent_tags=parent_tags)
+
+
+def test_process_tag_override():
+    converter = TrackingConverter()
+    assert converter.convert('<div><p><b>text</b></p></div>') == '**text**'
+    assert converter.tag_names == ['[document]', 'div', 'p', 'b']