fix(rss): preserve complete feed content and resolve relative link (#2432)

* fix(rss): preserve complete feed content and resolve relative link
* Fix extra whitespace, failing tests.

packages/markitdown/src/markitdown/converters/_rss_converter.py

@@ -1,11 +1,12 @@
 import textwrap
 import warnings
+from html import escape
+from urllib.parse import urljoin
 
 from defusedxml import minidom
-from xml.dom import Node
-from xml.dom.minidom import Document, Element
+from xml.dom.minidom import Document, Element, Node
 from typing import BinaryIO, Any, Union
-from bs4 import BeautifulSoup
+from bs4 import BeautifulSoup, Tag
 
 from ._markdownify import _CustomMarkdownify
 from .._stream_info import StreamInfo
@@ -31,6 +32,24 @@ CANDIDATE_FILE_EXTENSIONS = [
 
 ATOM_NAMESPACE = "http://www.w3.org/2005/Atom"
 XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml"
+CONTENT_NAMESPACE = "http://purl.org/rss/1.0/modules/content/"
+XML_NAMESPACE = "http://www.w3.org/XML/1998/namespace"
+
+# Boundaries to retain when reducing markup to readable heading/plain text.
+# Inline elements must not introduce spaces (e.g. co<b>op</b>erate).
+_TEXT_BREAK_ELEMENTS = frozenset(
+    "address article aside blockquote br dd div dl dt figcaption figure footer "
+    "h1 h2 h3 h4 h5 h6 header hr li main nav ol p pre section table td th tr ul".split()
+)
+
+
+def _resolve_url(base_url: str, reference: str) -> str:
+    """Resolve a reference without letting a malformed URI drop the body."""
+    try:
+        return urljoin(base_url, reference)
+    except ValueError:
+        # Leave malformed links to the Markdown converter's existing handling.
+        return reference
 
 
 def _atom_content_kind(content_type: str, *, allow_media_types: bool) -> str:
@@ -107,12 +126,14 @@ class RssConverter(DocumentConverter):
             file_stream.seek(cur_pos)
         return False
 
-    def _feed_type(self, doc: Any) -> str | None:
-        if doc.getElementsByTagName("rss"):
-            return "rss"
+    def _feed_type(self, doc: Document) -> str | None:
         root = doc.documentElement
+        if root is None:
+            return None
+        if root.tagName == "rss":
+            return "rss"
         if root.localName == "feed" and root.namespaceURI in (None, ATOM_NAMESPACE):
-            if root.getElementsByTagNameNS(root.namespaceURI, "entry"):
+            if self._get_children(root, "entry"):
                 # An Atom feed must have a root element of <feed> and at least one <entry>
                 return "atom"
         return None
@@ -128,6 +149,7 @@ class RssConverter(DocumentConverter):
         strict: bool = kwargs.pop("strict", False)
         self._kwargs = kwargs
         doc = minidom.parse(file_stream)
+        doc.documentURI = stream_info.url or kwargs.get("url")
         feed_type = self._feed_type(doc)
 
         if feed_type == "rss":
@@ -145,15 +167,16 @@ class RssConverter(DocumentConverter):
         Returns None if the feed type is not recognized or something goes wrong.
         """
         root = doc.documentElement
-        title = self._get_flattened_text(root, "title")
-        subtitle = self._get_flattened_text(root, "subtitle")
-        entries = root.getElementsByTagNameNS(root.namespaceURI, "entry")
+        assert root is not None
+        title = self._get_flattened_text(root, "title", atom_text=True)
+        subtitle = self._get_flattened_text(root, "subtitle", atom_text=True)
+        entries = self._get_children(root, "entry")
         md_text = f"# {title}\n" if title else ""
 
         if subtitle:
             md_text += f"{subtitle}\n"
         for entry in entries:
-            entry_title = self._get_flattened_text(entry, "title")
+            entry_title = self._get_flattened_text(entry, "title", atom_text=True)
             entry_summary, summary_is_markup = self._get_atom_content(entry, "summary")
             entry_updated = self._get_flattened_text(entry, "updated")
             entry_content, content_is_markup = self._get_atom_content(entry, "content")
@@ -162,14 +185,23 @@ class RssConverter(DocumentConverter):
                 md_text += f"\n## {entry_title}\n"
             if entry_updated:
                 md_text += f"Updated on: {entry_updated}\n"
-            if entry_summary:
-                md_text += self._render_atom_content(
-                    entry_summary, is_markup=summary_is_markup, strict=strict
+            body_parts = (
+                self._render_atom_content(
+                    value,
+                    is_markup=is_markup,
+                    base_url=self._get_field_base_url(entry, tag_name),
+                    strict=strict,
                 )
-            if entry_content:
-                md_text += self._render_atom_content(
-                    entry_content, is_markup=content_is_markup, strict=strict
+                for value, is_markup, tag_name in (
+                    (entry_summary, summary_is_markup, "summary"),
+                    (entry_content, content_is_markup, "content"),
                 )
+                if value
+            )
+            body = "\n\n".join(part for part in body_parts if part)
+            if body and md_text and not md_text.endswith("\n"):
+                md_text += "\n\n"
+            md_text += body
 
         return DocumentConverterResult(
             markdown=md_text,
@@ -179,35 +211,25 @@ class RssConverter(DocumentConverter):
     def _get_atom_content(
         self, entry: Element, tag_name: str
     ) -> tuple[Union[str, None], bool]:
-        """Return an Atom summary or content value, and whether it is markup.
+        """Return an Atom text construct or content, and whether it is markup.
 
         Values flagged as markup are converted by ``_parse_content``; plain text
         is returned verbatim for the caller to emit as-is.
         """
-        nodes = entry.getElementsByTagNameNS(entry.namespaceURI, tag_name)
-        if not nodes:
+        node = self._get_child(entry, tag_name)
+        if node is None:
             return None, True
 
-        node = nodes[0]
         kind = _atom_content_kind(
             node.getAttribute("type"), allow_media_types=tag_name == "content"
         )
-        if kind == "xhtml":
-            return (
-                "".join(
-                    self._localize_xhtml_names(child.cloneNode(True)).toxml()
-                    for child in node.childNodes
-                    if child.nodeType == Node.ELEMENT_NODE
-                ),
-                True,
-            )
         if kind == "binary":
             # A base64-encoded payload; there is no text to render.
             return None, True
 
-        text = self._get_data_by_tag_name(entry, tag_name)
-        if text is None or kind == "html":
-            return text, True
+        text = self._read_content(node, kind=kind)
+        if kind != "text":
+            return text or None, True
 
         # Plain text carries no markup.  Drop the whitespace the feed used to
         # lay the element out -- indentation carried into the output would read
@@ -220,32 +242,18 @@ class RssConverter(DocumentConverter):
         return text.strip(), False
 
     def _render_atom_content(
-        self, value: str, *, is_markup: bool, strict: bool = False
+        self, value: str, *, is_markup: bool, base_url: str = "", strict: bool = False
     ) -> str:
         """Render one Atom summary or content value as markdown."""
         if not is_markup:
             # Plain text is returned verbatim: routing it through the HTML
             # parser drops tag-shaped text such as ``<job_id>`` entirely.
             return value
-        return self._parse_content(value, strict=strict)
+        return self._parse_content(value, base_url=base_url, strict=strict)
 
-    def _localize_xhtml_names(self, node: Node) -> Node:
-        """Rewrite prefixed XHTML element names to their local HTML names.
-
-        Atom permits XHTML content to be namespace-prefixed (e.g. ``x:strong``).
-        The downstream HTML converter dispatches on HTML tag names, so the
-        prefix has to be dropped or the element is treated as an unknown tag
-        and its formatting is lost.
-        """
-        if node.nodeType == Node.ELEMENT_NODE:
-            if node.prefix and node.namespaceURI == XHTML_NAMESPACE:
-                node.tagName = node.nodeName = node.localName
-                node.prefix = None
-            for child in node.childNodes:
-                self._localize_xhtml_names(child)
-        return node
-
-    def _get_flattened_text(self, element: Element, tag_name: str) -> Union[str, None]:
+    def _get_flattened_text(
+        self, element: Element, tag_name: str, *, atom_text: bool = False
+    ) -> Union[str, None]:
         """Get a value that is rendered as a heading or a metadata line.
 
         Feeds are routinely pretty-printed, so a value written as
@@ -255,10 +263,23 @@ class RssConverter(DocumentConverter):
         text, so the layout whitespace is dropped here. A value that is
         nothing but whitespace is reported as absent.
         """
-        value = self._get_data_by_tag_name(element, tag_name)
+        if atom_text:
+            value, is_markup = self._get_atom_content(element, tag_name)
+            if value and is_markup:
+                soup = BeautifulSoup(value, "html.parser")
+                for block in soup.find_all(_TEXT_BREAK_ELEMENTS):
+                    block.insert_before("\n")
+                    block.append("\n")
+                value = soup.get_text()
+        else:
+            value = self._get_data_by_tag_name(element, tag_name)
         if value is None:
             return None
-        return " ".join(part.strip() for part in value.splitlines()).strip() or None
+        # Block boundaries become line breaks, and adjacent blocks leave blank
+        # lines between them; drop the empty parts so the flattened value is
+        # separated by single spaces rather than by runs of them.
+        parts = (part.strip() for part in value.splitlines())
+        return " ".join(part for part in parts if part) or None
 
     def _parse_rss_type(
         self, doc: Document, *, strict: bool = False
@@ -267,14 +288,16 @@ class RssConverter(DocumentConverter):
 
         Returns None if the feed type is not recognized or something goes wrong.
         """
-        root = doc.getElementsByTagName("rss")[0]
-        channel_list = root.getElementsByTagName("channel")
-        if not channel_list:
+        root = doc.documentElement
+        assert root is not None
+        channel = self._get_child(root, "channel")
+        if channel is None:
             raise ValueError("No channel found in RSS feed")
-        channel = channel_list[0]
         channel_title = self._get_flattened_text(channel, "title")
+        # A channel description is a text field; item descriptions carry HTML.
+        # Preserve its existing whitespace while collecting the complete value.
         channel_description = self._get_data_by_tag_name(channel, "description")
-        items = channel.getElementsByTagName("item")
+        items = self._get_children(channel, "item")
         md_text = ""
         if channel_title:
             md_text += f"# {channel_title}\n"
@@ -282,29 +305,44 @@ class RssConverter(DocumentConverter):
             md_text += f"{channel_description}\n"
         for item in items:
             title = self._get_flattened_text(item, "title")
-            description = self._get_data_by_tag_name(item, "description")
+            description = self._get_data_by_tag_name(item, "description", kind="html")
             pubDate = self._get_flattened_text(item, "pubDate")
-            content = self._get_data_by_tag_name(item, "content:encoded")
+            content = self._get_data_by_tag_name(item, "content:encoded", kind="html")
 
             if title:
                 md_text += f"\n## {title}\n"
             if pubDate:
                 md_text += f"Published on: {pubDate}\n"
-            if description:
-                md_text += self._parse_content(description, strict=strict)
-            if content:
-                md_text += self._parse_content(content, strict=strict)
+            body_parts = (
+                self._parse_content(
+                    value,
+                    base_url=self._get_field_base_url(item, tag_name),
+                    strict=strict,
+                )
+                for value, tag_name in (
+                    (description, "description"),
+                    (content, "content:encoded"),
+                )
+                if value
+            )
+            body = "\n\n".join(part for part in body_parts if part)
+            if body and md_text and not md_text.endswith("\n"):
+                md_text += "\n\n"
+            md_text += body
 
         return DocumentConverterResult(
             markdown=md_text,
             title=channel_title,
         )
 
-    def _parse_content(self, content: str, *, strict: bool = False) -> str:
+    def _parse_content(
+        self, content: str, *, base_url: str = "", strict: bool = False
+    ) -> str:
         """Parse the content of an RSS feed item"""
         try:
             # using bs4 because many RSS feeds have HTML-styled content
             soup = BeautifulSoup(content, "html.parser")
+            self._resolve_content_links(soup, base_url)
             return _CustomMarkdownify(**self._kwargs).convert_soup(soup)
         except RecursionError:
             if strict:
@@ -322,20 +360,129 @@ class RssConverter(DocumentConverter):
         except BaseException as _:
             return content
 
-    def _get_data_by_tag_name(
-        self, element: Element, tag_name: str
-    ) -> Union[str, None]:
-        """Get data from first child element with the given tag name.
-        Returns None when no such element is found.
+    def _get_field_base_url(self, element: Element, tag_name: str) -> str:
+        """Apply xml:base from the document root through the selected field.
+
+        Relative overrides resolve against their parent's base. An empty
+        xml:base inherits that base; it does not reset it to the document URL.
+        The document URI is local to this conversion, even when a converter is
+        reused. The source stream's URL also supplies a base for RSS HTML.
         """
-        if element.namespaceURI == ATOM_NAMESPACE:
-            nodes = element.getElementsByTagNameNS(ATOM_NAMESPACE, tag_name)
+        bases: list[str] = []
+        node: Node | None = self._get_child(element, tag_name)
+        while node is not None:
+            if isinstance(node, Element) and node.hasAttributeNS(XML_NAMESPACE, "base"):
+                bases.append(node.getAttributeNS(XML_NAMESPACE, "base"))
+            node = node.parentNode
+        document = element.ownerDocument
+        base_url = (document.documentURI or "") if document is not None else ""
+        for reference in reversed(bases):
+            base_url = _resolve_url(base_url, reference)
+        return base_url
+
+    def _resolve_content_links(self, soup: BeautifulSoup, base_url: str) -> None:
+        """Resolve rendered links/images, retaining nested XML Base scopes.
+
+        Inline XHTML keeps its xml:base attributes during serialization; HTML
+        carried in text/CDATA inherits the enclosing field's base. Traverse
+        iteratively so deeply nested content can still reach the fallback.
+        """
+        stack: list[tuple[Tag, str]] = [(soup, base_url)]
+        while stack:
+            node, inherited_base = stack.pop()
+            override = node.get("xml:base")
+            current_base = (
+                _resolve_url(inherited_base, override)
+                if isinstance(override, str)
+                else inherited_base
+            )
+            attributes: tuple[str, ...] = ()
+            if node.name == "a":
+                attributes = ("href",)
+            elif node.name == "img":
+                attributes = ("src", "data-src")
+            for attribute in attributes:
+                reference = node.get(attribute)
+                # Empty src must still allow markdownify's data-src fallback.
+                if isinstance(reference, str) and (reference or attribute == "href"):
+                    node[attribute] = _resolve_url(current_base, reference)
+            stack.extend(
+                (child, current_base)
+                for child in node.children
+                if isinstance(child, Tag)
+            )
+
+    def _get_children(self, element: Element, tag_name: str) -> list[Element]:
+        """Select fields on their owner, without borrowing descendant metadata."""
+        namespace: str | None
+        if tag_name == "content:encoded":
+            namespace, local_name = CONTENT_NAMESPACE, "encoded"
         else:
-            nodes = element.getElementsByTagName(tag_name)
-        if not nodes:
+            namespace, local_name = element.namespaceURI, tag_name
+        return [
+            child
+            for child in element.childNodes
+            if isinstance(child, Element)
+            and child.namespaceURI == namespace
+            and child.localName == local_name
+        ]
+
+    def _get_child(self, element: Element, tag_name: str) -> Element | None:
+        return next(iter(self._get_children(element, tag_name)), None)
+
+    def _get_data_by_tag_name(
+        self, element: Element, tag_name: str, *, kind: str = "text"
+    ) -> Union[str, None]:
+        """Read the complete value of the first matching direct child field."""
+        node = self._get_child(element, tag_name)
+        if node is None:
             return None
-        fc = nodes[0].firstChild
-        if fc:
-            if hasattr(fc, "data"):
-                return fc.data
-        return None
+        return self._read_content(node, kind=kind) or None
+
+    def _read_content(self, element: Element, *, kind: str) -> str:
+        """Collect plain text, encoded HTML, or inline XML in document order.
+
+        Encoded HTML's direct text/CDATA children are parts of one HTML string.
+        Text inside actual XML elements is already literal and must be escaped
+        when serializing those elements for the HTML parser. In XHTML, all text
+        is literal, including CDATA. Comments and processing instructions carry
+        no display content.
+
+        Use an explicit stack: minidom's recursive cloning/serialization can
+        fail on deep XML before the Markdown renderer's fallback is reached.
+        """
+        parts: list[str] = []
+        stack: list[Node | str] = list(reversed(element.childNodes))
+        while stack:
+            child = stack.pop()
+            if isinstance(child, str):
+                parts.append(child)
+            elif child.nodeType in (Node.TEXT_NODE, Node.CDATA_SECTION_NODE):
+                text = child.nodeValue or ""
+                if kind == "text" or (kind == "html" and child.parentNode is element):
+                    parts.append(text)
+                else:
+                    parts.append(escape(text, quote=False))
+            elif isinstance(child, Element):
+                if kind == "text":
+                    if child.localName.lower() in _TEXT_BREAK_ELEMENTS:
+                        parts.append("\n")
+                        stack.append("\n")
+                else:
+                    # markdownify recognizes local HTML names, not x:strong.
+                    name = (
+                        child.localName
+                        if child.namespaceURI == XHTML_NAMESPACE
+                        else child.tagName
+                    )
+                    attrs = "".join(
+                        f' {attr.name}="{escape(attr.value, quote=True)}"'
+                        for attr in child.attributes.values()
+                    )
+                    if child.childNodes:
+                        parts.append(f"<{name}{attrs}>")
+                        stack.append(f"</{name}>")
+                    else:
+                        parts.append(f"<{name}{attrs}/>")
+                stack.extend(reversed(child.childNodes))
+        return "".join(parts)

packages/markitdown/tests/test_rss_converter.py

@@ -1,4 +1,5 @@
 import io
+import sys
 
 import pytest
 
@@ -338,3 +339,352 @@ def test_atom_plain_text_layout_whitespace_is_removed() -> None:
         "",
         "Then check status.",
     ]
+
+
+def _feed_with_body(field: str, payload: str, *, atom_type: str = "html") -> bytes:
+    if field.startswith("rss-"):
+        tag = "description" if field == "rss-description" else "content:encoded"
+        return (
+            '<rss version="2.0" '
+            'xmlns:content="http://purl.org/rss/1.0/modules/content/">'
+            "<channel><title>Feed</title><description>About the feed.</description>"
+            f"<item><title>Entry</title><{tag}>{payload}</{tag}></item>"
+            "</channel></rss>"
+        ).encode()
+    tag = field.removeprefix("atom-")
+    return (
+        '<feed xmlns="http://www.w3.org/2005/Atom">'
+        "<title>Feed</title><entry><title>Entry</title>"
+        f'<{tag} type="{atom_type}">{payload}</{tag}>'
+        "</entry></feed>"
+    ).encode()
+
+
+@pytest.mark.parametrize(
+    "field", ["rss-description", "rss-content", "atom-summary", "atom-content"]
+)
+@pytest.mark.parametrize(
+    "payload, expected",
+    [
+        ("Plain body.", "Plain body."),
+        ("<![CDATA[<p>Only CDATA.</p>]]>", "Only CDATA."),
+        (
+            "\n    <![CDATA[<p>The <strong>body</strong>.</p>]]>\n  ",
+            "The **body**.",
+        ),
+        (
+            "Before <![CDATA[<str]]><![CDATA[ong>middle</strong>]]> after.",
+            "Before **middle** after.",
+        ),
+        ("<b>nested</b> trailing", "**nested** trailing"),
+        ("leading <b>nested</b> trailing", "leading **nested** trailing"),
+        ("leading <b>nested</b>", "leading **nested**"),
+        ("First<br/>Second", "First  \nSecond"),
+        ("<pre>line 1\n  line 2</pre>", "```\nline 1\n  line 2\n```"),
+        (
+            "\n <!--ignore--><?instruction ignore?><b>nested<!--ignore--></b> trailing",
+            "**nested** trailing",
+        ),
+        (
+            '<p>Read <a href="https://example.com/?a=1&amp;b=2">this</a>.</p>'
+            "<p>Then <em>continue</em>.</p>",
+            "Read [this](https://example.com/?a=1&b=2).\n\nThen *continue*.",
+        ),
+        (
+            "<p>Use &lt;slot&gt; and &amp;lt;literal&amp;gt;.</p>",
+            "Use <slot> and &lt;literal&gt;.",
+        ),
+    ],
+)
+def test_feed_body_preserves_complete_content(
+    field: str, payload: str, expected: str
+) -> None:
+    feed = _feed_with_body(field, payload)
+    stream_info = StreamInfo(extension=".rss" if field.startswith("rss-") else ".atom")
+    result = RssConverter().convert(io.BytesIO(feed), stream_info)
+
+    assert result.markdown.split("## Entry\n", 1)[1] == expected
+
+
+@pytest.mark.parametrize("field", ["atom-summary", "atom-content"])
+@pytest.mark.parametrize("atom_type", ["text", "text/plain"])
+def test_atom_plain_mixed_content_stays_literal(field: str, atom_type: str) -> None:
+    feed = _feed_with_body(
+        field,
+        "Run <![CDATA[<job_id>]]> with <b>&amp;lt;literal&amp;gt;</b>."
+        "<!--ignore--><?instruction ignore?>",
+        atom_type=atom_type,
+    )
+    result = RssConverter().convert(io.BytesIO(feed), StreamInfo(extension=".atom"))
+
+    assert result.markdown.split("## Entry\n", 1)[1] == (
+        "Run <job_id> with &lt;literal&gt;."
+    )
+
+
+@pytest.mark.parametrize("atom_type", ["xhtml", "application/xhtml+xml"])
+def test_atom_xhtml_preserves_literal_text_and_surrounding_content(
+    atom_type: str,
+) -> None:
+    feed = _feed_with_body(
+        "atom-content",
+        'Before &lt;slot&gt; <x:div xmlns:x="http://www.w3.org/1999/xhtml">'
+        "<x:p>Use <![CDATA[<slot> &lt;literal&gt;]]> and <x:b>bold</x:b>.</x:p>"
+        "<!--ignore--><?instruction ignore?>"
+        "<x:pre>line 1\n  line 2</x:pre></x:div> after.",
+        atom_type=atom_type,
+    )
+    result = RssConverter().convert(io.BytesIO(feed), StreamInfo(extension=".atom"))
+
+    body = result.markdown.split("## Entry\n", 1)[1]
+    assert body.startswith("Before <slot>")
+    assert "Use <slot> &lt;literal&gt; and **bold**." in body
+    assert "```\nline 1\n  line 2\n```" in body
+    assert body.endswith("after.")
+    assert "ignore" not in body
+
+
+@pytest.mark.parametrize("extension", [".rss", ".atom"])
+def test_feed_body_fields_are_separated(extension: str) -> None:
+    if extension == ".rss":
+        feed = _feed_with_body("rss-description", "Summary.").replace(
+            b"</item>", b"<content:encoded>Body.</content:encoded></item>"
+        )
+    else:
+        feed = _feed_with_body("atom-summary", "Summary.", atom_type="text").replace(
+            b"</entry>", b'<content type="text">Body.</content></entry>'
+        )
+    result = RssConverter().convert(io.BytesIO(feed), StreamInfo(extension=extension))
+
+    assert result.markdown.split("## Entry\n", 1)[1] == "Summary.\n\nBody."
+
+
+@pytest.mark.parametrize("extension", [".rss", ".atom"])
+@pytest.mark.parametrize(
+    "summary, content, expected",
+    [("<b/>", "Body.", "Body."), ("Summary.", "<b/>", "Summary."), ("<b/>", "", "")],
+)
+def test_empty_markup_fields_do_not_add_body_separators(
+    extension: str, summary: str, content: str, expected: str
+) -> None:
+    if extension == ".rss":
+        feed = _feed_with_body("rss-description", summary).replace(
+            b"</item>", f"<content:encoded>{content}</content:encoded></item>".encode()
+        )
+    else:
+        feed = _feed_with_body("atom-summary", summary).replace(
+            b"</entry>", f'<content type="html">{content}</content></entry>'.encode()
+        )
+    result = RssConverter().convert(io.BytesIO(feed), StreamInfo(extension=extension))
+
+    assert result.markdown.split("## Entry\n", 1)[1] == expected
+
+
+@pytest.mark.parametrize("field", ["rss-description", "atom-content"])
+def test_complete_feed_content_through_public_api(field: str) -> None:
+    feed = _feed_with_body(field, "Before <b>nested</b> <![CDATA[after.]]>")
+    result = MarkItDown().convert_stream(
+        io.BytesIO(feed), stream_info=StreamInfo(extension=".xml")
+    )
+
+    assert result.title == "Feed"
+    assert result.markdown.endswith("## Entry\nBefore **nested** after.")
+
+
+@pytest.mark.parametrize("field", ["rss-description", "atom-content"])
+def test_deep_xml_body_uses_rendering_fallback(field: str) -> None:
+    payload = "<div>" * 500 + "Deep <b>body</b>." + "</div>" * 500
+    feed = _feed_with_body(field, payload, atom_type="xhtml")
+    stream_info = StreamInfo(extension=".rss" if field.startswith("rss-") else ".atom")
+    original_limit = sys.getrecursionlimit()
+    try:
+        sys.setrecursionlimit(200)
+        with pytest.warns(UserWarning, match="too deeply nested"):
+            result = RssConverter().convert(io.BytesIO(feed), stream_info)
+        with pytest.raises(RecursionError):
+            RssConverter().convert(io.BytesIO(feed), stream_info, strict=True)
+    finally:
+        sys.setrecursionlimit(original_limit)
+
+    assert "Deep" in result.markdown
+    assert "body" in result.markdown
+    assert "<div>" not in result.markdown
+
+
+def test_rss_content_namespace_alias_and_field_ownership() -> None:
+    feed = b"""<rss xmlns:c="http://purl.org/rss/1.0/modules/content/">
+      <channel><title>Feed</title><item><title>Entry</title>
+        <extension><description>Wrong body.</description></extension>
+        <c:encoded>Before <b>nested</b> <![CDATA[after.]]></c:encoded>
+      </item></channel>
+    </rss>"""
+    result = RssConverter().convert(io.BytesIO(feed), StreamInfo(extension=".rss"))
+
+    assert result.markdown == "# Feed\n\n## Entry\nBefore **nested** after."
+
+
+@pytest.mark.parametrize(
+    "field", ["rss-description", "rss-content", "atom-summary", "atom-content"]
+)
+@pytest.mark.parametrize("cdata", [False, True])
+def test_feed_relative_links_use_document_url(field: str, cdata: bool) -> None:
+    payload = '<a href="../stories/café%20notes?x=1&amp;y=2#part">Read</a>'
+    if cdata:
+        payload = f"<![CDATA[{payload}]]>"
+    feed = _feed_with_body(field, payload)
+    result = MarkItDown().convert_stream(
+        io.BytesIO(feed),
+        stream_info=StreamInfo(
+            extension=".xml", url="https://example.com/news/feed.xml"
+        ),
+    )
+
+    assert result.markdown.endswith(
+        "[Read](https://example.com/stories/caf%C3%A9%20notes?x=1&y=2#part)"
+    )
+
+
+def test_atom_inherits_xml_base_and_scopes_nested_overrides() -> None:
+    feed = b"""<a:feed xmlns:a="http://www.w3.org/2005/Atom"
+        xmlns:x="http://www.w3.org/1999/xhtml" xml:base="https://example.com/root/">
+      <a:title>Feed</a:title>
+      <a:entry xml:base="entries/">
+        <a:title>Entry</a:title>
+        <a:summary type="html" xml:base="../summaries/">
+          <![CDATA[<a href="summary">Summary</a>]]>
+        </a:summary>
+        <a:content type="xhtml" xml:base="../articles/">
+          <x:div xml:base="chapter/">
+            <x:p xml:base="../appendix/">
+              <x:a xml:base="../../assets/" href="story">Story</x:a>
+              <x:a href="sibling">Sibling</x:a>
+            </x:p>
+            <x:a href="after">After</x:a>
+            <x:img src="diagram.png" alt="Diagram"/>
+            <x:img src="" data-src="../lazy.png" alt="Lazy"/>
+          </x:div>
+        </a:content>
+      </a:entry>
+      <a:entry><a:title>Next</a:title>
+        <a:content type="html">&lt;a href="next"&gt;Next link&lt;/a&gt;</a:content>
+      </a:entry>
+    </a:feed>"""
+    result = RssConverter().convert(
+        io.BytesIO(feed),
+        StreamInfo(extension=".atom", url="https://other.example/feed.xml"),
+    )
+
+    assert "[Summary](https://example.com/root/summaries/summary)" in result.markdown
+    assert "[Story](https://example.com/root/assets/story)" in result.markdown
+    assert (
+        "[Sibling](https://example.com/root/articles/appendix/sibling)"
+        in result.markdown
+    )
+    assert "[After](https://example.com/root/articles/chapter/after)" in result.markdown
+    assert (
+        "![Diagram](https://example.com/root/articles/chapter/diagram.png)"
+        in result.markdown
+    )
+    assert "![Lazy](https://example.com/root/articles/lazy.png)" in result.markdown
+    assert "[Next link](https://example.com/root/next)" in result.markdown
+
+
+@pytest.mark.parametrize(
+    "document_url, feed_base, entry_base, content_base, expected",
+    [
+        (
+            "https://example.com/news/feed.xml",
+            "../stories/",
+            "2026/",
+            "details/",
+            "https://example.com/stories/2026/details/story",
+        ),
+        (
+            "https://example.com/news/feed.xml",
+            "https://other.example/base/",
+            "section/",
+            "",
+            "https://other.example/base/section/story",
+        ),
+        (None, "https://example.com/", "", "", "https://example.com/story"),
+        (None, "", "", "", "story"),
+    ],
+)
+def test_atom_relative_and_empty_xml_base(
+    document_url: str | None,
+    feed_base: str,
+    entry_base: str,
+    content_base: str,
+    expected: str,
+) -> None:
+    xml = (
+        f'<feed xmlns="http://www.w3.org/2005/Atom" xml:base="{feed_base}">'
+        f'<entry xml:base="{entry_base}"><content type="html" xml:base="{content_base}">'
+        '<![CDATA[<a href="story">Read</a>]]></content></entry></feed>'
+    )
+    result = RssConverter().convert(
+        io.BytesIO(xml.encode()), StreamInfo(extension=".atom", url=document_url)
+    )
+
+    assert result.markdown == f"[Read]({expected})"
+
+
+@pytest.mark.parametrize(
+    "href, expected",
+    [
+        ("#section", "https://example.com/news/feed.xml?lang=en#section"),
+        ("?page=2", "https://example.com/news/feed.xml?page=2"),
+        ("/story", "https://example.com/story"),
+        ("//cdn.example.com/story", "https://cdn.example.com/story"),
+        ("https://other.example/story", "https://other.example/story"),
+    ],
+)
+def test_feed_link_reference_forms(href: str, expected: str) -> None:
+    feed = _feed_with_body("atom-content", f'<a href="{href}">Read</a>')
+    result = RssConverter().convert(
+        io.BytesIO(feed),
+        StreamInfo(extension=".atom", url="https://example.com/news/feed.xml?lang=en"),
+    )
+
+    assert result.markdown.endswith(f"[Read]({expected})")
+
+
+def test_link_resolution_preserves_plain_atom_text() -> None:
+    feed = _feed_with_body(
+        "atom-content",
+        '<![CDATA[<a href="story">literal</a>]]>',
+        atom_type="text",
+    )
+    result = RssConverter().convert(
+        io.BytesIO(feed), StreamInfo(extension=".atom", url="https://example.com/feed")
+    )
+
+    assert result.markdown.endswith('<a href="story">literal</a>')
+
+
+def test_link_resolution_retains_existing_invalid_link_handling() -> None:
+    feed = _feed_with_body(
+        "atom-content",
+        '<a href="http://[invalid">Malformed</a> '
+        '<a href="javascript:alert(1)">Script</a> '
+        '<a href="story">Read</a>',
+    )
+    result = RssConverter().convert(
+        io.BytesIO(feed), StreamInfo(extension=".atom", url="https://example.com/feed")
+    )
+
+    assert result.markdown.endswith(
+        "Malformed Script [Read](https://example.com/story)"
+    )
+
+
+def test_document_base_does_not_leak_between_conversions() -> None:
+    feed = _feed_with_body("atom-content", '<a href="story">Read</a>')
+    converter = RssConverter()
+    first = converter.convert(
+        io.BytesIO(feed), StreamInfo(extension=".atom"), url="https://example.com/feed"
+    )
+    second = converter.convert(io.BytesIO(feed), StreamInfo(extension=".atom"))
+
+    assert first.markdown.endswith("[Read](https://example.com/story)")
+    assert second.markdown.endswith("[Read](story)")

packages/markitdown/tests/test_rss_titles.py

@@ -2,6 +2,9 @@
 """Feed and entry titles must survive a pretty-printed feed."""
 
 import io
+import sys
+
+import pytest
 
 from markitdown import StreamInfo
 from markitdown.converters import RssConverter
@@ -146,3 +149,157 @@ def test_rss_single_line_titles_are_unchanged() -> None:
         "Published on: Mon, 01 Jan 2024 00:00:00 GMT\n"
         "Body text."
     )
+
+
+@pytest.mark.parametrize("extension", [".rss", ".atom"])
+@pytest.mark.parametrize(
+    "payload, expected",
+    [
+        ("Quarterly <![CDATA[R&D]]> report", "Quarterly R&D report"),
+        ("<![CDATA[Quarterly ]]><![CDATA[R&D]]> report", "Quarterly R&D report"),
+        ("\n  <![CDATA[Example]]>\n", "Example"),
+        ("<b>nested</b> trailing", "nested trailing"),
+        ("leading <b>nested</b> trailing", "leading nested trailing"),
+        ("leading <b>nested</b>", "leading nested"),
+        ("<b><i>nested</i></b>", "nested"),
+        (
+            " \n<!--ignore--><?instruction ignore?><b>nested</b> trailing",
+            "nested trailing",
+        ),
+        ("co<b>op</b>erate", "cooperate"),
+        ("<p>First</p><p>Second</p>", "First Second"),
+        (
+            "Use &lt;slot&gt; and &amp;lt;literal&amp;gt;",
+            "Use <slot> and &lt;literal&gt;",
+        ),
+        ("", None),
+        ("   \n", None),
+        ("<b/>", None),
+        ("<!--ignore--><?instruction ignore?>", None),
+    ],
+)
+def test_complete_feed_and_entry_titles(
+    extension: str, payload: str, expected: str | None
+) -> None:
+    if extension == ".rss":
+        xml = (
+            f"<rss><channel><title>{payload}</title>"
+            f"<item><title>{payload}</title><description>Body.</description></item>"
+            "</channel></rss>"
+        )
+    else:
+        xml = (
+            f'<feed xmlns="http://www.w3.org/2005/Atom"><title>{payload}</title>'
+            f"<entry><title>{payload}</title><content>Body.</content></entry></feed>"
+        )
+    result = RssConverter().convert(
+        io.BytesIO(xml.encode()), StreamInfo(extension=extension)
+    )
+
+    assert result.title == expected
+    assert result.markdown == (
+        f"# {expected}\n\n## {expected}\nBody." if expected else "Body."
+    )
+
+
+@pytest.mark.parametrize("prefix", ["", "a:"])
+@pytest.mark.parametrize(
+    "content_type, payload, expected",
+    [
+        ("html", "&lt;b&gt;nested&lt;/b&gt; trailing", "nested trailing"),
+        ("html", "<![CDATA[<b>nes]]><![CDATA[ted</b>]]> trailing", "nested trailing"),
+        ("html", "<b>nested</b> trailing", "nested trailing"),
+        ("html", "&lt;p&gt;First&lt;/p&gt;&lt;p&gt;Second&lt;/p&gt;", "First Second"),
+        ("html", "Use &amp;lt;slot&amp;gt;", "Use <slot>"),
+        (
+            "xhtml",
+            '<x:div xmlns:x="http://www.w3.org/1999/xhtml">co<x:b>op</x:b>erate'
+            " <![CDATA[<slot> &lt;literal&gt;]]></x:div>",
+            "cooperate <slot> &lt;literal&gt;",
+        ),
+        ("text/plain", "Use &lt;slot&gt;", "Use <slot>"),
+    ],
+)
+def test_atom_typed_titles_and_subtitles(
+    prefix: str, content_type: str, payload: str, expected: str
+) -> None:
+    xml = (
+        f'<{prefix}feed xmlns="http://www.w3.org/2005/Atom" '
+        'xmlns:a="http://www.w3.org/2005/Atom">'
+        f'<{prefix}title type="{content_type}">{payload}</{prefix}title>'
+        f'<{prefix}subtitle type="{content_type}">{payload}</{prefix}subtitle>'
+        f'<{prefix}entry><{prefix}title type="{content_type}">{payload}</{prefix}title>'
+        f"<{prefix}content>Body.</{prefix}content></{prefix}entry></{prefix}feed>"
+    )
+    result = RssConverter().convert(
+        io.BytesIO(xml.encode()), StreamInfo(extension=".atom")
+    )
+
+    assert result.title == expected
+    assert result.markdown == f"# {expected}\n{expected}\n\n## {expected}\nBody."
+
+
+@pytest.mark.parametrize("extension", [".rss", ".atom"])
+def test_feed_does_not_borrow_nested_metadata(extension: str) -> None:
+    if extension == ".rss":
+        xml = (
+            "<rss><channel><image><title>Wrong feed title</title></image>"
+            "<item><title>Entry</title><description>Body.</description></item>"
+            "<item><extension><title>Wrong entry title</title></extension>"
+            "<description>Second body.</description></item></channel></rss>"
+        )
+    else:
+        xml = (
+            '<feed xmlns="http://www.w3.org/2005/Atom">'
+            "<entry><title>Entry</title><content>Body.</content></entry>"
+            "<entry><source><title>Wrong entry title</title><summary>Wrong body.</summary>"
+            "</source><content>Second body.</content></entry></feed>"
+        )
+    result = RssConverter().convert(
+        io.BytesIO(xml.encode()), StreamInfo(extension=extension)
+    )
+
+    assert result.title is None
+    assert result.markdown.count("## Entry") == 1
+    assert "Wrong" not in result.markdown
+    assert not result.markdown.startswith("# ")
+    assert "Body.\n\nSecond body." in result.markdown
+
+
+@pytest.mark.parametrize("content_type", ["rss", "text", "html", "xhtml"])
+def test_deeply_nested_title_is_read_without_recursion(content_type: str) -> None:
+    payload = "<b>" * 500 + "Deep title" + "</b>" * 500
+    if content_type == "rss":
+        xml = f"<rss><channel><title>{payload}</title></channel></rss>".encode()
+        extension = ".rss"
+    else:
+        if content_type == "html":
+            payload = f"<![CDATA[{payload}]]>"
+        elif content_type == "xhtml":
+            payload = f'<div xmlns="http://www.w3.org/1999/xhtml">{payload}</div>'
+        xml = (
+            f'<feed xmlns="http://www.w3.org/2005/Atom">'
+            f'<title type="{content_type}">{payload}</title><entry/></feed>'
+        ).encode()
+        extension = ".atom"
+    original_limit = sys.getrecursionlimit()
+    try:
+        sys.setrecursionlimit(200)
+        result = RssConverter().convert(
+            io.BytesIO(xml), StreamInfo(extension=extension)
+        )
+    finally:
+        sys.setrecursionlimit(original_limit)
+
+    assert result.title == "Deep title"
+
+
+def test_rss_channel_description_and_date_preserve_complete_text() -> None:
+    xml = b"""<rss><channel><title>Feed</title>
+      <description>About <![CDATA[R&D]]> and <b>research</b>.</description>
+      <item><title>Entry</title><pubDate>Mon, <![CDATA[01 Jan 2024]]> 00:00:00 GMT</pubDate>
+      </item></channel></rss>"""
+    result = RssConverter().convert(io.BytesIO(xml), StreamInfo(extension=".rss"))
+
+    assert "About R&D and research." in result.markdown
+    assert "Published on: Mon, 01 Jan 2024 00:00:00 GMT" in result.markdown