fix: DocumentCleaner remove_regex strips only pages where the regex matched

haystack/components/preprocessors/document_cleaner.py

@@ -63,6 +63,7 @@ def __init__(
             which is supported by `TextFileToDocument` and `AzureOCRDocumentConverter`.
         :param remove_substrings: List of substrings to remove from the text.
         :param remove_regex: Regex to match and replace substrings by "".
+            Pages where the regex matches are also stripped of leading and trailing whitespace.
         :param keep_id: If `True`, keeps the IDs of the original documents.
         :param unicode_normalization: Unicode normalization form to apply to the text.
             Note: This will run before any other steps.
@@ -231,11 +232,15 @@ def _remove_regex(self, text: str, regex: str) -> str:
 
         :param text: Text to clean.
         :param regex: Regex to match and replace substrings by "".
-        :returns: The text without the substrings that match the regex.
+        :returns: The text without the substrings that match the regex. Pages with at least one match are also
+            stripped of leading and trailing whitespace; other pages are left unchanged.
         """
-        texts = text.split("\f")
-        cleaned_text = [re.sub(regex, "", text).strip() for text in texts]
-        return "\f".join(cleaned_text)
+        pages = text.split("\f")
+        cleaned_pages = []
+        for page in pages:
+            cleaned_page, n_matches = re.subn(regex, "", page)
+            cleaned_pages.append(cleaned_page.strip() if n_matches > 0 else cleaned_page)
+        return "\f".join(cleaned_pages)
 
     def _replace_regexes(self, text: str, replace_regexes: dict[str, str]) -> str:
         """

releasenotes/notes/document-cleaner-remove-regex-keep-whitespace-f012e562a73d8932.yaml

@@ -0,0 +1,7 @@
+---
+fixes:
+  - |
+    Fixed ``DocumentCleaner`` removing indentation and newlines at the start and end of every page when
+    ``remove_regex`` was set, even on pages where the regex matched nothing. Pages without a match are now left
+    unchanged, which preserves Markdown and code content when ``remove_extra_whitespaces``, ``remove_empty_lines``
+    and ``strip_whitespaces`` are disabled. Pages where the regex matches are still stripped as before.

test/components/preprocessors/test_document_cleaner.py

@@ -122,6 +122,27 @@ def test_remove_regex(self):
         assert len(result["documents"]) == 1
         assert result["documents"][0].content == "This is a text\fwith some words."
 
+    @pytest.mark.parametrize(
+        "content, expected",
+        [
+            ("    def f():\n        return 1\n", "    def f():\n        return 1\n"),
+            ("    keep indent [TODO] here\n", "keep indent  here"),
+            ("\n# Page 1 [TODO]\n\f\n## Page 2\n    indented\n", "# Page 1\f\n## Page 2\n    indented\n"),
+        ],
+        ids=["no-match", "match", "multi-page"],
+    )
+    def test_remove_regex_strips_only_matching_pages(self, content, expected):
+        cleaner = DocumentCleaner(remove_empty_lines=False, remove_extra_whitespaces=False, remove_regex=r"\[TODO\]")
+        result = cleaner.run(documents=[Document(content=content)])
+        assert result["documents"][0].content == expected
+
+    def test_remove_regex_with_strip_whitespaces(self):
+        cleaner = DocumentCleaner(
+            remove_empty_lines=False, remove_extra_whitespaces=False, remove_regex=r"Page \d$", strip_whitespaces=True
+        )
+        result = cleaner.run(documents=[Document(content="\n  Body 1\nPage 1\f  Body 2  \n")])
+        assert result["documents"][0].content == "Body 1\f  Body 2"
+
     def test_remove_repeated_substrings(self):
         cleaner = DocumentCleaner(
             remove_empty_lines=False, remove_extra_whitespaces=False, remove_repeated_substrings=True