From 3732f75134cbea8f178d2fe75c8c8b07aa1a22a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AD=99=E6=84=8F=E5=8D=9A?= Date: Wed, 21 Oct 2026 19:43:41 +1100 Subject: [PATCH 1/2] fix: keep surrounding whitespace in DocumentCleaner remove_regex `_remove_regex` called `.strip()` on every page, so `remove_regex` removed indentation and newlines at page edges even when the regex matched nothing and all whitespace cleaning options were disabled. Closes #13357 Co-Authored-By: Claude Opus 5.5 --- .../preprocessors/document_cleaner.py | 2 +- ...egex-keep-whitespace-f012e562a73d8932.yaml | 15 +++++++++++++ .../preprocessors/test_document_cleaner.py | 21 +++++++++++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/document-cleaner-remove-regex-keep-whitespace-f012e562a73d8932.yaml diff --git a/haystack/components/preprocessors/document_cleaner.py b/haystack/components/preprocessors/document_cleaner.py index c51ac13e13f..675d266800b 100644 --- a/haystack/components/preprocessors/document_cleaner.py +++ b/haystack/components/preprocessors/document_cleaner.py @@ -234,7 +234,7 @@ def _remove_regex(self, text: str, regex: str) -> str: :returns: The text without the substrings that match the regex. """ texts = text.split("\f") - cleaned_text = [re.sub(regex, "", text).strip() for text in texts] + cleaned_text = [re.sub(regex, "", text) for text in texts] return "\f".join(cleaned_text) def _replace_regexes(self, text: str, replace_regexes: dict[str, str]) -> str: diff --git a/releasenotes/notes/document-cleaner-remove-regex-keep-whitespace-f012e562a73d8932.yaml b/releasenotes/notes/document-cleaner-remove-regex-keep-whitespace-f012e562a73d8932.yaml new file mode 100644 index 00000000000..c2fbf267ab4 --- /dev/null +++ b/releasenotes/notes/document-cleaner-remove-regex-keep-whitespace-f012e562a73d8932.yaml @@ -0,0 +1,15 @@ +--- +upgrade: + - | + ``DocumentCleaner`` and ``DocumentPreprocessor`` no longer strip leading and trailing whitespace from each page + when ``remove_regex`` is set. Only the regex matches are removed, so cleaned content (and the generated document + IDs) can change if your text has indentation or newlines at the start or end of a page. + To keep the previous output, also pass ``replace_regexes={r"^\s+|\s+$": ""}``, which strips each page the same + way. Use ``strip_whitespaces=True`` instead if stripping the start and end of the whole document is enough. + ``DocumentPreprocessor`` does not expose these parameters; to keep the previous output there, connect a + ``DocumentSplitter`` to a ``DocumentCleaner`` configured as above. +fixes: + - | + Fixed ``DocumentCleaner`` removing indentation and newlines at the start and end of every page when + ``remove_regex`` was set, even if the regex matched nothing and ``remove_extra_whitespaces``, + ``remove_empty_lines`` and ``strip_whitespaces`` were all disabled. This damaged Markdown and code content. diff --git a/test/components/preprocessors/test_document_cleaner.py b/test/components/preprocessors/test_document_cleaner.py index 12dd1ca7e2b..1a88fedcd93 100644 --- a/test/components/preprocessors/test_document_cleaner.py +++ b/test/components/preprocessors/test_document_cleaner.py @@ -122,6 +122,27 @@ def test_remove_regex(self): assert len(result["documents"]) == 1 assert result["documents"][0].content == "This is a text\fwith some words." + @pytest.mark.parametrize( + "content, expected", + [ + (" def f():\n return 1\n", " def f():\n return 1\n"), + (" keep indent [TODO] here\n", " keep indent here\n"), + ("\n# Page 1 [TODO]\n\f\n## Page 2\n indented\n", "\n# Page 1 \n\f\n## Page 2\n indented\n"), + ], + ids=["no-match", "match", "multi-page"], + ) + def test_remove_regex_preserves_surrounding_whitespace(self, content, expected): + cleaner = DocumentCleaner(remove_empty_lines=False, remove_extra_whitespaces=False, remove_regex=r"\[TODO\]") + result = cleaner.run(documents=[Document(content=content)]) + assert result["documents"][0].content == expected + + def test_remove_regex_with_strip_whitespaces(self): + cleaner = DocumentCleaner( + remove_empty_lines=False, remove_extra_whitespaces=False, remove_regex=r"Page \d$", strip_whitespaces=True + ) + result = cleaner.run(documents=[Document(content="\n Body 1\nPage 1\fBody 2\nPage 2")]) + assert result["documents"][0].content == "Body 1\n\fBody 2" + def test_remove_repeated_substrings(self): cleaner = DocumentCleaner( remove_empty_lines=False, remove_extra_whitespaces=False, remove_repeated_substrings=True From fb8917bb4fcc0af0ec6da43888c2e97b42180b47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AD=99=E6=84=8F=E5=8D=9A?= Date: Mon, 26 Oct 2026 23:41:38 +1100 Subject: [PATCH 2/2] fix: strip only pages where remove_regex matched Use re.subn and strip a page only when the regex matched at least once, so pages without a match keep their whitespace while existing outputs for matching pages (e.g. footer removal) and their document IDs stay the same. Co-Authored-By: Claude Opus 5.5 --- .../components/preprocessors/document_cleaner.py | 13 +++++++++---- ...ove-regex-keep-whitespace-f012e562a73d8932.yaml | 14 +++----------- .../preprocessors/test_document_cleaner.py | 10 +++++----- 3 files changed, 17 insertions(+), 20 deletions(-) diff --git a/haystack/components/preprocessors/document_cleaner.py b/haystack/components/preprocessors/document_cleaner.py index 675d266800b..9a54fa7e9f4 100644 --- a/haystack/components/preprocessors/document_cleaner.py +++ b/haystack/components/preprocessors/document_cleaner.py @@ -63,6 +63,7 @@ def __init__( which is supported by `TextFileToDocument` and `AzureOCRDocumentConverter`. :param remove_substrings: List of substrings to remove from the text. :param remove_regex: Regex to match and replace substrings by "". + Pages where the regex matches are also stripped of leading and trailing whitespace. :param keep_id: If `True`, keeps the IDs of the original documents. :param unicode_normalization: Unicode normalization form to apply to the text. Note: This will run before any other steps. @@ -231,11 +232,15 @@ def _remove_regex(self, text: str, regex: str) -> str: :param text: Text to clean. :param regex: Regex to match and replace substrings by "". - :returns: The text without the substrings that match the regex. + :returns: The text without the substrings that match the regex. Pages with at least one match are also + stripped of leading and trailing whitespace; other pages are left unchanged. """ - texts = text.split("\f") - cleaned_text = [re.sub(regex, "", text) for text in texts] - return "\f".join(cleaned_text) + pages = text.split("\f") + cleaned_pages = [] + for page in pages: + cleaned_page, n_matches = re.subn(regex, "", page) + cleaned_pages.append(cleaned_page.strip() if n_matches > 0 else cleaned_page) + return "\f".join(cleaned_pages) def _replace_regexes(self, text: str, replace_regexes: dict[str, str]) -> str: """ diff --git a/releasenotes/notes/document-cleaner-remove-regex-keep-whitespace-f012e562a73d8932.yaml b/releasenotes/notes/document-cleaner-remove-regex-keep-whitespace-f012e562a73d8932.yaml index c2fbf267ab4..7ee7f5450de 100644 --- a/releasenotes/notes/document-cleaner-remove-regex-keep-whitespace-f012e562a73d8932.yaml +++ b/releasenotes/notes/document-cleaner-remove-regex-keep-whitespace-f012e562a73d8932.yaml @@ -1,15 +1,7 @@ --- -upgrade: - - | - ``DocumentCleaner`` and ``DocumentPreprocessor`` no longer strip leading and trailing whitespace from each page - when ``remove_regex`` is set. Only the regex matches are removed, so cleaned content (and the generated document - IDs) can change if your text has indentation or newlines at the start or end of a page. - To keep the previous output, also pass ``replace_regexes={r"^\s+|\s+$": ""}``, which strips each page the same - way. Use ``strip_whitespaces=True`` instead if stripping the start and end of the whole document is enough. - ``DocumentPreprocessor`` does not expose these parameters; to keep the previous output there, connect a - ``DocumentSplitter`` to a ``DocumentCleaner`` configured as above. fixes: - | Fixed ``DocumentCleaner`` removing indentation and newlines at the start and end of every page when - ``remove_regex`` was set, even if the regex matched nothing and ``remove_extra_whitespaces``, - ``remove_empty_lines`` and ``strip_whitespaces`` were all disabled. This damaged Markdown and code content. + ``remove_regex`` was set, even on pages where the regex matched nothing. Pages without a match are now left + unchanged, which preserves Markdown and code content when ``remove_extra_whitespaces``, ``remove_empty_lines`` + and ``strip_whitespaces`` are disabled. Pages where the regex matches are still stripped as before. diff --git a/test/components/preprocessors/test_document_cleaner.py b/test/components/preprocessors/test_document_cleaner.py index 1a88fedcd93..5ea8246a9fe 100644 --- a/test/components/preprocessors/test_document_cleaner.py +++ b/test/components/preprocessors/test_document_cleaner.py @@ -126,12 +126,12 @@ def test_remove_regex(self): "content, expected", [ (" def f():\n return 1\n", " def f():\n return 1\n"), - (" keep indent [TODO] here\n", " keep indent here\n"), - ("\n# Page 1 [TODO]\n\f\n## Page 2\n indented\n", "\n# Page 1 \n\f\n## Page 2\n indented\n"), + (" keep indent [TODO] here\n", "keep indent here"), + ("\n# Page 1 [TODO]\n\f\n## Page 2\n indented\n", "# Page 1\f\n## Page 2\n indented\n"), ], ids=["no-match", "match", "multi-page"], ) - def test_remove_regex_preserves_surrounding_whitespace(self, content, expected): + def test_remove_regex_strips_only_matching_pages(self, content, expected): cleaner = DocumentCleaner(remove_empty_lines=False, remove_extra_whitespaces=False, remove_regex=r"\[TODO\]") result = cleaner.run(documents=[Document(content=content)]) assert result["documents"][0].content == expected @@ -140,8 +140,8 @@ def test_remove_regex_with_strip_whitespaces(self): cleaner = DocumentCleaner( remove_empty_lines=False, remove_extra_whitespaces=False, remove_regex=r"Page \d$", strip_whitespaces=True ) - result = cleaner.run(documents=[Document(content="\n Body 1\nPage 1\fBody 2\nPage 2")]) - assert result["documents"][0].content == "Body 1\n\fBody 2" + result = cleaner.run(documents=[Document(content="\n Body 1\nPage 1\f Body 2 \n")]) + assert result["documents"][0].content == "Body 1\f Body 2" def test_remove_repeated_substrings(self): cleaner = DocumentCleaner(