Skip to content

Commit e8497db

Browse files
Elonon901001julian-rischclaudesjrl
authored
fix: keep docstring-only Python chunks valid (#12346)
Co-authored-by: Julian Risch <julian.risch@deepset.ai> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Sebastian Husch Lee <10526848+sjrl@users.noreply.github.com>
1 parent 7b23a9f commit e8497db

3 files changed

Lines changed: 21 additions & 0 deletions

File tree

‎haystack/components/preprocessors/python_code_splitter.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,12 @@ def _strip_docstring(
199199

200200
before = source_lines[unit_start - 1 : ds_start - 1]
201201
after = source_lines[ds_end:unit_end]
202+
# Removing the only statement from a function or class would leave an
203+
# invalid suite. Keep the generated chunk syntactically valid.
204+
if len(body) == 1:
205+
docstring_line = source_lines[ds_start - 1]
206+
indentation = docstring_line[: len(docstring_line) - len(docstring_line.lstrip())]
207+
after.insert(0, f"{indentation}pass\n")
202208
return "".join(before + after), docstring
203209

204210
def _emit_class_units(self, cls: ast.ClassDef, source_lines: list[str], cursor: int, units: list[_CodeUnit]) -> int:
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
fixes:
2+
- |
3+
Ensures ``PythonCodeSplitter`` emits syntactically valid chunks when ``strip_docstrings=True`` is used on a function or class whose docstring is its only statement.

‎test/components/preprocessors/test_python_code_splitter.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
#
33
# SPDX-License-Identifier: Apache-2.0
44

5+
import ast
56
import textwrap
67

78
import pytest
@@ -707,6 +708,17 @@ def method(self):
707708
assert "Class-level docstring." not in (header.content or "")
708709
assert "Class-level docstring." in " | ".join(header.meta.get("docstrings") or [])
709710

711+
@pytest.mark.parametrize(
712+
"source", ['def only_docstring():\n """Function docs."""\n', 'class OnlyDocstring:\n """Class docs."""\n']
713+
)
714+
def test_strip_docstrings_keeps_docstring_only_units_valid(self, source):
715+
splitter = PythonCodeSplitter(min_effective_lines=1, max_effective_lines=10, strip_docstrings=True)
716+
result = splitter.run(documents=[Document(content=source)])
717+
718+
assert result["documents"]
719+
for chunk in result["documents"]:
720+
ast.parse(chunk.content or "")
721+
710722

711723
class TestTopLevelStatements:
712724
@pytest.fixture

0 commit comments

Comments
 (0)