fix(converters): preserve literal NA strings in XLSXToDocument (#12945)

docs-website/docs/pipeline-components/converters/xlsxtodocument.mdx

@@ -29,6 +29,14 @@ The `XLSXToDocument` component converts XLSX files into Haystack Documents wit
 
 To see the additional parameters that you can specify with the component initialization, check out the [API Reference](/reference/converters-api#xlsxtodocument).
 
+### Missing values
+
+When neither `keep_default_na` nor `na_values` is supplied in `read_excel_kwargs`, `XLSXToDocument` uses `keep_default_na=False` and `na_values=[""]`. This preserves literal strings such as `"NA"`, `"N/A"`, `"NULL"`, and `"None"` while treating empty cells as missing. Missing values are rendered as blank cells unless you set `table_format_kwargs={"na_rep": "<missing>"}` for CSV or `table_format_kwargs={"missingval": "<missing>"}` for Markdown.
+
+If you supply either option, the converter passes your NA settings to pandas unchanged and adds neither default. For example, `read_excel_kwargs={"na_values": ["foo"]}` treats `"foo"` and pandas' built-in NA strings as missing. Setting `read_excel_kwargs={"keep_default_na": False}` preserves those strings and keeps empty cells as empty strings unless you explicitly include `""` in `na_values`. Setting `na_filter=False` disables missing-value detection entirely.
+
+To restore pandas' previous default NA parsing, which also treats strings such as `"NA"` and `"NULL"` as missing, set `read_excel_kwargs={"keep_default_na": True}`.
+
 ## Usage
 
 First, install the pandas, openpyxl, and tabulate packages to start using this converter:

haystack/components/converters/xlsx.py

@@ -63,6 +63,12 @@ def __init__(
         :param table_format: The format to convert the Excel file to.
         :param sheet_name: The name of the sheet to read. If None, all sheets are read.
         :param read_excel_kwargs: Additional arguments to pass to `pandas.read_excel`.
+            When neither `keep_default_na` nor `na_values` is supplied, `keep_default_na=False` and
+            `na_values=[""]` preserve literal text like "NA" and "N/A" while treating empty cells as missing.
+            If either option is supplied, both follow pandas' semantics without adding converter defaults
+            or modifying the supplied values. Custom `na_values` alone supplement pandas' default NA strings;
+            explicit `keep_default_na=False` disables those strings without adding missing values.
+            Set `keep_default_na=True` to restore pandas' default NA parsing.
             See https://pandas.pydata.org/docs/reference/api/pandas.read_excel.html#pandas-read-excel
         :param table_format_kwargs: Additional keyword arguments to pass to the table format function.
             - If `table_format` is "csv", these arguments are passed to `pandas.DataFrame.to_csv`.
@@ -169,6 +175,8 @@ def _extract_tables(self, bytestream: ByteStream) -> tuple[list[str], list[dict]
             "header": None,  # Don't assign any pandas column labels
             "engine": "openpyxl",  # Use openpyxl as the engine to read the Excel file
         }
+        if "keep_default_na" not in self.read_excel_kwargs and "na_values" not in self.read_excel_kwargs:
+            resolved_read_excel_kwargs.update(keep_default_na=False, na_values=[""])
         sheet_to_dataframe = pd.read_excel(io=file_bytes, **resolved_read_excel_kwargs)
         if isinstance(sheet_to_dataframe, pd.DataFrame):
             sheet_to_dataframe = {self.sheet_name: sheet_to_dataframe}

releasenotes/notes/fix-xlsx-drops-literal-na-values-b18412ebd88214fe.yaml

@@ -0,0 +1,15 @@
+---
+fixes:
+  - |
+    Fixed an issue where ``XLSXToDocument`` dropped literal text values such as ``"NA"`` and ``"N/A"``,
+    converting them into empty cells. When neither ``keep_default_na`` nor ``na_values`` is supplied in
+    ``read_excel_kwargs``, the converter uses ``keep_default_na=False`` and ``na_values=[""]`` to preserve
+    literal text while still recognizing empty cells as missing.
+upgrade:
+  - |
+    If you do not configure ``keep_default_na`` or ``na_values``, ``XLSXToDocument`` now preserves strings
+    such as ``"NA"``, ``"N/A"``, ``"NULL"``, and ``"None"`` instead of treating them as missing values.
+    Pass ``read_excel_kwargs={"keep_default_na": True}`` to restore pandas' previous default NA parsing.
+    If either option is already supplied, the converter preserves pandas' existing behavior: custom
+    ``na_values`` alone supplement the built-in NA strings, while explicit ``keep_default_na=False``
+    disables those strings without adding ``""`` to the missing values or changing numeric formatting.

test/components/converters/test_xlsx_to_document.py

@@ -5,8 +5,9 @@
 import csv
 import io
 import logging
+from copy import deepcopy
 from pathlib import Path
-from typing import Literal
+from typing import Any, Literal
 
 import pytest
 from openpyxl import Workbook
@@ -254,3 +255,195 @@ def test_link_extraction_from_typed_columns(
         ]
         assert rows == [linked_values + ["1.23"], ["43", "4.5", "False", "2.35"]]
         assert documents[0].meta["xlsx"] == {"sheet_name": "Sheet"}
+
+    @pytest.mark.parametrize("table_format", ["csv", "markdown"])
+    @pytest.mark.parametrize("read_excel_kwargs", [{}, {"skiprows": 0}], ids=["defaults", "unrelated-option"])
+    @pytest.mark.parametrize("missing_value", ["", "<missing>"])
+    def test_run_preserves_literal_na_strings(
+        self,
+        tmp_path: Path,
+        table_format: Literal["csv", "markdown"],
+        read_excel_kwargs: dict[str, Any],
+        missing_value: str,
+    ) -> None:
+        workbook = Workbook()
+        sheet = workbook.active
+        assert sheet is not None
+        sheet.append(["Code", "Status", "Blank"])
+        sheet.append(["NA", "OK", None])
+        sheet.append(["N/A", "pending", ""])
+        sheet.append(["NULL", "None", None])
+        sheet.append(["NaN", "null", ""])
+        sheet.append(["North America", "active", ""])
+        path = tmp_path / "literal_na.xlsx"
+        workbook.save(path)
+        workbook.close()
+
+        original_kwargs = deepcopy(read_excel_kwargs)
+        format_kwargs = {"na_rep": missing_value} if table_format == "csv" else {"missingval": missing_value}
+        converter = XLSXToDocument(
+            table_format=table_format, read_excel_kwargs=read_excel_kwargs, table_format_kwargs=format_kwargs
+        )
+        documents = converter.run(sources=[path])["documents"]
+
+        assert len(documents) == 1
+        content = documents[0].content
+        assert content is not None
+
+        if table_format == "csv":
+            rows = [row[1:] for row in list(csv.reader(io.StringIO(content)))[1:]]
+        else:
+            rows = [[cell.strip() for cell in row.split("|")[2:-1]] for row in content.splitlines()[2:]]
+        assert rows == [
+            ["Code", "Status", "Blank"],
+            ["NA", "OK", missing_value],
+            ["N/A", "pending", missing_value],
+            ["NULL", "None", missing_value],
+            ["NaN", "null", missing_value],
+            ["North America", "active", missing_value],
+        ]
+        assert read_excel_kwargs == original_kwargs
+
+    @pytest.mark.parametrize("table_format", ["csv", "markdown"])
+    @pytest.mark.parametrize(
+        ("na_values", "expected_custom_values"),
+        [
+            pytest.param(["foo"], ["<missing>", "<missing>"], id="list"),
+            pytest.param(["foo", ""], ["<missing>", "<missing>"], id="list-with-empty-string"),
+            pytest.param([], ["foo", "foo"], id="empty-list"),
+            pytest.param(("foo",), ["<missing>", "<missing>"], id="tuple"),
+            pytest.param({"foo"}, ["<missing>", "<missing>"], id="set"),
+            pytest.param("foo", ["<missing>", "<missing>"], id="string"),
+            pytest.param({0: ["foo"]}, ["<missing>", "foo"], id="column-dict"),
+            pytest.param(None, ["foo", "foo"], id="none"),
+        ],
+    )
+    def test_run_custom_na_values_preserves_empty_cells(
+        self,
+        tmp_path: Path,
+        table_format: Literal["csv", "markdown"],
+        na_values: Any,
+        expected_custom_values: list[str],
+    ) -> None:
+        workbook = Workbook()
+        sheet = workbook.active
+        assert sheet is not None
+        sheet.append(["Code", "Status", "Blank"])
+        sheet.append(["foo", "foo", None])
+        sheet.append(["NA", "N/A", "value"])
+        sheet.append(["NULL", "None", ""])
+        path = tmp_path / "custom_na.xlsx"
+        workbook.save(path)
+        workbook.close()
+
+        read_excel_kwargs = {"na_values": na_values}
+        original_kwargs = deepcopy(read_excel_kwargs)
+        format_kwargs = {"na_rep": "<missing>"} if table_format == "csv" else {"missingval": "<missing>"}
+        converter = XLSXToDocument(
+            table_format=table_format, read_excel_kwargs=read_excel_kwargs, table_format_kwargs=format_kwargs
+        )
+        documents = converter.run(sources=[path])["documents"]
+
+        assert len(documents) == 1
+        content = documents[0].content
+        assert content is not None
+        if table_format == "csv":
+            rows = [row[1:] for row in list(csv.reader(io.StringIO(content)))[1:]]
+        else:
+            rows = [[cell.strip() for cell in row.split("|")[2:-1]] for row in content.splitlines()[2:]]
+        assert rows == [
+            ["Code", "Status", "Blank"],
+            [*expected_custom_values, "<missing>"],
+            ["<missing>", "<missing>", "value"],
+            ["<missing>", "<missing>", "<missing>"],
+        ]
+        assert read_excel_kwargs == original_kwargs
+
+    @pytest.mark.parametrize("table_format", ["csv", "markdown"])
+    @pytest.mark.parametrize(
+        ("read_excel_kwargs", "expected_rows"),
+        [
+            pytest.param(
+                {"keep_default_na": True},
+                [["foo", "<missing>", "<missing>"], ["<missing>", "<missing>", "value"]],
+                id="keep-default-na-true",
+            ),
+            pytest.param(
+                {"keep_default_na": True, "na_values": ["foo"]},
+                [["<missing>", "<missing>", "<missing>"], ["<missing>", "<missing>", "value"]],
+                id="keep-default-na-true-with-custom-values",
+            ),
+            pytest.param(
+                {"keep_default_na": False}, [["foo", "NA", ""], ["N/A", "", "value"]], id="keep-default-na-false"
+            ),
+            pytest.param(
+                {"keep_default_na": False, "na_values": ["foo"]},
+                [["<missing>", "NA", ""], ["N/A", "", "value"]],
+                id="keep-default-na-false-with-custom-values",
+            ),
+            pytest.param(
+                {"keep_default_na": False, "na_values": ["foo", ""]},
+                [["<missing>", "NA", "<missing>"], ["N/A", "<missing>", "value"]],
+                id="explicit-empty-string",
+            ),
+            pytest.param({"na_filter": False}, [["foo", "NA", ""], ["N/A", "", "value"]], id="na-filter-false"),
+        ],
+    )
+    def test_run_na_parsing_overrides(
+        self,
+        tmp_path: Path,
+        table_format: Literal["csv", "markdown"],
+        read_excel_kwargs: dict[str, Any],
+        expected_rows: list[list[str]],
+    ) -> None:
+        workbook = Workbook()
+        sheet = workbook.active
+        assert sheet is not None
+        sheet.append(["Code", "Status", "Blank"])
+        sheet.append(["foo", "NA", None])
+        sheet.append(["N/A", "", "value"])
+        path = tmp_path / "override_na.xlsx"
+        workbook.save(path)
+        workbook.close()
+
+        original_kwargs = deepcopy(read_excel_kwargs)
+        format_kwargs = {"na_rep": "<missing>"} if table_format == "csv" else {"missingval": "<missing>"}
+        converter = XLSXToDocument(
+            table_format=table_format, read_excel_kwargs=read_excel_kwargs, table_format_kwargs=format_kwargs
+        )
+        documents = converter.run(sources=[path])["documents"]
+
+        assert len(documents) == 1
+        content = documents[0].content
+        assert content is not None
+        if table_format == "csv":
+            rows = [row[1:] for row in list(csv.reader(io.StringIO(content)))[1:]]
+        else:
+            rows = [[cell.strip() for cell in row.split("|")[2:-1]] for row in content.splitlines()[2:]]
+        assert rows == [["Code", "Status", "Blank"], *expected_rows]
+        assert read_excel_kwargs == original_kwargs
+
+    @pytest.mark.parametrize("na_values", [None, ["foo"]], ids=["no-custom-values", "custom-values"])
+    def test_run_keep_default_na_false_preserves_numeric_formatting(self, tmp_path: Path, na_values: Any) -> None:
+        workbook = Workbook()
+        sheet = workbook.active
+        assert sheet is not None
+        sheet.append([1, "x"])
+        sheet.append([None, "foo"])
+        sheet.append([2, "y"])
+        path = tmp_path / "numeric_gaps.xlsx"
+        workbook.save(path)
+        workbook.close()
+
+        read_excel_kwargs: dict[str, Any] = {"keep_default_na": False}
+        if na_values is not None:
+            read_excel_kwargs["na_values"] = na_values
+        converter = XLSXToDocument(read_excel_kwargs=read_excel_kwargs, table_format_kwargs={"na_rep": "<missing>"})
+        documents = converter.run(sources=[path])["documents"]
+
+        assert len(documents) == 1
+        content = documents[0].content
+        assert content is not None
+        rows = [row[1:] for row in list(csv.reader(io.StringIO(content)))[1:]]
+        expected_foo = "foo" if na_values is None else "<missing>"
+        assert rows == [["1", "x"], ["", expected_foo], ["2", "y"]]