From d2bd8c9a5b26e9fdbb8aa7d0d8280ced245826ba Mon Sep 17 00:00:00 2001 From: Jeremy Schoemaker Date: Wed, 5 Aug 2026 02:44:12 -0500 Subject: [PATCH 1/5] fix(gmail): restore base64url padding before decoding message bodies The Gmail API returns MessagePartBody.data as base64url that commonly omits the trailing '=', which base64.urlsafe_b64decode rejects with binascii.Error. _extract_body decoded that field directly at three sites with no padding normalization, and every one of them sits inside a try/except or contextlib.suppress -- so an unpadded body did not raise, it silently became "(Could not decode email body)" or "(No email body found)". gmail_get_attachment already restores the padding. This lifts that normalization into a shared _decode_base64url helper and routes the three _extract_body sites plus gmail_get_attachment through it, so the module handles the wire format consistently. Addresses the first half of #169. Two tests, both verified failing without the change: the single-part path returned "(Could not decode email body)" and the multipart text/plain path returned "(No email body found)". --- .../providers/google/gmail/tools.py | 22 ++++++--- tests/providers/google/gmail/test_tools.py | 48 +++++++++++++++++++ 2 files changed, 63 insertions(+), 7 deletions(-) diff --git a/src/apron_tools/providers/google/gmail/tools.py b/src/apron_tools/providers/google/gmail/tools.py index e9914e0..d2b9cf0 100644 --- a/src/apron_tools/providers/google/gmail/tools.py +++ b/src/apron_tools/providers/google/gmail/tools.py @@ -58,6 +58,17 @@ def _headers(token: str, *, content_type: bool = False) -> dict[str, str]: return h +def _decode_base64url(data: str) -> bytes: + """Decode Gmail base64url data, restoring the padding Gmail omits. + + The Gmail API returns ``MessagePartBody.data`` and attachment data as + base64url that commonly drops the trailing ``=``, which + ``base64.urlsafe_b64decode`` rejects. Restoring it is a no-op on input + that is already padded. + """ + return base64.urlsafe_b64decode(data + "=" * (-len(data) % 4)) + + def _extract_header(headers: list[dict[str, str]], name: str) -> str: """Extract a single header value from a Gmail payload headers list.""" for h in headers: @@ -76,7 +87,7 @@ def _extract_body(payload: dict) -> str: body_data = payload.get("body", {}).get("data", "") if body_data: try: - return base64.urlsafe_b64decode(body_data).decode("utf-8") + return _decode_base64url(body_data).decode("utf-8") except Exception: return "(Could not decode email body)" @@ -90,10 +101,10 @@ def _extract_body(payload: dict) -> str: if mime_type == "text/plain" and part_data: with contextlib.suppress(Exception): - plain_text = base64.urlsafe_b64decode(part_data).decode("utf-8") + plain_text = _decode_base64url(part_data).decode("utf-8") elif mime_type == "text/html" and part_data: with contextlib.suppress(Exception): - html_text = base64.urlsafe_b64decode(part_data).decode("utf-8") + html_text = _decode_base64url(part_data).decode("utf-8") elif part.get("parts"): nested = _extract_body(part) if nested and nested != "(No email body found)": @@ -292,11 +303,8 @@ async def gmail_get_attachment( if not encoded: return GetAttachmentResult(success=False, error="Attachment contained no data.") - # Gmail returns base64url data that often omits trailing padding, which - # base64.urlsafe_b64decode rejects; restore it before decoding. - encoded += "=" * (-len(encoded) % 4) try: - raw = base64.urlsafe_b64decode(encoded) + raw = _decode_base64url(encoded) except (binascii.Error, ValueError): return GetAttachmentResult(success=False, error="Could not decode attachment data.") diff --git a/tests/providers/google/gmail/test_tools.py b/tests/providers/google/gmail/test_tools.py index 9f9a622..d241aea 100644 --- a/tests/providers/google/gmail/test_tools.py +++ b/tests/providers/google/gmail/test_tools.py @@ -195,6 +195,54 @@ async def test_encodes_message_id_in_request_path(self, httpx_mock: HTTPXMock) - assert result.success is True + async def test_success_with_unpadded_single_part_body(self, httpx_mock: HTTPXMock) -> None: + # Gmail commonly returns MessagePartBody.data without trailing '=' padding. + body = "Hi Bob, following up on the thread." + padded = base64.urlsafe_b64encode(body.encode()).decode("ascii") + assert padded.endswith("="), "test payload must exercise padding removal" + + message = _load_json("get_message_full.json") + message["payload"]["mimeType"] = "text/plain" + message["payload"]["body"] = {"size": len(body), "data": padded.rstrip("=")} + message["payload"].pop("parts") + + httpx_mock.add_response( + url=f"{_GMAIL_BASE}/messages/msg-001?format=full", + json=message, + ) + + result = await gmail_read_email( + ReadEmailParams(message_id="msg-001"), + token=_TOKEN, + ) + + assert result.success is True + assert result.body == body + + async def test_success_with_unpadded_multipart_body(self, httpx_mock: HTTPXMock) -> None: + body = "Hi Bob, following up on the thread." + padded = base64.urlsafe_b64encode(body.encode()).decode("ascii") + assert padded.endswith("="), "test payload must exercise padding removal" + + message = _load_json("get_message_full.json") + message["payload"]["parts"][0]["body"] = { + "size": len(body), + "data": padded.rstrip("="), + } + + httpx_mock.add_response( + url=f"{_GMAIL_BASE}/messages/msg-001?format=full", + json=message, + ) + + result = await gmail_read_email( + ReadEmailParams(message_id="msg-001"), + token=_TOKEN, + ) + + assert result.success is True + assert result.body == body + async def test_api_error(self, httpx_mock: HTTPXMock) -> None: httpx_mock.add_response(status_code=404, text="Not Found") From 5eb67eeda071c45f96e248090589666c6028d8e3 Mon Sep 17 00:00:00 2001 From: Jeremy Schoemaker Date: Wed, 5 Aug 2026 23:35:59 -0500 Subject: [PATCH 2/5] test(gmail): cover unpadded text/html body and double-padding attachment data --- tests/providers/google/gmail/test_tools.py | 48 ++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tests/providers/google/gmail/test_tools.py b/tests/providers/google/gmail/test_tools.py index d241aea..dc983e0 100644 --- a/tests/providers/google/gmail/test_tools.py +++ b/tests/providers/google/gmail/test_tools.py @@ -243,6 +243,32 @@ async def test_success_with_unpadded_multipart_body(self, httpx_mock: HTTPXMock) assert result.success is True assert result.body == body + async def test_success_with_unpadded_html_body(self, httpx_mock: HTTPXMock) -> None: + # Unlike the other unpadded regression tests above, this payload requires + # two padding characters ('==') rather than just one. + body = "

Hi Bob, following up.

" + padded = base64.urlsafe_b64encode(body.encode()).decode("ascii") + assert padded.endswith("=="), "test payload must require two padding characters" + + message = _load_json("get_message_full.json") + message["payload"]["parts"][0] = { + "mimeType": "text/html", + "body": {"size": len(body), "data": padded.rstrip("=")}, + } + + httpx_mock.add_response( + url=f"{_GMAIL_BASE}/messages/msg-001?format=full", + json=message, + ) + + result = await gmail_read_email( + ReadEmailParams(message_id="msg-001"), + token=_TOKEN, + ) + + assert result.success is True + assert result.body == body + async def test_api_error(self, httpx_mock: HTTPXMock) -> None: httpx_mock.add_response(status_code=404, text="Not Found") @@ -939,6 +965,28 @@ async def test_success_with_unpadded_data(self, httpx_mock: HTTPXMock) -> None: assert result.data == raw assert result.size == len(raw) + async def test_success_with_unpadded_data_requiring_double_padding(self, httpx_mock: HTTPXMock) -> None: + # Unlike test_success_with_unpadded_data above, which only strips one + # padding character, this payload requires two ('=='). + raw = b"unpadded pdf report bytes" + padded = base64.urlsafe_b64encode(raw).decode("ascii") + assert padded.endswith("=="), "test payload must require two padding characters" + unpadded = padded.rstrip("=") + + httpx_mock.add_response( + url=f"{_GMAIL_BASE}/messages/msg-001/attachments/att-006", + json={"attachmentId": "att-006", "size": len(raw), "data": unpadded}, + ) + + result = await gmail_get_attachment( + GetAttachmentParams(message_id="msg-001", attachment_id="att-006"), + token=_TOKEN, + ) + + assert result.success is True + assert result.data == raw + assert result.size == len(raw) + async def test_non_json_response(self, httpx_mock: HTTPXMock) -> None: httpx_mock.add_response( url=f"{_GMAIL_BASE}/messages/msg-001/attachments/att-005", From 9db3ff64e313825dcfc85349732e2764a7053aaa Mon Sep 17 00:00:00 2001 From: Peter Wilson Date: Thu, 6 Aug 2026 15:12:33 +0100 Subject: [PATCH 3/5] refactor(gmail): catch ValueError consistently across base64url decode sites Review follow-up to the shared _decode_base64url helper. The four decode sites caught failure inconsistently: _extract_body used broad `except Exception` / `contextlib.suppress(Exception)`, while gmail_get_attachment caught `(binascii.Error, ValueError)`. Both binascii.Error (bad base64) and UnicodeDecodeError (bad UTF-8) subclass ValueError, so every site now catches ValueError -- narrower at the body sites (no longer swallowing unrelated bugs) and without the redundant tuple (binascii.Error subclasses ValueError), which lets the now-unused binascii import go. Document the helper's decode-failure contract, noting the decode is lenient and does not validate the payload, and add regression tests pinning both failure modes (invalid base64 length; valid base64 that decodes to non-UTF-8 bytes) at the narrowed catch sites. --- .../providers/google/gmail/tools.py | 23 +++++-- tests/providers/google/gmail/test_tools.py | 66 +++++++++++++++++++ 2 files changed, 83 insertions(+), 6 deletions(-) diff --git a/src/apron_tools/providers/google/gmail/tools.py b/src/apron_tools/providers/google/gmail/tools.py index d2b9cf0..ce956b3 100644 --- a/src/apron_tools/providers/google/gmail/tools.py +++ b/src/apron_tools/providers/google/gmail/tools.py @@ -3,7 +3,6 @@ from __future__ import annotations import base64 -import binascii import contextlib from email.mime.text import MIMEText @@ -64,7 +63,19 @@ def _decode_base64url(data: str) -> bytes: The Gmail API returns ``MessagePartBody.data`` and attachment data as base64url that commonly drops the trailing ``=``, which ``base64.urlsafe_b64decode`` rejects. Restoring it is a no-op on input - that is already padded. + that is already padded. Decoding is lenient and does not validate the + payload: characters outside the base64url alphabet are discarded rather + than rejected. + + Args: + data: The base64url-encoded string to decode, with or without padding. + + Returns: + The decoded bytes. + + Raises: + ValueError: If ``data`` cannot be decoded, such as an invalid length + or non-ASCII input. """ return base64.urlsafe_b64decode(data + "=" * (-len(data) % 4)) @@ -88,7 +99,7 @@ def _extract_body(payload: dict) -> str: if body_data: try: return _decode_base64url(body_data).decode("utf-8") - except Exception: + except ValueError: return "(Could not decode email body)" parts = payload.get("parts", []) @@ -100,10 +111,10 @@ def _extract_body(payload: dict) -> str: part_data = part.get("body", {}).get("data", "") if mime_type == "text/plain" and part_data: - with contextlib.suppress(Exception): + with contextlib.suppress(ValueError): plain_text = _decode_base64url(part_data).decode("utf-8") elif mime_type == "text/html" and part_data: - with contextlib.suppress(Exception): + with contextlib.suppress(ValueError): html_text = _decode_base64url(part_data).decode("utf-8") elif part.get("parts"): nested = _extract_body(part) @@ -305,7 +316,7 @@ async def gmail_get_attachment( try: raw = _decode_base64url(encoded) - except (binascii.Error, ValueError): + except ValueError: return GetAttachmentResult(success=False, error="Could not decode attachment data.") return GetAttachmentResult( diff --git a/tests/providers/google/gmail/test_tools.py b/tests/providers/google/gmail/test_tools.py index dc983e0..c0da490 100644 --- a/tests/providers/google/gmail/test_tools.py +++ b/tests/providers/google/gmail/test_tools.py @@ -269,6 +269,72 @@ async def test_success_with_unpadded_html_body(self, httpx_mock: HTTPXMock) -> N assert result.success is True assert result.body == body + async def test_single_part_body_with_undecodable_base64_returns_placeholder(self, httpx_mock: HTTPXMock) -> None: + # An invalid base64url length (a single data character) raises + # binascii.Error, a ValueError subclass, which the single-part path + # catches and reports as a decode failure. + message = _load_json("get_message_full.json") + message["payload"]["mimeType"] = "text/plain" + message["payload"]["body"] = {"size": 1, "data": "A"} + message["payload"].pop("parts") + + httpx_mock.add_response( + url=f"{_GMAIL_BASE}/messages/msg-001?format=full", + json=message, + ) + + result = await gmail_read_email( + ReadEmailParams(message_id="msg-001"), + token=_TOKEN, + ) + + assert result.success is True + assert result.body == "(Could not decode email body)" + + async def test_single_part_body_with_non_utf8_bytes_returns_placeholder(self, httpx_mock: HTTPXMock) -> None: + # Valid base64url that decodes to bytes that are not valid UTF-8 raises + # UnicodeDecodeError, also a ValueError subclass, on the single-part path. + non_utf8 = base64.urlsafe_b64encode(b"\xff\xfe").decode("ascii") + + message = _load_json("get_message_full.json") + message["payload"]["mimeType"] = "text/plain" + message["payload"]["body"] = {"size": 2, "data": non_utf8} + message["payload"].pop("parts") + + httpx_mock.add_response( + url=f"{_GMAIL_BASE}/messages/msg-001?format=full", + json=message, + ) + + result = await gmail_read_email( + ReadEmailParams(message_id="msg-001"), + token=_TOKEN, + ) + + assert result.success is True + assert result.body == "(Could not decode email body)" + + async def test_multipart_body_with_non_utf8_bytes_falls_through(self, httpx_mock: HTTPXMock) -> None: + # On the multipart path an undecodable part is suppressed, leaving no + # usable body, so extraction falls through to the sentinel. + non_utf8 = base64.urlsafe_b64encode(b"\xff\xfe").decode("ascii") + + message = _load_json("get_message_full.json") + message["payload"]["parts"][0]["body"] = {"size": 2, "data": non_utf8} + + httpx_mock.add_response( + url=f"{_GMAIL_BASE}/messages/msg-001?format=full", + json=message, + ) + + result = await gmail_read_email( + ReadEmailParams(message_id="msg-001"), + token=_TOKEN, + ) + + assert result.success is True + assert result.body == "(No email body found)" + async def test_api_error(self, httpx_mock: HTTPXMock) -> None: httpx_mock.add_response(status_code=404, text="Not Found") From 20ec75d36985a8514fee1b0548014e1493bbfb05 Mon Sep 17 00:00:00 2001 From: Jeremy Schoemaker Date: Thu, 6 Aug 2026 10:04:16 -0500 Subject: [PATCH 4/5] fix(gmail): validate the base64url alphabet when decoding Lenient urlsafe_b64decode silently discards characters outside the base64url alphabet, so corrupt Gmail data decoded to truncated bytes instead of raising. Decode with validate=True and altchars=b"-_" so those payloads surface as a decode failure, and cover the invalid- alphabet case alongside the existing invalid-length one. --- .../providers/google/gmail/tools.py | 12 +++++----- tests/providers/google/gmail/test_tools.py | 24 +++++++++++++++++++ 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/src/apron_tools/providers/google/gmail/tools.py b/src/apron_tools/providers/google/gmail/tools.py index ce956b3..ca1cf58 100644 --- a/src/apron_tools/providers/google/gmail/tools.py +++ b/src/apron_tools/providers/google/gmail/tools.py @@ -63,9 +63,9 @@ def _decode_base64url(data: str) -> bytes: The Gmail API returns ``MessagePartBody.data`` and attachment data as base64url that commonly drops the trailing ``=``, which ``base64.urlsafe_b64decode`` rejects. Restoring it is a no-op on input - that is already padded. Decoding is lenient and does not validate the - payload: characters outside the base64url alphabet are discarded rather - than rejected. + that is already padded. Decoding is strict: characters outside the + base64url alphabet are rejected rather than silently discarded, so + corrupt data surfaces as an error instead of truncated bytes. Args: data: The base64url-encoded string to decode, with or without padding. @@ -74,10 +74,10 @@ def _decode_base64url(data: str) -> bytes: The decoded bytes. Raises: - ValueError: If ``data`` cannot be decoded, such as an invalid length - or non-ASCII input. + ValueError: If ``data`` cannot be decoded, such as an invalid length, + a character outside the base64url alphabet, or non-ASCII input. """ - return base64.urlsafe_b64decode(data + "=" * (-len(data) % 4)) + return base64.b64decode(data + "=" * (-len(data) % 4), altchars=b"-_", validate=True) def _extract_header(headers: list[dict[str, str]], name: str) -> str: diff --git a/tests/providers/google/gmail/test_tools.py b/tests/providers/google/gmail/test_tools.py index c0da490..d4db431 100644 --- a/tests/providers/google/gmail/test_tools.py +++ b/tests/providers/google/gmail/test_tools.py @@ -291,6 +291,30 @@ async def test_single_part_body_with_undecodable_base64_returns_placeholder(self assert result.success is True assert result.body == "(Could not decode email body)" + async def test_single_part_body_with_invalid_base64_alphabet_returns_placeholder( + self, httpx_mock: HTTPXMock + ) -> None: + # A valid length but a character outside the base64url alphabet. Strict + # validation rejects it rather than discarding the character and + # returning truncated bytes. + message = _load_json("get_message_full.json") + message["payload"]["mimeType"] = "text/plain" + message["payload"]["body"] = {"size": 4, "data": "ab*d"} + message["payload"].pop("parts") + + httpx_mock.add_response( + url=f"{_GMAIL_BASE}/messages/msg-001?format=full", + json=message, + ) + + result = await gmail_read_email( + ReadEmailParams(message_id="msg-001"), + token=_TOKEN, + ) + + assert result.success is True + assert result.body == "(Could not decode email body)" + async def test_single_part_body_with_non_utf8_bytes_returns_placeholder(self, httpx_mock: HTTPXMock) -> None: # Valid base64url that decodes to bytes that are not valid UTF-8 raises # UnicodeDecodeError, also a ValueError subclass, on the single-part path. From 6d73fce46d9fa0fbecb1c979d79878bfa8677cb8 Mon Sep 17 00:00:00 2001 From: Peter Wilson Date: Fri, 7 Aug 2026 14:19:22 +0100 Subject: [PATCH 5/5] test(gmail): isolate base64url alphabet validation in decode-failure test The regression added in 20ec75d used `ab*d`, which the lenient decoder also rejects -- it discards `*`, leaving `abd`, which then fails the padding check -- so the test passed even without the validate=True change. Use `YWJj*`: a complete valid block plus a stray non-alphabet character that the lenient decoder silently accepts (decoding "abc") but strict validation rejects, so the test now fails without the fix. --- tests/providers/google/gmail/test_tools.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/providers/google/gmail/test_tools.py b/tests/providers/google/gmail/test_tools.py index d4db431..61a7051 100644 --- a/tests/providers/google/gmail/test_tools.py +++ b/tests/providers/google/gmail/test_tools.py @@ -294,12 +294,14 @@ async def test_single_part_body_with_undecodable_base64_returns_placeholder(self async def test_single_part_body_with_invalid_base64_alphabet_returns_placeholder( self, httpx_mock: HTTPXMock ) -> None: - # A valid length but a character outside the base64url alphabet. Strict - # validation rejects it rather than discarding the character and - # returning truncated bytes. + # A complete valid Base64 block ("YWJj" -> "abc") followed by a stray + # character outside the base64url alphabet. The lenient decoder would + # discard the stray character and still decode "abc"; strict validation + # rejects it. This isolates the alphabet check rather than tripping the + # padding check the way a shorter invalid payload would. message = _load_json("get_message_full.json") message["payload"]["mimeType"] = "text/plain" - message["payload"]["body"] = {"size": 4, "data": "ab*d"} + message["payload"]["body"] = {"size": 3, "data": "YWJj*"} message["payload"].pop("parts") httpx_mock.add_response(