Skip to content

feat(core): add LowercaseTextProcessor, TrimWhitespaceProcessor, and LanguageDetectProcessor #170

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Rajeev91691 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat(core): add LowercaseTextProcessor, TrimWhitespaceProcessor, and LanguageDetectProcessor #170

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 76 additions & 0 deletions genai_processors/core/text.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -495,6 +495,82 @@ async def call(
raise ValueError(f'Unsupported cleaning mode: {self._cleaning_mode}')


class LowercaseTextProcessor(processor.PartProcessor):
"""PartProcessor that lowercases text parts."""

def match(self, part: content_api.ProcessorPart) -> bool:
return content_api.is_text(part.mimetype)

async def call(
self, part: content_api.ProcessorPart
) -> AsyncIterable[content_api.ProcessorPartTypes]:
yield content_api.ProcessorPart(
part.text.lower(),
mimetype=part.mimetype,
role=part.role,
substream_name=part.substream_name,
metadata=part.metadata,
)


class TrimWhitespaceProcessor(processor.PartProcessor):
"""PartProcessor that trims leading and trailing whitespace from text parts."""

def match(self, part: content_api.ProcessorPart) -> bool:
return content_api.is_text(part.mimetype)

async def call(
self, part: content_api.ProcessorPart
) -> AsyncIterable[content_api.ProcessorPartTypes]:
yield content_api.ProcessorPart(
part.text.strip(),
mimetype=part.mimetype,
role=part.role,
substream_name=part.substream_name,
metadata=part.metadata,
)


class LanguageDetectProcessor(processor.PartProcessor):
"""PartProcessor that automatically detects language of text parts and adds it to metadata.

Uses the `langdetect` library to detect the language.
"""

def __init__(
self, *, metadata_key: str = 'language', fallback_lang: str | None = None
):
self._metadata_key = metadata_key
self._fallback_lang = fallback_lang
Comment on lines +540 to +544

@gemini-code-assist gemini-code-assist Bot Aug 17, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

The langdetect library's detection algorithm is non-deterministic by default because it uses a randomized initialization. This can lead to inconsistent language detection results for the same input text across different runs, which can cause flaky tests or unpredictable behavior in production.

To ensure deterministic and consistent results, initialize the DetectorFactory seed to 0 during the processor's initialization.

Suggested change
def __init__(
self, *, metadata_key: str = 'language', fallback_lang: str | None = None
):
self._metadata_key = metadata_key
self._fallback_lang = fallback_lang
def __init__(
self, *, metadata_key: str = 'language', fallback_lang: str | None = None
):
self._metadata_key = metadata_key
self._fallback_lang = fallback_lang
from langdetect import DetectorFactory
DetectorFactory.seed = 0


def match(self, part: content_api.ProcessorPart) -> bool:
return content_api.is_text(part.mimetype)

async def call(
self, part: content_api.ProcessorPart
) -> AsyncIterable[content_api.ProcessorPartTypes]:
import langdetect

@gemini-code-assist gemini-code-assist Bot Aug 17, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

Since langdetect is listed as a required dependency in pyproject.toml, it is guaranteed to be installed. Importing it inline inside the call method on every invocation adds unnecessary overhead and is non-idiomatic.

It is highly recommended to move the import langdetect statement to the top of the file along with other module-level imports, adhering to the Google Python Style Guide.

References
  1. Imports should be placed at the top of the file, after any module comments and docstrings, and before module globals and constants. (link)


lang = self._fallback_lang
if part.text:
try:
lang = langdetect.detect(part.text)
except Exception:
pass

meta = dict(part.metadata or {})
if lang:
meta[self._metadata_key] = lang

yield content_api.ProcessorPart(
part.text,
mimetype=part.mimetype,
role=part.role,
substream_name=part.substream_name,
metadata=meta,
)


@processor.source()
async def terminal_input(
prompt: str = '',
Expand Down
63 changes: 63 additions & 0 deletions genai_processors/tests/text_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -719,5 +719,68 @@ async def test_html_cleaner_html_mode(self, cleaning_mode, expected_output):
)


class LowercaseTextProcessorTest(
parameterized.TestCase, unittest.IsolatedAsyncioTestCase
):

async def test_lowercase_text(self):
processor = text.LowercaseTextProcessor()
input_part = content_api.ProcessorPart(
'HELLO World!', mimetype='text/plain', role='user'
)
self.assertSequenceEqual(
await processor(input_part).gather(),
[
content_api.ProcessorPart(
'hello world!', mimetype='text/plain', role='user'
)
],
)


class TrimWhitespaceProcessorTest(
parameterized.TestCase, unittest.IsolatedAsyncioTestCase
):

async def test_trim_whitespace(self):
processor = text.TrimWhitespaceProcessor()
input_part = content_api.ProcessorPart(
' \n hello \t ', mimetype='text/plain', role='user'
)
self.assertSequenceEqual(
await processor(input_part).gather(),
[
content_api.ProcessorPart(
'hello', mimetype='text/plain', role='user'
)
],
)


class LanguageDetectProcessorTest(
parameterized.TestCase, unittest.IsolatedAsyncioTestCase
):

async def test_detect_english(self):
processor = text.LanguageDetectProcessor()
input_part = content_api.ProcessorPart(
'This is a simple english text to detect language.',
mimetype='text/plain',
)
results = await processor(input_part).gather()
self.assertEqual(len(results), 1)
self.assertEqual(results[0].metadata.get('language'), 'en')

async def test_detect_spanish(self):
processor = text.LanguageDetectProcessor()
input_part = content_api.ProcessorPart(
'Hola, esto es un texto simple en español para detectar el idioma.',
mimetype='text/plain',
)
results = await processor(input_part).gather()
self.assertEqual(len(results), 1)
self.assertEqual(results[0].metadata.get('language'), 'es')
Comment on lines +780 to +782

@gemini-code-assist gemini-code-assist Bot Aug 17, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

To ensure the robustness of LanguageDetectProcessor, it is highly recommended to add unit tests covering the following edge cases:

  1. Fallback Language: Verifying that the fallback language is used when the input text is empty or contains no detectable language features (e.g., only numbers/punctuation).
  2. Custom Metadata Key: Verifying that the detected language is stored under a custom metadata key if provided.
  3. Non-Text Parts: Verifying that non-text parts (e.g., images) are ignored by the processor and passed through unchanged.
    results = await processor(input_part).gather()
    self.assertEqual(len(results), 1)
    self.assertEqual(results[0].metadata.get('language'), 'es')

  async def test_detect_fallback_on_empty_text(self):
    processor = text.LanguageDetectProcessor(fallback_lang='fr')
    input_part = content_api.ProcessorPart('', mimetype='text/plain')
    results = await processor(input_part).gather()
    self.assertEqual(len(results), 1)
    self.assertEqual(results[0].metadata.get('language'), 'fr')

  async def test_detect_custom_metadata_key(self):
    processor = text.LanguageDetectProcessor(metadata_key='detected_lang')
    input_part = content_api.ProcessorPart(
        'This is a simple english text.', mimetype='text/plain'
    )
    results = await processor(input_part).gather()
    self.assertEqual(len(results), 1)
    self.assertEqual(results[0].metadata.get('detected_lang'), 'en')

  async def test_detect_non_text_passed_through(self):
    processor = text.LanguageDetectProcessor()
    input_part = content_api.ProcessorPart(b'image_bytes', mimetype='image/png')
    results = await processor(input_part).gather()
    self.assertEqual(len(results), 1)
    self.assertEqual(results[0], input_part)



if __name__ == '__main__':
absltest.main()
1 change: 1 addition & 0 deletions pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ dependencies = [
"mcp>=1.26.0",
"sqlalchemy>=2.0.0",
"webrtcvad>=2.0.10",
"langdetect>=1.0.9",
]

# `version` is automatically set by flit to use `genai_processors.__version__`
Expand Down
Loading