diff --git a/recipes/python/voice-agents/v1/multilingual-agent/README.md b/recipes/python/voice-agents/v1/multilingual-agent/README.md new file mode 100644 index 00000000..89695c16 --- /dev/null +++ b/recipes/python/voice-agents/v1/multilingual-agent/README.md @@ -0,0 +1,63 @@ +# Multilingual Voice Agent (Voice Agents v1) + +Build a voice agent that automatically detects the speaker's language and dynamically switches its TTS voice and system prompt to match — no upfront language selection required. + +## What it does + +This recipe configures a Deepgram Voice Agent with Nova-3's multilingual STT (`language=multi`) to transcribe speech in any supported language. A `switch_language` function is registered with the LLM so it can signal when the user's language changes. When triggered, the agent dynamically updates its TTS voice and system prompt using `send_update_speak` and `send_update_prompt`, enabling seamless mid-conversation language switching across English, Spanish, and French. + +## Key parameters + +| Parameter | Value | Description | +|-----------|-------|-------------| +| `listen.provider.model` | `"nova-3"` | STT model with multilingual support | +| `listen.provider.language` | `"multi"` | Enables automatic language detection | +| `think.provider.model` | `"gpt-4o-mini"` | LLM for the think stage | +| `think.functions` | `[switch_language]` | Function the LLM calls on language change | +| `speak.provider.model` | `"aura-2-thalia-en"` | Initial TTS voice (English) | + +## Language configuration + +| Language | TTS Voice | Prompt Language | +|----------|-----------|-----------------| +| English (`en`) | `aura-2-thalia-en` | Reply in English | +| Spanish (`es`) | `aura-2-thalia-es` | Responde en español | +| French (`fr`) | `aura-2-thalia-fr` | Répondez en français | + +## How language switching works + +1. Nova-3 with `language=multi` transcribes speech regardless of language +2. The LLM detects the user's language from the transcript text +3. When the language changes, the LLM calls the `switch_language` function +4. The handler updates the TTS voice and system prompt for the new language +5. The agent continues the conversation in the detected language + +## Example output + +``` +Multilingual agent configured (en/es/fr) +Connection opened +Event: SettingsApplied +Switched to es: voice=aura-2-thalia-es +Received 4800 bytes of agent audio +Switched to fr: voice=aura-2-thalia-fr +Connection closed +``` + +## Prerequisites + +- Python 3.10+ +- Set `DEEPGRAM_API_KEY` environment variable +- Install: `pip install -r recipes/python/requirements.txt` + +## Run + +```bash +python example.py +``` + +## Test + +```bash +pytest example_test.py -v +``` diff --git a/recipes/python/voice-agents/v1/multilingual-agent/example.py b/recipes/python/voice-agents/v1/multilingual-agent/example.py new file mode 100644 index 00000000..a6aa605d --- /dev/null +++ b/recipes/python/voice-agents/v1/multilingual-agent/example.py @@ -0,0 +1,61 @@ +""" +Recipe: Multilingual Voice Agent — Nova-3 with language=multi for automatic +language detection, plus per-language TTS voice and prompt switching. +""" +import json + +from deepgram import DeepgramClient +from deepgram.agent.v1.types import ( + AgentV1FunctionCallRequest, AgentV1SendFunctionCallResponse, + AgentV1Settings, AgentV1SettingsAgent, AgentV1SettingsAgentListen, + AgentV1SettingsAgentListenProvider_V1, AgentV1SettingsAudio, + AgentV1SettingsAudioInput, AgentV1UpdatePrompt, AgentV1UpdateSpeak, +) +from deepgram.core.events import EventType +from deepgram.types.speak_settings_v1 import SpeakSettingsV1 +from deepgram.types.speak_settings_v1provider import SpeakSettingsV1Provider_Deepgram +from deepgram.types.think_settings_v1 import ThinkSettingsV1 +from deepgram.types.think_settings_v1provider import ThinkSettingsV1Provider_OpenAi + +LANGS = { + "en": {"voice": "aura-2-thalia-en", "prompt": "Reply in English."}, + "es": {"voice": "aura-2-thalia-es", "prompt": "Responde en español."}, + "fr": {"voice": "aura-2-thalia-fr", "prompt": "Répondez en français."}, +} +SWITCH_FN = {"name": "switch_language", "description": "Call when the user's language changes", + "parameters": {"type": "object", "properties": {"lang": {"type": "string", "enum": list(LANGS)}}, "required": ["lang"]}} +BASE_PROMPT = "You are a multilingual assistant. Detect the user's language and call switch_language when it changes. " + +def main(): + client = DeepgramClient() + with client.agent.v1.connect() as agent: + settings = AgentV1Settings( + audio=AgentV1SettingsAudio(input=AgentV1SettingsAudioInput(encoding="linear16", sample_rate=24000)), + agent=AgentV1SettingsAgent( + listen=AgentV1SettingsAgentListen(provider=AgentV1SettingsAgentListenProvider_V1(type="deepgram", model="nova-3", language="multi")), + think=ThinkSettingsV1(provider=ThinkSettingsV1Provider_OpenAi(type="open_ai", model="gpt-4o-mini"), prompt=BASE_PROMPT + LANGS["en"]["prompt"], functions=[SWITCH_FN]), + speak=SpeakSettingsV1(provider=SpeakSettingsV1Provider_Deepgram(type="deepgram", model="aura-2-thalia-en")), + )) + agent.send_settings(settings) + print("Multilingual agent configured (en/es/fr)") + + def on_message(msg) -> None: + if isinstance(msg, AgentV1FunctionCallRequest) and msg.name == "switch_language": + lang = json.loads(msg.input).get("lang", "en") + cfg = LANGS.get(lang, LANGS["en"]) + agent.send_update_speak(AgentV1UpdateSpeak(speak=SpeakSettingsV1(provider=SpeakSettingsV1Provider_Deepgram(type="deepgram", model=cfg["voice"])))) + agent.send_update_prompt(AgentV1UpdatePrompt(prompt=BASE_PROMPT + cfg["prompt"])) + agent.send_function_call_response(AgentV1SendFunctionCallResponse(type="FunctionCallResponse", id=msg.id, name=msg.name, content=f'{{"switched_to": "{lang}"}}')) + print(f"Switched to {lang}: voice={cfg['voice']}") + elif isinstance(msg, bytes): + print(f"Received {len(msg)} bytes of agent audio") + else: + print(f"Event: {getattr(msg, 'type', type(msg).__name__)}") + + agent.on(EventType.OPEN, lambda _: print("Connection opened")) + agent.on(EventType.MESSAGE, on_message) + agent.on(EventType.CLOSE, lambda _: print("Connection closed")) + agent.start_listening() + +if __name__ == "__main__": + main() diff --git a/recipes/python/voice-agents/v1/multilingual-agent/example_test.py b/recipes/python/voice-agents/v1/multilingual-agent/example_test.py new file mode 100644 index 00000000..cd63fa2c --- /dev/null +++ b/recipes/python/voice-agents/v1/multilingual-agent/example_test.py @@ -0,0 +1,16 @@ +import subprocess +from pathlib import Path + +def test_example_runs(): + """Runs the multilingual voice agent example and verifies it produces output.""" + example = Path(__file__).parent / "example.py" + result = subprocess.run( + ["python", str(example)], + capture_output=True, + text=True, + timeout=60, + ) + assert result.returncode == 0, ( + f"Example failed\nSTDOUT: {result.stdout}\nSTDERR: {result.stderr}" + ) + assert result.stdout.strip(), "Example produced no output"