diff --git a/submissions/google-lightspeed-agent/tasks/lightspeed-qa/environment/Dockerfile b/submissions/google-lightspeed-agent/tasks/lightspeed-qa/environment/Dockerfile new file mode 100644 index 0000000..842d023 --- /dev/null +++ b/submissions/google-lightspeed-agent/tasks/lightspeed-qa/environment/Dockerfile @@ -0,0 +1,13 @@ +# Minimal environment for A2A evaluation verifier +FROM python:3.12-slim + +WORKDIR /workspace + +# Install dependencies for LLM judge verifier +RUN pip install --no-cache-dir --root-user-action=ignore \ + litellm>=1.80.0 \ + pyyaml \ + requests + +# Keep container running for Harbor +CMD ["sleep", "infinity"] diff --git a/submissions/google-lightspeed-agent/tasks/lightspeed-qa/tests/test.sh b/submissions/google-lightspeed-agent/tasks/lightspeed-qa/tests/test.sh new file mode 100755 index 0000000..958971c --- /dev/null +++ b/submissions/google-lightspeed-agent/tasks/lightspeed-qa/tests/test.sh @@ -0,0 +1,40 @@ +#!/bin/bash +set -e + +# Run LLM judge verifier on the agent's response +# Supports both container paths (/logs/...) and local environment paths + +# Try container paths first, fall back to CWD-relative +if [ -d "/logs" ]; then + LOGS_BASE="/logs" + TESTS_BASE="/tests" +else + LOGS_BASE="$(pwd)/logs" + TESTS_BASE="$(pwd)/tests" +fi + +RESPONSE_FILE="${LOGS_BASE}/agent/a2a_response.txt" +REWARD_FILE="${LOGS_BASE}/verifier/reward.txt" + +# Ensure directories exist +mkdir -p "${LOGS_BASE}/verifier" + +echo "Looking for response at: $RESPONSE_FILE" +echo "Will write reward to: $REWARD_FILE" + +# Check if response file exists +if [ ! -f "$RESPONSE_FILE" ]; then + echo "ERROR: Agent response file not found at $RESPONSE_FILE" + ls -la "${LOGS_BASE}/" 2>/dev/null || echo "Logs base doesn't exist" + ls -la "${LOGS_BASE}/agent/" 2>/dev/null || echo "Agent dir doesn't exist" + echo "0" > "$REWARD_FILE" + exit 0 # Exit success but with 0 reward +fi + +echo "Response file found, running LLM judge..." + +# Run LLM judge +python3 "${TESTS_BASE}/llm_judge.py" "$RESPONSE_FILE" "$REWARD_FILE" + +echo "LLM judge completed" +exit 0