From 43970a95468e0dfd7da6190d99e98c19f0791812 Mon Sep 17 00:00:00 2001 From: ikrispin Date: Wed, 22 Jul 2026 11:44:18 +0300 Subject: [PATCH 1/2] Run evaluation as remote Pod on workload cluster Instead of running Harbor on the Konflux cluster (which can't reach LiteLLM or the agent internally), submit the eval as a Pod on cn-ai-lab where all services are co-located. The evaluate task creates a Pod, waits for completion, and retrieves results from logs. Also removes Route from deploy-agent (not needed for cluster-internal eval), adds eval Pod cleanup, and fixes emit-result ERROR->FAILURE mapping. --- pipeline/tasks/konflux/cleanup-agent.yaml | 2 +- pipeline/tasks/konflux/deploy-agent.yaml | 37 +- pipeline/tasks/konflux/emit-result.yaml | 2 +- pipeline/tasks/konflux/evaluate.yaml | 1163 ++++----------------- 4 files changed, 223 insertions(+), 981 deletions(-) diff --git a/pipeline/tasks/konflux/cleanup-agent.yaml b/pipeline/tasks/konflux/cleanup-agent.yaml index 7e3fb85..a1f5e5f 100644 --- a/pipeline/tasks/konflux/cleanup-agent.yaml +++ b/pipeline/tasks/konflux/cleanup-agent.yaml @@ -59,8 +59,8 @@ spec: OC_REMOTE="oc --server=$CLUSTER_URL --token=$WORKLOAD_TOKEN --insecure-skip-tls-verify=true" echo "Deleting: $AGENT_NAME in $NAMESPACE on $CLUSTER_URL" - $OC_REMOTE delete route/$AGENT_NAME -n $NAMESPACE --ignore-not-found=true $OC_REMOTE delete deployment/$AGENT_NAME -n $NAMESPACE --ignore-not-found=true $OC_REMOTE delete service/$AGENT_NAME -n $NAMESPACE --ignore-not-found=true + $OC_REMOTE delete pod -n $NAMESPACE -l abevalflow/temp=true,abevalflow/type=eval-job --ignore-not-found=true echo "Cleanup complete" diff --git a/pipeline/tasks/konflux/deploy-agent.yaml b/pipeline/tasks/konflux/deploy-agent.yaml index c7a39c3..8dc1372 100644 --- a/pipeline/tasks/konflux/deploy-agent.yaml +++ b/pipeline/tasks/konflux/deploy-agent.yaml @@ -140,40 +140,23 @@ spec: ports: - port: 8000 targetPort: 8000 - --- - apiVersion: route.openshift.io/v1 - kind: Route - metadata: - name: $AGENT_NAME - labels: - abevalflow/temp: "true" - spec: - to: - kind: Service - name: $AGENT_NAME - port: - targetPort: 8000 - tls: - termination: edge - insecureEdgeTerminationPolicy: Redirect YAML TIMEOUT=$(params.readiness-timeout) + ENDPOINT="http://${AGENT_NAME}.${NAMESPACE}.svc:8000" + printf "Waiting for agent readiness" for i in $(seq 1 $((TIMEOUT / 5))); do if $OC_REMOTE rollout status deployment/$AGENT_NAME -n $NAMESPACE --timeout=5s >/dev/null 2>&1; then - ROUTE_HOST=$($OC_REMOTE get route $AGENT_NAME -n $NAMESPACE -o jsonpath='{.spec.host}' 2>/dev/null || echo "") - if [ -n "$ROUTE_HOST" ]; then - ENDPOINT="https://${ROUTE_HOST}" - if curl -skf "$ENDPOINT/.well-known/agent.json" > /dev/null 2>&1; then - echo " ready!" - echo -n "$ENDPOINT" > "$(results.agent-endpoint.path)" - echo -n "$AGENT_NAME" > "$(results.agent-name.path)" - echo -n "true" > "$(results.deployed.path)" - echo "Agent endpoint: $ENDPOINT" - exit 0 - fi + READY=$($OC_REMOTE get deployment $AGENT_NAME -n $NAMESPACE -o jsonpath='{.status.readyReplicas}' 2>/dev/null || echo "0") + if [ "${READY:-0}" -ge 1 ]; then + echo " ready!" + echo -n "$ENDPOINT" > "$(results.agent-endpoint.path)" + echo -n "$AGENT_NAME" > "$(results.agent-name.path)" + echo -n "true" > "$(results.deployed.path)" + echo "Agent endpoint: $ENDPOINT" + exit 0 fi fi printf "." diff --git a/pipeline/tasks/konflux/emit-result.yaml b/pipeline/tasks/konflux/emit-result.yaml index 81dcb1a..89fc2ef 100644 --- a/pipeline/tasks/konflux/emit-result.yaml +++ b/pipeline/tasks/konflux/emit-result.yaml @@ -64,7 +64,7 @@ spec: NOTE="ABEvalFlow: recommendation=${REPORT_REC} (no scorecard)" else - RESULT="ERROR" + RESULT="FAILURE" NOTE="ABEvalFlow: no scorecard.json or report.json found" fi diff --git a/pipeline/tasks/konflux/evaluate.yaml b/pipeline/tasks/konflux/evaluate.yaml index 0ac44d5..436924a 100644 --- a/pipeline/tasks/konflux/evaluate.yaml +++ b/pipeline/tasks/konflux/evaluate.yaml @@ -7,21 +7,14 @@ metadata: app.kubernetes.io/component: konflux spec: description: >- - Evaluation phase: dispatches to the appropriate evaluation engine based on - eval-engine parameter. Handles Harbor (scaffold+build+eval), ASE (LLM-as-judge), - MCPChecker (MCP server testing), and A2A (agent evaluation) in a single composite task. - Adapted for Konflux — no hardcoded namespace, configurable image references. - stepTemplate: - computeResources: - requests: - cpu: 50m - memory: 64Mi - limits: - cpu: 100m - memory: 256Mi + Evaluation phase for Konflux integration. Submits the evaluation as a + remote Pod on the workload cluster (cn-ai-lab) where the agent, LiteLLM, + and Harbor infrastructure are co-located. Waits for completion and + retrieves results back to the Konflux workspace. params: - name: eval-engine type: string + default: "a2a" description: Evaluation engine (harbor, ase, mcpchecker, a2a, both) - name: submission-dir type: string @@ -31,7 +24,8 @@ spec: description: Validated submission name - name: commit-sha type: string - description: Git commit SHA for image tagging + default: "" + description: Git commit SHA for provenance - name: pipeline-run-id type: string description: PipelineRun name @@ -45,26 +39,6 @@ spec: type: string default: "quay.io/rh-ee-ikrispin/abevalflow-eval-base:latest" description: Eval base image with Harbor pre-installed - # Harbor params - - name: harbor-fork-url - type: string - default: "https://github.com/RHEcosystemAppEng/skills_eval_corrections.git" - - name: harbor-fork-revision - type: string - default: "main" - - name: eval-mode - type: string - default: "prebuilt" - - name: registry-url - type: string - default: "image-registry.openshift-image-registry.svc:5000" - - name: registry-namespace - type: string - default: "ab-eval-flow" - - name: base-image - type: string - default: "image-registry.openshift-image-registry.svc:5000/ab-eval-flow/eval-base:latest" - # LLM params - name: llm-model type: string default: "claude-sonnet" @@ -73,66 +47,24 @@ spec: default: "http://litellm.ab-eval-flow.svc:4000" - name: llm-api-key type: string - default: "mock" - - name: llm-agent-wrapper - type: string - default: "" - # ASE params - - name: ase-iterations - type: string - default: "5" - - name: ase-concurrency - type: string - default: "1" - - name: ase-judge-model - type: string - default: "" - - name: uplift-threshold - type: string - default: "0.0" - # MCPChecker params - - name: mcp-credentials-secret - type: string - default: "mcp-credentials-placeholder" - description: Secret containing MCP credentials (use placeholder when not using MCPChecker) - - name: mcpchecker-agent-model - type: string - default: "google:gemini-2.5-flash" - - name: mcpchecker-judge-model - type: string - default: "openai:gpt-4o" - - name: mcpchecker-task-timeout - type: string - default: "10m" - # A2A params + default: "sk-dummy" - name: agent-endpoint type: string default: "" - description: A2A agent endpoint URL (if empty, deploys agent from agent-image) + description: A2A agent endpoint URL (cluster-internal svc URL) - name: agent-timeout type: string default: "120" - description: A2A agent request timeout in seconds - - name: agent-image - type: string - default: "quay.io/ecosystem-appeng/google-lightspeed-agent" - description: Agent container image for deployment - - name: agent-tag + - name: workload-cluster-url type: string - default: "on-pr-b5535de9e1439da87e6610c8e3ae55f2d34231a8" - description: Agent image tag - - name: agent-namespace + default: "https://api.cn-ai-lab.2vn8.p1.openshiftapps.com:6443" + - name: workload-namespace type: string default: "ab-eval-flow" - description: Namespace to deploy agent - - name: agent-release-name + - name: eval-timeout type: string - default: "eval-agent" - description: Helm release name for agent deployment - - name: agent-chart-url - type: string - default: "https://github.com/RHEcosystemAppEng/google-lightspeed-agent.git" - description: Agent Helm chart Git URL + default: "1800" + description: Seconds to wait for the eval Pod to complete workspaces: - name: source description: Workspace containing the cloned submissions repository @@ -146,904 +78,231 @@ spec: - name: results-dir description: Path to evaluation results steps: - # Step 1: Setup and route to appropriate engine - - name: setup - image: registry.access.redhat.com/ubi9/python-311:9.6 - script: | - #!/usr/bin/env bash - set -euo pipefail - echo "=== EVALUATE PHASE: Setup ===" - echo "Eval engine: $(params.eval-engine)" - echo "Submission: $(params.submission-name)" - - # Clone/update pipeline repo - PIPELINE_DIR="$(workspaces.source.path)/_pipeline" - if [ -d "$PIPELINE_DIR/.git" ]; then - cd "$PIPELINE_DIR" - git fetch origin "$(params.pipeline-repo-revision)" --depth 1 2>/dev/null || true - git reset --hard FETCH_HEAD 2>/dev/null || true - else - git clone --depth 1 --branch "$(params.pipeline-repo-revision)" \ - "$(params.pipeline-repo-url)" "$PIPELINE_DIR" - fi - - # Create results directories - mkdir -p "$(workspaces.source.path)/eval-results/$(params.submission-name)" - mkdir -p "$(workspaces.source.path)/reports/$(params.submission-name)" - mkdir -p "$(workspaces.source.path)/_eval_tmp" - - # Initialize results with defaults - echo -n "0.0" > "$(results.treatment-mean-reward.path)" - echo -n "0.0" > "$(results.control-mean-reward.path)" - echo -n "fail" > "$(results.recommendation.path)" - echo -n "$(workspaces.source.path)/eval-results/$(params.submission-name)" > "$(results.results-dir.path)" - - # Step 2: Harbor - Scaffold - - name: harbor-scaffold - image: registry.access.redhat.com/ubi9/python-311:9.6 - script: | - #!/usr/bin/env bash - set -euo pipefail - - EVAL_ENGINE="$(params.eval-engine)" - if [ "$EVAL_ENGINE" != "harbor" ] && [ "$EVAL_ENGINE" != "both" ]; then - echo "Skipping scaffold (eval-engine=$EVAL_ENGINE)" - exit 0 - fi - - echo "=== EVALUATE PHASE: Harbor Scaffold ===" - PIPELINE_DIR="$(workspaces.source.path)/_pipeline" - SUBMISSION_DIR="$(workspaces.source.path)/submissions/$(params.submission-dir)" - OUTPUT_DIR="$(workspaces.source.path)" - - cd "$PIPELINE_DIR" - export PYTHONPATH="$PIPELINE_DIR" - pip install --quiet --no-cache-dir jinja2 pyyaml pydantic - - python scripts/scaffold.py "$SUBMISSION_DIR" "$OUTPUT_DIR" - - echo "Treatment: $OUTPUT_DIR/tasks-treatment/$(params.submission-name)" - echo "Control: $OUTPUT_DIR/tasks-control/$(params.submission-name)" - - # Step 3: Harbor - Build Treatment (using crane - instant layer append, no extraction) - - name: harbor-build-treatment - image: registry.access.redhat.com/ubi9/python-311:latest - script: | - #!/usr/bin/env bash - set -euo pipefail - - EVAL_ENGINE="$(params.eval-engine)" - if [ "$EVAL_ENGINE" != "harbor" ] && [ "$EVAL_ENGINE" != "both" ]; then - echo "Skipping build (eval-engine=$EVAL_ENGINE)" - exit 0 - fi - - echo "=== EVALUATE PHASE: Harbor Build Treatment (crane) ===" - - # Install crane (lightweight tool that appends layers without extracting) - curl -sL https://github.com/google/go-containerregistry/releases/download/v0.20.1/go-containerregistry_Linux_x86_64.tar.gz | tar xz -C /tmp crane - - LOCAL_BASE_IMAGE="local-registry.ab-eval-flow.svc.cluster.local:5000/eval-base:latest" - REGISTRY="$(params.registry-url)" - NS="$(params.registry-namespace)" - NAME="$(params.submission-name)" - SHA="$(echo '$(params.commit-sha)' | tr '/' '-')" - IMAGE_TAG="${REGISTRY}/${NS}/${NAME}:treatment-${SHA}" - CONTEXT="$(workspaces.source.path)/tasks-treatment/${NAME}/environment" - - # Create tar layer from scaffolded files - echo "Creating layer from: $CONTEXT" - tar -cvf /tmp/layer.tar -C "$CONTEXT" . - - # Login to destination registry (crane uses docker config) - mkdir -p ~/.docker - TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) - AUTH=$(echo -n "serviceaccount:${TOKEN}" | base64 -w0) - cat > ~/.docker/config.json <&1 | tee /tmp/crane-output.txt - - # Get digest from crane output - FULL_REF=$(tail -1 /tmp/crane-output.txt) - echo "$FULL_REF" > "$(workspaces.source.path)/_eval_tmp/treatment-image-ref" - echo "Treatment image: $FULL_REF" - - # Step 4: Harbor - Build Control (using crane - instant layer append, no extraction) - - name: harbor-build-control - image: registry.access.redhat.com/ubi9/python-311:latest - script: | - #!/usr/bin/env bash - set -euo pipefail - - EVAL_ENGINE="$(params.eval-engine)" - if [ "$EVAL_ENGINE" != "harbor" ] && [ "$EVAL_ENGINE" != "both" ]; then - exit 0 - fi - - echo "=== EVALUATE PHASE: Harbor Build Control (crane) ===" - - # Install crane - curl -sL https://github.com/google/go-containerregistry/releases/download/v0.20.1/go-containerregistry_Linux_x86_64.tar.gz | tar xz -C /tmp crane - - LOCAL_BASE_IMAGE="local-registry.ab-eval-flow.svc.cluster.local:5000/eval-base:latest" - REGISTRY="$(params.registry-url)" - NS="$(params.registry-namespace)" - NAME="$(params.submission-name)" - SHA="$(echo '$(params.commit-sha)' | tr '/' '-')" - IMAGE_TAG="${REGISTRY}/${NS}/${NAME}:control-${SHA}" - CONTEXT="$(workspaces.source.path)/tasks-control/${NAME}/environment" - - # Create tar layer from scaffolded files - echo "Creating layer from: $CONTEXT" - tar -cvf /tmp/layer.tar -C "$CONTEXT" . - - # Login to destination registry - mkdir -p ~/.docker - TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) - AUTH=$(echo -n "serviceaccount:${TOKEN}" | base64 -w0) - cat > ~/.docker/config.json <&1 | tee /tmp/crane-output.txt - - FULL_REF=$(tail -1 /tmp/crane-output.txt) - echo "$FULL_REF" > "$(workspaces.source.path)/_eval_tmp/control-image-ref" - echo "Control image: $FULL_REF" - - # Step 5: Harbor - Run Evaluation (uses pre-built image with Harbor installed) - - name: harbor-eval - image: $(params.eval-base-image) - computeResources: - requests: - memory: 512Mi - limits: - memory: 1Gi - env: - - name: HOME - value: /tmp - script: | - #!/usr/bin/env bash - set -euo pipefail - - EVAL_ENGINE="$(params.eval-engine)" - if [ "$EVAL_ENGINE" != "harbor" ] && [ "$EVAL_ENGINE" != "both" ]; then - exit 0 - fi - - echo "=== EVALUATE PHASE: Harbor Evaluation ===" - - # Check if Harbor is pre-installed (saves ~10 min) - if python -c "import harbor" 2>/dev/null; then - echo "Harbor already installed in base image" - else - echo "Installing Harbor from git..." - pip install --quiet --no-cache-dir \ - "git+$(params.harbor-fork-url)@$(params.harbor-fork-revision)" \ - "kubernetes>=32.0.0" \ - pydantic pyyaml - fi - - PIPELINE_DIR="$(workspaces.source.path)/_pipeline" - SUBMISSION_PATH="$(workspaces.source.path)/submissions/$(params.submission-dir)" - RESULTS_DIR="$(workspaces.source.path)/eval-results/$(params.submission-name)" - CONFIG_DIR="$(workspaces.source.path)/_eval-configs" - NAME="$(params.submission-name)" - - cd "$PIPELINE_DIR" - export PYTHONPATH="$PIPELINE_DIR" - - TREATMENT_REF=$(cat "$(workspaces.source.path)/_eval_tmp/treatment-image-ref" 2>/dev/null || echo "") - CONTROL_REF=$(cat "$(workspaces.source.path)/_eval_tmp/control-image-ref" 2>/dev/null || echo "") - - python scripts/generate_eval_config.py \ - --submission-dir "$SUBMISSION_PATH" \ - --treatment-task-dir "$(workspaces.source.path)/tasks-treatment/${NAME}" \ - --control-task-dir "$(workspaces.source.path)/tasks-control/${NAME}" \ - --output-dir "$CONFIG_DIR" \ - --eval-mode "$(params.eval-mode)" \ - --results-base-dir "$RESULTS_DIR" \ - --treatment-image-ref "$TREATMENT_REF" \ - --control-image-ref "$CONTROL_REF" \ - --llm-model "$(params.llm-model)" \ - --llm-api-base "$(params.llm-api-base)" \ - --llm-api-key "$(params.llm-api-key)" - - mkdir -p "$RESULTS_DIR/treatment" "$RESULTS_DIR/control" - - # Export LLM config for claude-code agent (Harbor reads from os.environ) - export ANTHROPIC_API_KEY="$(params.llm-api-key)" - export ANTHROPIC_BASE_URL="$(params.llm-api-base)" - - echo "=== Running treatment evaluation ===" - harbor run -c "$CONFIG_DIR/treatment-config.yaml" -y - - echo "=== Running control evaluation ===" - harbor run -c "$CONFIG_DIR/control-config.yaml" -y - - # Compute mean rewards from trial results (report generated by analyze task) - python3 - "$RESULTS_DIR" "$(workspaces.source.path)/_eval_tmp" <<'COMPUTE_REWARDS' - import json - import sys - from pathlib import Path - - results_dir = Path(sys.argv[1]) - output_dir = Path(sys.argv[2]) - - def compute_mean_reward(variant_dir: Path) -> float: - if not variant_dir.is_dir(): - return 0.0 - rewards = [] - for result_file in sorted(variant_dir.rglob("result.json")): - try: - data = json.loads(result_file.read_text()) - vr = data.get("verifier_result") or {} - rw = vr.get("rewards") or {} - reward = rw.get("reward") or vr.get("reward") - if reward is not None: - rewards.append(float(reward)) - except (json.JSONDecodeError, ValueError, TypeError): - pass - return sum(rewards) / len(rewards) if rewards else 0.0 - - t_mean = compute_mean_reward(results_dir / "treatment") - c_mean = compute_mean_reward(results_dir / "control") - rec = "pass" if t_mean >= c_mean else "fail" - - output_dir.mkdir(parents=True, exist_ok=True) - (output_dir / "treatment-reward").write_text(f"{t_mean:.4f}") - (output_dir / "control-reward").write_text(f"{c_mean:.4f}") - (output_dir / "recommendation").write_text(rec) - - print(f"Treatment mean reward: {t_mean:.4f}") - print(f"Control mean reward: {c_mean:.4f}") - print(f"Recommendation: {rec}") - COMPUTE_REWARDS - - # Step 6: ASE - Run Evaluation - - name: ase-eval - image: registry.access.redhat.com/ubi9/nodejs-20:latest - env: - - name: OPENAI_API_KEY - value: "$(params.llm-api-key)" - - name: PARAM_LLM_MODEL - value: "$(params.llm-model)" - - name: PARAM_LLM_API_BASE - value: "$(params.llm-api-base)" - - name: PARAM_ASE_ITERATIONS - value: "$(params.ase-iterations)" - - name: PARAM_ASE_CONCURRENCY - value: "$(params.ase-concurrency)" - - name: PARAM_ASE_JUDGE_MODEL - value: "$(params.ase-judge-model)" - script: | - #!/usr/bin/env bash - set -euo pipefail - - EVAL_ENGINE="$(params.eval-engine)" - if [ "$EVAL_ENGINE" != "ase" ] && [ "$EVAL_ENGINE" != "both" ]; then - exit 0 - fi - - echo "=== EVALUATE PHASE: ASE Evaluation ===" - npm install --global agent-skills-eval 2>&1 | tail -3 - - SUBMISSION_PATH="$(workspaces.source.path)/submissions/$(params.submission-dir)" - METADATA_FILE="$SUBMISSION_PATH/metadata.yaml" - - # Base: pipeline params - ITERATIONS="$PARAM_ASE_ITERATIONS" - LLM_MODEL="$PARAM_LLM_MODEL" - BASE_URL="$PARAM_LLM_API_BASE" - JUDGE_MODEL="${PARAM_ASE_JUDGE_MODEL:-$PARAM_LLM_MODEL}" - CONCURRENCY="$PARAM_ASE_CONCURRENCY" - - # Override from metadata.yaml if present - if [ -f "$METADATA_FILE" ]; then - pip install --quiet pyyaml 2>/dev/null || true - OVERRIDES=$(python3 -c "import yaml,sys;m=yaml.safe_load(open(sys.argv[1]))or{};e=m.get('experiment')or{};l=m.get('llm')or{};e.get('n_trials')and print('ITERATIONS='+str(e['n_trials']));l.get('model')and print('LLM_MODEL='+str(l['model']));l.get('api_base')and print('BASE_URL='+str(l['api_base']));l.get('judge_model')and print('JUDGE_MODEL='+str(l['judge_model']));e.get('concurrency')and print('CONCURRENCY='+str(e['concurrency']))" "$METADATA_FILE" 2>/dev/null || true) - eval "$OVERRIDES" - fi - - # Apply defaults for any empty values - ITERATIONS="${ITERATIONS:-5}" - LLM_MODEL="${LLM_MODEL:-claude-sonnet}" - BASE_URL="${BASE_URL:-http://litellm.ab-eval-flow.svc:4000}" - JUDGE_MODEL="${JUDGE_MODEL:-$LLM_MODEL}" - CONCURRENCY="${CONCURRENCY:-1}" - - [[ "$BASE_URL" != */v1 ]] && BASE_URL="${BASE_URL}/v1" - - echo "Config from metadata.yaml (with fallbacks):" - echo " Iterations: $ITERATIONS" - echo " Model: $LLM_MODEL" - echo " Judge: $JUDGE_MODEL" - echo " API Base: $BASE_URL" - echo " Concurrency: $CONCURRENCY" - - SKILL_DIR="" - if [ -f "$SUBMISSION_PATH/skills/SKILL.md" ]; then - SKILL_DIR="$SUBMISSION_PATH/skills" - else - for dir in "$SUBMISSION_PATH"/skills/*/; do - if [ -f "${dir}SKILL.md" ]; then - SKILL_DIR="$dir" - break - fi - done - fi - - if [ -z "$SKILL_DIR" ]; then - echo "ERROR: No SKILL.md found" - exit 1 - fi - - if [ ! -f "$SKILL_DIR/evals/evals.json" ] && [ -f "$SUBMISSION_PATH/evals/evals.json" ]; then - ln -sf "$SUBMISSION_PATH/evals" "$SKILL_DIR/evals" - fi - - RESULTS_DIR="$(workspaces.source.path)/eval-results/$(params.submission-name)" - - for i in $(seq 1 "$ITERATIONS"); do - echo "--- ASE Iteration $i/$ITERATIONS ---" - agent-skills-eval "$SKILL_DIR" --baseline --layout iteration --report \ - --base-url "$BASE_URL" --target "$LLM_MODEL" --judge "$JUDGE_MODEL" \ - --concurrency "$CONCURRENCY" --workspace "$RESULTS_DIR/iteration-$i" \ - --api-key-env OPENAI_API_KEY || true - done - - # Step 7: ASE - Aggregate - - name: ase-aggregate - image: registry.access.redhat.com/ubi9/python-311:9.6 - script: | - #!/usr/bin/env bash - set -euo pipefail - - EVAL_ENGINE="$(params.eval-engine)" - if [ "$EVAL_ENGINE" != "ase" ] && [ "$EVAL_ENGINE" != "both" ]; then - exit 0 - fi - - echo "=== EVALUATE PHASE: ASE Aggregation ===" - PIPELINE_DIR="$(workspaces.source.path)/_pipeline" - SUBMISSION_PATH="$(workspaces.source.path)/submissions/$(params.submission-dir)" - pip install --quiet --no-cache-dir pydantic pyyaml scipy - - cd "$PIPELINE_DIR" - export PYTHONPATH="$PIPELINE_DIR" - - # Read iterations from metadata.yaml with pipeline param as fallback - ITERATIONS=$(python3 -c " - import yaml, sys - try: - meta = yaml.safe_load(open('$SUBMISSION_PATH/metadata.yaml')) - exp = meta.get('experiment', {}) - print(exp.get('n_trials', $(params.ase-iterations))) - except: - print($(params.ase-iterations)) - ") - echo "Using iterations: $ITERATIONS" - - RESULTS_DIR="$(workspaces.source.path)/eval-results/$(params.submission-name)" - REPORT_DIR="$(workspaces.source.path)/reports/$(params.submission-name)" - - python3 scripts/aggregate_ase.py \ - --results-dir "$RESULTS_DIR" \ - --output-dir "$REPORT_DIR" \ - --submission-name "$(params.submission-name)" \ - --iterations "$ITERATIONS" \ - --threshold "$(params.uplift-threshold)" - - python3 - "$REPORT_DIR/report.json" <<'PARSE' - import json, sys - r = json.load(open(sys.argv[1])) - t = r["summary"]["treatment"].get("mean_reward", 0) - c = r["summary"]["control"].get("mean_reward", 0) - rec = r["summary"].get("recommendation", "fail") - open("$(workspaces.source.path)/_eval_tmp/treatment-reward", "w").write(f"{t:.4f}") - open("$(workspaces.source.path)/_eval_tmp/control-reward", "w").write(f"{c:.4f}") - open("$(workspaces.source.path)/_eval_tmp/recommendation", "w").write(rec) - PARSE - - # Step 8: MCPChecker - Run Evaluation - - name: mcpchecker-eval - image: registry.access.redhat.com/ubi9/python-311:9.6 + - name: submit-eval + image: registry.redhat.io/openshift4/ose-cli:latest env: - - name: MCP_URL - valueFrom: - secretKeyRef: - name: $(params.mcp-credentials-secret) - key: MCP_URL - optional: true - - name: MCP_BEARER_TOKEN - valueFrom: - secretKeyRef: - name: $(params.mcp-credentials-secret) - key: MCP_BEARER_TOKEN - optional: true - - name: OPENAI_API_KEY + - name: WORKLOAD_TOKEN valueFrom: secretKeyRef: - name: llm-credentials - key: api-key - optional: true - - name: LITELLM_PROXY_URL - value: "http://litellm.ab-eval-flow.svc:4000/v1" + name: workload-cluster-credentials + key: token script: | - #!/bin/bash + #!/usr/bin/env bash set -euo pipefail - - EVAL_ENGINE="$(params.eval-engine)" - if [ "$EVAL_ENGINE" != "mcpchecker" ]; then - exit 0 - fi - - echo "=== EVALUATE PHASE: MCPChecker ===" - - pip install --quiet --no-cache-dir mcpchecker 2>&1 | tail -3 + echo "=== EVALUATE PHASE: Submit Remote Eval ===" - if [ -z "${MCP_URL:-}" ]; then - echo "ERROR: MCP_URL not set" - exit 1 - fi - - cd "$(workspaces.source.path)/submissions/$(params.submission-dir)" - - # Expand env vars in mcp-config.yaml - envsubst < mcp-config.yaml > mcp-config-resolved.yaml - - if [ -n "$LITELLM_PROXY_URL" ]; then - export OPENAI_BASE_URL="$LITELLM_PROXY_URL" - fi - - OUTPUT_FILE="/tmp/mcpchecker-out.json" - RAW_OUTPUT="/tmp/mcpchecker-raw.txt" - - mcpchecker check eval.yaml --mcp-config-file mcp-config-resolved.yaml -o json > "$RAW_OUTPUT" 2>&1 || true - - # Extract JSON - if grep -n '^{' "$RAW_OUTPUT" > /dev/null; then - START_LINE=$(grep -n '^{' "$RAW_OUTPUT" | head -1 | cut -d: -f1) - tail -n +$START_LINE "$RAW_OUTPUT" > "$OUTPUT_FILE" - else - cp "$RAW_OUTPUT" "$OUTPUT_FILE" - fi - - RESULTS_DIR="$(workspaces.source.path)/eval-results/$(params.submission-name)" - mkdir -p "$RESULTS_DIR" - cp "$OUTPUT_FILE" "$RESULTS_DIR/mcpchecker-out.json" + CLUSTER_URL="$(params.workload-cluster-url)" + NAMESPACE="$(params.workload-namespace)" + OC="oc --server=$CLUSTER_URL --token=$WORKLOAD_TOKEN --insecure-skip-tls-verify=true" - # Step 9: MCPChecker - Aggregate - - name: mcpchecker-aggregate - image: registry.access.redhat.com/ubi9/python-311:9.6 - script: | - #!/bin/bash - set -euo pipefail - EVAL_ENGINE="$(params.eval-engine)" - if [ "$EVAL_ENGINE" != "mcpchecker" ]; then - exit 0 - fi - - echo "=== EVALUATE PHASE: MCPChecker Aggregation ===" - PIPELINE_DIR="$(workspaces.source.path)/_pipeline" - pip install --quiet --no-cache-dir pydantic pyyaml - - export PYTHONPATH="$PIPELINE_DIR" - - RESULTS_DIR="$(workspaces.source.path)/eval-results/$(params.submission-name)" - REPORT_DIR="$(workspaces.source.path)/reports/$(params.submission-name)" - mkdir -p "$REPORT_DIR" - - python "$PIPELINE_DIR/scripts/aggregate_mcpchecker.py" \ - --output-json "$RESULTS_DIR/mcpchecker-out.json" \ - --submission-name "$(params.submission-name)" \ - --report-dir "$REPORT_DIR" \ - --pipeline-run-id "$(params.pipeline-run-id)" \ - > /tmp/mcp-result.json - - python3 - /tmp/mcp-result.json <<'PARSE' - import json, sys - r = json.load(open(sys.argv[1])) - score = r.get("overall_score", 0) - rec = r.get("recommendation", "fail") - open("$(workspaces.source.path)/_eval_tmp/treatment-reward", "w").write(f"{score:.4f}") - open("$(workspaces.source.path)/_eval_tmp/control-reward", "w").write("0.0000") - open("$(workspaces.source.path)/_eval_tmp/recommendation", "w").write(rec) - PARSE - - # Step 10: A2A - Deploy Agent as Deployment+Service (if no endpoint provided) - - name: a2a-deploy-agent - image: registry.redhat.io/openshift4/ose-cli:latest - script: | - #!/usr/bin/env bash - set -euo pipefail + SUBMISSION_NAME="$(params.submission-name)" + SUBMISSION_DIR="$(params.submission-dir)" + AGENT_ENDPOINT="$(params.agent-endpoint)" + RUN_ID=$(echo "$(params.pipeline-run-id)" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g' | cut -c1-20) + POD_NAME="eval-job-${RUN_ID:-$(date +%s)}" - EVAL_ENGINE="$(params.eval-engine)" - if [ "$EVAL_ENGINE" != "a2a" ]; then - echo "Skipping A2A deploy (eval-engine=$EVAL_ENGINE)" - exit 0 - fi + echo "Engine: $EVAL_ENGINE" + echo "Submission: $SUBMISSION_NAME" + echo "Agent endpoint: $AGENT_ENDPOINT" + echo "Eval pod: $POD_NAME" + echo "Workload cluster: $CLUSTER_URL" - AGENT_ENDPOINT="$(params.agent-endpoint)" - if [ -n "$AGENT_ENDPOINT" ]; then - echo "Using external agent: $AGENT_ENDPOINT" - echo "$AGENT_ENDPOINT" > "$(workspaces.source.path)/_eval_tmp/agent-endpoint" - echo "false" > "$(workspaces.source.path)/_eval_tmp/agent-deployed" - exit 0 - fi + # Write defaults for results + echo -n "0.0" > "$(results.treatment-mean-reward.path)" + echo -n "0.0" > "$(results.control-mean-reward.path)" + echo -n "fail" > "$(results.recommendation.path)" + RESULTS_DIR="$(workspaces.source.path)/eval-results/$SUBMISSION_NAME" + mkdir -p "$RESULTS_DIR" + echo -n "$RESULTS_DIR" > "$(results.results-dir.path)" - # Generate unique name for this evaluation - RUN_ID=$(echo "$(params.pipeline-run-id)" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g' | cut -c1-20) - AGENT_NAME="a2a-agent-${RUN_ID}" - NAMESPACE=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace) - - echo "Deploying agent: $AGENT_NAME" - - # Create Deployment + Service - cat </dev/null 2>&1; then - ENDPOINT="http://${AGENT_NAME}.${NAMESPACE}.svc:8000" - if curl -sf "$ENDPOINT/.well-known/agent.json" > /dev/null 2>&1; then - echo " ready!" - echo "$ENDPOINT" > "$(workspaces.source.path)/_eval_tmp/agent-endpoint" - echo "$AGENT_NAME" > "$(workspaces.source.path)/_eval_tmp/agent-deployed" - exit 0 - fi - fi - printf "." - sleep 5 - done - - echo " FAILED" - oc delete deployment/$AGENT_NAME service/$AGENT_NAME --ignore-not-found=true - exit 1 + restartPolicy: Never + activeDeadlineSeconds: $(params.eval-timeout) + containers: + - name: eval + image: $(params.eval-base-image) + command: ["/bin/bash", "-c"] + args: + - | + set -euo pipefail + export HOME=/tmp + export PYTHONHTTPSVERIFY=0 - # Step 10.5: A2A - Pre-flight health check - - name: a2a-pre-flight - image: registry.access.redhat.com/ubi9/ubi-minimal:latest - script: | - #!/usr/bin/env bash - set -euo pipefail + echo "=== Remote Eval Pod Starting ===" + echo "Engine: $EVAL_ENGINE" + echo "Agent: $AGENT_ENDPOINT" - EVAL_ENGINE="$(params.eval-engine)" - if [ "$EVAL_ENGINE" != "a2a" ]; then - echo "Skipping A2A pre-flight (eval-engine=$EVAL_ENGINE)" - exit 0 - fi + # Clone ABEvalFlow + git clone --depth 1 --branch "$(params.pipeline-repo-revision)" \ + "$(params.pipeline-repo-url)" /tmp/abevalflow + cd /tmp/abevalflow + export PYTHONPATH=/tmp/abevalflow - ENDPOINT_FILE="$(workspaces.source.path)/_eval_tmp/agent-endpoint" - if [ ! -f "$ENDPOINT_FILE" ]; then - echo "ERROR: No agent endpoint file found — deploy step may have failed" - exit 1 - fi + SUBMISSION_PATH="/tmp/abevalflow/submissions/$SUBMISSION_DIR" + RESULTS_DIR="/tmp/eval-results" + REPORT_DIR="/tmp/eval-reports" + mkdir -p "\$RESULTS_DIR" "\$REPORT_DIR" - AGENT_ENDPOINT=$(cat "$ENDPOINT_FILE") - if [ -z "$AGENT_ENDPOINT" ]; then - echo "ERROR: Agent endpoint is empty" - exit 1 - fi + if [ "$EVAL_ENGINE" = "a2a" ]; then + # Read trial count from metadata + N_ATTEMPTS=\$(python3 -c " + import yaml + try: + meta = yaml.safe_load(open('\$SUBMISSION_PATH/metadata.yaml')) + print(meta.get('experiment', {}).get('n_trials', 5)) + except: + print(5) + ") - echo "=== A2A Pre-flight Health Check ===" - echo "Endpoint: $AGENT_ENDPOINT" + # Find task directory + TASK_DIR="" + if [ -f "\$SUBMISSION_PATH/task.toml" ]; then + TASK_DIR="\$SUBMISSION_PATH" + elif [ -d "\$SUBMISSION_PATH/tasks" ]; then + for subdir in "\$SUBMISSION_PATH/tasks"/*/; do + if [ -f "\${subdir}task.toml" ]; then + TASK_DIR="\${subdir%/}" + break + fi + done + fi - HEALTH_URL="${AGENT_ENDPOINT}/.well-known/agent.json" - HTTP_STATUS=$(curl -sf --max-time 15 -o /dev/null -w "%{http_code}" "$HEALTH_URL" 2>/dev/null || echo "000") + if [ -z "\$TASK_DIR" ]; then + echo "ERROR: No task.toml found" + exit 1 + fi - if [ "$HTTP_STATUS" != "200" ]; then - echo "ERROR: Agent unreachable — /.well-known/agent.json returned HTTP $HTTP_STATUS" - echo "Agent endpoint: $AGENT_ENDPOINT" - echo "This will cause the evaluation to fail. Check that the agent is running and accessible." - exit 1 - fi + echo "Task: \$(basename \$TASK_DIR) | Attempts: \$N_ATTEMPTS" - echo "Agent is healthy (HTTP $HTTP_STATUS)" + # Generate Harbor config + python3 - "\$RESULTS_DIR" "\$TASK_DIR" "\$N_ATTEMPTS" "$AGENT_ENDPOINT" "$(params.agent-timeout)" "$(params.llm-api-base)" "openai/$(params.llm-model)" <<'GENCFG' + import sys, yaml + results_dir, task_dir, n_attempts, endpoint, timeout, llm_api_base, llm_model = sys.argv[1:8] + config = { + "job_name": "a2a-eval", + "jobs_dir": results_dir, + "n_attempts": int(n_attempts), + "agents": [{"import_path": "abevalflow.harbor_agents.a2a_adapter:A2AAgent", "kwargs": {"endpoint": endpoint, "timeout": int(timeout)}}], + "tasks": [{"path": task_dir}], + "environment": {"type": "local"}, + "verifier": {"env": {"LLM_JUDGE_MODEL": llm_model, "LLM_API_BASE": llm_api_base, "OPENAI_API_KEY": "sk-dummy"}} + } + with open(results_dir + "/config.yaml", "w") as f: + yaml.dump(config, f, default_flow_style=False) + GENCFG - # Step 11: A2A Agent Evaluation - - name: a2a-eval - image: $(params.eval-base-image) - computeResources: - requests: - memory: 256Mi - limits: - cpu: 500m - memory: 1Gi - env: - - name: HOME - value: /tmp - - name: LLM_JUDGE_MODEL - value: "openai/$(params.llm-model)" - - name: LLM_BASE_URL - value: "$(params.llm-api-base)" - - name: OPENAI_API_KEY - valueFrom: - secretKeyRef: - name: llm-credentials - key: api-key - optional: true - script: | - #!/usr/bin/env bash - set -euo pipefail - - EVAL_ENGINE="$(params.eval-engine)" - if [ "$EVAL_ENGINE" != "a2a" ]; then - echo "Skipping A2A eval (eval-engine=$EVAL_ENGINE)" - exit 0 - fi - - # Read endpoint from deploy step or param - ENDPOINT_FILE="$(workspaces.source.path)/_eval_tmp/agent-endpoint" - if [ -f "$ENDPOINT_FILE" ]; then - AGENT_ENDPOINT=$(cat "$ENDPOINT_FILE") - else - AGENT_ENDPOINT="$(params.agent-endpoint)" - fi - - if [ -z "$AGENT_ENDPOINT" ]; then - echo "ERROR: No agent endpoint available (deploy failed or not provided)" - exit 1 - fi - - echo "=== A2A Evaluation ===" - - PIPELINE_DIR="$(workspaces.source.path)/_pipeline" - SUBMISSION_PATH="$(workspaces.source.path)/submissions/$(params.submission-dir)" - RESULTS_DIR="$(workspaces.source.path)/eval-results/$(params.submission-name)" - REPORT_DIR="$(workspaces.source.path)/reports/$(params.submission-name)" - - mkdir -p "$RESULTS_DIR" "$REPORT_DIR" - - # Harbor (with LocalEnvironment) is pre-installed in eval-base:local-env. - # Expose abevalflow (A2AAgent adapter) via PYTHONPATH — no install needed, - # avoids virtualenv --user restrictions and permission conflicts. - export PYTHONPATH="$PIPELINE_DIR:${PYTHONPATH:-}" - - cd "$PIPELINE_DIR" - - # Read evaluation config from metadata.yaml - N_ATTEMPTS=$(python3 -c " - import yaml, sys - try: - meta = yaml.safe_load(open('$SUBMISSION_PATH/metadata.yaml')) - exp = meta.get('experiment', {}) - print(exp.get('n_trials', 5)) - except: - print(5) - ") - - # Find task directory (look for task.toml) - TASK_DIR="" - if [ -f "$SUBMISSION_PATH/task.toml" ]; then - TASK_DIR="$SUBMISSION_PATH" - elif [ -d "$SUBMISSION_PATH/tasks" ]; then - for subdir in "$SUBMISSION_PATH/tasks"/*/; do - if [ -f "${subdir}task.toml" ]; then - TASK_DIR="${subdir%/}" + python3 -c "from abevalflow.harbor_agents.a2a_adapter import A2AAgent; print('A2AAgent imported OK')" + + harbor run -c "\$RESULTS_DIR/config.yaml" -y 2>&1 || true + + # Show exceptions if any + find "\$RESULTS_DIR" -name "exception.txt" -exec echo "--- {} ---" \; -exec cat {} \; 2>/dev/null || true + + # Compute mean reward + python3 - "\$RESULTS_DIR" <<'COMPUTE' + import json, sys + from pathlib import Path + results_dir = Path(sys.argv[1]) + rewards = [] + for f in sorted(results_dir.rglob("result.json")): + try: + data = json.loads(f.read_text()) + vr = data.get("verifier_result") or {} + rw = vr.get("rewards") or {} + reward = rw.get("reward") or vr.get("reward") + if reward is not None: + rewards.append(float(reward)) + except: + pass + mean_reward = sum(rewards) / len(rewards) if rewards else 0.0 + rec = "pass" if mean_reward >= 0.5 else "fail" + result = {"mean_reward": mean_reward, "recommendation": rec, "n_trials": len(rewards)} + Path("/tmp/eval-summary.json").write_text(json.dumps(result)) + print(json.dumps(result, indent=2)) + COMPUTE + fi + + echo "=== Remote Eval Pod Complete ===" + env: + - name: HOME + value: /tmp + - name: LLM_JUDGE_MODEL + value: "openai/$(params.llm-model)" + - name: LLM_BASE_URL + value: "$(params.llm-api-base)" + - name: LLM_API_BASE + value: "$(params.llm-api-base)" + - name: OPENAI_API_KEY + value: "$(params.llm-api-key)" + resources: + requests: + cpu: 200m + memory: 512Mi + limits: + cpu: "1" + memory: 2Gi + PODSPEC + + echo "Eval pod submitted. Waiting for completion..." + + TIMEOUT=$(params.eval-timeout) + POLL_INTERVAL=15 + ELAPSED=0 + + while [ $ELAPSED -lt $TIMEOUT ]; do + PHASE=$($OC get pod $POD_NAME -n $NAMESPACE -o jsonpath='{.status.phase}' 2>/dev/null || echo "Unknown") + case "$PHASE" in + Succeeded) + echo "Eval pod completed successfully (${ELAPSED}s)" break - fi - done - fi - - if [ -z "$TASK_DIR" ]; then - echo "ERROR: No task.toml found in submission" + ;; + Failed) + echo "Eval pod failed (${ELAPSED}s)" + $OC logs $POD_NAME -n $NAMESPACE 2>&1 | tail -30 + break + ;; + *) + printf "." + sleep $POLL_INTERVAL + ELAPSED=$((ELAPSED + POLL_INTERVAL)) + ;; + esac + done + echo "" + + if [ $ELAPSED -ge $TIMEOUT ]; then + echo "TIMEOUT: Eval pod did not complete in ${TIMEOUT}s" + $OC logs $POD_NAME -n $NAMESPACE 2>&1 | tail -30 + $OC delete pod $POD_NAME -n $NAMESPACE --ignore-not-found=true exit 1 fi - - TASK_NAME=$(basename "$TASK_DIR") - echo "Task: $TASK_NAME | Attempts: $N_ATTEMPTS" - - # Generate Harbor config YAML - CONFIG_FILE="$RESULTS_DIR/a2a-config.yaml" - LLM_API_BASE="$(params.llm-api-base)" - LLM_MODEL="openai/$(params.llm-model)" - - python3 - "$CONFIG_FILE" "$TASK_DIR" "$RESULTS_DIR" "$N_ATTEMPTS" "$AGENT_ENDPOINT" "$(params.agent-timeout)" "$LLM_API_BASE" "$LLM_MODEL" <<'GENCFG' - import sys - import yaml - - config_file, task_dir, results_dir, n_attempts, endpoint, timeout, llm_api_base, llm_model = sys.argv[1:9] - - config = { - "job_name": "a2a-eval", - "jobs_dir": results_dir, - "n_attempts": int(n_attempts), - "agents": [{ - "import_path": "abevalflow.harbor_agents.a2a_adapter:A2AAgent", - "kwargs": { - "endpoint": endpoint, - "timeout": int(timeout) - } - }], - "tasks": [{"path": task_dir}], - "environment": { - "type": "local" - }, - "verifier": { - "env": { - "LLM_JUDGE_MODEL": llm_model, - "LLM_API_BASE": llm_api_base, - "OPENAI_API_KEY": "sk-dummy" - } - } - } - - with open(config_file, "w") as f: - yaml.dump(config, f, default_flow_style=False) - GENCFG - - # Verify A2A adapter can be imported - python3 -c "from abevalflow.harbor_agents.a2a_adapter import A2AAgent; print('A2AAgent imported OK')" - - # Run Harbor evaluation and capture any errors - harbor run -c "$CONFIG_FILE" -y 2>&1 || true - - # Show exception details if any - echo "=== Exception details ===" - find "$RESULTS_DIR" -name "exception.txt" -exec echo "--- {} ---" \; -exec cat {} \; 2>/dev/null || echo "No exception files" - - # Compute mean reward from Harbor results - python3 - "$RESULTS_DIR" "$(workspaces.source.path)/_eval_tmp" <<'COMPUTE_REWARDS' - import json - import sys - from pathlib import Path - - results_dir = Path(sys.argv[1]) - output_dir = Path(sys.argv[2]) - - def compute_mean_reward(base_dir: Path) -> float: - if not base_dir.is_dir(): - return 0.0 - rewards = [] - for result_file in sorted(base_dir.rglob("result.json")): - try: - data = json.loads(result_file.read_text()) - vr = data.get("verifier_result") or {} - rw = vr.get("rewards") or {} - reward = rw.get("reward") or vr.get("reward") - if reward is not None: - rewards.append(float(reward)) - except (json.JSONDecodeError, ValueError, TypeError): - pass - return sum(rewards) / len(rewards) if rewards else 0.0 - - mean_reward = compute_mean_reward(results_dir) - rec = "pass" if mean_reward >= 0.5 else "fail" - - output_dir.mkdir(parents=True, exist_ok=True) - (output_dir / "treatment-reward").write_text(f"{mean_reward:.4f}") - (output_dir / "control-reward").write_text("0.0000") - (output_dir / "recommendation").write_text(rec) - - print(f"A2A mean reward: {mean_reward:.4f}") - print(f"Recommendation: {rec}") - COMPUTE_REWARDS - # Step 12: Finalize results - - name: finalize - image: registry.access.redhat.com/ubi9/python-311:9.6 - script: | - #!/usr/bin/env bash - set -euo pipefail - echo "=== EVALUATE PHASE: Finalize ===" - - # Read results from temp files - EVAL_TMP="$(workspaces.source.path)/_eval_tmp" - T_REWARD=$(cat "$EVAL_TMP/treatment-reward" 2>/dev/null || echo "0.0") - C_REWARD=$(cat "$EVAL_TMP/control-reward" 2>/dev/null || echo "0.0") - REC=$(cat "$EVAL_TMP/recommendation" 2>/dev/null || echo "fail") - - echo -n "$T_REWARD" > "$(results.treatment-mean-reward.path)" - echo -n "$C_REWARD" > "$(results.control-mean-reward.path)" - echo -n "$REC" > "$(results.recommendation.path)" - - echo "Treatment reward: $T_REWARD" - echo "Control reward: $C_REWARD" - echo "Recommendation: $REC" - echo "Results: $(cat "$(results.results-dir.path)")" + echo "=== Retrieving results ===" + $OC logs $POD_NAME -n $NAMESPACE 2>&1 | tee /tmp/eval-pod-logs.txt - # Step 13: Cleanup A2A agent deployment (always runs) - - name: a2a-cleanup - image: registry.redhat.io/openshift4/ose-cli:latest - script: | - #!/usr/bin/env bash - - EVAL_ENGINE="$(params.eval-engine)" - if [ "$EVAL_ENGINE" != "a2a" ]; then - exit 0 - fi - - DEPLOYED_FILE="$(workspaces.source.path)/_eval_tmp/agent-deployed" - if [ ! -f "$DEPLOYED_FILE" ] || [ "$(cat "$DEPLOYED_FILE")" = "false" ]; then - echo "No agent to cleanup (external endpoint used)" - exit 0 + SUMMARY=$(grep -A5 '"mean_reward"' /tmp/eval-pod-logs.txt | head -6 || echo "") + if [ -n "$SUMMARY" ]; then + echo "$SUMMARY" > /tmp/eval-summary-extracted.json + MEAN_REWARD=$(python3 -c "import json; print(json.loads(open('/tmp/eval-summary-extracted.json').read().strip())['mean_reward'])" 2>/dev/null || echo "0.0") + REC=$(python3 -c "import json; print(json.loads(open('/tmp/eval-summary-extracted.json').read().strip())['recommendation'])" 2>/dev/null || echo "fail") + echo -n "$MEAN_REWARD" > "$(results.treatment-mean-reward.path)" + echo -n "0.0" > "$(results.control-mean-reward.path)" + echo -n "$REC" > "$(results.recommendation.path)" + echo "Mean reward: $MEAN_REWARD" + echo "Recommendation: $REC" + else + echo "WARNING: Could not extract eval summary from pod logs" fi - - AGENT_NAME=$(cat "$DEPLOYED_FILE") - echo "Cleaning up agent: $AGENT_NAME" - oc delete deployment/$AGENT_NAME service/$AGENT_NAME --ignore-not-found=true - echo "Cleanup complete" + + $OC delete pod $POD_NAME -n $NAMESPACE --ignore-not-found=true + echo "=== Evaluate phase complete ===" From 001e9f1a97f9206984c49283e7cda8f3e7234634 Mon Sep 17 00:00:00 2001 From: ikrispin Date: Wed, 22 Jul 2026 12:12:03 +0300 Subject: [PATCH 2/2] fix: update agent image tag to latest available commit --- pipeline/integration/konflux-eval-pipelinerun.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline/integration/konflux-eval-pipelinerun.yaml b/pipeline/integration/konflux-eval-pipelinerun.yaml index ef385c6..7d3762b 100644 --- a/pipeline/integration/konflux-eval-pipelinerun.yaml +++ b/pipeline/integration/konflux-eval-pipelinerun.yaml @@ -51,7 +51,7 @@ spec: value: task params: - name: agent-image - value: quay.io/ecosystem-appeng/google-lightspeed-agent:on-pr-4af1ac0fab1c823cf2d034961c902a90c54ba4f4 + value: quay.io/ecosystem-appeng/google-lightspeed-agent:176a268e783a84f903ce680a26e031294ea985f7 - name: llm-api-base value: "http://litellm.ab-eval-flow.svc:4000" - name: llm-model