From 2fb3c892adf83c84985801a13d13928c16bd7aa4 Mon Sep 17 00:00:00 2001 From: ikrispin Date: Thu, 16 Jul 2026 15:08:12 +0300 Subject: [PATCH 1/5] Remove explicit serviceAccountName from PipelineRun Konflux injects the integration-runner SA automatically. The custom abevalflow-integration SA cannot be created on managed Konflux clusters due to RBAC restrictions. --- pipeline/integration/konflux-eval-pipelinerun.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/pipeline/integration/konflux-eval-pipelinerun.yaml b/pipeline/integration/konflux-eval-pipelinerun.yaml index 279fe3f..922660a 100644 --- a/pipeline/integration/konflux-eval-pipelinerun.yaml +++ b/pipeline/integration/konflux-eval-pipelinerun.yaml @@ -6,8 +6,6 @@ spec: timeouts: pipeline: "4h" tasks: "3h" - taskRunTemplate: - serviceAccountName: abevalflow-integration pipelineSpec: params: - name: SNAPSHOT From 8c602604f38aaf23a4800bc6fa6e7b30234abff7 Mon Sep 17 00:00:00 2001 From: ikrispin Date: Tue, 21 Jul 2026 15:23:12 +0300 Subject: [PATCH 2/5] Adapt deploy/cleanup tasks for cross-cluster deployment The Konflux managed cluster (stone-prod-p02) doesn't allow creating Deployments/Services. The agent is now deployed to the workload cluster (cn-ai-lab/ab-eval-flow) via a remote oc command using a stored token. A Route is created to expose the agent externally so the eval pipeline on Konflux can reach it. --- pipeline/tasks/konflux/cleanup-agent.yaml | 37 +++++++++-- pipeline/tasks/konflux/deploy-agent.yaml | 78 +++++++++++++++++------ 2 files changed, 89 insertions(+), 26 deletions(-) diff --git a/pipeline/tasks/konflux/cleanup-agent.yaml b/pipeline/tasks/konflux/cleanup-agent.yaml index 793e91c..7e3fb85 100644 --- a/pipeline/tasks/konflux/cleanup-agent.yaml +++ b/pipeline/tasks/konflux/cleanup-agent.yaml @@ -7,8 +7,9 @@ metadata: app.kubernetes.io/component: konflux spec: description: >- - Cleans up the A2A agent Deployment and Service created by the deploy-agent - task. Runs in the pipeline's finally block to ensure cleanup even on failure. + Cleans up the A2A agent Deployment and Service on the remote workload + cluster. Runs in the pipeline's finally block to ensure cleanup even + on failure. params: - name: agent-name type: string @@ -18,9 +19,24 @@ spec: type: string default: "false" description: Whether an agent was actually deployed + - name: workload-cluster-url + type: string + default: "https://api.cn-ai-lab.2vn8.p1.openshiftapps.com:6443" + description: API URL of the workload cluster + - name: workload-namespace + type: string + default: "ab-eval-flow" + description: Namespace on the workload cluster steps: - name: cleanup image: registry.redhat.io/openshift4/ose-cli:latest + env: + - name: WORKLOAD_TOKEN + valueFrom: + secretKeyRef: + name: workload-cluster-credentials + key: token + optional: true script: | #!/usr/bin/env bash @@ -32,10 +48,19 @@ spec: exit 0 fi - echo "=== CLEANUP AGENT ===" - echo "Deleting: $AGENT_NAME" + if [ -z "${WORKLOAD_TOKEN:-}" ]; then + echo "WARNING: No workload cluster token, cannot clean up" + exit 0 + fi + + echo "=== CLEANUP AGENT (cross-cluster) ===" + CLUSTER_URL="$(params.workload-cluster-url)" + NAMESPACE="$(params.workload-namespace)" + OC_REMOTE="oc --server=$CLUSTER_URL --token=$WORKLOAD_TOKEN --insecure-skip-tls-verify=true" - oc delete deployment/$AGENT_NAME --ignore-not-found=true - oc delete service/$AGENT_NAME --ignore-not-found=true + echo "Deleting: $AGENT_NAME in $NAMESPACE on $CLUSTER_URL" + $OC_REMOTE delete route/$AGENT_NAME -n $NAMESPACE --ignore-not-found=true + $OC_REMOTE delete deployment/$AGENT_NAME -n $NAMESPACE --ignore-not-found=true + $OC_REMOTE delete service/$AGENT_NAME -n $NAMESPACE --ignore-not-found=true echo "Cleanup complete" diff --git a/pipeline/tasks/konflux/deploy-agent.yaml b/pipeline/tasks/konflux/deploy-agent.yaml index 62b050e..c7a39c3 100644 --- a/pipeline/tasks/konflux/deploy-agent.yaml +++ b/pipeline/tasks/konflux/deploy-agent.yaml @@ -7,9 +7,10 @@ metadata: app.kubernetes.io/component: konflux spec: description: >- - Deploys an A2A agent as a Deployment + Service in the current namespace - using the container image from the Konflux snapshot. Waits for the agent - to become ready and outputs its endpoint URL. + Deploys an A2A agent as a Deployment + Service on a remote workload + cluster using the container image from the Konflux snapshot. The agent + is deployed to the ab-eval-flow namespace on the workload cluster where + LiteLLM and other eval infrastructure already live. params: - name: agent-image type: string @@ -30,9 +31,17 @@ spec: type: string default: "300" description: Seconds to wait for agent readiness + - name: workload-cluster-url + type: string + default: "https://api.cn-ai-lab.2vn8.p1.openshiftapps.com:6443" + description: API URL of the workload cluster where the agent is deployed + - name: workload-namespace + type: string + default: "ab-eval-flow" + description: Namespace on the workload cluster to deploy the agent results: - name: agent-endpoint - description: HTTP endpoint URL of the deployed agent + description: HTTP endpoint URL of the deployed agent (cluster-internal) - name: agent-name description: Name of the Deployment/Service (for cleanup) - name: deployed @@ -40,26 +49,37 @@ spec: steps: - name: deploy image: registry.redhat.io/openshift4/ose-cli:latest + env: + - name: WORKLOAD_TOKEN + valueFrom: + secretKeyRef: + name: workload-cluster-credentials + key: token script: | #!/usr/bin/env bash set -euo pipefail - echo "=== DEPLOY AGENT ===" + echo "=== DEPLOY AGENT (cross-cluster) ===" + + CLUSTER_URL="$(params.workload-cluster-url)" + NAMESPACE="$(params.workload-namespace)" + OC_REMOTE="oc --server=$CLUSTER_URL --token=$WORKLOAD_TOKEN --insecure-skip-tls-verify=true" RUN_ID=$(echo "$(params.pipeline-run-id)" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g' | cut -c1-20) AGENT_NAME="a2a-eval-${RUN_ID:-$(date +%s)}" - NAMESPACE=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace) AGENT_IMAGE="$(params.agent-image)" - echo "Agent name: $AGENT_NAME" + echo "Workload cluster: $CLUSTER_URL" echo "Namespace: $NAMESPACE" + echo "Agent name: $AGENT_NAME" echo "Image: $AGENT_IMAGE" - cat </dev/null 2>&1; then - if curl -sf "$ENDPOINT/.well-known/agent.json" > /dev/null 2>&1; then - echo " ready!" - echo -n "$ENDPOINT" > "$(results.agent-endpoint.path)" - echo -n "$AGENT_NAME" > "$(results.agent-name.path)" - echo -n "true" > "$(results.deployed.path)" - echo "Agent endpoint: $ENDPOINT" - exit 0 + if $OC_REMOTE rollout status deployment/$AGENT_NAME -n $NAMESPACE --timeout=5s >/dev/null 2>&1; then + ROUTE_HOST=$($OC_REMOTE get route $AGENT_NAME -n $NAMESPACE -o jsonpath='{.spec.host}' 2>/dev/null || echo "") + if [ -n "$ROUTE_HOST" ]; then + ENDPOINT="https://${ROUTE_HOST}" + if curl -skf "$ENDPOINT/.well-known/agent.json" > /dev/null 2>&1; then + echo " ready!" + echo -n "$ENDPOINT" > "$(results.agent-endpoint.path)" + echo -n "$AGENT_NAME" > "$(results.agent-name.path)" + echo -n "true" > "$(results.deployed.path)" + echo "Agent endpoint: $ENDPOINT" + exit 0 + fi fi fi printf "." @@ -144,7 +182,7 @@ spec: echo " TIMED OUT" echo "Cleaning up failed deployment..." - oc delete deployment/$AGENT_NAME service/$AGENT_NAME --ignore-not-found=true + $OC_REMOTE delete deployment/$AGENT_NAME service/$AGENT_NAME -n $NAMESPACE --ignore-not-found=true echo -n "" > "$(results.agent-endpoint.path)" echo -n "$AGENT_NAME" > "$(results.agent-name.path)" echo -n "false" > "$(results.deployed.path)" From 15c51c214626b903ae46fc8c42c6ba0bf95c28d8 Mon Sep 17 00:00:00 2001 From: ikrispin Date: Tue, 21 Jul 2026 16:25:19 +0300 Subject: [PATCH 3/5] Fix Quay bundle paths: use flat repos instead of nested Quay doesn't support nested repository paths like org/repo/subrepo. Changed from quay.io/rh-ee-ikrispin/abevalflow-catalog/task-X to quay.io/rh-ee-ikrispin/abevalflow-task-X for all bundle references. --- pipeline/integration/Makefile | 14 +++++++------- .../integration/konflux-eval-pipelinerun.yaml | 18 +++++++++--------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/pipeline/integration/Makefile b/pipeline/integration/Makefile index 9e87d8e..71f5311 100644 --- a/pipeline/integration/Makefile +++ b/pipeline/integration/Makefile @@ -1,4 +1,4 @@ -QUAY_REPO ?= quay.io/rh-ee-ikrispin/abevalflow-catalog +QUAY_NS ?= quay.io/rh-ee-ikrispin VERSION ?= 0.1 TASKS_DIR := ../tasks/konflux @@ -12,21 +12,21 @@ help: ## Show this help bundles: $(addprefix bundle-,$(TASKS)) ## Build and push all Tekton Bundles bundle-%: ## Push a single task bundle (e.g. make bundle-parse-snapshot) - @echo "=== Pushing task-$* ===" - tkn bundle push $(QUAY_REPO)/task-$*:$(VERSION) \ + @echo "=== Pushing abevalflow-task-$* ===" + tkn bundle push $(QUAY_NS)/abevalflow-task-$*:$(VERSION) \ -f $(TASKS_DIR)/$*.yaml @echo "" list: ## List all bundles in the registry @for task in $(TASKS); do \ - echo "--- task-$$task ---"; \ - tkn bundle list $(QUAY_REPO)/task-$$task 2>/dev/null || echo " (not found)"; \ + echo "--- abevalflow-task-$$task ---"; \ + tkn bundle list $(QUAY_NS)/abevalflow-task-$$task 2>/dev/null || echo " (not found)"; \ done digests: ## Print SHA digests for all pushed bundles @for task in $(TASKS); do \ - DIGEST=$$(skopeo inspect --format '{{.Digest}}' docker://$(QUAY_REPO)/task-$$task:$(VERSION) 2>/dev/null || echo "NOT_FOUND"); \ - echo "$(QUAY_REPO)/task-$$task:$(VERSION)@$$DIGEST"; \ + DIGEST=$$(skopeo inspect --format '{{.Digest}}' docker://$(QUAY_NS)/abevalflow-task-$$task:$(VERSION) 2>/dev/null || echo "NOT_FOUND"); \ + echo "$(QUAY_NS)/abevalflow-task-$$task:$(VERSION)@$$DIGEST"; \ done clean: ## Remove local tkn bundle cache diff --git a/pipeline/integration/konflux-eval-pipelinerun.yaml b/pipeline/integration/konflux-eval-pipelinerun.yaml index 922660a..fc7d5ff 100644 --- a/pipeline/integration/konflux-eval-pipelinerun.yaml +++ b/pipeline/integration/konflux-eval-pipelinerun.yaml @@ -28,7 +28,7 @@ spec: - name: name value: parse-snapshot - name: bundle - value: quay.io/rh-ee-ikrispin/abevalflow-catalog/task-parse-snapshot:0.1 + value: quay.io/rh-ee-ikrispin/abevalflow-task-parse-snapshot:0.1 - name: kind value: task params: @@ -46,7 +46,7 @@ spec: - name: name value: deploy-agent - name: bundle - value: quay.io/rh-ee-ikrispin/abevalflow-catalog/task-deploy-agent:0.1 + value: quay.io/rh-ee-ikrispin/abevalflow-task-deploy-agent:0.1 - name: kind value: task params: @@ -70,7 +70,7 @@ spec: - name: name value: prepare - name: bundle - value: quay.io/rh-ee-ikrispin/abevalflow-catalog/task-prepare:0.1 + value: quay.io/rh-ee-ikrispin/abevalflow-task-prepare:0.1 - name: kind value: task params: @@ -105,7 +105,7 @@ spec: - name: name value: test - name: bundle - value: quay.io/rh-ee-ikrispin/abevalflow-catalog/task-test:0.1 + value: quay.io/rh-ee-ikrispin/abevalflow-task-test:0.1 - name: kind value: task params: @@ -149,7 +149,7 @@ spec: - name: name value: evaluate - name: bundle - value: quay.io/rh-ee-ikrispin/abevalflow-catalog/task-evaluate:0.1 + value: quay.io/rh-ee-ikrispin/abevalflow-task-evaluate:0.1 - name: kind value: task params: @@ -190,7 +190,7 @@ spec: - name: name value: analyze-scorecard - name: bundle - value: quay.io/rh-ee-ikrispin/abevalflow-catalog/task-analyze-scorecard:0.1 + value: quay.io/rh-ee-ikrispin/abevalflow-task-analyze-scorecard:0.1 - name: kind value: task params: @@ -227,7 +227,7 @@ spec: - name: name value: store - name: bundle - value: quay.io/rh-ee-ikrispin/abevalflow-catalog/task-store:0.1 + value: quay.io/rh-ee-ikrispin/abevalflow-task-store:0.1 - name: kind value: task params: @@ -262,7 +262,7 @@ spec: - name: name value: emit-result - name: bundle - value: quay.io/rh-ee-ikrispin/abevalflow-catalog/task-emit-result:0.1 + value: quay.io/rh-ee-ikrispin/abevalflow-task-emit-result:0.1 - name: kind value: task params: @@ -283,7 +283,7 @@ spec: - name: name value: cleanup-agent - name: bundle - value: quay.io/rh-ee-ikrispin/abevalflow-catalog/task-cleanup-agent:0.1 + value: quay.io/rh-ee-ikrispin/abevalflow-task-cleanup-agent:0.1 - name: kind value: task params: From 15d824abeb58c91873d0c942f6ab5d1de9b268d6 Mon Sep 17 00:00:00 2001 From: ikrispin Date: Tue, 21 Jul 2026 16:59:18 +0300 Subject: [PATCH 4/5] Temporarily hardcode public agent image for deploy-agent Use quay.io/ecosystem-appeng/google-lightspeed-agent until image pull auth for redhat-user-workloads is set up on the workload cluster. --- pipeline/integration/konflux-eval-pipelinerun.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline/integration/konflux-eval-pipelinerun.yaml b/pipeline/integration/konflux-eval-pipelinerun.yaml index fc7d5ff..ef385c6 100644 --- a/pipeline/integration/konflux-eval-pipelinerun.yaml +++ b/pipeline/integration/konflux-eval-pipelinerun.yaml @@ -51,7 +51,7 @@ spec: value: task params: - name: agent-image - value: $(tasks.parse-snapshot.results.component-image) + value: quay.io/ecosystem-appeng/google-lightspeed-agent:on-pr-4af1ac0fab1c823cf2d034961c902a90c54ba4f4 - name: llm-api-base value: "http://litellm.ab-eval-flow.svc:4000" - name: llm-model From 0478617aa285c2fe617ce4314ea4a61562d1801c Mon Sep 17 00:00:00 2001 From: ikrispin Date: Tue, 21 Jul 2026 18:08:58 +0300 Subject: [PATCH 5/5] Fix mcpchecker-eval step image pull failure Tekton pulls all step images upfront even for skipped steps. Replace private mcpchecker-runner image with UBI Python and install mcpchecker via pip only when the engine is mcpchecker. --- pipeline/tasks/konflux/evaluate.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pipeline/tasks/konflux/evaluate.yaml b/pipeline/tasks/konflux/evaluate.yaml index 0fe416d..0ac44d5 100644 --- a/pipeline/tasks/konflux/evaluate.yaml +++ b/pipeline/tasks/konflux/evaluate.yaml @@ -561,7 +561,7 @@ spec: # Step 8: MCPChecker - Run Evaluation - name: mcpchecker-eval - image: quay.io/rhecosystemappeng/mcpchecker-runner:latest + image: registry.access.redhat.com/ubi9/python-311:9.6 env: - name: MCP_URL valueFrom: @@ -594,6 +594,8 @@ spec: echo "=== EVALUATE PHASE: MCPChecker ===" + pip install --quiet --no-cache-dir mcpchecker 2>&1 | tail -3 + if [ -z "${MCP_URL:-}" ]; then echo "ERROR: MCP_URL not set" exit 1