diff --git a/.github/workflows/push-bundles.yaml b/.github/workflows/push-bundles.yaml new file mode 100644 index 0000000..9832fb8 --- /dev/null +++ b/.github/workflows/push-bundles.yaml @@ -0,0 +1,50 @@ +name: Push Tekton Bundles + +on: + push: + branches: [main] + paths: + - 'pipeline/tasks/konflux/**' + - 'pipeline/integration/Makefile' + workflow_dispatch: + inputs: + version: + description: 'Bundle version tag' + required: false + default: '0.1' + +env: + QUAY_REPO: quay.io/rh-ee-ikrispin/abevalflow-catalog + VERSION: ${{ github.event.inputs.version || '0.1' }} + +jobs: + push-bundles: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install tkn CLI + run: | + TKN_VERSION="0.37.0" + curl -sLO "https://github.com/tektoncd/cli/releases/download/v${TKN_VERSION}/tkn_${TKN_VERSION}_Linux_x86_64.tar.gz" + tar xzf "tkn_${TKN_VERSION}_Linux_x86_64.tar.gz" tkn + sudo mv tkn /usr/local/bin/ + tkn version + + - name: Install skopeo + run: sudo apt-get update && sudo apt-get install -y skopeo + + - name: Login to Quay.io + run: | + echo "${{ secrets.QUAY_TOKEN }}" | tkn bundle push --help > /dev/null 2>&1 || true + mkdir -p ~/.docker + echo '{"auths":{"quay.io":{"auth":"'$(echo -n "${{ secrets.QUAY_USERNAME }}:${{ secrets.QUAY_TOKEN }}" | base64 -w0)'"}}}' > ~/.docker/config.json + + - name: Push all bundles + working-directory: pipeline/integration + run: make bundles VERSION=${{ env.VERSION }} + + - name: Print digests + working-directory: pipeline/integration + run: make digests VERSION=${{ env.VERSION }} diff --git a/Docs/konflux-integration-guide.md b/Docs/konflux-integration-guide.md new file mode 100644 index 0000000..fd8d96d --- /dev/null +++ b/Docs/konflux-integration-guide.md @@ -0,0 +1,340 @@ +# ABEvalFlow Konflux Integration Guide + +This guide explains how to integrate ABEvalFlow evaluation into any Konflux +application pipeline. ABEvalFlow provides generic evaluation tasks as Tekton +Bundles that can evaluate A2A agents, MCP servers, and skills. + +## Architecture + +ABEvalFlow publishes **7 core tasks** as Tekton Bundles: + +``` +parse-snapshot → prepare → test → evaluate → analyze-scorecard → store → emit-result +``` + +These tasks handle the entire evaluation lifecycle: + +| Task | Purpose | +|------|---------| +| `parse-snapshot` | Extract component image and git info from a Konflux Snapshot | +| `prepare` | Clone and validate the submission definition | +| `test` | Run security scans and quality review (optional) | +| `evaluate` | Execute the evaluation engine (A2A, MCPChecker, Harbor, ASE) | +| `analyze-scorecard` | Produce a certification scorecard from results | +| `store` | Persist results to PostgreSQL/MinIO (optional) | +| `emit-result` | Map scorecard to Konflux's `TEST_OUTPUT` format | + +Your application adds its own deployment and cleanup logic around these core +tasks. ABEvalFlow never deploys or manages your application. + +## Parameter Contract + +### Pipeline Parameters + +```yaml +# Required by Konflux (provided automatically) +SNAPSHOT: "" + +# What to evaluate +EVAL_ENGINE: "a2a" # a2a | harbor | ase | mcpchecker +SUBMISSION_REPO_URL: "" # Git repo containing the submission definition +SUBMISSION_DIR: "" # Directory name under submissions/ +SUBMISSION_REVISION: "main" # Git ref for the submission repo + +# Target endpoints (provide based on engine) +AGENT_ENDPOINT: "" # Required for a2a: HTTP endpoint of the agent +MCP_URL: "" # Required for mcpchecker: URL of the MCP server + +# LLM infrastructure (for judging) +LLM_API_BASE: "" # LLM proxy URL (e.g. http://litellm.ns.svc:4000) +LLM_MODEL: "claude-sonnet" # Model name for LLM-as-judge + +# Execution mode +EVAL_MODE: "local" # "local" or "remote" +WORKLOAD_CLUSTER_URL: "" # Required when EVAL_MODE=remote +WORKLOAD_NAMESPACE: "" # Required when EVAL_MODE=remote +WORKLOAD_CREDENTIALS_SECRET: "workload-cluster-credentials" # Secret name + +# Pipeline repo (for evaluation scripts) +PIPELINE_REPO_URL: "https://github.com/RHEcosystemAppEng/ABEvalFlow.git" +PIPELINE_REPO_REVISION: "main" +``` + +### When to Use Each Parameter + +| Eval Engine | Required Parameters | +|-------------|-------------------| +| `a2a` | `AGENT_ENDPOINT`, `SUBMISSION_*`, `LLM_*` | +| `mcpchecker` | `MCP_URL`, `SUBMISSION_*`, `LLM_*` | +| `harbor` | `SUBMISSION_*`, `LLM_*` | +| `ase` | `SUBMISSION_*`, `LLM_*` | + +### Engine x Mode Validation Matrix + +| Engine | Local | Remote | +|--------|-------|--------| +| `a2a` | Supported | **Fully tested** (E2E on Konflux) | +| `mcpchecker` | Supported | Supported (untested) | +| `ase` | Supported (no external endpoint needed) | Not yet implemented | +| `harbor` | Limited (local environment only, no scaffold/build) | Not supported (use standalone pipeline) | + +For `harbor` in Konflux, the local mode runs with `environment.type: local` which +does not perform the full scaffold/build/eval cycle. For full Harbor A/B testing +with container registry support, use the standalone ABEvalFlow pipeline on OpenShift. + +### Multi-component Applications + +The `parse-snapshot` task defaults to `.components[0]` from the Snapshot. For +applications with multiple components, set the `component-name` parameter in +parse-snapshot to target the specific component you want to evaluate. Failing +to do so may result in evaluating the wrong component image. + +## Evaluation Modes + +### Local Mode (`EVAL_MODE=local`) + +The evaluation runs directly inside the Tekton task step on the pipeline +cluster. Use this when: + +- The target (agent/MCP server) is reachable from the pipeline cluster +- The target has a public Route or Ingress +- You're evaluating skills (Harbor/ASE) that don't need an external endpoint + +No workload cluster credentials are needed in this mode. + +### Remote Mode (`EVAL_MODE=remote`) + +The evaluation runs as a Pod on a separate workload cluster. Use this when: + +- The pipeline cluster (Konflux) can't reach the target's cluster-internal Services +- The target is deployed on a different cluster from where the pipeline runs +- You need the eval Pod co-located with the target for network access + +Required in this mode: +- `WORKLOAD_CLUSTER_URL` — API URL of the workload cluster +- `WORKLOAD_NAMESPACE` — Namespace to create the eval Pod in +- A Secret (named by `WORKLOAD_CREDENTIALS_SECRET`) with a `token` key containing + a ServiceAccount token for the workload cluster + +## Submissions + +A **submission** is the evaluation definition package. It tells ABEvalFlow what +to test and how to judge results. Structure depends on the eval engine: + +### A2A Agent Submission + +``` +submission/ + metadata.yaml # name, eval_engine: a2a, experiment config + tasks/ + / + task.toml # Harbor task configuration + instruction.md # Multi-turn conversation instructions + tests/test.sh # Verifier entry point + tests/llm_judge.py # LLM-as-judge scorer + environment/Dockerfile +``` + +### MCP Server Submission + +``` +submission/ + metadata.yaml # name, eval_engine: mcpchecker + eval.yaml # MCPChecker evaluation config + mcp-config.yaml # MCP server connection config (can use $MCP_URL) +``` + +### Skill Submission (ASE) + +``` +submission/ + metadata.yaml # name, eval_engine: ase + skills/ + / + SKILL.md # Skill definition + evals/evals.json # Evaluation scenarios +``` + +### Skill Submission (Harbor) + +``` +submission/ + metadata.yaml # name, eval_engine: harbor + tasks/ + / + task.toml + instruction.md + tests/test.sh + environment/Dockerfile +``` + +### metadata.yaml Reference + +```yaml +name: my-evaluation # Unique evaluation name +description: What this evaluates +version: "1.0.0" +eval_engine: a2a # a2a | harbor | ase | mcpchecker + +experiment: + n_trials: 5 # Number of evaluation trials + +security_scan: disabled # disabled | warn | block +skip_quality_review: true # Skip LLM quality review + +gate_policy: # Optional certification gates + default_mode: warn + combination: all_pass + gates: + evaluation: + mode: block + threshold: 0.0 +``` + +Submissions can live in any Git repository. The pipeline accepts +`SUBMISSION_REPO_URL` and `SUBMISSION_DIR` to locate them. + +## Integration Patterns + +### Pattern 1: Pre-deployed Target (simplest) + +If your agent or MCP server is already running (e.g., a long-lived service), +use ABEvalFlow's reference pipeline directly: + +```yaml +apiVersion: appstudio.redhat.com/v1beta2 +kind: IntegrationTestScenario +metadata: + name: abevalflow-eval + namespace: + labels: + test.appstudio.openshift.io/optional: "true" +spec: + application: + contexts: + - description: AI evaluation via ABEvalFlow + name: application + resolverRef: + resolver: git + resourceKind: pipelinerun + params: + - name: url + value: https://github.com/RHEcosystemAppEng/ABEvalFlow + - name: revision + value: main + - name: pathInRepo + value: pipeline/integration/konflux-eval-pipelinerun.yaml + params: + - name: EVAL_ENGINE + value: "a2a" + - name: AGENT_ENDPOINT + value: "http://my-agent.my-namespace.svc:8000" + - name: SUBMISSION_REPO_URL + value: "https://github.com/myorg/my-submissions.git" + - name: SUBMISSION_DIR + value: "my-agent-eval" + - name: LLM_API_BASE + value: "http://litellm.my-namespace.svc:4000" +``` + +### Pattern 2: Pipeline-deployed Target + +If your target needs to be deployed for each evaluation run, create your own +pipeline that wraps ABEvalFlow's core tasks with deploy/cleanup steps. + +See the [full working example](https://github.com/ikrispin/abevalflow-konflux-example) +for the Google Lightspeed Agent. + +Key steps: +1. Create a `deploy-.yaml` task that deploys your application and outputs + the endpoint URL as a task result +2. Create a `cleanup-.yaml` task for the `finally:` block +3. Create a PipelineRun that chains: deploy → ABEvalFlow core tasks → cleanup +4. Create a submission definition for your evaluation scenarios +5. Create an IntegrationTestScenario pointing to your pipeline + +### Pattern 3: MCP Server Evaluation + +```yaml +# In your IntegrationTestScenario params: +- name: EVAL_ENGINE + value: "mcpchecker" +- name: MCP_URL + value: "http://my-mcp-server.my-namespace.svc:3000" +- name: SUBMISSION_REPO_URL + value: "https://github.com/myorg/my-submissions.git" +- name: SUBMISSION_DIR + value: "my-mcp-server-eval" +``` + +## Secrets + +### Required Secrets by Mode + +| Secret | When Required | +|--------|--------------| +| `workload-cluster-credentials` | `EVAL_MODE=remote` only | +| `llm-credentials` | When LLM proxy needs a real API key | + +### Optional Secrets + +| Secret | Purpose | +|--------|---------| +| `compass-facts-api` | Push scorecard facts to Red Hat Compass | +| `ab-eval-db-credentials` | Store results in PostgreSQL | +| `minio-credentials` | Upload artifacts to MinIO/S3 | +| `monitoring-slack-webhook` | Send degradation alerts to Slack | + +### Creating Workload Cluster Credentials + +On the workload cluster: +```bash +oc create sa abevalflow-deployer -n +oc adm policy add-role-to-user edit -z abevalflow-deployer -n +oc create token abevalflow-deployer -n --duration=8760h +``` + +Store the token in your Konflux tenant namespace: +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: workload-cluster-credentials + namespace: +type: Opaque +stringData: + token: "" +``` + +See `config/konflux/secrets-template.yaml` for the full template. + +## Tekton Bundles + +The core tasks are published as Tekton Bundles to Quay.io: + +| Bundle | Task | +|--------|------| +| `quay.io/rh-ee-ikrispin/abevalflow-task-parse-snapshot:0.1` | parse-snapshot | +| `quay.io/rh-ee-ikrispin/abevalflow-task-prepare:0.1` | prepare | +| `quay.io/rh-ee-ikrispin/abevalflow-task-test:0.1` | test | +| `quay.io/rh-ee-ikrispin/abevalflow-task-evaluate:0.1` | evaluate | +| `quay.io/rh-ee-ikrispin/abevalflow-task-analyze-scorecard:0.1` | analyze-scorecard | +| `quay.io/rh-ee-ikrispin/abevalflow-task-store:0.1` | store | +| `quay.io/rh-ee-ikrispin/abevalflow-task-emit-result:0.1` | emit-result | + +To rebuild bundles after editing task YAML: +```bash +cd pipeline/integration +make bundles +``` + +## Quick Start + +1. **Choose your pattern** from the Integration Patterns section above +2. **Create a submission** defining your evaluation scenarios +3. **Provision secrets** in your Konflux tenant namespace +4. **Create an IntegrationTestScenario** in your tenant namespace +5. **Push a change** to your application — Konflux triggers the evaluation + +For a complete working example, see: +[github.com/ikrispin/abevalflow-konflux-example](https://github.com/ikrispin/abevalflow-konflux-example) diff --git a/config/konflux/secrets-template.yaml b/config/konflux/secrets-template.yaml new file mode 100644 index 0000000..5b49131 --- /dev/null +++ b/config/konflux/secrets-template.yaml @@ -0,0 +1,55 @@ +# ABEvalFlow Konflux Secrets Template +# +# Apply these secrets to your Konflux tenant namespace. Not all secrets are +# required -- it depends on which eval-mode and features you use. +# +# Required secrets by mode: +# EVAL_MODE=local : llm-credentials (if LLM proxy requires a real API key) +# EVAL_MODE=remote : llm-credentials + workload-cluster-credentials +# +# Optional secrets (for additional features): +# compass-facts-api : Push scorecard facts to Red Hat Compass +# ab-eval-db-credentials : Store results in PostgreSQL +# minio-credentials : Upload artifacts to MinIO/S3 +# monitoring-slack-webhook: Send degradation alerts to Slack +--- +# workload-cluster-credentials +# ONLY required when EVAL_MODE=remote (cross-cluster evaluation). +# Contains a ServiceAccount token from the workload cluster. +# +# To create the SA and token on the workload cluster: +# oc create sa abevalflow-deployer -n +# oc adm policy add-role-to-user edit -z abevalflow-deployer -n +# oc create token abevalflow-deployer -n --duration=8760h +apiVersion: v1 +kind: Secret +metadata: + name: workload-cluster-credentials + namespace: +type: Opaque +stringData: + token: "" +--- +# llm-credentials +# API key for the LLM proxy. When using LiteLLM proxy (which handles real +# auth to providers like Vertex AI), set this to "sk-dummy". +apiVersion: v1 +kind: Secret +metadata: + name: llm-credentials + namespace: +type: Opaque +stringData: + api-key: "" +--- +# compass-facts-api (OPTIONAL) +# Bearer token for the Red Hat Compass Soundcheck Facts API. +# If not configured, scorecard is computed but facts are not pushed. +apiVersion: v1 +kind: Secret +metadata: + name: compass-facts-api + namespace: +type: Opaque +stringData: + token: "" diff --git a/pipeline/integration/Makefile b/pipeline/integration/Makefile new file mode 100644 index 0000000..75b7997 --- /dev/null +++ b/pipeline/integration/Makefile @@ -0,0 +1,36 @@ +QUAY_NS ?= quay.io/rh-ee-ikrispin +VERSION ?= 0.1 +TASKS_DIR := ../tasks/konflux + +TASKS = parse-snapshot prepare test evaluate analyze-scorecard store emit-result + +.PHONY: bundles login list clean help + +help: ## Show this help + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' + +bundles: $(addprefix bundle-,$(TASKS)) ## Build and push all Tekton Bundles + +bundle-%: ## Push a single task bundle (e.g. make bundle-parse-snapshot) + @echo "=== Pushing abevalflow-task-$* ===" + tkn bundle push $(QUAY_NS)/abevalflow-task-$*:$(VERSION) \ + -f $(TASKS_DIR)/$*.yaml + @echo "" + +list: ## List all bundles in the registry + @for task in $(TASKS); do \ + echo "--- abevalflow-task-$$task ---"; \ + tkn bundle list $(QUAY_NS)/abevalflow-task-$$task 2>/dev/null || echo " (not found)"; \ + done + +digests: ## Print SHA digests for all pushed bundles + @for task in $(TASKS); do \ + DIGEST=$$(skopeo inspect --format '{{.Digest}}' docker://$(QUAY_NS)/abevalflow-task-$$task:$(VERSION) 2>/dev/null || echo "NOT_FOUND"); \ + echo "$(QUAY_NS)/abevalflow-task-$$task:$(VERSION)@$$DIGEST"; \ + done + +clean: ## Remove local tkn bundle cache + rm -rf ~/.cache/tekton/bundles/ + +login: ## Login to Quay.io + podman login quay.io diff --git a/pipeline/integration/konflux-eval-pipelinerun.yaml b/pipeline/integration/konflux-eval-pipelinerun.yaml new file mode 100644 index 0000000..29624c5 --- /dev/null +++ b/pipeline/integration/konflux-eval-pipelinerun.yaml @@ -0,0 +1,377 @@ +# ABEvalFlow Generic Evaluation Pipeline for Konflux +# +# This is a REFERENCE pipeline that any Konflux application can use for +# AI evaluation (agents, MCP servers, skills). It provides 7 core stages: +# parse-snapshot → prepare → test → evaluate → analyze → store → emit-result +# +# USAGE: +# 1. For pre-deployed targets (agents, MCP servers): use this pipeline directly +# via an IntegrationTestScenario and pass the target endpoint as a parameter. +# 2. For targets that need deployment: create your own pipeline that wraps this +# one, adding deploy/cleanup tasks specific to your application. +# See https://github.com/ikrispin/abevalflow-konflux-example for a full example. +# +# NOTE: Dual-publish requirement +# This PipelineRun is resolved from git (via IntegrationTestScenario), but +# the tasks it references are published as Tekton Bundles to Quay.io. +# When updating task logic: +# 1. Edit task YAML in pipeline/tasks/konflux/ +# 2. Push bundles: cd pipeline/integration && make bundles +# 3. Commit and push to git +# Pushing to git alone does NOT update the task logic inside bundles. +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + name: abevalflow-eval +spec: + timeouts: + pipeline: "4h" + tasks: "3h" + pipelineSpec: + params: + # === Konflux integration === + - name: SNAPSHOT + type: string + description: Konflux Snapshot JSON with component details (provided automatically) + + # === Evaluation target === + - name: EVAL_ENGINE + type: string + default: "a2a" + description: "Evaluation engine: harbor, ase, mcpchecker, a2a" + - name: SUBMISSION_REPO_URL + type: string + default: "https://github.com/RHEcosystemAppEng/ABEvalFlow.git" + description: Git repo containing the submission definition + - name: SUBMISSION_DIR + type: string + default: "" + description: Submission directory name under submissions/ + - name: SUBMISSION_REVISION + type: string + default: "main" + description: Git branch/tag/SHA of the submission repo + + # === LLM credentials === + - name: LLM_API_KEY + type: string + default: "sk-dummy" + description: >- + LLM API key. When using LiteLLM proxy (which handles real auth), + keep as "sk-dummy". For direct LLM API calls, set the real key + or provision llm-credentials Secret (preferred). + + # === Target endpoints (provide based on engine) === + - name: AGENT_ENDPOINT + type: string + default: "" + description: "For a2a engine: HTTP endpoint of the deployed agent" + - name: MCP_URL + type: string + default: "" + description: "For mcpchecker engine: URL of the MCP server" + + # === LLM infrastructure === + - name: LLM_API_BASE + type: string + default: "" + description: LLM proxy base URL for judging (e.g. http://litellm.ns.svc:4000) + - name: LLM_MODEL + type: string + default: "claude-sonnet" + description: Model for LLM-as-judge + + # === Execution mode === + - name: EVAL_MODE + type: string + default: "local" + description: >- + "local" runs eval directly in the task step (target must be reachable + from the pipeline cluster). "remote" submits an eval Pod to the + workload cluster. + - name: WORKLOAD_CLUSTER_URL + type: string + default: "" + description: API URL of the workload cluster (required when EVAL_MODE=remote) + - name: WORKLOAD_NAMESPACE + type: string + default: "" + description: Namespace on the workload cluster (required when EVAL_MODE=remote) + - name: WORKLOAD_CREDENTIALS_SECRET + type: string + default: "workload-cluster-credentials" + description: >- + Name of the Secret containing 'token' key for the workload cluster. + Only used when EVAL_MODE=remote. + + # === Pipeline repo (for scripts) === + - name: PIPELINE_REPO_URL + type: string + default: "https://github.com/RHEcosystemAppEng/ABEvalFlow.git" + description: URL of the ABEvalFlow pipeline repo containing evaluation scripts + - name: PIPELINE_REPO_REVISION + type: string + default: "main" + description: Branch or SHA of the pipeline repo + + workspaces: + - name: shared-workspace + results: + - name: TEST_OUTPUT + description: Standardized Konflux test output + value: $(tasks.emit-result.results.TEST_OUTPUT) + + tasks: + # ================================================================ + # Stage 1: Parse the Konflux Snapshot + # ================================================================ + - name: parse-snapshot + taskRef: + resolver: bundles + params: + - name: name + value: parse-snapshot + - name: bundle + value: quay.io/rh-ee-ikrispin/abevalflow-task-parse-snapshot:0.1 + - name: kind + value: task + params: + - name: SNAPSHOT + value: $(params.SNAPSHOT) + + # ================================================================ + # Stage 2: Prepare (clone + validate submission) + # ================================================================ + - name: prepare + runAfter: [parse-snapshot] + taskRef: + resolver: bundles + params: + - name: name + value: prepare + - name: bundle + value: quay.io/rh-ee-ikrispin/abevalflow-task-prepare:0.1 + - name: kind + value: task + params: + - name: repo-url + value: $(params.SUBMISSION_REPO_URL) + - name: revision + value: $(params.SUBMISSION_REVISION) + - name: submission-dir + value: $(params.SUBMISSION_DIR) + - name: eval-engine + value: $(params.EVAL_ENGINE) + - name: pipeline-repo-url + value: $(params.PIPELINE_REPO_URL) + - name: pipeline-repo-revision + value: $(params.PIPELINE_REPO_REVISION) + - name: pipeline-run-name + value: $(context.pipelineRun.name) + - name: enable-generation + value: "false" + workspaces: + - name: source + workspace: shared-workspace + + # ================================================================ + # Stage 3: Test (security scan + quality review) + # ================================================================ + - name: test + runAfter: [prepare] + taskRef: + resolver: bundles + params: + - name: name + value: test + - name: bundle + value: quay.io/rh-ee-ikrispin/abevalflow-task-test:0.1 + - name: kind + value: task + params: + - name: submission-dir + value: $(params.SUBMISSION_DIR) + - name: submission-name + value: $(tasks.prepare.results.submission-name) + - name: eval-engine + value: $(params.EVAL_ENGINE) + - name: report-prefix + value: $(tasks.prepare.results.report-prefix) + - name: pipeline-run-name + value: $(context.pipelineRun.name) + - name: pipeline-repo-url + value: $(params.PIPELINE_REPO_URL) + - name: pipeline-repo-revision + value: $(params.PIPELINE_REPO_REVISION) + # Security and quality are disabled in this reference pipeline. + # For a stricter gate, set security-scan-mode to "warn" or "block" + # and enable-quality-review to "true". + - name: security-scan-mode + value: disabled + - name: submission-security-scan + value: $(tasks.prepare.results.security-scan) + - name: submission-skip-quality-review + value: $(tasks.prepare.results.skip-quality-review) + - name: enable-quality-review + value: "false" + - name: llm-base-url + value: $(params.LLM_API_BASE) + workspaces: + - name: source + workspace: shared-workspace + + # ================================================================ + # Stage 4: Evaluate + # ================================================================ + - name: evaluate + runAfter: [test] + timeout: "3h" + taskRef: + resolver: bundles + params: + - name: name + value: evaluate + - name: bundle + value: quay.io/rh-ee-ikrispin/abevalflow-task-evaluate:0.1 + - name: kind + value: task + params: + - name: eval-engine + value: $(params.EVAL_ENGINE) + - name: submission-dir + value: $(params.SUBMISSION_DIR) + - name: submission-name + value: $(tasks.prepare.results.submission-name) + - name: commit-sha + value: $(tasks.parse-snapshot.results.git-revision) + - name: pipeline-run-id + value: $(context.pipelineRun.name) + - name: pipeline-repo-url + value: $(params.PIPELINE_REPO_URL) + - name: pipeline-repo-revision + value: $(params.PIPELINE_REPO_REVISION) + - name: llm-model + value: $(params.LLM_MODEL) + - name: llm-api-base + value: $(params.LLM_API_BASE) + - name: agent-endpoint + value: $(params.AGENT_ENDPOINT) + - name: mcp-url + value: $(params.MCP_URL) + - name: submission-repo-url + value: $(params.SUBMISSION_REPO_URL) + - name: submission-repo-revision + value: $(params.SUBMISSION_REVISION) + - name: eval-mode + value: $(params.EVAL_MODE) + - name: workload-cluster-url + value: $(params.WORKLOAD_CLUSTER_URL) + - name: workload-namespace + value: $(params.WORKLOAD_NAMESPACE) + - name: workload-credentials-secret + value: $(params.WORKLOAD_CREDENTIALS_SECRET) + workspaces: + - name: source + workspace: shared-workspace + + # ================================================================ + # Stage 5: Analyze + Scorecard + # ================================================================ + - name: analyze-scorecard + runAfter: [evaluate] + taskRef: + resolver: bundles + params: + - name: name + value: analyze-scorecard + - name: bundle + value: quay.io/rh-ee-ikrispin/abevalflow-task-analyze-scorecard:0.1 + - name: kind + value: task + params: + - name: submission-name + value: $(tasks.prepare.results.submission-name) + - name: eval-engine + value: $(params.EVAL_ENGINE) + - name: commit-sha + value: $(tasks.parse-snapshot.results.git-revision) + - name: pipeline-run-id + value: $(context.pipelineRun.name) + - name: uplift-threshold + value: "0.0" + - name: pipeline-repo-url + value: $(params.PIPELINE_REPO_URL) + - name: pipeline-repo-revision + value: $(params.PIPELINE_REPO_REVISION) + - name: enable-scorecard + value: "true" + - name: enable-degradation-check + value: "false" + workspaces: + - name: source + workspace: shared-workspace + + # ================================================================ + # Stage 6: Store (optional -- graceful when secrets missing) + # ================================================================ + - name: store + runAfter: [analyze-scorecard] + taskRef: + resolver: bundles + params: + - name: name + value: store + - name: bundle + value: quay.io/rh-ee-ikrispin/abevalflow-task-store:0.1 + - name: kind + value: task + params: + - name: submission-name + value: $(tasks.prepare.results.submission-name) + - name: pipeline-run-id + value: $(context.pipelineRun.name) + - name: report-prefix + value: $(tasks.prepare.results.report-prefix) + - name: recommendation + value: $(tasks.evaluate.results.recommendation) + - name: eval-engine + value: $(params.EVAL_ENGINE) + - name: commit-sha + value: $(tasks.parse-snapshot.results.git-revision) + - name: pipeline-repo-url + value: $(params.PIPELINE_REPO_URL) + - name: pipeline-repo-revision + value: $(params.PIPELINE_REPO_REVISION) + workspaces: + - name: source + workspace: shared-workspace + + # ================================================================ + # Stage 7: Emit Konflux TEST_OUTPUT + # ================================================================ + - name: emit-result + runAfter: [store] + taskRef: + resolver: bundles + params: + - name: name + value: emit-result + - name: bundle + value: quay.io/rh-ee-ikrispin/abevalflow-task-emit-result:0.1 + - name: kind + value: task + params: + - name: submission-name + value: $(tasks.prepare.results.submission-name) + workspaces: + - name: source + workspace: shared-workspace + + workspaces: + - name: shared-workspace + volumeClaimTemplate: + spec: + accessModes: [ReadWriteOnce] + resources: + requests: + storage: 1Gi diff --git a/pipeline/tasks/konflux/analyze-scorecard.yaml b/pipeline/tasks/konflux/analyze-scorecard.yaml new file mode 100644 index 0000000..5698072 --- /dev/null +++ b/pipeline/tasks/konflux/analyze-scorecard.yaml @@ -0,0 +1,407 @@ +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: analyze-scorecard + labels: + app.kubernetes.io/name: abevalflow + app.kubernetes.io/component: konflux +spec: + description: >- + Analyzes A/B evaluation results and optionally checks for performance + degradation against historical runs. Step 1 always runs analyze.py to + produce report.json and Tekton results. Step 2 runs aggregate scorecard. + Step 3 runs monitor.py and sends Slack alerts when degradation checking + is enabled and the DB is configured; failures in step 3 are non-blocking. + Adapted for Konflux — no hardcoded namespace, optional secrets, scorecard + enabled by default. + params: + - name: submission-name + type: string + description: Validated submission name + - name: eval-engine + type: string + default: "harbor" + description: >- + Evaluation engine used: 'harbor', 'ase', 'a2a', or 'both'. Controls + which analysis path runs. + - name: commit-sha + type: string + default: "" + description: Git commit SHA for provenance + - name: pipeline-run-id + type: string + default: "" + description: Tekton PipelineRun name for provenance and alerts + - name: harbor-fork-revision + type: string + default: "" + description: Harbor fork revision used for provenance + - name: uplift-threshold + type: string + default: "0.0" + description: >- + Minimum mean-reward gap (treatment - control) for a 'pass' + recommendation. Set to 0.0 to pass whenever treatment >= control. + - name: pipeline-repo-url + type: string + default: "https://github.com/RHEcosystemAppEng/ABEvalFlow.git" + description: URL of the ABEvalFlow pipeline repository + - name: pipeline-repo-revision + type: string + default: "main" + description: Branch or SHA of the pipeline repo to use for scripts + - name: security-scan-mode + type: string + default: "disabled" + description: Security scan mode (disabled/warn/block) for including scan results + - name: llm-model + type: string + default: "" + description: LLM model name for report metadata + - name: repo-name + type: string + default: "" + description: GitHub repo (org/name) for constructing PR URL + - name: pr-number + type: string + default: "" + description: PR number that triggered this pipeline run + - name: enable-degradation-check + type: string + default: "false" + description: >- + When 'true', run historical degradation check after analysis (monitoring + pipelines). CI pipelines leave this false. + - name: degradation-threshold + type: string + default: "0.85" + description: >- + Degradation threshold as a ratio. Alert if current/previous < threshold. + - name: openshift-console-url + type: string + default: "" + description: Base URL for OpenShift console (for Slack message links) + - name: enable-scorecard + type: string + default: "true" + description: >- + When 'true', aggregate all gates into a unified scorecard after analysis. + Produces scorecard.json with combined engine, security, and quality gates. + - name: certification-profile + type: string + default: "" + description: >- + Certification profile name (e.g., 'skill', 'agent', 'mcp_server', 'plugin'). + Provides artifact-type-specific check defaults. If empty, uses hardcoded defaults + unless submission's metadata.yaml specifies certification_policy. + workspaces: + - name: source + description: Workspace containing the evaluation results and report output + results: + - name: recommendation + description: "'pass' or 'fail' based on uplift threshold" + - name: treatment-mean-reward + description: Treatment variant mean reward as a decimal string (e.g. "0.8500") + - name: control-mean-reward + description: Control variant mean reward as a decimal string (e.g. "0.6000") + - name: mean-reward-gap + description: "Mean reward gap (treatment - control) as a signed string (e.g. \"+0.1500\")" + - name: ttest-p-value + description: "Welch's t-test p-value (e.g. \"0.0342\"), or \"N/A\" if not computable" + - name: fisher-p-value + description: "Fisher's exact test p-value (e.g. \"0.0271\"), or \"N/A\" if not computable" + - name: report-path + description: Path to the directory containing report.json and report.md + - name: degraded + description: Whether degradation was detected (true or false) + - name: message + description: Human-readable degradation check result message + - name: scorecard-recommendation + description: Unified scorecard recommendation (pass/warn/fail) + - name: scorecard-gates-passed + description: Number of gates that passed + - name: scorecard-gates-failed + description: Number of gates that failed + steps: + - name: analyze + image: registry.access.redhat.com/ubi9/python-311:9.6 + script: | + #!/usr/bin/env bash + set -euo pipefail + + # --- 1. Clone pipeline repo (or update to requested revision) --- + PIPELINE_DIR="$(workspaces.source.path)/_pipeline" + if [ -d "$PIPELINE_DIR" ]; then + echo "Pipeline repo exists, updating to requested revision" + git -C "$PIPELINE_DIR" fetch origin "$(params.pipeline-repo-revision)" --depth 1 + git -C "$PIPELINE_DIR" -c advice.detachedHead=false checkout FETCH_HEAD + else + git clone --depth 1 --branch "$(params.pipeline-repo-revision)" \ + "$(params.pipeline-repo-url)" "$PIPELINE_DIR" + fi + + cd "$PIPELINE_DIR" + export PYTHONPATH="$PIPELINE_DIR" + pip install --quiet --no-cache-dir pydantic scipy pyyaml + + # --- 2. Use pre-computed report or run analysis --- + REPORT_DIR="$(workspaces.source.path)/reports/$(params.submission-name)" + RESULTS_DIR="$(workspaces.source.path)/eval-results/$(params.submission-name)" + EVAL_ENGINE="$(params.eval-engine)" + mkdir -p "$REPORT_DIR" + + if [ -f "$REPORT_DIR/report.json" ]; then + echo "=== report.json already exists (computed by remote eval Pod) ===" + python3 -c " + import json, sys + r = json.load(open(sys.argv[1])) + s = r.get('summary', {}) + t = s.get('treatment', {}) + print(f\"Recommendation: {s.get('recommendation', 'N/A')}\") + print(f\"Mean reward: {t.get('mean_reward', 'N/A')}\") + print(f\"Trials: {t.get('n_trials', 0)}\") + print(f\"Pass rate: {t.get('pass_rate', 'N/A')}\") + " "$REPORT_DIR/report.json" + else + echo "=== No pre-computed report, running analyze.py locally ===" + ARGS=( + --results-dir "$RESULTS_DIR" + --output-dir "$REPORT_DIR" + --submission-name "$(params.submission-name)" + --threshold "$(params.uplift-threshold)" + --eval-engine "$EVAL_ENGINE" + ) + [ -n "$(params.commit-sha)" ] && ARGS+=(--commit-sha "$(params.commit-sha)") + [ -n "$(params.pipeline-run-id)" ] && ARGS+=(--pipeline-run-id "$(params.pipeline-run-id)") + [ -n "$(params.harbor-fork-revision)" ] && ARGS+=(--harbor-fork-revision "$(params.harbor-fork-revision)") + python scripts/analyze.py "${ARGS[@]}" + fi + + # --- 3. Enrich report with PR/LLM metadata (ASE mode) --- + PR_URL="" + if [ -n "$(params.repo-name)" ] && [ -n "$(params.pr-number)" ]; then + PR_URL="https://github.com/$(params.repo-name)/pull/$(params.pr-number)" + fi + LLM_LABEL="" + case "$(params.llm-model)" in + claude-sonnet*) LLM_LABEL="Claude Sonnet 4.6 (vertex_ai)" ;; + claude-haiku*) LLM_LABEL="Claude Haiku 3.5 (vertex_ai)" ;; + ?*) LLM_LABEL="$(params.llm-model)" ;; + esac + + if [ "$EVAL_ENGINE" = "ase" ] && { [ -n "$PR_URL" ] || [ -n "$LLM_LABEL" ]; }; then + echo "Enriching ASE report with PR/LLM metadata..." + python3 -c "import json;from pathlib import Path;import sys;p=Path(sys.argv[1])/'report.json';r=json.loads(p.read_text());r.setdefault('summary',{});sys.argv[2] and r['summary'].update({'related_pr':sys.argv[2]});sys.argv[3] and r['summary'].update({'llm':sys.argv[3]});p.write_text(json.dumps(r,indent=2))" "$REPORT_DIR" "$PR_URL" "$LLM_LABEL" + fi + + # --- 4. Enrich with security scan results (ASE/both only) --- + if [ "$EVAL_ENGINE" != "harbor" ] && [ "$(params.security-scan-mode)" != "disabled" ] && [ -f "$REPORT_DIR/security-scan.json" ]; then + echo "Enriching report with security scan results..." + python3 -c "import json;from pathlib import Path;import sys;d=Path(sys.argv[1]);m=sys.argv[2];r=json.loads((d/'report.json').read_text());s=json.loads((d/'security-scan.json').read_text());f=[{'rule_id':x.get('rule_id','?'),'severity':x.get('severity','info').lower(),'message':x.get('message',''),'scanner':'cisco'} for x in s.get('findings',[])];p=m!='block' or sum(1 for x in f if x['severity'] in('critical','high'))==0;r.setdefault('security_scans',[]).append({'scanner':'cisco','scan_mode':m,'findings':f,'passed':p});(d/'report.json').write_text(json.dumps(r,indent=2));print(f'Added {len(f)} findings')" "$REPORT_DIR" "$(params.security-scan-mode)" + fi + + # --- 5. Regenerate markdown from enriched JSON (ASE/both only) --- + if [ "$EVAL_ENGINE" != "harbor" ] && [ "$EVAL_ENGINE" != "a2a" ]; then + echo "Regenerating report.md from enriched JSON..." + python3 -c "import sys;sys.path.insert(0,'$PIPELINE_DIR');from pathlib import Path;from abevalflow.report import AnalysisResult;from scripts.analyze import render_markdown;d=Path(sys.argv[1]);r=AnalysisResult.model_validate_json((d/'report.json').read_text());(d/'report.md').write_text(render_markdown(r))" "$REPORT_DIR" + fi + + # --- 6. Extract results for Tekton --- + echo -n "$REPORT_DIR" > "$(results.report-path.path)" + + python3 -c "import json,sys;r=json.loads(open(sys.argv[1]).read());s=r['summary'];open(sys.argv[2],'w').write(s['recommendation']);t,c=s['treatment'].get('mean_reward'),s['control'].get('mean_reward');open(sys.argv[3],'w').write(f'{t:.4f}' if t is not None else 'N/A');open(sys.argv[4],'w').write(f'{c:.4f}' if c is not None else 'N/A');g=s.get('mean_reward_gap');open(sys.argv[5],'w').write(f'{g:+.4f}' if g is not None else 'N/A');tt,fi=s.get('ttest_p_value'),s.get('fisher_p_value');open(sys.argv[6],'w').write(f'{tt:.4f}' if tt is not None else 'N/A');open(sys.argv[7],'w').write(f'{fi:.4f}' if fi is not None else 'N/A')" \ + "$REPORT_DIR/report.json" \ + "$(results.recommendation.path)" \ + "$(results.treatment-mean-reward.path)" \ + "$(results.control-mean-reward.path)" \ + "$(results.mean-reward-gap.path)" \ + "$(results.ttest-p-value.path)" \ + "$(results.fisher-p-value.path)" + + echo "=== Analysis complete ===" + echo "Report: $REPORT_DIR" + cat "$REPORT_DIR/report.md" 2>/dev/null || echo "(report.md not available)" + + - name: aggregate-scorecard + image: registry.access.redhat.com/ubi9/python-311:9.6 + env: + - name: COMPASS_API_TOKEN + valueFrom: + secretKeyRef: + name: compass-facts-api + key: token + optional: true + script: | + #!/usr/bin/env bash + set -euo pipefail + + ENABLE_SCORECARD="$(params.enable-scorecard)" + if [ "$ENABLE_SCORECARD" != "true" ]; then + echo "Scorecard aggregation disabled, skipping" + echo -n "skipped" > "$(results.scorecard-recommendation.path)" + echo -n "0" > "$(results.scorecard-gates-passed.path)" + echo -n "0" > "$(results.scorecard-gates-failed.path)" + exit 0 + fi + + echo "=== Aggregating Unified Scorecard ===" + + PIPELINE_DIR="$(workspaces.source.path)/_pipeline" + cd "$PIPELINE_DIR" + export PYTHONPATH="$PIPELINE_DIR" + pip install --quiet --no-cache-dir pydantic pyyaml + + SUBMISSION_DIR="$(workspaces.source.path)/submissions/$(params.submission-name)" + RESULTS_DIR="$(workspaces.source.path)/eval-results/$(params.submission-name)" + REPORT_DIR="$(workspaces.source.path)/reports/$(params.submission-name)" + WORKSPACE_ROOT="$(workspaces.source.path)" + + SCORECARD_ARGS=( + --submission-dir "$SUBMISSION_DIR" + --results-dir "$RESULTS_DIR" + --reports-dir "$REPORT_DIR" + --workspace-root "$WORKSPACE_ROOT" + --eval-engine "$(params.eval-engine)" + --pipeline-run-id "$(params.pipeline-run-id)" + ) + if [ -n "$(params.certification-profile)" ]; then + SCORECARD_ARGS+=(--certification-profile "$(params.certification-profile)") + fi + python scripts/aggregate_scorecard.py "${SCORECARD_ARGS[@]}" + + if [ -f "$REPORT_DIR/scorecard.json" ]; then + echo "" + echo "Scorecard written to $REPORT_DIR/scorecard.json" + python3 -m json.tool "$REPORT_DIR/scorecard.json" 2>/dev/null | head -30 || true + echo "..." + + python3 -c "import json,sys;sc=json.load(open(sys.argv[1]));open(sys.argv[2],'w').write(sc['recommendation']);open(sys.argv[3],'w').write(str(sc['gates_passed']));open(sys.argv[4],'w').write(str(sc['gates_failed']))" \ + "$REPORT_DIR/scorecard.json" \ + "$(results.scorecard-recommendation.path)" \ + "$(results.scorecard-gates-passed.path)" \ + "$(results.scorecard-gates-failed.path)" + else + echo "WARNING: scorecard.json not created" + echo -n "error" > "$(results.scorecard-recommendation.path)" + echo -n "0" > "$(results.scorecard-gates-passed.path)" + echo -n "0" > "$(results.scorecard-gates-failed.path)" + fi + + echo "=== Scorecard aggregation complete ===" + + - name: check-degradation + image: registry.access.redhat.com/ubi9/python-311:9.6 + env: + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: ab-eval-db-credentials + key: database-url + optional: true + - name: SLACK_WEBHOOK_URL + valueFrom: + secretKeyRef: + name: monitoring-slack-webhook + key: webhook-url + optional: true + script: | + #!/usr/bin/env bash + set -euo pipefail + + write_skip_results() { + echo "false" > "$(results.degraded.path)" + echo "$1" > "$(results.message.path)" + } + + if [ "$(params.enable-degradation-check)" != "true" ]; then + echo "Degradation check disabled (enable-degradation-check != true)" + write_skip_results "Degradation check disabled" + exit 0 + fi + + if [ -z "${DATABASE_URL:-}" ]; then + echo "WARNING: DATABASE_URL not configured, skipping degradation check" + write_skip_results "DB not configured" + exit 0 + fi + + PIPELINE_DIR="$(workspaces.source.path)/_pipeline" + if [ ! -d "$PIPELINE_DIR" ]; then + echo "Pipeline repo missing after analyze step, skipping degradation check" + write_skip_results "Pipeline repo unavailable" + exit 0 + fi + + echo "=== Degradation Check ===" + echo "Submission: $(params.submission-name)" + echo "Pipeline Run: $(params.pipeline-run-id)" + echo "Threshold: $(params.degradation-threshold)" + + pip install --quiet --no-cache-dir sqlalchemy "psycopg[binary]" "tenacity>=8.2" scipy pydantic pyyaml + + cd "$PIPELINE_DIR" + export PYTHONPATH="$PIPELINE_DIR" + + MONITOR_OUTPUT=/tmp/monitor_result.json + + REPORT_DIR="$(workspaces.source.path)/reports/$(params.submission-name)" + CURRENT_SCORE=$(python3 -c "import json,sys;r=json.load(open(sys.argv[1]));s=r.get('summary',{});t=s.get('treatment',{}).get('mean_reward');print(t if t is not None else s.get('treatment_pass_rate',0.0))" "$REPORT_DIR/report.json" 2>/dev/null || echo "0.0") + + MONITOR_EXIT=0 + python scripts/monitor.py \ + --submission-name "$(params.submission-name)" \ + --threshold "$(params.degradation-threshold)" \ + --db-url "$DATABASE_URL" \ + --current-score "$CURRENT_SCORE" \ + --eval-engine "$(params.eval-engine)" \ + --run-id "$(params.pipeline-run-id)" \ + --output "$MONITOR_OUTPUT" || MONITOR_EXIT=$? + if [ "$MONITOR_EXIT" -eq 2 ]; then + echo "Monitor errored (non-blocking), continuing" + write_skip_results "Monitor errored" + exit 0 + fi + + cat "$MONITOR_OUTPUT" + + if [ -f "$REPORT_DIR/report.json" ]; then + echo "Merging degradation results into report.json..." + python scripts/analyze.py \ + --merge-degradation-from "$MONITOR_OUTPUT" \ + --report-json "$REPORT_DIR/report.json" + else + echo "WARNING: report.json not found at $REPORT_DIR/report.json, skipping merge" + fi + + DEGRADED=$(python3 -c "import json; print(json.load(open('$MONITOR_OUTPUT'))['degraded'])") + MESSAGE=$(python3 -c "import json; print(json.load(open('$MONITOR_OUTPUT'))['message'])") + + echo "$DEGRADED" > "$(results.degraded.path)" + echo "$MESSAGE" > "$(results.message.path)" + + if [ "$DEGRADED" = "True" ]; then + echo "" + echo "!!! DEGRADATION DETECTED !!!" + echo "" + fi + + if [ -n "${SLACK_WEBHOOK_URL:-}" ] && [ "$SLACK_WEBHOOK_URL" != "https://hooks.slack.com/services/REPLACE/WITH/ACTUAL_WEBHOOK" ]; then + PIPELINE_RUN_URL="$(params.openshift-console-url)/k8s/ns/ab-eval-flow/tekton.dev~v1~PipelineRun/$(params.pipeline-run-id)" + + python scripts/alert.py \ + --payload "$MONITOR_OUTPUT" \ + --webhook-url "$SLACK_WEBHOOK_URL" \ + --pipeline-run-url "$PIPELINE_RUN_URL" \ + --eval-engine "$(params.eval-engine)" || { + echo "Failed to send Slack alert (continuing anyway)" + } + else + echo "Slack webhook not configured, skipping alert" + fi + + if [ "$DEGRADED" != "True" ]; then + echo "No degradation detected" + fi + + echo "" + echo "=== Check complete ===" diff --git a/pipeline/tasks/konflux/emit-result.yaml b/pipeline/tasks/konflux/emit-result.yaml new file mode 100644 index 0000000..89fc2ef --- /dev/null +++ b/pipeline/tasks/konflux/emit-result.yaml @@ -0,0 +1,95 @@ +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: emit-result + labels: + app.kubernetes.io/name: abevalflow + app.kubernetes.io/component: konflux +spec: + description: >- + Maps ABEvalFlow scorecard results to Konflux's standardized TEST_OUTPUT + format. Reads the scorecard.json from the workspace and emits a JSON + result with SUCCESS/WARNING/FAILURE status. + params: + - name: submission-name + type: string + description: Submission name to locate scorecard in workspace + workspaces: + - name: source + description: Workspace containing the evaluation reports + results: + - name: TEST_OUTPUT + description: Standardized Konflux test output in JSON format + steps: + - name: emit + image: registry.access.redhat.com/ubi9/ubi-minimal:latest + script: | + #!/usr/bin/env bash + set -euo pipefail + echo "=== EMIT KONFLUX TEST RESULT ===" + + microdnf install -y jq --nodocs 2>/dev/null || true + + REPORT_DIR="$(workspaces.source.path)/reports/$(params.submission-name)" + SCORECARD="$REPORT_DIR/scorecard.json" + REPORT="$REPORT_DIR/report.json" + + RESULT="ERROR" + NOTE="" + + if [ -f "$SCORECARD" ]; then + SCORECARD_REC=$(jq -r '.recommendation // "fail"' "$SCORECARD") + CERT_LEVEL=$(jq -r '.highest_certification // "none"' "$SCORECARD") + GATES_PASSED=$(jq -r '.gates_passed // 0' "$SCORECARD") + GATES_FAILED=$(jq -r '.gates_failed // 0' "$SCORECARD") + + case "$SCORECARD_REC" in + pass) RESULT="SUCCESS" ;; + warn) RESULT="WARNING" ;; + fail) RESULT="FAILURE" ;; + *) RESULT="ERROR" ;; + esac + + NOTE="ABEvalFlow: recommendation=${SCORECARD_REC}, certification=${CERT_LEVEL}, gates_passed=${GATES_PASSED}, gates_failed=${GATES_FAILED}" + + elif [ -f "$REPORT" ]; then + REPORT_REC=$(jq -r '.summary.recommendation // "fail"' "$REPORT") + + case "$REPORT_REC" in + pass) RESULT="SUCCESS" ;; + fail) RESULT="FAILURE" ;; + *) RESULT="ERROR" ;; + esac + + NOTE="ABEvalFlow: recommendation=${REPORT_REC} (no scorecard)" + + else + RESULT="FAILURE" + NOTE="ABEvalFlow: no scorecard.json or report.json found" + fi + + echo "Result: $RESULT" + echo "Note: $NOTE" + + SUCCESSES=0 + FAILURES=0 + WARNINGS=0 + case "$RESULT" in + SUCCESS) SUCCESSES=1 ;; + FAILURE) FAILURES=1 ;; + WARNING) WARNINGS=1 ;; + ERROR) FAILURES=1 ;; + esac + + TEST_OUTPUT=$(jq -rcn \ + --arg date "$(date -u --iso-8601=seconds)" \ + --arg result "$RESULT" \ + --arg note "$NOTE" \ + --argjson successes "$SUCCESSES" \ + --argjson failures "$FAILURES" \ + --argjson warnings "$WARNINGS" \ + '{result: $result, timestamp: $date, note: $note, successes: $successes, failures: $failures, warnings: $warnings}') + + echo -n "$TEST_OUTPUT" | tee "$(results.TEST_OUTPUT.path)" + echo "" + echo "=== TEST_OUTPUT emitted ===" diff --git a/pipeline/tasks/konflux/evaluate.yaml b/pipeline/tasks/konflux/evaluate.yaml new file mode 100644 index 0000000..6bde9ff --- /dev/null +++ b/pipeline/tasks/konflux/evaluate.yaml @@ -0,0 +1,768 @@ +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: evaluate + labels: + app.kubernetes.io/name: abevalflow + app.kubernetes.io/component: konflux +spec: + description: >- + Generic evaluation task for Konflux integration. Dispatches to the + appropriate evaluation engine (a2a, harbor, ase, mcpchecker) and supports + two execution modes: "local" runs evaluation directly in the task step, + "remote" submits an evaluation Pod to a workload cluster. Consumers + provide the target endpoint (AGENT_ENDPOINT / MCP_URL) as a parameter; + this task does not deploy or manage any target services. + + Validated engine x mode combinations: + a2a + remote: fully tested (E2E on Konflux) + a2a + local: supported (agent must be reachable from pipeline cluster) + mcpchecker + local: supported (MCP server must be reachable) + mcpchecker + remote: supported (untested) + ase + local: supported (no external endpoint needed) + harbor + local: limited (no scaffold/build; uses local environment only) + harbor + remote: not supported (use standalone ABEvalFlow pipeline) + params: + - name: eval-engine + type: string + description: "Evaluation engine: harbor, ase, mcpchecker, a2a" + - name: submission-dir + type: string + description: Submission directory name under submissions/ + - name: submission-name + type: string + description: Validated submission name from prepare task + - name: commit-sha + type: string + default: "" + - name: pipeline-run-id + type: string + - name: pipeline-repo-url + type: string + default: "https://github.com/RHEcosystemAppEng/ABEvalFlow.git" + - name: pipeline-repo-revision + type: string + default: "main" + - name: submission-repo-url + type: string + default: "" + description: >- + Git repo containing the submission definition. Required for remote + mode when submissions are in a different repo from the pipeline. + If empty, falls back to pipeline-repo-url. + - name: submission-repo-revision + type: string + default: "main" + description: Git branch/tag/SHA of the submission repo + - name: eval-base-image + type: string + default: "quay.io/rh-ee-ikrispin/abevalflow-eval-base:latest" + description: Container image with Harbor + dependencies pre-installed + - name: llm-model + type: string + default: "claude-sonnet" + - name: llm-api-base + type: string + default: "" + description: LLM proxy base URL for judging (e.g. http://litellm.ns.svc:4000) + - name: llm-api-key + type: string + default: "sk-dummy" + - name: agent-endpoint + type: string + default: "" + description: "HTTP endpoint of the A2A agent (required when eval-engine=a2a)" + - name: agent-timeout + type: string + default: "120" + description: A2A agent request timeout in seconds + - name: mcp-url + type: string + default: "" + description: "URL of the MCP server (required when eval-engine=mcpchecker)" + - name: mcpchecker-agent-model + type: string + default: "" + description: "MCPChecker agent model (e.g. google:gemini-2.5-flash). Empty = mcpchecker default." + - name: mcpchecker-judge-model + type: string + default: "" + description: "MCPChecker judge model (e.g. openai:gpt-4o). Empty = mcpchecker default." + - name: mcpchecker-task-timeout + type: string + default: "10m" + - name: ase-iterations + type: string + default: "5" + - name: ase-concurrency + type: string + default: "1" + - name: ase-judge-model + type: string + default: "" + - name: uplift-threshold + type: string + default: "0.0" + - name: eval-mode + type: string + default: "local" + description: >- + "local" runs eval directly in the task step (target must be reachable + from the pipeline cluster). "remote" submits an eval Pod to the + workload cluster (for cross-cluster scenarios). + - name: workload-cluster-url + type: string + default: "" + description: API URL of the workload cluster (required when eval-mode=remote) + - name: workload-namespace + type: string + default: "" + description: Namespace on the workload cluster (required when eval-mode=remote) + - name: workload-credentials-secret + type: string + default: "workload-cluster-credentials" + description: >- + Name of the Secret containing 'token' key for the workload cluster. + Only used when eval-mode=remote. + - name: eval-timeout + type: string + default: "1800" + description: Timeout in seconds for remote eval Pod + workspaces: + - name: source + results: + - name: treatment-mean-reward + description: Treatment/with-skill mean reward + - name: control-mean-reward + description: Control/without-skill mean reward + - name: recommendation + description: Pass or fail recommendation + - name: results-dir + description: Path to evaluation results + steps: + - name: run-eval + image: registry.redhat.io/openshift4/ose-cli:latest + env: + - name: WORKLOAD_TOKEN + valueFrom: + secretKeyRef: + name: $(params.workload-credentials-secret) + key: token + optional: true + - name: OPENAI_API_KEY + valueFrom: + secretKeyRef: + name: llm-credentials + key: api-key + optional: true + script: | + #!/usr/bin/env bash + set -euo pipefail + echo "=== EVALUATE PHASE ===" + + EVAL_ENGINE="$(params.eval-engine)" + EVAL_MODE="$(params.eval-mode)" + SUBMISSION_NAME="$(params.submission-name)" + SUBMISSION_DIR="$(params.submission-dir)" + AGENT_ENDPOINT="$(params.agent-endpoint)" + MCP_URL="$(params.mcp-url)" + COMMIT_SHA="$(params.commit-sha)" + PIPELINE_RUN_ID="$(params.pipeline-run-id)" + UPLIFT_THRESHOLD="$(params.uplift-threshold)" + LLM_API_KEY="${OPENAI_API_KEY:-$(params.llm-api-key)}" + export OPENAI_API_KEY="$LLM_API_KEY" + + SUBMISSION_REPO_URL="$(params.submission-repo-url)" + SUBMISSION_REPO_REV="$(params.submission-repo-revision)" + if [ -z "$SUBMISSION_REPO_URL" ]; then + SUBMISSION_REPO_URL="$(params.pipeline-repo-url)" + SUBMISSION_REPO_REV="$(params.pipeline-repo-revision)" + fi + + echo "Engine: $EVAL_ENGINE" + echo "Mode: $EVAL_MODE" + echo "Submission: $SUBMISSION_NAME" + + RESULTS_DIR="$(workspaces.source.path)/eval-results/$SUBMISSION_NAME" + REPORT_DIR="$(workspaces.source.path)/reports/$SUBMISSION_NAME" + mkdir -p "$RESULTS_DIR" "$REPORT_DIR" + + echo -n "0.0" > "$(results.treatment-mean-reward.path)" + echo -n "0.0" > "$(results.control-mean-reward.path)" + echo -n "fail" > "$(results.recommendation.path)" + echo -n "$RESULTS_DIR" > "$(results.results-dir.path)" + + # ---- Clone pipeline repo ---- + PIPELINE_DIR="$(workspaces.source.path)/_pipeline" + if [ -d "$PIPELINE_DIR/.git" ]; then + git -C "$PIPELINE_DIR" fetch origin "$(params.pipeline-repo-revision)" --depth 1 2>/dev/null || true + git -C "$PIPELINE_DIR" -c advice.detachedHead=false checkout FETCH_HEAD 2>/dev/null || true + else + git clone --depth 1 --branch "$(params.pipeline-repo-revision)" \ + "$(params.pipeline-repo-url)" "$PIPELINE_DIR" + fi + + # ================================================================ + # REMOTE MODE: submit an eval Pod to the workload cluster + # ================================================================ + if [ "$EVAL_MODE" = "remote" ]; then + echo "=== Remote eval mode ===" + + if [ -z "${WORKLOAD_TOKEN:-}" ]; then + echo "ERROR: workload-credentials-secret has no token for remote mode" + exit 1 + fi + + CLUSTER_URL="$(params.workload-cluster-url)" + NAMESPACE="$(params.workload-namespace)" + if [ -z "$CLUSTER_URL" ] || [ -z "$NAMESPACE" ]; then + echo "ERROR: workload-cluster-url and workload-namespace required for remote mode" + exit 1 + fi + + # TODO: Replace --insecure-skip-tls-verify with --certificate-authority + oc login --server="$CLUSTER_URL" --token="$WORKLOAD_TOKEN" --insecure-skip-tls-verify=true 2>/dev/null + + RUN_ID=$(echo "$PIPELINE_RUN_ID" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g' | cut -c1-20) + POD_NAME="eval-job-${RUN_ID:-$(date +%s)}" + echo "Eval pod: $POD_NAME" + echo "Submission repo: $SUBMISSION_REPO_URL @ $SUBMISSION_REPO_REV" + + cat <&1 | tail -1 + + # Clone submission repo if different from pipeline repo + SUBMISSION_PATH="/tmp/abevalflow/submissions/$SUBMISSION_DIR" + if [ "$SUBMISSION_REPO_URL" != "$(params.pipeline-repo-url)" ] || [ "$SUBMISSION_REPO_REV" != "$(params.pipeline-repo-revision)" ]; then + echo "Cloning submission repo: $SUBMISSION_REPO_URL @ $SUBMISSION_REPO_REV" + git clone --depth 1 --branch "$SUBMISSION_REPO_REV" \ + "$SUBMISSION_REPO_URL" /tmp/submission-repo + SUBMISSION_PATH="/tmp/submission-repo/submissions/$SUBMISSION_DIR" + fi + + if [ ! -d "\$SUBMISSION_PATH" ]; then + echo "ERROR: Submission not found at \$SUBMISSION_PATH" + exit 1 + fi + + RESULTS_DIR="/tmp/eval-results" + REPORT_DIR="/tmp/eval-reports" + mkdir -p "\$RESULTS_DIR" "\$REPORT_DIR" + + if [ "$EVAL_ENGINE" = "a2a" ]; then + N_ATTEMPTS=\$(python3 -c " + import yaml + try: + meta = yaml.safe_load(open('\$SUBMISSION_PATH/metadata.yaml')) + print(meta.get('experiment', {}).get('n_trials', 5)) + except: + print(5) + ") + + TASK_DIR="" + if [ -f "\$SUBMISSION_PATH/task.toml" ]; then + TASK_DIR="\$SUBMISSION_PATH" + elif [ -d "\$SUBMISSION_PATH/tasks" ]; then + for subdir in "\$SUBMISSION_PATH/tasks"/*/; do + if [ -f "\${subdir}task.toml" ]; then + TASK_DIR="\${subdir%/}" + break + fi + done + fi + + if [ -z "\$TASK_DIR" ]; then + echo "ERROR: No task.toml found" + exit 1 + fi + + echo "Task: \$(basename \$TASK_DIR) | Attempts: \$N_ATTEMPTS" + + python3 - "\$RESULTS_DIR" "\$TASK_DIR" "\$N_ATTEMPTS" "$AGENT_ENDPOINT" "$(params.agent-timeout)" "$(params.llm-api-base)" "openai/$(params.llm-model)" <<'GENCFG' + import sys, yaml + results_dir, task_dir, n_attempts, endpoint, timeout, llm_api_base, llm_model = sys.argv[1:8] + config = { + "job_name": "a2a-eval", + "jobs_dir": results_dir, + "n_attempts": int(n_attempts), + "agents": [{"import_path": "abevalflow.harbor_agents.a2a_adapter:A2AAgent", "kwargs": {"endpoint": endpoint, "timeout": int(timeout)}}], + "tasks": [{"path": task_dir}], + "environment": {"type": "local"}, + "verifier": {"env": {"LLM_JUDGE_MODEL": llm_model, "LLM_API_BASE": llm_api_base, "OPENAI_API_KEY": "sk-dummy"}} + } + with open(results_dir + "/config.yaml", "w") as f: + yaml.dump(config, f, default_flow_style=False) + GENCFG + + python3 -c "from abevalflow.harbor_agents.a2a_adapter import A2AAgent; print('A2AAgent imported OK')" + HARBOR_EXIT=0 + harbor run -c "\$RESULTS_DIR/config.yaml" -y 2>&1 || HARBOR_EXIT=\$? + echo "Harbor exit code: \$HARBOR_EXIT" + if [ "\$HARBOR_EXIT" -ne 0 ] && [ ! -f "\$RESULTS_DIR/a2a-eval/result.json" ]; then + find "\$RESULTS_DIR" -name "exception.txt" -exec echo "--- {} ---" \; -exec cat {} \; 2>/dev/null || true + echo "ERROR: Harbor failed and produced no results" + exit 1 + fi + find "\$RESULTS_DIR" -name "exception.txt" -exec echo "--- {} ---" \; -exec cat {} \; 2>/dev/null || true + + elif [ "$EVAL_ENGINE" = "mcpchecker" ]; then + echo "=== MCPChecker Evaluation ===" + pip install --quiet --no-cache-dir mcpchecker 2>&1 | tail -1 + cd "\$SUBMISSION_PATH" + export MCP_URL="$MCP_URL" + envsubst < mcp-config.yaml > mcp-config-resolved.yaml 2>/dev/null || cp mcp-config.yaml mcp-config-resolved.yaml + CHECKER_EXIT=0 + mcpchecker check eval.yaml --mcp-config-file mcp-config-resolved.yaml -o json > "\$RESULTS_DIR/mcpchecker-out.json" 2>&1 || CHECKER_EXIT=\$? + if [ "\$CHECKER_EXIT" -ne 0 ]; then + echo "WARNING: mcpchecker exited with code \$CHECKER_EXIT" + fi + fi + + echo "=== Running analyze.py ===" + ANALYZE_ARGS=( + --results-dir "\$RESULTS_DIR" + --output-dir "\$REPORT_DIR" + --submission-name "$SUBMISSION_NAME" + --threshold "$UPLIFT_THRESHOLD" + --eval-engine "$EVAL_ENGINE" + ) + [ -n "$COMMIT_SHA" ] && ANALYZE_ARGS+=(--commit-sha "$COMMIT_SHA") + [ -n "$PIPELINE_RUN_ID" ] && ANALYZE_ARGS+=(--pipeline-run-id "$PIPELINE_RUN_ID") + + python scripts/analyze.py "\${ANALYZE_ARGS[@]}" 2>&1 + if [ ! -f "\$REPORT_DIR/report.json" ]; then + echo "ERROR: analyze.py did not produce report.json" + exit 1 + fi + + echo "=== REPORT_JSON_START ===" + cat "\$REPORT_DIR/report.json" + echo "" + echo "=== REPORT_JSON_END ===" + + python3 -c " + import json + r = json.load(open('\$REPORT_DIR/report.json')) + s = r.get('summary', {}) + t = s.get('treatment', {}).get('mean_reward') + rec = s.get('recommendation', 'fail') + n = s.get('treatment', {}).get('n_trials', 0) + print(json.dumps({'mean_reward': t if t is not None else 0.0, 'recommendation': rec, 'n_trials': n})) + " + + echo "=== Remote Eval Pod Complete ===" + env: + - name: HOME + value: /tmp + - name: LLM_JUDGE_MODEL + value: "openai/$(params.llm-model)" + - name: LLM_BASE_URL + value: "$(params.llm-api-base)" + - name: LLM_API_BASE + value: "$(params.llm-api-base)" + - name: OPENAI_API_KEY + value: "$(params.llm-api-key)" + resources: + requests: + cpu: 200m + memory: 512Mi + limits: + cpu: "1" + memory: 2Gi + PODSPEC + + echo "Eval pod submitted. Waiting for completion..." + + TIMEOUT=$(params.eval-timeout) + POLL_INTERVAL=15 + ELAPSED=0 + + while [ $ELAPSED -lt $TIMEOUT ]; do + PHASE=$(oc get pod $POD_NAME -n $NAMESPACE -o jsonpath='{.status.phase}' 2>/dev/null || echo "Unknown") + case "$PHASE" in + Succeeded) + echo "Eval pod completed successfully (${ELAPSED}s)" + break + ;; + Failed) + echo "Eval pod failed (${ELAPSED}s)" + oc logs $POD_NAME -n $NAMESPACE 2>&1 | tail -50 + break + ;; + *) + printf "." + sleep $POLL_INTERVAL + ELAPSED=$((ELAPSED + POLL_INTERVAL)) + ;; + esac + done + echo "" + + if [ $ELAPSED -ge $TIMEOUT ]; then + echo "TIMEOUT: Eval pod did not complete in ${TIMEOUT}s" + oc logs $POD_NAME -n $NAMESPACE 2>&1 | tail -30 + oc delete pod $POD_NAME -n $NAMESPACE --ignore-not-found=true + exit 1 + fi + + if [ "$PHASE" = "Failed" ]; then + echo "ERROR: Remote eval pod failed" + oc delete pod $POD_NAME -n $NAMESPACE --ignore-not-found=true + exit 1 + fi + + echo "=== Retrieving results ===" + oc logs $POD_NAME -n $NAMESPACE 2>&1 | tee /tmp/eval-pod-logs.txt + + python3 - /tmp/eval-pod-logs.txt "$REPORT_DIR" "$(results.treatment-mean-reward.path)" "$(results.control-mean-reward.path)" "$(results.recommendation.path)" <<'EXTRACT' + import json, re, sys + + logs_path, report_dir, treat_path, ctrl_path, rec_path = sys.argv[1:6] + logs = open(logs_path).read() + + report_match = re.search(r'=== REPORT_JSON_START ===\n(.*?)\n=== REPORT_JSON_END ===', logs, re.DOTALL) + if report_match: + report_json = report_match.group(1).strip() + try: + report = json.loads(report_json) + with open(f"{report_dir}/report.json", "w") as f: + json.dump(report, f, indent=2) + print(f"Wrote report.json to {report_dir}/report.json") + + s = report.get("summary", {}) + t_reward = s.get("treatment", {}).get("mean_reward") + recommendation = s.get("recommendation", "fail") + print(f"Treatment mean reward: {t_reward}") + print(f"Recommendation: {recommendation}") + + open(treat_path, "w").write(str(t_reward if t_reward is not None else 0.0)) + open(ctrl_path, "w").write("0.0") + open(rec_path, "w").write(recommendation) + except json.JSONDecodeError as e: + print(f"ERROR: Failed to parse report.json: {e}") + sys.exit(1) + else: + print("ERROR: Could not extract report.json from pod logs") + sys.exit(1) + EXTRACT + + oc delete pod $POD_NAME -n $NAMESPACE --ignore-not-found=true + echo "=== Remote evaluate phase complete ===" + exit 0 + fi + + # ================================================================ + # LOCAL MODE: run evaluation directly in the task step + # ================================================================ + echo "=== Local eval mode ===" + + cd "$PIPELINE_DIR" + export PYTHONPATH="$PIPELINE_DIR" + python3 -m ensurepip --default-pip 2>/dev/null || true + pip install --quiet --no-cache-dir pydantic scipy pyyaml 2>&1 | tail -3 + + SUBMISSION_PATH="$(workspaces.source.path)/submissions/$SUBMISSION_DIR" + + # ---------- A2A Engine ---------- + if [ "$EVAL_ENGINE" = "a2a" ]; then + if [ -z "$AGENT_ENDPOINT" ]; then + echo "ERROR: agent-endpoint is required for a2a engine" + exit 1 + fi + + echo "=== A2A Evaluation (local) ===" + echo "Agent endpoint: $AGENT_ENDPOINT" + + N_ATTEMPTS=$(python3 -c " + import yaml + try: + meta = yaml.safe_load(open('$SUBMISSION_PATH/metadata.yaml')) + print(meta.get('experiment', {}).get('n_trials', 5)) + except: + print(5) + ") + + TASK_DIR="" + if [ -f "$SUBMISSION_PATH/task.toml" ]; then + TASK_DIR="$SUBMISSION_PATH" + elif [ -d "$SUBMISSION_PATH/tasks" ]; then + for subdir in "$SUBMISSION_PATH/tasks"/*/; do + if [ -f "${subdir}task.toml" ]; then + TASK_DIR="${subdir%/}" + break + fi + done + fi + + if [ -z "$TASK_DIR" ]; then + echo "ERROR: No task.toml found in submission" + exit 1 + fi + + echo "Task: $(basename $TASK_DIR) | Attempts: $N_ATTEMPTS" + + CONFIG_FILE="$RESULTS_DIR/a2a-config.yaml" + LLM_API_BASE="$(params.llm-api-base)" + LLM_MODEL="openai/$(params.llm-model)" + + python3 - "$CONFIG_FILE" "$TASK_DIR" "$RESULTS_DIR" "$N_ATTEMPTS" "$AGENT_ENDPOINT" "$(params.agent-timeout)" "$LLM_API_BASE" "$LLM_MODEL" <<'GENCFG' + import sys, yaml + config_file, task_dir, results_dir, n_attempts, endpoint, timeout, llm_api_base, llm_model = sys.argv[1:9] + config = { + "job_name": "a2a-eval", + "jobs_dir": results_dir, + "n_attempts": int(n_attempts), + "agents": [{"import_path": "abevalflow.harbor_agents.a2a_adapter:A2AAgent", "kwargs": {"endpoint": endpoint, "timeout": int(timeout)}}], + "tasks": [{"path": task_dir}], + "environment": {"type": "local"}, + "verifier": {"env": {"LLM_JUDGE_MODEL": llm_model, "LLM_API_BASE": llm_api_base, "OPENAI_API_KEY": "sk-dummy"}} + } + with open(config_file, "w") as f: + yaml.dump(config, f, default_flow_style=False) + GENCFG + + python3 -c "from abevalflow.harbor_agents.a2a_adapter import A2AAgent; print('A2AAgent imported OK')" + + pip install --quiet --no-cache-dir harbor-bench 2>&1 | tail -3 || true + HARBOR_EXIT=0 + harbor run -c "$CONFIG_FILE" -y 2>&1 || HARBOR_EXIT=$? + echo "Harbor exit code: $HARBOR_EXIT" + find "$RESULTS_DIR" -name "exception.txt" -exec echo "--- {} ---" \; -exec cat {} \; 2>/dev/null || true + if [ "$HARBOR_EXIT" -ne 0 ]; then + RESULT_COUNT=$(find "$RESULTS_DIR" -name "result.json" 2>/dev/null | wc -l) + if [ "$RESULT_COUNT" -eq 0 ]; then + echo "ERROR: Harbor failed with no results produced" + exit 1 + fi + echo "WARNING: Harbor exit $HARBOR_EXIT but $RESULT_COUNT result(s) found, continuing" + fi + + # ---------- MCPChecker Engine ---------- + elif [ "$EVAL_ENGINE" = "mcpchecker" ]; then + if [ -z "$MCP_URL" ]; then + echo "ERROR: mcp-url is required for mcpchecker engine" + exit 1 + fi + + echo "=== MCPChecker Evaluation (local) ===" + echo "MCP URL: $MCP_URL" + pip install --quiet --no-cache-dir mcpchecker 2>&1 | tail -3 + + cd "$SUBMISSION_PATH" + export MCP_URL + envsubst < mcp-config.yaml > mcp-config-resolved.yaml 2>/dev/null || cp mcp-config.yaml mcp-config-resolved.yaml + + if [ -n "$(params.llm-api-base)" ]; then + export OPENAI_BASE_URL="$(params.llm-api-base)" + fi + + CHECKER_EXIT=0 + mcpchecker check eval.yaml --mcp-config-file mcp-config-resolved.yaml -o json > "$RESULTS_DIR/mcpchecker-out.json" 2>&1 || CHECKER_EXIT=$? + if [ "$CHECKER_EXIT" -ne 0 ]; then + echo "WARNING: mcpchecker exited with code $CHECKER_EXIT" + fi + if [ ! -s "$RESULTS_DIR/mcpchecker-out.json" ]; then + echo "ERROR: mcpchecker produced no output" + exit 1 + fi + + python3 - "$RESULTS_DIR/mcpchecker-out.json" "$SUBMISSION_NAME" "$REPORT_DIR" "$PIPELINE_RUN_ID" <<'AGGREGATE' + import json, sys + from pathlib import Path + output_file, sub_name, report_dir, run_id = sys.argv[1:5] + try: + data = json.loads(Path(output_file).read_text()) + total = data.get("total_checks", 0) + passed = data.get("passed_checks", 0) + score = passed / total if total > 0 else 0.0 + rec = "pass" if score >= 0.7 else "fail" + report = {"summary": {"submission_name": sub_name, "recommendation": rec, "treatment": {"mean_reward": score, "n_trials": total}, "control": {"mean_reward": 0.0}}, "pipeline_run_id": run_id} + Path(report_dir).mkdir(parents=True, exist_ok=True) + (Path(report_dir) / "report.json").write_text(json.dumps(report, indent=2)) + print(f"MCPChecker score: {score:.4f}, recommendation: {rec}") + except Exception as e: + print(f"ERROR: MCPChecker aggregation failed: {e}") + sys.exit(1) + AGGREGATE + + cd "$PIPELINE_DIR" + + # ---------- ASE Engine ---------- + elif [ "$EVAL_ENGINE" = "ase" ]; then + echo "=== ASE Evaluation (local) ===" + + ITERATIONS="$(params.ase-iterations)" + LLM_MODEL="$(params.llm-model)" + BASE_URL="$(params.llm-api-base)" + JUDGE_MODEL="$(params.ase-judge-model)" + CONCURRENCY="$(params.ase-concurrency)" + + if [ -f "$SUBMISSION_PATH/metadata.yaml" ]; then + OVERRIDES=$(python3 -c "import yaml,sys;m=yaml.safe_load(open(sys.argv[1]))or{};e=m.get('experiment')or{};l=m.get('llm')or{};e.get('n_trials')and print('ITERATIONS='+str(e['n_trials']));l.get('model')and print('LLM_MODEL='+str(l['model']));l.get('api_base')and print('BASE_URL='+str(l['api_base']));l.get('judge_model')and print('JUDGE_MODEL='+str(l['judge_model']));e.get('concurrency')and print('CONCURRENCY='+str(e['concurrency']))" "$SUBMISSION_PATH/metadata.yaml" 2>/dev/null || true) + eval "$OVERRIDES" + fi + + ITERATIONS="${ITERATIONS:-5}" + LLM_MODEL="${LLM_MODEL:-claude-sonnet}" + BASE_URL="${BASE_URL:-}" + JUDGE_MODEL="${JUDGE_MODEL:-$LLM_MODEL}" + CONCURRENCY="${CONCURRENCY:-1}" + [[ -n "$BASE_URL" && "$BASE_URL" != */v1 ]] && BASE_URL="${BASE_URL}/v1" + + echo " Iterations: $ITERATIONS | Model: $LLM_MODEL | Judge: $JUDGE_MODEL" + + SKILL_DIR="" + if [ -f "$SUBMISSION_PATH/skills/SKILL.md" ]; then + SKILL_DIR="$SUBMISSION_PATH/skills" + else + for dir in "$SUBMISSION_PATH"/skills/*/; do + if [ -f "${dir}SKILL.md" ]; then + SKILL_DIR="$dir" + break + fi + done + fi + + if [ -z "$SKILL_DIR" ]; then + echo "ERROR: No SKILL.md found in submission" + exit 1 + fi + + if [ ! -f "$SKILL_DIR/evals/evals.json" ] && [ -f "$SUBMISSION_PATH/evals/evals.json" ]; then + ln -sf "$SUBMISSION_PATH/evals" "$SKILL_DIR/evals" + fi + + npm install --global agent-skills-eval 2>&1 | tail -3 + + ASE_FAILURES=0 + for i in $(seq 1 "$ITERATIONS"); do + echo "--- ASE Iteration $i/$ITERATIONS ---" + ASE_ARGS=("$SKILL_DIR" --baseline --layout iteration --report) + [ -n "$BASE_URL" ] && ASE_ARGS+=(--base-url "$BASE_URL") + ASE_ARGS+=(--target "$LLM_MODEL" --judge "$JUDGE_MODEL") + ASE_ARGS+=(--concurrency "$CONCURRENCY" --workspace "$RESULTS_DIR/iteration-$i") + ASE_ARGS+=(--api-key-env OPENAI_API_KEY) + agent-skills-eval "${ASE_ARGS[@]}" || ASE_FAILURES=$((ASE_FAILURES + 1)) + done + if [ "$ASE_FAILURES" -eq "$ITERATIONS" ]; then + echo "ERROR: All $ITERATIONS ASE iterations failed" + exit 1 + fi + [ "$ASE_FAILURES" -gt 0 ] && echo "WARNING: $ASE_FAILURES/$ITERATIONS ASE iterations failed" + + # ---------- Harbor Engine ---------- + elif [ "$EVAL_ENGINE" = "harbor" ]; then + echo "=== Harbor Evaluation (local) ===" + echo "NOTE: Local Harbor mode uses environment.type=local (no scaffold/build)." + echo "For full Harbor A/B with container registry, use the standalone pipeline." + + pip install --quiet --no-cache-dir harbor-bench 2>&1 | tail -3 || true + + TASK_DIR="" + if [ -f "$SUBMISSION_PATH/task.toml" ]; then + TASK_DIR="$SUBMISSION_PATH" + elif [ -d "$SUBMISSION_PATH/tasks" ]; then + for subdir in "$SUBMISSION_PATH/tasks"/*/; do + if [ -f "${subdir}task.toml" ]; then + TASK_DIR="${subdir%/}" + break + fi + done + fi + + if [ -n "$TASK_DIR" ]; then + echo "Running Harbor with local environment for: $(basename $TASK_DIR)" + N_ATTEMPTS=$(python3 -c "import yaml;m=yaml.safe_load(open('$SUBMISSION_PATH/metadata.yaml'));print(m.get('experiment',{}).get('n_trials',5))" 2>/dev/null || echo "5") + + python3 - "$RESULTS_DIR" "$TASK_DIR" "$N_ATTEMPTS" "$(params.llm-api-base)" "openai/$(params.llm-model)" <<'HARBORCFG' + import sys, yaml + results_dir, task_dir, n_attempts, llm_api_base, llm_model = sys.argv[1:6] + config = { + "job_name": "harbor-eval", + "jobs_dir": results_dir, + "n_attempts": int(n_attempts), + "tasks": [{"path": task_dir}], + "environment": {"type": "local"}, + "verifier": {"env": {"LLM_JUDGE_MODEL": llm_model, "LLM_API_BASE": llm_api_base, "OPENAI_API_KEY": "sk-dummy"}} + } + with open(results_dir + "/config.yaml", "w") as f: + yaml.dump(config, f, default_flow_style=False) + HARBORCFG + HARBOR_EXIT=0 + harbor run -c "$RESULTS_DIR/config.yaml" -y 2>&1 || HARBOR_EXIT=$? + if [ "$HARBOR_EXIT" -ne 0 ]; then + echo "WARNING: Harbor exited with code $HARBOR_EXIT" + fi + else + echo "ERROR: No task.toml found, cannot run Harbor eval" + exit 1 + fi + + else + echo "ERROR: Unsupported eval engine: $EVAL_ENGINE" + exit 1 + fi + + # ---- Run analyze.py (for local mode) ---- + echo "=== Running analyze.py ===" + ANALYZE_ARGS=( + --results-dir "$RESULTS_DIR" + --output-dir "$REPORT_DIR" + --submission-name "$SUBMISSION_NAME" + --threshold "$UPLIFT_THRESHOLD" + --eval-engine "$EVAL_ENGINE" + ) + [ -n "$COMMIT_SHA" ] && ANALYZE_ARGS+=(--commit-sha "$COMMIT_SHA") + [ -n "$PIPELINE_RUN_ID" ] && ANALYZE_ARGS+=(--pipeline-run-id "$PIPELINE_RUN_ID") + + python scripts/analyze.py "${ANALYZE_ARGS[@]}" 2>&1 + + # ---- Extract results for Tekton ---- + if [ -f "$REPORT_DIR/report.json" ]; then + python3 -c " + import json, sys + r = json.load(open(sys.argv[1])) + s = r.get('summary', {}) + t = s.get('treatment', {}).get('mean_reward') + rec = s.get('recommendation', 'fail') + open(sys.argv[2], 'w').write(str(t if t is not None else 0.0)) + open(sys.argv[3], 'w').write('0.0') + open(sys.argv[4], 'w').write(rec) + print(f'Recommendation: {rec}, Mean reward: {t}') + " "$REPORT_DIR/report.json" \ + "$(results.treatment-mean-reward.path)" \ + "$(results.control-mean-reward.path)" \ + "$(results.recommendation.path)" + else + echo "ERROR: report.json not generated after evaluation" + exit 1 + fi + + echo "=== Evaluate phase complete ===" diff --git a/pipeline/tasks/konflux/parse-snapshot.yaml b/pipeline/tasks/konflux/parse-snapshot.yaml new file mode 100644 index 0000000..573d9b4 --- /dev/null +++ b/pipeline/tasks/konflux/parse-snapshot.yaml @@ -0,0 +1,75 @@ +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: parse-snapshot + labels: + app.kubernetes.io/name: abevalflow + app.kubernetes.io/component: konflux +spec: + description: >- + Parses a Konflux Snapshot JSON to extract the component container image, + git source URL, git revision, and component name. This bridges the Konflux + SNAPSHOT model with ABEvalFlow's parameter-based pipeline. + params: + - name: SNAPSHOT + type: string + description: Konflux Snapshot JSON containing component details + - name: component-name + type: string + default: "" + description: >- + Specific component name to extract from the snapshot. If empty, + the first component (.components[0]) is used. For multi-component + applications, set this to the component you want to evaluate to + avoid accidentally selecting the wrong one. + results: + - name: component-image + description: Full container image reference (with digest) from the snapshot + - name: git-url + description: Git repository URL of the component source + - name: git-revision + description: Git commit SHA of the component source + - name: component-name + description: Name of the component extracted from the snapshot + steps: + - name: parse + image: registry.access.redhat.com/ubi9/ubi-minimal:latest + env: + - name: SNAPSHOT + value: $(params.SNAPSHOT) + - name: TARGET_COMPONENT + value: $(params.component-name) + script: | + #!/usr/bin/env bash + set -euo pipefail + echo "=== PARSE SNAPSHOT ===" + + microdnf install -y jq --nodocs 2>/dev/null || true + + if [ -n "$TARGET_COMPONENT" ]; then + JQ_FILTER=".components[] | select(.name == \"$TARGET_COMPONENT\")" + else + JQ_FILTER=".components[0]" + fi + + COMPONENT_IMAGE=$(echo "${SNAPSHOT}" | jq -r "$JQ_FILTER | .containerImage") + GIT_URL=$(echo "${SNAPSHOT}" | jq -r "$JQ_FILTER | .source.git.url // empty") + GIT_REVISION=$(echo "${SNAPSHOT}" | jq -r "$JQ_FILTER | .source.git.revision // empty") + COMPONENT_NAME=$(echo "${SNAPSHOT}" | jq -r "$JQ_FILTER | .name") + + if [ -z "$COMPONENT_IMAGE" ] || [ "$COMPONENT_IMAGE" = "null" ]; then + echo "ERROR: Could not extract container image from snapshot" + echo "Snapshot contents:" + echo "${SNAPSHOT}" | jq . 2>/dev/null || echo "${SNAPSHOT}" + exit 1 + fi + + echo -n "$COMPONENT_IMAGE" > "$(results.component-image.path)" + echo -n "$GIT_URL" > "$(results.git-url.path)" + echo -n "$GIT_REVISION" > "$(results.git-revision.path)" + echo -n "$COMPONENT_NAME" > "$(results.component-name.path)" + + echo "Component: $COMPONENT_NAME" + echo "Image: $COMPONENT_IMAGE" + echo "Git URL: $GIT_URL" + echo "Git Revision: $GIT_REVISION" diff --git a/pipeline/tasks/konflux/prepare.yaml b/pipeline/tasks/konflux/prepare.yaml new file mode 100644 index 0000000..a52ce68 --- /dev/null +++ b/pipeline/tasks/konflux/prepare.yaml @@ -0,0 +1,257 @@ +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: prepare + labels: + app.kubernetes.io/name: abevalflow + app.kubernetes.io/component: konflux +spec: + description: >- + Preparation phase: clones the submission repository, optionally generates + evaluation files, and validates the submission structure. Adapted for Konflux + integration — no hardcoded namespace, configurable pipeline repo URL. + params: + - name: repo-url + type: string + description: Git URL of the submissions repository + - name: revision + type: string + description: Git revision (branch, tag, SHA) to checkout + - name: submission-dir + type: string + description: Submission directory name under submissions/ + - name: eval-engine + type: string + default: "harbor" + description: Evaluation engine (harbor, ase, mcpchecker, a2a, both) + - name: pipeline-repo-url + type: string + default: "https://github.com/RHEcosystemAppEng/ABEvalFlow.git" + - name: pipeline-repo-revision + type: string + default: "main" + - name: pipeline-run-name + type: string + description: Name of the PipelineRun + - name: enable-generation + type: string + default: "true" + - name: llm-base-url + type: string + default: "" + - name: llm-model + type: string + default: "claude-sonnet" + - name: agent-type + type: string + default: "api" + - name: max-generation-retries + type: string + default: "3" + - name: max-workspace-mb + type: string + default: "500" + workspaces: + - name: source + description: Workspace for the cloned repository + results: + - name: submission-name + description: Validated submission name from metadata.yaml + - name: report-prefix + description: MinIO report prefix (timestamp_name_runid) + - name: security-scan + description: Security scan mode from metadata (disabled/warn/block) + - name: security-scan-use-llm + description: Whether to use LLM in security scanning + - name: skip-quality-review + description: Whether to skip quality review (from metadata) + - name: mcp-credentials-secret + description: Secret name for MCP credentials (MCPChecker only) + - name: generated-files + description: JSON array of generated file paths + - name: validation-result + description: JSON validation result object + steps: + - name: clone-repo + image: registry.access.redhat.com/ubi9/python-311:9.6 + script: | + #!/usr/bin/env bash + set -euo pipefail + echo "=== PREPARE PHASE: Clone Repository ===" + CHECKOUT_DIR="$(workspaces.source.path)" + REVISION="$(params.revision)" + git config --global --add safe.directory "$CHECKOUT_DIR" + rm -rf "${CHECKOUT_DIR:?}"/{,.[!.],..?}* 2>/dev/null || true + echo "Cloning $(params.repo-url) @ ${REVISION}" + if git clone --depth 1 --branch "$REVISION" "$(params.repo-url)" "$CHECKOUT_DIR" 2>/dev/null; then + echo "Cloned branch/tag $REVISION" + else + git clone "$(params.repo-url)" "$CHECKOUT_DIR" + cd "$CHECKOUT_DIR" + git checkout "$REVISION" + fi + cd "$CHECKOUT_DIR" + echo "Cloned at $(git rev-parse HEAD)" + + - name: clone-pipeline-repo + image: registry.access.redhat.com/ubi9/python-311:9.6 + script: | + #!/usr/bin/env bash + set -euo pipefail + echo "=== PREPARE PHASE: Clone Pipeline Repo ===" + PIPELINE_DIR="$(workspaces.source.path)/_pipeline" + if [ -d "$PIPELINE_DIR/.git" ]; then + echo "Pipeline repo already cloned, updating..." + cd "$PIPELINE_DIR" + git fetch origin "$(params.pipeline-repo-revision)" + git checkout "$(params.pipeline-repo-revision)" 2>/dev/null || git checkout FETCH_HEAD + else + git clone --depth 1 --branch "$(params.pipeline-repo-revision)" \ + "$(params.pipeline-repo-url)" "$PIPELINE_DIR" + fi + echo "Pipeline repo ready at $(params.pipeline-repo-revision)" + + - name: generate-tests + image: registry.access.redhat.com/ubi9/python-311:9.6 + env: + - name: LLM_BASE_URL + value: "$(params.llm-base-url)" + - name: LLM_MODEL + value: "$(params.llm-model)" + - name: AGENT_TYPE + value: "$(params.agent-type)" + - name: EVAL_ENGINE + value: "$(params.eval-engine)" + - name: ENABLE_GENERATION + value: "$(params.enable-generation)" + - name: LLM_API_KEY + valueFrom: + secretKeyRef: + name: llm-credentials + key: api-key + optional: true + script: | + #!/usr/bin/env bash + set -euo pipefail + echo "=== PREPARE PHASE: Generate Tests ===" + if [ "$ENABLE_GENERATION" != "true" ]; then + echo "Generation disabled, skipping" + echo "[]" > "$(results.generated-files.path)" + exit 0 + fi + if [ "$EVAL_ENGINE" = "mcpchecker" ]; then + echo "MCPChecker mode: no generation needed" + echo "[]" > "$(results.generated-files.path)" + exit 0 + fi + PIPELINE_DIR="$(workspaces.source.path)/_pipeline" + SUBMISSION_PATH="$(workspaces.source.path)/submissions/$(params.submission-dir)" + WORKSPACE_DIR="$(workspaces.source.path)" + cd "$PIPELINE_DIR" + pip install --quiet --no-cache-dir pydantic pyyaml openai pytest + export PYTHONPATH="$PIPELINE_DIR" + GENERATED='[]' + if [ "$EVAL_ENGINE" = "ase" ] || [ "$EVAL_ENGINE" = "both" ]; then + if [ ! -f "$SUBMISSION_PATH/evals/evals.json" ]; then + echo "ASE mode: generating evals.json from SKILL.md..." + if python3 scripts/generate_ase_evals.py "$SUBMISSION_PATH" \ + > /tmp/ase-stdout.json 2>/tmp/ase-stderr.txt; then + cat /tmp/ase-stdout.json + GENERATED=$(python3 -c "import json; print(json.dumps(json.load(open('/tmp/ase-stdout.json')).get('generated', [])))" 2>/dev/null || echo "[]") + else + echo "WARNING: ASE evals.json generation failed:" + cat /tmp/ase-stderr.txt || true + fi + else + echo "ASE mode: evals.json exists, skipping generation" + fi + fi + if [ "$EVAL_ENGINE" = "harbor" ] || [ "$EVAL_ENGINE" = "both" ]; then + if python scripts/generate_tests.py "$SUBMISSION_PATH" \ + --workspace-dir "$WORKSPACE_DIR" \ + --agent-type "$(params.agent-type)" \ + --max-retries "$(params.max-generation-retries)" \ + > /tmp/harbor-stdout.json 2>/tmp/harbor-stderr.txt; then + cat /tmp/harbor-stdout.json + HARBOR_GENERATED=$(python3 -c "import json; print(json.dumps(json.load(open('/tmp/harbor-stdout.json')).get('generated', [])))" 2>/dev/null || echo "[]") + GENERATED=$(python3 -c "import json,sys; a=json.loads(sys.argv[1]); b=json.loads(sys.argv[2]); print(json.dumps(list(set(a+b))))" "$GENERATED" "$HARBOR_GENERATED") + else + echo "WARNING: Harbor test generation failed:" + cat /tmp/harbor-stderr.txt || true + fi + fi + echo "$GENERATED" > "$(results.generated-files.path)" + echo "Generated files: $GENERATED" + + - name: validate + image: registry.access.redhat.com/ubi9/python-311:9.6 + script: | + #!/usr/bin/env bash + set -euo pipefail + echo "=== PREPARE PHASE: Validate Submission ===" + PIPELINE_DIR="$(workspaces.source.path)/_pipeline" + SUBMISSION_PATH="$(workspaces.source.path)/submissions/$(params.submission-dir)" + MAX_MB=$(params.max-workspace-mb) + SIZE_KB=$(du -sk "$(workspaces.source.path)" | cut -f1) + SIZE_MB=$((SIZE_KB / 1024)) + if [ "$SIZE_MB" -gt "$MAX_MB" ]; then + echo "ERROR: Workspace size ${SIZE_MB}MB exceeds limit ${MAX_MB}MB" + exit 1 + fi + echo "Workspace size: ${SIZE_MB}MB (limit: ${MAX_MB}MB)" + cd "$PIPELINE_DIR" + export PYTHONPATH="$PIPELINE_DIR" + pip install --quiet --no-cache-dir pydantic pyyaml + python scripts/validate.py "$SUBMISSION_PATH" \ + --eval-engine "$(params.eval-engine)" \ + | tee /tmp/validation-output.json + cat /tmp/validation-output.json | tr -d '\n' > "$(results.validation-result.path)" + REPORTS_DIR="$(workspaces.source.path)/reports/$(params.submission-dir)" + mkdir -p "$REPORTS_DIR" + cp /tmp/validation-output.json "$REPORTS_DIR/validation.json" + VALID=$(python3 -c "import json; print(json.load(open('/tmp/validation-output.json'))['valid'])") + if [ "$VALID" = "True" ]; then + python3 -c " + import yaml, sys + meta = yaml.safe_load(open(sys.argv[1])) + print(meta['name'], end='') + " "$SUBMISSION_PATH/metadata.yaml" > "$(results.submission-name.path)" + python3 -c " + import yaml, sys + meta = yaml.safe_load(open(sys.argv[1])) + print(meta.get('security_scan', 'warn'), end='') + " "$SUBMISSION_PATH/metadata.yaml" > "$(results.security-scan.path)" + python3 -c " + import yaml, sys + meta = yaml.safe_load(open(sys.argv[1])) + print(str(meta.get('security_scan_use_llm', True)).lower(), end='') + " "$SUBMISSION_PATH/metadata.yaml" > "$(results.security-scan-use-llm.path)" + python3 -c " + import yaml, sys + meta = yaml.safe_load(open(sys.argv[1])) + print(str(meta.get('skip_quality_review', False)).lower(), end='') + " "$SUBMISSION_PATH/metadata.yaml" > "$(results.skip-quality-review.path)" + SUBMISSION_NAME=$(cat "$(results.submission-name.path)") + TIMESTAMP=$(date -u +%Y%m%d_%H%M%S) + REPORT_PREFIX="${TIMESTAMP}_${SUBMISSION_NAME}_$(params.pipeline-run-name)" + echo -n "$REPORT_PREFIX" > "$(results.report-prefix.path)" + python3 -c " + import yaml, sys + meta = yaml.safe_load(open(sys.argv[1])) + mcp = meta.get('mcp') or {} + secret = mcp.get('credentials_secret', '') + print(secret if secret else 'mcp-credentials-placeholder', end='') + " "$SUBMISSION_PATH/metadata.yaml" > "$(results.mcp-credentials-secret.path)" + echo "Validation PASSED" + echo "Submission: $SUBMISSION_NAME" + echo "Report prefix: $REPORT_PREFIX" + else + echo "INVALID" > "$(results.submission-name.path)" + echo "warn" > "$(results.security-scan.path)" + echo "false" > "$(results.security-scan-use-llm.path)" + echo "false" > "$(results.skip-quality-review.path)" + echo "INVALID" > "$(results.report-prefix.path)" + echo "" > "$(results.mcp-credentials-secret.path)" + echo "Validation FAILED" + exit 1 + fi diff --git a/pipeline/tasks/konflux/store.yaml b/pipeline/tasks/konflux/store.yaml new file mode 100644 index 0000000..5a5de2e --- /dev/null +++ b/pipeline/tasks/konflux/store.yaml @@ -0,0 +1,248 @@ +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: store + labels: + app.kubernetes.io/name: abevalflow + app.kubernetes.io/component: konflux +spec: + description: >- + Persists A/B evaluation results and publishes artifacts. Step 1 writes + report.json to PostgreSQL via store_results.py (skipped for engines + without a DB report). Step 2 uploads reports to MinIO, optionally + promotes images to Quay.io, posts GitHub PR comments, and cleans up + ephemeral registry images via publish.py. Adapted for Konflux — all + secrets are optional with graceful no-op when missing. + params: + - name: submission-name + type: string + description: Submission name + - name: pipeline-run-id + type: string + description: Tekton PipelineRun name + - name: report-prefix + type: string + default: "" + description: >- + MinIO report prefix (YYYYMMDD_HHMMSS_submissionname_runid). + If empty, one will be generated (legacy behavior). + - name: recommendation + type: string + description: "'pass' or 'fail' from the analyze step" + - name: treatment-image-ref + type: string + default: "" + description: Treatment image digest ref for Quay promotion and cleanup + - name: control-image-ref + type: string + default: "" + description: Control image digest ref for cleanup + - name: commit-sha + type: string + default: "" + description: Git commit SHA for image tagging + - name: uplift-threshold + type: string + default: "-1.0" + description: >- + Minimum uplift for Quay promotion. Default -1.0 disables promotion. + - name: quay-ttl-days + type: string + default: "7" + description: TTL in days for promoted Quay images + - name: quay-repo + type: string + default: "" + description: Quay.io repo for image promotion (e.g. quay.io/myorg) + - name: repo-name + type: string + default: "" + description: GitHub repo (org/name) for PR comment + - name: pr-number + type: string + default: "" + description: PR number for GitHub comment + - name: results-dir + type: string + default: "" + description: Path to results dir for debug artifact upload (Harbor or ASE) + - name: eval-engine + type: string + default: "harbor" + description: >- + Evaluation engine ('harbor', 'ase', 'both', 'mcpchecker', 'a2a'). + Controls whether DB storage runs and publish artifact layout. + - name: minio-bucket + type: string + default: "ab-eval-reports" + description: MinIO bucket for report storage + - name: pipeline-repo-url + type: string + default: "https://github.com/RHEcosystemAppEng/ABEvalFlow.git" + description: URL of the pipeline repo containing scripts + - name: pipeline-repo-revision + type: string + default: "main" + description: Branch or SHA of the pipeline repo to use for scripts + workspaces: + - name: source + description: Shared workspace containing evaluation artifacts + steps: + - name: store-to-db + image: registry.access.redhat.com/ubi9/python-311:9.6 + env: + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: ab-eval-db-credentials + key: database-url + optional: true + script: | + #!/usr/bin/env bash + set -euo pipefail + + EVAL_ENGINE="$(params.eval-engine)" + if [ "$EVAL_ENGINE" = "mcpchecker" ]; then + echo "Skipping DB store for eval-engine=$EVAL_ENGINE" + exit 0 + fi + + if [ -z "${DATABASE_URL:-}" ]; then + echo "DATABASE_URL not configured, skipping DB store" + exit 0 + fi + + # --- 1. Clone pipeline repo (or update to requested revision) --- + PIPELINE_DIR="$(workspaces.source.path)/_pipeline" + if [ -d "$PIPELINE_DIR" ]; then + echo "Pipeline repo exists, updating to requested revision" + git -C "$PIPELINE_DIR" fetch origin "$(params.pipeline-repo-revision)" --depth 1 + git -C "$PIPELINE_DIR" -c advice.detachedHead=false checkout FETCH_HEAD + else + git clone --depth 1 --branch "$(params.pipeline-repo-revision)" \ + "$(params.pipeline-repo-url)" "$PIPELINE_DIR" + fi + + cd "$PIPELINE_DIR" + export PYTHONPATH="$PIPELINE_DIR" + + # --- 2. Install dependencies --- + pip install --quiet --no-cache-dir pydantic sqlalchemy "psycopg[binary]" "tenacity>=8.2" + + # --- 3. Run store script --- + REPORT_DIR="$(workspaces.source.path)/reports/$(params.submission-name)" + echo "Storing results for submission=$(params.submission-name) run=$(params.pipeline-run-id)" + echo "Report dir: $REPORT_DIR" + python scripts/store_results.py \ + --report-dir "$REPORT_DIR" \ + --run-id "$(params.pipeline-run-id)" + + echo "Results stored successfully" + + - name: upload-artifacts + image: registry.access.redhat.com/ubi9/python-311:9.6 + env: + - name: MINIO_ENDPOINT + valueFrom: + secretKeyRef: + name: minio-credentials + key: endpoint-url + optional: true + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: root-user + optional: true + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: root-password + optional: true + - name: GITHUB_TOKEN + valueFrom: + secretKeyRef: + name: github-token + key: token + optional: true + script: | + #!/usr/bin/env bash + set -euo pipefail + + # Check if MinIO credentials are available + if [ -z "${MINIO_ENDPOINT:-}" ] || [ -z "${MINIO_ACCESS_KEY:-}" ] || [ -z "${MINIO_SECRET_KEY:-}" ]; then + echo "MinIO credentials not available, skipping artifact upload" + echo "To enable artifact upload, create a 'minio-credentials' secret with endpoint-url, root-user, and root-password keys" + exit 0 + fi + + # --- 1. Clone pipeline repo (or update to requested revision) --- + PIPELINE_DIR="$(workspaces.source.path)/_pipeline" + if [ -d "$PIPELINE_DIR/.git" ]; then + echo "Pipeline repo exists, updating to requested revision" + git -C "$PIPELINE_DIR" fetch origin "$(params.pipeline-repo-revision)" --depth 1 + git -C "$PIPELINE_DIR" -c advice.detachedHead=false checkout FETCH_HEAD + else + git clone --depth 1 --branch "$(params.pipeline-repo-revision)" \ + "$(params.pipeline-repo-url)" "$PIPELINE_DIR" + fi + + cd "$PIPELINE_DIR" + export PYTHONPATH="$PIPELINE_DIR" + + # --- 2. Install dependencies --- + pip install --quiet --no-cache-dir minio + + # --- 3. Run publish script (MinIO upload, Quay promotion, PR comment) --- + REPORT_DIR="$(workspaces.source.path)/reports/$(params.submission-name)" + echo "Publishing artifacts for submission=$(params.submission-name) run=$(params.pipeline-run-id)" + echo "Report dir: $REPORT_DIR" + + ARGS=( + --report-dir "$REPORT_DIR" + --submission-name "$(params.submission-name)" + --pipeline-run-id "$(params.pipeline-run-id)" + --recommendation "$(params.recommendation)" + --uplift-threshold "$(params.uplift-threshold)" + --minio-bucket "$(params.minio-bucket)" + ) + + if [ -n "$(params.treatment-image-ref)" ]; then + ARGS+=(--treatment-image-ref "$(params.treatment-image-ref)") + fi + if [ -n "$(params.control-image-ref)" ]; then + ARGS+=(--control-image-ref "$(params.control-image-ref)") + fi + if [ -n "$(params.commit-sha)" ]; then + ARGS+=(--commit-sha "$(params.commit-sha)") + fi + if [ -n "$(params.quay-repo)" ]; then + ARGS+=(--quay-repo "$(params.quay-repo)") + fi + if [ -n "$(params.quay-ttl-days)" ]; then + ARGS+=(--quay-ttl-days "$(params.quay-ttl-days)") + fi + if [ -n "$(params.repo-name)" ]; then + ARGS+=(--repo-name "$(params.repo-name)") + fi + if [ -n "$(params.pr-number)" ]; then + ARGS+=(--pr-number "$(params.pr-number)") + fi + RESULTS_DIR="$(params.results-dir)" + if [ -z "$RESULTS_DIR" ] && [ "$(params.eval-engine)" != "harbor" ]; then + RESULTS_DIR="$(workspaces.source.path)/eval-results/$(params.submission-name)" + fi + if [ -n "$RESULTS_DIR" ]; then + ARGS+=(--results-dir "$RESULTS_DIR") + fi + + ARGS+=(--eval-engine "$(params.eval-engine)") + ARGS+=(--workspace-root "$(workspaces.source.path)") + if [ -n "$(params.report-prefix)" ]; then + ARGS+=(--report-prefix "$(params.report-prefix)") + fi + + python scripts/publish.py "${ARGS[@]}" + + echo "Publish step completed successfully" diff --git a/pipeline/tasks/konflux/test.yaml b/pipeline/tasks/konflux/test.yaml new file mode 100644 index 0000000..2bc663f --- /dev/null +++ b/pipeline/tasks/konflux/test.yaml @@ -0,0 +1,263 @@ +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: test + labels: + app.kubernetes.io/name: abevalflow + app.kubernetes.io/component: konflux +spec: + description: >- + Testing phase: runs security scanning and quality review checks. + Adapted for Konflux — MinIO/DB persistence is optional (graceful no-op + when secrets are not present). + params: + - name: submission-dir + type: string + description: Submission directory name under submissions/ + - name: submission-name + type: string + description: Validated submission name from metadata.yaml + - name: eval-engine + type: string + description: Evaluation engine (harbor, ase, mcpchecker, a2a, both) + - name: report-prefix + type: string + description: MinIO report prefix + - name: pipeline-run-name + type: string + description: PipelineRun name for DB records + - name: pipeline-repo-url + type: string + default: "https://github.com/RHEcosystemAppEng/ABEvalFlow.git" + - name: pipeline-repo-revision + type: string + default: "main" + - name: security-scan-mode + type: string + default: "" + - name: submission-security-scan + type: string + default: "warn" + - name: security-scan-use-llm + type: string + default: "true" + - name: enable-quality-review + type: string + default: "true" + - name: submission-skip-quality-review + type: string + default: "false" + - name: llm-base-url + type: string + default: "" + - name: llm-model + type: string + default: "claude-sonnet" + - name: llm-api-key + type: string + default: "" + - name: minio-endpoint + type: string + default: "minio.ab-eval-flow.svc:9000" + - name: minio-bucket + type: string + default: "ab-eval-reports" + workspaces: + - name: source + description: Workspace containing the cloned submissions repository + results: + - name: security-passed + description: Whether security scan passed + - name: security-mode + description: Effective security scan mode used + - name: security-findings + description: Number of security findings + - name: quality-passed + description: Whether quality review passed + - name: tests-passed + description: Overall tests passed (all checks) + steps: + - name: setup + image: registry.access.redhat.com/ubi9/python-311:9.6 + script: | + #!/usr/bin/env bash + set -euo pipefail + echo "=== TEST PHASE: Setup ===" + EVAL_ENGINE="$(params.eval-engine)" + if [ "$EVAL_ENGINE" = "mcpchecker" ]; then + echo "MCPChecker mode: skipping test phase" + echo -n "true" > "$(results.security-passed.path)" + echo -n "disabled" > "$(results.security-mode.path)" + echo -n "0" > "$(results.security-findings.path)" + echo -n "true" > "$(results.quality-passed.path)" + echo -n "true" > "$(results.tests-passed.path)" + exit 0 + fi + PIPELINE_DIR="$(workspaces.source.path)/_pipeline" + if [ -d "$PIPELINE_DIR/.git" ]; then + cd "$PIPELINE_DIR" + git fetch origin "$(params.pipeline-repo-revision)" 2>/dev/null || true + git reset --hard FETCH_HEAD 2>/dev/null || true + else + git clone --depth 1 --branch "$(params.pipeline-repo-revision)" \ + "$(params.pipeline-repo-url)" "$PIPELINE_DIR" + fi + echo "Pipeline repo ready" + echo -n "true" > "$(results.security-passed.path)" + echo -n "warn" > "$(results.security-mode.path)" + echo -n "0" > "$(results.security-findings.path)" + echo -n "true" > "$(results.quality-passed.path)" + echo -n "true" > "$(results.tests-passed.path)" + + - name: security-scan + image: registry.access.redhat.com/ubi9/python-311:9.6 + env: + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: root-user + optional: true + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: root-password + optional: true + - name: DB_URL + valueFrom: + secretKeyRef: + name: ab-eval-db-credentials + key: database-url + optional: true + - name: LLM_API_KEY + value: "$(params.llm-api-key)" + script: | + #!/usr/bin/env bash + set -euo pipefail + echo "=== TEST PHASE: Security Scan ===" + EVAL_ENGINE="$(params.eval-engine)" + if [ "$EVAL_ENGINE" = "mcpchecker" ]; then + echo "MCPChecker mode: security scan skipped" + exit 0 + fi + PIPELINE_MODE="$(params.security-scan-mode)" + SUBMISSION_MODE="$(params.submission-security-scan)" + SCAN_MODE="${PIPELINE_MODE:-$SUBMISSION_MODE}" + echo -n "$SCAN_MODE" > "$(results.security-mode.path)" + if [ "$SCAN_MODE" = "disabled" ]; then + echo "Security scanning disabled" + echo -n "true" > "$(results.security-passed.path)" + echo -n "0" > "$(results.security-findings.path)" + exit 0 + fi + SUBMISSION_PATH="$(workspaces.source.path)/submissions/$(params.submission-dir)" + SUBMISSION_NAME="$(params.submission-name)" + REPORT_DIR="$(workspaces.source.path)/reports/$SUBMISSION_NAME" + JSON_PATH="$REPORT_DIR/security-scan.json" + SARIF_PATH="$REPORT_DIR/security-scan.sarif" + mkdir -p "$REPORT_DIR" + if [ -d "$SUBMISSION_PATH/skills" ] && [ -f "$SUBMISSION_PATH/skills/SKILL.md" ]; then + SUBMISSION_PATH="$SUBMISSION_PATH/skills" + fi + pip install --quiet --no-cache-dir 'cisco-ai-skill-scanner==2.0.11' 2>&1 | tail -3 + SCANNER_CMD=(skill-scanner scan "$SUBMISSION_PATH" --format json --output-json "$JSON_PATH" --output-sarif "$SARIF_PATH" --lenient --verbose) + if [ "$SCAN_MODE" = "block" ]; then + SCANNER_CMD+=(--fail-on-severity high) + fi + USE_LLM="$(params.security-scan-use-llm)" + if [ "$USE_LLM" = "true" ]; then + SCANNER_CMD+=(--use-llm) + export SKILL_SCANNER_LLM_MODEL="openai/$(params.llm-model)" + export SKILL_SCANNER_LLM_API_KEY="${LLM_API_KEY:-}" + export OPENAI_API_KEY="${LLM_API_KEY:-}" + export OPENAI_BASE_URL="$(params.llm-base-url)" + fi + set +e + "${SCANNER_CMD[@]}" 2>&1 | tee /tmp/scan-output.txt + SCAN_EXIT=$? + set -e + FINDINGS_COUNT=0 + PASSED="true" + if [ -f "$JSON_PATH" ]; then + FINDINGS_COUNT=$(python3 -c "import json; print(len(json.load(open('$JSON_PATH')).get('findings', [])))" 2>/dev/null || echo "0") + else + PASSED="false" + fi + if [ "$SCAN_MODE" = "block" ] && [ "$SCAN_EXIT" -ne 0 ]; then + PASSED="false" + fi + echo -n "$PASSED" > "$(results.security-passed.path)" + echo -n "$FINDINGS_COUNT" > "$(results.security-findings.path)" + echo "Security scan: $FINDINGS_COUNT findings, Passed: $PASSED" + + # Optional MinIO persistence + if [ -n "${MINIO_ACCESS_KEY:-}" ] && [ -n "${MINIO_SECRET_KEY:-}" ]; then + pip install --quiet --no-cache-dir minio 2>&1 | tail -1 + REPORT_PREFIX="$(params.report-prefix)" + python3 - "$JSON_PATH" "$SARIF_PATH" "$REPORT_PREFIX" "$(params.minio-endpoint)" "$(params.minio-bucket)" <<'UPLOAD' + import os, sys + from minio import Minio + json_path, sarif_path, prefix, endpoint, bucket = sys.argv[1:6] + client = Minio(endpoint, access_key=os.environ["MINIO_ACCESS_KEY"], secret_key=os.environ["MINIO_SECRET_KEY"], secure=False) + for path, name in [(json_path, "security-scan.json"), (sarif_path, "security-scan.sarif")]: + if os.path.isfile(path): + obj = f"{prefix}/security_scans/{name}" + client.fput_object(bucket, obj, path) + print(f"Uploaded: {obj}") + UPLOAD + else + echo "MinIO credentials not available, skipping artifact upload" + fi + + - name: quality-review + image: registry.access.redhat.com/ubi9/python-311:9.6 + env: + - name: LLM_API_KEY + value: "$(params.llm-api-key)" + script: | + #!/usr/bin/env bash + set -euo pipefail + echo "=== TEST PHASE: Quality Review ===" + EVAL_ENGINE="$(params.eval-engine)" + ENABLE_REVIEW="$(params.enable-quality-review)" + SUBMISSION_SKIP="$(params.submission-skip-quality-review)" + if [ "$EVAL_ENGINE" = "mcpchecker" ]; then + echo "MCPChecker mode: quality review skipped" + exit 0 + fi + if [ "$ENABLE_REVIEW" != "true" ] || [ "$SUBMISSION_SKIP" = "true" ]; then + echo "Quality review disabled" + echo -n "true" > "$(results.quality-passed.path)" + exit 0 + fi + PIPELINE_DIR="$(workspaces.source.path)/_pipeline" + SUBMISSION_PATH="$(workspaces.source.path)/submissions/$(params.submission-dir)" + cd "$PIPELINE_DIR" + pip install --quiet --no-cache-dir pydantic pyyaml openai + export PYTHONPATH="$PIPELINE_DIR" + export LLM_BASE_URL="$(params.llm-base-url)" + export LLM_MODEL="$(params.llm-model)" + set +e + python scripts/test_quality_review.py "$SUBMISSION_PATH" | tee /tmp/review-output.json + set -e + cp /tmp/review-output.json "$(workspaces.source.path)/_ai_review.json" 2>/dev/null || true + PASSED=$(python3 -c "import json; print(str(json.load(open('/tmp/review-output.json')).get('passed', False)).lower())" 2>/dev/null || echo "true") + echo -n "$PASSED" > "$(results.quality-passed.path)" + echo "Quality review: passed=$PASSED" + + - name: finalize + image: registry.access.redhat.com/ubi9/python-311:9.6 + script: | + #!/usr/bin/env bash + set -euo pipefail + echo "=== TEST PHASE: Finalize ===" + SECURITY_PASSED=$(cat "$(results.security-passed.path)") + QUALITY_PASSED=$(cat "$(results.quality-passed.path)") + if [ "$SECURITY_PASSED" = "true" ] && [ "$QUALITY_PASSED" = "true" ]; then + echo -n "true" > "$(results.tests-passed.path)" + echo "All tests PASSED" + else + echo -n "false" > "$(results.tests-passed.path)" + echo "Tests FAILED (security=$SECURITY_PASSED, quality=$QUALITY_PASSED)" + fi