feat: add PyRIT Crescendo adaptive multi-turn to red-team task

abevalflow/schemas.py

@@ -39,6 +39,45 @@ class SecurityScanMode(StrEnum):
     BLOCK = "block"
 
 
+class RedTeamMode(StrEnum):
+    SMOKE = "smoke"
+    FULL = "full"
+
+
+class RedTeamConfig(BaseModel):
+    """Optional red-team adversarial testing configuration in metadata.yaml."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    enabled: bool = Field(
+        default=True,
+        description="Whether red-team evaluation is enabled for this submission",
+    )
+    mode: RedTeamMode | None = Field(
+        default=None,
+        description="Override pipeline red-team mode: smoke (fast) or full (comprehensive)",
+    )
+    purpose: str | None = Field(
+        default=None,
+        description="What the agent/server does; drives domain-aware attack generation",
+    )
+    auth_context: str | None = Field(
+        default=None,
+        description="Assumed authenticated user context for nuanced policy grading",
+    )
+    policy: str | None = Field(
+        default=None,
+        description="Security policy the agent must uphold under adversarial probing",
+    )
+    crescendo_objectives: list[str] | None = Field(
+        default=None,
+        description=(
+            "Optional explicit PyRIT Crescendo objectives. If omitted, objectives "
+            "are auto-derived from purpose/policy/auth_context at runtime."
+        ),
+    )
+
+
 # Import GateMode from canonical location to avoid duplication
 from abevalflow.gates.base import GateMode  # noqa: E402
 
@@ -579,6 +618,15 @@ def _validate_name(cls, v: str) -> str:
         ),
     )
 
+    red_team: RedTeamConfig | None = Field(
+        default=None,
+        description=(
+            "Optional red-team adversarial testing configuration. "
+            "When present, drives Promptfoo attack generation and PyRIT "
+            "Crescendo objectives (purpose, policy, auth_context, crescendo_objectives)."
+        ),
+    )
+
     gate_policy: GatePolicy | None = Field(
         default=None,
         description=(

pipeline/images/pyrit/Containerfile

@@ -0,0 +1,22 @@
+FROM python:3.12-slim
+
+WORKDIR /app
+
+# OpenShift-friendly: writable dirs for arbitrary UID
+ENV HOME=/tmp \
+    PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1
+
+# Adaptive Crescendo loop uses httpx + LiteLLM HTTP — not the full PyRIT package
+# (native CrescendoOrchestrator had API issues in the POC; we keep a lean runtime).
+RUN pip install --no-cache-dir \
+        "httpx>=0.27.0" \
+        "PyYAML>=6.0" \
+    && mkdir -p /app/results /tmp \
+    && chmod -R g+rwX /app /tmp
+
+COPY scripts/pyrit_crescendo/ /app/pyrit_crescendo/
+
+WORKDIR /app/pyrit_crescendo
+
+ENTRYPOINT ["python", "run_crescendo.py"]

pipeline/integration/konflux-eval-pipelinerun.yaml

@@ -110,6 +110,10 @@ spec:
         type: string
         default: "12"
         description: Number of parallel Promptfoo evaluations
+      - name: RED_TEAM_CRESCENDO_MAX_TURNS
+        type: string
+        default: "7"
+        description: Max turns per PyRIT Crescendo objective (full mode only)
 
       # === Pipeline repo (for scripts) ===
       - name: PIPELINE_REPO_URL
@@ -265,6 +269,8 @@ spec:
             value: $(params.PIPELINE_REPO_REVISION)
           - name: concurrency
             value: $(params.RED_TEAM_CONCURRENCY)
+          - name: crescendo-max-turns
+            value: $(params.RED_TEAM_CRESCENDO_MAX_TURNS)
         workspaces:
           - name: source
             workspace: shared-workspace

pipeline/pipelines/ci-pipeline.yaml

@@ -82,6 +82,24 @@ spec:
       type: string
       default: "true"
       description: Enable unified scorecard aggregation
+    - name: enable-red-team
+      type: string
+      default: "false"
+      description: Enable red team adversarial evaluation (Promptfoo + PyRIT Crescendo in full mode)
+    - name: red-team-mode
+      type: string
+      default: "full"
+      description: >-
+        "smoke" generates a quick Promptfoo suite (~25 tests). "full" runs
+        comprehensive Promptfoo then PyRIT Crescendo.
+    - name: red-team-concurrency
+      type: string
+      default: "12"
+      description: Number of parallel Promptfoo evaluations
+    - name: red-team-crescendo-max-turns
+      type: string
+      default: "7"
+      description: Max turns per PyRIT Crescendo objective (full mode only)
     - name: agent-type
       type: string
       default: "api"
@@ -300,11 +318,53 @@ spec:
         - name: source
           workspace: shared-workspace
 
+    # ======================================================================
+    # PHASE 2.5: RED TEAM (adversarial testing, optional)
+    # ======================================================================
+    - name: red-team
+      runAfter: ["test"]
+      when:
+        - input: $(params.enable-red-team)
+          operator: in
+          values: ["true"]
+      taskRef:
+        name: red-team
+      params:
+        - name: eval-engine
+          value: $(params.eval-engine)
+        - name: submission-dir
+          value: $(params.submission-dir)
+        - name: submission-name
+          value: $(tasks.prepare.results.submission-name)
+        - name: agent-endpoint
+          value: $(params.agent-endpoint)
+        - name: red-team-mode
+          value: $(params.red-team-mode)
+        - name: llm-api-base
+          value: $(params.llm-api-base)
+        - name: llm-model
+          value: $(params.llm-model)
+        - name: llm-api-key
+          value: $(params.llm-api-key)
+        - name: pipeline-run-id
+          value: $(context.pipelineRun.name)
+        - name: pipeline-repo-url
+          value: $(params.pipeline-repo-url)
+        - name: pipeline-repo-revision
+          value: $(params.pipeline-repo-revision)
+        - name: concurrency
+          value: $(params.red-team-concurrency)
+        - name: crescendo-max-turns
+          value: $(params.red-team-crescendo-max-turns)
+      workspaces:
+        - name: source
+          workspace: shared-workspace
+
     # ======================================================================
     # PHASE 3: EVALUATE (dispatches to harbor/ase/mcpchecker)
     # ======================================================================
     - name: evaluate
-      runAfter: ["test"]
+      runAfter: ["red-team", "test"]
       timeout: "3h"
       taskRef:
         name: evaluate

pipeline/tasks/konflux/red-team.yaml

@@ -7,11 +7,12 @@ metadata:
     app.kubernetes.io/component: konflux
 spec:
   description: >-
-    Generic red team evaluation using Promptfoo. Generates domain-aware adversarial
-    attacks based on submission metadata and scores responses with LLM-as-judge.
-    Supports A2A agents (JSON-RPC), MCP servers, and generic HTTP endpoints.
-    Two modes: "smoke" generates a quick suite (~25 tests, basic strategy);
-    "full" produces comprehensive attacks with all strategies (~1750 tests).
+    Generic red team evaluation using Promptfoo (broad coverage) plus PyRIT
+    Crescendo (adaptive multi-turn) in full mode. Generates domain-aware
+    adversarial attacks from submission metadata and scores responses with
+    LLM-as-judge. Supports A2A agents (JSON-RPC) and generic HTTP endpoints.
+    Two modes: "smoke" (~25 Promptfoo tests, basic strategy, no Crescendo);
+    "full" (comprehensive Promptfoo + PyRIT Crescendo after Promptfoo).
   params:
     - name: eval-engine
       type: string
@@ -30,8 +31,8 @@ spec:
       type: string
       default: "full"
       description: >-
-        "smoke" generates a quick suite from metadata (~25 tests, basic strategy, ~2 min).
-        "full" generates comprehensive attacks with all strategies (~1750 tests, ~90 min).
+        "smoke" generates a quick Promptfoo suite (~25 tests, basic strategy, ~2 min).
+        "full" runs comprehensive Promptfoo then PyRIT Crescendo (~90+ min).
     - name: llm-api-base
       type: string
       default: ""
@@ -56,6 +57,10 @@ spec:
       type: string
       default: "12"
       description: Number of parallel Promptfoo test evaluations
+    - name: crescendo-max-turns
+      type: string
+      default: "7"
+      description: Max turns per PyRIT Crescendo objective (full mode only)
   workspaces:
     - name: source
       description: Shared workspace with cloned submission and pipeline repos
@@ -183,7 +188,7 @@ spec:
         CONCURRENCY="$(params.concurrency)"
 
         if [ "$MODE" = "full" ]; then
-          echo "Mode: full (comprehensive attacks, all strategies)"
+          echo "Mode: full (comprehensive attacks; Crescendo via PyRIT step)"
         else
           echo "Mode: smoke (quick coverage, basic strategy)"
         fi
@@ -222,8 +227,83 @@ spec:
           PASSED="false"
         fi
 
+        # Stash Promptfoo findings for the Crescendo merge step
+        echo -n "$FINDINGS" > "$REPORT_DIR/.promptfoo-findings"
         echo -n "$PASSED" > "$(results.redteam-passed.path)"
         echo -n "$FINDINGS" > "$(results.redteam-findings.path)"
 
         echo ""
-        echo "Red team complete: $FINDINGS findings, passed=$PASSED"
+        echo "Promptfoo complete: $FINDINGS findings, passed=$PASSED"
+
+    - name: run-crescendo
+      image: quay.io/rh-ee-ikrispin/abevalflow-pyrit:0.1
+      script: |
+        #!/usr/bin/env bash
+        set -euo pipefail
+        echo "=== RED TEAM: PyRIT Crescendo ==="
+
+        EVAL_ENGINE="$(params.eval-engine)"
+        if [ "$EVAL_ENGINE" = "ase" ]; then
+          echo "Skipped (ase)"
+          exit 0
+        fi
+
+        ENDPOINT="$(params.agent-endpoint)"
+        if [ -z "$ENDPOINT" ]; then
+          echo "Skipped (no endpoint)"
+          exit 0
+        fi
+
+        MODE="$(params.red-team-mode)"
+        if [ "$MODE" != "full" ]; then
+          echo "Skipped (Crescendo runs only in full mode; mode=$MODE)"
+          exit 0
+        fi
+
+        REPORT_DIR="$(workspaces.source.path)/reports/$(params.submission-name)"
+        SUBMISSION_PATH="$(workspaces.source.path)/submissions/$(params.submission-dir)"
+        # Prefer workspace pipeline clone (has scripts); fall back to image-baked copy
+        PIPELINE_DIR="$(workspaces.source.path)/_pipeline"
+        if [ -d "$PIPELINE_DIR/scripts/pyrit_crescendo" ]; then
+          CRESCENDO_DIR="$PIPELINE_DIR/scripts/pyrit_crescendo"
+        else
+          CRESCENDO_DIR="/app/pyrit_crescendo"
+        fi
+        mkdir -p "$REPORT_DIR"
+
+        echo "Running Crescendo from $CRESCENDO_DIR"
+        set +e
+        python "$CRESCENDO_DIR/run_crescendo.py" \
+          --endpoint "$ENDPOINT" \
+          --eval-engine "$EVAL_ENGINE" \
+          --submission-path "$SUBMISSION_PATH" \
+          --llm-api-base "$(params.llm-api-base)" \
+          --llm-model "$(params.llm-model)" \
+          --llm-api-key "$(params.llm-api-key)" \
+          --max-turns "$(params.crescendo-max-turns)" \
+          --output "$REPORT_DIR/pyrit-crescendo-results.json"
+        CRESCENDO_EXIT=$?
+        set -e
+
+        PF_FINDINGS=0
+        if [ -f "$REPORT_DIR/.promptfoo-findings" ]; then
+          PF_FINDINGS=$(cat "$REPORT_DIR/.promptfoo-findings")
+        fi
+
+        CRESCENDO_FINDINGS=0
+        if [ -f "$REPORT_DIR/pyrit-crescendo-results.json" ]; then
+          CRESCENDO_FINDINGS=$(python -c "import json;d=json.load(open('$REPORT_DIR/pyrit-crescendo-results.json'));print(int(d.get('summary',{}).get('achieved',0)))" 2>/dev/null || echo "0")
+        fi
+
+        TOTAL=$((PF_FINDINGS + CRESCENDO_FINDINGS))
+        PASSED="true"
+        if [ "$TOTAL" -gt "0" ]; then
+          PASSED="false"
+        fi
+
+        echo -n "$PASSED" > "$(results.redteam-passed.path)"
+        echo -n "$TOTAL" > "$(results.redteam-findings.path)"
+
+        echo ""
+        echo "Crescendo exit=$CRESCENDO_EXIT findings=$CRESCENDO_FINDINGS"
+        echo "Combined red-team: promptfoo=$PF_FINDINGS crescendo=$CRESCENDO_FINDINGS total=$TOTAL passed=$PASSED"