Repository navigation
The tool-contract guard sees a gitlink rollback, with the cells tested (refresh of #1090) #8090
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Tests | |
| on: | |
| push: | |
| branches: [ v1.0.x ] | |
| # No branches filter on pull_request: stacked PRs (base = another feature | |
| # branch) previously got ZERO test runs — every number on them was | |
| # single-sourced to the author's local sweep, and the 2026-08-22 threading | |
| # train shipped two real failures that only surfaced once CI was dispatched | |
| # by hand. Merge-to-main guards (base-freshness, version-guard) stay scoped | |
| # to main; the test tiers run for every PR regardless of base. | |
| pull_request: | |
| # The merge queue on main runs the required check (Test & Coverage) on the | |
| # queue's own commit, which is main plus the PRs ahead of it. Without this | |
| # trigger a queued PR never gets that check and the queue stalls. The | |
| # concurrency group below already falls back to github.sha for this event, | |
| # so each queue commit gets its own group. | |
| merge_group: | |
| # Coverage is collected on this nightly schedule only — not on pull_request | |
| # or merge_group, where --coverage added wall time to every required-check | |
| # run for a report nobody reads on the PR. 08:00 UTC ~= 01:00 PT, off-peak. | |
| schedule: | |
| - cron: '0 8 * * *' | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| # The push fallback is github.sha, NOT github.ref. On a push event `github.ref` | |
| # is the branch, so concurrent commits would cancel each other. This caused | |
| # 18 of 39 Tests runs on main to be cancelled (2026-08-04 → 08-06). | |
| # | |
| # The loss is ordered by job depth, not random, which is what makes it worth | |
| # fixing rather than tolerating: | |
| # | |
| # Chart Lint no needs, ~8s survived 12 of 14 | |
| # old Test & Coverage no needs, ~5.5min survived 3 of 14 | |
| # Service Tests formerly needs:test survived 0 of 14 | |
| # | |
| # Service Tests is the ADR-009 Tier 1 real-DB tier — the deepest job, and the | |
| # only one that never once completed in a cancelled run. So the tier main was | |
| # silently skipping is the tier hardest to reproduce anywhere else. | |
| # | |
| # Per-SHA grouping lets every commit finish; PR runs still supersede correctly | |
| # because pull_request.number short-circuits before the fallback is read. | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} | |
| cancel-in-progress: true | |
| jobs: | |
| backend-tests: | |
| name: Backend tests (${{ matrix.shard }}/3) | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| shard: [1, 2, 3] | |
| runs-on: ubuntu-latest | |
| env: | |
| JWT_SECRET: test-jwt-secret | |
| SENDGRID_API_KEY: fake-api-key | |
| SENDGRID_FROM_EMAIL: test@example.com | |
| FRONTEND_URL: http://localhost:3000 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: '22' | |
| cache: npm | |
| cache-dependency-path: backend/package-lock.json | |
| - run: cd backend && npm ci --include=dev | |
| - name: Run backend tests (coverage on the nightly schedule only) | |
| run: | | |
| cd backend | |
| if [ "${{ github.event_name }}" = "schedule" ]; then | |
| npx jest --coverage --forceExit --shard=${{ matrix.shard }}/3 | |
| else | |
| npx jest --forceExit --shard=${{ matrix.shard }}/3 | |
| fi | |
| - uses: actions/upload-artifact@v4 | |
| if: github.event_name == 'schedule' | |
| with: | |
| name: backend-coverage-${{ matrix.shard }} | |
| path: backend/coverage/ | |
| backend-checks: | |
| name: Backend typecheck and lint | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: '22' | |
| cache: npm | |
| cache-dependency-path: backend/package-lock.json | |
| - run: cd backend && npm ci --include=dev | |
| - run: cd backend && npm run tsc:check | |
| - run: cd packages/types && ../../backend/node_modules/.bin/tsc --noEmit -p tsconfig.json | |
| - run: cd backend && npm run lint:ts | |
| frontend-tests: | |
| name: Frontend tests | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: '22' | |
| cache: npm | |
| cache-dependency-path: | | |
| backend/package-lock.json | |
| frontend/package-lock.json | |
| # Frontend typecheck follows shared imports into backend TypeScript. | |
| - run: cd backend && npm ci --include=dev | |
| - run: cd frontend && npm install --legacy-peer-deps | |
| - run: cd frontend && npm run typecheck | |
| - name: Run frontend tests (coverage on the nightly schedule only) | |
| run: | | |
| cd frontend | |
| if [ "${{ github.event_name }}" = "schedule" ]; then | |
| npx jest --coverage --watchAll=false --forceExit | |
| else | |
| npx jest --watchAll=false --forceExit | |
| fi | |
| - run: cd frontend && node --test scripts/generate-seo-pages.test.mjs scripts/nginx-routing.test.mjs | |
| tool-tests: | |
| name: CLI, MCP and hosted runtime tests | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| # The tool-contract guard compares the submodule pin against the | |
| # merge base with origin/main to catch a ROLLBACK, which containment | |
| # structurally cannot see. At the default depth 1 there is no | |
| # origin/main and no shared history, so that check could not run at | |
| # all (#2497). The guard now says so loudly instead of passing, and | |
| # this is what lets it answer. | |
| fetch-depth: 0 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: '22' | |
| cache: npm | |
| cache-dependency-path: | | |
| package-lock.json | |
| backend/package-lock.json | |
| commonly-mcp/package-lock.json | |
| cli/package-lock.json | |
| workers/agent-runtime/package-lock.json | |
| - run: npm ci --include=dev | |
| # CLI tests load backend TypeScript through ts-node. | |
| - run: cd backend && npm ci --include=dev | |
| - run: cd commonly-mcp && npm ci --include=dev | |
| - run: cd cli && npm ci --include=dev | |
| - run: cd workers/agent-runtime && npm ci --include=dev | |
| - run: npm run verify:moltbot-tools | |
| - run: npm run verify:litellm-patch-runner | |
| # These node:test files lived only in the retired all-in-one job. | |
| - name: Run agent service tests | |
| run: | | |
| node --test \ | |
| external/commonly-agent-services/shared/delivery-nonce.test.js \ | |
| external/commonly-agent-services/shared/summary-text.test.js \ | |
| external/commonly-agent-services/commonly-bot/index.test.js | |
| - run: cd commonly-mcp && npm test | |
| - run: cd cli && npm run lint && npm test | |
| - run: cd workers/agent-runtime && npm run typecheck && npm test | |
| test-gate: | |
| name: Test & Coverage | |
| if: always() | |
| needs: [backend-tests, backend-checks, frontend-tests, tool-tests, service-test, chart-lint] | |
| runs-on: ubuntu-latest | |
| env: | |
| NEEDS_JSON: ${{ toJSON(needs) }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Verify gate covers every suite | |
| run: | | |
| diff -u \ | |
| <(awk '/^jobs:$/ { jobs=1; next } jobs && /^ [[:alnum:]_-]+:$/ { id=$1; sub(/:$/, "", id); if (id != "test-gate") print id }' .github/workflows/tests.yml | sort) \ | |
| <(printf '%s\n' "$NEEDS_JSON" | jq -r 'keys[]' | sort) | |
| - name: Verify every check passed | |
| run: | | |
| # Suite jobs run unconditionally; a skipped result is a failed gate. | |
| # Revisit this predicate if a later PR adds path-based job skipping. | |
| echo "$NEEDS_JSON" | jq -e ' | |
| [.[].result] | all(. == "success") | |
| ' | |
| service-test: | |
| name: Service Tests (Tier 1 — real DBs) | |
| runs-on: ubuntu-latest | |
| services: | |
| mongo: | |
| image: mongo:7 | |
| ports: ["27017:27017"] | |
| options: >- | |
| --health-cmd "mongosh --eval 'db.runCommand(\"ping\").ok'" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| postgres: | |
| image: postgres:16 | |
| ports: ["5432:5432"] | |
| env: | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: commonly-test | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| env: | |
| JWT_SECRET: test-jwt-secret | |
| SENDGRID_API_KEY: fake-api-key | |
| SENDGRID_FROM_EMAIL: test@example.com | |
| FRONTEND_URL: http://localhost:3000 | |
| INTEGRATION_TEST: "true" | |
| MONGO_URI: mongodb://localhost:27017/commonly-test | |
| PG_HOST: localhost | |
| PG_PORT: "5432" | |
| PG_DATABASE: commonly-test | |
| PG_USER: postgres | |
| PG_PASSWORD: postgres | |
| PG_SSL_ENABLED: "false" | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '22' | |
| cache: 'npm' | |
| cache-dependency-path: '**/package-lock.json' | |
| - name: Install backend dependencies | |
| run: cd backend && npm ci --include=dev | |
| - name: Run service tests (Tier 1) | |
| run: | | |
| cd backend | |
| npx jest --testPathPattern="__tests__/service" --forceExit --runInBand | |
| chart-lint: | |
| name: Chart Lint (Tier 1.5) | |
| runs-on: ubuntu-latest | |
| env: | |
| # Pin CRD schemas to a known commit to avoid drift. | |
| # Update manually by running: gh api /repos/datreeio/CRDs-catalog/commits/main --jq .sha | |
| CRDS_CATALOG_SHA: 15e9b72701ce617b0ac15da611a7fb2e54b980dd | |
| KUBECONFORM_VERSION: v0.7.0 | |
| YQ_VERSION: v4.44.3 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| - name: Install Helm | |
| uses: azure/setup-helm@v4 | |
| with: | |
| version: v3.14.0 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: '22' | |
| cache: npm | |
| cache-dependency-path: package-lock.json | |
| - run: npm ci --ignore-scripts --include=dev | |
| - name: Install kubeconform | |
| run: | | |
| curl -sSL "https://github.com/yannh/kubeconform/releases/download/${KUBECONFORM_VERSION}/kubeconform-linux-amd64.tar.gz" -o kc.tgz | |
| sudo tar -xzf kc.tgz -C /usr/local/bin/ kubeconform | |
| kubeconform -v | |
| - name: Install yq | |
| run: | | |
| curl -sSL "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64" -o yq | |
| chmod +x yq | |
| sudo mv yq /usr/local/bin/yq | |
| yq --version | |
| - name: Render dev chart | |
| run: | | |
| helm template commonly-dev k8s/helm/commonly \ | |
| -f k8s/helm/commonly/values.yaml \ | |
| -f k8s/helm/commonly/values-dev.yaml \ | |
| > /tmp/dev-rendered.yaml | |
| - name: Lint rendered dev chart | |
| run: | | |
| kubeconform --strict --summary -output text \ | |
| --schema-location "https://raw.githubusercontent.com/datreeio/CRDs-catalog/${CRDS_CATALOG_SHA}/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json" \ | |
| --schema-location default \ | |
| /tmp/dev-rendered.yaml | |
| # The dev render has kubeconform coverage, but staging is otherwise first | |
| # rendered at deploy time. Parse both values combinations before merge. | |
| - name: Strictly parse dev and staging chart renders | |
| run: | | |
| set -o pipefail | |
| for environment in dev staging; do | |
| error_file="/tmp/${environment}-helm.err" | |
| if ! helm template "commonly-${environment}" k8s/helm/commonly \ | |
| -f k8s/helm/commonly/values.yaml \ | |
| -f "k8s/helm/commonly/values-${environment}.yaml" \ | |
| 2>"$error_file" | node -e ' | |
| const yaml = require("js-yaml"); | |
| let rendered = ""; | |
| process.stdin.setEncoding("utf8"); | |
| process.stdin.on("data", chunk => { rendered += chunk; }); | |
| process.stdin.on("end", () => { | |
| try { | |
| let objects = 0; | |
| yaml.loadAll(rendered, object => { if (object !== undefined) objects++; }); | |
| if (objects === 0) throw new Error("Helm rendered no objects"); | |
| console.log(`Parsed ${objects} objects`); | |
| } catch (error) { | |
| console.error(error.reason || error.message); | |
| if (error.mark?.snippet) console.error(error.mark.snippet); | |
| process.exitCode = 1; | |
| } | |
| }); | |
| '; then | |
| cat "$error_file" | |
| exit 1 | |
| fi | |
| cat "$error_file" | |
| done | |
| - name: Validate alerting config (offline) | |
| # k8s/monitoring/ is applied by hand with scripts/monitoring/apply-alerting.py | |
| # (docs/runbooks/alerting.md). --check makes no API call: it asserts every | |
| # policy links a runbook heading, every metric/uptime reference resolves, | |
| # and no project or account identifier is committed. | |
| run: python3 scripts/monitoring/apply-alerting.py --check | |
| - name: Assert reliability rollout and scheduling invariants | |
| run: | | |
| test "$(yq 'select(.kind == "Deployment" and .metadata.name == "backend") | .spec.strategy.type' /tmp/dev-rendered.yaml)" = "RollingUpdate" | |
| test "$(yq 'select(.kind == "Deployment" and .metadata.name == "backend") | .spec.strategy.rollingUpdate.maxUnavailable' /tmp/dev-rendered.yaml)" = "0" | |
| test "$(yq 'select(.kind == "Deployment" and .metadata.name == "backend") | .spec.strategy.rollingUpdate.maxSurge' /tmp/dev-rendered.yaml)" = "1" | |
| test "$(yq 'select(.kind == "Deployment" and .metadata.name == "backend") | .spec.template.spec.volumes | map(select(has("persistentVolumeClaim"))) | length' /tmp/dev-rendered.yaml)" = "0" | |
| test "$(yq 'select(.kind == "Deployment" and .metadata.name == "redis") | .spec.template.spec.nodeSelector.pool' /tmp/dev-rendered.yaml)" = "dev" | |
| test "$(yq 'select(.kind == "Deployment" and .metadata.name == "redis") | .spec.template.spec.tolerations | map(select(.key == "workload-tier")) | length' /tmp/dev-rendered.yaml)" = "0" | |
| test "$(yq 'select(.kind == "Deployment" and .metadata.name == "redis") | .spec.template.spec.tolerations | map(select(.key == "pool" and .value == "dev" and .effect == "NoSchedule")) | length' /tmp/dev-rendered.yaml)" = "1" | |
| test "$(yq 'select(.kind == "Deployment" and .metadata.name == "backend") | .spec.minReadySeconds' /tmp/dev-rendered.yaml)" = "10" | |
| test "$(yq 'select(.kind == "Deployment" and .metadata.name == "backend") | .spec.template.spec.terminationGracePeriodSeconds' /tmp/dev-rendered.yaml)" = "30" | |
| test "$(yq 'select(.kind == "Deployment" and .metadata.name == "backend") | .spec.template.spec.containers[] | select(.name == "backend") | .readinessProbe.httpGet.path' /tmp/dev-rendered.yaml)" = "/api/health/ready" | |
| test "$(yq 'select(.kind == "Deployment" and .metadata.name == "backend") | .spec.template.spec.containers[] | select(.name == "backend") | .lifecycle.preStop.exec.command[2]' /tmp/dev-rendered.yaml)" = "sleep 10" | |
| # Two frontend replicas never share a node, so a spot reclaim leaves one serving; with the spot | |
| # pool capped at 2 nodes, a rollout must replace in place (maxSurge 0) or it deadlocks (2026-10-19). | |
| test "$(yq 'select(.kind == "Deployment" and .metadata.name == "frontend") | .spec.replicas' /tmp/dev-rendered.yaml)" = "2" | |
| test "$(yq 'select(.kind == "Deployment" and .metadata.name == "frontend") | .spec.strategy.rollingUpdate.maxSurge' /tmp/dev-rendered.yaml)" = "0" | |
| test "$(yq 'select(.kind == "Deployment" and .metadata.name == "frontend") | .spec.strategy.rollingUpdate.maxUnavailable' /tmp/dev-rendered.yaml)" = "1" | |
| test "$(yq 'select(.kind == "Deployment" and .metadata.name == "frontend") | .spec.template.spec.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[0].topologyKey' /tmp/dev-rendered.yaml)" = "kubernetes.io/hostname" | |
| test "$(yq 'select(.kind == "Deployment" and .metadata.name == "frontend") | .spec.template.spec.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[0].labelSelector.matchLabels.app' /tmp/dev-rendered.yaml)" = "frontend" | |
| # A frontend pod keeps serving until ingress-nginx has dropped it, or a request waits out the 5 s connect timeout. | |
| test "$(yq 'select(.kind == "Deployment" and .metadata.name == "frontend") | .spec.template.spec.containers[] | select(.name == "frontend") | .lifecycle.preStop.exec.command[2]' /tmp/dev-rendered.yaml)" = "sleep 10" | |
| # LiteLLM is public for its /v1 API only; `/` exposed the master-key admin login (2026-10-07). | |
| test "$(yq 'select(.kind == "Ingress") | .spec.rules[] | select(.host == "litellm.commonly.me") | .http.paths | map(.path) | join(",")' /tmp/dev-rendered.yaml)" = "/v1" | |
| # LiteLLM reads its config only at boot, so its pod must roll when the config changes. | |
| test -n "$(yq -N 'select(.kind == "Deployment" and .metadata.name == "litellm") | .spec.template.metadata.annotations."checksum/config" // ""' /tmp/dev-rendered.yaml)" | |
| # Nightly DB backups (2026-10-14): both jobs render, run on spot-pool (the only | |
| # pool with the GKE metadata server, so the only one where backup-sa's | |
| # Workload Identity works), and dump the same Postgres the backend uses. | |
| test "$(yq -N 'select(.kind == "CronJob" and .metadata.labels.component == "backup") | .metadata.name' /tmp/dev-rendered.yaml | sort | paste -sd, -)" = "mongodb-backup,postgres-backup" | |
| test "$(yq -N 'select(.kind == "CronJob" and .metadata.labels.component == "backup") | .spec.jobTemplate.spec.template.spec.nodeSelector."workload-tier"' /tmp/dev-rendered.yaml | sort -u)" = "spot" | |
| for pair in PG_HOST:PG_HOST PG_PORT:PGPORT PG_USER:PG_USER PG_DATABASE:PG_DATABASE; do | |
| backend=$(B="${pair%%:*}" yq -N 'select(.kind == "Deployment" and .metadata.name == "backend") | .spec.template.spec.containers[] | select(.name == "backend") | .env[] | select(.name == strenv(B)) | .value' /tmp/dev-rendered.yaml) | |
| dump=$(D="${pair##*:}" yq -N 'select(.kind == "CronJob" and .metadata.name == "postgres-backup") | .spec.jobTemplate.spec.template.spec.initContainers[0].env[] | select(.name == strenv(D)) | .value' /tmp/dev-rendered.yaml) | |
| test -n "$backend" && test "$backend" = "$dump" | |
| done | |
| # Stranger journey (2026-10-15): renders, and never retries (a retry would sign | |
| # up a second account and could hide a flaky step behind a later pass). | |
| test "$(yq -N 'select(.kind == "CronJob" and .metadata.name == "stranger-journey") | .spec.jobTemplate.spec.backoffLimit' /tmp/dev-rendered.yaml)" = "0" | |
| - name: Render Slack OAuth secret wiring | |
| run: | | |
| helm template commonly-slack k8s/helm/commonly \ | |
| -f k8s/helm/commonly/values.yaml \ | |
| -f k8s/helm/commonly/values-dev.yaml \ | |
| --set slack.oauth.enabled=true \ | |
| > /tmp/slack-rendered.yaml | |
| for key in slack-client-id slack-client-secret slack-signing-secret connector-secret-keys connector-secret-active-key; do | |
| test "$(yq 'select(.kind == "ExternalSecret" and .metadata.name == "api-keys") | .spec.data[].secretKey' /tmp/slack-rendered.yaml | grep -Fx "$key")" = "$key" | |
| done | |
| for name in SLACK_CLIENT_ID SLACK_CLIENT_SECRET SLACK_SIGNING_SECRET CONNECTOR_SECRET_KEYS CONNECTOR_SECRET_ACTIVE_KEY; do | |
| test "$(yq 'select(.kind == "Deployment" and .metadata.name == "backend") | .spec.template.spec.containers[] | select(.name == "backend") | .env[].name' /tmp/slack-rendered.yaml | grep -Fx "$name")" = "$name" | |
| done | |
| - name: Render and lint prod chart (if values-prod.yaml exists) | |
| run: | | |
| if [ -f k8s/helm/commonly/values-prod.yaml ]; then | |
| helm template commonly-prod k8s/helm/commonly \ | |
| -f k8s/helm/commonly/values.yaml \ | |
| -f k8s/helm/commonly/values-prod.yaml \ | |
| | kubeconform --strict --summary -output text \ | |
| --schema-location "https://raw.githubusercontent.com/datreeio/CRDs-catalog/${CRDS_CATALOG_SHA}/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json" \ | |
| --schema-location default | |
| else | |
| echo "values-prod.yaml not present yet (lands in ADR-009 Phase 5); skipping prod lint." | |
| fi |