ran/.github/workflows/e2e.yml at main · chaxus/ran · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
name: E2E

on:
  push:
    branches: [main]
    paths:
      - "packages/docs/**"
      - "packages/ranui/**"
      - "plugins/ran/**"
      - "design/**"
      - ".agents/plugins/**"
      - "packages/ranuts/**"
      - "packages/ranview/**"
      - "packages/im/**"
      # The docs build is ranpress; the ranui and ranview builds run
      # bin/prepare-declarations.ts; every package extends tsconfig.base.json.
      - "packages/ranpress/**"
      - "bin/**"
      - "tsconfig.base.json"
      - ".npmrc"
      - "pnpm-lock.yaml"
      - "pnpm-workspace.yaml"
      - "package.json"
      - ".github/workflows/e2e.yml"
  pull_request:
    branches: [main]
    paths:
      - "packages/docs/**"
      - "packages/ranui/**"
      - "plugins/ran/**"
      - "design/**"
      - ".agents/plugins/**"
      - "packages/ranuts/**"
      - "packages/ranview/**"
      - "packages/im/**"
      # The docs build is ranpress; the ranui and ranview builds run
      # bin/prepare-declarations.ts; every package extends tsconfig.base.json.
      - "packages/ranpress/**"
      - "bin/**"
      - "tsconfig.base.json"
      - ".npmrc"
      - "pnpm-lock.yaml"
      - "pnpm-workspace.yaml"
      - "package.json"
      - ".github/workflows/e2e.yml"

permissions:
  contents: read

# E2E is the longest job in the repository: three browsers, each rebuilding ranuts
# and ranui. A superseded run has nothing left to tell us, so a new commit on the
# same ref cancels it instead of letting both finish.
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  e2e:
    if: github.repository == 'chaxus/ran'
    timeout-minutes: 30
    runs-on: ubuntu-latest
    # The image carries the browsers and their system libraries for the pinned Playwright
    # (pnpm-workspace.yaml), so nothing is fetched from the distribution's mirror: that
    # step used to stall for the whole job timeout when the mirror slowed down.
    container:
      image: mcr.microsoft.com/playwright:v1.62.1-noble
    # Firefox refuses to start when HOME belongs to another user than the process. The
    # runner sets its own HOME for the container, so it is set here rather than in its options.
    env:
      HOME: /root
    name: "E2E (${{ matrix.browser }}, ${{ matrix.suite }})"
    # One job per browser used to run the four suites back to back: about 15 minutes, of which
    # 1.5 was setup. Split by suite, with ranui's own (the longest) in two shards, each job is
    # a few minutes and the run is as long as its slowest one.
    strategy:
      fail-fast: false
      matrix:
        browser: [chromium, firefox, webkit]
        suite: [ranui-1, ranui-2, docs, im]
    steps:
      # Shallow clone on purpose. This repository's history is hundreds of megabytes, and
      # nothing in this job reads it: the specs, both Playwright configs and the ranui
      # build all work off the working tree.
      - uses: actions/checkout@v7

      # The checkout belongs to the runner's user and the container runs as root, so git
      # refuses the repository ("dubious ownership") until it is marked safe.
      - name: Trust the checkout inside the container
        run: git config --global --add safe.directory "$GITHUB_WORKSPACE"

      - uses: pnpm/action-setup@v6
        name: Install pnpm

      - name: Setup Node.js
        uses: actions/setup-node@v7
        with:
          node-version-file: .nvmrc
          cache: pnpm

      - name: Install dependencies
        run: pnpm install --frozen-lockfile

      - name: Build ranuts
        run: pnpm --filter ranuts run build

      - name: Verify agent-evaluation browser oracles (no model calls)
        if: matrix.browser == 'chromium' && matrix.suite == 'docs'
        run: pnpm verify:agent-evals

      - name: Build ranui
        run: pnpm --filter ranui run build

      - name: Build documentation preview integration
        if: matrix.suite == 'docs'
        run: pnpm --filter docs run build

      - name: Verify standalone preview compatibility in the built documentation site
        if: matrix.suite == 'docs'
        run: pnpm --filter docs run test:preview --project=${{ matrix.browser }}

      - name: Exercise compaction cancellation without retries
        if: matrix.suite == 'im'
        run: pnpm --filter im exec playwright test test/e2e/compaction-cancel.spec.ts --project=${{ matrix.browser }} --repeat-each=5 --retries=0

      # Engine assertions remain independent from the canonical screenshot gate below.
      - name: Run E2E tests
        if: startsWith(matrix.suite, 'ranui-')
        run: pnpm --filter ranui exec playwright test --project=${{ matrix.browser }} --shard=${{ matrix.suite == 'ranui-1' && '1/2' || '2/2' }}

      # im is the assembled application, and its defects have all been integration-level —
      # what the browser draws once the pieces are wired together, which no unit test sees.
      # The suite runs keyless against the built-in sample answer, so it needs no secret.
      - name: Run im E2E tests
        if: matrix.suite == 'im'
        run: pnpm --filter im exec playwright test --project=${{ matrix.browser }} --grep-invert @soak

      - name: Upload Playwright report
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: playwright-report-${{ matrix.browser }}-${{ matrix.suite }}
          path: |
            packages/ranui/playwright-report/
            packages/ranui/test-results/
            packages/im/test-results/
            .temp/docs-preview-results/
          retention-days: 7

  visual:
    if: github.repository == 'chaxus/ran'
    name: Visual (Linux arm64)
    runs-on: ubuntu-24.04-arm
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@v7
      - uses: actions/setup-node@v7
        with:
          node-version-file: .nvmrc
      # BuildKit's gha cache needs buildx and the runtime token the cache API uses.
      - uses: docker/setup-buildx-action@v4
      - uses: crazy-max/ghaction-github-runtime@v4
      # The launcher installs/builds the three required workspaces inside the same
      # digest-pinned renderer used for local baseline updates. Missing PNGs fail.
      - name: Compare reviewed screenshot baselines
        run: node packages/ranui/bin/visual.ts check
        env:
          RANUI_VISUAL_CACHE: gha
      - name: Upload visual differences and environment
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: visual-linux-arm64
          path: packages/ranui/test-results/visual/
          retention-days: 7