From 7c6ccaa7bbdbd53b467101b62534b9ceced01bea Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 11 Aug 2026 09:37:58 +0000 Subject: [PATCH 1/2] docs(qa): state the positive secret contract and de-flake the chart measurement (#7573, #7569) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two checklist-item corrections in `platform-core.json`, one file, nothing else. `settings-hub-roundtrip` (#7573): the secret clause only stated a NEGATIVE — "GET /api/settings never returns the plaintext". Masking, omitting the value and dropping the key all satisfy that, and two of the three are regressions the console renders "configured vs not configured" from. The clauses now mirror the ten route-level pins PR #7554 added in `settings-routes.test.ts`: set → the mask, unset → null (presence-preserving), masking across `values..value` AND every `cascadeChain` entry, `source`/`locked`/ `lockedReason` unchanged, both specifier flavours — plus the distinct echoed-mask-PUT-is-a-no-op round-trip clause and the guard that the in-process read path is deliberately NOT masked. `nav-surfaces-render` (#7569): a null chart `boundingBox` read once at a fixed instant measures the recharts ResponsiveContainer layout window, not the product (null only at t=1.5s, non-null on all 19 later samples, 5 real charts on each of 3 fresh loads). The clause now requires a bounded poll-until-non-null and the steps tell the runner to re-measure before recording a red. The harness half — replacing `waitForTimeout(1500)` + a single `boundingBox()` read in the pinned spec — is recorded in `knownGaps`; it lives outside this PR's file surface, so #7569 stays open. --- .../areas/platform-core.json | 59 ++++++++++++++----- 1 file changed, 45 insertions(+), 14 deletions(-) diff --git a/docs/qa/platform-checklist/areas/platform-core.json b/docs/qa/platform-checklist/areas/platform-core.json index 8a9ef2f1a2..12bb04e0ef 100644 --- a/docs/qa/platform-checklist/areas/platform-core.json +++ b/docs/qa/platform-checklist/areas/platform-core.json @@ -213,14 +213,15 @@ "title": "Every showcase nav surface renders without page errors, and failures surface the error boundary — never a blank page", "since": "v15", "status": "active", - "revision": 2, + "revision": 3, "priority": "P0", "surface": "browser", "personas": ["seeded admin (admin@objectos.ai / admin123)"], "fixtures": { "app": "showcase", "knownGaps": [ - "no stock showcase fixture deliberately throws inside a route, so the ErrorBoundary fallback ('Something went wrong' + Try Again / Go Home — objectui packages/app-shell/src/chrome/ErrorBoundary.tsx) is verified opportunistically on any failure encountered, plus via the bad-route probe below" + "no stock showcase fixture deliberately throws inside a route, so the ErrorBoundary fallback ('Something went wrong' + Try Again / Go Home — objectui packages/app-shell/src/chrome/ErrorBoundary.tsx) is verified opportunistically on any failure encountered, plus via the bad-route probe below", + "the pinned suite still MEASURES charts with a fixed `page.waitForTimeout(1500)` followed by a SINGLE `boundingBox()` read (examples/app-showcase/e2e/showcase-smoke.spec.ts) — a coin-flip red whenever that fixed instant lands inside the recharts ResponsiveContainer initial-layout window. Until that read is replaced by a bounded poll-until-non-null (the #7472 / #7409 de-flake shape: loop the predicate against a DEADLINE derived from the case's own budget, never sleep-then-read-once — a bigger fixed number is not the fix), a null-box chart red must be RE-MEASURED per the steps before it is recorded. Tracked by #7569, whose harness half lands in the spec file, not in this checklist" ] }, "steps": [ @@ -228,15 +229,16 @@ "GET /api/v1/meta/app?id=com.example.showcase and extract every nav destination from the response", "diff the served nav destinations against the SURFACES array; hand-walk any destination the suite does not cover (screenshot first, then DOM)", "on each hand-walked surface: wait for render, screenshot, then check for pageerror / empty
/ placeholder leaks ('no actions configured')", + "chart surfaces (the chart:true entries in SURFACES — Delivery Operations, Chart Gallery, Command Center): if the suite reports a NULL chart boundingBox, do NOT record it as a product red — RE-MEASURE it by polling boundingBox until non-null within a bounded budget. #7569 measured the difference on this exact assertion: 3 fresh loads each drew 5 real charts, and polling every 400 ms returned null ONLY at t = 1.5 s — the suite's fixed measurement instant, and exactly the recharts ResponsiveContainer initial-layout window — and non-null on all 19 later samples; the same item re-run in isolation is 4/4 green. A box that stays null through the whole bounded poll IS a product finding; one that resolves on a later sample is the harness's measurement window, not the product", "probe the failure path: navigate to a nonexistent route under the app (/_console/apps/com.example.showcase/object_that_does_not_exist); screenshot what renders", "if ANY surface fails during the sweep, verify the failure presents as the shell's error boundary or a named empty/error state — capture it" ], "acceptance": [ { - "clause": "no surface throws a pageerror, renders an empty
, or leaks a 'no actions configured' placeholder; chart surfaces draw a real SVG", + "clause": "no surface throws a pageerror, renders an empty
, or leaks a 'no actions configured' placeholder; chart surfaces draw a real SVG whose box is measured by POLLING until non-null within a bounded budget — never by a fixed wall-clock wait followed by a single boundingBox read, which measures the layout window rather than the product", "oracle": "test", - "verify": "pnpm -C examples/app-showcase test:smoke (SURFACES array in e2e/showcase-smoke.spec.ts) — green", - "evidence": "test run output" + "verify": "pnpm -C examples/app-showcase test:smoke (SURFACES array in e2e/showcase-smoke.spec.ts) — green; a chart clause that failed on a NULL box counts only after the bounded re-measure in the steps confirms it stays null (the pinned suite still reads the box once after a fixed 1500 ms wait — see knownGaps, #7569)", + "evidence": "test run output +, for any null-box chart failure, the bounded re-measure's sample series" }, { "clause": "the smoke's coverage is CURRENT: every nav destination served in meta/app is either in the SURFACES array or hand-walked this run — no surface silently outside the net", @@ -259,7 +261,8 @@ ], "negative": [ "a surface that renders a blank
is a FAIL even with no console error — transitional emptiness must be ruled out by the screenshot-first protocol, then the persistent blank is the finding", - "a bad route producing a white page (shell gone) is a FAIL of the error-boundary clause" + "a bad route producing a white page (shell gone) is a FAIL of the error-boundary clause", + "a chart red obtained from a SINGLE boundingBox read at a fixed instant is NOT a product finding and must not be filed as one — it measures the ResponsiveContainer layout window, and it costs a triage cycle every run (#7569). Re-measure with a bounded poll first. The converse still holds both ways: a box that stays null through the whole poll, or a resolved width ≤ 200 (the #2616 D collapsed-panel case), IS a FAIL" ], "automated": { "kind": "e2e", "ref": "examples/app-showcase/e2e/showcase-smoke.spec.ts" }, "traps": ["hydration-race", "single-datapoint", "stale-console-bundle"], @@ -271,7 +274,8 @@ ], "history": [ { "revision": 1, "date": "2026-08-07", "change": "initial — wraps the existing automated smoke as a checklist row so runs report it alongside manual items", "ref": "#3358" }, - { "revision": 2, "date": "2026-08-07", "change": "expanded to deep-test contract: concrete steps, multi-clause acceptance, negatives, variants", "ref": "claude/platform-test-checklist-ocwugl" } + { "revision": 2, "date": "2026-08-07", "change": "expanded to deep-test contract: concrete steps, multi-clause acceptance, negatives, variants", "ref": "claude/platform-test-checklist-ocwugl" }, + { "revision": 3, "date": "2026-08-11", "change": "de-flaked the chart clause's MEASUREMENT: a null boundingBox read once at a fixed instant is the recharts ResponsiveContainer layout window, not a product defect (measured: null only at t=1.5s, non-null on all 19 later samples, 5 real charts on each of 3 fresh loads). The clause now requires a bounded poll-until-non-null, the steps tell the runner to re-measure before recording a red, and knownGaps records that the pinned suite's fixed 1500 ms wait + single read is still there", "ref": "#7569" } ] }, { @@ -535,10 +539,10 @@ }, { "id": "platform-core.settings-hub-roundtrip", - "title": "Settings hub round-trip: a value saves, PERSISTS, reaches an observable consumer, audits, env-locks, tests honestly, and stores secrets as handles — anonymous denied", + "title": "Settings hub round-trip: a value saves, PERSISTS, reaches an observable consumer, audits, env-locks, tests honestly, and stores secrets as handles served MASKED — anonymous denied", "since": "v15", "status": "active", - "revision": 1, + "revision": 2, "priority": "P1", "surface": "mixed", "personas": [ @@ -564,7 +568,10 @@ "source badge: open /apps/setup/system/settings/localization and confirm the timezone field's source indicator flips from default to the persisted scope after save; screenshot", "env-lock: set OS_LOCALIZATION_TIMEZONE=Europe/Paris (envKeyOf convention: OS__, settings-service.types.ts) and restart; GET /api/settings/localization now reports timezone source='env', locked=true, lockedReason 'Set via env: OS_LOCALIZATION_TIMEZONE', and the console renders the EnvLockBadge; PUT /api/settings/localization { timezone: 'UTC' } is REFUSED 409 SETTINGS_LOCKED (server-side, effectiveEnvOverride)", "test action at the dev transport: POST /api/settings/mail/test with provider=log (or no email plugin mounted) — the handler answers ok:false and NO mail is faked (mail.manifest.ts mailTestActionHandler / plugin-email honest degradation, framework#5087)", - "secret handling: on a namespace with an encrypted specifier (mail.smtp_password or ai.*_api_key — type 'password' or encrypted:true), PUT a value; then read sys_secret + sys_setting — the ciphertext lands in sys_secret keyed by a 'sec_' handle and sys_setting.value_enc holds the handle id, NOT plaintext; GET /api/settings never returns the plaintext", + "secret handling — AT REST: on a namespace with an encrypted specifier (mail.smtp_password or ai.*_api_key — type 'password' or encrypted:true), PUT a value; then read sys_secret + sys_setting — the ciphertext lands in sys_secret keyed by a 'sec_' handle and sys_setting.value_enc holds the handle id, NOT plaintext", + "secret handling — the MASKED READ (positive contract, #7522/PR #7554): GET /api/settings/ and check what IS served, not merely that plaintext is absent — the SET secret's values..value is exactly SETTINGS_SECRET_MASK ('••••••••', eight U+2022 BULLETs — settings-secret-redaction.ts); an UNSET secret's value is still null (the redaction is presence-preserving, and the console renders 'configured vs not configured' from exactly this field); EVERY cascadeChain entry of a secret-backed key is the mask or null (walk a namespace whose scope is 'user' so the chain carries more than one entry); and the whole serialised body contains no plaintext. Do BOTH specifier flavours — type:'password' AND an explicit encrypted:true", + "secret handling — metadata survives the redaction: on an env-pinned secret (OS__) confirm the GET still reports source='env', locked=true and lockedReason naming that env var WHILE value is the mask, and that PUT is still refused 409 SETTINGS_LOCKED — masking must not disturb the lock affordances", + "secret handling — the ECHOED-MASK round-trip (the half a redaction fix most often breaks): PUT the mask straight back in BOTH shapes the console can send — { : '••••••••' } and the read-shape envelope { values: { : { value: '••••••••' } } } — and confirm 200 plus an UNCHANGED stored secret: re-read it through an in-process consumer (settings client / getNamespace, which is deliberately NOT redacted) and confirm it is still the original plaintext, and that no new sys_secret ciphertext row appeared. Then PUT a genuinely NEW secret: it takes effect, but the PUT RESPONSE is redacted too. Control: a NON-encrypted key whose value genuinely is '••••••••' is written verbatim — the drop is scoped to secret keys and must not swallow a legal write", "anonymous deny: GET /api/settings/localization with no session → 403 SETTINGS_FORBIDDEN (assertPermitted read); GET /api/settings (list) as anon returns an EMPTY manifest set — zero namespaces enumerated (Finding-1 secure default)", "guardrails: PUT /api/settings/localization { timezone: 'Mars/Phobos' } → 400 SETTINGS_VALIDATION invalid_option (declared options table); PUT { bogus_key: 1 } → 400 UNKNOWN_KEY" ], @@ -612,10 +619,28 @@ "evidence": "the action response" }, { - "clause": "an encrypted specifier's value lands in sys_secret as a handle, never plaintext: sys_setting.value_enc holds a 'sec_' handle id and sys_secret holds the ciphertext; GET /api/settings never echoes the plaintext", + "clause": "an encrypted specifier's value lands in sys_secret as a handle, never plaintext: sys_setting.value_enc holds a 'sec_' handle id and sys_secret holds the ciphertext", + "oracle": "api", + "verify": "the sys_secret row (id starts 'sec_', ciphertext present) + sys_setting.value_enc = that handle (materialiseRow dereferences through sys_secret)", + "evidence": "the sys_secret row + sys_setting.value_enc" + }, + { + "clause": "the REST read states a POSITIVE contract, not just an absent plaintext: a SET secret is SERVED AS the mask SETTINGS_SECRET_MASK ('••••••••'), an UNSET one stays null (presence-preserving), and the masking covers values..value AND every cascadeChain entry — an OMITTED value or a VANISHED key also satisfies 'never returns the plaintext' and is a FAIL, because the console renders configured-vs-not-configured from exactly this field", + "oracle": "api", + "verify": "GET /api/settings/: values..value === '••••••••' for the set secret and === null for the unset one; every cascadeChain entry of a secret-backed key is the mask or null; the serialised body contains no plaintext; both flavours (type:'password' and encrypted:true) — mirrors the route-level pins in packages/services/service-settings/src/settings-routes.test.ts (#7522 / PR #7554)", + "evidence": "the GET body showing values..value and the full cascadeChain per secret key, set and unset" + }, + { + "clause": "the redaction is METADATA-preserving: source, locked and lockedReason are unchanged by it, so an env-pinned secret still reports source='env' / locked=true / lockedReason naming OS__ and still refuses the write 409 SETTINGS_LOCKED — exactly as a non-secret key does", + "oracle": "api", + "verify": "the env-pinned secret's GET (value = mask, source='env', locked=true, lockedReason names the env var, cascadeChain entry scope='env' masked and locked) + the 409 on its PUT", + "evidence": "the locked GET + the 409" + }, + { + "clause": "PUTting the echoed mask back is a NO-OP, not an overwrite: a form save that returns the masked value verbatim leaves the stored secret intact — in BOTH shapes ({key: mask} and the read-shape {values:{key:{value: mask}}}) — while a genuinely NEW value still writes and its PUT response is redacted too; the in-process read path is deliberately NOT masked (mail/sms/storage/auth plugins read their credentials there), and a non-encrypted key whose value really is the mask is written verbatim", "oracle": "api", - "verify": "the sys_secret row (id starts 'sec_', ciphertext present) + sys_setting.value_enc = that handle + the redacted GET (materialiseRow dereferences through sys_secret)", - "evidence": "the sys_secret row + sys_setting.value_enc + the redacted GET" + "verify": "after each echoed-mask PUT (200): the in-process consumer still resolves the ORIGINAL plaintext and the sys_secret row inventory is unchanged — never the mask's literal text, which would decrypt back to itself and look fine until the dependent transport failed; after the real new value: it took effect in the store but the PUT response body carries the mask; the non-encrypted control key stores '••••••••' literally", + "evidence": "the echoed-mask PUT responses + the resolved in-process value + the sys_secret inventory before/after + the new-secret PUT response" }, { "clause": "anonymous is denied: GET /api/settings/:namespace → 403 SETTINGS_FORBIDDEN, and GET /api/settings lists ZERO namespaces for an unauthenticated caller (no enumeration — Finding-1)", @@ -634,6 +659,9 @@ "a saved settings form treated as proof of effect (no observable consumer checked) is the ai.json anti-pattern — ticking the effect clause on the form alone is a FALSE PASS", "a 200 on an env-locked PUT is a FAIL (the lock is a server contract, effectiveEnvOverride)", "an encrypted value returned as plaintext by GET /api/settings, or stored inline in sys_setting.value rather than sys_secret, is a FAIL", + "a masked read satisfied by OMITTING the value or dropping the KEY from the response is a FAIL, not a pass — omission satisfies 'never returns the plaintext' just as masking does, and it breaks the console's configured-vs-not-configured rendering while a negative-only clause stays green (#7573)", + "an echoed-mask PUT that OVERWRITES the stored secret with the mask's literal text is a FAIL — it re-encrypts and decrypts back to itself, so nothing looks wrong until the dependent transport (SMTP login, provider API key) fails", + "a mask on the IN-PROCESS read path (settings client / getNamespace / snapshot) is a FAIL — the redaction is a REST-boundary contract only; masking there breaks every plugin that reads its credentials through it", "a test action answering ok:true for a send that did not happen is a FAIL (framework#5087)", "GET /api/settings enumerating namespaces for an anonymous caller is a FAIL (Finding-1 — the old default trusted x-user-id/x-permissions headers)" ], @@ -657,10 +685,13 @@ "packages/services/service-settings/src/settings-service.types.ts (envKeyOf OS__)", "packages/services/service-settings/src/manifests/{localization,mail,ai,branding}.manifest.ts", "packages/core/src/security/resolve-authz-context.ts (localization timezone/locale/currency → ExecutionContext — the observable consumer)", + "packages/services/service-settings/src/settings-secret-redaction.ts (SETTINGS_SECRET_MASK = eight U+2022 BULLETs; READ: set→mask, unset→null; WRITE: a submitted value equal to the mask means 'unchanged' and the key is DROPPED from the patch)", + "packages/services/service-settings/src/settings-routes.test.ts (#7522 / PR #7554 — the ten route-level pins these secret clauses mirror rather than re-inventing)", "framework#5087 (a test action must not fake success), #5204 (env override enforcement)" ], "history": [ - { "revision": 1, "date": "2026-08-08", "change": "initial — settings hub round-trip with persistence + observable-consumer effect (honoring the ai.json form-is-not-effect caveat), sys_setting_audit, env-lock server refusal, honest test action, sys_secret handle-not-plaintext, anonymous deny", "ref": "claude/platform-test-checklist-ocwugl" } + { "revision": 1, "date": "2026-08-08", "change": "initial — settings hub round-trip with persistence + observable-consumer effect (honoring the ai.json form-is-not-effect caveat), sys_setting_audit, env-lock server refusal, honest test action, sys_secret handle-not-plaintext, anonymous deny", "ref": "claude/platform-test-checklist-ocwugl" }, + { "revision": 2, "date": "2026-08-11", "change": "secret clauses strengthened from negative-only ('never returns the plaintext', which omission and a vanished key also satisfy) to the POSITIVE contract #7522/PR #7554 pinned at the route level: set→mask / unset→null presence-preserving, masking across values..value AND every cascadeChain entry, source/locked/lockedReason unchanged, both specifier flavours, plus the distinct echoed-mask-PUT-is-a-no-op round-trip clause and the in-process-path-is-NOT-masked guard", "ref": "#7573" } ] }, { From 9185685c36f891ef22cd868d9864a527eddda861 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 11 Aug 2026 10:19:47 +0000 Subject: [PATCH 2/2] test(showcase-smoke): poll the chart box against a bounded deadline instead of a fixed 1500 ms wait (#7569) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The harness half of #7569, on the PM ruling that extended this claim's file surface to the spec file after the premise check found the fixed wait was never under `docs/qa/platform-checklist/`. `page.waitForTimeout(1500)` followed by a single `boundingBox()` read measured the recharts `ResponsiveContainer` initial-layout window, not the product: across 3 fresh loads each drawing 5 real charts, polling every 400 ms returned null ONLY at t = 1.5 s — exactly where the suite measured — and non-null on all 19 later samples; the same surfaces in isolation were 4/4 green. Both go, per the #7472 / #7409 shape: loop the predicate against a deadline derived from the case's own budget, never sleep-then-read-once. A bigger fixed number is not the fix. Three properties kept deliberately: - the last non-null box is retained, so a chart that IS laid out but collapsed still fails on its width (#2616 D) instead of being reported as "no chart SVG" — those are different defects and now read differently; - `pollUntil` returns its last sample rather than throwing, so a timeout fails as the assertion it belongs to, naming the surface; - the `pageErrors` / placeholder / main-content assertions moved after the waits, so the error-collection window is as long as the render actually took rather than a fixed slice of it. `nav-surfaces-render` is reconciled in the same commit: its knownGaps entry recording this harness gap is removed now that the gap is closed, and its revision-3 history entry covers both halves. --- .../areas/platform-core.json | 13 ++- .../app-showcase/e2e/showcase-smoke.spec.ts | 87 ++++++++++++++++--- 2 files changed, 83 insertions(+), 17 deletions(-) diff --git a/docs/qa/platform-checklist/areas/platform-core.json b/docs/qa/platform-checklist/areas/platform-core.json index 12bb04e0ef..923a076bb2 100644 --- a/docs/qa/platform-checklist/areas/platform-core.json +++ b/docs/qa/platform-checklist/areas/platform-core.json @@ -220,8 +220,7 @@ "fixtures": { "app": "showcase", "knownGaps": [ - "no stock showcase fixture deliberately throws inside a route, so the ErrorBoundary fallback ('Something went wrong' + Try Again / Go Home — objectui packages/app-shell/src/chrome/ErrorBoundary.tsx) is verified opportunistically on any failure encountered, plus via the bad-route probe below", - "the pinned suite still MEASURES charts with a fixed `page.waitForTimeout(1500)` followed by a SINGLE `boundingBox()` read (examples/app-showcase/e2e/showcase-smoke.spec.ts) — a coin-flip red whenever that fixed instant lands inside the recharts ResponsiveContainer initial-layout window. Until that read is replaced by a bounded poll-until-non-null (the #7472 / #7409 de-flake shape: loop the predicate against a DEADLINE derived from the case's own budget, never sleep-then-read-once — a bigger fixed number is not the fix), a null-box chart red must be RE-MEASURED per the steps before it is recorded. Tracked by #7569, whose harness half lands in the spec file, not in this checklist" + "no stock showcase fixture deliberately throws inside a route, so the ErrorBoundary fallback ('Something went wrong' + Try Again / Go Home — objectui packages/app-shell/src/chrome/ErrorBoundary.tsx) is verified opportunistically on any failure encountered, plus via the bad-route probe below" ] }, "steps": [ @@ -229,7 +228,7 @@ "GET /api/v1/meta/app?id=com.example.showcase and extract every nav destination from the response", "diff the served nav destinations against the SURFACES array; hand-walk any destination the suite does not cover (screenshot first, then DOM)", "on each hand-walked surface: wait for render, screenshot, then check for pageerror / empty
/ placeholder leaks ('no actions configured')", - "chart surfaces (the chart:true entries in SURFACES — Delivery Operations, Chart Gallery, Command Center): if the suite reports a NULL chart boundingBox, do NOT record it as a product red — RE-MEASURE it by polling boundingBox until non-null within a bounded budget. #7569 measured the difference on this exact assertion: 3 fresh loads each drew 5 real charts, and polling every 400 ms returned null ONLY at t = 1.5 s — the suite's fixed measurement instant, and exactly the recharts ResponsiveContainer initial-layout window — and non-null on all 19 later samples; the same item re-run in isolation is 4/4 green. A box that stays null through the whole bounded poll IS a product finding; one that resolves on a later sample is the harness's measurement window, not the product", + "chart surfaces (the chart:true entries in SURFACES — Delivery Operations, Chart Gallery, Command Center): the suite now POLLS boundingBox against a bounded deadline (SURFACE_BUDGET_MS in the spec) instead of reading it once after a fixed wait, so a chart red here is a real finding and not the measurement window — #7569 measured the difference: 3 fresh loads each drew 5 real charts, and polling every 400 ms returned null ONLY at t = 1.5 s, the old fixed measurement instant and exactly the recharts ResponsiveContainer initial-layout window, then non-null on all 19 later samples. Read WHICH assertion failed: 'no chart SVG laid out' means the box stayed null for the whole budget; a width/height failure means the chart laid out but collapsed (#2616 D). If you hand-walk a chart surface outside the suite, poll the same way — never sleep-then-read-once", "probe the failure path: navigate to a nonexistent route under the app (/_console/apps/com.example.showcase/object_that_does_not_exist); screenshot what renders", "if ANY surface fails during the sweep, verify the failure presents as the shell's error boundary or a named empty/error state — capture it" ], @@ -237,8 +236,8 @@ { "clause": "no surface throws a pageerror, renders an empty
, or leaks a 'no actions configured' placeholder; chart surfaces draw a real SVG whose box is measured by POLLING until non-null within a bounded budget — never by a fixed wall-clock wait followed by a single boundingBox read, which measures the layout window rather than the product", "oracle": "test", - "verify": "pnpm -C examples/app-showcase test:smoke (SURFACES array in e2e/showcase-smoke.spec.ts) — green; a chart clause that failed on a NULL box counts only after the bounded re-measure in the steps confirms it stays null (the pinned suite still reads the box once after a fixed 1500 ms wait — see knownGaps, #7569)", - "evidence": "test run output +, for any null-box chart failure, the bounded re-measure's sample series" + "verify": "pnpm -C examples/app-showcase test:smoke (SURFACES array in e2e/showcase-smoke.spec.ts) — green; the chart assertion polls boundingBox against a bounded deadline and keeps the last non-null sample, so a null-box failure and a collapsed-panel failure report as different assertions (#7569)", + "evidence": "test run output + the failing assertion's message for any chart red" }, { "clause": "the smoke's coverage is CURRENT: every nav destination served in meta/app is either in the SURFACES array or hand-walked this run — no surface silently outside the net", @@ -262,7 +261,7 @@ "negative": [ "a surface that renders a blank
is a FAIL even with no console error — transitional emptiness must be ruled out by the screenshot-first protocol, then the persistent blank is the finding", "a bad route producing a white page (shell gone) is a FAIL of the error-boundary clause", - "a chart red obtained from a SINGLE boundingBox read at a fixed instant is NOT a product finding and must not be filed as one — it measures the ResponsiveContainer layout window, and it costs a triage cycle every run (#7569). Re-measure with a bounded poll first. The converse still holds both ways: a box that stays null through the whole poll, or a resolved width ≤ 200 (the #2616 D collapsed-panel case), IS a FAIL" + "a chart red obtained from a SINGLE boundingBox read at a fixed instant is NOT a product finding and must not be filed as one — it measures the ResponsiveContainer layout window, and it cost a triage cycle every run until #7569 replaced that read with a bounded poll. Re-introducing any fixed sleep-then-read-once in this suite re-opens it. The converse holds both ways: a box that stays null through the WHOLE bounded poll, or a laid-out width ≤ 200 (the #2616 D collapsed-panel case), IS a FAIL" ], "automated": { "kind": "e2e", "ref": "examples/app-showcase/e2e/showcase-smoke.spec.ts" }, "traps": ["hydration-race", "single-datapoint", "stale-console-bundle"], @@ -275,7 +274,7 @@ "history": [ { "revision": 1, "date": "2026-08-07", "change": "initial — wraps the existing automated smoke as a checklist row so runs report it alongside manual items", "ref": "#3358" }, { "revision": 2, "date": "2026-08-07", "change": "expanded to deep-test contract: concrete steps, multi-clause acceptance, negatives, variants", "ref": "claude/platform-test-checklist-ocwugl" }, - { "revision": 3, "date": "2026-08-11", "change": "de-flaked the chart clause's MEASUREMENT: a null boundingBox read once at a fixed instant is the recharts ResponsiveContainer layout window, not a product defect (measured: null only at t=1.5s, non-null on all 19 later samples, 5 real charts on each of 3 fresh loads). The clause now requires a bounded poll-until-non-null, the steps tell the runner to re-measure before recording a red, and knownGaps records that the pinned suite's fixed 1500 ms wait + single read is still there", "ref": "#7569" } + { "revision": 3, "date": "2026-08-11", "change": "de-flaked the chart clause's MEASUREMENT, both halves in one change: a null boundingBox read once at a fixed instant is the recharts ResponsiveContainer layout window, not a product defect (measured: null only at t=1.5s, non-null on all 19 later samples, 5 real charts on each of 3 fresh loads). The clause now requires a bounded poll-until-non-null, and the pinned suite was changed to match — showcase-smoke.spec.ts's fixed page.waitForTimeout(1500) plus single boundingBox() read are gone, replaced by deadline-bounded polling (the #7472/#7409 shape), keeping the last non-null sample so a collapsed panel (#2616 D) still fails on its width rather than as 'no chart SVG'", "ref": "#7569" } ] }, { diff --git a/examples/app-showcase/e2e/showcase-smoke.spec.ts b/examples/app-showcase/e2e/showcase-smoke.spec.ts index 427d0baa7e..2b6635f174 100644 --- a/examples/app-showcase/e2e/showcase-smoke.spec.ts +++ b/examples/app-showcase/e2e/showcase-smoke.spec.ts @@ -43,32 +43,99 @@ const SURFACES: { name: string; path: string; chart?: boolean }[] = [ { name: 'Work Map', path: base('page/showcase_task_map') }, ]; +/** + * The budget every wait in this file spends, measured from the start of the + * surface that calls it — and the SOURCE of each wait's deadline, rather than + * one more number chosen in advance. It sits inside the config's 45s per-test + * `timeout`, which stays the only wall-clock budget left to expire after it. + * + * ⛔ Never put a fixed wall-clock wait back in front of an assertion here. The + * shape this replaced was `await page.waitForTimeout(1500)` followed by a + * SINGLE `boundingBox()` read, and #7569 measured what that costs: across 3 + * fresh loads each drawing 5 real charts, polling every 400 ms returned null + * ONLY at t = 1.5 s — precisely the instant this suite measured at, and + * precisely the recharts `ResponsiveContainer` initial-layout window — and + * non-null on all 19 later samples; the same surfaces re-run in isolation were + * 4/4 green. So the assertion reported a failing surface whenever the fixed + * wait landed inside the layout window: a coin-flip red on a correct product, + * costing a triage cycle every run. A bigger fixed number is not the fix — the + * budget is a constant while the thing it times is not. Wait on a predicate + * the page can actually satisfy, bounded by a deadline. + */ +const SURFACE_BUDGET_MS = 25_000; +const POLL_MS = 100; + +const surfaceDeadline = (): number => Date.now() + SURFACE_BUDGET_MS; +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +/** + * Poll `probe` until it returns a truthy value, or until `deadline`. Returns + * whatever the last sample was either way, so the CALLER still asserts on it — + * a timeout then fails as the assertion it belongs to, naming the surface, + * instead of as a bare timeout that says nothing about what was missing. + */ +async function pollUntil(probe: () => Promise, deadline: number): Promise { + let sample = await probe(); + while (!sample && Date.now() < deadline) { + await sleep(POLL_MS); + sample = await probe(); + } + return sample; +} + for (const surface of SURFACES) { test(`surface renders cleanly: ${surface.name}`, async ({ page }) => { const pageErrors: string[] = []; page.on('pageerror', (e) => pageErrors.push(e.message)); + const deadline = surfaceDeadline(); await page.goto(surface.path, { waitUntil: 'domcontentloaded' }); - await page.locator('main').first().waitFor({ state: 'visible', timeout: 25_000 }); - await page.waitForTimeout(1500); + const main = page.locator('main').first(); + await main.waitFor({ state: 'visible', timeout: SURFACE_BUDGET_MS }); + + // Replaces the fixed 1500 ms settle with a wait on the thing the assertion + // below actually needs — rendered main content — on the same deadline. + const mainText = await pollUntil( + async () => ((await main.innerText().catch(() => '')) || '').trim(), + deadline, + ); + let chartBox: { width: number; height: number } | null = null; + if (surface.chart) { + const svg = page.locator('.recharts-wrapper svg, .recharts-surface').first(); + await svg.waitFor({ state: 'visible', timeout: SURFACE_BUDGET_MS }); + // `visible` commits before ResponsiveContainer has sized its child, which + // is why reading the box once lands inside the layout window. Poll it + // instead, and keep the last non-null sample: a chart that IS laid out but + // collapsed must still fail on its width (#2616 D below) rather than be + // reported as "no chart SVG", which is a different defect. + let lastSeen: { width: number; height: number } | null = null; + const laidOut = await pollUntil(async () => { + const box = await svg.boundingBox(); + if (box) lastSeen = box; + return box && box.width > 200 && box.height > 0 ? box : null; + }, deadline); + chartBox = laidOut ?? lastSeen; + } + + // Asserted after the waits above, so the error-collection window is as long + // as the render actually took rather than a fixed slice of it. expect(pageErrors, `uncaught errors on ${surface.name}`).toEqual([]); await expect( page.getByText(/no actions configured/i), `leaked placeholder on ${surface.name}`, ).toHaveCount(0); - const mainText = (await page.locator('main').first().innerText().catch(() => '')) || ''; - expect(mainText.trim().length, `${surface.name} rendered no main content`).toBeGreaterThan(0); + expect(mainText.length, `${surface.name} rendered no main content`).toBeGreaterThan(0); if (surface.chart) { - const svg = page.locator('.recharts-wrapper svg, .recharts-surface').first(); - await svg.waitFor({ state: 'visible', timeout: 25_000 }); - const box = await svg.boundingBox(); - expect(box, `${surface.name}: no chart SVG`).not.toBeNull(); + expect( + chartBox, + `${surface.name}: no chart SVG laid out within ${SURFACE_BUDGET_MS}ms`, + ).not.toBeNull(); // >0 alone previously passed even on a collapsed ~130px-wide chart panel // (#2616 D) — require a width a real chart panel would actually have. - expect(box!.width, `${surface.name}: chart width`).toBeGreaterThan(200); - expect(box!.height, `${surface.name}: chart height`).toBeGreaterThan(0); + expect(chartBox!.width, `${surface.name}: chart width`).toBeGreaterThan(200); + expect(chartBox!.height, `${surface.name}: chart height`).toBeGreaterThan(0); } }); }