PROTO(objectui#5403): read the equivalence numbers back through artifact NAMES [verdict]

Three channels have now failed from the measuring session: the Actions log CDN,
artifact bodies, and a branch push with the workflow token. Artifact metadata is
readable, so the comparison encodes its numbers in the artifact name it uploads.
Two comparisons: merged vs unsharded on the same commit (run 32388915404), and
merged vs merged across two identical sharded rounds (the noise floor).

Author: os-support-ai

.github/workflows/proto-5403-shard-coverage.yml

@@ -39,7 +39,7 @@ jobs:
   # ── A: the shipped recipe, 4 ways ─────────────────────────────────────────
   shard:
     name: PROTO shard ${{ matrix.shard }}/4
-    if: ${{ !contains(github.event.head_commit.message, '[threshold-probe]') && !contains(github.event.head_commit.message, '[diag]') }}
+    if: ${{ !contains(github.event.head_commit.message, '[threshold-probe]') && !contains(github.event.head_commit.message, '[diag]') && !contains(github.event.head_commit.message, '[verdict]') }}
     runs-on: ubuntu-latest
     timeout-minutes: 40
     strategy:
@@ -82,7 +82,7 @@ jobs:
   # and that has to be said out loud rather than shipped as decoration.
   probe-no-override:
     name: PROTO shard 1/4 WITHOUT the threshold override
-    if: ${{ !contains(github.event.head_commit.message, '[threshold-probe]') && !contains(github.event.head_commit.message, '[diag]') }}
+    if: ${{ !contains(github.event.head_commit.message, '[threshold-probe]') && !contains(github.event.head_commit.message, '[diag]') && !contains(github.event.head_commit.message, '[verdict]') }}
     runs-on: ubuntu-latest
     timeout-minutes: 40
     outputs:
@@ -112,7 +112,7 @@ jobs:
     # Measurement-only relaxation: the SHIPPED job refuses to merge unless every
     # shard is green. Here the point is to obtain a merged report to compare
     # against the unsharded one, so a red shard must not abort the measurement.
-    if: ${{ always() && !contains(github.event.head_commit.message, '[threshold-probe]') && !contains(github.event.head_commit.message, '[diag]') }}
+    if: ${{ always() && !contains(github.event.head_commit.message, '[threshold-probe]') && !contains(github.event.head_commit.message, '[diag]') && !contains(github.event.head_commit.message, '[verdict]') }}
     runs-on: ubuntu-latest
     timeout-minutes: 20
     outputs:
@@ -185,7 +185,7 @@ jobs:
   # ── B: the status quo on the same commit — the baseline to compare against ─
   baseline:
     name: PROTO unsharded coverage baseline
-    if: ${{ !contains(github.event.head_commit.message, '[threshold-probe]') && !contains(github.event.head_commit.message, '[diag]') && !contains(github.event.head_commit.message, '[merge-only]') }}
+    if: ${{ !contains(github.event.head_commit.message, '[threshold-probe]') && !contains(github.event.head_commit.message, '[diag]') && !contains(github.event.head_commit.message, '[merge-only]') && !contains(github.event.head_commit.message, '[verdict]') }}
     runs-on: ubuntu-latest
     # Instrument, not a gate: production `test-coverage` keeps its 40 minutes.
     # A timeout here would truncate the measurement, which is the one thing
@@ -225,7 +225,7 @@ jobs:
   compare:
     name: PROTO compare merged vs unsharded
     needs: [merge, baseline]
-    if: ${{ (always()) && !contains(github.event.head_commit.message, '[threshold-probe]') && !contains(github.event.head_commit.message, '[diag]') && !contains(github.event.head_commit.message, '[merge-only]') }}
+    if: ${{ (always()) && !contains(github.event.head_commit.message, '[threshold-probe]') && !contains(github.event.head_commit.message, '[diag]') && !contains(github.event.head_commit.message, '[merge-only]') && !contains(github.event.head_commit.message, '[verdict]') }}
     runs-on: ubuntu-latest
     timeout-minutes: 10
     outputs:
@@ -325,7 +325,7 @@ jobs:
   report:
     name: PROTO write results
     needs: [shard, probe-no-override, merge, baseline, compare, red-arm-shard, red-arm-merge, red-arm-merge-without-verdict]
-    if: ${{ (always()) && !contains(github.event.head_commit.message, '[threshold-probe]') && !contains(github.event.head_commit.message, '[diag]') && !contains(github.event.head_commit.message, '[merge-only]') }}
+    if: ${{ (always()) && !contains(github.event.head_commit.message, '[threshold-probe]') && !contains(github.event.head_commit.message, '[diag]') && !contains(github.event.head_commit.message, '[merge-only]') && !contains(github.event.head_commit.message, '[verdict]') }}
     runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:
@@ -534,3 +534,65 @@ jobs:
           git add -f proto-results/diag-shard3.md
           git commit -m "PROTO diag shard3 output ${GITHUB_RUN_ID}"
           git push origin "HEAD:refs/heads/proto-5403-diag-shard3-${GITHUB_RUN_ID}"
+
+  # [verdict]: the equivalence numbers, encoded in ARTIFACT NAMES because that
+  # is the one channel out of a run this session can actually read.
+  verdict-equivalence:
+    name: PROTO verdict - merged vs unsharded (same commit)
+    if: ${{ contains(github.event.head_commit.message, '[verdict]') }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/download-artifact@v7
+        with:
+          name: proto-coverage-unsharded
+          path: unsharded
+          run-id: 32388915404
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: actions/download-artifact@v7
+        with:
+          name: proto-coverage-merged
+          path: merged
+          run-id: 32388915404
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+      - id: cmp
+        run: node scripts-proto/verdict.mjs unsharded/coverage-final.json merged/coverage-final.json EQUIV
+      - run: 'echo "$RESULT" > result.txt'
+        env:
+          RESULT: ${{ steps.cmp.outputs.name }}
+      - uses: actions/upload-artifact@v7
+        with:
+          name: ${{ steps.cmp.outputs.name }}
+          path: result.txt
+          retention-days: 1
+
+  verdict-noise-floor:
+    name: PROTO verdict - merged vs merged (two identical sharded rounds)
+    if: ${{ contains(github.event.head_commit.message, '[verdict]') }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/download-artifact@v7
+        with:
+          name: proto-coverage-merged
+          path: roundA
+          run-id: 32388915404
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: actions/download-artifact@v7
+        with:
+          name: proto-coverage-merged
+          path: roundB
+          run-id: 32391890554
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+      - id: cmp
+        run: node scripts-proto/verdict.mjs roundA/coverage-final.json roundB/coverage-final.json NOISE
+      - run: 'echo "$RESULT" > result.txt'
+        env:
+          RESULT: ${{ steps.cmp.outputs.name }}
+      - uses: actions/upload-artifact@v7
+        with:
+          name: ${{ steps.cmp.outputs.name }}
+          path: result.txt
+          retention-days: 1

scripts-proto/verdict.mjs

@@ -0,0 +1,43 @@
+// THROWAWAY — objectui#5403 measurement instrument. Not to be merged.
+// Compares two Istanbul coverage-final.json files and encodes the answer in a
+// string safe to use as an ARTIFACT NAME. The measuring session can read
+// artifact names through the REST API; it cannot read Actions logs, artifact
+// bodies, or (as this instrument found) push a results branch with the
+// workflow token. So the name is the channel.
+import { readFileSync, existsSync, appendFileSync } from 'node:fs';
+
+const [aPath, bPath, label] = process.argv.slice(2);
+const out = (k, v) => { if (process.env.GITHUB_OUTPUT) appendFileSync(process.env.GITHUB_OUTPUT, `${k}=${v}\n`); };
+for (const p of [aPath, bPath]) {
+  if (!existsSync(p)) { out('name', `${label}__MISSING-INPUT`); console.log('missing', p); process.exit(1); }
+}
+const norm = (o) => { const r = {}; for (const k of Object.keys(o)) r[k.replace(/^.*?\/objectui[^/]*\//, '')] = o[k]; return r; };
+const A = norm(JSON.parse(readFileSync(aPath, 'utf8')));
+const B = norm(JSON.parse(readFileSync(bPath, 'utf8')));
+const ka = new Set(Object.keys(A)), kb = new Set(Object.keys(B));
+const onlyA = [...ka].filter((k) => !kb.has(k));
+const onlyB = [...kb].filter((k) => !ka.has(k));
+const tot = (o) => { let s = 0, c = 0; for (const e of Object.values(o)) for (const v of Object.values(e.s ?? {})) { s++; if (v > 0) c++; } return { s, c }; };
+const ta = tot(A), tb = tot(B);
+const pct = (x, t) => (t ? (100 * x) / t : 0);
+const deltaPP = (pct(tb.c, tb.s) - pct(ta.c, ta.s)).toFixed(4);
+let regressed = 0;
+for (const k of [...ka].filter((x) => kb.has(x))) {
+  const ca = Object.values(A[k].s ?? {}).filter((v) => v > 0).length;
+  const cb = Object.values(B[k].s ?? {}).filter((v) => v > 0).length;
+  if (cb < ca) regressed++;
+}
+const name = [
+  label,
+  `filesA-${ka.size}`,
+  `filesB-${kb.size}`,
+  `onlyA-${onlyA.length}`,
+  `onlyB-${onlyB.length}`,
+  `stmtA-${ta.c}of${ta.s}`,
+  `stmtB-${tb.c}of${tb.s}`,
+  `stmtDiff-${tb.c - ta.c}`,
+  `deltaPP-${deltaPP}`.replace('.', 'p').replace('-0p', 'minus0p'),
+  `regressed-${regressed}`,
+].join('__');
+console.log(name);
+out('name', name);