lilyshen0722 · Oct 23, 2026 · Oct 23, 2026 · Oct 23, 2026 · Oct 23, 2026 · coderabbitai
diff --git a/cli/__tests__/spawn-retry.test.mjs b/cli/__tests__/spawn-retry.test.mjs
@@ -173,3 +173,149 @@ describe('classifies real provider-exhaustion strings (2026-08-03 outage)', () =
       .toBe(SPAWN_FAILURE_CLASS.RUNTIME);
   });
 });
+
+describe('classifies the possessive-exhaustion phrase (2026-08-25, third miss)', () => {
+  // Verbatim from the fleet's own logs, measured 2026-08-25 across every seat
+  // that had failed at all. Three wordings, one sentence shape, and the
+  // allowlist carried exactly one of them:
+  //
+  //   283 × "You've hit your weekly limit"      <- unmatched, RUNTIME ladder
+  //    55 × "You've hit your session limit"     <- matched (the 08-18 fix)
+  //     6 × "You've reached your Fable 5 limit" <- unmatched, RUNTIME ladder
+  //
+  // Re-measured on main 54eb2abc8 before this change: all four wordings in this
+  // file's QUOTA cases below still classified RUNTIME, so the gap is live.
+  const WEEKLY = "claude exited with code 1: You've hit your weekly limit "
+    + '· resets Aug 27 at 2pm (America/Los_Angeles)';
+  const PER_MODEL = "claude exited with code 1: You've reached your Fable 5 limit";
+
+  test.each([
+    ['weekly, the wording behind 283 of 365 fleet failures', WEEKLY],
+    ['per-model, whose variable part is a model name', PER_MODEL],
+  ])('%s is QUOTA, not RUNTIME', (_label, message) => {
+    expect(classifySpawnFailure(new Error(message))).toBe(SPAWN_FAILURE_CLASS.QUOTA);
+  });
+
+  test('so the circuit opens at the full cooldown on the FIRST failure', () => {
+    const { circuitOpen, delayMs } = spawnRetryPolicy({
+      error: new Error(WEEKLY),
+      consecutiveFailures: 1,
+      intervalMs: 5000,
+    });
+    expect(circuitOpen).toBe(true);
+    expect(delayMs).toBe(SPAWN_RETRY_MAX_MS);
+  });
+
+  // This is the specific harm the misclassification caused, and it is not the
+  // latency. The RUNTIME ladder is 5s / 10s / 60s, so three consecutive
+  // failures land inside ~75 seconds — and the event is left unacked on each
+  // (agent.js, "the kernel must retain the event for at-least-once delivery"),
+  // which burns all three requeue attempts and retires it to `status: 'failed'`.
+  // Under QUOTA the first failure alone opens a 15-minute circuit.
+  test('the RUNTIME ladder would burn all three requeue attempts inside 90s', () => {
+    const runtimeLadder = [1, 2, 3].map((n) => spawnRetryPolicy({
+      error: new Error('claude exited with code 1: something transient'),
+      consecutiveFailures: n,
+      intervalMs: 5000,
+    }).delayMs);
+    expect(runtimeLadder.reduce((a, b) => a + b, 0)).toBeLessThan(90 * 1000);
+
+    const quotaLadder = [1, 2, 3].map((n) => spawnRetryPolicy({
+      error: new Error(WEEKLY),
+      consecutiveFailures: n,
+      intervalMs: 5000,
+    }).delayMs);
+    expect(quotaLadder.every((ms) => ms === SPAWN_RETRY_MAX_MS)).toBe(true);
+  });
+
+  // Guard on the widening. The phrase requires "your", so a server-side
+  // throttle phrased the same way must not be swallowed into the 15-minute
+  // cooldown — QUOTA is tested before RATE_LIMIT, so the ordering is load-bearing.
+  test.each([
+    "You've hit your rate limit, retry shortly",
+    "You've reached your rate-limit for this model",
+    // Capitalised, because the lookahead's `i` is load-bearing and invisible.
+    "you've hit your Rate Limit",
+    // The lookahead only guards the word IMMEDIATELY after "your", so these
+    // two walked past it into the 15-minute circuit while main correctly gave
+    // the 60-second probe — a regression this branch introduced and otto's
+    // gate on #2491 caught. The call site now refuses the possessive shape
+    // whenever a throttle wording appears anywhere in the text.
+    "You've hit your API rate limit",
+    "You've reached your per-model rate limit",
+    "You've hit your organisation's rate limit for this model",
+  ])('a possessive RATE-limit string stays RATE_LIMIT: %s', (message) => {
+    expect(classifySpawnFailure(new Error(message))).toBe(SPAWN_FAILURE_CLASS.RATE_LIMIT);
+  });
+
+  // The boundary of that refusal, in the other direction: an explicit quota
+  // WORD still wins, because QUOTA_RE is tested before the possessive shape.
+  // Otherwise a provider sentence that names both would lose the long cooldown
+  // it needs.
+  test('an explicit quota word beats a rate-limit mention in the same message', () => {
+    expect(classifySpawnFailure(new Error(
+      'Claude usage limit reached; standard rate limits also apply.',
+    ))).toBe(SPAWN_FAILURE_CLASS.QUOTA);
+  });
+
+  test('the phrase does not span a sentence boundary', () => {
+    expect(classifySpawnFailure(new Error("You've hit your stride. Now describe the limit.")))
+      .toBe(SPAWN_FAILURE_CLASS.RUNTIME);
+  });
+
+  describe('the dotted-model-name gap (sprint-review, 2026-08-25)', () => {
+    // `[^.\n]` was chosen to keep the match inside one sentence, and silently
+    // also excluded every model name we run, because they are all dotted.
+    // RUNTIME is the fallthrough with the SHORTEST backoff, so the net effect
+    // was to probe a quota-blocked seat hardest.
+    test.each([
+      ["You've reached your Haiku 4.5 limit"],
+      ["you've hit your gpt-5.4-mini limit"],
+      ["Error: You've reached your Claude Opus 4.8 limit for today"],
+    ])('classifies %s as QUOTA', (text) => {
+      expect(classifySpawnFailure(new Error(text))).toBe(SPAWN_FAILURE_CLASS.QUOTA);
+    });
+
+    test('still refuses a rate limit, which QUOTA is tested before', () => {
+      expect(classifySpawnFailure(new Error("you've hit your rate limit")))
+        .toBe(SPAWN_FAILURE_CLASS.RATE_LIMIT);
+    });
+
+    // RENAMED from "does not let the dot carve-out span a sentence boundary",
+    // which sprint-review's gate on #1215 showed was vacuous: it passed
+    // identically with and without the carve-out, because a LETTER follows the
+    // period there and plain `[^.\n]` already excluded that. What the pattern
+    // actually has is weaker, and this is it — the space after the period is
+    // what ends the match.
+    test('a period followed by a SPACE still terminates the match', () => {
+      expect(classifySpawnFailure(new Error("You've reached your goal. 5 things have no limit")))
+        .not.toBe(SPAWN_FAILURE_CLASS.QUOTA);
+    });
+
+    // The documented limit, asserted so it cannot drift unnoticed: a digit
+    // immediately after the period IS admitted, even at a sentence boundary.
+    // Kept rather than tightened — adjacent `.`+digit is rare in provider copy,
+    // and the alternative predicate is harder to read than the bug is costly.
+    test('a period followed immediately by a DIGIT is admitted — known and accepted', () => {
+      expect(classifySpawnFailure(new Error("You've reached your goal.5 things have no limit")))
+        .toBe(SPAWN_FAILURE_CLASS.QUOTA);
+    });
+
+    // The {0,40} bound counts alternation units, so a long model name falls
+    // through to RUNTIME by design. Pinned because nothing else pins it.
+    test('past the length bound it falls through to RUNTIME', () => {
+      expect(classifySpawnFailure(new Error(
+        "You've reached your extremely verbose vendor model name 4.5 preview limit",
+      ))).toBe(SPAWN_FAILURE_CLASS.RUNTIME);
+    });
+
+    test('a dotted model name costs the quota ladder, not the runtime ladder', () => {
+      const policy = spawnRetryPolicy({
+        error: new Error("You've reached your Haiku 4.5 limit"),
+        consecutiveFailures: 1,
+        intervalMs: 5000,
+      });
+      expect(policy).toMatchObject({ circuitOpen: true, delayMs: SPAWN_RETRY_MAX_MS });
+    });
+  });
+});
diff --git a/cli/package-lock.json b/cli/package-lock.json
diff --git a/cli/package.json b/cli/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@commonlyai/cli",
-  "version": "0.1.122",
+  "version": "0.1.130",
   "license": "Apache-2.0",
   "description": "The Commonly CLI — connect agents, manage pods, iterate fast",
   "type": "module",

diff --git a/cli/src/lib/spawn-retry.js b/cli/src/lib/spawn-retry.js
@@ -27,6 +27,7 @@ export const SPAWN_RETRY_JITTER_MAX_RATIO = 0.2;
 // outage has already been misclassified. Twice now:
 //   2026-08-03  codex  "Your workspace is out of credits."   → `out of credits`
 //   2026-08-18  claude "You've hit your session limit"       → `session limit`
+//   2026-08-25  claude "You've hit your weekly limit"        → the phrase below
 // The second one is the instructive failure: `usage limit` was already here —
 // it is Claude's OTHER exhaustion wording — so the fleet stalled for an hour on
 // a string one word away from a pattern we had. Both times the miss meant
@@ -62,7 +63,38 @@ export const SPAWN_RETRY_JITTER_MAX_RATIO = 0.2;
 // nothing else until the kernel retired the event. Only the refusal wordings
 // themselves are matched, so an ordinary failure that mentions a flag is not.
 const CONTENT_REFUSED_RE = /(?:flagged for possible cybersecurity risk|this content was flagged|content[_ ]policy[_ ]violation|cyber[_ ]policy)/i;
-const QUOTA_RE = /(?:quota|usage limit|session limit|credit balance|out of credits|billing|insufficient[_ -]?quota|resource exhausted|spending limit)/i;
+// The third miss is why there is now a PHRASE as well as a word list, and it is
+// a different failure from the first two. Measured across the fleet's logs on
+// 2026-08-25: 283 failures reading "You've hit your weekly limit", 55 reading
+// "You've hit your session limit" (matched), and 6 reading "You've reached your
+// Fable 5 limit" (not matched). Three wordings, one sentence shape, and the list
+// had caught exactly one of them. Enumerating per-wording keeps failing because
+// the variable part is a BILLING PERIOD or a MODEL NAME, both of which the
+// providers keep adding, so a list is structurally one release behind.
+//
+// `QUOTA_POSSESSIVE_RE` matches that shape and nothing looser: the subject must
+// be the caller's own allowance ("you've hit/reached YOUR ... limit"), which is
+// what distinguishes exhaustion from a server-side throttle. The negative
+// lookahead keeps "you've hit your rate limit" out — QUOTA is tested first, and
+// without it that string would take the 15-minute cooldown instead of the
+// 60-second rate-limit backoff. The length bound stops it spanning sentences.
+//
+// The character class admits a dot ONLY when a digit follows it, and that
+// carve-out is load-bearing rather than cosmetic (sprint-review, 2026-08-25):
+// a plain `[^.\n]` reads as "stay inside one sentence" and silently also means
+// "no model names". Every model we run is dotted, so "you've reached your Haiku
+// 4.5 limit" fell through to RUNTIME, the shortest backoff — the class excluded
+// the exact variable part this paragraph says keeps changing.
+//
+// Known limit of `\.(?=\d)`, kept rather than papered over: a period with a
+// digit immediately after it is admitted even at a sentence boundary, so
+// "You've reached your goal.5 things have no limit" classes as QUOTA. The space
+// in ordinary prose ("goal. 5 things") is what terminates the match, and
+// adjacent `.`+digit is rare enough in provider copy to accept. The test below
+// pins what the pattern ACTUALLY has — a space after the period ends the match —
+// rather than the stronger property it does not have.
+const QUOTA_RE = /(?:quota|usage limit|session limit|weekly limit|credit balance|out of credits|billing|insufficient[_ -]?quota|resource exhausted|spending limit)/i;
+const QUOTA_POSSESSIVE_RE = /you'?ve (?:hit|reached) your (?!rate[ -]?limit)(?:[^.\n]|\.(?=\d)){0,40}?limit/i;
 const RATE_LIMIT_RE = /(?:rate[ -]?limit|too many requests|\b429\b|overloaded|capacity)/i;
 const CONFIGURATION_RE = /(?:ENOENT|command not found|not on PATH|login required|not logged in|invalid api key|authentication failed|unauthori[sz]ed|forbidden|\b40[13]\b)/i;
 
@@ -85,6 +117,17 @@ export const classifySpawnFailure = (error) => {
   // Prefer the more specific body/message over the generic status code so a
   // hard quota failure gets the long cooldown rather than a one-minute probe.
   if (QUOTA_RE.test(text)) return SPAWN_FAILURE_CLASS.QUOTA;
+  // The possessive shape is deliberately loose about what sits between "your"
+  // and "limit", and its `(?!rate[ -]?limit)` lookahead only guards the word
+  // IMMEDIATELY after "your" — so "you've hit your API rate limit" walked past
+  // it into the 15-minute circuit, where main correctly gives the 60-second
+  // rate-limit probe (otto, gate on #2491; it was a regression, not a gap).
+  // A throttle wording anywhere in the text therefore wins over the shape.
+  // QUOTA_RE is still tested first, so an explicit quota word keeps QUOTA even
+  // when the same message also mentions rate limits.
+  if (QUOTA_POSSESSIVE_RE.test(text) && !RATE_LIMIT_RE.test(text)) {
+    return SPAWN_FAILURE_CLASS.QUOTA;
+  }
   if (error?.status === 429 || RATE_LIMIT_RE.test(text)) {
     return SPAWN_FAILURE_CLASS.RATE_LIMIT;
   }