From 306ab1037cf2cff0ad066e06074992ed09b54dea Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Mon, 17 Aug 2026 13:31:06 -0400 Subject: [PATCH 1/3] fix(catalog): survive an unreachable registry, and ask for the gap Three things a search that goes wrong could not do. A registry the network cannot reach emptied the catalog. The cache read returned undefined past its 24h TTL, so an expired entry and no entry at all were indistinguishable, and one timeout against the registry host reported the whole catalog as unreachable while a usable copy sat on disk. Freshness is now the caller's decision: a fresh entry still skips the network, a stale one is served once revalidation fails, and only a name never fetched on this machine can still fail outright. The gap-report channel was never offered at the moment it is for. `feedback --search-miss` names moves the catalog does not have, which is the one signal install counts cannot produce, and it was documented only in a skill file and a help screen, neither of them open when a query comes back wrong. `catalog --query` now prints the line pre-filled, and every --json envelope carries it as `report_gap`, on hits as well as on zero results: the reports worth having come from searches that returned plausible items where none of them did the job. The registry skill owns `hyperframes catalog` and never mentioned the channel at all; it does now, alongside what a stale-served registry means. The CLI skill gains the three commands nothing named, and says plainly which two entries in --help are not for agents to run. Test plan: 8 new tests in remote.test.ts, 3 of which fail without the cache change; 4 new in catalog.test.ts pinning report_gap and the shell-escaping of a non-ASCII query. Full CLI suite: 2632 passed, 3 pre-existing failures (2 transcribe, 1 hostname-dependent) unchanged from the base commit. Both output paths exercised against the real CLI. --- packages/cli/src/commands/catalog.test.ts | 41 +++++- packages/cli/src/commands/catalog.ts | 44 +++++++ packages/cli/src/registry/remote.test.ts | 148 ++++++++++++++++++++++ packages/cli/src/registry/remote.ts | 57 ++++++--- skills-manifest.json | 4 +- skills/hyperframes-cli/SKILL.md | 13 +- skills/hyperframes-registry/SKILL.md | 16 +++ 7 files changed, 301 insertions(+), 22 deletions(-) create mode 100644 packages/cli/src/registry/remote.test.ts diff --git a/packages/cli/src/commands/catalog.test.ts b/packages/cli/src/commands/catalog.test.ts index 27f0b61e67..39c5bf12d2 100644 --- a/packages/cli/src/commands/catalog.test.ts +++ b/packages/cli/src/commands/catalog.test.ts @@ -1,6 +1,6 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; -import { countUnindexed, pickByName } from "./catalog.js"; +import { countUnindexed, pickByName, searchMissCommand } from "./catalog.js"; /** The whole registry, which is what "in this registry" has to be measured against. */ const registryNames = new Set(["fade-through", "whip-pan", "count-up"]); @@ -347,6 +347,45 @@ describe("catalog --json meaning search", () => { "on-device search is using the previous catalog vectors because the update failed", ]); }); + + it("hands back the gap-report command even when the search found things", async () => { + // The reports we actually want come from searches that returned plausible + // items where none of them did the job. If the command only appeared on + // zero results it would be absent from every case worth reporting. + const envelope = await runEnvelope({ query: "make a number count up" }); + + expect(envelope.shown).toBeGreaterThan(0); + expect(envelope.report_gap).toBe( + 'npx hyperframes feedback --search-miss "make a number count up" ' + + '--wanted "" --tier on-device', + ); + }); + + it("names the tier that actually answered in the gap-report command", async () => { + state.modelStatus = "declined"; + state.ranking = null; + + const envelope = await runEnvelope({ query: "count up" }); + + expect(envelope.tier).toBe("words"); + expect(envelope.report_gap).toContain("--tier words"); + }); +}); + +describe("searchMissCommand", () => { + it("keeps a non-ASCII query intact", () => { + // Half of the gap reports received so far were CJK. A query mangled on the + // way into the command is a report nobody can act on. + expect(searchMissCommand("実写写真のみ 9:16 生活ハック", "on-device")).toContain( + '--search-miss "実写写真のみ 9:16 生活ハック"', + ); + }); + + it("escapes shell metacharacters so the printed line is safe to paste", () => { + const cmd = searchMissCommand('a "quoted" $VAR `sub` \\ thing', "words"); + + expect(cmd).toContain('--search-miss "a \\"quoted\\" \\$VAR \\`sub\\` \\\\ thing"'); + }); }); describe("catalog meaning search, on a terminal", () => { diff --git a/packages/cli/src/commands/catalog.ts b/packages/cli/src/commands/catalog.ts index 07e5481b77..764e64712d 100644 --- a/packages/cli/src/commands/catalog.ts +++ b/packages/cli/src/commands/catalog.ts @@ -259,6 +259,7 @@ export default defineCommand({ shown: 0, total: tagged.length, ...(warnings.length ? { warnings } : {}), + report_gap: searchMissCommand(query, tierToken(searched)), results: [], }, null, @@ -274,6 +275,13 @@ export default defineCommand({ args.tag ? `tag "${args.tag}"` : null, ].filter(Boolean); console.log(`No items match ${criteria.join(" and ")}.`); + // Zero results is the unambiguous case: no tier judgement to make and + // nothing to install, so name the gap channel outright. + if (query) { + console.log(""); + console.log(c.dim(" Nothing in the catalog does this? Report the gap:")); + console.log(c.dim(` ${searchMissCommand(query, tierToken(searched))}`)); + } } if (query) await offerLocalModel(0, json, config.registry, artifactRevision); return; @@ -307,6 +315,12 @@ export default defineCommand({ shown: output.length, total: tagged.length, ...(warnings.length ? { warnings } : {}), + // Carried on hits too, not just on zero results. The gaps that + // actually get reported are the ones where the search returned + // plausible items and none of them did the thing — a judgement + // only the reader can make, so the command has to already be in + // the envelope by the time they make it. + report_gap: searchMissCommand(query, tierToken(searched)), results: output, }, null, @@ -349,6 +363,11 @@ export default defineCommand({ reportLocalModelOption(json); await offerLocalModel(matching.length, json, config.registry, artifactRevision); } + } else if (query) { + // On-device only. A thin result on the word tier is expected and not + // worth reporting, so nudging there would train people to ignore the + // line; a thin result once meaning search has answered is a real gap. + console.log(c.dim(` None of these do it? ${searchMissCommand(query, "on-device")}`)); } } @@ -559,6 +578,31 @@ function tierDetail(searched: { localMode: LocalMode } | null): string { return searched?.localMode === "local-model" ? "on-device meaning search" : "local word match"; } +/** + * The command that turns a fruitless search into a catalog gap report. + * + * A search that returns nothing usable is the only moment anyone knows what + * the catalog is missing, and it was also the one moment we said nothing: + * `feedback --search-miss` was documented in the skill and printed by + * `feedback --help`, neither of which is open when a query comes back wrong. + * Handing back the exact line, with the query already in it, is the whole + * difference between a gap someone reports and a gap someone works around. + * + * Only the command is built here. Sending it stays a separate deliberate act, + * so plain `catalog --query` keeps its promise that the query text never + * leaves the machine. + */ +export function searchMissCommand(query: string, tier: "on-device" | "words"): string { + // Double quotes with the shell metacharacters escaped: the queries that + // matter are plain-language phrases, and half the real ones so far were + // CJK, which single-quoting renders no more safely and reads worse. + const quoted = query.replace(/(["\\$`])/g, "\\$1"); + return ( + `npx hyperframes feedback --search-miss "${quoted}" ` + + `--wanted "" --tier ${tier}` + ); +} + type LocalMode = "local-model" | "words"; /** diff --git a/packages/cli/src/registry/remote.test.ts b/packages/cli/src/registry/remote.test.ts new file mode 100644 index 0000000000..5ddbba86f1 --- /dev/null +++ b/packages/cli/src/registry/remote.test.ts @@ -0,0 +1,148 @@ +import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +// The cache lives under homedir(), so the whole suite runs against a scratch +// home rather than the developer's own ~/.hyperframes. +const scratchHome = mkdtempSync(join(tmpdir(), "hf-remote-")); +vi.mock("node:os", async (importOriginal) => ({ + ...(await importOriginal()), + homedir: () => scratchHome, +})); + +const { fetchItemManifest, fetchRegistryManifest, DEFAULT_REGISTRY_URL } = + await import("./remote.js"); + +const MANIFEST = { name: "hyperframes", items: [{ name: "count-up" }] }; +const ITEM = { name: "count-up", type: "hyperframes:component", files: [] }; + +const ONE_DAY_MS = 24 * 60 * 60 * 1000; + +function ok(body: unknown): Response { + return { ok: true, status: 200, json: async () => body } as unknown as Response; +} + +/** + * Prime the cache with one good fetch, then jump past the 24h TTL with every + * later fetch failing: the exact shape of "the registry host stopped answering + * and the copy on disk is a day old". Returns the failing spy so a caller can + * assert the network was actually attempted — without that the fallback + * assertions pass even if the clock never moved. + */ +async function staleAfterPriming( + body: unknown, + prime: () => Promise, +): Promise>> { + vi.spyOn(globalThis, "fetch").mockResolvedValueOnce(ok(body)); + await prime(); + + vi.useFakeTimers({ shouldAdvanceTime: true }); + vi.setSystemTime(new Date(Date.now() + ONE_DAY_MS + 60_000)); + return vi.spyOn(globalThis, "fetch").mockRejectedValue(new Error("The operation was aborted")); +} + +beforeEach(() => { + rmSync(join(scratchHome, ".hyperframes"), { recursive: true, force: true }); + vi.restoreAllMocks(); + vi.useRealTimers(); +}); + +afterEach(() => { + vi.useRealTimers(); +}); + +afterAll(() => { + rmSync(scratchHome, { recursive: true, force: true }); +}); + +describe("fetchRegistryManifest", () => { + it("serves a fresh cache without touching the network", async () => { + const fetchSpy = vi.spyOn(globalThis, "fetch").mockResolvedValue(ok(MANIFEST)); + await fetchRegistryManifest(DEFAULT_REGISTRY_URL); + expect(fetchSpy).toHaveBeenCalledTimes(1); + + const second = await fetchRegistryManifest(DEFAULT_REGISTRY_URL); + + expect(second).toEqual(MANIFEST); + expect(fetchSpy).toHaveBeenCalledTimes(1); + }); + + it("serves the expired cache when the registry host times out", async () => { + // The failure this exists for: raw.githubusercontent.com stops answering, + // the entry is a day and a bit old, and before this fix the caller was + // told the whole catalog was unreachable while a usable copy sat on disk. + const fetchSpy = await staleAfterPriming(MANIFEST, () => + fetchRegistryManifest(DEFAULT_REGISTRY_URL), + ); + + await expect(fetchRegistryManifest(DEFAULT_REGISTRY_URL)).resolves.toEqual(MANIFEST); + expect(fetchSpy).toHaveBeenCalledTimes(1); + }); + + it("still reports unreachable when the network fails and nothing was cached", async () => { + vi.spyOn(globalThis, "fetch").mockRejectedValue(new Error("getaddrinfo ENOTFOUND")); + + await expect(fetchRegistryManifest(DEFAULT_REGISTRY_URL)).resolves.toBeUndefined(); + }); + + it("falls back to the stale copy under skipCache too", async () => { + // skipCache asks for something newer. It has never meant "rather have + // nothing than this", so a failed revalidation must not empty the result. + vi.spyOn(globalThis, "fetch").mockResolvedValueOnce(ok(MANIFEST)); + await fetchRegistryManifest(DEFAULT_REGISTRY_URL); + + vi.spyOn(globalThis, "fetch").mockRejectedValue(new Error("HTTP 503")); + + await expect(fetchRegistryManifest(DEFAULT_REGISTRY_URL, { skipCache: true })).resolves.toEqual( + MANIFEST, + ); + }); + + it("prefers a successful refetch over the cached copy", async () => { + vi.spyOn(globalThis, "fetch").mockResolvedValueOnce(ok(MANIFEST)); + await fetchRegistryManifest(DEFAULT_REGISTRY_URL); + + const fresher = { ...MANIFEST, items: [{ name: "count-up" }, { name: "push-in" }] }; + vi.spyOn(globalThis, "fetch").mockResolvedValue(ok(fresher)); + + await expect(fetchRegistryManifest(DEFAULT_REGISTRY_URL, { skipCache: true })).resolves.toEqual( + fresher, + ); + }); +}); + +describe("fetchItemManifest", () => { + it("serves the expired cache when the item fetch fails", async () => { + const fetchSpy = await staleAfterPriming(ITEM, () => + fetchItemManifest("count-up", "hyperframes:component", DEFAULT_REGISTRY_URL), + ); + + await expect( + fetchItemManifest("count-up", "hyperframes:component", DEFAULT_REGISTRY_URL), + ).resolves.toEqual(ITEM); + expect(fetchSpy).toHaveBeenCalledTimes(1); + }); + + it("still throws when the item fetch fails and nothing was cached", async () => { + // The documented contract for a genuinely unknown item, unchanged: callers + // that install by name have to be able to tell "offline" from "no such item". + vi.spyOn(globalThis, "fetch").mockRejectedValue(new Error("The operation was aborted")); + + await expect( + fetchItemManifest("never-fetched", "hyperframes:component", DEFAULT_REGISTRY_URL), + ).rejects.toThrow("The operation was aborted"); + }); + + it("surfaces an HTTP error for an item that does not exist", async () => { + vi.spyOn(globalThis, "fetch").mockResolvedValue({ + ok: false, + status: 404, + json: async () => ({}), + } as unknown as Response); + + await expect( + fetchItemManifest("no-such-move", "hyperframes:component", DEFAULT_REGISTRY_URL), + ).rejects.toThrow("HTTP 404"); + }); +}); diff --git a/packages/cli/src/registry/remote.ts b/packages/cli/src/registry/remote.ts index b7cd0bab7e..d8251847ec 100644 --- a/packages/cli/src/registry/remote.ts +++ b/packages/cli/src/registry/remote.ts @@ -45,18 +45,30 @@ function cachePath(baseUrl: string, key: string): string { return join(CACHE_DIR, `${slug}__${key}.json`); } -function readCache(path: string): T | undefined { +/** + * Read a cache entry regardless of age. Freshness is deliberately NOT decided + * here: a fresh entry lets a caller skip the network, and a stale one is still + * the best answer available once the network has already failed. Collapsing + * the two (returning undefined past the TTL) made a 25-hour-old manifest and + * no manifest at all indistinguishable, so one timeout against the registry + * host reported the entire catalog as unreachable and sent authors off to + * hand-write what they already had on disk. + */ +function readCacheEntry(path: string): CacheEntry | undefined { try { const entry = JSON.parse(readFileSync(path, "utf-8")) as CacheEntry; if (typeof entry.fetchedAt !== "number") return undefined; - if (Date.now() - entry.fetchedAt > CACHE_TTL_MS) return undefined; - return entry.data; + return entry; } catch { // Missing file or corrupt JSON → cache miss. return undefined; } } +function isFresh(entry: CacheEntry): boolean { + return Date.now() - entry.fetchedAt <= CACHE_TTL_MS; +} + function writeCache(path: string, data: T): void { try { mkdirSync(dirname(path), { recursive: true }); @@ -79,31 +91,37 @@ async function fetchJson(url: string): Promise { } /** - * Fetch the top-level registry.json manifest. Cached for 24h. - * Returns undefined if the registry is unreachable (offline / 404). + * Fetch the top-level registry.json manifest. Served from cache while fresh, + * revalidated after 24h, and — when revalidation fails — served stale rather + * than not at all. Returns undefined only when the registry is unreachable AND + * nothing was ever cached. + * + * `skipCache` forces revalidation; it does not forbid the stale fallback, + * because "check for something newer" and "rather have nothing than this" are + * different requests and only the first one is ever made. */ export async function fetchRegistryManifest( baseUrl: string = DEFAULT_REGISTRY_URL, options?: { skipCache?: boolean }, ): Promise { const cacheFile = cachePath(baseUrl, "registry"); - if (!options?.skipCache) { - const cached = readCache(cacheFile); - if (cached) return cached; - } + const cached = readCacheEntry(cacheFile); + if (!options?.skipCache && cached && isFresh(cached)) return cached.data; try { const manifest = await fetchJson(`${baseUrl}/registry.json`); writeCache(cacheFile, manifest); return manifest; } catch { - return undefined; + return cached?.data; } } /** - * Fetch a single item's `registry-item.json` manifest. Cached for 24h. - * Throws on network failure (callers decide whether to degrade gracefully). + * Fetch a single item's `registry-item.json` manifest. Same freshness policy as + * the top-level manifest: fresh from cache, else revalidate, else serve stale. + * Throws on network failure only when nothing was ever cached for this item + * (callers decide whether to degrade gracefully). */ export async function fetchItemManifest( name: string, @@ -112,13 +130,18 @@ export async function fetchItemManifest( ): Promise { const dir = ITEM_TYPE_DIRS[type]; const cacheFile = cachePath(baseUrl, `${dir}__${name}`); - const cached = readCache(cacheFile); - if (cached) return cached; + const cached = readCacheEntry(cacheFile); + if (cached && isFresh(cached)) return cached.data; const url = `${baseUrl}/${dir}/${name}/registry-item.json`; - const item = await fetchJson(url); - writeCache(cacheFile, item); - return item; + try { + const item = await fetchJson(url); + writeCache(cacheFile, item); + return item; + } catch (err) { + if (cached) return cached.data; + throw err; + } } /** diff --git a/skills-manifest.json b/skills-manifest.json index d6291f0e8c..343883997a 100644 --- a/skills-manifest.json +++ b/skills-manifest.json @@ -30,7 +30,7 @@ "files": 6 }, "hyperframes-cli": { - "hash": "e042fcaaa3f9767f", + "hash": "12acd85eafd0ab63", "files": 11 }, "hyperframes-core": { @@ -46,7 +46,7 @@ "files": 3 }, "hyperframes-registry": { - "hash": "9a4049ea33f0b914", + "hash": "5e127844efa5f0aa", "files": 12 }, "media-use": { diff --git a/skills/hyperframes-cli/SKILL.md b/skills/hyperframes-cli/SKILL.md index b01ee482c1..997b98cbc4 100644 --- a/skills/hyperframes-cli/SKILL.md +++ b/skills/hyperframes-cli/SKILL.md @@ -65,7 +65,7 @@ Treat tiny unstyled content, canvas-sized icons, missing hero elements, or timel - **Search the catalog before writing motion by hand.** `npx hyperframes catalog --query ""`. Search is entirely local: there is no hosted tier, no account, and the query text is never sent anywhere. By default it ranks on vocabulary shared with the item's name, title and description, which misses any phrasing that does not reuse the catalog's own wording. Add `--on-device` to rank by meaning instead (see the offline tier below). - **Read which tier answered; never infer it from results appearing.** With `--json` the envelope carries `query`, `tier` (`on-device` or `words`), `tier_detail`, `dropped`, `unindexed`, `shown`, `total` and `results`, plus `top_score` when the answering tier produces one and `warnings` when a tier was asked for and could not run. A weak result on `words` is expected; the same result on `on-device` is a bug. `top_score` is on-device only and has no threshold behind it: the ranker returns the whole catalog in some order for every query, so read it as evidence rather than as a pass or fail. - **`dropped` and `unindexed` are opposite skews between the registry and the on-device index, and rewording the query fixes neither.** `dropped` counts ranked names this registry cannot install, so the strongest matches are the ones being lost. `unindexed` counts registry moves the index cannot see at all, which no query can ever return. Refreshing the registry is not the answer to either: its manifest carries a 24h TTL and heals itself, while the vectors are a separately published artifact fetched into `~/.hyperframes/catalog/`. Re-running with `--on-device` refetches that index when `unindexed` is above zero, so that is the remedy to hand the user. A pure over-coverage skew (`dropped` above zero while `unindexed` is zero) does not trigger the refetch; clearing `~/.hyperframes/catalog/` is the only way out of that one. Both counts are of names rather than of results, so either can exceed `total`. -- **When meaning search comes back with nothing worth installing, say so.** `npx hyperframes feedback --search-miss "" --wanted "" --tier on-device`. This is the only path that sends a query anywhere, and it is a separate deliberate command precisely so plain `catalog --query` keeps its promise of sending nothing. Report the miss when the on-device tier answered and the top hits still do not do the thing; a weak result on the `words` tier is expected and is not worth reporting. What comes back is a list of moves the catalog does not have yet, read directly rather than guessed from install counts, so the phrasing that matters is the effect you wanted, not the item name you imagined. It carries no rating and never lands in the rating metric. +- **When meaning search comes back with nothing worth installing, say so.** `npx hyperframes feedback --search-miss "" --wanted "" --tier on-device`. You do not have to assemble that line: `catalog --query` prints it pre-filled once meaning search has answered, and every `--json` search envelope carries it as `report_gap` with the query and tier already correct — fill in `--wanted` and send. This is the only path that sends a query anywhere, and it is a separate deliberate command precisely so plain `catalog --query` keeps its promise of sending nothing. Report the miss when the on-device tier answered and the top hits still do not do the thing; a weak result on the `words` tier is expected and is not worth reporting. What comes back is a list of moves the catalog does not have yet, read directly rather than guessed from install counts, so the phrasing that matters is the effect you wanted, not the item name you imagined. It carries no rating and never lands in the rating metric. - **Offer the offline tier; never enable it silently.** A one-time ~33 MB download (a quantized ONNX build of `bge-small-en-v1.5` plus its tokenizer, pinned to a fixed revision) and the catalog vectors from the registry, both cached under `~/.hyperframes/`, neither added to the project or any package. Once cached it ranks by meaning with nothing sent. Say the size out loud and let the person decide, then pass `--on-device` (with `-y` to skip the prompt) once they agree. The interactive offer only fires on a TTY, and under `--json` nothing about it is printed at all, so in an agent run you have to raise it with the user yourself. - Prefer `--json` for agent and CI calls. Server-mode `render`, `preview`, and `play` do not provide ordinary JSON output; `preview --selection --json` and `preview --context --json` are query-mode exceptions. @@ -141,6 +141,15 @@ The specialized commands are deliberately documented by their owning workflows: npx hyperframes present --port 3004 --no-open npx hyperframes beats --json npx hyperframes keyframes --json +npx hyperframes media-treatment --capabilities +npx hyperframes figma asset KEY:10-20 ``` -`present` serves a navigable deck with presenter and audience synchronization. `beats` is the standalone Studio beat-grid utility defined in `references/beats.md`. `keyframes` surfaces seek-safe animation and motion-path diagnostics. +`present` serves a navigable deck with presenter and audience synchronization. `beats` is the standalone Studio beat-grid utility defined in `references/beats.md`. `keyframes` surfaces seek-safe animation and motion-path diagnostics. `media-treatment` discovers, applies, and clears deterministic looks on local footage — start with `--capabilities` for the overview and `--capability ` for one family; `/media-use` owns which treatment a brief is asking for. `figma` imports over the REST API with the `asset`, `tokens`, and `component` subcommands and needs `FIGMA_TOKEN`; motion and shader import have no REST endpoint and are agent-only, so `/figma` owns those. + +## Commands you should not run + +Two entries in `hyperframes --help` are not part of the authoring loop, and reaching for them wastes a turn: + +- `events` is the telemetry endpoint skills use to report their **own** invocation, ideally from a bundled script. It emits an anonymous event and exits 0 no matter what you pass it. It is not a way to read telemetry back, and an agent has no reason to call it by hand. +- `validate`, `inspect`, and `layout` are deprecated aliases kept for old scripts. `check` is the one that is maintained, and it is what every reference in this skill assumes. diff --git a/skills/hyperframes-registry/SKILL.md b/skills/hyperframes-registry/SKILL.md index 57c2f6cad2..7ae5ce0da3 100644 --- a/skills/hyperframes-registry/SKILL.md +++ b/skills/hyperframes-registry/SKILL.md @@ -110,12 +110,28 @@ npx hyperframes catalog --human-friendly The normal table and `--json` modes only list matches; install a selected name with `hyperframes add `. `--human-friendly` opens an interactive picker and installs the selected item immediately. In CI or agent workflows, prefer `--json` followed by an explicit `add`. +### Report what the catalog does not have + +When the search comes back and nothing in it does the job, say so before you hand-author the move: + +```bash +npx hyperframes feedback --search-miss "" --wanted "" --tier on-device +``` + +`catalog --query` prints this line for you, pre-filled, and `--json` carries it as `report_gap` — so it is already in hand at the moment you decide nothing fits. + +Report the miss when the **on-device** tier answered and the top hits still do not do the thing. A thin result on the `words` tier is expected and is not worth reporting. Describe the effect you wanted, not the item name you imagined: what comes back is a list of moves worth building, and a report naming a non-existent item teaches nothing. This is the only path that sends a query anywhere, which is exactly why it is a separate deliberate command rather than something the search does on its own. It carries no rating and never lands in the rating metric. + +This is the whole demand signal for the catalog. Skipping it means the gap you hit gets guessed at from install counts instead, which cannot see a move nobody could install. + If the CLI cannot reach the configured registry, inspect the raw manifest as a fallback: ```bash curl -s https://raw.githubusercontent.com/heygen-com/hyperframes/main/registry/registry.json ``` +A registry the CLI cannot reach does **not** empty the catalog: a previously fetched manifest keeps serving past its 24h refresh window whenever revalidation fails, so search and install still work against the last copy on disk. An item never fetched on this machine is the only thing a network failure can genuinely block. + Each item's `registry-item.json` contains: name, type, title, description, tags, dimensions (blocks only), duration (blocks only), and file list. See [discovery.md](./references/discovery.md) for details on filtering by type and tags. From e922ff4324ce90fef5becaced421fdd1b2d086df Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Mon, 17 Aug 2026 14:08:47 -0400 Subject: [PATCH 2/3] fix(catalog): typecheck the new tests, and correct the offline claim Review follow-ups on the three points raised. The package did not compile. `Envelope` in catalog.test.ts never gained `report_gap`, and the explicit return annotation on `staleAfterPriming` asserted "fetch" against `keyof typeof globalThis`, which does not hold even though the `vi.spyOn` calls themselves infer fine. `report_gap` is non-optional because both envelope-shaped emit sites sit inside `if (query)` branches and both set it; the annotation is now `MockInstance`, which says the same thing without the keyof constraint. Worth naming why a green suite hid this: vitest transpiles without typechecking, so the two signals are independent, and the pre-commit typecheck hook covers core, studio and scripts but not packages/cli -- which is why the commit passed locally and CI did not. The registry skill claimed more than the code delivers. Item files are never cached and `installer.ts` fetches every one of them unconditionally (the only skip is the local-edits guard, which protects a file you changed rather than serving one from disk), so a re-install offline still fails -- a step later, not less often. The doc now says discovery degrades gracefully and `add` does not, because that file is what an agent loads and acts on, and as written it would have sent someone to run `add` offline. An empty cached payload is a miss, not an answer. The callers now test the entry rather than the payload, so a parseable file with a numeric fetchedAt and a null body would short-circuit the fetch and hand `null` back typed as RegistryItem. Rejecting it in readCacheEntry keeps the miss failing toward "go ask the network". Test plan: 1 new test pinning the empty-payload guard, verified to fail with the guard removed. remote.test.ts 9, catalog.test.ts 27, both green. `bun run typecheck` in packages/cli is clean, which is the check that was red. --- packages/cli/src/commands/catalog.test.ts | 4 ++++ packages/cli/src/registry/remote.test.ts | 16 +++++++++++++++- packages/cli/src/registry/remote.ts | 6 ++++++ skills-manifest.json | 2 +- skills/hyperframes-registry/SKILL.md | 4 +++- 5 files changed, 29 insertions(+), 3 deletions(-) diff --git a/packages/cli/src/commands/catalog.test.ts b/packages/cli/src/commands/catalog.test.ts index 39c5bf12d2..dcf5589383 100644 --- a/packages/cli/src/commands/catalog.test.ts +++ b/packages/cli/src/commands/catalog.test.ts @@ -177,6 +177,10 @@ interface Envelope { top_score?: number; shown: number; warnings?: string[]; + // Not optional: both envelope-shaped emit sites are inside `if (query)` + // branches and both set it, so a search envelope without it is a bug rather + // than a shape the caller has to handle. + report_gap: string; } async function runCatalog(args: Record): Promise { diff --git a/packages/cli/src/registry/remote.test.ts b/packages/cli/src/registry/remote.test.ts index 5ddbba86f1..a6b046caf1 100644 --- a/packages/cli/src/registry/remote.test.ts +++ b/packages/cli/src/registry/remote.test.ts @@ -1,4 +1,5 @@ import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { MockInstance } from "vitest"; import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -33,7 +34,7 @@ function ok(body: unknown): Response { async function staleAfterPriming( body: unknown, prime: () => Promise, -): Promise>> { +): Promise> { vi.spyOn(globalThis, "fetch").mockResolvedValueOnce(ok(body)); await prime(); @@ -80,6 +81,19 @@ describe("fetchRegistryManifest", () => { expect(fetchSpy).toHaveBeenCalledTimes(1); }); + it("treats an empty cached payload as a miss rather than an answer", async () => { + // The callers test the entry, not the payload, so a file carrying a valid + // fetchedAt and a null body would otherwise short-circuit the fetch and be + // handed back as a manifest. + vi.spyOn(globalThis, "fetch").mockResolvedValueOnce(ok(null)); + await fetchRegistryManifest(DEFAULT_REGISTRY_URL); + + const fetchSpy = vi.spyOn(globalThis, "fetch").mockResolvedValue(ok(MANIFEST)); + + await expect(fetchRegistryManifest(DEFAULT_REGISTRY_URL)).resolves.toEqual(MANIFEST); + expect(fetchSpy).toHaveBeenCalledTimes(1); + }); + it("still reports unreachable when the network fails and nothing was cached", async () => { vi.spyOn(globalThis, "fetch").mockRejectedValue(new Error("getaddrinfo ENOTFOUND")); diff --git a/packages/cli/src/registry/remote.ts b/packages/cli/src/registry/remote.ts index d8251847ec..6bc0f3b1c4 100644 --- a/packages/cli/src/registry/remote.ts +++ b/packages/cli/src/registry/remote.ts @@ -58,6 +58,12 @@ function readCacheEntry(path: string): CacheEntry | undefined { try { const entry = JSON.parse(readFileSync(path, "utf-8")) as CacheEntry; if (typeof entry.fetchedAt !== "number") return undefined; + // The callers now test the entry rather than the payload, so an empty + // payload would satisfy them: `null` data would short-circuit the fetch + // and be handed back as a RegistryItem. Rejecting it here keeps the miss + // failing toward "go ask the network" rather than toward "the catalog is + // empty", which is the whole point of the change around it. + if (entry.data === undefined || entry.data === null) return undefined; return entry; } catch { // Missing file or corrupt JSON → cache miss. diff --git a/skills-manifest.json b/skills-manifest.json index 343883997a..04b25d430e 100644 --- a/skills-manifest.json +++ b/skills-manifest.json @@ -46,7 +46,7 @@ "files": 3 }, "hyperframes-registry": { - "hash": "5e127844efa5f0aa", + "hash": "2e8e0c7034dc4f6d", "files": 12 }, "media-use": { diff --git a/skills/hyperframes-registry/SKILL.md b/skills/hyperframes-registry/SKILL.md index 7ae5ce0da3..b643efdf9d 100644 --- a/skills/hyperframes-registry/SKILL.md +++ b/skills/hyperframes-registry/SKILL.md @@ -130,7 +130,9 @@ If the CLI cannot reach the configured registry, inspect the raw manifest as a f curl -s https://raw.githubusercontent.com/heygen-com/hyperframes/main/registry/registry.json ``` -A registry the CLI cannot reach does **not** empty the catalog: a previously fetched manifest keeps serving past its 24h refresh window whenever revalidation fails, so search and install still work against the last copy on disk. An item never fetched on this machine is the only thing a network failure can genuinely block. +A registry the CLI cannot reach does **not** empty the catalog for **discovery**: a previously fetched manifest keeps serving past its 24h refresh window whenever revalidation fails, so `catalog` and `catalog --query` still list and rank against the last copy on disk. + +**`add` still needs the network, even for an item you installed yesterday.** Only manifests are cached; the item's actual files are fetched on every install. So offline you can search, and you can see what an item is, but installing it fails at the file fetch. Do not promise a user an offline install. Each item's `registry-item.json` contains: name, type, title, description, tags, dimensions (blocks only), duration (blocks only), and file list. From f313bac8e167b329b5d45d0008d5ec530bd715fa Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Mon, 17 Aug 2026 14:37:44 -0400 Subject: [PATCH 3/3] fix(catalog): offer the gap report on the word tier too The nudge added earlier in this branch only printed once the on-device tier had answered, on the reasoning that a thin word-match result is explainable while a thin meaning-match result is a real gap. That reasoning silences the line in the only case that actually occurs. The on-device tier needs a consented 33 MB download, and under --json the offer is not even printed, so an agent run ranks on `words` unless it explicitly passed --on-device --yes. Every catalog gap reported since the channel opened came from the word tier; none came from on-device. Gating the prompt on on-device therefore guaranteed it would never be seen by the population that files these reports, and the accompanying skill text went further and told agents outright not to report a word-tier miss -- so an agent handed `report_gap` with `--tier words` would have followed the docs straight into discarding it. The nudge now prints on both tiers with the tier that actually answered, and both skills say to report whenever nothing fits rather than to hold out for a tier that is usually off. The tier still rides along in the report, which is what keeps a vocabulary miss distinguishable from a meaning miss when these are read -- that distinction belongs in the analysis, not in a gate that suppresses the data. Test plan: 2 new tests pinning the nudge on each tier, including the `--tier words` token. catalog.test.ts 29, remote.test.ts 9, both green; packages/cli typecheck clean. Confirmed against the real CLI: a word-tier query now prints the pre-filled command with --tier words. --- packages/cli/src/commands/catalog.test.ts | 20 ++++++++++++++++++++ packages/cli/src/commands/catalog.ts | 18 +++++++++++++----- skills-manifest.json | 4 ++-- skills/hyperframes-cli/SKILL.md | 2 +- skills/hyperframes-registry/SKILL.md | 2 +- 5 files changed, 37 insertions(+), 9 deletions(-) diff --git a/packages/cli/src/commands/catalog.test.ts b/packages/cli/src/commands/catalog.test.ts index dcf5589383..4a2260bba4 100644 --- a/packages/cli/src/commands/catalog.test.ts +++ b/packages/cli/src/commands/catalog.test.ts @@ -407,6 +407,26 @@ describe("catalog meaning search, on a terminal", () => { expect(output).not.toContain("missing from the on-device index"); }); + + it("offers the gap report on the word tier, not just on-device", async () => { + // The tier that answers almost every real search, because on-device needs + // a consented download. Gating the nudge on on-device left it unprinted in + // the only case that occurs, which is how the gap channel stayed silent. + state.modelStatus = "declined"; + state.ranking = null; + + const output = await runCatalog({ query: "count up" }); + + expect(output).toContain("None of these do it?"); + expect(output).toContain("--tier words"); + }); + + it("offers the gap report on the on-device tier too", async () => { + const output = await runCatalog({ query: "make a number count up" }); + + expect(output).toContain("None of these do it?"); + expect(output).toContain("--tier on-device"); + }); }); describe("the on-device download offer", () => { diff --git a/packages/cli/src/commands/catalog.ts b/packages/cli/src/commands/catalog.ts index 764e64712d..f041700365 100644 --- a/packages/cli/src/commands/catalog.ts +++ b/packages/cli/src/commands/catalog.ts @@ -363,11 +363,19 @@ export default defineCommand({ reportLocalModelOption(json); await offerLocalModel(matching.length, json, config.registry, artifactRevision); } - } else if (query) { - // On-device only. A thin result on the word tier is expected and not - // worth reporting, so nudging there would train people to ignore the - // line; a thin result once meaning search has answered is a real gap. - console.log(c.dim(` None of these do it? ${searchMissCommand(query, "on-device")}`)); + } + if (query) { + // Both tiers, deliberately. Gating this on on-device sounded right -- + // a thin word-match result is explainable, a thin meaning-match result + // is a real gap -- but it silences the line in the case that produces + // essentially every search: the on-device tier needs a consented 33 MB + // download, so an agent run is on `words` unless it explicitly opted + // in. Every catalog gap reported to date came from the word tier. The + // tier rides along in the command so a vocabulary miss stays + // distinguishable from a meaning miss when the reports are read. + console.log( + c.dim(` None of these do it? ${searchMissCommand(query, tierToken(searched))}`), + ); } } diff --git a/skills-manifest.json b/skills-manifest.json index 04b25d430e..2f6e534ed3 100644 --- a/skills-manifest.json +++ b/skills-manifest.json @@ -30,7 +30,7 @@ "files": 6 }, "hyperframes-cli": { - "hash": "12acd85eafd0ab63", + "hash": "7f61f9a970a94cc1", "files": 11 }, "hyperframes-core": { @@ -46,7 +46,7 @@ "files": 3 }, "hyperframes-registry": { - "hash": "2e8e0c7034dc4f6d", + "hash": "549017b6d436aaf8", "files": 12 }, "media-use": { diff --git a/skills/hyperframes-cli/SKILL.md b/skills/hyperframes-cli/SKILL.md index 997b98cbc4..8b776925ef 100644 --- a/skills/hyperframes-cli/SKILL.md +++ b/skills/hyperframes-cli/SKILL.md @@ -65,7 +65,7 @@ Treat tiny unstyled content, canvas-sized icons, missing hero elements, or timel - **Search the catalog before writing motion by hand.** `npx hyperframes catalog --query ""`. Search is entirely local: there is no hosted tier, no account, and the query text is never sent anywhere. By default it ranks on vocabulary shared with the item's name, title and description, which misses any phrasing that does not reuse the catalog's own wording. Add `--on-device` to rank by meaning instead (see the offline tier below). - **Read which tier answered; never infer it from results appearing.** With `--json` the envelope carries `query`, `tier` (`on-device` or `words`), `tier_detail`, `dropped`, `unindexed`, `shown`, `total` and `results`, plus `top_score` when the answering tier produces one and `warnings` when a tier was asked for and could not run. A weak result on `words` is expected; the same result on `on-device` is a bug. `top_score` is on-device only and has no threshold behind it: the ranker returns the whole catalog in some order for every query, so read it as evidence rather than as a pass or fail. - **`dropped` and `unindexed` are opposite skews between the registry and the on-device index, and rewording the query fixes neither.** `dropped` counts ranked names this registry cannot install, so the strongest matches are the ones being lost. `unindexed` counts registry moves the index cannot see at all, which no query can ever return. Refreshing the registry is not the answer to either: its manifest carries a 24h TTL and heals itself, while the vectors are a separately published artifact fetched into `~/.hyperframes/catalog/`. Re-running with `--on-device` refetches that index when `unindexed` is above zero, so that is the remedy to hand the user. A pure over-coverage skew (`dropped` above zero while `unindexed` is zero) does not trigger the refetch; clearing `~/.hyperframes/catalog/` is the only way out of that one. Both counts are of names rather than of results, so either can exceed `total`. -- **When meaning search comes back with nothing worth installing, say so.** `npx hyperframes feedback --search-miss "" --wanted "" --tier on-device`. You do not have to assemble that line: `catalog --query` prints it pre-filled once meaning search has answered, and every `--json` search envelope carries it as `report_gap` with the query and tier already correct — fill in `--wanted` and send. This is the only path that sends a query anywhere, and it is a separate deliberate command precisely so plain `catalog --query` keeps its promise of sending nothing. Report the miss when the on-device tier answered and the top hits still do not do the thing; a weak result on the `words` tier is expected and is not worth reporting. What comes back is a list of moves the catalog does not have yet, read directly rather than guessed from install counts, so the phrasing that matters is the effect you wanted, not the item name you imagined. It carries no rating and never lands in the rating metric. +- **When a search comes back with nothing worth installing, say so.** `npx hyperframes feedback --search-miss "" --wanted "" --tier `. You do not have to assemble that line: `catalog --query` prints it pre-filled, and every `--json` search envelope carries it as `report_gap` with the query and tier already correct — fill in `--wanted` and send. This is the only path that sends a query anywhere, and it is a separate deliberate command precisely so plain `catalog --query` keeps its promise of sending nothing. **Report on either tier**, whenever the results do not do the thing; do not hold out for the on-device tier, which needs a consented 33 MB download and is therefore off in most agent runs — waiting for it means never reporting at all. The tier rides along in the report, so a vocabulary miss stays distinguishable from a meaning miss without you having to judge which one you hit. What comes back is a list of moves the catalog does not have yet, read directly rather than guessed from install counts, so the phrasing that matters is the effect you wanted, not the item name you imagined. It carries no rating and never lands in the rating metric. - **Offer the offline tier; never enable it silently.** A one-time ~33 MB download (a quantized ONNX build of `bge-small-en-v1.5` plus its tokenizer, pinned to a fixed revision) and the catalog vectors from the registry, both cached under `~/.hyperframes/`, neither added to the project or any package. Once cached it ranks by meaning with nothing sent. Say the size out loud and let the person decide, then pass `--on-device` (with `-y` to skip the prompt) once they agree. The interactive offer only fires on a TTY, and under `--json` nothing about it is printed at all, so in an agent run you have to raise it with the user yourself. - Prefer `--json` for agent and CI calls. Server-mode `render`, `preview`, and `play` do not provide ordinary JSON output; `preview --selection --json` and `preview --context --json` are query-mode exceptions. diff --git a/skills/hyperframes-registry/SKILL.md b/skills/hyperframes-registry/SKILL.md index b643efdf9d..a6d571c992 100644 --- a/skills/hyperframes-registry/SKILL.md +++ b/skills/hyperframes-registry/SKILL.md @@ -120,7 +120,7 @@ npx hyperframes feedback --search-miss "" --wanted "