diff --git a/packages/cli/src/commands/catalog.test.ts b/packages/cli/src/commands/catalog.test.ts index 4a2260bba4..3d8410b984 100644 --- a/packages/cli/src/commands/catalog.test.ts +++ b/packages/cli/src/commands/catalog.test.ts @@ -376,6 +376,74 @@ describe("catalog --json meaning search", () => { }); }); +describe("a query with no searchable words", () => { + // Runs the command capturing stderr, and reports the exit code the CLI would + // have used. finishCommand throws a signal rather than calling process.exit. + async function runForExit( + args: Record, + ): Promise<{ exitCode: number; err: string }> { + const command = (await import("./catalog.js")).default as unknown as { + run: (context: { args: Record }) => Promise; + }; + const lines: string[] = []; + const capture = (...parts: unknown[]): void => { + lines.push(parts.map(String).join(" ")); + }; + const log = vi.spyOn(console, "log").mockImplementation(capture); + const error = vi.spyOn(console, "error").mockImplementation(capture); + let exitCode = 0; + try { + await command.run({ args }); + } catch (thrown) { + const signal = thrown as { result?: { exitCode?: number }; exitCode?: number }; + exitCode = signal.result?.exitCode ?? signal.exitCode ?? -1; + } finally { + log.mockRestore(); + error.mockRestore(); + } + // eslint-disable-next-line no-control-regex + const esc = String.fromCharCode(27); + return { exitCode, err: lines.join("\n").split(`${esc}[`).join("").replace(/\d+m/g, "") }; + } + + it("exits non-zero, because it is bad input rather than an empty shelf", async () => { + // The flag is word-tier only, so the stubbed on-device ranker has to be off + // or it answers with hits and the branch never runs. + state.modelStatus = "declined"; + state.ranking = null; + + const { exitCode } = await runForExit({ query: "実写写真のみ 9:16 生活ハック" }); + + // An agent that only reads the exit code would otherwise take "success, no + // results" at face value and hand-author a move already in the registry. + expect(exitCode).toBe(1); + }); + + it("says to search in English and does not blame the catalog", async () => { + state.modelStatus = "declined"; + state.ranking = null; + + const { err } = await runForExit({ query: "実写写真のみ 9:16 生活ハック" }); + + expect(err).toContain("No searchable words in query"); + expect(err).toContain("Search in English"); + expect(err).toContain("not a gap in"); + // The gap channel must not be offered: nothing was searched, so a report + // here is noise in the one signal that tells us what to build. + expect(err).not.toContain("--search-miss"); + }); + + it("leaves a genuine empty result exiting zero", async () => { + state.ranking = []; + state.modelStatus = "declined"; + + const { exitCode, err } = await runForExit({ query: "quantum entanglement reactor" }); + + expect(exitCode).toBe(0); + expect(err).toContain("No items match"); + }); +}); + describe("searchMissCommand", () => { it("keeps a non-ASCII query intact", () => { // Half of the gap reports received so far were CJK. A query mangled on the diff --git a/packages/cli/src/commands/catalog.ts b/packages/cli/src/commands/catalog.ts index f041700365..213578a57f 100644 --- a/packages/cli/src/commands/catalog.ts +++ b/packages/cli/src/commands/catalog.ts @@ -18,7 +18,7 @@ import { loadProjectConfig, DEFAULT_PROJECT_CONFIG } from "../utils/projectConfi import { resolve } from "node:path"; import { finishCommand } from "../utils/commandResult.js"; import { runAdd } from "./add.js"; -import { searchByWords } from "../registry/localSearch.js"; +import { hasNoSearchableTokens, searchByWords } from "../registry/localSearch.js"; import { downloadOfferMessage, ensureLocalModel, @@ -243,6 +243,13 @@ export default defineCommand({ const matching = searched ? searched.items : tagged; if (matching.length === 0) { + // "We could not read your query" is a different answer from "the catalog + // has nothing like this", and only one of them is worth reporting as a + // gap. Word matching indexes English, so a query in another script parses + // to no tokens at all and used to return the same empty list as a genuine + // miss -- sending an author off to report a move that may well exist. + const unsearchable = + Boolean(query) && searched?.localMode === "words" && hasNoSearchableTokens(query as string); // An empty result is exactly when the tier matters most: nothing found on // the weakest tier means something different from nothing found on the // best one. @@ -259,7 +266,13 @@ export default defineCommand({ shown: 0, total: tagged.length, ...(warnings.length ? { warnings } : {}), - report_gap: searchMissCommand(query, tierToken(searched)), + ...(unsearchable ? { unsearchable_query: true } : {}), + // Withheld when the query never parsed: the catalog has not been + // shown to be missing anything, so inviting a gap report here + // would file noise against a search that never ran. + ...(unsearchable + ? {} + : { report_gap: searchMissCommand(query, tierToken(searched)) }), results: [], }, null, @@ -274,16 +287,34 @@ export default defineCommand({ query ? `query "${query}"` : null, args.tag ? `tag "${args.tag}"` : null, ].filter(Boolean); - console.log(`No items match ${criteria.join(" and ")}.`); - // Zero results is the unambiguous case: no tier judgement to make and - // nothing to install, so name the gap channel outright. - if (query) { - console.log(""); - console.log(c.dim(" Nothing in the catalog does this? Report the gap:")); - console.log(c.dim(` ${searchMissCommand(query, tierToken(searched))}`)); + if (unsearchable) { + console.error(c.error(`No searchable words in query "${query}".`)); + console.error(""); + console.error( + c.warn( + " Word matching indexes the catalog in English, so a query written in another\n" + + " script produces no terms to match and returns nothing. This is not a gap in\n" + + " the catalog. Search in English; the on-screen copy of your video can stay\n" + + " in any language.", + ), + ); + } else { + console.log(`No items match ${criteria.join(" and ")}.`); + // Zero results is the unambiguous case: no tier judgement to make and + // nothing to install, so name the gap channel outright. + if (query) { + console.log(""); + console.log(c.dim(" Nothing in the catalog does this? Report the gap:")); + console.log(c.dim(` ${searchMissCommand(query, tierToken(searched))}`)); + } } } if (query) await offerLocalModel(0, json, config.registry, artifactRevision); + // A query with no searchable words is bad input, not an empty shelf, so it + // exits non-zero like an invalid --type does. An agent that only checks the + // exit code would otherwise read "searched successfully, catalog has + // nothing" and go hand-author a move that is sitting in the registry. + if (unsearchable) finishCommand(1); return; } @@ -561,11 +592,11 @@ async function applySearch< // Shared vocabulary, not substring presence. A user asking to "make the // pace feel faster" shares no literal substring with any description, so // the old test returned nothing at all for exactly the phrasing people use. - const words = searchByWords( - query, - items, - (item) => `${item.name} ${item.title} ${item.description} ${(item.tags ?? []).join(" ")}`, - ); + const words = searchByWords(query, items, (item) => ({ + // Name and title are what an author types when they already know the move. + strong: `${item.name} ${item.title}`, + weak: `${item.description} ${(item.tags ?? []).join(" ")}`, + })); // Word matching ranks the items in hand, so nothing can go missing. return { items: words, localMode: "words", warnings, missing: 0, unindexed: 0, topScore: null }; } diff --git a/packages/cli/src/registry/localSearch.test.ts b/packages/cli/src/registry/localSearch.test.ts index 056846de11..9cd94f2458 100644 --- a/packages/cli/src/registry/localSearch.test.ts +++ b/packages/cli/src/registry/localSearch.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; -import { rankByWords, searchByWords, tokenize } from "./localSearch.js"; +import { hasNoSearchableTokens, rankByWords, searchByWords, tokenize } from "./localSearch.js"; interface Item { name: string; @@ -13,7 +13,7 @@ const ITEMS: Item[] = [ { name: "text-reveal", text: "Type reveals line by line beneath a mask." }, ]; -const textOf = (item: Item) => item.text; +const textOf = (item: Item) => ({ strong: "", weak: item.text }); describe("tokenize", () => { it("drops stop words", () => { @@ -25,7 +25,7 @@ describe("tokenize", () => { }); it("lowercases and strips punctuation and digits", () => { - expect(tokenize("Camera, pushes 42 times!")).toEqual(["camera", "pushes", "times"]); + expect(tokenize("Camera, pushes 42 times!")).toEqual(["camera", "push", "time"]); }); it("returns nothing for a query made only of stop words", () => { @@ -95,3 +95,91 @@ describe("searchByWords", () => { expect(searchByWords("quantum entanglement", ITEMS, textOf)).toEqual([]); }); }); + +// ── The three ranking defects this scorer exists to fix ───────────────────── +// Each case is a query that failed against the real catalog, reduced to the +// smallest fixture that still reproduces it. + +const named = (name: string, title: string, body: string) => ({ name, title, body }); +const fieldsOf = (i: { name: string; title: string; body: string }) => ({ + strong: `${i.name} ${i.title}`, + weak: i.body, +}); + +describe("name and title outrank description", () => { + const items = [ + named("typewriter", "Typewriter", "Character-by-character text reveal with a blinking cursor."), + named("mk-emphasis-type", "Emphasis Type", "A phrase types on with emphasis on the key word."), + named("variable-axis-type", "Variable Axis Type", "Type that shifts along a variable axis."), + ]; + + it("puts the item actually called typewriter first", () => { + // Ranked seventh against the real catalog before field weighting: an author + // typing a move's name got entries that merely mention typing. + expect(rankByWords("typewriter effect on a title", items, fieldsOf)[0]?.item.name).toBe( + "typewriter", + ); + }); +}); + +describe("plurals fold to the singular", () => { + const items = [ + named( + "count-up", + "Count Up", + "A stat counter that eases between values and lands with a pulse.", + ), + named("skeleton-reveal", "Skeleton Reveal", "Placeholder blocks resolve into content once."), + ]; + + it("matches a pluralised query against a singular description", () => { + // "counts" is not "count" and "pulses" is not "pulse", so adding detail to + // a query used to make the result strictly worse. + expect( + rankByWords("a stat that counts up and then pulses once", items, fieldsOf)[0]?.item.name, + ).toBe("count-up"); + }); + + it("leaves a word that merely ends in s alone", () => { + expect(tokenize("press glass")).toEqual(["press", "glass"]); + }); +}); + +describe("common words count for less than distinctive ones", () => { + const items = [ + named("line-by-line-slide", "Line By Line Slide", "A headline arrives one line at a time."), + named("ui-3d-reveal", "UI 3D Reveal", "A panel reveals in three dimensions."), + named("panel-reveal", "Panel Reveal", "A panel reveals itself."), + named("pull-back-reveal", "Pull Back Reveal", "The camera pulls back to reveal the scene."), + ]; + + it("does not let one hit on a catalog-wide word beat several precise ones", () => { + // Field weighting alone made this worse: every item merely NAMED *-reveal + // outranked the one that does the thing, because "reveal" is the catalog's + // most common word. Inverse document frequency is what settles it. + expect(rankByWords("reveal a headline one line at a time", items, fieldsOf)[0]?.item.name).toBe( + "line-by-line-slide", + ); + }); +}); + +describe("hasNoSearchableTokens", () => { + it("flags a query in a script this ranker cannot index", () => { + // The catalog is written in English and tokenising on [a-z]+ leaves nothing + // of a Japanese query. Reporting that as "no items match" told the author + // the catalog lacked a move it may well have. + expect(hasNoSearchableTokens("実写写真のみ 9:16 生活ハック")).toBe(true); + }); + + it("does not flag an ordinary query that simply matches nothing", () => { + expect(hasNoSearchableTokens("quantum entanglement")).toBe(false); + }); + + it("does not flag an empty query, which is a listing rather than a search", () => { + expect(hasNoSearchableTokens(" ")).toBe(false); + }); + + it("flags a query that is only stop words and punctuation", () => { + expect(hasNoSearchableTokens("the and of !!!")).toBe(true); + }); +}); diff --git a/packages/cli/src/registry/localSearch.ts b/packages/cli/src/registry/localSearch.ts index fde4bbc9d7..7c94b9e7fc 100644 --- a/packages/cli/src/registry/localSearch.ts +++ b/packages/cli/src/registry/localSearch.ts @@ -6,12 +6,26 @@ * so the honest result was zero matches. Scoring shared vocabulary answers it * partially, offline, with no model and no network. * - * The scorer is the one the retrieval evaluation used, reproduced so the local - * arm and the remote fallback rank identically rather than merely similarly: - * lowercase alphabetic tokens, stop words dropped, anything three characters or - * shorter dropped, and the shared-token count divided by the square root of the - * entry's token count. That divisor is load-bearing. Without it the wordiest - * entry wins every query on sheer surface area. + * The scorer is the one the retrieval evaluation used: lowercase tokens, stop + * words dropped, anything three characters or shorter dropped, and the shared + * token count divided by the square root of the entry's token count. That + * divisor is load-bearing. Without it the wordiest entry wins every query on + * sheer surface area. + * + * Two things sit on top of that, both measured against a fixed eval set of + * catalog queries rather than adjusted by feel: + * + * 1. A token matching an item's NAME or TITLE counts for more than one + * matching its description. Without this, searching "typewriter effect on a + * title" ranked the item literally called `typewriter` seventh, behind + * entries that merely mention typing. An author who types a move's name is + * giving the strongest signal available and it was being averaged away. + * + * 2. Plurals fold to their singular on both sides. "a stat that counts up and + * then pulses once" shares no token with a description reading "lands with + * a restrained scale pulse", because `counts` is not `count` and `pulses` + * is not `pulse`. Adding detail to a query made results strictly worse, + * which is the opposite of what a search should do. * * Ties break on descending name, matching the evaluation's sort. */ @@ -26,9 +40,51 @@ const STOP = new Set( ).split(" "), ); +/** How much more a name/title token is worth than a description token. */ +const STRONG_FIELD_WEIGHT = 3; + +/** + * Fold a plural to its singular, and nothing else. + * + * Deliberately not a real stemmer. Porter would fold `counter` to `count` and + * `values` to `valu`, which merges moves that mean different things and makes + * the failure harder to read when it happens. Plurals are the case that + * actually bit: every extra descriptive word an author adds tends to arrive + * pluralised while the catalog writes its descriptions in the singular. + */ +const PLURAL_RULES: ReadonlyArray = [ + [/([^aeiou])ies$/, "$1y"], // stories -> story + [/(ch|sh|ss|x)es$/, "$1"], // matches -> match, glasses -> glass + [/(ss|us|is)$/, "$&"], // press, status, axis: the trailing s is not a plural + [/s$/, ""], // counts -> count, pulses -> pulse +]; + +function singularize(word: string): string { + if (word.length <= 3) return word; + // First rule wins, so the guard above `s$` is what protects press and status. + for (const [pattern, replacement] of PLURAL_RULES) { + if (pattern.test(word)) return word.replace(pattern, replacement); + } + return word; +} + export function tokenize(text: string): string[] { const words = text.toLowerCase().match(/[a-z]+/g) ?? []; - return words.filter((word) => word.length > 2 && !STOP.has(word)); + return words.filter((word) => word.length > 2 && !STOP.has(word)).map(singularize); +} + +/** + * True when a query contains no token this ranker can search on. + * + * The catalog is written in English and tokenising on `[a-z]+` means a query + * in another script produces nothing at all. That used to be indistinguishable + * from "the catalog has nothing like this", so a Japanese query printed the + * same "no items match" as a genuine gap and sent the author off to report a + * missing move that may well exist. Callers use this to say which of the two + * actually happened. + */ +export function hasNoSearchableTokens(query: string): boolean { + return tokenize(query).length === 0 && query.trim().length > 0; } export interface Scored { @@ -36,6 +92,14 @@ export interface Scored { score: number; } +/** The searchable text of an item, split by how much a match in it counts. */ +export interface ItemText { + /** Name and title: what an author types when they know what they want. */ + strong: string; + /** Description, tags, everything else. */ + weak: string; +} + /** * Rank every item by shared vocabulary, best first. * @@ -45,25 +109,51 @@ export interface Scored { export function rankByWords( query: string, items: T[], - textOf: (item: T) => string, + textOf: (item: T) => ItemText, ): Scored[] { const want = new Set(tokenize(query)); if (want.size === 0) return items.map((item) => ({ item, score: 0 })); - return items - .map((item) => { - const have = new Set(tokenize(textOf(item))); + const parsed = items.map((item) => { + const { strong, weak } = textOf(item); + const strongTokens = new Set(tokenize(strong)); + return { item, strongTokens, allTokens: new Set([...strongTokens, ...tokenize(weak)]) }; + }); + + // How rare each queried word is across the catalog. Without this a common + // word carries the same weight as a distinctive one, and field weighting + // makes that worse rather than better: searching "reveal a headline one line + // at a time" put every item merely NAMED `*-reveal` on top, because one + // strong hit on the catalog's most common word outscored several weak hits + // on the words that actually narrowed it down. + const idf = new Map(); + for (const token of want) { + const df = parsed.reduce((count, p) => count + (p.allTokens.has(token) ? 1 : 0), 0); + // +1 inside the log keeps a token present in every item at a small + // positive weight rather than exactly zero: still nearly worthless, but + // never able to flip a tie on its own. + idf.set(token, Math.log((parsed.length + 1) / (df + 1)) + 1); + } + + return parsed + .map(({ item, strongTokens, allTokens }) => { let shared = 0; - for (const token of want) if (have.has(token)) shared += 1; + for (const token of want) { + const weight = idf.get(token) ?? 1; + if (strongTokens.has(token)) shared += STRONG_FIELD_WEIGHT * weight; + else if (allTokens.has(token)) shared += weight; + } // sqrt normalization: a longer entry has more chances to overlap, and - // without this the wordiest blurb ranks first for every query. - return { item, score: shared / (Math.sqrt(have.size) || 1) }; + // without this the wordiest blurb ranks first for every query. Measured + // over the whole token set, so weighting a field cannot be gamed by + // moving words into the name. + return { item, score: shared / (Math.sqrt(allTokens.size) || 1) }; }) .sort((a, b) => b.score - a.score || nameOf(b.item).localeCompare(nameOf(a.item))); } /** Only items sharing at least one token, best first. */ -export function searchByWords(query: string, items: T[], textOf: (item: T) => string): T[] { +export function searchByWords(query: string, items: T[], textOf: (item: T) => ItemText): T[] { return rankByWords(query, items, textOf) .filter((scored) => scored.score > 0) .map((scored) => scored.item); diff --git a/skills-manifest.json b/skills-manifest.json index 2f6e534ed3..bd26200334 100644 --- a/skills-manifest.json +++ b/skills-manifest.json @@ -30,7 +30,7 @@ "files": 6 }, "hyperframes-cli": { - "hash": "7f61f9a970a94cc1", + "hash": "dc900e2ef0aa16b2", "files": 11 }, "hyperframes-core": { @@ -46,7 +46,7 @@ "files": 3 }, "hyperframes-registry": { - "hash": "549017b6d436aaf8", + "hash": "d6dfcc8ceb7c8178", "files": 12 }, "media-use": { diff --git a/skills/hyperframes-cli/SKILL.md b/skills/hyperframes-cli/SKILL.md index 8b776925ef..0c05d814df 100644 --- a/skills/hyperframes-cli/SKILL.md +++ b/skills/hyperframes-cli/SKILL.md @@ -62,7 +62,8 @@ Treat tiny unstyled content, canvas-sized icons, missing hero elements, or timel ## Agent conventions -- **Search the catalog before writing motion by hand.** `npx hyperframes catalog --query ""`. Search is entirely local: there is no hosted tier, no account, and the query text is never sent anywhere. By default it ranks on vocabulary shared with the item's name, title and description, which misses any phrasing that does not reuse the catalog's own wording. Add `--on-device` to rank by meaning instead (see the offline tier below). +- **Search the catalog before writing motion by hand.** `npx hyperframes catalog --query ""`. Search is entirely local: there is no hosted tier, no account, and the query text is never sent anywhere. By default it ranks on vocabulary shared with the item's name, title and description, which misses any phrasing that does not reuse the catalog's own wording. Add `--on-device` to rank by meaning instead (see the offline tier below). +- **Query in English even when the video is not.** Both tiers index an English catalog, so a query in another script produces no searchable terms and returns nothing. Describe the move in English; the on-screen copy stays in whatever language the video needs. `No searchable words in query` means exactly this and is not a missing component, so do not report it as a catalog gap. - **Read which tier answered; never infer it from results appearing.** With `--json` the envelope carries `query`, `tier` (`on-device` or `words`), `tier_detail`, `dropped`, `unindexed`, `shown`, `total` and `results`, plus `top_score` when the answering tier produces one and `warnings` when a tier was asked for and could not run. A weak result on `words` is expected; the same result on `on-device` is a bug. `top_score` is on-device only and has no threshold behind it: the ranker returns the whole catalog in some order for every query, so read it as evidence rather than as a pass or fail. - **`dropped` and `unindexed` are opposite skews between the registry and the on-device index, and rewording the query fixes neither.** `dropped` counts ranked names this registry cannot install, so the strongest matches are the ones being lost. `unindexed` counts registry moves the index cannot see at all, which no query can ever return. Refreshing the registry is not the answer to either: its manifest carries a 24h TTL and heals itself, while the vectors are a separately published artifact fetched into `~/.hyperframes/catalog/`. Re-running with `--on-device` refetches that index when `unindexed` is above zero, so that is the remedy to hand the user. A pure over-coverage skew (`dropped` above zero while `unindexed` is zero) does not trigger the refetch; clearing `~/.hyperframes/catalog/` is the only way out of that one. Both counts are of names rather than of results, so either can exceed `total`. - **When a search comes back with nothing worth installing, say so.** `npx hyperframes feedback --search-miss "" --wanted "" --tier `. You do not have to assemble that line: `catalog --query` prints it pre-filled, and every `--json` search envelope carries it as `report_gap` with the query and tier already correct — fill in `--wanted` and send. This is the only path that sends a query anywhere, and it is a separate deliberate command precisely so plain `catalog --query` keeps its promise of sending nothing. **Report on either tier**, whenever the results do not do the thing; do not hold out for the on-device tier, which needs a consented 33 MB download and is therefore off in most agent runs — waiting for it means never reporting at all. The tier rides along in the report, so a vocabulary miss stays distinguishable from a meaning miss without you having to judge which one you hit. What comes back is a list of moves the catalog does not have yet, read directly rather than guessed from install counts, so the phrasing that matters is the effect you wanted, not the item name you imagined. It carries no rating and never lands in the rating metric. diff --git a/skills/hyperframes-registry/SKILL.md b/skills/hyperframes-registry/SKILL.md index a6d571c992..42a54d918a 100644 --- a/skills/hyperframes-registry/SKILL.md +++ b/skills/hyperframes-registry/SKILL.md @@ -95,6 +95,8 @@ npx hyperframes add caption-clip-wipe Search is local and sends nothing. By default it ranks on vocabulary shared with the item's name, title and description, so it only finds items that reuse your words; `--on-device` ranks by meaning instead, after a one-time model download. With `--json` the envelope names which tier answered, so check that rather than assuming a ranking happened. +**Always query in English, whatever language the video is in.** The catalog is written in English and both tiers index it that way (the on-device model is English-only too). A query in another script produces no searchable terms and returns nothing at all. This is easy to get wrong on a Japanese or Chinese project, where the brief, the captions and the narration are all in that language and the query naturally follows: describe the _move_ in English, then write the on-screen copy in whatever language the video needs. If a query does come back with `No searchable words in query`, that is this rule, not a missing component, and it is not worth a gap report. + Installability is applied after ranking, not before it: a name the vectors carry but this registry cannot serve is dropped from the results and counted in `dropped`, so a non-zero `dropped` means the two are different generations. See `/hyperframes-cli` for the offline tier, the consent gates, and how to refresh a stale index. To browse or filter instead of search: