From 7f4f0b59636ed06913ca70ab716b96130a12cd11 Mon Sep 17 00:00:00 2001 From: dan sinclair Date: Wed, 8 Jul 2026 11:22:00 -0400 Subject: [PATCH 1/5] Remove code using the listing_meta.json file. This CL removes the infrastructure which validates and uses the listing_meta.json file. The listing_meta.json file itself will be removed in a followup CL. Issue: #4592 --- docs/adding_timing_metadata.md | 148 --------------------- src/common/framework/metadata.ts | 29 ---- src/common/internal/file_loader.ts | 4 +- src/common/internal/tree.ts | 45 +------ src/common/tools/crawl.ts | 119 ----------------- src/common/tools/gen_wpt_cts_html.ts | 17 --- src/common/tools/merge_listing_times.ts | 169 ------------------------ src/common/tools/validate.ts | 6 +- tools/gen_wpt_cfg_chunked2sec.json | 1 - 9 files changed, 6 insertions(+), 532 deletions(-) delete mode 100644 docs/adding_timing_metadata.md delete mode 100644 src/common/framework/metadata.ts delete mode 100644 src/common/tools/merge_listing_times.ts diff --git a/docs/adding_timing_metadata.md b/docs/adding_timing_metadata.md deleted file mode 100644 index e251524177d0..000000000000 --- a/docs/adding_timing_metadata.md +++ /dev/null @@ -1,148 +0,0 @@ -# Adding Timing Metadata - -## listing_meta.json files - -`listing_meta.json` files are SEMI AUTO-GENERATED. - -The raw data may be edited manually, to add entries or change timing values. - -The list of tests in this file is **not** guaranteed to stay up to date. -Use the generated `gen/*_variant_list*.json` if you need a complete list. - -The `subcaseMS` values are estimates. They can be set to 0 or omitted if for some reason -you can't estimate the time (or there's an existing test with a long name and -slow subcases that would result in query strings that are too long). -It's OK if the number is estimated too high. - -These entries are estimates for the amount of time that subcases take to run, -and are used as inputs into the WPT tooling to attempt to portion out tests into -approximately same-sized chunks. High estimates are OK, they just may generate -more chunks than necessary. - -To check for missing or 0 entries, run -`tools/validate --print-metadata-warnings src/webgpu` -and look at the resulting warnings. - -### Performance - -Note this data is typically captured by developers using higher-end -computers, so typical test machines might execute more slowly. For this -reason, the WPT chunking should be configured to generate chunks much shorter -than 5 seconds (a typical default time limit in WPT test executors) so they -should still execute in under 5 seconds on lower-end computers. - -## Problem - -When renaming or removing tests from the CTS you will see an error like this -when running `npm test` or `npm run standalone`: - -``` -ERROR: Non-existent tests found in listing_meta.json. Please update: - webgpu:api,operation,adapter,requestAdapter:old_test_that_got_renamed:* -``` - -## Solution - -This means there is a stale line in `src/webgpu/listing_meta.json` that needs -to be deleted, or updated to match the rename that you did. - -## Problem - -You run `tools/validate --print-metadata-warnings src/webgpu` -and want to fix the warnings. - -## Solution 1 (manual, best for one-off updates of simple tests) - -If you're developing new tests and need to update this file, it is sometimes -easiest to do so manually. Run your tests under your usual development workflow -and see how long they take. In the standalone web runner `npm start`, the total -time for a test case is reported on the right-hand side when the case logs are -expanded. - -Record the average time per *subcase* across all cases of the test (you may need -to compute this) into the `listing_meta.json` file. - -## Solution 2 (semi-automated) - -There exists tooling in the CTS repo for generating appropriate estimates for -these values, though they do require some manual intervention. The rest of this -doc will be a walkthrough of running these tools. - -Timing data can be captured in bulk and "merged" into this file using -the `merge_listing_times` tool. This is -This is useful when a large number of tests -change or otherwise a lot of tests need to be updated, but it also automates the -manual steps above. - -The tool can also be used without any inputs to reformat `listing_meta.json`. -Please read the help message of `merge_listing_times` for more information. - -### Websocket Logger - -The first tool that needs to be run is `websocket-logger`, which receives data -on a WebSocket channel at `localhost:59497` to capture timing data when CTS is run. This -should be run in a separate process/terminal, since it needs to stay running -throughout the following steps. - -In the `tools/websocket-logger/` directory: - -``` -npm ci -npm start -``` - -The output from this command will indicate where the results are being logged, -which will be needed later. For example: - -``` -... -Writing to wslog-2023-09-12T18-57-34.txt -... -``` - -See also [tools/websocket-logger/README.md](../tools/websocket-logger/README.md). - -### Running CTS - -Now we need to run the specific cases in CTS that we need to time. - -This should be possible under any development workflow by logging through a -side-channel (as long as its runtime environment, like Node, supports WebSockets). -Regardless of development workflow, you need to enable logToWebSocket flag -(`?log_to_web_socket=1` in browser, `--log-to-web-socket` on command line, or -just hack it in by switching the default in `options.ts`). - -The most well-tested way to do this is using the standalone web runner. - -This requires serving the CTS locally. In the project root: - -``` -npm run standalone -npm start -``` - -Once this is started you can then direct a WebGPU enabled browser to the -specific CTS entry and run the tests, for example: - -``` -http://localhost:8080/standalone/?log_to_web_socket=1&q=webgpu:* -``` - -If the tests have a high variance in runtime, you can run them multiple times. -The longest recorded time will be used. - -### Merging metadata - -The final step is to merge the new data that has been captured into the JSON -file. - -This can be done using the following command: - -``` -tools/merge_listing_times webgpu -- tools/websocket-logger/wslog-2023-09-12T18-57-34.txt -tools/merge_listing_times webgpu -- tools/websocket-logger/wslog-*.txt -``` - -Or, you can point it to one of the log files from a specific invocation of websocket-logger. - -Now you just need to commit the pending diff in your repo. diff --git a/src/common/framework/metadata.ts b/src/common/framework/metadata.ts deleted file mode 100644 index 5143cbf4c4f0..000000000000 --- a/src/common/framework/metadata.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { assert } from '../util/util.js'; - -/** Metadata about tests (that can't be derived at runtime). */ -export type TestMetadata = { - /** - * Estimated average time-per-subcase, in milliseconds. - * This is used to determine chunking granularity when exporting to WPT with - * chunking enabled (like out-wpt/cts-chunked2sec.https.html). - */ - subcaseMS: number; -}; - -export type TestMetadataListing = { - [testQuery: string]: TestMetadata; -}; - -export function loadMetadataForSuite(suiteDir: string): TestMetadataListing | null { - assert(typeof require !== 'undefined', 'loadMetadataForSuite is only implemented on Node'); - /* eslint-disable-next-line n/no-restricted-require */ - const fs = require('fs'); - - const metadataFile = `${suiteDir}/listing_meta.json`; - if (!fs.existsSync(metadataFile)) { - return null; - } - - const metadata: TestMetadataListing = JSON.parse(fs.readFileSync(metadataFile, 'utf8')); - return metadata; -} diff --git a/src/common/internal/file_loader.ts b/src/common/internal/file_loader.ts index aae4b8799549..6a918fa5ea48 100644 --- a/src/common/internal/file_loader.ts +++ b/src/common/internal/file_loader.ts @@ -74,8 +74,7 @@ export abstract class TestFileLoader extends EventTarget { { subqueriesToExpand = [], fullyExpandSubtrees = [], - maxChunkTime = Infinity, - }: { subqueriesToExpand?: string[]; fullyExpandSubtrees?: string[]; maxChunkTime?: number } = {} + }: { subqueriesToExpand?: string[]; fullyExpandSubtrees?: string[] } = {} ): Promise { const tree = await loadTreeForQuery(this, query, { subqueriesToExpand: subqueriesToExpand.map(s => { @@ -84,7 +83,6 @@ export abstract class TestFileLoader extends EventTarget { return q; }), fullyExpandSubtrees: fullyExpandSubtrees.map(s => parseQuery(s)), - maxChunkTime, }); this.dispatchEvent(new MessageEvent('finish')); return tree; diff --git a/src/common/internal/tree.ts b/src/common/internal/tree.ts index f2fad590373c..f2827e2f391e 100644 --- a/src/common/internal/tree.ts +++ b/src/common/internal/tree.ts @@ -1,4 +1,3 @@ -import { loadMetadataForSuite, TestMetadataListing } from '../framework/metadata.js'; import { globalTestConfig } from '../framework/test_config.js'; import { RunCase, RunFn } from '../internal/test_group.js'; import { assert, now } from '../util/util.js'; @@ -104,18 +103,10 @@ export class TestTree { static async create( forQuery: TestQuery, root: TestSubtree, - maxChunkTime: number ): Promise { const suite = forQuery.suite; - let chunking = undefined; - if (Number.isFinite(maxChunkTime)) { - const metadata = loadMetadataForSuite(`./src/${suite}`); - assert(metadata !== null, `metadata for ${suite} is missing, but maxChunkTime was requested`); - chunking = { metadata, maxChunkTime }; - } - await TestTree.propagateCounts(root, chunking); - + await TestTree.propagateCounts(root); return new TestTree(forQuery, root); } @@ -207,13 +198,12 @@ export class TestTree { /** Propagate the subtreeTODOs/subtreeTests state upward from leaves to parent nodes. */ static async propagateCounts( subtree: TestSubtree, - chunking: { metadata: TestMetadataListing; maxChunkTime: number } | undefined ): Promise<{ tests: number; nodesWithTODO: number; totalTimeMS: number; subcaseCount: number }> { subtree.subtreeCounts ??= { tests: 0, nodesWithTODO: 0, totalTimeMS: 0 }; subtree.subcaseCount = 0; for (const [, child] of subtree.children) { if ('children' in child) { - const counts = await TestTree.propagateCounts(child, chunking); + const counts = await TestTree.propagateCounts(child); subtree.subtreeCounts.tests += counts.tests; subtree.subtreeCounts.nodesWithTODO += counts.nodesWithTODO; subtree.subtreeCounts.totalTimeMS += counts.totalTimeMS; @@ -223,32 +213,6 @@ export class TestTree { } } - // If we're chunking based on a maxChunkTime, then at each - // TestQueryMultiCase node of the tree we look at its total time. If the - // total time is larger than the maxChunkTime, we set collapsible=false to - // make sure it gets split up in the output. Note: - // - TestQueryMultiTest and higher nodes are never set to collapsible anyway, so we ignore them. - // - TestQuerySingleCase nodes can't be collapsed, so we ignore them. - if (chunking && subtree.query instanceof TestQueryMultiCase) { - const testLevelQuery = new TestQueryMultiCase( - subtree.query.suite, - subtree.query.filePathParts, - subtree.query.testPathParts, - {} - ).toString(); - - const metadata = chunking.metadata; - - const subcaseTiming: number | undefined = metadata[testLevelQuery]?.subcaseMS; - if (subcaseTiming !== undefined) { - const totalTiming = subcaseTiming * subtree.subcaseCount; - subtree.subtreeCounts.totalTimeMS = totalTiming; - if (totalTiming > chunking.maxChunkTime) { - subtree.collapsible = false; - } - } - } - return { ...subtree.subtreeCounts, subcaseCount: subtree.subcaseCount ?? 0 }; } @@ -287,8 +251,7 @@ export async function loadTreeForQuery( { subqueriesToExpand, fullyExpandSubtrees = [], - maxChunkTime = Infinity, - }: { subqueriesToExpand: TestQuery[]; fullyExpandSubtrees?: TestQuery[]; maxChunkTime?: number } + }: { subqueriesToExpand: TestQuery[]; fullyExpandSubtrees?: TestQuery[] } ): Promise { const suite = queryToLoad.suite; const specs = await loader.listing(suite); @@ -454,7 +417,7 @@ export async function loadTreeForQuery( } assert(foundCase, `Query \`${queryToLoad.toString()}\` does not match any cases`); - return TestTree.create(queryToLoad, subtreeL0, maxChunkTime); + return TestTree.create(queryToLoad, subtreeL0); } function setSubtreeDescriptionAndCountTODOs( diff --git a/src/common/tools/crawl.ts b/src/common/tools/crawl.ts index 7cfbfa7f65ab..191bec411cfd 100644 --- a/src/common/tools/crawl.ts +++ b/src/common/tools/crawl.ts @@ -5,7 +5,6 @@ import * as fs from 'fs'; import * as path from 'path'; -import { loadMetadataForSuite } from '../framework/metadata.js'; import { SpecFile } from '../internal/file_loader.js'; import { TestQueryMultiCase, TestQueryMultiFile } from '../internal/query/query.js'; import { validQueryPart } from '../internal/query/validQueryPart.js'; @@ -47,30 +46,11 @@ async function crawlFilesRecursively(dir: string): Promise { export async function crawl( suiteDir: string, - opts: { - validate: boolean; - printMetadataWarnings: boolean; - printCaseCountReport: boolean; - } | null = null ): Promise { if (!fs.existsSync(suiteDir)) { throw new Error(`Could not find suite: ${suiteDir}`); } - let totalCases = 0; - let totalSubcases = 0; - - let validateTimingsEntries; - if (opts?.validate) { - const metadata = loadMetadataForSuite(suiteDir); - if (metadata) { - validateTimingsEntries = { - metadata, - testsFoundInFiles: new Set(), - }; - } - } - // Crawl files and convert paths to be POSIX-style, relative to suiteDir. const filesToEnumerate = (await crawlFilesRecursively(suiteDir)) .map(f => path.relative(suiteDir, f).replace(/\\/g, '/')) @@ -84,52 +64,6 @@ export async function crawl( const pathSegments = filepathWithoutExtension.split('/'); const suite = path.basename(suiteDir); - - if (opts?.validate) { - const filename = `../../${suite}/${filepathWithoutExtension}.spec.js`; - - assert(!process.env.STANDALONE_DEV_SERVER); - const mod = (await import(filename)) as SpecFile; - assert(mod.description !== undefined, 'Test spec file missing description: ' + filename); - assert(mod.g !== undefined, 'Test spec file missing TestGroup definition: ' + filename); - - mod.g.validate(new TestQueryMultiFile(suite, pathSegments)); - - if (opts?.printCaseCountReport || validateTimingsEntries) { - for (const t of mod.g.iterate()) { - const testQuery = new TestQueryMultiCase( - suite, - pathSegments, - t.testPath, - {} - ).toString(); - - let cases = 0; - let subcases = 0; - for (const c of t.iterate(null)) { - cases++; - if (opts?.printCaseCountReport) { - subcases += c.computeSubcaseCount(); - } - } - - if (opts?.printCaseCountReport) { - const perCase = (subcases / cases).toFixed(0); - console.log(`${testQuery} - ${cases} cases, ${subcases} subcases (~${perCase}/case)`); - totalCases += cases; - totalSubcases += subcases; - } - - if (validateTimingsEntries && cases > 0) { - validateTimingsEntries.testsFoundInFiles.add(testQuery); - } - } - } - } - - for (const p of pathSegments) { - assert(validQueryPart.test(p), `Invalid directory name ${p}; must match ${validQueryPart}`); - } entries.push({ file: pathSegments }); } else if (path.basename(file) === 'README.txt') { const dirname = path.dirname(file); @@ -142,59 +76,6 @@ export async function crawl( } } - if (validateTimingsEntries) { - const zeroEntries = []; - const staleEntries = []; - for (const [metadataKey, metadataValue] of Object.entries(validateTimingsEntries.metadata)) { - if (metadataKey.startsWith('_')) { - // Ignore json "_comments". - continue; - } - if (metadataValue.subcaseMS <= 0) { - zeroEntries.push(metadataKey); - } - if (!validateTimingsEntries.testsFoundInFiles.has(metadataKey)) { - staleEntries.push(metadataKey); - } - } - if (zeroEntries.length && opts?.printMetadataWarnings) { - console.warn( - 'WARNING: subcaseMS ≤ 0 found in listing_meta.json (see docs/adding_timing_metadata.md):' - ); - for (const metadataKey of zeroEntries) { - console.warn(` ${metadataKey}`); - } - } - - if (opts?.printMetadataWarnings) { - const missingEntries = []; - for (const metadataKey of validateTimingsEntries.testsFoundInFiles) { - if (!(metadataKey in validateTimingsEntries.metadata)) { - missingEntries.push(metadataKey); - } - } - if (missingEntries.length) { - console.error( - 'WARNING: Tests missing from listing_meta.json (see docs/adding_timing_metadata.md):' - ); - for (const metadataKey of missingEntries) { - console.error(` ${metadataKey}`); - } - } - } - - if (staleEntries.length) { - console.error('ERROR: Non-existent tests found in listing_meta.json. Please update:'); - for (const metadataKey of staleEntries) { - console.error(` ${metadataKey}`); - } - unreachable(); - } - } - - if (opts?.printCaseCountReport) { - console.log(`-----\nTOTAL: ${totalCases} cases, ${totalSubcases} subcases`); - } return entries; } diff --git a/src/common/tools/gen_wpt_cts_html.ts b/src/common/tools/gen_wpt_cts_html.ts index 8e5460f6b44f..0fc9860f26a2 100644 --- a/src/common/tools/gen_wpt_cts_html.ts +++ b/src/common/tools/gen_wpt_cts_html.ts @@ -30,7 +30,6 @@ gen_wpt_cts_html.ts. Example: "out": "path/to/output/cts.https.html", "outJSON": "path/to/output/webgpu_variant_list.json", "template": "path/to/template/cts.https.html", - "maxChunkTimeMS": 2000 } Usage (advanced) (deprecated, use config file): @@ -63,14 +62,6 @@ interface ConfigJSON { outVariantList?: string; /** Input template filename, relative to config file. */ template: string; - /** - * Maximum time for a single WPT "variant" chunk, in milliseconds. Defaults to infinity. - * - * This data is typically captured by developers on higher-end computers, so typical test - * machines might execute more slowly. For this reason, use a time much less than 5 seconds - * (a typical default time limit in WPT test executors). - */ - maxChunkTimeMS?: number; /** * List of argument prefixes (what comes before the test query), and optionally a list of * test queries to run under that prefix. Defaults to `['?q=']`. @@ -97,7 +88,6 @@ interface Config { out: string; outVariantList?: string; template: string; - maxChunkTimeMS: number; argumentsPrefixes: ArgumentsPrefixConfig[]; noLongPathAssert: boolean; expectations?: { @@ -144,7 +134,6 @@ let config: Config; suite: configJSON.suite, out: path.resolve(jsonFileDir, configJSON.out), template: path.resolve(jsonFileDir, configJSON.template), - maxChunkTimeMS: configJSON.maxChunkTimeMS ?? Infinity, argumentsPrefixes: configJSON.argumentsPrefixes ? [...reifyArgumentsPrefixesConfig(configJSON.argumentsPrefixes)] : [{ prefix: '?q=' }], @@ -185,7 +174,6 @@ let config: Config; suite, out: outFile, template: templateFile, - maxChunkTimeMS: Infinity, argumentsPrefixes: [{ prefix: '?q=' }], noLongPathAssert: false, }; @@ -206,8 +194,6 @@ let config: Config; printUsageAndExit(1); } - const useChunking = Number.isFinite(config.maxChunkTimeMS); - // Sort prefixes from longest to shortest config.argumentsPrefixes.sort((a, b) => b.prefix.length - a.prefix.length); @@ -247,12 +233,10 @@ let config: Config; const tree = await loader.loadTree(rootQuery, { subqueriesToExpand, fullyExpandSubtrees: fullyExpand.get(prefix), - maxChunkTime: config.maxChunkTimeMS, }); lines.push(undefined); // output blank line between prefixes const prefixComment = { comment: `Prefix: "${prefix}"` }; // contents will be updated later - if (useChunking) lines.push(prefixComment); const filesSeen = new Set(); const testsSeen = new Set(); @@ -288,7 +272,6 @@ let config: Config; lines.push({ urlQueryString: prefix + query.toString(), // "?debug=0&q=..." - comment: useChunking ? `estimated: ${subtreeCounts?.totalTimeMS.toFixed(3)} ms` : undefined, }); variantCount++; diff --git a/src/common/tools/merge_listing_times.ts b/src/common/tools/merge_listing_times.ts deleted file mode 100644 index a8bef354cc4f..000000000000 --- a/src/common/tools/merge_listing_times.ts +++ /dev/null @@ -1,169 +0,0 @@ -import * as fs from 'fs'; -import * as process from 'process'; -import * as readline from 'readline'; - -import { TestMetadataListing } from '../framework/metadata.js'; -import { parseQuery } from '../internal/query/parseQuery.js'; -import { TestQueryMultiCase, TestQuerySingleCase } from '../internal/query/query.js'; -import { CaseTimingLogLine } from '../internal/test_group.js'; -import { assert } from '../util/util.js'; - -// For information on listing_meta.json file maintenance, please read -// tools/merge_listing_times first. - -function usage(rc: number): never { - console.error(`Usage: tools/merge_listing_times [options] SUITES... -- [TIMING_LOG_FILES...] - -Options: - --help Print this message and exit. - -Reads raw case timing data for each suite in SUITES, from all TIMING_LOG_FILES -(see below), and merges it into the src/*/listing_meta.json files checked into -the repository. The timing data in the listing_meta.json files is updated with -the newly-observed timing data *if the new timing is slower*. That is, it will -only increase the values in the listing_meta.json file, and will only cause WPT -chunks to become smaller. - -If there are no TIMING_LOG_FILES, this just regenerates (reformats) the file -using the data already present. - -In more detail: - -- Reads per-case timing data in any of the SUITES, from all TIMING_LOG_FILES - (ignoring skipped cases), and averages it over the number of subcases. - In the case of cases that have run multiple times, takes the max of each. -- Compiles the average time-per-subcase for each test seen. -- For each suite seen, loads its listing_meta.json, takes the max of the old and - new data, and writes it back out. - -See 'docs/adding_timing_metadata.md' for how to generate TIMING_LOG_FILES files. -`); - process.exit(rc); -} - -const kHeader = `{ - "_comment": "SEMI AUTO-GENERATED. This list is NOT exhaustive. Please read docs/adding_timing_metadata.md.", -`; -const kFooter = `\ - "_end": "" -} -`; - -const argv = process.argv; -if (argv.some(v => v.startsWith('-') && v !== '--') || argv.every(v => v !== '--')) { - usage(0); -} -const suites = []; -const timingLogFilenames = []; -let seenDashDash = false; -for (const arg of argv.slice(2)) { - if (arg === '--') { - seenDashDash = true; - continue; - } else if (arg.startsWith('-')) { - usage(0); - } - - if (seenDashDash) { - timingLogFilenames.push(arg); - } else { - suites.push(arg); - } -} -if (!seenDashDash) { - usage(0); -} - -void (async () => { - // Read the log files to find the log line for each *case* query. If a case - // ran multiple times, take the one with the largest average subcase time. - const caseTimes = new Map(); - for (const timingLogFilename of timingLogFilenames) { - const rl = readline.createInterface({ - input: fs.createReadStream(timingLogFilename), - crlfDelay: Infinity, - }); - - for await (const line of rl) { - const parsed: CaseTimingLogLine = JSON.parse(line); - - const prev = caseTimes.get(parsed.q); - if (prev !== undefined) { - const timePerSubcase = parsed.timems / Math.max(1, parsed.nonskippedSubcaseCount); - const prevTimePerSubcase = prev.timems / Math.max(1, prev.nonskippedSubcaseCount); - - if (timePerSubcase > prevTimePerSubcase) { - caseTimes.set(parsed.q, parsed); - } - } else { - caseTimes.set(parsed.q, parsed); - } - } - } - - // Accumulate total times per test. Map of suite -> query -> {totalTimeMS, caseCount}. - const testTimes = new Map>(); - for (const suite of suites) { - testTimes.set(suite, new Map()); - } - for (const [caseQString, caseTime] of caseTimes) { - const caseQ = parseQuery(caseQString); - assert(caseQ instanceof TestQuerySingleCase); - const suite = caseQ.suite; - const suiteTestTimes = testTimes.get(suite); - if (suiteTestTimes === undefined) { - continue; - } - - const testQ = new TestQueryMultiCase(suite, caseQ.filePathParts, caseQ.testPathParts, {}); - const testQString = testQ.toString(); - - const prev = suiteTestTimes.get(testQString); - if (prev !== undefined) { - prev.totalTimeMS += caseTime.timems; - prev.subcaseCount += caseTime.nonskippedSubcaseCount; - } else { - suiteTestTimes.set(testQString, { - totalTimeMS: caseTime.timems, - subcaseCount: caseTime.nonskippedSubcaseCount, - }); - } - } - - for (const suite of suites) { - const currentMetadata: TestMetadataListing = JSON.parse( - fs.readFileSync(`./src/${suite}/listing_meta.json`, 'utf8') - ); - - const metadata = { ...currentMetadata }; - for (const [testQString, { totalTimeMS, subcaseCount }] of testTimes.get(suite)!) { - const avgTime = totalTimeMS / Math.max(1, subcaseCount); - if (testQString in metadata) { - metadata[testQString].subcaseMS = Math.max(metadata[testQString].subcaseMS, avgTime); - } else { - metadata[testQString] = { subcaseMS: avgTime }; - } - } - - writeListings(suite, metadata); - } -})(); - -function writeListings(suite: string, metadata: TestMetadataListing) { - const output = fs.createWriteStream(`./src/${suite}/listing_meta.json`); - try { - output.write(kHeader); - const keys = Object.keys(metadata).sort(); - for (const k of keys) { - if (k.startsWith('_')) { - // Ignore json "_comments". - continue; - } - assert(k.indexOf('"') === -1); - output.write(` "${k}": { "subcaseMS": ${metadata[k].subcaseMS.toFixed(3)} },\n`); - } - output.write(kFooter); - } finally { - output.close(); - } -} diff --git a/src/common/tools/validate.ts b/src/common/tools/validate.ts index 5486bf4602aa..6575343477ff 100644 --- a/src/common/tools/validate.ts +++ b/src/common/tools/validate.ts @@ -54,9 +54,5 @@ if (suiteDirs.length === 0) { } for (const suiteDir of suiteDirs) { - void crawl(suiteDir, { - validate: true, - printMetadataWarnings, - printCaseCountReport, - }); + void crawl(suiteDir); } diff --git a/tools/gen_wpt_cfg_chunked2sec.json b/tools/gen_wpt_cfg_chunked2sec.json index 123a06e9cd1d..3c2616617f7e 100644 --- a/tools/gen_wpt_cfg_chunked2sec.json +++ b/tools/gen_wpt_cfg_chunked2sec.json @@ -3,5 +3,4 @@ "out": "../out-wpt/cts-chunked2sec.https.html", "outVariantList": "../gen/webgpu_variant_list_chunked2sec.json", "template": "../src/common/templates/cts.https.html", - "maxChunkTimeMS": 2000 } From 69ba8ff9003f3dfe4876b0a5d55e9deb334320aa Mon Sep 17 00:00:00 2001 From: dan sinclair Date: Wed, 8 Jul 2026 11:27:41 -0400 Subject: [PATCH 2/5] remove chunked2sec --- Gruntfile.js | 5 ----- tools/gen_wpt_cfg_chunked2sec.json | 6 ------ 2 files changed, 11 deletions(-) delete mode 100644 tools/gen_wpt_cfg_chunked2sec.json diff --git a/Gruntfile.js b/Gruntfile.js index 54092f99d1b7..a8d903b8e8c6 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -58,10 +58,6 @@ module.exports = function (grunt) { cmd: 'node', args: ['tools/gen_wpt_cts_html', 'tools/gen_wpt_cfg_unchunked.json'], }, - 'write-out-wpt-cts-html-chunked2sec': { - cmd: 'node', - args: ['tools/gen_wpt_cts_html', 'tools/gen_wpt_cfg_chunked2sec.json'], - }, 'write-out-wpt-cts-html-withsomeworkers': { cmd: 'node', args: ['tools/gen_wpt_cts_html', 'tools/gen_wpt_cfg_withsomeworkers.json'], @@ -205,7 +201,6 @@ module.exports = function (grunt) { 'write-out-wpt-cts-html-all': { tasks: [ 'run:write-out-wpt-cts-html', - 'run:write-out-wpt-cts-html-chunked2sec', 'run:write-out-wpt-cts-html-withsomeworkers', ], }, diff --git a/tools/gen_wpt_cfg_chunked2sec.json b/tools/gen_wpt_cfg_chunked2sec.json deleted file mode 100644 index 3c2616617f7e..000000000000 --- a/tools/gen_wpt_cfg_chunked2sec.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "suite": "webgpu", - "out": "../out-wpt/cts-chunked2sec.https.html", - "outVariantList": "../gen/webgpu_variant_list_chunked2sec.json", - "template": "../src/common/templates/cts.https.html", -} From dca6bf424ec68de7871a160a3dfa5d2d897cdc67 Mon Sep 17 00:00:00 2001 From: dan sinclair Date: Wed, 8 Jul 2026 11:36:27 -0400 Subject: [PATCH 3/5] run lint fix --- Gruntfile.js | 5 +---- src/common/internal/tree.ts | 7 ++----- src/common/tools/crawl.ts | 4 +--- 3 files changed, 4 insertions(+), 12 deletions(-) diff --git a/Gruntfile.js b/Gruntfile.js index a8d903b8e8c6..8dfcf191a792 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -199,10 +199,7 @@ module.exports = function (grunt) { concurrent: { 'write-out-wpt-cts-html-all': { - tasks: [ - 'run:write-out-wpt-cts-html', - 'run:write-out-wpt-cts-html-withsomeworkers', - ], + tasks: ['run:write-out-wpt-cts-html', 'run:write-out-wpt-cts-html-withsomeworkers'], }, 'all-builds': { tasks: ['build-standalone', 'build-wpt', 'run:build-out-node'], diff --git a/src/common/internal/tree.ts b/src/common/internal/tree.ts index f2827e2f391e..2fd4bd9ab4bb 100644 --- a/src/common/internal/tree.ts +++ b/src/common/internal/tree.ts @@ -100,10 +100,7 @@ export class TestTree { ); } - static async create( - forQuery: TestQuery, - root: TestSubtree, - ): Promise { + static async create(forQuery: TestQuery, root: TestSubtree): Promise { const suite = forQuery.suite; await TestTree.propagateCounts(root); @@ -197,7 +194,7 @@ export class TestTree { /** Propagate the subtreeTODOs/subtreeTests state upward from leaves to parent nodes. */ static async propagateCounts( - subtree: TestSubtree, + subtree: TestSubtree ): Promise<{ tests: number; nodesWithTODO: number; totalTimeMS: number; subcaseCount: number }> { subtree.subtreeCounts ??= { tests: 0, nodesWithTODO: 0, totalTimeMS: 0 }; subtree.subcaseCount = 0; diff --git a/src/common/tools/crawl.ts b/src/common/tools/crawl.ts index 191bec411cfd..8d6af287794d 100644 --- a/src/common/tools/crawl.ts +++ b/src/common/tools/crawl.ts @@ -44,9 +44,7 @@ async function crawlFilesRecursively(dir: string): Promise { ); } -export async function crawl( - suiteDir: string, -): Promise { +export async function crawl(suiteDir: string): Promise { if (!fs.existsSync(suiteDir)) { throw new Error(`Could not find suite: ${suiteDir}`); } From 76faa39ebbf83cdef302fc6c4aba439e301ba527 Mon Sep 17 00:00:00 2001 From: dan sinclair Date: Wed, 8 Jul 2026 11:55:38 -0400 Subject: [PATCH 4/5] fix lint warnings --- src/common/internal/tree.ts | 2 -- src/common/tools/crawl.ts | 6 +----- src/common/tools/gen_wpt_cts_html.ts | 2 +- src/common/tools/validate.ts | 4 ---- 4 files changed, 2 insertions(+), 12 deletions(-) diff --git a/src/common/internal/tree.ts b/src/common/internal/tree.ts index 2fd4bd9ab4bb..a284c56a4153 100644 --- a/src/common/internal/tree.ts +++ b/src/common/internal/tree.ts @@ -101,8 +101,6 @@ export class TestTree { } static async create(forQuery: TestQuery, root: TestSubtree): Promise { - const suite = forQuery.suite; - await TestTree.propagateCounts(root); return new TestTree(forQuery, root); } diff --git a/src/common/tools/crawl.ts b/src/common/tools/crawl.ts index 8d6af287794d..4dc74e50a31a 100644 --- a/src/common/tools/crawl.ts +++ b/src/common/tools/crawl.ts @@ -5,11 +5,8 @@ import * as fs from 'fs'; import * as path from 'path'; -import { SpecFile } from '../internal/file_loader.js'; -import { TestQueryMultiCase, TestQueryMultiFile } from '../internal/query/query.js'; -import { validQueryPart } from '../internal/query/validQueryPart.js'; import { TestSuiteListingEntry, TestSuiteListing } from '../internal/test_suite_listing.js'; -import { assert, unreachable } from '../util/util.js'; +import { unreachable } from '../util/util.js'; const specFileSuffix = __filename.endsWith('.ts') ? '.spec.ts' : '.spec.js'; @@ -61,7 +58,6 @@ export async function crawl(suiteDir: string): Promise const filepathWithoutExtension = file.substring(0, file.length - specFileSuffix.length); const pathSegments = filepathWithoutExtension.split('/'); - const suite = path.basename(suiteDir); entries.push({ file: pathSegments }); } else if (path.basename(file) === 'README.txt') { const dirname = path.dirname(file); diff --git a/src/common/tools/gen_wpt_cts_html.ts b/src/common/tools/gen_wpt_cts_html.ts index 0fc9860f26a2..ee65cac9653a 100644 --- a/src/common/tools/gen_wpt_cts_html.ts +++ b/src/common/tools/gen_wpt_cts_html.ts @@ -243,7 +243,7 @@ let config: Config; let variantCount = 0; const alwaysExpandThroughLevel = 2; // expand to, at minimum, every test. - loopOverNodes: for (const { query, subtreeCounts } of tree.iterateCollapsedNodes({ + loopOverNodes: for (const { query, _subtreeCounts } of tree.iterateCollapsedNodes({ alwaysExpandThroughLevel, })) { assert(query instanceof TestQueryMultiCase); diff --git a/src/common/tools/validate.ts b/src/common/tools/validate.ts index 6575343477ff..ee4762987eee 100644 --- a/src/common/tools/validate.ts +++ b/src/common/tools/validate.ts @@ -32,16 +32,12 @@ if (args.indexOf('--help') !== -1) { usage(0); } -let printMetadataWarnings = false; -let printCaseCountReport = false; const suiteDirs = []; for (const arg of args) { switch (arg) { case '--print-metadata-warnings': - printMetadataWarnings = true; break; case '--print-case-count-report': - printCaseCountReport = true; break; default: suiteDirs.push(arg); From 508a4689ec6175eb4ccce5ae3c9ff0521269d32a Mon Sep 17 00:00:00 2001 From: dan sinclair Date: Wed, 8 Jul 2026 16:19:21 -0400 Subject: [PATCH 5/5] Restore case count report --- src/common/tools/crawl.ts | 56 +++++++++++++++++++++++++++- src/common/tools/gen_wpt_cts_html.ts | 2 +- src/common/tools/validate.ts | 7 +++- 3 files changed, 61 insertions(+), 4 deletions(-) diff --git a/src/common/tools/crawl.ts b/src/common/tools/crawl.ts index 4dc74e50a31a..ef41772ee527 100644 --- a/src/common/tools/crawl.ts +++ b/src/common/tools/crawl.ts @@ -5,8 +5,11 @@ import * as fs from 'fs'; import * as path from 'path'; +import { SpecFile } from '../internal/file_loader.js'; +import { TestQueryMultiCase, TestQueryMultiFile } from '../internal/query/query.js'; +import { validQueryPart } from '../internal/query/validQueryPart.js'; import { TestSuiteListingEntry, TestSuiteListing } from '../internal/test_suite_listing.js'; -import { unreachable } from '../util/util.js'; +import { assert, unreachable } from '../util/util.js'; const specFileSuffix = __filename.endsWith('.ts') ? '.spec.ts' : '.spec.js'; @@ -41,11 +44,17 @@ async function crawlFilesRecursively(dir: string): Promise { ); } -export async function crawl(suiteDir: string): Promise { +export async function crawl( + suiteDir: string, + opts: { validate: boolean; printCaseCountReport: boolean } | null = null +): Promise { if (!fs.existsSync(suiteDir)) { throw new Error(`Could not find suite: ${suiteDir}`); } + let totalCases = 0; + let totalSubcases = 0; + // Crawl files and convert paths to be POSIX-style, relative to suiteDir. const filesToEnumerate = (await crawlFilesRecursively(suiteDir)) .map(f => path.relative(suiteDir, f).replace(/\\/g, '/')) @@ -58,6 +67,45 @@ export async function crawl(suiteDir: string): Promise const filepathWithoutExtension = file.substring(0, file.length - specFileSuffix.length); const pathSegments = filepathWithoutExtension.split('/'); + if (opts?.validate) { + const suite = path.basename(suiteDir); + const filename = `../../${suite}/${filepathWithoutExtension}.spec.js`; + + assert(!process.env.STANDALONE_DEV_SERVER); + const mod = (await import(filename)) as SpecFile; + assert(mod.description !== undefined, 'Test spec file missing description: ' + filename); + assert(mod.g !== undefined, 'Test spec file missing TestGroup definition: ' + filename); + + mod.g.validate(new TestQueryMultiFile(suite, pathSegments)); + + if (opts?.printCaseCountReport) { + for (const t of mod.g.iterate()) { + const testQuery = new TestQueryMultiCase( + suite, + pathSegments, + t.testPath, + {} + ).toString(); + + let cases = 0; + let subcases = 0; + for (const c of t.iterate(null)) { + cases++; + subcases += c.computeSubcaseCount(); + } + + const perCase = (subcases / cases).toFixed(0); + console.log(`${testQuery} - ${cases} cases, ${subcases} subcases (~${perCase}/case)`); + totalCases += cases; + totalSubcases += subcases; + } + } + } + + for (const p of pathSegments) { + assert(validQueryPart.test(p), `Invalid directory name ${p}; must match ${validQueryPart}`); + } + entries.push({ file: pathSegments }); } else if (path.basename(file) === 'README.txt') { const dirname = path.dirname(file); @@ -70,6 +118,10 @@ export async function crawl(suiteDir: string): Promise } } + if (opts?.printCaseCountReport) { + console.log(`-----\nTOTAL: ${totalCases} cases, ${totalSubcases} subcases`); + } + return entries; } diff --git a/src/common/tools/gen_wpt_cts_html.ts b/src/common/tools/gen_wpt_cts_html.ts index ee65cac9653a..4e65e628402c 100644 --- a/src/common/tools/gen_wpt_cts_html.ts +++ b/src/common/tools/gen_wpt_cts_html.ts @@ -243,7 +243,7 @@ let config: Config; let variantCount = 0; const alwaysExpandThroughLevel = 2; // expand to, at minimum, every test. - loopOverNodes: for (const { query, _subtreeCounts } of tree.iterateCollapsedNodes({ + loopOverNodes: for (const { query } of tree.iterateCollapsedNodes({ alwaysExpandThroughLevel, })) { assert(query instanceof TestQueryMultiCase); diff --git a/src/common/tools/validate.ts b/src/common/tools/validate.ts index ee4762987eee..7fbc548bcadc 100644 --- a/src/common/tools/validate.ts +++ b/src/common/tools/validate.ts @@ -32,12 +32,14 @@ if (args.indexOf('--help') !== -1) { usage(0); } +let printCaseCountReport = false; const suiteDirs = []; for (const arg of args) { switch (arg) { case '--print-metadata-warnings': break; case '--print-case-count-report': + printCaseCountReport = true; break; default: suiteDirs.push(arg); @@ -50,5 +52,8 @@ if (suiteDirs.length === 0) { } for (const suiteDir of suiteDirs) { - void crawl(suiteDir); + void crawl(suiteDir, { + validate: true, + printCaseCountReport, + }); }