diff --git a/Gruntfile.js b/Gruntfile.js index 54092f99d1b7..8dfcf191a792 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -58,10 +58,6 @@ module.exports = function (grunt) { cmd: 'node', args: ['tools/gen_wpt_cts_html', 'tools/gen_wpt_cfg_unchunked.json'], }, - 'write-out-wpt-cts-html-chunked2sec': { - cmd: 'node', - args: ['tools/gen_wpt_cts_html', 'tools/gen_wpt_cfg_chunked2sec.json'], - }, 'write-out-wpt-cts-html-withsomeworkers': { cmd: 'node', args: ['tools/gen_wpt_cts_html', 'tools/gen_wpt_cfg_withsomeworkers.json'], @@ -203,11 +199,7 @@ module.exports = function (grunt) { concurrent: { 'write-out-wpt-cts-html-all': { - tasks: [ - 'run:write-out-wpt-cts-html', - 'run:write-out-wpt-cts-html-chunked2sec', - 'run:write-out-wpt-cts-html-withsomeworkers', - ], + tasks: ['run:write-out-wpt-cts-html', 'run:write-out-wpt-cts-html-withsomeworkers'], }, 'all-builds': { tasks: ['build-standalone', 'build-wpt', 'run:build-out-node'], diff --git a/docs/adding_timing_metadata.md b/docs/adding_timing_metadata.md deleted file mode 100644 index e251524177d0..000000000000 --- a/docs/adding_timing_metadata.md +++ /dev/null @@ -1,148 +0,0 @@ -# Adding Timing Metadata - -## listing_meta.json files - -`listing_meta.json` files are SEMI AUTO-GENERATED. - -The raw data may be edited manually, to add entries or change timing values. - -The list of tests in this file is **not** guaranteed to stay up to date. -Use the generated `gen/*_variant_list*.json` if you need a complete list. - -The `subcaseMS` values are estimates. They can be set to 0 or omitted if for some reason -you can't estimate the time (or there's an existing test with a long name and -slow subcases that would result in query strings that are too long). -It's OK if the number is estimated too high. - -These entries are estimates for the amount of time that subcases take to run, -and are used as inputs into the WPT tooling to attempt to portion out tests into -approximately same-sized chunks. High estimates are OK, they just may generate -more chunks than necessary. - -To check for missing or 0 entries, run -`tools/validate --print-metadata-warnings src/webgpu` -and look at the resulting warnings. - -### Performance - -Note this data is typically captured by developers using higher-end -computers, so typical test machines might execute more slowly. For this -reason, the WPT chunking should be configured to generate chunks much shorter -than 5 seconds (a typical default time limit in WPT test executors) so they -should still execute in under 5 seconds on lower-end computers. - -## Problem - -When renaming or removing tests from the CTS you will see an error like this -when running `npm test` or `npm run standalone`: - -``` -ERROR: Non-existent tests found in listing_meta.json. Please update: - webgpu:api,operation,adapter,requestAdapter:old_test_that_got_renamed:* -``` - -## Solution - -This means there is a stale line in `src/webgpu/listing_meta.json` that needs -to be deleted, or updated to match the rename that you did. - -## Problem - -You run `tools/validate --print-metadata-warnings src/webgpu` -and want to fix the warnings. - -## Solution 1 (manual, best for one-off updates of simple tests) - -If you're developing new tests and need to update this file, it is sometimes -easiest to do so manually. Run your tests under your usual development workflow -and see how long they take. In the standalone web runner `npm start`, the total -time for a test case is reported on the right-hand side when the case logs are -expanded. - -Record the average time per *subcase* across all cases of the test (you may need -to compute this) into the `listing_meta.json` file. - -## Solution 2 (semi-automated) - -There exists tooling in the CTS repo for generating appropriate estimates for -these values, though they do require some manual intervention. The rest of this -doc will be a walkthrough of running these tools. - -Timing data can be captured in bulk and "merged" into this file using -the `merge_listing_times` tool. This is -This is useful when a large number of tests -change or otherwise a lot of tests need to be updated, but it also automates the -manual steps above. - -The tool can also be used without any inputs to reformat `listing_meta.json`. -Please read the help message of `merge_listing_times` for more information. - -### Websocket Logger - -The first tool that needs to be run is `websocket-logger`, which receives data -on a WebSocket channel at `localhost:59497` to capture timing data when CTS is run. This -should be run in a separate process/terminal, since it needs to stay running -throughout the following steps. - -In the `tools/websocket-logger/` directory: - -``` -npm ci -npm start -``` - -The output from this command will indicate where the results are being logged, -which will be needed later. For example: - -``` -... -Writing to wslog-2023-09-12T18-57-34.txt -... -``` - -See also [tools/websocket-logger/README.md](../tools/websocket-logger/README.md). - -### Running CTS - -Now we need to run the specific cases in CTS that we need to time. - -This should be possible under any development workflow by logging through a -side-channel (as long as its runtime environment, like Node, supports WebSockets). -Regardless of development workflow, you need to enable logToWebSocket flag -(`?log_to_web_socket=1` in browser, `--log-to-web-socket` on command line, or -just hack it in by switching the default in `options.ts`). - -The most well-tested way to do this is using the standalone web runner. - -This requires serving the CTS locally. In the project root: - -``` -npm run standalone -npm start -``` - -Once this is started you can then direct a WebGPU enabled browser to the -specific CTS entry and run the tests, for example: - -``` -http://localhost:8080/standalone/?log_to_web_socket=1&q=webgpu:* -``` - -If the tests have a high variance in runtime, you can run them multiple times. -The longest recorded time will be used. - -### Merging metadata - -The final step is to merge the new data that has been captured into the JSON -file. - -This can be done using the following command: - -``` -tools/merge_listing_times webgpu -- tools/websocket-logger/wslog-2023-09-12T18-57-34.txt -tools/merge_listing_times webgpu -- tools/websocket-logger/wslog-*.txt -``` - -Or, you can point it to one of the log files from a specific invocation of websocket-logger. - -Now you just need to commit the pending diff in your repo. diff --git a/src/common/framework/metadata.ts b/src/common/framework/metadata.ts deleted file mode 100644 index 5143cbf4c4f0..000000000000 --- a/src/common/framework/metadata.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { assert } from '../util/util.js'; - -/** Metadata about tests (that can't be derived at runtime). */ -export type TestMetadata = { - /** - * Estimated average time-per-subcase, in milliseconds. - * This is used to determine chunking granularity when exporting to WPT with - * chunking enabled (like out-wpt/cts-chunked2sec.https.html). - */ - subcaseMS: number; -}; - -export type TestMetadataListing = { - [testQuery: string]: TestMetadata; -}; - -export function loadMetadataForSuite(suiteDir: string): TestMetadataListing | null { - assert(typeof require !== 'undefined', 'loadMetadataForSuite is only implemented on Node'); - /* eslint-disable-next-line n/no-restricted-require */ - const fs = require('fs'); - - const metadataFile = `${suiteDir}/listing_meta.json`; - if (!fs.existsSync(metadataFile)) { - return null; - } - - const metadata: TestMetadataListing = JSON.parse(fs.readFileSync(metadataFile, 'utf8')); - return metadata; -} diff --git a/src/common/internal/file_loader.ts b/src/common/internal/file_loader.ts index aae4b8799549..6a918fa5ea48 100644 --- a/src/common/internal/file_loader.ts +++ b/src/common/internal/file_loader.ts @@ -74,8 +74,7 @@ export abstract class TestFileLoader extends EventTarget { { subqueriesToExpand = [], fullyExpandSubtrees = [], - maxChunkTime = Infinity, - }: { subqueriesToExpand?: string[]; fullyExpandSubtrees?: string[]; maxChunkTime?: number } = {} + }: { subqueriesToExpand?: string[]; fullyExpandSubtrees?: string[] } = {} ): Promise { const tree = await loadTreeForQuery(this, query, { subqueriesToExpand: subqueriesToExpand.map(s => { @@ -84,7 +83,6 @@ export abstract class TestFileLoader extends EventTarget { return q; }), fullyExpandSubtrees: fullyExpandSubtrees.map(s => parseQuery(s)), - maxChunkTime, }); this.dispatchEvent(new MessageEvent('finish')); return tree; diff --git a/src/common/internal/tree.ts b/src/common/internal/tree.ts index f2fad590373c..a284c56a4153 100644 --- a/src/common/internal/tree.ts +++ b/src/common/internal/tree.ts @@ -1,4 +1,3 @@ -import { loadMetadataForSuite, TestMetadataListing } from '../framework/metadata.js'; import { globalTestConfig } from '../framework/test_config.js'; import { RunCase, RunFn } from '../internal/test_group.js'; import { assert, now } from '../util/util.js'; @@ -101,21 +100,8 @@ export class TestTree { ); } - static async create( - forQuery: TestQuery, - root: TestSubtree, - maxChunkTime: number - ): Promise { - const suite = forQuery.suite; - - let chunking = undefined; - if (Number.isFinite(maxChunkTime)) { - const metadata = loadMetadataForSuite(`./src/${suite}`); - assert(metadata !== null, `metadata for ${suite} is missing, but maxChunkTime was requested`); - chunking = { metadata, maxChunkTime }; - } - await TestTree.propagateCounts(root, chunking); - + static async create(forQuery: TestQuery, root: TestSubtree): Promise { + await TestTree.propagateCounts(root); return new TestTree(forQuery, root); } @@ -206,14 +192,13 @@ export class TestTree { /** Propagate the subtreeTODOs/subtreeTests state upward from leaves to parent nodes. */ static async propagateCounts( - subtree: TestSubtree, - chunking: { metadata: TestMetadataListing; maxChunkTime: number } | undefined + subtree: TestSubtree ): Promise<{ tests: number; nodesWithTODO: number; totalTimeMS: number; subcaseCount: number }> { subtree.subtreeCounts ??= { tests: 0, nodesWithTODO: 0, totalTimeMS: 0 }; subtree.subcaseCount = 0; for (const [, child] of subtree.children) { if ('children' in child) { - const counts = await TestTree.propagateCounts(child, chunking); + const counts = await TestTree.propagateCounts(child); subtree.subtreeCounts.tests += counts.tests; subtree.subtreeCounts.nodesWithTODO += counts.nodesWithTODO; subtree.subtreeCounts.totalTimeMS += counts.totalTimeMS; @@ -223,32 +208,6 @@ export class TestTree { } } - // If we're chunking based on a maxChunkTime, then at each - // TestQueryMultiCase node of the tree we look at its total time. If the - // total time is larger than the maxChunkTime, we set collapsible=false to - // make sure it gets split up in the output. Note: - // - TestQueryMultiTest and higher nodes are never set to collapsible anyway, so we ignore them. - // - TestQuerySingleCase nodes can't be collapsed, so we ignore them. - if (chunking && subtree.query instanceof TestQueryMultiCase) { - const testLevelQuery = new TestQueryMultiCase( - subtree.query.suite, - subtree.query.filePathParts, - subtree.query.testPathParts, - {} - ).toString(); - - const metadata = chunking.metadata; - - const subcaseTiming: number | undefined = metadata[testLevelQuery]?.subcaseMS; - if (subcaseTiming !== undefined) { - const totalTiming = subcaseTiming * subtree.subcaseCount; - subtree.subtreeCounts.totalTimeMS = totalTiming; - if (totalTiming > chunking.maxChunkTime) { - subtree.collapsible = false; - } - } - } - return { ...subtree.subtreeCounts, subcaseCount: subtree.subcaseCount ?? 0 }; } @@ -287,8 +246,7 @@ export async function loadTreeForQuery( { subqueriesToExpand, fullyExpandSubtrees = [], - maxChunkTime = Infinity, - }: { subqueriesToExpand: TestQuery[]; fullyExpandSubtrees?: TestQuery[]; maxChunkTime?: number } + }: { subqueriesToExpand: TestQuery[]; fullyExpandSubtrees?: TestQuery[] } ): Promise { const suite = queryToLoad.suite; const specs = await loader.listing(suite); @@ -454,7 +412,7 @@ export async function loadTreeForQuery( } assert(foundCase, `Query \`${queryToLoad.toString()}\` does not match any cases`); - return TestTree.create(queryToLoad, subtreeL0, maxChunkTime); + return TestTree.create(queryToLoad, subtreeL0); } function setSubtreeDescriptionAndCountTODOs( diff --git a/src/common/tools/crawl.ts b/src/common/tools/crawl.ts index 7cfbfa7f65ab..ef41772ee527 100644 --- a/src/common/tools/crawl.ts +++ b/src/common/tools/crawl.ts @@ -5,7 +5,6 @@ import * as fs from 'fs'; import * as path from 'path'; -import { loadMetadataForSuite } from '../framework/metadata.js'; import { SpecFile } from '../internal/file_loader.js'; import { TestQueryMultiCase, TestQueryMultiFile } from '../internal/query/query.js'; import { validQueryPart } from '../internal/query/validQueryPart.js'; @@ -47,11 +46,7 @@ async function crawlFilesRecursively(dir: string): Promise { export async function crawl( suiteDir: string, - opts: { - validate: boolean; - printMetadataWarnings: boolean; - printCaseCountReport: boolean; - } | null = null + opts: { validate: boolean; printCaseCountReport: boolean } | null = null ): Promise { if (!fs.existsSync(suiteDir)) { throw new Error(`Could not find suite: ${suiteDir}`); @@ -60,17 +55,6 @@ export async function crawl( let totalCases = 0; let totalSubcases = 0; - let validateTimingsEntries; - if (opts?.validate) { - const metadata = loadMetadataForSuite(suiteDir); - if (metadata) { - validateTimingsEntries = { - metadata, - testsFoundInFiles: new Set(), - }; - } - } - // Crawl files and convert paths to be POSIX-style, relative to suiteDir. const filesToEnumerate = (await crawlFilesRecursively(suiteDir)) .map(f => path.relative(suiteDir, f).replace(/\\/g, '/')) @@ -83,9 +67,8 @@ export async function crawl( const filepathWithoutExtension = file.substring(0, file.length - specFileSuffix.length); const pathSegments = filepathWithoutExtension.split('/'); - const suite = path.basename(suiteDir); - if (opts?.validate) { + const suite = path.basename(suiteDir); const filename = `../../${suite}/${filepathWithoutExtension}.spec.js`; assert(!process.env.STANDALONE_DEV_SERVER); @@ -95,7 +78,7 @@ export async function crawl( mod.g.validate(new TestQueryMultiFile(suite, pathSegments)); - if (opts?.printCaseCountReport || validateTimingsEntries) { + if (opts?.printCaseCountReport) { for (const t of mod.g.iterate()) { const testQuery = new TestQueryMultiCase( suite, @@ -108,21 +91,13 @@ export async function crawl( let subcases = 0; for (const c of t.iterate(null)) { cases++; - if (opts?.printCaseCountReport) { - subcases += c.computeSubcaseCount(); - } - } - - if (opts?.printCaseCountReport) { - const perCase = (subcases / cases).toFixed(0); - console.log(`${testQuery} - ${cases} cases, ${subcases} subcases (~${perCase}/case)`); - totalCases += cases; - totalSubcases += subcases; + subcases += c.computeSubcaseCount(); } - if (validateTimingsEntries && cases > 0) { - validateTimingsEntries.testsFoundInFiles.add(testQuery); - } + const perCase = (subcases / cases).toFixed(0); + console.log(`${testQuery} - ${cases} cases, ${subcases} subcases (~${perCase}/case)`); + totalCases += cases; + totalSubcases += subcases; } } } @@ -130,6 +105,7 @@ export async function crawl( for (const p of pathSegments) { assert(validQueryPart.test(p), `Invalid directory name ${p}; must match ${validQueryPart}`); } + entries.push({ file: pathSegments }); } else if (path.basename(file) === 'README.txt') { const dirname = path.dirname(file); @@ -142,59 +118,10 @@ export async function crawl( } } - if (validateTimingsEntries) { - const zeroEntries = []; - const staleEntries = []; - for (const [metadataKey, metadataValue] of Object.entries(validateTimingsEntries.metadata)) { - if (metadataKey.startsWith('_')) { - // Ignore json "_comments". - continue; - } - if (metadataValue.subcaseMS <= 0) { - zeroEntries.push(metadataKey); - } - if (!validateTimingsEntries.testsFoundInFiles.has(metadataKey)) { - staleEntries.push(metadataKey); - } - } - if (zeroEntries.length && opts?.printMetadataWarnings) { - console.warn( - 'WARNING: subcaseMS ≤ 0 found in listing_meta.json (see docs/adding_timing_metadata.md):' - ); - for (const metadataKey of zeroEntries) { - console.warn(` ${metadataKey}`); - } - } - - if (opts?.printMetadataWarnings) { - const missingEntries = []; - for (const metadataKey of validateTimingsEntries.testsFoundInFiles) { - if (!(metadataKey in validateTimingsEntries.metadata)) { - missingEntries.push(metadataKey); - } - } - if (missingEntries.length) { - console.error( - 'WARNING: Tests missing from listing_meta.json (see docs/adding_timing_metadata.md):' - ); - for (const metadataKey of missingEntries) { - console.error(` ${metadataKey}`); - } - } - } - - if (staleEntries.length) { - console.error('ERROR: Non-existent tests found in listing_meta.json. Please update:'); - for (const metadataKey of staleEntries) { - console.error(` ${metadataKey}`); - } - unreachable(); - } - } - if (opts?.printCaseCountReport) { console.log(`-----\nTOTAL: ${totalCases} cases, ${totalSubcases} subcases`); } + return entries; } diff --git a/src/common/tools/gen_wpt_cts_html.ts b/src/common/tools/gen_wpt_cts_html.ts index 8e5460f6b44f..4e65e628402c 100644 --- a/src/common/tools/gen_wpt_cts_html.ts +++ b/src/common/tools/gen_wpt_cts_html.ts @@ -30,7 +30,6 @@ gen_wpt_cts_html.ts. Example: "out": "path/to/output/cts.https.html", "outJSON": "path/to/output/webgpu_variant_list.json", "template": "path/to/template/cts.https.html", - "maxChunkTimeMS": 2000 } Usage (advanced) (deprecated, use config file): @@ -63,14 +62,6 @@ interface ConfigJSON { outVariantList?: string; /** Input template filename, relative to config file. */ template: string; - /** - * Maximum time for a single WPT "variant" chunk, in milliseconds. Defaults to infinity. - * - * This data is typically captured by developers on higher-end computers, so typical test - * machines might execute more slowly. For this reason, use a time much less than 5 seconds - * (a typical default time limit in WPT test executors). - */ - maxChunkTimeMS?: number; /** * List of argument prefixes (what comes before the test query), and optionally a list of * test queries to run under that prefix. Defaults to `['?q=']`. @@ -97,7 +88,6 @@ interface Config { out: string; outVariantList?: string; template: string; - maxChunkTimeMS: number; argumentsPrefixes: ArgumentsPrefixConfig[]; noLongPathAssert: boolean; expectations?: { @@ -144,7 +134,6 @@ let config: Config; suite: configJSON.suite, out: path.resolve(jsonFileDir, configJSON.out), template: path.resolve(jsonFileDir, configJSON.template), - maxChunkTimeMS: configJSON.maxChunkTimeMS ?? Infinity, argumentsPrefixes: configJSON.argumentsPrefixes ? [...reifyArgumentsPrefixesConfig(configJSON.argumentsPrefixes)] : [{ prefix: '?q=' }], @@ -185,7 +174,6 @@ let config: Config; suite, out: outFile, template: templateFile, - maxChunkTimeMS: Infinity, argumentsPrefixes: [{ prefix: '?q=' }], noLongPathAssert: false, }; @@ -206,8 +194,6 @@ let config: Config; printUsageAndExit(1); } - const useChunking = Number.isFinite(config.maxChunkTimeMS); - // Sort prefixes from longest to shortest config.argumentsPrefixes.sort((a, b) => b.prefix.length - a.prefix.length); @@ -247,19 +233,17 @@ let config: Config; const tree = await loader.loadTree(rootQuery, { subqueriesToExpand, fullyExpandSubtrees: fullyExpand.get(prefix), - maxChunkTime: config.maxChunkTimeMS, }); lines.push(undefined); // output blank line between prefixes const prefixComment = { comment: `Prefix: "${prefix}"` }; // contents will be updated later - if (useChunking) lines.push(prefixComment); const filesSeen = new Set(); const testsSeen = new Set(); let variantCount = 0; const alwaysExpandThroughLevel = 2; // expand to, at minimum, every test. - loopOverNodes: for (const { query, subtreeCounts } of tree.iterateCollapsedNodes({ + loopOverNodes: for (const { query } of tree.iterateCollapsedNodes({ alwaysExpandThroughLevel, })) { assert(query instanceof TestQueryMultiCase); @@ -288,7 +272,6 @@ let config: Config; lines.push({ urlQueryString: prefix + query.toString(), // "?debug=0&q=..." - comment: useChunking ? `estimated: ${subtreeCounts?.totalTimeMS.toFixed(3)} ms` : undefined, }); variantCount++; diff --git a/src/common/tools/merge_listing_times.ts b/src/common/tools/merge_listing_times.ts deleted file mode 100644 index a8bef354cc4f..000000000000 --- a/src/common/tools/merge_listing_times.ts +++ /dev/null @@ -1,169 +0,0 @@ -import * as fs from 'fs'; -import * as process from 'process'; -import * as readline from 'readline'; - -import { TestMetadataListing } from '../framework/metadata.js'; -import { parseQuery } from '../internal/query/parseQuery.js'; -import { TestQueryMultiCase, TestQuerySingleCase } from '../internal/query/query.js'; -import { CaseTimingLogLine } from '../internal/test_group.js'; -import { assert } from '../util/util.js'; - -// For information on listing_meta.json file maintenance, please read -// tools/merge_listing_times first. - -function usage(rc: number): never { - console.error(`Usage: tools/merge_listing_times [options] SUITES... -- [TIMING_LOG_FILES...] - -Options: - --help Print this message and exit. - -Reads raw case timing data for each suite in SUITES, from all TIMING_LOG_FILES -(see below), and merges it into the src/*/listing_meta.json files checked into -the repository. The timing data in the listing_meta.json files is updated with -the newly-observed timing data *if the new timing is slower*. That is, it will -only increase the values in the listing_meta.json file, and will only cause WPT -chunks to become smaller. - -If there are no TIMING_LOG_FILES, this just regenerates (reformats) the file -using the data already present. - -In more detail: - -- Reads per-case timing data in any of the SUITES, from all TIMING_LOG_FILES - (ignoring skipped cases), and averages it over the number of subcases. - In the case of cases that have run multiple times, takes the max of each. -- Compiles the average time-per-subcase for each test seen. -- For each suite seen, loads its listing_meta.json, takes the max of the old and - new data, and writes it back out. - -See 'docs/adding_timing_metadata.md' for how to generate TIMING_LOG_FILES files. -`); - process.exit(rc); -} - -const kHeader = `{ - "_comment": "SEMI AUTO-GENERATED. This list is NOT exhaustive. Please read docs/adding_timing_metadata.md.", -`; -const kFooter = `\ - "_end": "" -} -`; - -const argv = process.argv; -if (argv.some(v => v.startsWith('-') && v !== '--') || argv.every(v => v !== '--')) { - usage(0); -} -const suites = []; -const timingLogFilenames = []; -let seenDashDash = false; -for (const arg of argv.slice(2)) { - if (arg === '--') { - seenDashDash = true; - continue; - } else if (arg.startsWith('-')) { - usage(0); - } - - if (seenDashDash) { - timingLogFilenames.push(arg); - } else { - suites.push(arg); - } -} -if (!seenDashDash) { - usage(0); -} - -void (async () => { - // Read the log files to find the log line for each *case* query. If a case - // ran multiple times, take the one with the largest average subcase time. - const caseTimes = new Map(); - for (const timingLogFilename of timingLogFilenames) { - const rl = readline.createInterface({ - input: fs.createReadStream(timingLogFilename), - crlfDelay: Infinity, - }); - - for await (const line of rl) { - const parsed: CaseTimingLogLine = JSON.parse(line); - - const prev = caseTimes.get(parsed.q); - if (prev !== undefined) { - const timePerSubcase = parsed.timems / Math.max(1, parsed.nonskippedSubcaseCount); - const prevTimePerSubcase = prev.timems / Math.max(1, prev.nonskippedSubcaseCount); - - if (timePerSubcase > prevTimePerSubcase) { - caseTimes.set(parsed.q, parsed); - } - } else { - caseTimes.set(parsed.q, parsed); - } - } - } - - // Accumulate total times per test. Map of suite -> query -> {totalTimeMS, caseCount}. - const testTimes = new Map>(); - for (const suite of suites) { - testTimes.set(suite, new Map()); - } - for (const [caseQString, caseTime] of caseTimes) { - const caseQ = parseQuery(caseQString); - assert(caseQ instanceof TestQuerySingleCase); - const suite = caseQ.suite; - const suiteTestTimes = testTimes.get(suite); - if (suiteTestTimes === undefined) { - continue; - } - - const testQ = new TestQueryMultiCase(suite, caseQ.filePathParts, caseQ.testPathParts, {}); - const testQString = testQ.toString(); - - const prev = suiteTestTimes.get(testQString); - if (prev !== undefined) { - prev.totalTimeMS += caseTime.timems; - prev.subcaseCount += caseTime.nonskippedSubcaseCount; - } else { - suiteTestTimes.set(testQString, { - totalTimeMS: caseTime.timems, - subcaseCount: caseTime.nonskippedSubcaseCount, - }); - } - } - - for (const suite of suites) { - const currentMetadata: TestMetadataListing = JSON.parse( - fs.readFileSync(`./src/${suite}/listing_meta.json`, 'utf8') - ); - - const metadata = { ...currentMetadata }; - for (const [testQString, { totalTimeMS, subcaseCount }] of testTimes.get(suite)!) { - const avgTime = totalTimeMS / Math.max(1, subcaseCount); - if (testQString in metadata) { - metadata[testQString].subcaseMS = Math.max(metadata[testQString].subcaseMS, avgTime); - } else { - metadata[testQString] = { subcaseMS: avgTime }; - } - } - - writeListings(suite, metadata); - } -})(); - -function writeListings(suite: string, metadata: TestMetadataListing) { - const output = fs.createWriteStream(`./src/${suite}/listing_meta.json`); - try { - output.write(kHeader); - const keys = Object.keys(metadata).sort(); - for (const k of keys) { - if (k.startsWith('_')) { - // Ignore json "_comments". - continue; - } - assert(k.indexOf('"') === -1); - output.write(` "${k}": { "subcaseMS": ${metadata[k].subcaseMS.toFixed(3)} },\n`); - } - output.write(kFooter); - } finally { - output.close(); - } -} diff --git a/src/common/tools/validate.ts b/src/common/tools/validate.ts index 5486bf4602aa..7fbc548bcadc 100644 --- a/src/common/tools/validate.ts +++ b/src/common/tools/validate.ts @@ -32,13 +32,11 @@ if (args.indexOf('--help') !== -1) { usage(0); } -let printMetadataWarnings = false; let printCaseCountReport = false; const suiteDirs = []; for (const arg of args) { switch (arg) { case '--print-metadata-warnings': - printMetadataWarnings = true; break; case '--print-case-count-report': printCaseCountReport = true; @@ -56,7 +54,6 @@ if (suiteDirs.length === 0) { for (const suiteDir of suiteDirs) { void crawl(suiteDir, { validate: true, - printMetadataWarnings, printCaseCountReport, }); } diff --git a/tools/gen_wpt_cfg_chunked2sec.json b/tools/gen_wpt_cfg_chunked2sec.json deleted file mode 100644 index 123a06e9cd1d..000000000000 --- a/tools/gen_wpt_cfg_chunked2sec.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "suite": "webgpu", - "out": "../out-wpt/cts-chunked2sec.https.html", - "outVariantList": "../gen/webgpu_variant_list_chunked2sec.json", - "template": "../src/common/templates/cts.https.html", - "maxChunkTimeMS": 2000 -}