From c7a7fcb90bf76d3eab2dd0f8f71134d8ff9ad8e0 Mon Sep 17 00:00:00 2001 From: Demetrio Girardi Date: Tue, 4 Aug 2026 07:01:33 -0700 Subject: [PATCH 1/3] CI: classify PR files as core by component ownership Extract the "is this file core?" logic out of getPRProperties.js into coreFiles.js, and replace the heuristic it used. The old rule treated everything as core except test/example files, modules matching an adapter naming convention, and libraries referenced by a single vendor. That flagged as core the autogenerated per-module metadata, the vendor-specific libraries shared by an adapter and its aliases, and the registries every new module has to touch. Core is now what no outside component owns: a module is core when its metadata declares no component or only `prebid` ones, and a library is core when a core module pulls it in (prebid-core included). Modules with no metadata yet - newly added ones - fall back to the naming conventions, and to whether their name starts with a registered component name. Over the 112 PRs opened or updated in the last two weeks this flags 25 as core, down from 38, with none newly flagged. Repo-wide it flags 260 files and 38 modules, down from 1063 files. coreFiles.js also runs as a CLI, to classify an arbitrary list of files: gh pr diff --name-only 1234 | node .github/workflows/scripts/coreFiles.js Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/scripts/coreFiles.js | 316 +++++++++++++++++++ .github/workflows/scripts/getPRProperties.js | 57 +--- 2 files changed, 318 insertions(+), 55 deletions(-) create mode 100644 .github/workflows/scripts/coreFiles.js diff --git a/.github/workflows/scripts/coreFiles.js b/.github/workflows/scripts/coreFiles.js new file mode 100644 index 00000000000..4c74f2643ed --- /dev/null +++ b/.github/workflows/scripts/coreFiles.js @@ -0,0 +1,316 @@ +/** + * Decides whether a given repository file counts as a "core" change. + * + * A "core" PR needs more scrutiny than a module PR (see `reviewRequirements` in getPRProperties.js), + * so the goal is to classify as core only the files that are not owned by an outside vendor: a module + * is core when it declares no component of its own (or only `prebid` ones), and a library is core when + * a core module pulls it in. + * + * This file doubles as a CLI - see `usage` at the bottom - so that the classification can be run over + * an arbitrary list of files. + */ + +const fs = require('fs'); +const path = require('path'); + +const MODULE_PATTERNS = [ + /^modules\/([^\/]+)BidAdapter(\.(\w+)|\/)/, + /^modules\/([^\/]+)AnalyticsAdapter(\.(\w+)|\/)/, + /^modules\/([^\/]+)RtdProvider(\.(\w+)|\/)/, + /^modules\/([^\/]+)IdSystem(\.(\w+)|\/)/, + // a video provider is an integration with a particular player, so it always belongs to its vendor + /^modules\/([^\/]+)VideoProvider(\.(\w+)|\/)/ +]; + +const EXCLUDE_PATTERNS = [ + /^test\//, + /^integrationExamples\//, + /^[^\/]+$/, + /^.github\//, + // registries and per-module data that every new module has to touch; a change here is about the + // module being registered, not about the file itself + /^metadata\/modules\.json$/, + /^metadata\/disclosures\/modules\//, + /^modules\/\.submodules\.json$/, +]; + +const LIBRARY_PATTERN = /^libraries\/([^\/]+)\//; +const MODULE_FILE_PATTERN = /^modules\/([^\/.]+)/; +const MODULE_METADATA_PATTERN = /^metadata\/modules\/([^\/.]+)\.json$/; + +const REPO_ROOT = path.join(__dirname, '..', '..', '..'); +const DEFAULT_DEPENDENCIES_JSON = path.join(REPO_ROOT, 'build', 'dist', 'dependencies.json'); +const DEFAULT_METADATA_DIR = path.join(REPO_ROOT, 'metadata', 'modules'); +const DEFAULT_COMPONENTS_JSON = path.join(REPO_ROOT, 'metadata', 'modules.json'); + +// shortest name that is distinctive enough to identify a vendor by itself +const MIN_VENDOR_NAME_LENGTH = 3; + +// Modules that belong to a vendor but carry no sign of it: they declare no component, their name follows +// none of the module naming conventions, and no component is registered under their vendor's name. +// Giving them metadata is the way to take them off this list. +const VENDOR_MODULES = [ + 'seenthisBrandStories' +]; + +/** + * Loads the dependency graph (entry point -> chunk files) built by webpack's manifest plugin. + * + * @param {string} [file] path to dependencies.json; defaults to $DEPENDENCIES_JSON, then to the local build output. + */ +function loadDependencies(file = process.env.DEPENDENCIES_JSON || DEFAULT_DEPENDENCIES_JSON) { + if (!fs.existsSync(file)) { + throw new Error(`Cannot find dependency graph '${file}'; run 'gulp build' or set DEPENDENCIES_JSON`); + } + return JSON.parse(fs.readFileSync(file).toString()); +} + +/** + * The name of the module a repository file belongs to - the first path element under `modules/`, + * without its extension (`modules/foo.js` and `modules/foo/bar/baz.js` both belong to module `foo`), + * or the module a metadata file describes (`metadata/modules/foo.json` -> `foo`). + * + * @returns {string|null} module name, or null if the file does not belong to a module. + */ +function moduleName(path) { + for (const pat of [MODULE_FILE_PATTERN, MODULE_METADATA_PATTERN]) { + const match = pat.exec(path); + if (match != null) { + return match[1]; + } + } + return null; +} + +/** + * @param {string} entry name of a dependencies.json entry point (e.g. `appnexusBidAdapter.js` or + * `appnexusBidAdapter.metadata.js`) + * @returns {string} the module it builds (e.g. `appnexusBidAdapter`). + */ +function entryModule(entry) { + return entry.replace(/\.js$/, '').replace(/\.metadata$/, ''); +} + +/** + * Reads the components a module declares in its metadata. + * + * @param {object} [options] + * @param {string} [options.metadataDir] directory containing the per-module metadata JSON. + * @returns {function(string): Array|null} module name -> its components, or null if it has no metadata. + */ +function moduleComponents({metadataDir = DEFAULT_METADATA_DIR} = {}) { + const cache = {}; + return function (module) { + if (!cache.hasOwnProperty(module)) { + const file = path.join(metadataDir, `${module}.json`); + cache[module] = fs.existsSync(file) ? (JSON.parse(fs.readFileSync(file).toString()).components || []) : null; + } + return cache[module]; + }; +} + +/** + * Tells whether a module name begins with the name of a component registered anywhere in the repo - + * `adlooxAdServerVideo` starts with `adloox`, which is registered as an rtd and an analytics component, + * so the module belongs to that vendor. The remainder has to start on a camelCase boundary, so that a + * component named e.g. `currency` does not make `modules/currency.js` look vendor-owned. + * + * @param {object} [options] + * @param {Array} [options.components] the component registry, as found in metadata/modules.json. + * @returns {function(string): boolean} module name -> whether a registered vendor owns it. + */ +function vendorNamePrefix({components} = {}) { + let names; + return function (module) { + if (names == null) { + const registry = components ?? JSON.parse(fs.readFileSync(DEFAULT_COMPONENTS_JSON).toString()).components; + names = Array.from(new Set( + registry + .filter(component => component.componentType !== 'prebid') + .flatMap(component => [component.componentName, component.aliasOf]) + .filter(name => name != null && name.length >= MIN_VENDOR_NAME_LENGTH) + .map(name => name.toLowerCase()) + )); + } + return names.some(name => module.toLowerCase().startsWith(name) && /^[A-Z]/.test(module.charAt(name.length))); + }; +} + +/** + * Core is what is not owned by an outside component: a module is core if it declares no component + * (or has no metadata at all), or if every component it declares is a `prebid` one; a library is core + * if it's pulled in by a core module - which includes prebid-core itself. + * + * @param {object} [options] + * @param {object} [options.dependencies] dependency graph, as loaded from dependencies.json. + * @param {string} [options.metadataDir] directory containing the per-module metadata JSON. + * @param {Array} [options.components] the component registry, as found in metadata/modules.json. + * @param {Array} [options.vendorModules] modules known to belong to a vendor, for the ones no + * naming convention can pick out. + * @param {boolean} [options.unknownLibrariesAreCore] how to classify a library that no entry point + * pulls in - it has no known owner, so by default it's assumed to be core. + * @param {string} [options.missingMetadata] how to classify a module that has no metadata file at all. + * Metadata is generated separately from the module it describes, so a newly added module does not have + * any yet; `by-name` (the default) falls back to the naming conventions - a module named `BidAdapter` + * & co, or one whose name starts with a registered component name, belongs to a vendor, anything else is + * core - while `core` treats them all like a module with no components, and `not-core` keeps them all out. + * @returns {function(string): boolean} true if the given path should count as a core change. + */ +function coreFileMatcher({ + dependencies, + metadataDir, + components, + vendorModules = VENDOR_MODULES, + unknownLibrariesAreCore = true, + missingMetadata = 'by-name' +} = {}) { + const componentsOf = moduleComponents({metadataDir}); + const belongsToVendor = vendorNamePrefix({components}); + let deps = dependencies; + const libraryUsers = {}; + + function isCoreModule(module) { + const declared = componentsOf(module); + if (declared == null) { + switch (missingMetadata) { + case 'core': return true; + case 'not-core': return false; + default: return !vendorModules.includes(module) && + !MODULE_PATTERNS.find(pat => pat.test(`modules/${module}.js`)) && + !belongsToVendor(module); + } + } + return declared.length === 0 || + declared.every(component => component.componentType === 'prebid'); + } + + function usersOf(library) { + if (!libraryUsers.hasOwnProperty(library)) { + if (deps == null) deps = loadDependencies(); + libraryUsers[library] = Object.entries(deps) + .filter(([entry, chunks]) => chunks.includes(`${library}.js`)) + .map(([entry]) => entryModule(entry)); + } + return libraryUsers[library]; + } + + return function isCoreFile(path) { + if (EXCLUDE_PATTERNS.find(pat => pat.test(path))) { + return false; + } + const module = moduleName(path); + if (module != null) { + return isCoreModule(module); + } + const lib = LIBRARY_PATTERN.exec(path); + if (lib != null) { + const users = usersOf(lib[1]); + return users.length === 0 ? unknownLibrariesAreCore : users.some(isCoreModule); + } + return true; + }; +} + +module.exports = { + coreFileMatcher, + MODULE_PATTERNS, + EXCLUDE_PATTERNS, + LIBRARY_PATTERN, + VENDOR_MODULES, + loadDependencies, + moduleName, + entryModule, + moduleComponents, + vendorNamePrefix, +}; + +function usage() { + return [ + 'Classify repository files as "core" or not, the way PR assignment does.', + '', + 'Usage: node .github/workflows/scripts/coreFiles.js [options] [file...]', + '', + 'Files may also be piped in, one per line, e.g.:', + ' gh pr diff --name-only 1234 | node .github/workflows/scripts/coreFiles.js', + '', + 'Options:', + ' -d, --deps path to dependencies.json (default: $DEPENDENCIES_JSON, then build/dist)', + ' -o, --option pass an option to the matcher (repeatable; values are JSON when parseable)', + ' -c, --core-only print only the files classified as core', + ' -j, --json print results as JSON', + ' -h, --help show this message', + '', + 'Exit code is 0 if any file is core, 1 otherwise - matching `isCoreChange` in getPRProperties.js.', + ].join('\n'); +} + +function parseArgs(argv) { + const opts = {files: [], options: {}}; + while (argv.length) { + const arg = argv.shift(); + switch (arg) { + case '-d': case '--deps': opts.deps = argv.shift(); break; + case '-c': case '--core-only': opts.coreOnly = true; break; + case '-j': case '--json': opts.json = true; break; + case '-h': case '--help': opts.help = true; break; + case '-o': case '--option': { + const [key, ...rest] = argv.shift().split('='); + const value = rest.join('='); + try { + opts.options[key] = JSON.parse(value); + } catch (e) { + opts.options[key] = value; + } + break; + } + default: opts.files.push(arg); + } + } + return opts; +} + +function readStdin() { + try { + return fs.readFileSync(0).toString(); + } catch (e) { + return ''; + } +} + +function main(argv) { + const opts = parseArgs(argv); + if (opts.help) { + console.log(usage()); + return 0; + } + let files = opts.files; + if (!files.length && !process.stdin.isTTY) { + files = readStdin().split('\n').map(line => line.trim()).filter(Boolean); + } + if (!files.length) { + console.error(usage()); + return 2; + } + const isCore = coreFileMatcher(Object.assign( + opts.deps ? {dependencies: loadDependencies(opts.deps)} : {}, + opts.options + )); + const results = files.map(file => ({file, core: isCore(file)})); + if (opts.json) { + console.log(JSON.stringify(opts.coreOnly ? results.filter(({core}) => core) : results, null, 2)); + } else { + results + .filter(({core}) => core || !opts.coreOnly) + .forEach(({file, core}) => console.log(opts.coreOnly ? file : `${core ? 'CORE' : ' '} ${file}`)); + } + return results.some(({core}) => core) ? 0 : 1; +} + +if (require.main === module) { + try { + process.exitCode = main(process.argv.slice(2)); + } catch (e) { + console.error(e.message); + process.exitCode = 2; + } +} diff --git a/.github/workflows/scripts/getPRProperties.js b/.github/workflows/scripts/getPRProperties.js index 4961acd3c6a..34616a53fe1 100644 --- a/.github/workflows/scripts/getPRProperties.js +++ b/.github/workflows/scripts/getPRProperties.js @@ -1,61 +1,8 @@ const ghRequester = require('./ghRequest.js'); const AWS = require("@aws-sdk/client-s3"); -const fs = require('fs'); +const { coreFileMatcher } = require('./coreFiles.js'); -const MODULE_PATTERNS = [ - /^modules\/([^\/]+)BidAdapter(\.(\w+)|\/)/, - /^modules\/([^\/]+)AnalyticsAdapter(\.(\w+)|\/)/, - /^modules\/([^\/]+)RtdProvider(\.(\w+)|\/)/, - /^modules\/([^\/]+)IdSystem(\.(\w+)|\/)/ -] - -const EXCLUDE_PATTERNS = [ - /^test\//, - /^integrationExamples\//, - /^[^\/]+$/, - /^.github\//, -] - -const LIBRARY_PATTERN = /^libraries\/([^\/]+)\//; - -function extractVendor(chunkName) { - for (const pat of MODULE_PATTERNS) { - const match = pat.exec(`modules/${chunkName}`); - if (match != null) { - return match[1]; - } - } - return chunkName; -} - -const getLibraryRefs = (() => { - const deps = JSON.parse(fs.readFileSync(process.env.DEPENDENCIES_JSON).toString()); - const refs = {}; - return function (libraryName) { - if (!refs.hasOwnProperty(libraryName)) { - refs[libraryName] = new Set(); - Object.entries(deps) - .filter(([name, deps]) => deps.includes(`${libraryName}.js`)) - .forEach(([name]) => refs[libraryName].add(extractVendor(name))) - } - return refs[libraryName]; - } -})(); - -function isCoreFile(path) { - if (EXCLUDE_PATTERNS.find(pat => pat.test(path))) { - return false; - } - if (MODULE_PATTERNS.find(pat => pat.test(path)) ) { - return false; - } - const lib = LIBRARY_PATTERN.exec(path); - if (lib != null) { - // a library is "core" if it's used by more than one vendor - return getLibraryRefs(lib[1]).size > 1; - } - return true; -} +const isCoreFile = coreFileMatcher(); async function isPrebidMember(ghHandle) { const client = new AWS.S3({region: 'us-east-2'}); From 0a34542eaf54a5e35065f29c342ecf8581b359ab Mon Sep 17 00:00:00 2001 From: Demetrio Girardi Date: Tue, 4 Aug 2026 07:26:00 -0700 Subject: [PATCH 2/3] CI: default libraries to core unless they belong to a vendor Attributing a library through the chunk graph alone misses prebid-maintained shared code whose consumers all happen to be vendor modules: ortbConverter (135 consumers), dnt (47), boundingClientRect (37), sizeUtils (33) and about thirty others were not flagged. Counting consumers, as the previous heuristic did, does not separate those from white-label libraries shared between one vendor's brands - teqblazeUtils has 55 consumers, more than pbsExtensions or devicePixelRatio. Neither does counting distinct component owners or gvl ids, nor the mix of component types among the consumers. So libraries now default to core, and are excluded when they belong to a vendor: when their name begins with a registered component name, or when they are listed in VENDOR_LIBRARIES - white labels whose brand is not itself a component, or whose components are registered under a longer name than the library. A library that a core module pulls in stays core whatever its name suggests, which is what keeps timeoutQueue - it reads as an extension of the `timeout` rtd component - on the right side. This keeps all 21 libraries that were core before and adds 35, while dropping the 47 vendor-owned ones. Repo-wide the classification covers 318 files; the 112 PRs from the last two weeks are unaffected, still 25 flagged core. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/scripts/coreFiles.js | 43 ++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/.github/workflows/scripts/coreFiles.js b/.github/workflows/scripts/coreFiles.js index 4c74f2643ed..3bb76d30c8f 100644 --- a/.github/workflows/scripts/coreFiles.js +++ b/.github/workflows/scripts/coreFiles.js @@ -8,6 +8,8 @@ * * This file doubles as a CLI - see `usage` at the bottom - so that the classification can be run over * an arbitrary list of files. + * + * This file was written by a bot (Claude Code). */ const fs = require('fs'); @@ -53,6 +55,24 @@ const VENDOR_MODULES = [ 'seenthisBrandStories' ]; +// Libraries that belong to a vendor - typically a white label serving several brands - but whose name +// does not begin with any registered component name, either because the vendor's own brand is not a +// component (`teqblaze`, `vizionik`) or because its components are registered under a longer name +// (`intentIqId`, `advangelists`). Everything else under libraries/ is taken to be shared code. +const VENDOR_LIBRARIES = [ + 'advangUtils', + 'agenticxUtils', + 'audUtils', + 'dxUtils', + 'intentIqConstants', + 'intentIqUtils', + 'pageInfosUtils', + 'teqblazeUtils', + 'utiqUtils', + 'vizionikUtils', + 'xeUtils' +]; + /** * Loads the dependency graph (entry point -> chunk files) built by webpack's manifest plugin. * @@ -139,7 +159,10 @@ function vendorNamePrefix({components} = {}) { /** * Core is what is not owned by an outside component: a module is core if it declares no component * (or has no metadata at all), or if every component it declares is a `prebid` one; a library is core - * if it's pulled in by a core module - which includes prebid-core itself. + * if a core module pulls it in - prebid-core included - or, failing that, if it does not belong to a + * vendor. Libraries default to core because most of them are shared code that happens to be used only + * by vendor modules, and because a library extracted tomorrow should be reviewed until someone says + * otherwise; the vendor ones are recognizable by name. * * @param {object} [options] * @param {object} [options.dependencies] dependency graph, as loaded from dependencies.json. @@ -147,8 +170,7 @@ function vendorNamePrefix({components} = {}) { * @param {Array} [options.components] the component registry, as found in metadata/modules.json. * @param {Array} [options.vendorModules] modules known to belong to a vendor, for the ones no * naming convention can pick out. - * @param {boolean} [options.unknownLibrariesAreCore] how to classify a library that no entry point - * pulls in - it has no known owner, so by default it's assumed to be core. + * @param {Array} [options.vendorLibraries] libraries known to belong to a vendor, likewise. * @param {string} [options.missingMetadata] how to classify a module that has no metadata file at all. * Metadata is generated separately from the module it describes, so a newly added module does not have * any yet; `by-name` (the default) falls back to the naming conventions - a module named `BidAdapter` @@ -161,7 +183,7 @@ function coreFileMatcher({ metadataDir, components, vendorModules = VENDOR_MODULES, - unknownLibrariesAreCore = true, + vendorLibraries = VENDOR_LIBRARIES, missingMetadata = 'by-name' } = {}) { const componentsOf = moduleComponents({metadataDir}); @@ -194,6 +216,15 @@ function coreFileMatcher({ return libraryUsers[library]; } + function isCoreLibrary(library) { + // a library a core module depends on is core whatever its name suggests - `timeoutQueue` reads as + // an extension of the `timeout` rtd component, but core modules use it + if (usersOf(library).some(isCoreModule)) { + return true; + } + return !vendorLibraries.includes(library) && !belongsToVendor(library); + } + return function isCoreFile(path) { if (EXCLUDE_PATTERNS.find(pat => pat.test(path))) { return false; @@ -204,8 +235,7 @@ function coreFileMatcher({ } const lib = LIBRARY_PATTERN.exec(path); if (lib != null) { - const users = usersOf(lib[1]); - return users.length === 0 ? unknownLibrariesAreCore : users.some(isCoreModule); + return isCoreLibrary(lib[1]); } return true; }; @@ -217,6 +247,7 @@ module.exports = { EXCLUDE_PATTERNS, LIBRARY_PATTERN, VENDOR_MODULES, + VENDOR_LIBRARIES, loadDependencies, moduleName, entryModule, From cc2fb6d0d00b7637b94b7a660b1036b3c831ed23 Mon Sep 17 00:00:00 2001 From: Demetrio Girardi Date: Tue, 4 Aug 2026 08:36:40 -0700 Subject: [PATCH 3/3] CI: treat a library with a single consumer as that consumer's own A library added together with the adapter that uses it was flagged core: PR-assignment.yml runs from master, where the new vendor is not yet in metadata/modules.json, so no name test can recognize the library as theirs. Of the new-adapter-with-its-own-library PRs merged since the component registry exists, five would have been mislabelled this way - #13731, #13815, #14267, #14807, #14896 - and four of them are not flagged core today. A library with exactly one consumer now belongs to that consumer, whatever its name suggests; it becomes core as soon as a second module picks it up. Libraries a core module uses are still core, so this only applies to code owned outright by one vendor module. Counting consumers beyond that is still avoided - a threshold cannot separate shared infrastructure from a white label, since teqblazeUtils has more consumers than pbsExtensions - so ortbConverter, dnt and the other shared libraries stay core. Four existing libraries move with this: cryptoUtils, htmlEscape, interpretResponseUtils and vastTrackers, each a single vendor's helper with a generic name. Core libraries go from 56 to 52, repo-wide from 318 files to 314; the last two weeks of PRs are unchanged at 25 flagged core. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/scripts/coreFiles.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/scripts/coreFiles.js b/.github/workflows/scripts/coreFiles.js index 3bb76d30c8f..3557cb3d0ee 100644 --- a/.github/workflows/scripts/coreFiles.js +++ b/.github/workflows/scripts/coreFiles.js @@ -217,11 +217,18 @@ function coreFileMatcher({ } function isCoreLibrary(library) { + const users = usersOf(library); // a library a core module depends on is core whatever its name suggests - `timeoutQueue` reads as // an extension of the `timeout` rtd component, but core modules use it - if (usersOf(library).some(isCoreModule)) { + if (users.some(isCoreModule)) { return true; } + // a single consumer owns the library outright; this is how a vendor library added together with + // its adapter is recognized, before any component of that vendor is registered. It becomes core + // as soon as a second module picks it up. + if (users.length === 1) { + return false; + } return !vendorLibraries.includes(library) && !belongsToVendor(library); }