diff --git a/.github/workflows/scripts/coreFiles.js b/.github/workflows/scripts/coreFiles.js new file mode 100644 index 00000000000..3557cb3d0ee --- /dev/null +++ b/.github/workflows/scripts/coreFiles.js @@ -0,0 +1,354 @@ +/** + * Decides whether a given repository file counts as a "core" change. + * + * A "core" PR needs more scrutiny than a module PR (see `reviewRequirements` in getPRProperties.js), + * so the goal is to classify as core only the files that are not owned by an outside vendor: a module + * is core when it declares no component of its own (or only `prebid` ones), and a library is core when + * a core module pulls it in. + * + * This file doubles as a CLI - see `usage` at the bottom - so that the classification can be run over + * an arbitrary list of files. + * + * This file was written by a bot (Claude Code). + */ + +const fs = require('fs'); +const path = require('path'); + +const MODULE_PATTERNS = [ + /^modules\/([^\/]+)BidAdapter(\.(\w+)|\/)/, + /^modules\/([^\/]+)AnalyticsAdapter(\.(\w+)|\/)/, + /^modules\/([^\/]+)RtdProvider(\.(\w+)|\/)/, + /^modules\/([^\/]+)IdSystem(\.(\w+)|\/)/, + // a video provider is an integration with a particular player, so it always belongs to its vendor + /^modules\/([^\/]+)VideoProvider(\.(\w+)|\/)/ +]; + +const EXCLUDE_PATTERNS = [ + /^test\//, + /^integrationExamples\//, + /^[^\/]+$/, + /^.github\//, + // registries and per-module data that every new module has to touch; a change here is about the + // module being registered, not about the file itself + /^metadata\/modules\.json$/, + /^metadata\/disclosures\/modules\//, + /^modules\/\.submodules\.json$/, +]; + +const LIBRARY_PATTERN = /^libraries\/([^\/]+)\//; +const MODULE_FILE_PATTERN = /^modules\/([^\/.]+)/; +const MODULE_METADATA_PATTERN = /^metadata\/modules\/([^\/.]+)\.json$/; + +const REPO_ROOT = path.join(__dirname, '..', '..', '..'); +const DEFAULT_DEPENDENCIES_JSON = path.join(REPO_ROOT, 'build', 'dist', 'dependencies.json'); +const DEFAULT_METADATA_DIR = path.join(REPO_ROOT, 'metadata', 'modules'); +const DEFAULT_COMPONENTS_JSON = path.join(REPO_ROOT, 'metadata', 'modules.json'); + +// shortest name that is distinctive enough to identify a vendor by itself +const MIN_VENDOR_NAME_LENGTH = 3; + +// Modules that belong to a vendor but carry no sign of it: they declare no component, their name follows +// none of the module naming conventions, and no component is registered under their vendor's name. +// Giving them metadata is the way to take them off this list. +const VENDOR_MODULES = [ + 'seenthisBrandStories' +]; + +// Libraries that belong to a vendor - typically a white label serving several brands - but whose name +// does not begin with any registered component name, either because the vendor's own brand is not a +// component (`teqblaze`, `vizionik`) or because its components are registered under a longer name +// (`intentIqId`, `advangelists`). Everything else under libraries/ is taken to be shared code. +const VENDOR_LIBRARIES = [ + 'advangUtils', + 'agenticxUtils', + 'audUtils', + 'dxUtils', + 'intentIqConstants', + 'intentIqUtils', + 'pageInfosUtils', + 'teqblazeUtils', + 'utiqUtils', + 'vizionikUtils', + 'xeUtils' +]; + +/** + * Loads the dependency graph (entry point -> chunk files) built by webpack's manifest plugin. + * + * @param {string} [file] path to dependencies.json; defaults to $DEPENDENCIES_JSON, then to the local build output. + */ +function loadDependencies(file = process.env.DEPENDENCIES_JSON || DEFAULT_DEPENDENCIES_JSON) { + if (!fs.existsSync(file)) { + throw new Error(`Cannot find dependency graph '${file}'; run 'gulp build' or set DEPENDENCIES_JSON`); + } + return JSON.parse(fs.readFileSync(file).toString()); +} + +/** + * The name of the module a repository file belongs to - the first path element under `modules/`, + * without its extension (`modules/foo.js` and `modules/foo/bar/baz.js` both belong to module `foo`), + * or the module a metadata file describes (`metadata/modules/foo.json` -> `foo`). + * + * @returns {string|null} module name, or null if the file does not belong to a module. + */ +function moduleName(path) { + for (const pat of [MODULE_FILE_PATTERN, MODULE_METADATA_PATTERN]) { + const match = pat.exec(path); + if (match != null) { + return match[1]; + } + } + return null; +} + +/** + * @param {string} entry name of a dependencies.json entry point (e.g. `appnexusBidAdapter.js` or + * `appnexusBidAdapter.metadata.js`) + * @returns {string} the module it builds (e.g. `appnexusBidAdapter`). + */ +function entryModule(entry) { + return entry.replace(/\.js$/, '').replace(/\.metadata$/, ''); +} + +/** + * Reads the components a module declares in its metadata. + * + * @param {object} [options] + * @param {string} [options.metadataDir] directory containing the per-module metadata JSON. + * @returns {function(string): Array|null} module name -> its components, or null if it has no metadata. + */ +function moduleComponents({metadataDir = DEFAULT_METADATA_DIR} = {}) { + const cache = {}; + return function (module) { + if (!cache.hasOwnProperty(module)) { + const file = path.join(metadataDir, `${module}.json`); + cache[module] = fs.existsSync(file) ? (JSON.parse(fs.readFileSync(file).toString()).components || []) : null; + } + return cache[module]; + }; +} + +/** + * Tells whether a module name begins with the name of a component registered anywhere in the repo - + * `adlooxAdServerVideo` starts with `adloox`, which is registered as an rtd and an analytics component, + * so the module belongs to that vendor. The remainder has to start on a camelCase boundary, so that a + * component named e.g. `currency` does not make `modules/currency.js` look vendor-owned. + * + * @param {object} [options] + * @param {Array} [options.components] the component registry, as found in metadata/modules.json. + * @returns {function(string): boolean} module name -> whether a registered vendor owns it. + */ +function vendorNamePrefix({components} = {}) { + let names; + return function (module) { + if (names == null) { + const registry = components ?? JSON.parse(fs.readFileSync(DEFAULT_COMPONENTS_JSON).toString()).components; + names = Array.from(new Set( + registry + .filter(component => component.componentType !== 'prebid') + .flatMap(component => [component.componentName, component.aliasOf]) + .filter(name => name != null && name.length >= MIN_VENDOR_NAME_LENGTH) + .map(name => name.toLowerCase()) + )); + } + return names.some(name => module.toLowerCase().startsWith(name) && /^[A-Z]/.test(module.charAt(name.length))); + }; +} + +/** + * Core is what is not owned by an outside component: a module is core if it declares no component + * (or has no metadata at all), or if every component it declares is a `prebid` one; a library is core + * if a core module pulls it in - prebid-core included - or, failing that, if it does not belong to a + * vendor. Libraries default to core because most of them are shared code that happens to be used only + * by vendor modules, and because a library extracted tomorrow should be reviewed until someone says + * otherwise; the vendor ones are recognizable by name. + * + * @param {object} [options] + * @param {object} [options.dependencies] dependency graph, as loaded from dependencies.json. + * @param {string} [options.metadataDir] directory containing the per-module metadata JSON. + * @param {Array} [options.components] the component registry, as found in metadata/modules.json. + * @param {Array} [options.vendorModules] modules known to belong to a vendor, for the ones no + * naming convention can pick out. + * @param {Array} [options.vendorLibraries] libraries known to belong to a vendor, likewise. + * @param {string} [options.missingMetadata] how to classify a module that has no metadata file at all. + * Metadata is generated separately from the module it describes, so a newly added module does not have + * any yet; `by-name` (the default) falls back to the naming conventions - a module named `BidAdapter` + * & co, or one whose name starts with a registered component name, belongs to a vendor, anything else is + * core - while `core` treats them all like a module with no components, and `not-core` keeps them all out. + * @returns {function(string): boolean} true if the given path should count as a core change. + */ +function coreFileMatcher({ + dependencies, + metadataDir, + components, + vendorModules = VENDOR_MODULES, + vendorLibraries = VENDOR_LIBRARIES, + missingMetadata = 'by-name' +} = {}) { + const componentsOf = moduleComponents({metadataDir}); + const belongsToVendor = vendorNamePrefix({components}); + let deps = dependencies; + const libraryUsers = {}; + + function isCoreModule(module) { + const declared = componentsOf(module); + if (declared == null) { + switch (missingMetadata) { + case 'core': return true; + case 'not-core': return false; + default: return !vendorModules.includes(module) && + !MODULE_PATTERNS.find(pat => pat.test(`modules/${module}.js`)) && + !belongsToVendor(module); + } + } + return declared.length === 0 || + declared.every(component => component.componentType === 'prebid'); + } + + function usersOf(library) { + if (!libraryUsers.hasOwnProperty(library)) { + if (deps == null) deps = loadDependencies(); + libraryUsers[library] = Object.entries(deps) + .filter(([entry, chunks]) => chunks.includes(`${library}.js`)) + .map(([entry]) => entryModule(entry)); + } + return libraryUsers[library]; + } + + function isCoreLibrary(library) { + const users = usersOf(library); + // a library a core module depends on is core whatever its name suggests - `timeoutQueue` reads as + // an extension of the `timeout` rtd component, but core modules use it + if (users.some(isCoreModule)) { + return true; + } + // a single consumer owns the library outright; this is how a vendor library added together with + // its adapter is recognized, before any component of that vendor is registered. It becomes core + // as soon as a second module picks it up. + if (users.length === 1) { + return false; + } + return !vendorLibraries.includes(library) && !belongsToVendor(library); + } + + return function isCoreFile(path) { + if (EXCLUDE_PATTERNS.find(pat => pat.test(path))) { + return false; + } + const module = moduleName(path); + if (module != null) { + return isCoreModule(module); + } + const lib = LIBRARY_PATTERN.exec(path); + if (lib != null) { + return isCoreLibrary(lib[1]); + } + return true; + }; +} + +module.exports = { + coreFileMatcher, + MODULE_PATTERNS, + EXCLUDE_PATTERNS, + LIBRARY_PATTERN, + VENDOR_MODULES, + VENDOR_LIBRARIES, + loadDependencies, + moduleName, + entryModule, + moduleComponents, + vendorNamePrefix, +}; + +function usage() { + return [ + 'Classify repository files as "core" or not, the way PR assignment does.', + '', + 'Usage: node .github/workflows/scripts/coreFiles.js [options] [file...]', + '', + 'Files may also be piped in, one per line, e.g.:', + ' gh pr diff --name-only 1234 | node .github/workflows/scripts/coreFiles.js', + '', + 'Options:', + ' -d, --deps path to dependencies.json (default: $DEPENDENCIES_JSON, then build/dist)', + ' -o, --option pass an option to the matcher (repeatable; values are JSON when parseable)', + ' -c, --core-only print only the files classified as core', + ' -j, --json print results as JSON', + ' -h, --help show this message', + '', + 'Exit code is 0 if any file is core, 1 otherwise - matching `isCoreChange` in getPRProperties.js.', + ].join('\n'); +} + +function parseArgs(argv) { + const opts = {files: [], options: {}}; + while (argv.length) { + const arg = argv.shift(); + switch (arg) { + case '-d': case '--deps': opts.deps = argv.shift(); break; + case '-c': case '--core-only': opts.coreOnly = true; break; + case '-j': case '--json': opts.json = true; break; + case '-h': case '--help': opts.help = true; break; + case '-o': case '--option': { + const [key, ...rest] = argv.shift().split('='); + const value = rest.join('='); + try { + opts.options[key] = JSON.parse(value); + } catch (e) { + opts.options[key] = value; + } + break; + } + default: opts.files.push(arg); + } + } + return opts; +} + +function readStdin() { + try { + return fs.readFileSync(0).toString(); + } catch (e) { + return ''; + } +} + +function main(argv) { + const opts = parseArgs(argv); + if (opts.help) { + console.log(usage()); + return 0; + } + let files = opts.files; + if (!files.length && !process.stdin.isTTY) { + files = readStdin().split('\n').map(line => line.trim()).filter(Boolean); + } + if (!files.length) { + console.error(usage()); + return 2; + } + const isCore = coreFileMatcher(Object.assign( + opts.deps ? {dependencies: loadDependencies(opts.deps)} : {}, + opts.options + )); + const results = files.map(file => ({file, core: isCore(file)})); + if (opts.json) { + console.log(JSON.stringify(opts.coreOnly ? results.filter(({core}) => core) : results, null, 2)); + } else { + results + .filter(({core}) => core || !opts.coreOnly) + .forEach(({file, core}) => console.log(opts.coreOnly ? file : `${core ? 'CORE' : ' '} ${file}`)); + } + return results.some(({core}) => core) ? 0 : 1; +} + +if (require.main === module) { + try { + process.exitCode = main(process.argv.slice(2)); + } catch (e) { + console.error(e.message); + process.exitCode = 2; + } +} diff --git a/.github/workflows/scripts/getPRProperties.js b/.github/workflows/scripts/getPRProperties.js index 4961acd3c6a..34616a53fe1 100644 --- a/.github/workflows/scripts/getPRProperties.js +++ b/.github/workflows/scripts/getPRProperties.js @@ -1,61 +1,8 @@ const ghRequester = require('./ghRequest.js'); const AWS = require("@aws-sdk/client-s3"); -const fs = require('fs'); +const { coreFileMatcher } = require('./coreFiles.js'); -const MODULE_PATTERNS = [ - /^modules\/([^\/]+)BidAdapter(\.(\w+)|\/)/, - /^modules\/([^\/]+)AnalyticsAdapter(\.(\w+)|\/)/, - /^modules\/([^\/]+)RtdProvider(\.(\w+)|\/)/, - /^modules\/([^\/]+)IdSystem(\.(\w+)|\/)/ -] - -const EXCLUDE_PATTERNS = [ - /^test\//, - /^integrationExamples\//, - /^[^\/]+$/, - /^.github\//, -] - -const LIBRARY_PATTERN = /^libraries\/([^\/]+)\//; - -function extractVendor(chunkName) { - for (const pat of MODULE_PATTERNS) { - const match = pat.exec(`modules/${chunkName}`); - if (match != null) { - return match[1]; - } - } - return chunkName; -} - -const getLibraryRefs = (() => { - const deps = JSON.parse(fs.readFileSync(process.env.DEPENDENCIES_JSON).toString()); - const refs = {}; - return function (libraryName) { - if (!refs.hasOwnProperty(libraryName)) { - refs[libraryName] = new Set(); - Object.entries(deps) - .filter(([name, deps]) => deps.includes(`${libraryName}.js`)) - .forEach(([name]) => refs[libraryName].add(extractVendor(name))) - } - return refs[libraryName]; - } -})(); - -function isCoreFile(path) { - if (EXCLUDE_PATTERNS.find(pat => pat.test(path))) { - return false; - } - if (MODULE_PATTERNS.find(pat => pat.test(path)) ) { - return false; - } - const lib = LIBRARY_PATTERN.exec(path); - if (lib != null) { - // a library is "core" if it's used by more than one vendor - return getLibraryRefs(lib[1]).size > 1; - } - return true; -} +const isCoreFile = coreFileMatcher(); async function isPrebidMember(ghHandle) { const client = new AWS.S3({region: 'us-east-2'});